我们从Python开源项目中，提取了以下12个代码示例，用于说明如何使用scipy.sparse.indices()。
def run(self): it = iter(self.corpus) while True: chunk = itertools.islice(it, self.chunksize) if self.as_numpy: # HACK XXX convert documents to numpy arrays, to save memory. # This also gives a scipy warning at runtime: # "UserWarning: indices array has non-integer dtype (float64)" wrapped_chunk = [[np.asarray(doc) for doc in chunk]] else: wrapped_chunk = [list(chunk)] if not wrapped_chunk[0]: self.q.put(None, block=True) break try: qsize = self.q.qsize() except NotImplementedError: qsize = '?' logger.debug("prepared another chunk of %i documents (qsize=%s)" % (len(wrapped_chunk[0]), qsize)) self.q.put(wrapped_chunk.pop(), block=True) #endclass InputQueue
def load(cls, fname, mmap=None): """ Load a previously saved object from file (also see `save`). If the object was saved with large arrays stored separately, you can load these arrays via mmap (shared memory) using `mmap='r'`. Default: don't use mmap, load large arrays as normal objects. """ logger.info("loading %s object from %s" % (cls.__name__, fname)) subname = lambda suffix: fname + '.' + suffix + '.npy' obj = unpickle(fname) for attrib in getattr(obj, '__numpys', []): logger.info("loading %s from %s with mmap=%s" % (attrib, subname(attrib), mmap)) setattr(obj, attrib, numpy.load(subname(attrib), mmap_mode=mmap)) for attrib in getattr(obj, '__scipys', []): logger.info("loading %s from %s with mmap=%s" % (attrib, subname(attrib), mmap)) sparse = unpickle(subname(attrib)) sparse.data = numpy.load(subname(attrib) + '.data.npy', mmap_mode=mmap) sparse.indptr = numpy.load(subname(attrib) + '.indptr.npy', mmap_mode=mmap) sparse.indices = numpy.load(subname(attrib) + '.indices.npy', mmap_mode=mmap) setattr(obj, attrib, sparse) for attrib in getattr(obj, '__ignoreds', []): logger.info("setting ignored attribute %s to None" % (attrib)) setattr(obj, attrib, None) return obj
def run(self): if self.as_numpy: import numpy # don't clutter the global namespace with a dependency on numpy it = iter(self.corpus) while True: chunk = itertools.islice(it, self.chunksize) if self.as_numpy: # HACK XXX convert documents to numpy arrays, to save memory. # This also gives a scipy warning at runtime: # "UserWarning: indices array has non-integer dtype (float64)" wrapped_chunk = [[numpy.asarray(doc) for doc in chunk]] else: wrapped_chunk = [list(chunk)] if not wrapped_chunk[0]: self.q.put(None, block=True) break try: qsize = self.q.qsize() except NotImplementedError: qsize = '?' logger.debug("prepared another chunk of %i documents (qsize=%s)" % (len(wrapped_chunk[0]), qsize)) self.q.put(wrapped_chunk.pop(), block=True) #endclass InputQueue
def _load_specials(self, fname, mmap, compress, subname): """ Loads any attributes that were stored specially, and gives the same opportunity to recursively included SaveLoad instances. """ mmap_error = lambda x, y: IOError( 'Cannot mmap compressed object %s in file %s. ' % (x, y) + 'Use `load(fname, mmap=None)` or uncompress files manually.') for attrib in getattr(self, '__recursive_saveloads', []): cfname = '.'.join((fname, attrib)) logger.info("loading %s recursively from %s.* with mmap=%s" % ( attrib, cfname, mmap)) getattr(self, attrib)._load_specials(cfname, mmap, compress, subname) for attrib in getattr(self, '__numpys', []): logger.info("loading %s from %s with mmap=%s" % ( attrib, subname(fname, attrib), mmap)) if compress: if mmap: raise mmap_error(attrib, subname(fname, attrib)) val = np.load(subname(fname, attrib))['val'] else: val = np.load(subname(fname, attrib), mmap_mode=mmap) setattr(self, attrib, val) for attrib in getattr(self, '__scipys', []): logger.info("loading %s from %s with mmap=%s" % ( attrib, subname(fname, attrib), mmap)) sparse = unpickle(subname(fname, attrib)) if compress: if mmap: raise mmap_error(attrib, subname(fname, attrib)) with np.load(subname(fname, attrib, 'sparse')) as f: sparse.data = f['data'] sparse.indptr = f['indptr'] sparse.indices = f['indices'] else: sparse.data = np.load(subname(fname, attrib, 'data'), mmap_mode=mmap) sparse.indptr = np.load(subname(fname, attrib, 'indptr'), mmap_mode=mmap) sparse.indices = np.load(subname(fname, attrib, 'indices'), mmap_mode=mmap) setattr(self, attrib, sparse) for attrib in getattr(self, '__ignoreds', []): logger.info("setting ignored attribute %s to None" % (attrib)) setattr(self, attrib, None)
def _load_specials(self, fname, mmap, compress, subname): """ Loads any attributes that were stored specially, and gives the same opportunity to recursively included SaveLoad instances. """ mmap_error = lambda x, y: IOError( 'Cannot mmap compressed object %s in file %s. ' % (x, y) + 'Use `load(fname, mmap=None)` or uncompress files manually.') for attrib in getattr(self, '__recursive_saveloads', []): cfname = '.'.join((fname, attrib)) logger.info("loading %s recursively from %s.* with mmap=%s" % ( attrib, cfname, mmap)) getattr(self, attrib)._load_specials(cfname, mmap, compress, subname) for attrib in getattr(self, '__numpys', []): logger.info("loading %s from %s with mmap=%s" % ( attrib, subname(fname, attrib), mmap)) if compress: if mmap: raise mmap_error(attrib, subname(fname, attrib)) val = numpy.load(subname(fname, attrib))['val'] else: val = numpy.load(subname(fname, attrib), mmap_mode=mmap) setattr(self, attrib, val) for attrib in getattr(self, '__scipys', []): logger.info("loading %s from %s with mmap=%s" % ( attrib, subname(fname, attrib), mmap)) sparse = unpickle(subname(fname, attrib)) if compress: if mmap: raise mmap_error(attrib, subname(fname, attrib)) with numpy.load(subname(fname, attrib, 'sparse')) as f: sparse.data = f['data'] sparse.indptr = f['indptr'] sparse.indices = f['indices'] else: sparse.data = numpy.load(subname(fname, attrib, 'data'), mmap_mode=mmap) sparse.indptr = numpy.load(subname(fname, attrib, 'indptr'), mmap_mode=mmap) sparse.indices = numpy.load(subname(fname, attrib, 'indices'), mmap_mode=mmap) setattr(self, attrib, sparse) for attrib in getattr(self, '__ignoreds', []): logger.info("setting ignored attribute %s to None" % (attrib)) setattr(self, attrib, None)
