我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用numpy.any()。
def random_walk_rec(current, trace, length, successor_fn): import numpy.random as random if length == 0: return current else: sucs = successor_fn(current) first = random.randint(len(sucs)) now = first while True: suc = sucs[now] try: assert not np.any([np.all(np.equal(suc, t)) for t in trace]) result = random_walk_rec(suc, [*trace, suc], length-1, successor_fn) assert result is not None return result except AssertionError: now = (now+1)%len(sucs) if now == first: print("B",end="") return None else: continue
def test_against_numpy(self): """ Test iany against numpy.any """ stream = [np.zeros((8, 16, 2)) for _ in range(11)] stream[3][3,0,1] = 1 # so that np.all(axis = None) evaluates to False stack = np.stack(stream, axis = -1) with self.subTest('axis = None'): from_numpy = np.any(stack, axis = None) from_stream = last(iany(stream, axis = None)) self.assertEqual(from_numpy, from_stream) for axis in range(stack.ndim): with self.subTest('axis = {}'.format(axis)): from_numpy = np.any(stack, axis = axis) from_stream = last(iany(stream, axis = axis)) self.assertTrue(np.allclose(from_numpy, from_stream))
def _set_barcode_reads_metrics(self, read_type, read_type_set, bc): for genome in self.genomes: is_read_type = (genome, cr_constants.TRANSCRIPTOME_REGION) in read_type_set if is_read_type: barcode_reads = self._get_metric_attr( 'barcode_reads', genome, cr_constants.TRANSCRIPTOME_REGION, read_type) barcode_reads.add(bc) # Don't always-report the multi prefix for the barcode_reads metrics if self.has_multiple_genomes: is_read_type = any([(genome, cr_constants.TRANSCRIPTOME_REGION) in read_type_set for genome in self.genomes]) if is_read_type: multi_barcode_reads = self._get_metric_attr( 'barcode_reads', cr_constants.MULTI_REFS_PREFIX, cr_constants.TRANSCRIPTOME_REGION, read_type) multi_barcode_reads.add(bc)
def updateSpots(self, dataSet=None): if dataSet is None: dataSet = self.data invalidate = False if self.opts['pxMode']: mask = np.equal(dataSet['sourceRect'], None) if np.any(mask): invalidate = True opts = self.getSpotOpts(dataSet[mask]) sourceRect = self.fragmentAtlas.getSymbolCoords(opts) dataSet['sourceRect'][mask] = sourceRect self.fragmentAtlas.getAtlas() # generate atlas so source widths are available. dataSet['width'] = np.array(list(imap(QtCore.QRectF.width, dataSet['sourceRect'])))/2 dataSet['targetRect'] = None self._maxSpotPxWidth = self.fragmentAtlas.max_width else: self._maxSpotWidth = 0 self._maxSpotPxWidth = 0 self.measureSpotSizes(dataSet) if invalidate: self.invalidate()
def test_mask_value(self): result = self.model.predict(self.data) np.testing.assert_array_almost_equal( result[:, 1:, :], np.zeros(( self.data_size, self.max_length - 1, self.encoding_size )) ) np.testing.assert_equal( np.any( np.not_equal( result[:, 0:1, self.cell_units:], np.zeros((self.data_size, 1, self.cell_units)) ) ), True )
def test_image_data_mask(self): mask_cache_key = str(id(self.model.input)) + '_' + str(id(None)) mask_tensor = self.model._output_mask_cache[mask_cache_key] mask = mask_tensor.eval( session=K.get_session(), feed_dict={self.model.input: self.data} ) self.assertTrue( np.all( mask[:, self.x_start:self.x_end] ) ) self.assertFalse( np.any( mask[:, :self.x_start] ) ) self.assertFalse( np.any( mask[:, self.x_end:] ) )
def test_seq_data_mask(self): mask_cache_key = str(id(self.model.input)) + '_' + str(id(None)) mask_tensor = self.model._output_mask_cache[mask_cache_key] mask = mask_tensor.eval( session=K.get_session(), feed_dict={self.model.input: self.seq_data} ) self.assertTrue( np.all( mask[:, :self.seq_data_max_length] ) ) self.assertFalse( np.any( mask[:, self.seq_data_max_length:] ) )
def check_for_normalization(self, data_header): channels = [c.upper() for c in data_header.ch_names] if not data_header.info['sfreq'] == 100 and not self.resample: print('WARNING: Data not with 100hz. Use resample=True for resampling') # if not data_header.info['lowpass'] == 50: # print('WARNING: lowpass not at 50') if (not self.channels['EEG'] in channels) and not np.any(([ch in channels for ch in self.channels['EEG']])): print('WARNING: EEG channel missing') if not self.channels['EMG'] in channels: print('WARNING: EMG channel missing') if not self.channels['EOG'] in channels: print('WARNING: EOG channel missing') if self.references['RefEEG'] and not self.references['RefEEG'] in channels: print('WARNING: RefEEG channel missing') if self.references['RefEMG'] and not self.references['RefEMG'] in channels: print('WARNING: RefEMG channel missing') if self.references['RefEOG'] and not self.references['RefEOG'] in channels: print('WARNING: RefEOG channel missing')
def __init__(self, min_pt, max_pt, frame='unspecified'): """Initialize a box. Parameters ---------- min_pt : :obj:`numpy.ndarray` of float The minimum x, y, and (optionally) z points. max_pt : :obj:`numpy.ndarray` of float The maximum x, y, and (optionally) z points. frame : :obj:`str` The frame in which this box is placed. Raises ------ ValueError If max_pt is not strictly larger than min_pt in all dims. """ if np.any((max_pt - min_pt) < 0): raise ValueError('Min point must be smaller than max point') self._min_pt = min_pt self._max_pt = max_pt self._frame = frame
def _check_valid_data(self, data): """Checks that the incoming data is a 3 x #elements ndarray of normal vectors. Parameters ---------- data : :obj:`numpy.ndarray` The data to verify. Raises ------ ValueError If the data is not of the correct shape or type, or if the vectors therein are not normalized. """ if data.dtype.type != np.float32 and data.dtype.type != np.float64: raise ValueError('Must initialize normals clouds with a numpy float ndarray') if data.shape[0] != 3: raise ValueError('Illegal data array passed to normal cloud. Must have 3 coordinates') if len(data.shape) > 2: raise ValueError('Illegal data array passed to normal cloud. Must have 1 or 2 dimensions') if np.any((np.abs(np.linalg.norm(data, axis=0) - 1) > 1e-4) & (np.linalg.norm(data, axis=0) != 0)): raise ValueError('Illegal data array passed to normal cloud. Must have norm=1.0 or norm=0.0')
def process(self, nodemeta, timestamp, data, description): if self._job.getdata('perf')['active'] != True: self._error = ProcessingError.RAW_COUNTER_UNAVAILABLE return False if len(data[0]) == 0: # Ignore datapoints where no data stored return True if nodemeta.nodename not in self._data: self._data[nodemeta.nodename] = {"x": [], "t": []} info = self._data[nodemeta.nodename] info['x'].append(1.0 * numpy.sum(data[0])) info['t'].append(timestamp) if len(info['x']) > 1: if numpy.any(info['x'][-1] - info['x'][-2] < 0.0): self._error = ProcessingError.PMDA_RESTARTED_DURING_JOB return False return True
def process(self, nodemeta, timestamp, data, description): if self._job.getdata('perf')['active'] != True: self._error = ProcessingError.RAW_COUNTER_UNAVAILABLE return False ndata = numpy.array(data) if nodemeta.nodename not in self._first: self._first[nodemeta.nodename] = ndata return True if ndata.shape == self._first[nodemeta.nodename].shape: self._data[nodemeta.nodename] = numpy.sum(ndata - self._first[nodemeta.nodename]) if numpy.any(numpy.fabs(self._data[nodemeta.nodename]) != self._data[nodemeta.nodename]): self._error = ProcessingError.PMDA_RESTARTED_DURING_JOB return False else: # Perf counters changed during the job self._error = ProcessingError.RAW_COUNTER_UNAVAILABLE return False return True
def dspneumann(n, kr): """Derivative spherical Neumann (Bessel second kind) of order n at kr Parameters ---------- n : array_like Order kr: array_like Argument Returns ------- Yv' : complex float Derivative of spherical Neumann (Bessel second kind) """ n, kr = scalar_broadcast_match(n, kr) if _np.any(n < 0) | _np.any(_np.mod(n, 1) != 0) | _np.any(_np.mod(kr, 1) != 0): return spneumann(n, kr) * n / kr - spneumann(n + 1, kr) else: return scy.spherical_yn(n.astype(_np.int), kr.astype(_np.complex), derivative=True)
def crop_pad(image, corner, shape): ndim = len(corner) corner = [int(round(c)) for c in corner] shape = [int(round(s)) for s in shape] original = image.shape[-ndim:] zipped = zip(corner, shape, original) if np.any(c < 0 or c + s > o for (c, s, o) in zipped): no_padding = [(0, 0)] * (image.ndim - ndim) padding = [(max(-c, 0), max(c + s - o, 0)) for (c, s, o) in zipped] corner = [c + max(-c, 0) for c in corner] image_temp = np.pad(image, no_padding + padding, mode=str('constant')) else: image_temp = image no_crop = [slice(o+1) for o in image.shape[:-ndim]] crop = [slice(c, c+s) for (c, s) in zip(corner, shape)] return image_temp[no_crop + crop]
def slice_image(pos, image, radius): """ Slice a box around a group of features from an image. The box is the smallest box that contains all coordinates up to `radius` from any coordinate. Parameters ---------- image : ndarray The image that will be sliced pos : iterable An iterable (e.g. list or ndarray) that contains the feature positions radius : number or tuple of numbers Defines the size of the slice. Every pixel that has a distance lower or equal to `radius` to a feature position is included. Returns ------- tuple of: - the sliced image - the coordinate of the slice origin (top-left pixel) """ slices, origin = get_slice(pos, image.shape, radius) return image[slices], origin
def _logcdf(self, samples): lower = np.full(2, -np.inf) upper = norm.ppf(samples) limit_flags = np.zeros(2) if upper.shape[0] > 0: def func1d(upper1d): ''' Calculates the multivariate normal cumulative distribution function of a single sample. ''' return mvn.mvndst(lower, upper1d, limit_flags, self.theta)[1] vals = np.apply_along_axis(func1d, -1, upper) else: vals = np.empty((0, )) old_settings = np.seterr(divide='ignore') vals = np.log(vals) np.seterr(**old_settings) vals[np.any(samples == 0.0, axis=1)] = -np.inf vals[samples[:, 0] == 1.0] = np.log(samples[samples[:, 0] == 1.0, 1]) vals[samples[:, 1] == 1.0] = np.log(samples[samples[:, 1] == 1.0, 0]) return vals
def eval(self, t): # given a time vector t, return the design matrix column vector(s) if self.type is None: return np.array([]) hl = np.zeros((t.shape[0],)) ht = np.zeros((t.shape[0],)) if self.type in (0,2): hl[t >= self.year] = np.log10(1 + (t[t >= self.year] - self.year) / self.T) if self.type in (1,2): ht[t >= self.year] = 1 return np.append(ht,hl) if np.any(hl) else ht
def validate_transitions_cpu_old(transitions, **kwargs): pre = np.array(transitions[0]) suc = np.array(transitions[1]) base = setting['base'] width = pre.shape[1] // base height = pre.shape[1] // base load(width,height) pre_validation = validate_states(pre, **kwargs) suc_validation = validate_states(suc, **kwargs) results = [] for pre, suc, pre_validation, suc_validation in zip(pre, suc, pre_validation, suc_validation): if pre_validation and suc_validation: c = to_configs(np.array([pre, suc]), verbose=False) succs = successors(c[0], width, height) results.append(np.any(np.all(np.equal(succs, c[1]), axis=1))) else: results.append(False) return results
def validate_transitions(transitions, check_states=True, **kwargs): pre = np.array(transitions[0]) suc = np.array(transitions[1]) if check_states: pre_validation = validate_states(pre, verbose=False, **kwargs) suc_validation = validate_states(suc, verbose=False, **kwargs) pre_configs = to_configs(pre, verbose=False, **kwargs) suc_configs = to_configs(suc, verbose=False, **kwargs) results = [] if check_states: for pre_c, suc_c, pre_validation, suc_validation in zip(pre_configs, suc_configs, pre_validation, suc_validation): if pre_validation and suc_validation: succs = successors(pre_c) results.append(np.any(np.all(np.equal(succs, suc_c), axis=1))) else: results.append(False) else: for pre_c, suc_c in zip(pre_configs, suc_configs): succs = successors(pre_c) results.append(np.any(np.all(np.equal(succs, suc_c), axis=1))) return results
def transform(self, X): feature_range = self.feature_range X = check_array(X, copy=self.copy, ensure_2d=False, dtype=FLOAT_DTYPES) if X.ndim == 1: warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning) if np.any(X > feature_range[1]) or np.any(X < feature_range[0]): warnings.warn( "You got data that are out of the range: {}" .format(feature_range) ) X[X > feature_range[1]] = feature_range[1] X[X < feature_range[0]] = feature_range[0] return X
def vorEdges(vor, far): """ Given a voronoi tesselation, retuns the set of voronoi edges. far is the length of the "infinity" edges """ edges = [] for simplex in vor.ridge_vertices: simplex = numpy.asarray(simplex) if numpy.all(simplex >= 0): edge = {} edge['p1'], edge['p2'] = vor.vertices[simplex, 0], vor.vertices[simplex, 1] edge['p1'] = numpy.array([vor.vertices[simplex, 0][0], vor.vertices[simplex, 1][0]]) edge['p2'] = numpy.array([vor.vertices[simplex, 0][1], vor.vertices[simplex, 1][1]]) edge['t'] = (edge['p2'] - edge['p1']) / numpy.linalg.norm(edge['p2'] - edge['p1']) edges.append(edge) ptp_bound = vor.points.ptp(axis=0) center = vor.points.mean(axis=0) for pointidx, simplex in zip(vor.ridge_points, vor.ridge_vertices): simplex = numpy.asarray(simplex) if numpy.any(simplex < 0): i = simplex[simplex >= 0][0] # finite end Voronoi vertex t = vor.points[pointidx[1]] - vor.points[pointidx[0]] # tangent t /= numpy.linalg.norm(t) n = numpy.array([-t[1], t[0]]) # normal midpoint = vor.points[pointidx].mean(axis=0) direction = numpy.sign(numpy.dot(midpoint - center, n)) * n far_point = vor.vertices[i] + direction * ptp_bound.max() * far edge = {} edge['p1'], edge['p2'] = numpy.array([vor.vertices[i, 0], far_point[0]]), numpy.array( [vor.vertices[i, 1], far_point[1]]) edge['p1'], edge['p2'] = vor.vertices[i, :], far_point edge['t'] = (edge['p2'] - edge['p1']) / numpy.linalg.norm(edge['p2'] - edge['p1']) edges.append(edge) return edges
def sometrue(a, axis=None, out=None, keepdims=False): """ Check whether some values are true. Refer to `any` for full documentation. See Also -------- any : equivalent function """ arr = asanyarray(a) try: return arr.any(axis=axis, out=out, keepdims=keepdims) except TypeError: return arr.any(axis=axis, out=out)
def test_ddof_too_big(self): nanfuncs = [np.nanvar, np.nanstd] stdfuncs = [np.var, np.std] dsize = [len(d) for d in _rdat] for nf, rf in zip(nanfuncs, stdfuncs): for ddof in range(5): with warnings.catch_warnings(record=True) as w: warnings.simplefilter('always') tgt = [ddof >= d for d in dsize] res = nf(_ndat, axis=1, ddof=ddof) assert_equal(np.isnan(res), tgt) if any(tgt): assert_(len(w) == 1) assert_(issubclass(w[0].category, RuntimeWarning)) else: assert_(len(w) == 0)
def __iadd__(self, other): """ Add other to self in-place. """ m = getmask(other) if self._mask is nomask: if m is not nomask and m.any(): self._mask = make_mask_none(self.shape, self.dtype) self._mask += m else: if m is not nomask: self._mask += m self._data.__iadd__(np.where(self._mask, self.dtype.type(0), getdata(other))) return self
def __idiv__(self, other): """ Divide self by other in-place. """ other_data = getdata(other) dom_mask = _DomainSafeDivide().__call__(self._data, other_data) other_mask = getmask(other) new_mask = mask_or(other_mask, dom_mask) # The following 3 lines control the domain filling if dom_mask.any(): (_, fval) = ufunc_fills[np.divide] other_data = np.where(dom_mask, fval, other_data) self._mask |= new_mask self._data.__idiv__(np.where(self._mask, self.dtype.type(1), other_data)) return self
def __ifloordiv__(self, other): """ Floor divide self by other in-place. """ other_data = getdata(other) dom_mask = _DomainSafeDivide().__call__(self._data, other_data) other_mask = getmask(other) new_mask = mask_or(other_mask, dom_mask) # The following 3 lines control the domain filling if dom_mask.any(): (_, fval) = ufunc_fills[np.floor_divide] other_data = np.where(dom_mask, fval, other_data) self._mask |= new_mask self._data.__ifloordiv__(np.where(self._mask, self.dtype.type(1), other_data)) return self
def __ipow__(self, other): """ Raise self to the power other, in place. """ other_data = getdata(other) other_mask = getmask(other) with np.errstate(divide='ignore', invalid='ignore'): self._data.__ipow__(np.where(self._mask, self.dtype.type(1), other_data)) invalid = np.logical_not(np.isfinite(self._data)) if invalid.any(): if self._mask is not nomask: self._mask |= invalid else: self._mask = invalid np.copyto(self._data, self.fill_value, where=invalid) new_mask = mask_or(other_mask, invalid) self._mask = mask_or(self._mask, new_mask) return self
def test_comparisons(self): A = np.arange(100).reshape(10, 10) mA = matrix(A) mB = matrix(A) + 0.1 assert_(np.all(mB == A+0.1)) assert_(np.all(mB == matrix(A+0.1))) assert_(not np.any(mB == matrix(A-0.1))) assert_(np.all(mA < mB)) assert_(np.all(mA <= mB)) assert_(np.all(mA <= mA)) assert_(not np.any(mA < mA)) assert_(not np.any(mB < mA)) assert_(np.all(mB >= mA)) assert_(np.all(mB >= mB)) assert_(not np.any(mB > mB)) assert_(np.all(mA == mA)) assert_(not np.any(mA == mB)) assert_(np.all(mB != mA)) assert_(not np.all(abs(mA) > 0)) assert_(np.all(abs(mB > 0)))
def loadData (self, filename, verbose=True, replace_missing=True): ''' Get the data from a text file in one of 3 formats: matrix, sparse, binary_sparse''' if verbose: print("========= Reading " + filename) start = time.time() if self.use_pickle and os.path.exists (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle")): with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "r") as pickle_file: vprint (verbose, "Loading pickle file : " + os.path.join(self.tmp_dir, os.path.basename(filename) + ".pickle")) return pickle.load(pickle_file) if 'format' not in self.info.keys(): self.getFormatData(filename) if 'feat_num' not in self.info.keys(): self.getNbrFeatures(filename) data_func = {'dense':data_io.data, 'sparse':data_io.data_sparse, 'sparse_binary':data_io.data_binary_sparse} data = data_func[self.info['format']](filename, self.info['feat_num']) # INPORTANT: when we replace missing values we double the number of variables if self.info['format']=='dense' and replace_missing and np.any(map(np.isnan,data)): vprint (verbose, "Replace missing values by 0 (slow, sorry)") data = data_converter.replace_missing(data) if self.use_pickle: with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "wb") as pickle_file: vprint (verbose, "Saving pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle")) p = pickle.Pickler(pickle_file) p.fast = True p.dump(data) end = time.time() if verbose: print( "[+] Success in %5.2f sec" % (end - start)) return data
def getTypeProblem (self, solution_filename): ''' Get the type of problem directly from the solution file (in case we do not have an info file)''' if 'task' not in self.info.keys(): solution = np.array(data_converter.file_to_array(solution_filename)) target_num = solution.shape[1] self.info['target_num']=target_num if target_num == 1: # if we have only one column solution = np.ravel(solution) # flatten nbr_unique_values = len(np.unique(solution)) if nbr_unique_values < len(solution)/8: # Classification self.info['label_num'] = nbr_unique_values if nbr_unique_values == 2: self.info['task'] = 'binary.classification' self.info['target_type'] = 'Binary' else: self.info['task'] = 'multiclass.classification' self.info['target_type'] = 'Categorical' else: # Regression self.info['label_num'] = 0 self.info['task'] = 'regression' self.info['target_type'] = 'Numerical' else: # Multilabel or multiclass self.info['label_num'] = target_num self.info['target_type'] = 'Binary' if any(item > 1 for item in map(np.sum,solution.astype(int))): self.info['task'] = 'multilabel.classification' else: self.info['task'] = 'multiclass.classification' return self.info['task']
def conv2d(x,W,strides=[1,1,1,1],name=None): # return an op that convolves x with W strides = np.array(strides) if strides.size == 1: strides = np.array([1,strides,strides,1]) elif strides.size == 2: strides = np.array([1,strides[0],strides[1],1]) if np.any(strides < 1): strides = np.around(1./strides).astype(np.uint8) return tf.nn.conv2d_transpose(x,W,strides=strides.tolist(),padding='SAME',name=name) else: return tf.nn.conv2d(x,W,strides=strides.tolist(),padding='SAME',name=name)
def conv3d(x,W,strides=1,name=None): # return an op that convolves x with W strides = np.array(strides) if strides.size == 1: strides = np.array([1,strides,strides,strides[0],1]) elif strides.size == 3: strides = np.array([1,strides[0],strides[1],strides[2],1]) if np.any(strides < 1): strides = np.around(1./strides).astype(np.uint8) return tf.nn.conv3d_transpose(x,W,strides=strides.tolist(),padding='SAME',name=name) else: return tf.nn.conv3d(x,W,strides=strides.tolist(),padding='SAME',name=name)
def initial_guess(self, ranges): """Computes an initial position guess based on range measurements. The initial position is computed using Gauss-Newton method. The behavior can be modified with some parameters: `self.initial_guess_...`. Args: ranges (list of floats): Range measurements. Returns: initial_state (numpy.ndarray): Initial state vector (velocity components are zero). """ num_of_units = len(ranges) position = self.initial_guess_position H = np.zeros((num_of_units, position.size)) z = np.zeros((num_of_units, 1)) h = np.zeros((num_of_units, 1)) residuals = np.zeros((num_of_units, 1)) for i in xrange(self.initial_guess_iterations): self._compute_measurements_and_jacobians(ranges, position, h, H, z) new_residuals = z - h position = position + np.dot(self._solve_equation_least_squares(np.dot(H.T, H), H.T), new_residuals) if np.sum((new_residuals - residuals) ** 2) < self.initial_guess_tolerance: break residuals = new_residuals rospy.loginfo('initial guess residuals: {}'.format(residuals)) if np.any(np.abs(residuals) > self.initial_guess_residuals_threshold): # This initial guess is not good enough return None initial_state = np.zeros((6, 1)) initial_state[0:3] = position return initial_state
def overwrite_text(cursor, text): text_length = len(text) cursor.clearSelection() # Select the text after the current position (if any) current_position = cursor.position() cursor.movePosition(QTextCursor.Right, mode=QTextCursor.MoveAnchor, n=text_length) cursor.movePosition(QTextCursor.Left, mode=QTextCursor.KeepAnchor, n=cursor.position()-current_position) # Insert the text (will overwrite the selected text) cursor.insertText(text)
def store_tasks(self): self.stored_tasks = [cb.isChecked() for cb in self.task_comboboxes] if not numpy.any(self.stored_tasks): self.ui.btn_run.setEnabled(False) elif str(self.ui.edit_file.text()) != '': self.ui.btn_run.setEnabled(True) self.ui.btn_plots.setEnabled(True)
def _scale_data_to_float32(self, data): ''' This function will convert data from local data dtype into float32, the default format of the algorithm ''' if self.data_dtype != numpy.float32: data = data.astype(numpy.float32) if self.dtype_offset != 0: data -= self.dtype_offset if numpy.any(self.gain != 1): data *= self.gain return numpy.ascontiguousarray(data)
def _unscale_data_from_float32(self, data): ''' This function will convert data from float32 back to the original format of the file ''' if numpy.any(self.gain != 1): data /= self.gain if self.dtype_offset != 0: data += self.dtype_offset if (data.dtype != self.data_dtype) and (self.data_dtype != numpy.float32): data = data.astype(self.data_dtype) return data
def test_ignore_nan(self): """ Test that ignore_nan is working """ for axis in (0, 1, 2, 3, None): with self.subTest('axis = {}'.format(axis)): out = last(ireduce_ufunc(self.source, np.add, axis = axis, ignore_nan = True)) self.assertFalse(np.any(np.isnan(out))) # Dynamics generation of tests on binary ufuncs
def ffill_buffer_from_prior_values(freq, field, buffer_frame, digest_frame, pv_frame, raw=False): """ Forward-fill a buffer frame, falling back to the end-of-period values of a digest frame if the buffer frame has leading NaNs. """ # convert to ndarray if necessary digest_values = digest_frame if raw and isinstance(digest_frame, pd.DataFrame): digest_values = digest_frame.values buffer_values = buffer_frame if raw and isinstance(buffer_frame, pd.DataFrame): buffer_values = buffer_frame.values nan_sids = pd.isnull(buffer_values[0]) if np.any(nan_sids) and len(digest_values): # If we have any leading nans in the buffer and we have a non-empty # digest frame, use the oldest digest values as the initial buffer # values. buffer_values[0, nan_sids] = digest_values[-1, nan_sids] nan_sids = pd.isnull(buffer_values[0]) if np.any(nan_sids): # If we still have leading nans, fall back to the last known values # from before the digest. key_loc = pv_frame.index.get_loc((freq.freq_str, field)) filler = pv_frame.values[key_loc, nan_sids] buffer_values[0, nan_sids] = filler if raw: filled = ffill(buffer_values) return filled return buffer_frame.ffill()
def ffill_digest_frame_from_prior_values(freq, field, digest_frame, pv_frame, raw=False): """ Forward-fill a digest frame, falling back to the last known prior values if necessary. """ # convert to ndarray if necessary values = digest_frame if raw and isinstance(digest_frame, pd.DataFrame): values = digest_frame.values nan_sids = pd.isnull(values[0]) if np.any(nan_sids): # If we have any leading nans in the frame, use values from pv_frame to # seed values for those sids. key_loc = pv_frame.index.get_loc((freq.freq_str, field)) filler = pv_frame.values[key_loc, nan_sids] values[0, nan_sids] = filler if raw: filled = ffill(values) return filled return digest_frame.ffill()
def __new__(cls, field=None, frequency_delta=None, length_delta=None): """ field is a new field that was added. frequency is a FrequencyDelta representing a new frequency was added. length is a bar LengthDelta which is a frequency and a bar_count. If any field is None, then no change occurred of that type. """ return super(HistoryContainerDelta, cls).__new__( cls, field, frequency_delta, length_delta, )
def create_digest_panels(self, initial_sids, initial_dt): """ Initialize a RollingPanel for each unique panel frequency being stored by this container. Each RollingPanel pre-allocates enough storage space to service the highest bar-count of any history call that it serves. """ # Map from frequency -> first/last minute of the next digest to be # rolled for that frequency. first_window_starts = {} first_window_closes = {} # Map from frequency -> digest_panels. panels = {} for freq, largest_spec in iteritems(self.largest_specs): if largest_spec.bar_count == 1: # No need to allocate a digest panel; this frequency will only # ever use data drawn from self.buffer_panel. first_window_starts[freq] = freq.normalize(initial_dt) first_window_closes[freq] = freq.window_close( first_window_starts[freq] ) continue dt = initial_dt rp = self._create_digest_panel( dt, spec=largest_spec, window_starts=first_window_starts, window_closes=first_window_closes, ) panels[freq] = rp return panels, first_window_starts, first_window_closes
def update(self, data, algo_dt): """ Takes the bar at @algo_dt's @data, checks to see if we need to roll any new digests, then adds new data to the buffer panel. """ frame = self.frame_from_bardata(data, algo_dt) self.update_last_known_values() self.update_digest_panels(algo_dt, self.buffer_panel) self.buffer_panel.add_frame(algo_dt, frame)
def _set_mapping_metrics(self, read_type, read_type_set): for genome, region in itertools.product(self.genomes, self.regions): is_read_type = (genome, region) in read_type_set reads_frac = self._get_metric_attr('reads_frac', genome, region, read_type) reads_frac.add(1, filter=is_read_type) for region in self.regions: is_read_type = any([(genome, region) in read_type_set for genome in self.genomes]) multi_reads_frac = self._get_metric_attr('reads_frac', cr_constants.MULTI_REFS_PREFIX, region, read_type) multi_reads_frac.add(1, filter=is_read_type)
def mark_dupes_group_cb(self, gene_id, umis, dupe_type): total_counts = sum(umis.values()) total_umis = len(umis) if any([count > 1 for count in umis.itervalues()]): umi_hamming_distance = 0 else: umi_hamming_distance = cr_utils.get_kmers_hamming_distance(umis.keys()) for reference in [cr_utils.get_genome_from_str(gene_id, self.genomes), cr_constants.MULTI_REFS_PREFIX]: if total_counts > 0: reads_per_dupe_group_histogram = self._get_metric_attr( 'reads_per_dupe_group_histogram', reference, dupe_type) reads_per_dupe_group_histogram.add(total_counts) if total_umis > 0: umis_per_dupe_group_histogram = self._get_metric_attr( 'umis_per_dupe_group_histogram', reference, dupe_type) umis_per_dupe_group_histogram.add(total_umis) reads_per_molecule_histogram = self._get_metric_attr( 'reads_per_molecule_histogram', reference, dupe_type) for count in umis.itervalues(): reads_per_molecule_histogram.add(count) if umi_hamming_distance is not None: umi_hamming_distance_per_dupe_group_histogram = self._get_metric_attr( 'umi_hamming_distance_per_dupe_group_histogram', reference, dupe_type) umi_hamming_distance_per_dupe_group_histogram.add(umi_hamming_distance)
def merge_h5(in_filenames, out_filename): """ Merge a list of h5 files """ out_h5 = h5.File(out_filename, 'a') for filename in in_filenames: if filename is None: continue in_h5 = h5.File(filename, 'r') for name in in_h5.keys(): # If the dataset already exists, # They must be equal or one must be all-zero. if name in out_h5.keys(): src_data, dest_data = in_h5[name][()], out_h5[name][()] if src_data.dtype.kind != 'S' and dest_data.dtype.kind != 'S': # Both numeric if not np.any(src_data): # Source is all zero. Do nothing. continue elif not np.any(dest_data): # Dest is all zero. Overwrite. del out_h5[name] h5.h5o.copy(in_h5.id, name, out_h5.id, name) else: # Both non-zero. Assert equality and do nothing. assert np.array_equal(src_data, dest_data) else: # Either are non-numeric. Assert equality and do nothing. assert np.array_equal(src_data, dest_data) else: # Only exists in src. Copy to dest. h5.h5o.copy(in_h5.id, name, out_h5.id, name) out_h5.flush() out_h5.close()
def assert_allclose(x, y, rtol=1e-10, atol=1e-8): """Drop in replacement for `numpy.testing.assert_allclose` that shows the nonmatching elements""" if np.isscalar(x) and np.isscalar(y) == 1: return np.testing.assert_allclose(x, y, rtol=rtol, atol=atol) if x.shape != y.shape: raise AssertionError("Shape mismatch: %s vs %s" % (str(x.shape), str(y.shape))) d = ~np.isclose(x, y, rtol, atol) if np.any(d): miss = np.where(d)[0] raise AssertionError("""Mismatch of %d elements (%g %%) at the level of rtol=%g, atol=%g %s %s %s""" % (len(miss), len(miss)/x.size, rtol, atol, repr(miss), str(x[d]), str(y[d])))
def _nn_pose_fill(valid): """ Looks up closest True for each False and returns indices for fill-in-lookup In: [True, False, True, ... , False, True] Out: [0, 0, 2, ..., 212, 212] """ valid_inds, = np.where(valid) invalid_inds, = np.where(~valid) all_inds = np.arange(len(valid)) all_inds[invalid_inds] = -1 for j in range(10): fwd_inds = valid_inds + j bwd_inds = valid_inds - j # Forward fill invalid_inds, = np.where(all_inds < 0) fwd_fill_inds = np.intersect1d(fwd_inds, invalid_inds) all_inds[fwd_fill_inds] = all_inds[fwd_fill_inds-j] # Backward fill invalid_inds, = np.where(all_inds < 0) if not len(invalid_inds): break bwd_fill_inds = np.intersect1d(bwd_inds, invalid_inds) all_inds[bwd_fill_inds] = all_inds[bwd_fill_inds+j] # Check if any missing invalid_inds, = np.where(all_inds < 0) if not len(invalid_inds): break # np.set_printoptions(threshold=np.nan) # print valid.astype(np.int) # print np.array_str(all_inds) # print np.where(all_inds < 0) return all_inds
def roidb(self, target_hash, targets=[], every_k_frames=1, verbose=True, skip_empty=True): """ @param target_hash: target hash map (name -> unique id) @param targets: return only provided target names Returns (img, bbox, targets [hashed with target_hash (int32)]) """ self.check_ground_truth_availability() if every_k_frames > 1 and skip_empty: raise RuntimeError('roidb not meant for skipping frames,' 'and skipping empty simultaneously ') # Iterate through all images for idx, (t,ch,data) in enumerate(self.iterframes()): # Skip every k frames, if requested if idx % every_k_frames != 0: continue # Annotations may be empty, if # unlabeled, however we can request # to yield if its empty or not bboxes = data.annotation.bboxes if not len(bboxes) and skip_empty: continue target_names = data.annotation.pretty_names if len(targets): inds, = np.where([np.any([t in name for t in targets]) for name in target_names]) target_names = [target_names[ind] for ind in inds] bboxes = bboxes[inds] yield (data.img, bboxes, np.int32(map(lambda key: target_hash.get(key, -1), target_names)))
