我们从Python开源项目中,提取了以下50个代码示例,用于说明如何使用cPickle.HIGHEST_PROTOCOL。
def lcdict_to_pickle(lcdict, outfile=None): '''This just writes the lcdict to a pickle. If outfile is None, then will try to get the name from the lcdict['objectid'] and write to <objectid>-hptxtlc.pkl. If that fails, will write to a file named hptxtlc.pkl'. ''' if not outfile and lcdict['objectid']: outfile = '%s-hplc.pkl' % lcdict['objectid'] elif not outfile and not lcdict['objectid']: outfile = 'hplc.pkl' with open(outfile,'wb') as outfd: pickle.dump(lcdict, outfd, protocol=pickle.HIGHEST_PROTOCOL) if os.path.exists(outfile): LOGINFO('lcdict for object: %s -> %s OK' % (lcdict['objectid'], outfile)) return outfile else: LOGERROR('could not make a pickle for this lcdict!') return None
def set(self, key, value, timeout=None): if timeout is None: timeout = int(time() + self.default_timeout) elif timeout != 0: timeout = int(time() + timeout) filename = self._get_filename(key) self._prune() try: fd, tmp = tempfile.mkstemp(suffix=self._fs_transaction_suffix, dir=self._path) with os.fdopen(fd, 'wb') as f: pickle.dump(timeout, f, 1) pickle.dump(value, f, pickle.HIGHEST_PROTOCOL) rename(tmp, filename) os.chmod(filename, self._mode) except (IOError, OSError): return False else: return True
def gt_roidb(self): """ Return the database of ground-truth regions of interest. This function loads/saves from/to a cache file to speed up future calls. """ cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl') if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) print '{} gt roidb loaded from {}'.format(self.name, cache_file) return roidb # gt_roidb = [self._load_pascal_annotation(index) gt_roidb = [self._load_pascal_labels(index) for index in self.image_index] with open(cache_file, 'wb') as fid: cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL) print 'wrote gt roidb to {}'.format(cache_file) return gt_roidb
def selective_search_IJCV_roidb(self): """ Return the database of selective search regions of interest. Ground-truth ROIs are also included. This function loads/saves from/to a cache file to speed up future calls. """ cache_file = os.path.join(self.cache_path, '{:s}_selective_search_IJCV_top_{:d}_roidb.pkl'. format(self.name, self.config['top_k'])) if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) print '{} ss roidb loaded from {}'.format(self.name, cache_file) return roidb gt_roidb = self.gt_roidb() ss_roidb = self._load_selective_search_IJCV_roidb(gt_roidb) roidb = datasets.imdb.merge_roidbs(gt_roidb, ss_roidb) with open(cache_file, 'wb') as fid: cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL) print 'wrote ss roidb to {}'.format(cache_file) return roidb
def gt_roidb(self): """ return ground truth image regions database :return: imdb[image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] """ cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl') if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) print '{} gt roidb loaded from {}'.format(self.name, cache_file) return roidb gt_roidb = [self.load_pascal_annotation(index) for index in self.image_set_index] with open(cache_file, 'wb') as fid: cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL) print 'wrote gt roidb to {}'.format(cache_file) return gt_roidb
def selective_search_roidb(self, gt_roidb): """ get selective search roidb and ground truth roidb :param gt_roidb: ground truth roidb :return: roidb of selective search (ground truth included) """ cache_file = os.path.join(self.cache_path, self.name + '_ss_roidb.pkl') if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) print '{} ss roidb loaded from {}'.format(self.name, cache_file) return roidb if self.image_set != 'test': ss_roidb = self.load_selective_search_roidb(gt_roidb) roidb = IMDB.merge_roidbs(gt_roidb, ss_roidb) else: roidb = self.load_selective_search_roidb(None) with open(cache_file, 'wb') as fid: cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL) print 'wrote ss roidb to {}'.format(cache_file) return roidb
def gt_roidb(self): """ return ground truth image regions database :return: imdb[image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] """ cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl') if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) print '{} gt roidb loaded from {}'.format(self.name, cache_file) return roidb gt_roidb = [self.load_annotation(index) for index in self.image_set_index] with open(cache_file, 'wb') as fid: cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL) print 'wrote gt roidb to {}'.format(cache_file) return gt_roidb
def gt_roidb(self): """ Return the database of ground-truth regions of interest. This function loads/saves from/to a cache file to speed up future calls. """ cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl') if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) print '{} gt roidb loaded from {}'.format(self.name, cache_file) return roidb gt_roidb = [self._load_pascal_annotation(index) for index in self.image_index] with open(cache_file, 'wb') as fid: cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL) print 'wrote gt roidb to {}'.format(cache_file) return gt_roidb
def gt_roidb(self): """ Return the database of ground-truth regions of interest. This function loads/saves from/to a cache file to speed up future calls. """ cache_file = osp.join(self.cache_path, self.name + '_gt_roidb.pkl') if osp.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) print '{} gt roidb loaded from {}'.format(self.name, cache_file) return roidb gt_roidb = [self._load_coco_annotation(index) for index in self._image_index] with open(cache_file, 'wb') as fid: cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL) print 'wrote gt roidb to {}'.format(cache_file) return gt_roidb
def run(self): state = self._get(None) self._put(None, None) try: state = yield self.errors | self.kill_sessions() | self.main(state) except Stop: state = None finally: self._put(None, state) if self.file is not None: self.file.seek(0) self.file.truncate(0) pickle.dump(self.state, self.file, pickle.HIGHEST_PROTOCOL) self.file.flush() unlock_file(self.file) self.file.close()
def distribute_encode(socks): writable = [] while True: to_all, msg = yield idiokit.next() msg_bytes = cPickle.dumps(msg, cPickle.HIGHEST_PROTOCOL) data = struct.pack("!I", len(msg_bytes)) + msg_bytes if to_all: for sock in socks: yield sock.sendall(data) writable = [] else: while not writable: _, writable, _ = yield select.select((), socks, ()) writable = list(writable) yield writable.pop().sendall(data)
def save_params(self, weights_file, catched=False): """Save the model's parameters.""" f_dump = open(weights_file, "w") params_vls = [] if catched: if self.catched_params != []: params_vls = self.catched_params else: raise ValueError( "You asked to save catched params," + "but you didn't catch any!!!!!!!") else: for param in self.params: params_vls.append(param.get_value()) pkl.dump(params_vls, f_dump, protocol=pkl.HIGHEST_PROTOCOL) f_dump.close()
def save_params(self, weights_file, catched=False): """Save the model's params.""" with open(weights_file, "w") as f: if catched: if self.catched_params != []: params_vl = self.catched_params else: raise ValueError( "You asked to save catched params," + "but you didn't catch any!!!!!!!") else: params_vl = [param.get_value() for param in self.params] ft_extractor = False if self.ft_extractor is not None: ft_extractor = True stuff = {"layers_infos": self.layers_infos, "params_vl": params_vl, "tag": self.tag, "dropout": self.dropout, "ft_extractor": ft_extractor} pkl.dump(stuff, f, protocol=pkl.HIGHEST_PROTOCOL)
def save_params(self, weights_file, catched=False): """Save the model's params.""" with open(weights_file, "w") as f: if catched: if self.catched_params != []: params_vl = self.catched_params else: raise ValueError( "You asked to save catched params," + "but you didn't catch any!!!!!!!") else: params_vl = [param.get_value() for param in self.params] ft_extractor = False if self.ft_extractor is not None: ft_extractor = True stuff = {"layers_infos": self.layers_infos, "params_vl": params_vl, "tag": self.tag, "dropout": self.dropout, "ft_extractor": ft_extractor, "dic_keys": self.dic_keys, "config_arch": self.config_arch, "crop_size": self.crop_size} pkl.dump(stuff, f, protocol=pkl.HIGHEST_PROTOCOL)
def gt_roidb(self): """ Return the database of ground-truth regions of interest. This function loads/saves from/to a cache file to speed up future calls. """ cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl') if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) print '{} gt roidb loaded from {}'.format(self.name, cache_file) return roidb gt_roidb = [self._load_nyud2_annotation(index) for index in self.image_index] with open(cache_file, 'wb') as fid: cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL) print 'wrote gt roidb to {}'.format(cache_file) return gt_roidb
def set(self, key, value, timeout=None): timeout = self._normalize_timeout(timeout) filename = self._get_filename(key) self._prune() try: fd, tmp = tempfile.mkstemp(suffix=self._fs_transaction_suffix, dir=self._path) with os.fdopen(fd, 'wb') as f: pickle.dump(timeout, f, 1) pickle.dump(value, f, pickle.HIGHEST_PROTOCOL) rename(tmp, filename) os.chmod(filename, self._mode) except (IOError, OSError): return False else: return True
def pickle_transitions_matrix_data(): transitions = pickle.load( open( "/ssd/ddimitrov/pickle/transitions", "rb" ) ) vocab = pickle.load( open( "/ssd/ddimitrov/pickle/vocab", "rb" ) ) i_indices = array.array(str("l")) j_indices = array.array(str("l")) values = array.array(str("d")) for s, targets in transitions.iteritems(): for t, v in targets.iteritems(): i_indices.append(vocab[s]) j_indices.append(vocab[t]) values.append(v) i_indices = np.frombuffer(i_indices, dtype=np.int_) j_indices = np.frombuffer(j_indices, dtype=np.int_) values = np.frombuffer(values, dtype=np.float64) transition_matrix=[i_indices,j_indices,values] pickle.dump(transition_matrix, open("/ssd/ddimitrov/pickle/transition_matrix", "wb"), protocol=pickle.HIGHEST_PROTOCOL) print "transition_matrix"
def save(self, file_path, gsums=None, learning_rate=None, validation_ppl_history=None, best_validation_ppl=None, epoch=None, random_state=None): import cPickle state = { "type": self.__class__.__name__, "n_hidden": self.n_hidden, "x_vocabulary": self.x_vocabulary, "y_vocabulary": self.y_vocabulary, "stage1_model_file_name": self.stage1_model_file_name if hasattr(self, "stage1_model_file_name") else None, "params": [p.get_value(borrow=True) for p in self.params], "gsums": [s.get_value(borrow=True) for s in gsums] if gsums else None, "learning_rate": learning_rate, "validation_ppl_history": validation_ppl_history, "epoch": epoch, "random_state": random_state } with open(file_path, 'wb') as f: cPickle.dump(state, f, protocol=cPickle.HIGHEST_PROTOCOL)
def cache_it(self, key, f, time_expire): if self.debug: self.r_server.incr('web2py_cache_statistics:misses') cache_set_key = self.cache_set_key expire_at = int(time.time() + time_expire) + 120 bucket_key = "%s:%s" % (cache_set_key, expire_at / 60) value = f() value_ = pickle.dumps(value, pickle.HIGHEST_PROTOCOL) if time_expire == 0: time_expire = 1 self.r_server.setex(key, time_expire, value_) # print '%s will expire on %s: it goes in bucket %s' % (key, time.ctime(expire_at)) # print 'that will expire on %s' % (bucket_key, time.ctime(((expire_at / 60) + 1) * 60)) p = self.r_server.pipeline() # add bucket to the fixed set p.sadd(cache_set_key, bucket_key) # sets the key p.setex(key, time_expire, value_) # add the key to the bucket p.sadd(bucket_key, key) # expire the bucket properly p.expireat(bucket_key, ((expire_at / 60) + 1) * 60) p.execute() return value
def set(self, key, value, timeout=None): if timeout is None: timeout = self.default_timeout filename = self._get_filename(key) self._prune() try: fd, tmp = tempfile.mkstemp(suffix=self._fs_transaction_suffix, dir=self._path) f = os.fdopen(fd, 'wb') try: pickle.dump(int(time() + timeout), f, 1) pickle.dump(value, f, pickle.HIGHEST_PROTOCOL) finally: f.close() rename(tmp, filename) os.chmod(filename, self._mode) except (IOError, OSError): pass
def set(self, key, value, timeout=None): if timeout is None: timeout = self.default_timeout filename = self._get_filename(key) self._prune() try: fd, tmp = tempfile.mkstemp(suffix=self._fs_transaction_suffix, dir=self._path) with os.fdopen(fd, 'wb') as f: pickle.dump(int(time() + timeout), f, 1) pickle.dump(value, f, pickle.HIGHEST_PROTOCOL) rename(tmp, filename) os.chmod(filename, self._mode) except (IOError, OSError): return False else: return True
def save_track_proto_to_zip(track_proto, save_file): zf = zipfile.ZipFile(save_file, 'w', allowZip64=True) print "Writing to zip file {}...".format(save_file) track_id = 0 for track in track_proto['tracks']: track_obj = {} for key in track[0]: try: track_obj[key] = np.asarray([box[key] for box in track]) except KeyError: continue zf.writestr('{:06d}.pkl'.format(track_id), cPickle.dumps(track_obj, cPickle.HIGHEST_PROTOCOL)) track_id += 1 if (track_id + 1) % 1000 == 0: print "\t{} tracks written.".format(track_id + 1) print "\tTotally {} tracks written.".format(track_id + 1) zf.close()
def imdb_proposals(net, imdb, rank, count, output_dir): """Generate RPN proposals on all images in an imdb.""" _t = Timer() for i in xrange(rank, imdb.num_images, count): # imdb.num_images im = cv2.imread(imdb.image_path_at(i)) _t.tic() imdb_boxes, scores = im_proposals(net, im) with open(osp.join(output_dir, "{}.pkl".format(i)), "wb") as fp: cPickle.dump(imdb_boxes, fp, cPickle.HIGHEST_PROTOCOL) _t.toc() print 'im_proposals: {:d}/{:d} {:.3f}s' \ .format(i + 1, imdb.num_images, _t.average_time) if 0: dets = np.hstack((imdb_boxes, scores)) # from IPython import embed; embed() _vis_proposals(im, dets[:3, :], thresh=0.9) plt.show()
def gt_roidb(self): """ Return the database of ground-truth regions of interest. This function loads/saves from/to a cache file to speed up future calls. """ cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl') if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) print '{} gt roidb loaded from {}'.format(self.name, cache_file) return roidb gt_roidb = [self._load_imagenet_annotation(index) for index in self.image_index] with open(cache_file, 'wb') as fid: cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL) print 'wrote gt roidb to {}'.format(cache_file) return gt_roidb
def gt_segdb(self): """ return ground truth image regions database :return: imdb[image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] """ cache_file = os.path.join(self.cache_path, self.name + '_gt_segdb.pkl') if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: segdb = cPickle.load(fid) print '{} gt segdb loaded from {}'.format(self.name, cache_file) return segdb gt_segdb = [self.load_pascal_segmentation_annotation(index) for index in self.image_set_index] with open(cache_file, 'wb') as fid: cPickle.dump(gt_segdb, fid, cPickle.HIGHEST_PROTOCOL) print 'wrote gt segdb to {}'.format(cache_file) return gt_segdb
def selective_search_roidb(self, gt_roidb, append_gt=False): """ get selective search roidb and ground truth roidb :param gt_roidb: ground truth roidb :param append_gt: append ground truth :return: roidb of selective search """ cache_file = os.path.join(self.cache_path, self.name + '_ss_roidb.pkl') if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) print '{} ss roidb loaded from {}'.format(self.name, cache_file) return roidb if append_gt: print 'appending ground truth annotations' ss_roidb = self.load_selective_search_roidb(gt_roidb) roidb = IMDB.merge_roidbs(gt_roidb, ss_roidb) else: roidb = self.load_selective_search_roidb(gt_roidb) with open(cache_file, 'wb') as fid: cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL) print 'wrote ss roidb to {}'.format(cache_file) return roidb
def gt_roidb(self): """ Return the database of ground-truth regions of interest. This function loads/saves from/to a cache file to speed up future calls. """ cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl') if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) print '{} gt roidb loaded from {}'.format(self.name, cache_file) return roidb print self._image_index gt_roidb = [self._load_kitti_annotation(index) for index in self._image_index] with open(cache_file, 'wb') as fid: cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL) print 'wrote gt roidb to {}'.format(cache_file) return gt_roidb
def selective_search_IJCV_roidb(self): """ Return the database of selective search regions of interest. Ground-truth ROIs are also included. This function loads/saves from/to a cache file to speed up future calls. """ cache_file = os.path.join(self.cache_path, '{:s}_selective_search_IJCV_top_{:d}_roidb.pkl'. format(self.name, self.config['top_k'])) if os.path.exists(cache_file): with open(cache_file, 'rb') as fid: roidb = cPickle.load(fid) print '{} ss roidb loaded from {}'.format(self.name, cache_file) return roidb gt_roidb = self.gt_roidb() ss_roidb = self._load_selective_search_IJCV_roidb(gt_roidb) roidb = imdb.merge_roidbs(gt_roidb, ss_roidb) with open(cache_file, 'wb') as fid: cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL) print 'wrote ss roidb to {}'.format(cache_file) return roidb
def cache_it(self, key, f, time_expire): if self.debug: self.r_server.incr('web2py_cache_statistics:misses') cache_set_key = self.cache_set_key expireat = int(time.time() + time_expire) + 120 bucket_key = "%s:%s" % (cache_set_key, expireat / 60) value = f() value_ = pickle.dumps(value, pickle.HIGHEST_PROTOCOL) if time_expire == 0: time_expire = 1 self.r_server.setex(key, time_expire, value_) #print '%s will expire on %s: it goes in bucket %s' % (key, time.ctime(expireat)) #print 'that will expire on %s' % (bucket_key, time.ctime(((expireat/60) + 1)*60)) p = self.r_server.pipeline() #add bucket to the fixed set p.sadd(cache_set_key, bucket_key) #sets the key p.setex(key, time_expire, value_) #add the key to the bucket p.sadd(bucket_key, key) #expire the bucket properly p.expireat(bucket_key, ((expireat / 60) + 1) * 60) p.execute() return value
def set_global(self, name, value): """Adds a global, or updates it if it already exists. Also removes the global from the list of unpicklable names. Args: name: the name of the global to remove value: any picklable value """ blob = pickle.dumps(value, pickle.HIGHEST_PROTOCOL) if name in self.global_names: index = self.global_names.index(name) self.globals[index] = blob else: self.global_names.append(name) self.globals.append(blob) self.remove_unpicklable_name(name)
def safe_apply(self, key, function, default_value=None): """ Safely apply a function to the value of a key in storage and set the return value of the function to it. Return the result of applying the function. """ key = self.key_filter_in(key) exists = True try: val_file = recfile.open(key, mode='r+b', path=self.folder) except IOError: exists = False val_file = recfile.open(key, mode='wb', path=self.folder) self.wait_portalock(val_file) if exists: timestamp, value = pickle.load(val_file) else: value = default_value new_value = function(value) val_file.seek(0) pickle.dump((time.time(), new_value), val_file, pickle.HIGHEST_PROTOCOL) val_file.truncate() val_file.close() return new_value