我们从Python开源项目中，提取了以下19个代码示例，用于说明如何使用torch.multiprocessing()。
def share_memory_(self): """Moves the storage to shared memory. This is a no-op for storages already in shared memory and for CUDA storages, which do not need to be moved for sharing across processes. Storages in shared memory cannot be resized. Returns: self """ from torch.multiprocessing import get_sharing_strategy if self.is_cuda: pass # CUDA doesn't use POSIX shared memory elif get_sharing_strategy() == 'file_system': self._share_filename_() else: self._share_fd_() return self
def __init__(self, loader): self.dataset = loader.dataset self.batch_size = loader.batch_size self.collate_fn = loader.collate_fn self.sampler = loader.sampler self.num_workers = loader.num_workers self.samples_remaining = len(self.sampler) self.sample_iter = iter(self.sampler) if self.num_workers > 0: self.index_queue = multiprocessing.Queue() self.data_queue = multiprocessing.Queue() self.batches_outstanding = 0 self.shutdown = False self.send_idx = 0 self.rcvd_idx = 0 self.reorder_dict = {} self.workers = [ multiprocessing.Process( target=_worker_loop, args=(self.dataset, self.index_queue, self.data_queue, self.collate_fn)) for _ in range(self.num_workers)] for w in self.workers: w.daemon = True # ensure that the worker exits on process exit w.start() # prime the prefetch loop for _ in range(2 * self.num_workers): self._put_indices()
def _new_shared(cls, size): """Creates a new storage in shared memory with the same data type""" from torch.multiprocessing import get_sharing_strategy if cls.is_cuda: return cls(size) elif get_sharing_strategy() == 'file_system': return cls._new_using_filename(size) else: return cls._new_using_fd(size)
def fill_folder(folder, links): sigils = [] pool = torch.multiprocessing.Pool(3) result = pool.map_async(createsymbol, [x[1] for x in links]) sigils = result.get() pool.close() pool.join() for idx,im in enumerate(sigils): im.save(folder+links[idx][0]+".png")
def __init__(self, loader): self.dataset = loader.dataset self.collate_fn = loader.collate_fn self.batch_sampler = loader.batch_sampler self.num_workers = loader.num_workers self.pin_memory = loader.pin_memory self.done_event = threading.Event() self.sample_iter = iter(self.batch_sampler) if self.num_workers > 0: self.index_queue = multiprocessing.SimpleQueue() self.data_queue = multiprocessing.SimpleQueue() self.batches_outstanding = 0 self.shutdown = False self.send_idx = 0 self.rcvd_idx = 0 self.reorder_dict = {} self.workers = [ multiprocessing.Process( target=_worker_loop, args=(self.dataset, self.index_queue, self.data_queue, self.collate_fn)) for _ in range(self.num_workers)] for w in self.workers: w.daemon = True # ensure that the worker exits on process exit w.start() if self.pin_memory: in_data = self.data_queue self.data_queue = queue.Queue() self.pin_thread = threading.Thread( target=_pin_memory_loop, args=(in_data, self.data_queue, self.done_event)) self.pin_thread.daemon = True self.pin_thread.start() # prime the prefetch loop for _ in range(2 * self.num_workers): self._put_indices()
def __call__(self,pixel_no): ''' call instance so that train_pixel can be called with multiprocessing and still have all of the class instance variables :params pixel_no: Pixel number that is going to be trained ''' return self.train_pixel(pixel_no)
def __init__(self, dataset, batch_size=1, shuffle=False, sampler=None, batch_sampler=None, num_workers=0, collate_fn=default_collate, pin_memory=False, drop_last=False, timeout=0, worker_init_fn=None): self.dataset = dataset self.batch_size = batch_size self.num_workers = num_workers self.collate_fn = collate_fn self.pin_memory = pin_memory self.drop_last = drop_last self.timeout = timeout self.worker_init_fn = worker_init_fn if timeout < 0: raise ValueError('timeout option should be non-negative') if batch_sampler is not None: if batch_size > 1 or shuffle or sampler is not None or drop_last: raise ValueError('batch_sampler is mutually exclusive with ' 'batch_size, shuffle, sampler, and drop_last') if sampler is not None and shuffle: raise ValueError('sampler is mutually exclusive with shuffle') if self.num_workers < 0: raise ValueError('num_workers cannot be negative; ' 'use num_workers=0 to disable multiprocessing.') if batch_sampler is None: if sampler is None: if shuffle: sampler = RandomSampler(dataset) else: sampler = SequentialSampler(dataset) batch_sampler = BatchSampler(sampler, batch_size, drop_last) self.sampler = sampler self.batch_sampler = batch_sampler
def __init__(self, loader): self.dataset = loader.dataset self.collate_fn = loader.collate_fn self.batch_sampler = loader.batch_sampler self.num_workers = loader.num_workers self.pin_memory = loader.pin_memory self.done_event = threading.Event() self.sample_iter = iter(self.batch_sampler) if self.num_workers > 0: self.index_queue = multiprocessing.Queue() self.data_queue = multiprocessing.Queue() self.batches_outstanding = 0 self.shutdown = False self.send_idx = 0 self.rcvd_idx = 0 self.reorder_dict = {} self.workers = [ multiprocessing.Process( target=_worker_loop, args=(self.dataset, self.index_queue, self.data_queue, self.collate_fn)) for _ in range(self.num_workers)] for w in self.workers: w.daemon = True # ensure that the worker exits on process exit w.start() # prime the prefetch loop for _ in range(2 * self.num_workers): self._put_indices()
def __init__(self, loader): self.dataset = loader.dataset self.batch_size = loader.batch_size self.collate_fn = loader.collate_fn self.sampler = loader.sampler self.num_workers = loader.num_workers self.pin_memory = loader.pin_memory self.done_event = threading.Event() self.samples_remaining = len(self.sampler) self.sample_iter = iter(self.sampler) if self.num_workers > 0: self.index_queue = multiprocessing.SimpleQueue() self.data_queue = multiprocessing.SimpleQueue() self.batches_outstanding = 0 self.shutdown = False self.send_idx = 0 self.rcvd_idx = 0 self.reorder_dict = {} self.workers = [ multiprocessing.Process( target=_worker_loop, args=(self.dataset, self.index_queue, self.data_queue, self.collate_fn)) for _ in range(self.num_workers)] for w in self.workers: w.daemon = True # ensure that the worker exits on process exit w.start() if self.pin_memory: in_data = self.data_queue self.data_queue = queue.Queue() self.pin_thread = threading.Thread( target=_pin_memory_loop, args=(in_data, self.data_queue, self.done_event)) self.pin_thread.daemon = True self.pin_thread.start() # prime the prefetch loop for _ in range(2 * self.num_workers): self._put_indices()
def __init__(self, loader): self.dataset = loader.dataset self.batch_size = loader.batch_size self.collate_fn = loader.collate_fn self.sampler = loader.sampler self.num_workers = loader.num_workers self.pin_memory = loader.pin_memory self.drop_last = loader.drop_last self.done_event = threading.Event() self.samples_remaining = len(self.sampler) self.sample_iter = iter(self.sampler) if self.num_workers > 0: self.index_queue = multiprocessing.SimpleQueue() self.data_queue = multiprocessing.SimpleQueue() self.batches_outstanding = 0 self.shutdown = False self.send_idx = 0 self.rcvd_idx = 0 self.reorder_dict = {} self.workers = [ multiprocessing.Process( target=_worker_loop, args=(self.dataset, self.index_queue, self.data_queue, self.collate_fn)) for _ in range(self.num_workers)] for w in self.workers: w.daemon = True # ensure that the worker exits on process exit w.start() if self.pin_memory: in_data = self.data_queue self.data_queue = queue.Queue() self.pin_thread = threading.Thread( target=_pin_memory_loop, args=(in_data, self.data_queue, self.done_event)) self.pin_thread.daemon = True self.pin_thread.start() # prime the prefetch loop for _ in range(2 * self.num_workers): self._put_indices()
def create_circle(): IMAGE_WIDTH = 300 IMAGE_HEIGHT = IMAGE_WIDTH IMAGE_SIZE = (IMAGE_WIDTH, IMAGE_HEIGHT) FONT_SIZE = 100 NSYMBOLS = 15 BORDER_WIDTH = int(0.6 * FONT_SIZE) linksize = 3 full_link = "".join(["".join(random.sample(string.ascii_uppercase, linksize)) for i in range(NSYMBOLS)]) links = [full_link[i:i+linksize] for i in range(0, len(full_link), linksize)] nrotations = 8 rotations = [int(i*360/nrotations) for i in range(nrotations)] sigils = [] totensor = torchvision.transforms.ToTensor() pool = torch.multiprocessing.Pool(3) result = pool.map_async(lambda x:createsymbol(x, IMAGE_SIZE), links) while not result.ready(): print("# Symbols not yet processed: {}".format(result._number_left)) time.sleep(5) sigils = result.get() pool.close() pool.join() maxwidth = 0 for im in sigils: maxwidth = max(im.size[0], maxwidth) # 0.18 for touching radius = 0.25 * maxwidth * math.ceil(NSYMBOLS / 4) baseim = Image.new("L", (int(2*radius + 2*BORDER_WIDTH), int(2*radius + 2*BORDER_WIDTH))) center = BORDER_WIDTH + radius for idx,im in enumerate(sigils): t = Image.new("L", baseim.size) x = center + radius*math.sin(2*math.pi*idx/NSYMBOLS) - im.size[0]/2 y = center + radius*math.cos(2*math.pi*idx/NSYMBOLS) - im.size[1]/2 t.paste(im, (int(x), int(y))) baseim = ImageChops.lighter(baseim, t) IMAGE_WIDTH = 300 IMAGE_HEIGHT = IMAGE_WIDTH IMAGE_SIZE = (IMAGE_WIDTH, IMAGE_HEIGHT) FONT_SIZE = 150 im = createsymbol("".join(random.sample(string.ascii_uppercase, 3))) t = Image.new("L", baseim.size) x = center - im.size[0]/2 y = center - im.size[1]/2 t.paste(im, (int(x), int(y))) baseim = ImageChops.lighter(baseim, t) baseim.save("out.png")
def __init__(self, loader): self.dataset = loader.dataset self.collate_fn = loader.collate_fn self.batch_sampler = loader.batch_sampler self.num_workers = loader.num_workers self.pin_memory = loader.pin_memory and torch.cuda.is_available() self.timeout = loader.timeout self.done_event = threading.Event() self.sample_iter = iter(self.batch_sampler) if self.num_workers > 0: self.worker_init_fn = loader.worker_init_fn self.index_queue = multiprocessing.SimpleQueue() self.worker_result_queue = multiprocessing.SimpleQueue() self.batches_outstanding = 0 self.worker_pids_set = False self.shutdown = False self.send_idx = 0 self.rcvd_idx = 0 self.reorder_dict = {} base_seed = torch.LongTensor(1).random_()[0] self.workers = [ multiprocessing.Process( target=_worker_loop, args=(self.dataset, self.index_queue, self.worker_result_queue, self.collate_fn, base_seed + i, self.worker_init_fn, i)) for i in range(self.num_workers)] if self.pin_memory or self.timeout > 0: self.data_queue = queue.Queue() self.worker_manager_thread = threading.Thread( target=_worker_manager_loop, args=(self.worker_result_queue, self.data_queue, self.done_event, self.pin_memory, torch.cuda.current_device())) self.worker_manager_thread.daemon = True self.worker_manager_thread.start() else: self.data_queue = self.worker_result_queue for w in self.workers: w.daemon = True # ensure that the worker exits on process exit w.start() _update_worker_pids(id(self), tuple(w.pid for w in self.workers)) _set_SIGCHLD_handler() self.worker_pids_set = True # prime the prefetch loop for _ in range(2 * self.num_workers): self._put_indices()
