我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用six.moves.queue.Empty()。
def run_once(self): """Accumulate records and flush when accumulator is ready.""" try: record = self.queue.get(timeout=0.05) except queue.Empty: record = None else: success = self._accumulator.try_append(record) if not success: self.flush() success = self._accumulator.try_append(record) assert success, "Failed to accumulate even after flushing" self.queue.task_done() is_ready = self._accumulator.is_ready() force_flush = not self._running and record is None if is_ready or force_flush: self.flush()
def _commit(self): bulk = [] stop = False while True: while len(bulk) < 50 and not stop: try: obj = self.elastic_bulk.get(timeout=3) except queue.Empty: break if obj is None: stop = True else: bulk.append(obj) if bulk: try: self.helper.bulk(self.elastic, bulk) except Exception as err: LOGGER.exception('es index error: %s', err) bulk = [] if stop: break
def run(self): while True: if self.is_shutdown: return try: while True: run_at, i, task = self._queue.get(block=True, timeout=None) if self.is_shutdown: if task: log.debug("Not executing scheduled task due to Scheduler shutdown") return if run_at <= time.time(): self._scheduled_tasks.discard(task) fn, args, kwargs = task kwargs = dict(kwargs) future = self._executor.submit(fn, *args, **kwargs) future.add_done_callback(self._log_if_failed) else: self._queue.put_nowait((run_at, i, task)) break except Queue.Empty: pass time.sleep(0.1)
def run(self): futures = queue.Queue(maxsize=121) self.start_profile() for i in range(self.num_queries): if i >= 120: old_future = futures.get_nowait() old_future.result() key = "{}-{}".format(self.thread_num, i) future = self.run_query(key) futures.put_nowait(future) while True: try: futures.get_nowait().result() except queue.Empty: break self.finish_profile
def run(self): futures = queue.Queue(maxsize=121) self.start_profile() for i in range(self.num_queries): if i > 0 and i % 120 == 0: # clear the existing queue while True: try: futures.get_nowait().result() except queue.Empty: break key = "{0}-{1}".format(self.thread_num, i) future = self.run_query(key) futures.put_nowait(future) while True: try: futures.get_nowait().result() except queue.Empty: break self.finish_profile()
def worker_exec(self, queue_timeout=2, **kwargs): while True: if self.signal.get('reach_max_num'): self.logger.info('downloaded image reached max num, thread %s' ' exit', threading.current_thread().name) break try: url = self.in_queue.get(timeout=queue_timeout) except queue.Empty: if self.signal.get('feeder_exited'): self.logger.info('no more page urls to parse, thread %s' ' exit', threading.current_thread().name) break else: self.logger.info('%s is waiting for new page urls', threading.current_thread().name) continue except Exception as e: self.logger.error('exception caught in thread %s: %s', threading.current_thread().name, e) continue else: self.logger.debug('start downloading page {}'.format(url)) self.output({'file_url': url})
def watch_once(self, key, timeout=None, **kwargs): """Watch a key and stops after the first event. :param key: key to watch :param timeout: (optional) timeout in seconds. :returns: event """ event_queue = queue.Queue() def callback(event): event_queue.put(event) w = watch.Watcher(self, key, callback, **kwargs) try: return event_queue.get(timeout=timeout) except queue.Empty: raise exceptions.WatchTimedOut() finally: w.stop()
def test_cluster_proxy_pool(): with patch('django_nameko.rpc.ClusterRpcProxy') as FakeClusterRpcProxy: pool = rpc.ClusterRpcProxyPool(dict(), pool_size=2) pool.start() assert pool.queue.qsize() == 2 with pool.next() as client: assert pool.queue.qsize() == 1 client.foo.bar() assert call().start().foo.bar() in FakeClusterRpcProxy.mock_calls with pool.next(): assert pool.queue.qsize() == 0 tools.assert_raises(queue_six.Empty, pool.next, timeout=1) assert pool.queue.qsize() == 1 assert pool.queue.qsize() == 2 pool.stop()
def events(self, start_date, end_date, frequency): running = True self.clock_engine_thread.start() self.quotation_engine_thread.start() while running: real_dt = datetime.datetime.now() while True: try: dt, event_type = self.event_queue.get(timeout=1) break except Empty: continue system_log.debug("real_dt {}, dt {}, event {}", real_dt, dt, event_type) yield Event(event_type, real_dt, dt)
def events(self, start_date, end_date, frequency): running = True self.clock_engine_thread.start() self.quotation_engine_thread.start() while running: real_dt = datetime.datetime.now() while True: try: dt, event_type = self.event_queue.get(timeout=1) break except Empty: continue system_log.debug("real_dt {}, dt {}, event {}", real_dt, dt, event_type) yield Event(event_type, calendar_dt=real_dt, trading_dt=dt)
def consume_queue(queue, cascade_stop): """Consume the queue by reading lines off of it and yielding them.""" while True: try: item = queue.get(timeout=0.1) except Empty: yield None continue # See https://github.com/docker/compose/issues/189 except thread.error: raise ShutdownException() if item.exc: raise item.exc if item.is_stop: if cascade_stop: raise StopIteration else: continue yield item.item
def run(self): self._is_running = True while self._is_running: if self.consumer.is_connected(): producer = kombu.Producer(self.consumer._channel, on_return=self.consumer._handle_return) try: queued_request = self._out_queue.get(timeout=0.5) if True: # with kombu.producers[self.consumer.get_connection()].acquire(block=True) as producer: # producer.on_return = print try: self._dispatch_request(queued_request, producer) except Exception as e: # except ConnectionResetError: log.debug('Failed to dispatch request, re-enqueueing again, error was: {}'.format( str(e) )) self.enqueue(queued_request) except Empty: continue else: sleep(0.5) log.debug('Waiting for consumer to be ready...')
def refresh(self, data=None, force=False): widget = self.frame.body while isinstance(widget, urwid.Overlay): widget = widget.contents[0][0] interested = force invalidate = False try: while True: event = self.sync.result_queue.get(0) if widget.interested(event): interested = True if hasattr(event, 'held_changed') and event.held_changed: invalidate = True except queue.Empty: pass if interested: widget.refresh() if invalidate: self.updateStatusQueries() self.status.refresh()
def get(self): '''Get a task from queue when bucket available''' if self.bucket.get() < 1: return None now = time.time() self.mutex.acquire() try: task = self.priority_queue.get_nowait() self.bucket.desc() except Queue.Empty: self.mutex.release() return None task.exetime = now + self.processing_timeout self.processing.put(task) self.mutex.release() return task.taskid
def _check_task_done(self): '''Check status queue''' cnt = 0 try: while True: task = self.status_queue.get_nowait() # check _on_get_info result here if task.get('taskid') == '_on_get_info' and 'project' in task and 'track' in task: if task['project'] not in self.projects: continue project = self.projects[task['project']] project.on_get_info(task['track'].get('save') or {}) logger.info( '%s on_get_info %r', task['project'], task['track'].get('save', {}) ) continue elif not self.task_verify(task): continue self.on_task_status(task) cnt += 1 except Queue.Empty: pass return cnt
def run(self): '''Run loop''' logger.info("result_worker starting...") while not self._quit: try: task, result = self.inqueue.get(timeout=1) self.on_result(task, result) except Queue.Empty as e: continue except KeyboardInterrupt: break except AssertionError as e: logger.error(e) continue except Exception as e: logger.exception(e) continue logger.info("result_worker exiting...")
def get(self, block=True, timeout=None): if not block: return self.get_nowait() start_time = time.time() while True: try: return self.get_nowait() except self.Empty: if timeout: lasted = time.time() - start_time if timeout > lasted: time.sleep(min(self.max_timeout, timeout - lasted)) else: raise else: time.sleep(self.max_timeout)
def get(self, block=True, timeout=None, ack=False): if not block: return self.get_nowait() start_time = time.time() while True: try: return self.get_nowait(ack) except BaseQueue.Empty: if timeout: lasted = time.time() - start_time if timeout > lasted: time.sleep(min(self.max_timeout, timeout - lasted)) else: raise else: time.sleep(self.max_timeout)
def get(self, block=True, timeout=None): if not block: return self.get_nowait() start_time = time.time() while True: try: return self.get_nowait() except BaseQueue.Empty: if timeout: lasted = time.time() - start_time if timeout > lasted: time.sleep(min(self.max_timeout, timeout - lasted)) else: raise else: time.sleep(self.max_timeout)
def run(self): '''Run loop''' logger.info("processor starting...") while not self._quit: try: task, response = self.inqueue.get(timeout=1) self.on_task(task, response) self._exceptions = 0 except Queue.Empty as e: continue except KeyboardInterrupt: break except Exception as e: logger.exception(e) self._exceptions += 1 if self._exceptions > self.EXCEPTION_LIMIT: break continue logger.info("processor exiting...")
def add_watch_callback(self, *args, **kwargs): """ Watch a key or range of keys and call a callback on every event. If timeout was declared during the client initialization and the watch cannot be created during that time the method raises a ``WatchTimedOut`` exception. :param key: key to watch :param callback: callback function :returns: watch_id. Later it could be used for cancelling watch. """ try: return self.watcher.add_callback(*args, **kwargs) except queue.Empty: raise exceptions.WatchTimedOut()
def watch_once(self, key, timeout=None, **kwargs): """ Watch a key and stops after the first event. If the timeout was specified and event didn't arrived method will raise ``WatchTimedOut`` exception. :param key: key to watch :param timeout: (optional) timeout in seconds. :returns: ``Event`` """ event_queue = queue.Queue() def callback(event): event_queue.put(event) watch_id = self.add_watch_callback(key, callback, **kwargs) try: return event_queue.get(timeout=timeout) except queue.Empty: raise exceptions.WatchTimedOut() finally: self.cancel_watch(watch_id)
def _audio_data_generator(buff): """A generator that yields all available data in the given buffer. Args: buff - a Queue object, where each element is a chunk of data. Yields: A chunk of data that is the aggregate of all chunks of data in `buff`. The function will block until at least one data chunk is available. """ while True: # Use a blocking get() to ensure there's at least one chunk of data chunk = buff.get() if not chunk: # A falsey value indicates the stream is closed. break data = [chunk] # Now consume whatever other data's still buffered. while True: try: data.append(buff.get(block=False)) except queue.Empty: break yield b''.join(data)
def pull_batch_from_queue(self): """ Take a rollout from the queue of the thread runner. """ # get top rollout from queue (FIFO) rollout = self.runner.queue.get(timeout=600.0) while not rollout.terminal: try: # Now, get remaining *available* rollouts from queue and append them into # the same one above. If queue.Queue(5): len=5 and everything is # superfast (not usually the case), then all 5 will be returned and # exception is raised. In such a case, effective batch_size would become # constants['ROLLOUT_MAXLEN'] * queue_maxlen(5). But it is almost never the # case, i.e., collecting a rollout of length=ROLLOUT_MAXLEN takes more time # than get(). So, there are no more available rollouts in queue usually and # exception gets always raised. Hence, one should keep queue_maxlen = 1 ideally. # Also note that the next rollout generation gets invoked automatically because # its a thread which is always running using 'yield' at end of generation process. # To conclude, effective batch_size = constants['ROLLOUT_MAXLEN'] rollout.extend(self.runner.queue.get_nowait()) except queue.Empty: break return rollout
def _thread_body(self): while True: event = self._queue.get() if isinstance(event, EventFinish): break self._handle_event(event) while True: try: event = self._queue.get(True, 1) except queue.Empty: event = None if event: self._handle_event(event) elif not self._jobs: # Queue was empty and no jobs left. break
def _get_items(self): """Get multiple items from a Queue. Gets at least one (blocking) and at most ``max_items`` items (non-blocking) from a given Queue. Does not mark the items as done. :rtype: Sequence :returns: A sequence of items retrieved from the queue. """ items = [self._queue.get()] while len(items) < self._max_batch_size: try: items.append(self._queue.get_nowait()) except queue.Empty: break return items
def _get_connection_from_queue(self, initial_timeout, next_timeout): try: return self._queue.get(True, initial_timeout) except Empty: try: self._lock.acquire() if self._current_connections == self._max_connections: raise ClientUnavailableError("Too many connections in use") cb = self._make_connection() return cb except ClientUnavailableError as ex: try: return self._queue.get(True, next_timeout) except Empty: raise ex finally: self._lock.release()
def terminate(self): """Terminate data feeding early. Since TensorFlow applications can often terminate on conditions unrelated to the training data (e.g. steps, accuracy, etc), this method signals the data feeding process to ignore any further incoming data. Note that Spark itself does not have a mechanism to terminate an RDD operation early, so the extra partitions will still be sent to the executors (but will be ignored). Because of this, you should size your input data accordingly to avoid excessive overhead. """ logging.info("terminate() invoked") self.mgr.set('state', 'terminating') # drop remaining items in the queue queue = self.mgr.get_queue(self.qname_in) count = 0 done = False while not done: try: queue.get(block=True, timeout=5) queue.task_done() count += 1 except Empty: logging.info("dropped {0} items from queue".format(count)) done = True
def events(self, start_date, end_date, frequency): running = True self.clock_engine_thread.start() if not self.mod_config.redis_uri: self.quotation_engine_thread.start() while running: real_dt = datetime.datetime.now() while True: try: dt, event_type = self.event_queue.get(timeout=1) break except Empty: continue system_log.debug("real_dt {}, dt {}, event {}", real_dt, dt, event_type) yield Event(event_type, calendar_dt=real_dt, trading_dt=dt)
def fetch_batch(self): """ Fetch a batch of data without waiting""" inp, f = self.queue.get() nr_input_var = len(inp) batched, futures = [[] for _ in range(nr_input_var)], [] for k in range(nr_input_var): batched[k].append(inp[k]) futures.append(f) cnt = 1 while cnt < self.batch_size: try: inp, f = self.queue.get_nowait() for k in range(nr_input_var): batched[k].append(inp[k]) futures.append(f) except queue.Empty: break cnt += 1 return batched, futures
def run(self): """Run any background task scheduled for execution.""" while self.__running: try: try: # A brief timeout here is necessary # to reduce CPU usage and to ensure # that shutdown doesn't wait forever # for a new task to appear. task, args, kwargs = \ self.__q.get(timeout=.5) except queue.Empty: continue task(*args, **kwargs) if hasattr(self.__q, "task_done"): # Task is done; mark it so. self.__q.task_done() except: self.bus.log("Failure encountered executing " "background task {0!r}.".format(self), traceback=True)
def run(self): """Run any background task scheduled for execution.""" while self.__running: try: try: # A brief timeout here is necessary # to reduce CPU usage and to ensure # that shutdown doesn't wait forever # for a new task to appear. task, args, kwargs = \ self.__q.get(timeout=.5) except queue.Empty: continue task(*args, **kwargs) if hasattr(self.__q, "task_done"): # Task is done; mark it so. self.__q.task_done() if self.__q.unfinished_tasks == 0: self.__keep_busy = False except Exception as e: print("Failure encountered executing " "background task {0!r}.".format(self))
def test_run_empty(self, m_count): events = [mock.sentinel.event1, mock.sentinel.event2] group = mock.sentinel.group m_queue = mock.Mock() m_queue.empty.return_value = True m_queue.get.side_effect = events + [six_queue.Empty()] m_handler = mock.Mock() m_count.return_value = list(range(5)) async_handler = h_async.Async(m_handler, mock.Mock(), mock.Mock()) with mock.patch('time.sleep'): async_handler._run(group, m_queue) m_handler.assert_has_calls([mock.call(event) for event in events]) self.assertEqual(len(events), m_handler.call_count)
def test_run_stale(self, m_count): events = [mock.sentinel.event1, mock.sentinel.event2] group = mock.sentinel.group m_queue = mock.Mock() m_queue.empty.side_effect = [False, True, True] m_queue.get.side_effect = events + [six_queue.Empty()] m_handler = mock.Mock() m_count.return_value = list(range(5)) async_handler = h_async.Async(m_handler, mock.Mock(), mock.Mock()) with mock.patch('time.sleep'): async_handler._run(group, m_queue) m_handler.assert_called_once_with(mock.sentinel.event2)
def _run(self, group, queue): LOG.debug("Asynchronous handler started processing %s", group) for _ in itertools.count(): # NOTE(ivc): this is a mock-friendly replacement for 'while True' # to allow more controlled environment for unit-tests (e.g. to # avoid tests getting stuck in infinite loops) try: event = queue.get(timeout=self._grace_period) except six_queue.Empty: break # FIXME(ivc): temporary workaround to skip stale events # If K8s updates resource while the handler is processing it, # when the handler finishes its work it can fail to update an # annotation due to the 'resourceVersion' conflict. K8sClient # was updated to allow *new* annotations to be set ignoring # 'resourceVersion', but it leads to another problem as the # Handler will receive old events (i.e. before annotation is set) # and will start processing the event 'from scratch'. # It has negative effect on handlers' performance (VIFHandler # creates ports only to later delete them and LBaaS handler also # produces some excess requests to Neutron, although with lesser # impact). # Possible solutions (can be combined): # - use K8s ThirdPartyResources to store data/annotations instead # of native K8s resources (assuming Kuryr-K8s will own those # resources and no one else would update them) # - use the resulting 'resourceVersion' received from K8sClient's # 'annotate' to provide feedback to Async to skip all events # until that version # - stick to the 'get-or-create' behaviour in handlers and # also introduce cache for long operations time.sleep(STALE_PERIOD) while not queue.empty(): event = queue.get() if queue.empty(): time.sleep(STALE_PERIOD) self._handler(event)
def pull_batch_from_queue(self): """ self explanatory: take a rollout from the queue of the thread runner. """ rollout = self.runner.queue.get(timeout=600.0) while not rollout.terminal: try: rollout.extend(self.runner.queue.get_nowait()) except queue.Empty: break return rollout
def _consume_queue(self, terminate_evt): """ This is the main thread function that consumes functions that are inside the _queue object. To use, execute self._queue(fn), where fn is a function that performs some kind of network IO or otherwise benefits from threading and is independent. terminate_evt is automatically passed in on thread creation and is a common event for this generation of threads. The threads will terminate when the event is set and the queue burns down. Returns: void """ interface = self._initialize_interface() while not terminate_evt.is_set(): try: fn = self._queue.get(block=True, timeout=0.01) except Queue.Empty: continue # periodically check if the thread is supposed to die fn = partial(fn, interface) try: self._consume_queue_execution(fn) except Exception as err: self._error_queue.put(err) self._close_interface(interface)
def _check_errors(self): try: err = self._error_queue.get(block=False) self._error_queue.task_done() self.kill_threads() raise err except Queue.Empty: pass
def get_connection(self): with self._lock: try: conn = self.pool.get(block=False) self.pool.task_done() except Queue.Empty: conn = self._create_connection() finally: self.outstanding += 1 return conn
def reset_pool(self): closefn = self._close_function() while True: if not self.pool.qsize(): break try: conn = self.pool.get() closefn(conn) self.pool.task_done() except Queue.Empty: break with self._lock: self.outstanding = 0
def validate(self, proxy_scanner, expected_num=20, queue_timeout=3, val_timeout=5): """Target function of validation threads Args: proxy_scanner: A ProxyScanner object. expected_num: Max number of valid proxies to be scanned. queue_timeout: Timeout for getting a proxy from the queue. val_timeout: An integer passed to `is_valid` as argument `timeout`. """ while self.proxy_num() < expected_num: try: candidate_proxy = proxy_scanner.proxy_queue.get( timeout=queue_timeout) except queue.Empty: if proxy_scanner.is_scanning(): continue else: break addr = candidate_proxy['addr'] protocol = candidate_proxy['protocol'] ret = self.is_valid(addr, protocol, val_timeout) if self.proxy_num() >= expected_num: self.logger.info('Enough valid proxies, thread {} exit.' .format(threading.current_thread().name)) break if ret['valid']: self.add_proxy(Proxy(addr, protocol)) self.logger.info('{} ok, {:.2f}s'.format(addr, ret[ 'response_time'])) else: self.logger.info('{} invalid, {}'.format(addr, ret['msg']))
def run(self): while 1: try: msg = self.input_q.get(True, 10.0) LOG.info("SyncThread: received message %s " % msg) self.proc_sync_msg(msg) except queue.Empty: LOG.debug("SyncThread: Queue timeout") except ValueError: LOG.error("Error processing sync message") break LOG.error("SyncThread exiting") SyncData.sync_thread_running = False
def stop(self): """ Stop queue and remove all connections from pool. """ while True: try: ctx = self.queue.get_nowait() ctx.stop() except queue_six.Empty: break self.queue.queue.clear() self.queue = None
def generator(self): while not self.closed: # Use a blocking get() to ensure there's at least one chunk of # data, and stop iteration if the chunk is None, indicating the # end of the audio stream. chunk = self._buff.get() if chunk is None: return if self.isPause: continue data = [chunk] # Now consume whatever other data's still buffered. while True: try: chunk = self._buff.get(block=False) if chunk is None: return data.append(chunk) except queue.Empty: break yield b''.join(data) #?? ??
