Python rq 模块，Connection() 实例源码

def worker(config_uri):
    """ Console entry script that starts a worker process
    """
    # TODO: import spacy's model to share it between workers

    pyramid_env = bootstrap(config_uri)

    # this conflicts with normal worker output
    # TODO: solve logging for the console
    # Setup logging to allow log output from command methods
    # from pyramid.paster import setup_logging
    # setup_logging(config_uri)

    try:
        qs = ['default']
        conn = redis_connection()
        with Connection(conn):
            w = Worker(qs)
            w.work()
    finally:
        pyramid_env['closer']()

def worker():
    with Connection(Redis("jobqueue.local")):
        qs = sys.argv[1:] or ['default']
        print("foo")
        w = Worker(qs)
        w.work()

def main():
    """
        Init workers
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('--queues', nargs='+', type=unicode, dest='queues', required=True)
    args = parser.parse_args()

    with Connection(connection=StrictRedis(**settings.REDIS)):

        qs = map(Queue, args.queues) or [Queue()]
        worker = Worker(qs)
        worker.work()

def main():
    with Connection(redis_connection):
        worker = Worker(Queue('default'))
        worker.work()

def main(config):
    global worker_config
    worker_config = config

    listen = config["listen"].values()

    queue_dsn = config["queue"]["dsn"]

    conn = redis.from_url(queue_dsn)

    with Connection(conn):
        worker = Worker(map(Queue, listen))
        worker.work()

def main(config=None):

    listen = config["listen"].values()

    queue_dsn = config["queue"]["dsn"]

    conn = redis.from_url(queue_dsn)

    with Connection(conn):
        worker = Worker(map(Queue, listen))
        worker.work()

def main():
    with Connection(Redis(settings['redis server'])):
        qs = ['default']
        w = Worker(qs)
        w.work()

def main(cli_config, raw, by_queue, queues, **options):
    """RQ command-line monitor."""
    try:
        with Connection(redis_conn):
            refresh(0.1, show_prism_info, queues, raw, by_queue,
                    cli_config.queue_class, cli_config.worker_class)
    except ConnectionError as e:
        click.echo(e)
        sys.exit(1)
    except KeyboardInterrupt:
        click.echo()
        sys.exit(0)

def runworker(app):

    REDIS_HOST = app.config['REDIS_HOST']
    REDIS_PORT = app.config['REDIS_PORT']
    REDIS_DB = app.config['REDIS_DB']
    QUEUES = app.config['QUEUES']

    redis_conn = Connection(Redis(REDIS_HOST,
                                  REDIS_PORT,
                                  REDIS_DB))
    with redis_conn:
        w = Worker(QUEUES)
        w.work()

def __init__(self):
        super(TrainCenter, self).__init__()

        info = config['pubsub']
        self.host = info[0]
        self.port = int(info[1])
        self.raw_conn = redis.StrictRedis(host=self.host, port=self.port, db=0)
        self.conn = Connection(self.raw_conn)

        self.train_sessions = []

def __init__(self):
        self.measure_helper = MeasureHelper()
        self.r = None
        self.p = None
        self.rc = None
        self.group_dict = {}  # group_id : Group object

        info = config['pubsub']
        host = info[0]
        port = int(info[1])
        self.raw_conn = redis.StrictRedis(host=host, port=port, db=0)
        self.conn = Connection(self.raw_conn)

def launch_rq_worker() -> None:
    """
    Blocking function to launch a worker using Python RQ's internal API
    """
    with Connection():
        w = Worker(
            get_available_rq_worker_name()
        )

        w.work()

def run_worker():
    """Initializes a slim rq task queue."""
    listen = ['default']
    conn = Redis(
        host=app.config['RQ_DEFAULT_HOST'],
        port=app.config['RQ_DEFAULT_PORT'],
        db=0,
        password=app.config['RQ_DEFAULT_PASSWORD'])

    with Connection(conn):
        worker = Worker(map(Queue, listen))
        worker.work()

def work():
    print("Hello from the worker side.")
    with Connection(REDIS):
        worker = Worker(map(Queue, QUEUES))
        worker.work()

def start_worker(queue_name):
    print "starting worker '{}'...".format(queue_name)

    with Connection(redis_rq_conn):
        worker = Worker(Queue(queue_name), exc_handler=failed_job_handler)
        worker.work()

def main():
    utils.setup_logging()
    config.log()
    gh_pr.monkeypatch_github()
    if config.FLUSH_REDIS_ON_STARTUP:
        utils.get_redis().flushall()
    with rq.Connection(utils.get_redis()):
        worker = rq.worker.HerokuWorker([rq.Queue('default')],
                                        exception_handlers=[error_handler])
        worker.work()

def run_worker():
    """Initializes a slim rq task queue."""
    listen = ['default']
    conn = Redis(
        host=app.config['RQ_DEFAULT_HOST'],
        port=app.config['RQ_DEFAULT_PORT'],
        db=0,
        password=app.config['RQ_DEFAULT_PASSWORD'])

    with Connection(conn):
        worker = Worker(map(Queue, listen))
        worker.work()

def create_base_learner(id):
    """This creates a single base learner from a base learner origin and queues it up"""
    path = functions.get_path_from_query_string(request)

    with functions.DBContextManager(path) as session:
        base_learner_origin = session.query(models.BaseLearnerOrigin).filter_by(id=id).first()
        if base_learner_origin is None:
            raise exceptions.UserError('Base learner origin {} not found'.format(id), 404)

        if not base_learner_origin.final:
            raise exceptions.UserError('Base learner origin {} is not final'.format(id))

        req_body = request.get_json()

        # Retrieve full hyperparameters
        est = base_learner_origin.return_estimator()
        hyperparameters = functions.import_object_from_string_code(req_body['source'],
                                                                   'params')
        est.set_params(**hyperparameters)
        hyperparameters = functions.make_serializable(est.get_params())

        base_learners = session.query(models.BaseLearner).\
            filter_by(base_learner_origin_id=id,
                      hyperparameters=hyperparameters).all()
        if base_learners:
            raise exceptions.UserError('Base learner exists with given hyperparameters')

        base_learner = models.BaseLearner(hyperparameters,
                                          'queued',
                                          base_learner_origin)

        if 'single_searches' not in base_learner_origin.description:
            base_learner_origin.description['single_searches'] = []
        base_learner_origin.description['single_searches'] += ([req_body['source']])

        session.add(base_learner)
        session.add(base_learner_origin)
        session.commit()

        with Connection(get_redis_connection()):
            rqtasks.generate_meta_features.delay(path, base_learner.id)

        return jsonify(base_learner.serialize)

def get_automated_runs():
    """Return all automated runs"""
    path = functions.get_path_from_query_string(request)

    if request.method == 'GET':
        with functions.DBContextManager(path) as session:
            automated_runs = session.query(models.AutomatedRun).all()
            return jsonify(list(map(lambda x: x.serialize, automated_runs)))

    if request.method == 'POST':
        req_body = request.get_json()
        with functions.DBContextManager(path) as session:
            base_learner_origin = None

            if req_body['category'] == 'bayes' or req_body['category'] == 'greedy_ensemble_search':
                base_learner_origin = session.query(models.BaseLearnerOrigin).\
                    filter_by(id=req_body['base_learner_origin_id']).first()
                if base_learner_origin is None:
                    raise exceptions.UserError('Base learner origin {} not found'.format(
                        req_body['base_learner_origin_id']
                    ), 404)
                if not base_learner_origin.final:
                    raise exceptions.UserError('Base learner origin {} is not final'.format(
                        req_body['base_learner_origin_id']
                    ))

            elif req_body['category'] == 'tpot':
                pass

            else:
                raise exceptions.UserError('Automated run category'
                                           ' {} not recognized'.format(req_body['category']))

            # Check for any syntax errors
            module = functions.import_string_code_as_module(req_body['source'])
            del module

            automated_run = models.AutomatedRun(req_body['source'],
                                                'queued',
                                                req_body['category'],
                                                base_learner_origin)

            session.add(automated_run)
            session.commit()

            with Connection(get_redis_connection()):
                rqtasks.start_automated_run.delay(path, automated_run.id)

            return jsonify(automated_run.serialize)

def create_new_stacked_ensemble():
    path = functions.get_path_from_query_string(request)
    req_body = request.get_json()

    with functions.DBContextManager(path) as session:
        if request.method == 'GET':
                return jsonify(
                    list(map(lambda x: x.serialize, session.query(models.StackedEnsemble).all()))
                )

        if request.method == 'POST':
            base_learners = session.query(models.BaseLearner).\
                filter(models.BaseLearner.id.in_(req_body['base_learner_ids'])).all()
            if len(base_learners) != len(req_body['base_learner_ids']):
                raise exceptions.UserError('Not all base learners found')
            for learner in base_learners:
                if learner.job_status != 'finished':
                    raise exceptions.UserError('Not all base learners have finished')

            base_learner_origin = session.query(models.BaseLearnerOrigin).\
                filter_by(id=req_body['base_learner_origin_id']).first()
            if base_learner_origin is None:
                raise exceptions.UserError('Base learner origin {} not '
                                           'found'.format(req_body['base_learner_origin_id']), 404)

            # Retrieve full hyperparameters
            est = base_learner_origin.return_estimator()
            params = functions.import_object_from_string_code\
                (req_body['secondary_learner_hyperparameters_source'], 'params')
            est.set_params(**params)
            hyperparameters = functions.make_serializable(est.get_params())

            stacked_ensembles = session.query(models.StackedEnsemble).\
                filter_by(base_learner_origin_id=req_body['base_learner_origin_id'],
                          secondary_learner_hyperparameters=hyperparameters,
                          base_learner_ids=sorted([bl.id for bl in base_learners])).all()
            if stacked_ensembles:
                raise exceptions.UserError('Stacked ensemble exists')

            stacked_ensemble = models.StackedEnsemble(
                secondary_learner_hyperparameters=hyperparameters,
                base_learners=base_learners,
                base_learner_origin=base_learner_origin,
                job_status='queued'
            )

            session.add(stacked_ensemble)
            session.commit()

            with Connection(get_redis_connection()):
                rqtasks.evaluate_stacked_ensemble.delay(path, stacked_ensemble.id)

            return jsonify(stacked_ensemble.serialize)