我们从Python开源项目中，提取了以下27个代码示例，用于说明如何使用pymongo.Connection()。
def process_item(self, item, spider): valid = True for data in item: if not data: valid = False raise DropItem('Missing{0}!'.format(data)) if valid: self.collection.insert(dict(item)) log.msg('??????!', level=log.DEBUG, spider=spider) return item # def testdb(self): # # ???MongoHQ # con = pymongo.Connection("paulo.mongohq.com",10042) # db = con.mytest # db.authenticate("root", "sa123") # db.urllist.drop()
def mongo_connect(host=None, port=None,ssl=False, user=None,passwd=None,replica=None): try: # ssl connection for pymongo > 2.1 if pymongo.version >= "2.1": if replica is None: con = pymongo.Connection(host, port, read_preference=pymongo.ReadPreference.SECONDARY, ssl=ssl, network_timeout=10) else: con = pymongo.Connection(host, port, read_preference=pymongo.ReadPreference.SECONDARY, ssl=ssl, replicaSet=replica, network_timeout=10) else: if replica is None: con = pymongo.Connection(host, port, slave_okay=True, network_timeout=10) else: con = pymongo.Connection(host, port, slave_okay=True, replicaSet=replica, network_timeout=10) if user and passwd: db = con["admin"] if not db.authenticate(user, passwd): sys.exit("Username/Password incorrect") except Exception, e: if isinstance(e,pymongo.errors.AutoReconnect) and str(e).find(" is an arbiter") != -1: # We got a pymongo AutoReconnect exception that tells us we connected to an Arbiter Server # This means: Arbiter is reachable and can answer requests/votes - this is all we need to know from an arbiter print "OK - State: 7 (Arbiter)" sys.exit(0) return exit_with_general_critical(e),None return 0,con
def init_db(self): self.redis = None self.mongo_conf = None self.mongo = None try: self.mongo_conf = yield txmongo.MongoConnection(host=self.setting.CONF_MONGO_HOST, port=self.setting.CONF_MONGO_PORT) self.mongo = yield txmongo.MongoConnection(host=self.setting.DATA_MONGO_HOST, port=self.setting.DATA_MONGO_PORT) self.redis = yield txredisapi.Connection(host=self.setting.QUEUE_REDIS_HOST, port=self.setting.QUEUE_REDIS_PORT) yield self.mongo.admin.authenticate("root", "chenfuzhi") yield self.mongo_conf.admin.authenticate("root", "chenfuzhi") defer.returnValue(True) except Exception, e: import traceback traceback.print_exc() defer.returnValue(False)
def _get_connection(self): """Connect to the MongoDB server.""" if self._connection is None: from pymongo import MongoClient # The first pymongo.Connection() argument (host) can be # a list of ['host:port'] elements or a mongodb connection # URI. If this is the case, don't use self.port # but let pymongo get the port(s) from the URI instead. # This enables the use of replica sets and sharding. # See pymongo.Connection() for more info. url = self.host if isinstance(url, string_t) \ and not url.startswith('mongodb://'): url = 'mongodb://{0}:{1}'.format(url, self.port) if url == 'mongodb://': url = url + 'localhost' if detect_environment() != 'default': self.options['use_greenlets'] = True self._connection = MongoClient(host=url, **self.options) return self._connection
def __init__(self): asyn_client = asyncmongo.Client( pool_id='isdb', host=setting.MONGO_HOST, port=setting.MONGO_PORT, dbuser=setting.MONGO_USER, dbpass=setting.MONGO_PASS, dbname='admin', maxcached=150, maxconnections=150, ) connection = Connection(setting.MONGO_HOST, setting.MONGO_PORT) options["asyn_client"] = asyn_client AsyncBaseModel.configure(asyn_client) options["mono_conn"] = connection logging.error("{}".format("[init]Mongodb init success"))
def __new__(cls,size=5,dbname='',*args,**kwargs): if not hasattr(cls,'_instance'): cls._instance = object.__new__(cls) cls._instance.dbname = dbname cls._instance.queue = PriorityQueue(size) for x in xrange(size): try: cls._instance.queue.put( (time.time(),pymongo.Connection(*args,**kwargs)[dbname]) ) except Exception,e: raise MongoPoolCantConnect('Can\'t connect to mongo servers: %s' % e) return cls._instance
def nosql(): import pymongo import json conn = pymongo.Connection('127.0.0.1', port=27017) df = ts.get_tick_data('600848',date='2014-12-22') print(df.to_json(orient='records')) conn.db.tickdata.insert(json.loads(df.to_json(orient='records'))) # print conn.db.tickdata.find()
def insert_log(): connection = Connection('localhost', 27017) db = connection.mydb collection = db.logdata alf = ApacheLogFile(r'D:\work\gitproject\python3-cookbook\configs\app.log') for lg_line in alf.my_generator(): collection.insert(props(lg_line))
def query_log(): connection = Connection('localhost', 27017) db = connection.mydb collection = db.logdata for doc in collection.find(): print(doc) connection.close()
def check_connect(host, port, warning, critical, perf_data, user, passwd, conn_time): warning = warning or 3 critical = critical or 6 message = "Connection took %i seconds" % conn_time message += performance_data(perf_data,[(conn_time,"connection_time",warning,critical)]) return check_levels(conn_time,warning,critical,message)
def check_connect_primary(con, warning, critical, perf_data): warning = warning or 3 critical = critical or 6 try: try: set_read_preference(con.admin) data = con.admin.command(pymongo.son_manipulator.SON([('isMaster', 1)])) except: data = con.admin.command(son.SON([('isMaster', 1)])) if data['ismaster'] == True : print "OK - This server is primary" return 0 phost = data['primary'].split(':')[0] pport = int(data['primary'].split(':')[1]) start = time.time() err,con=mongo_connect(phost, pport) if err!=0: return err pconn_time = time.time() - start pconn_time = round(pconn_time, 0) message = "Connection to primary server "+data['primary']+" took %i seconds" % pconn_time message += performance_data(perf_data,[(pconn_time,"connection_time",warning,critical)]) return check_levels(pconn_time,warning,critical,message) except Exception, e: return exit_with_general_critical(e)
def __init__(self, host = db_config['DB_HOST'], port = db_config['DB_PORT'], user = db_config['DB_USER'], passwd = db_config['DB_PSW'], db = db_config['DB_NAME'], charset = db_config['DB_CHARSET']): self.connection = pymongo.Connection(host, port) self.db = self.connection[db] self.db.authenticate(user, passwd)
def __init__(self,hostname,db,collection,port=27017): self.conn = pymongo.Connection(hostname,port) self.db = self.conn[db] self.collection = collection
def load_conf(self): spider_conf = {} conn = pymongo.Connection(self.setting.CONF_MONGO_HOST, self.setting.CONF_MONGO_PORT) row = conn.taobao.spider.find_one({"name":self.spider}) spider_conf = dict(row) open("models/get_seed_%s.py" % str(spider_conf["_id"]), "w").write(spider_conf["get_seed"].encode("utf-8")) spider_conf["navi_models"] = [] for navi in spider_conf["navi_list"]: navi_row = conn.taobao.spider_navi.find_one({"_id":navi}) open("models/navi_%s.py" % str(navi_row["_id"]), "w").write(navi_row["python_code"].encode("utf-8")) spider_conf["navi_models"].append(dict(navi_row)) redis_queue = conn.taobao.redis_queue.find({"_id":spider_conf["_id"]}) spider_conf["redis_queue"] = list(redis_queue) return spider_conf
def init_database(self): self.mongo = yield txmongo.MongoConnection(host=self.setting.DATA_MONGO_HOST, port=self.setting.DATA_MONGO_PORT) self.redis = yield txredisapi.Connection(host=self.setting.QUEUE_REDIS_HOST, port=self.setting.QUEUE_REDIS_PORT) #yield self.mongo.admin.authenticate("root", "chenfuzhi") defer.returnValue(True)
def load_conf(self): conn = pymongo.Connection(host=self.setting.CONF_MONGO_HOST, port=self.setting.CONF_MONGO_PORT) row = conn.taobao.spider.find_one({"name":self.spider}) self.conf = dict(row) rows = conn.taobao.redis_queue.find({"_id":self.conf["_id"]}, sort=("prio", 1)) self.queues = list(rows)
def mongoDB_connect(self,ip,username,password,port): crack=0 try: connection=pymongo.Connection(ip,port) db=connection.admin db.collection_names() self.lock.acquire() printRed('%s mongodb service at %s allow login Anonymous login!!\r\n' %(ip,port)) self.result.append('%s mongodb service at %s allow login Anonymous login!!\r\n' %(ip,port)) self.lock.release() crack=1 except Exception,e: if e[0]=='database error: not authorized for query on admin.system.namespaces': try: r=db.authenticate(username,password) if r!=False: crack=2 else: self.lock.acquire() crack=3 print "%s mongodb service 's %s:%s login fail " %(ip,username,password) self.lock.release() except Exception,e: pass else: printRed('%s mongodb service at %s not connect' %(ip,port)) crack=4 return crack
def __init__(self, col, index=None): connection = pymongo.Connection(settings.MONGODB_SERVER, settings.MONGODB_PORT) self.db = connection[settings.MONGODB_DB] self.collection = self.db[col] if index: self.collection.create_index(index, unique=True)
def __init__(self): import pymongo connection = pymongo.Connection(settings['MONGODB_SERVER'], settings['MONGODB_PORT']) self.db = connection[settings['MONGODB_DB']] self.collection = self.db[settings['MONGODB_COLLECTION']] if self.__get_uniq_key() is not None: self.collection.create_index(self.__get_uniq_key(), unique=True)
def load_mongo(self): self.mongo_conn = pymongo.Connection('localhost', 27017) self.mongo_db = self.mongo_conn[mongo_db_name(self.base_fname)]
def __init__(self, db_name, collection_name='mongo_dict_data', connection=None): """ :param db_name: database name (be careful with production databases) :param collection_name: collection name (default: mongo_dict_data) :param connection: ``pymongo.Connection`` instance. If it's ``None`` (default) new connection with default options will be created """ if connection is not None: self.connection = connection else: self.connection = Connection() self.db = self.connection[db_name] self.collection = self.db[collection_name]
def get_mongodb_connection(cls): if cls._connection is not None: return cls._connection try: cls._connection = pymongo.Connection( "{0}:{1}".format( REQUEST_LOGGING_BACKEND['host'], REQUEST_LOGGING_BACKEND['port'] ) ) return cls._connection except Exception as e: # TODO: Add log here. return None
def get_client(self): if self.replica_set: return MongoClient(self.login_host, self.login_port, replicaset=self.replica_set, ssl=self.ssl) else: return MongoClient(self.login_host, self.login_port, ssl=self.ssl)
def get_data_from_mongo(): conn = pymongo.Connection('192.168.17.128',27017) db = conn.db_lianjia results = db.scrapy_zlzp_info.find({},{"zwlb":1,"zwyx":1,"gsdz":1,"gsxz":1,"_id":0}) #results = db.scrapy_zlzp_info.find({}) zwlb = ['C??','C++','C#','PYTHON','RUBY','JAVA','IOS','ANDROID','HTML','PHP'] #zwlb = ['SCALA'] zwnum_set = {} zwyx_set = {} je_re = re.compile('([0-9 ]*)-([0-9 ]*)') def get_average_salary(slary): r_slary = slary.replace(',','') m = je_re.match(r_slary) try: if m: low = m.group(1) high = m.group(2) return (float(low) + float(high)) / 2 except: return 0 return 0 #print(get_average_salary('??')) for result in results: zw = result.get('zwlb') yx = result.get('zwyx') if isinstance(yx,str) and isinstance(zw,str): uzw = zw.upper() for zwfl in zwlb: if uzw.rfind(zwfl) != -1: zwnum_set[zwfl] = zwnum_set.get(zwfl,0) + 1 zwyx_set[zwfl] = zwyx_set.get(zwfl,0) + get_average_salary(yx) #print(zwfl,yx,get_average_salary(yx)) for key in zwnum_set.keys(): zwyx_set[key] = zwyx_set[key]/float(zwnum_set[key]) return zwyx_set,zwnum_set
