/** * Marshals a Cassandra row into a Document object. * * @param row Row to marshal. * @return A document based on the provided row. */ public static Document marshalRow(Row row) { if (row == null) { return null; } Document d = new Document(); d.setUuid(row.getUUID(DocumentRepositoryImpl.Columns.ID)); ByteBuffer b = row.getBytes(DocumentRepositoryImpl.Columns.OBJECT); if (b != null && b.hasArray()) { byte[] result = new byte[b.remaining()]; b.get(result); BSONObject o = BSON.decode(result); d.setObject(o); } d.setCreatedAt(row.getDate(DocumentRepositoryImpl.Columns.CREATED_AT)); d.setUpdatedAt(row.getDate(DocumentRepositoryImpl.Columns.UPDATED_AT)); return d; }
@Override public Object getObject(String columnLabel) throws SQLException { if (isSum) { if (isGroupBy){ Object ob=dblist.get(_row-1); if (ob instanceof BSONObject) { return ((BSONObject)ob).get(columnLabel); } else { return "0"; } } else{ return this._sum; } } else { return this._cur.get(columnLabel); } }
private int InsertData(SQLInsertStatement state) { if (state.getValues().getValues().size() ==0 ){ throw new RuntimeException("number of columns error"); } if (state.getValues().getValues().size() != state.getColumns().size()){ throw new RuntimeException("number of values and columns have to match"); } SQLTableSource table=state.getTableSource(); BSONObject o = new BasicBSONObject(); int i=0; for(SQLExpr col : state.getColumns()) { o.put(getFieldName2(col), getExpValue(state.getValues().getValues().get(i))); i++; } DBCollection coll =this._db.getCollection(table.toString()); //coll.insert(new DBObject[] { o }); coll.insert(o); return 1; }
private int UpData(SQLUpdateStatement state) { SQLTableSource table=state.getTableSource(); DBCollection coll =this._db.getCollection(table.toString()); SQLExpr expr=state.getWhere(); BSONObject query = parserWhere(expr); BasicBSONObject set = new BasicBSONObject(); for(SQLUpdateSetItem col : state.getItems()){ set.put(getFieldName2(col.getColumn()), getExpValue(col.getValue())); } BSONObject mod = new BasicBSONObject("$set", set); //coll.updateMulti(query, mod); coll.update(query, mod, null); //System.out.println("changs count:"+coll.getStats().size()); return 1; }
public boolean checkElements(BSONObject index, BSONObject feature) { if (srcIpMask == 0 && dstIpMask == 0) { //marking according to featureConstraint return featureDatabaseMgmtManager.isSatisfyOnlineEvent( (Document) index, (Document) feature, featureConstraint); } if (srcIpMask != 0 && index.get(AthenaIndexField.MATCH_IPV4_SRC) != null ) { int ipsrc = (Integer) index.get(AthenaIndexField.MATCH_IPV4_SRC); if ((ipsrc & this.srcIpMask) == srcComparator) { return true; } } if (dstIpMask != 0 && index.get(AthenaIndexField.MATCH_IPV4_DST) != null) { int ipdst = (Integer) index.get(AthenaIndexField.MATCH_IPV4_DST); if ((ipdst & this.dstIpMask) == dstComparator) { return true; } } return false; }
public int checkClassificationMarkingElements(BSONObject index, BSONObject feature) { ClassificationMarkingElement classificationMarkingElement; boolean check = false; for (int i = 1 ; i < listofClassificationMarkingElement.size() ; i++){ classificationMarkingElement = listofClassificationMarkingElement.get(i); check = checkElements(index,feature, classificationMarkingElement.featureConstraint, classificationMarkingElement.getSrcIpMask(), classificationMarkingElement.getSrcComparator(), classificationMarkingElement.getDstIpMask(), classificationMarkingElement.getDstComparator()); if(check){ return classificationMarkingElement.getLabel(); } } return 0; }
public LogisticRegressionModel generateDecisionTreeWithPreprocessing(JavaPairRDD<Object, BSONObject> mongoRDD, AthenaMLFeatureConfiguration athenaMLFeatureConfiguration, LogisticRegressionDetectionAlgorithm logisticRegressionDetectionAlgorithm, Marking marking, LogisticRegressionModelSummary logisticRegressionModelSummary) { return generateKMeansModel( rddPreProcessing(mongoRDD, athenaMLFeatureConfiguration, logisticRegressionModelSummary, marking), logisticRegressionDetectionAlgorithm, logisticRegressionModelSummary ); }
/** * * @param dbcollection * @return */ public List<BSONObject> queryAll() { List<BSONObject> list = new ArrayList<BSONObject>(); DBCursor cursor = null; Sequoiadb sdb = DbConnectionPool.getConnection(); try { cursor = getDbCollection(sdb).query(); while (cursor.hasNext()) { BSONObject record = cursor.getNext(); list.add(record); } } catch (BaseException e) { e.printStackTrace(); } finally { DbConnectionPool.free(sdb); } return list; }
/** * * 查询多少条数据 * * @param matcher * @param selector * @param order * @param skipRows * @param returnRows * @return */ public List<BSONObject> query(BSONObject matcher, BSONObject selector, BSONObject order, long skipRows, long returnRows) { Sequoiadb sdb = DbConnectionPool.getConnection(); List<BSONObject> list = new ArrayList<BSONObject>(); DBCursor cursor = null; try { cursor = getDbCollection(sdb).query(matcher, selector, order, null, skipRows, returnRows); while (cursor.hasNext()) { BSONObject record = cursor.getNext(); list.add(record); } } catch (BaseException e) { e.printStackTrace(); } finally { DbConnectionPool.free(sdb); } return list; }
public List<T> findAll(BSONObject matcher, BSONObject orderBy) { List<T> result1 = new ArrayList<T>(); if (matcher == null || matcher.toMap().size() == 0) { return result1; } try { List<BSONObject> result = query(matcher, null, orderBy); for (BSONObject bsonObject : result) { result1.add(bsonObject.as(entityClass)); } return result1; } catch (Exception e) { e.printStackTrace(); } return result1; }
public synchronized void generateId(T obj) { String id = null; while (true) { BSONObject matcher = new BasicBSONObject(); id = CommEncode.generateId(); matcher.put(this.primaryKey, id); BSONObject bson = queryOne(matcher, null, null); if (bson == null) { break; } } try { Class cls = obj.getClass(); Method method = cls.getMethod(this.getGenerateIdKeySetMethodName(), String.class); method.invoke(obj, id); } catch (Exception e) { e.printStackTrace(); } }
public List<T> search(String key, String value, BSONObject order) { BSONObject arg = new BasicBSONObject(); BSONObject regex = new BasicBSONObject(); regex.put(MatchConst._REGEX, value); arg.put(key, regex); List<BSONObject> record = query(arg, null, null); List<T> result1 = new ArrayList<T>(); for (BSONObject bsonObject : record) { try { result1.add(bsonObject.as(entityClass)); } catch (Exception e) { e.printStackTrace(); } } return result1; }
public <V> List<V> search(String key, String value, BSONObject order,Class<V> clazz) { BSONObject arg = new BasicBSONObject(); BSONObject regex = new BasicBSONObject(); regex.put(MatchConst._REGEX, value); arg.put(key, regex); List<BSONObject> record = query(arg, null, null); List<V> result1 = new ArrayList<V>(); for (BSONObject bsonObject : record) { try { result1.add(bsonObject.as(clazz)); } catch (Exception e) { e.printStackTrace(); } } return result1; }
/** * 更新一个数组的字段 * * @param primaryKeyValue * @param field * @param value * @return */ public <T> boolean pushAllByPk(String primaryKeyValue, String field, List<T> value) { BSONObject query = new BasicBSONObject(); query.put(this.primaryKey, primaryKeyValue); BSONObject update = new BasicBSONObject(); update.put(field, value); update.put(field, value); BSONObject modifier = new BasicBSONObject(); modifier.put("$push_all", update); try { super.updateWithUserDefine(query, modifier); return true; } catch (Exception e) { e.printStackTrace(); } return false; }
/** * 取出来的值要按照主键ID的顺序 * * @param primaryKeyValues * @param cls * @return */ public List<T> findListByPkArray(List<String> primaryKeyValues) { BSONObject selector = null; BSONObject query = new BasicBSONObject(); BSONObject matcher = new BasicBSONObject(); matcher.put(MatchConst._IN, primaryKeyValues); query.put(this.primaryKey, matcher); try { List<BSONObject> result = query(query, selector, null); if (result != null && result.size() > 0) { return this.caseBsonsToList(result, entityClass); } } catch (Exception e) { e.printStackTrace(); } return new ArrayList<>(); }
/** * 取出来的值要按照主键ID的顺序 * * @param primaryKeyValues * @param cls * @return */ public List<T> findListByPkArray(String[] primaryKeyValues) { BSONObject selector = null; BSONObject query = new BasicBSONObject(); BSONObject matcher = new BasicBSONObject(); matcher.put(MatchConst._IN, primaryKeyValues); query.put(this.primaryKey, matcher); try { List<BSONObject> result = query(query, selector, null); if (result != null && result.size() > 0) { return this.caseBsonsToList(result, entityClass); } } catch (Exception e) { e.printStackTrace(); } return null; }
public <V> List<V> findListByPkArray(List<String> primaryKeyValues, Class<V> clazz) { BSONObject query = new BasicBSONObject(); BSONObject matcher = new BasicBSONObject(); matcher.put(MatchConst._IN, primaryKeyValues); query.put(this.primaryKey, matcher); try { BSONObject selector = BasicBSONObject.typeToBson(clazz.newInstance()); List<BSONObject> result = query(query, selector, null); if (result != null && result.size() > 0) { return this.caseBsonsToList(result, clazz); } } catch (Exception e) { e.printStackTrace(); } return null; }
public <V> V findAttributeByPk(String primaryKeyValue, Class<V> clazz) { BSONObject query = new BasicBSONObject(); query.put(this.primaryKey, primaryKeyValue); BSONObject selector; try { selector = BasicBSONObject.typeToBson(clazz.newInstance()); BSONObject bsonObejct = queryOne(query, selector, null); if (bsonObejct != null) { return bsonObejct.as(clazz); } } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } return null; }
/** * 删除某个对象 * * @param object * @return */ public boolean delete(T object) { BSONObject matcher = null; try { matcher = BasicBSONObject.typeToBson(object); if (matcher != null) { beforeDelete(object); delete(matcher); afterDelete(object); return true; } return false; } catch (IllegalArgumentException | IllegalAccessException | InvocationTargetException | IntrospectionException e) { e.printStackTrace(); } return false; }
/** * Determines if an indexed field has changed as part of an update. This * would be private but keeping public for ease of testing. * * @param oldObject the old BSON object. * @param index Index containing the getFields to check for changes. * @param entity New version of a document. * @return True if an indexed field has changed. False if there is no change * of indexed getFields. */ public static boolean hasIndexedFieldChanged(BSONObject oldObject, Index index, Document entity) { //DocumentRepository docRepo = new DocumentRepositoryImpl(session); BSONObject newObject = entity.getObject(); //BSONObject oldObject = (BSONObject) JSON.parse(docRepo.read(entity.getId()).object()); for (IndexField indexField : index.getFields()) { String field = indexField.getField(); if (newObject.get(field) == null && oldObject.get(field) == null)//this shouldn't happen? {//if there is not a field in either index return false;//if it's not in ether doc, it couldn't have changed } else if (newObject.get(field) == null || oldObject.get(field) == null) {//there is a field in one of the indexes, but not the other. return true;//the index field must have changed, it either went from missing to present or present to missing. } if (!newObject.get(field).equals(oldObject.get(field))) { return true;//fail early } } return false; }
/** * Test of query method, of class QueryDao. */ @Test public void testDoQueryWithResults() throws IndexParseException { logger.debug("testDoQueryWithResults"); Document doc = Fixtures.createTestDocument(); //put a test doc in DocumentRepository docRepo = new DocumentRepositoryImpl(f.getSession()); docRepo.create(doc); QueryRepositoryImpl instance = new QueryRepositoryImpl(f.getSession()); QueryResponseWrapper result = instance.query(Fixtures.createTestParsedQuery()); assertNotNull(result); assertTrue(!result.isEmpty()); assertTrue(result.size() == 1); Document res = result.get(0); assertNotNull(res); assertNotNull(res.getCreatedAt()); assertNotNull(res.getUpdatedAt()); assertNotNull(res.getUuid()); assertNotNull(res.getId()); assertNotNull(res.getObject()); BSONObject expected = doc.getObject(); BSONObject actual = res.getObject(); assertEquals(expected, actual); }
/** * Test of query method, of class QueryService. */ @Test public void testQuery() throws IndexParseException, FieldNotIndexedException { logger.debug("query"); Document doc = Fixtures.createTestDocument(); //put a test doc in DocumentRepositoryImpl docRepo = new DocumentRepositoryImpl(f.getSession()); docRepo.create(doc); List<Document> result = instance.query(Fixtures.DB, Fixtures.createTestQuery()); assertNotNull(result); assertTrue(!result.isEmpty()); assertTrue(result.size() == 1); Document res = result.get(0); assertNotNull(res); assertNotNull(res.getCreatedAt()); assertNotNull(res.getUpdatedAt()); assertNotNull(res.getUuid()); assertNotNull(res.getId()); assertNotNull(res.getObject()); BSONObject expected = doc.getObject(); BSONObject actual = res.getObject(); assertEquals(expected, actual); }
@Override Iterator<DBObject> __find( DBObject ref , DBObject fields , int numToSkip , int batchSize, int limit , int options ) throws MongoException { if ( ref == null ) ref = new BasicDBObject(); if ( willTrace() ) trace( "find: " + _fullNameSpace + " " + JSON.serialize( ref ) ); OutMessage query = OutMessage.query( _mongo , options , _fullNameSpace , numToSkip , chooseBatchSize(batchSize, limit, 0) , ref , fields ); Response res = _connector.call( _db , this , query , null , 2 ); if ( res.size() == 0 ) return null; if ( res.size() == 1 ){ BSONObject foo = res.get(0); MongoException e = MongoException.parse( foo ); if ( e != null && ! _name.equals( "$cmd" ) ) throw e; } return new Result( this , res , batchSize, limit , options ); }
/** * Uses the Hadoop API to store results * @param result */ @Override public void store(JavaRDD<ObjectValue> result) { logger.info("Storing result."); JavaSparkContext sc = NotaQL.SparkFactory.getSparkContext(); String mongoDBHost = "mongodb://" + NotaQL.prop.getProperty("mongodb_host", "localhost") + ":27017/"; Configuration config = new Configuration(); config.set("mongo.output.uri", mongoDBHost + databaseName + "." + collectionName); JavaPairRDD<Object,BSONObject> output = result.mapToPair( o -> new Tuple2<>(null, (DBObject)ValueConverter.convertFromNotaQL(o)) ); if(NotaQL.prop.getProperty("log_output") != null && NotaQL.prop.getProperty("log_output").equals("true")) output.foreach(t -> logger.info("Storing object: " + t._2.toString())); else logger.info("Storing objects."); output.saveAsNewAPIHadoopFile("file:///notapplicable", Object.class, Object.class, MongoOutputFormat.class, config); logger.info("Stored result."); }
/** * Translates compatible predicates to MongoDB queries * @param predicate * @return */ private static BSONObject translatePredicate(Predicate predicate) { if(predicate == null) return new BasicBSONObject(); if(predicate instanceof LogicalOperationPredicate) return translateLogicalOperationPredicate((LogicalOperationPredicate) predicate); if(predicate instanceof NegatedPredicate) return translateNegatedPredicate((NegatedPredicate) predicate); if(predicate instanceof PathExistencePredicate) return translatePathExistencePredicate((PathExistencePredicate) predicate); if(predicate instanceof RelationalPredicate) return translateRelationalPredicate((RelationalPredicate) predicate); throw new EvaluationException("Unknown predicate: " + predicate.toString()); }
private static BSONObject translateLogicalOperationPredicate(LogicalOperationPredicate predicate) { final BasicBSONList opList = new BasicBSONList(); opList.add(translatePredicate(predicate.getLeft())); opList.add(translatePredicate(predicate.getRight())); final String op; if(predicate.getOperator().equals(LogicalOperationPredicate.Operator.AND)) op = "$and"; else if(predicate.getOperator().equals(LogicalOperationPredicate.Operator.OR)) op = "$or"; else throw new EvaluationException("Unexpected operator: " + predicate.getOperator()); return new BasicBSONObject(op, opList); }
public BSONObject getUpdater( Object[] row, RowMetaInterface rowMeta ) throws KettleValueException { BSONObject updater = new BasicBSONObject() ; for(Map.Entry<String, List<SequoiaDBUpdateFieldInfo>> entry:m_updateFields.entrySet()) { BSONObject fieldsObj = new BasicBSONObject() ; int fieldNum = entry.getValue().size() ; for( int i = 0 ; i < fieldNum ; i++ ) { SequoiaDBUpdateFieldInfo fieldTmp = entry.getValue().get(i) ; int index = rowMeta.indexOfValue( fieldTmp.getName() ) ; ValueMetaInterface vmi = rowMeta.getValueMeta( index ) ; try{ fieldsObj.put(fieldTmp.getPath(), fieldTmp.getBsonValue(row[index], vmi)) ; } catch( KettleValueException e ){ throw new KettleValueException( BaseMessages.getString( PKG, "SequoiaDBOutput.Msg.Err.FailedToGetTheFieldVal" + "(" + entry.getKey() + ":" + row[index].toString() + ")" ) ); } } updater.put( entry.getKey(), fieldsObj ) ; } if ( updater.isEmpty()) { return null ; } return updater ; }
public BSONObject getUpdateCond( Object[] row, RowMetaInterface rowMeta ) throws KettleValueException { BSONObject condition = new BasicBSONObject() ; int fieldNum = m_condFields.size() ; for( int i = 0 ; i < fieldNum ; i++ ) { SequoiaDBUpdateFieldInfo fieldTmp = m_condFields.get(i) ; int index = rowMeta.indexOfValue( fieldTmp.getName() ) ; ValueMetaInterface vmi = rowMeta.getValueMeta( index ) ; try{ condition.put(fieldTmp.getPath(), fieldTmp.getBsonValue(row[index], vmi)) ; } catch( KettleValueException e ){ throw new KettleValueException( BaseMessages.getString( PKG, "SequoiaDBOutput.Msg.Err.FailedToGetTheFieldVal" + "(" + fieldTmp.getName() + ":" + row[index].toString() + ")" ) ); } } if ( condition.isEmpty() ) { return null ; } return condition ; }
@Test public void testDefaultValues() throws Exception { Schema schema = Defaults.SCHEMA$; GenericRecordBuilder builder = new GenericRecordBuilder(schema).set("id", "1234"); Record record1 = builder.build(); String json = "{\"id\": \"1234\"}"; BSONObject object = (BSONObject) JSON.parse(json); Record record2 = RecordConverter.toRecord(schema, object, getClass().getClassLoader()); assertThat(record2, is(record1)); assertThat(AvroHelper.toSimpleJson(schema, record2), is(AvroHelper.toSimpleJson(schema, record1))); assertEquals(record2.get("id"), "1234"); assertNull(record2.get("s")); assertTrue((Boolean) record2.get("b")); assertEquals(((Record) record2.get("r")).get("f"), "value"); assertEquals(((Record) record2.get("r")).get("l"), 1234l); }
@Test public void testArrays() throws Exception { Schema schema = Arrays.SCHEMA$; GenericRecordBuilder builder = new GenericRecordBuilder(schema); builder.set("arrays", ImmutableList.of(ImmutableList.of(ImmutableList.of(1, 2, 3), ImmutableList.of()), ImmutableList.of(ImmutableList.of(4), ImmutableList.of()), ImmutableList.of(ImmutableList.of()))); Record record1 = builder.build(); String json = "{\"arrays\": [[[1, 2, 3], []], [[4], []], [[]]]}"; BSONObject object = (BSONObject) JSON.parse(json); Record record2 = RecordConverter.toRecord(schema, object, getClass().getClassLoader()); assertThat(record2, is(record1)); assertThat(AvroHelper.toSimpleJson(schema, record2), is(AvroHelper.toSimpleJson(schema, record1))); }
@Test public void testEnums() throws Exception { Schema schema = Enums.SCHEMA$; String avroJson = "{\"enum1\": \"X\", \"enum2\": {\"test.Enum2\": \"A\"}, \"enum3\": {\"null\": null}, \"enum4\": [{\"test.Enum4\": \"SAT\"}, {\"test.Enum4\": \"SUN\"}]}}"; Decoder decoder = DecoderFactory.get().jsonDecoder(schema, avroJson); GenericDatumReader<Record> reader = new GenericDatumReader<Record>(schema); Record record1 = reader.read(null, decoder); String mongoJson = "{\"enum1\": \"X\", \"enum2\": \"A\", \"enum3\": null, \"enum4\": [\"SAT\", \"SUN\"]}}"; BSONObject object = (BSONObject) JSON.parse(mongoJson); Record record2 = RecordConverter.toRecord(schema, object, getClass().getClassLoader()); assertThat(record2, is(record1)); assertThat(AvroHelper.toSimpleJson(schema, record2), is(AvroHelper.toSimpleJson(schema, record1))); }
@Override public DBCallback getDBCallback(DBCollection collection) { return new DefaultDBCallback(collection) { @Override public BSONObject create() { return new MObject(); } @Override public BSONObject create(boolean array, List<String> path) { if (array) return new MList(); else return new MObject(); } }; }
@Test public void testParseEnum() throws IOException { String enumName = "FakeEnum"; String enumValue1 = "FakeEnumValue1"; String enumValue2 = "FakeEnumValue2"; BSONObject enumsNode = (BSONObject) parser.newNode(); enumsNode.put("name", enumName); enumsNode.put("values", Arrays.asList(enumValue1, enumValue2)); Enum e = parser.parseEnum(enumsNode); Assert.assertEquals(enumName, e.getName()); Set<EnumValue> values = e.getEnumValues(); Assert.assertEquals(2, values.size()); Assert.assertTrue(values.contains(new EnumValue(enumValue1, null))); Assert.assertTrue(values.contains(new EnumValue(enumValue2, null))); }
public List<Long> findObjectIds(String type, String searchJson) { if (getMongoDb() == null) return null; BasicDBObject searchObj = new BasicDBObject(k_objectType, type); searchObj.putAll((BSONObject) JSON.parse(searchJson)); mongoClient.getDB(DB).requestStart(); mongoClient.getDB(DB).requestEnsureConnection(); DBCursor cursor = getMongoDb() .find(searchObj, new BasicDBObject(k_objectId, 1).append("_id", 0)); List<Long> result = new ArrayList<>(); try { while (cursor.hasNext()) { BasicDBObject o = (BasicDBObject) cursor.next(); result.add(o.getLong(k_objectId)); } } finally { cursor.close(); mongoClient.getDB(DB).requestDone(); } return result; }
/** * @param buf * @param o * @return * @throws MongoException */ @Override public int writeObject(final OutputBuffer buf, BSONObject o) { if (!(o instanceof LazyDBObject)) { throw new IllegalArgumentException("LazyDBEncoder can only encode BSONObject instances of type LazyDBObject"); } LazyDBObject lazyDBObject = (LazyDBObject) o; try { lazyDBObject.pipe(buf); } catch (IOException e) { throw new MongoException("Exception serializing a LazyDBObject", e); } return lazyDBObject.getBSONSize(); }
@Test public void testArrays() throws Exception { Schema schema = Arrays.SCHEMA$; GenericRecordBuilder builder = new GenericRecordBuilder(schema); builder.set("arrays", ImmutableList.of(ImmutableList.of(ImmutableList.of(1, 2, 3), ImmutableList.of()), ImmutableList.of(ImmutableList.of(4), ImmutableList.of()), ImmutableList.of(ImmutableList.of()))); Record record1 = builder.build(); String json = "{\"arrays\": [[[1, 2, 3], []], [[4], []], [[]]]}"; BSONObject object = (BSONObject) JSON.parse(json); Record record2 = RecordConverter.toRecord(schema, object, getClass().getClassLoader()); assertThat(record2).isEqualTo(record1); assertThat(AvroHelper.toJson(schema, record2)).isEqualTo(AvroHelper.toJson(schema, record1)); }
@Test public void testEnums() throws Exception { Schema schema = Enums.SCHEMA$; String avroJson = "{\"enum1\": \"X\", \"enum2\": {\"test.Enum2\": \"A\"}, \"enum3\": {\"null\": null}, \"enum4\": [{\"test.Enum4\": \"SAT\"}, {\"test.Enum4\": \"SUN\"}]}}"; Decoder decoder = DecoderFactory.get().jsonDecoder(schema, avroJson); GenericDatumReader<Record> reader = new GenericDatumReader<Record>(schema); Record record1 = reader.read(null, decoder); String mongoJson = "{\"enum1\": \"X\", \"enum2\": \"A\", \"enum3\": null, \"enum4\": [\"SAT\", \"SUN\"]}}"; BSONObject object = (BSONObject) JSON.parse(mongoJson); Record record2 = RecordConverter.toRecord(schema, object, getClass().getClassLoader()); assertThat(record2).isEqualTo(record1); assertThat(AvroHelper.toJson(schema, record2)).isEqualTo(AvroHelper.toJson(schema, record1)); }
/** * Retrieve a list of pending content items to process * * @param queueId ID of processing queue * @param markItems * @return list of pending content items to process (newest first) * @throws EumssiException with a specific StatusType, if one of the following scenarios occurs: * <br> * <br><code>StatusType.ERROR_INVALID_QUEUE_ID</code> (Error 102) if the specified queue id does not correspond to a valid queue. * <br> * <br><code>StatusType.ERROR_DB_CONNECT</code> (Error 400) if an unhandled error occurred during acquisition of the database connection. * <br><code>StatusType.ERROR_DB_QUERY</code> (Error 401) if an unhandled error occurred during the query execution. * <br><code>StatusType.ERROR_UNKNOWN</code> (Error 999) if an unhandled exception is thrown. */ public List<String> getQueuePending(String queueId, Integer maxItems, Boolean markItems, String filters) throws EumssiException { DBObject query = null; if (this.queues.containsKey(queueId)) { query = (DBObject) JSON.parse(this.queues.getProperty(queueId)); // check that item is not yet (being) processed String testPending = String.format("{\"processing.queues.%s\":{\"$nin\":[\"in_process\",\"processed\"]}}",queueId); query.putAll((BSONObject) JSON.parse(testPending)); query.putAll((BSONObject) JSON.parse(filters)); // apply user-provided filters } else { throw new EumssiException(StatusType.ERROR_INVALID_QUEUE_ID); } log.info("performing query "+query.toString()+" on collection "+this.collection.toString()); DBCursor resCursor = this.collection.find(query, new BasicDBObject("_id", 1)) .sort(new BasicDBObject("meta.source.datePublished", -1)) .limit(maxItems); List<String> resList = new ArrayList<String>(); for (DBObject res : resCursor) { resList.add(res.get("_id").toString()); if (markItems) { collection.update(new BasicDBObject("_id", res.get("_id")), new BasicDBObject("$set", new BasicDBObject("processing.queues."+queueId,"in_process"))); } } return resList; }
/** * Reset list of pending content items to process * * @param queueId ID of processing queue * @param inProcessOnly only reset items marked as "in_process" * @return number of reset items * @throws EumssiException with a specific StatusType, if one of the following scenarios occurs: * <br> * <br><code>StatusType.ERROR_INVALID_QUEUE_ID</code> (Error 102) if the specified queue id does not correspond to a valid queue. * <br> * <br><code>StatusType.ERROR_DB_CONNECT</code> (Error 400) if an unhandled error occurred during acquisition of the database connection. * <br><code>StatusType.ERROR_DB_QUERY</code> (Error 401) if an unhandled error occurred during the query execution. * <br><code>StatusType.ERROR_UNKNOWN</code> (Error 999) if an unhandled exception is thrown. */ public Integer resetQueue(String queueId, Boolean inProcessOnly, String filters) throws EumssiException { DBObject query = null; if (this.queues.containsKey(queueId)) { query = (DBObject) JSON.parse(this.queues.getProperty(queueId)); // check that item is marked as in_process String testReset = String.format("{\"processing.queues.%s\":\"in_process\"}", queueId); if (!inProcessOnly) { // reset all items, even if already processed testReset = String.format("{\"processing.queues.%s\":{\"$in\":[\"in_process\",\"processed\"]}}", queueId); } query.putAll((BSONObject) JSON.parse(testReset)); query.putAll((BSONObject) JSON.parse(filters)); // apply user-provided filters } else { throw new EumssiException(StatusType.ERROR_INVALID_QUEUE_ID); } try { log.info("performing query "+query.toString()+" on collection "+this.collection.toString()); WriteResult r = collection.update(query, new BasicDBObject("$set", new BasicDBObject("processing.results."+queueId,"pending"))); Integer updatedCount = r.getN(); return updatedCount;} catch (Exception e) { // TODO: better handle possible failures throw new EumssiException(StatusType.ERROR_UNKNOWN); } }
/** * Retrieve a list of already processed content items * * @param queueId ID of processing queue * @param markItems * @return list of already processed content items * @throws EumssiException with a specific StatusType, if one of the following scenarios occurs: * <br> * <br><code>StatusType.ERROR_INVALID_QUEUE_ID</code> (Error 102) if the specified queue id does not correspond to a valid queue. * <br> * <br><code>StatusType.ERROR_DB_CONNECT</code> (Error 400) if an unhandled error occurred during acquisition of the database connection. * <br><code>StatusType.ERROR_DB_QUERY</code> (Error 401) if an unhandled error occurred during the query execution. * <br><code>StatusType.ERROR_UNKNOWN</code> (Error 999) if an unhandled exception is thrown. */ public List<String> getQueueProcessed(String queueId, Integer maxItems, Boolean markItems, String filters) throws EumssiException { DBObject query = null; if (this.queues.containsKey(queueId)) { query = (DBObject) JSON.parse(this.queues.getProperty(queueId)); // check that item is already processed String testPending = String.format("{\"processing.queues.%s\":\"processed\"}",queueId); query.putAll((BSONObject) JSON.parse(testPending)); query.putAll((BSONObject) JSON.parse(filters)); // apply user-provided filters } else { throw new EumssiException(StatusType.ERROR_INVALID_QUEUE_ID); } log.info("performing query "+query.toString()+" on collection "+this.collection.toString()); DBCursor resCursor = this.collection.find(query, new BasicDBObject("_id", 1)).limit(maxItems); List<String> resList = new ArrayList<String>(); for (DBObject res : resCursor) { resList.add(res.get("_id").toString()); if (markItems) { collection.update(new BasicDBObject("_id", res.get("_id")), new BasicDBObject("$set", new BasicDBObject("processing.queues."+queueId,"in_process"))); } } return resList; }