/** * Creates the query to execute on the collection. * * @return the query */ private Bson createQuery() { // timestamps are used as offsets, saved as a concatenation of seconds and order Integer timestamp = 0; Integer order = 0; if(!start.equals("0")){ final String[] splitted = start.split("_"); timestamp = Integer.valueOf(splitted[0]); order = Integer.valueOf(splitted[1]); } Bson query = Filters.and( Filters.exists("fromMigrate", false), Filters.gt("ts", new BSONTimestamp(timestamp, order)), Filters.or( Filters.eq("op", "i"), Filters.eq("op", "u"), Filters.eq("op", "d") ), Filters.eq("ns", db) ); return query; }
@Override public Void call() throws Exception { final Date now = new Date(); final Document query = new Document("ns", ns) .append("ts", new Document("$gt", new BSONTimestamp((int) (now.getTime() / 1000), 0))); final MongoCursor<Document> cursor = oplog.find(query) .cursorType(CursorType.TailableAwait).iterator(); while (cursor.hasNext()) { final Document doc = cursor.next(); for (final OplogListener listener : listeners) { listener.onOplog(doc); } } return null; }
protected static int mongoToKettleType( Object fieldValue ) { if ( fieldValue == null ) { return ValueMetaInterface.TYPE_STRING; } if ( fieldValue instanceof Symbol || fieldValue instanceof String || fieldValue instanceof Code || fieldValue instanceof ObjectId || fieldValue instanceof MinKey || fieldValue instanceof MaxKey ) { return ValueMetaInterface.TYPE_STRING; } else if ( fieldValue instanceof Date ) { return ValueMetaInterface.TYPE_DATE; } else if ( fieldValue instanceof Number ) { // try to parse as an Integer try { Integer.parseInt( fieldValue.toString() ); return ValueMetaInterface.TYPE_INTEGER; } catch ( NumberFormatException e ) { return ValueMetaInterface.TYPE_NUMBER; } } else if ( fieldValue instanceof Binary ) { return ValueMetaInterface.TYPE_BINARY; } else if ( fieldValue instanceof BSONTimestamp ) { return ValueMetaInterface.TYPE_INTEGER; } return ValueMetaInterface.TYPE_STRING; }
private DBCursor createCursor() { DBCollection oplog = _mongo.getDB("local").getCollection("oplog.rs"); BSONTimestamp startingTimestamp = getStartingTimestamp(); DBCursor cursor; if (startingTimestamp == null) { log.info("Tailing the oplog from the beginning..."); cursor = oplog.find(); } else { log.info("Tailing the oplog from " + startingTimestamp); BasicDBObject query = new BasicDBObject("ts", new BasicDBObject("$gt", startingTimestamp)); cursor = oplog.find(query); cursor.addOption(Bytes.QUERYOPTION_OPLOGREPLAY); } cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT); cursor.addOption(Bytes.QUERYOPTION_TAILABLE); cursor.addOption(Bytes.QUERYOPTION_AWAITDATA); return cursor; }
private BSONTimestamp getStartingTimestamp() { Get get = new Get(_tailerID); Result res; try { res = _stateTable.get(get); } catch (IOException e) { log.error("Failed to get a starting timestamp for tailer ID: " + _tailerID); return null; } if (res.isEmpty()) { if(ConfigUtil.getSkipBacklog(_conf)) return new BSONTimestamp((int) (System.currentTimeMillis() / 1000L), 0); return null; } else { byte[] raw_optime = res.getValue(STATE_TABLE_COL_FAMILY, STATE_TABLE_COL_QUALIFIER_OPTIME); byte[] raw_inc = res.getValue(STATE_TABLE_COL_FAMILY, STATE_TABLE_COL_QUALIFIER_INC); _optime = Integer.parseInt(new String(raw_optime)); _inc = Integer.parseInt(new String(raw_inc)); _optimeSet = true; return new BSONTimestamp(_optime, _inc); } }
/** * convert bson types to java primitives. BasicBSONList, Binary, * BSONTimestamp, Code, CodeWScope, MinKey, MaxKey, Symbol, ObjectId */ private Object convert(Object o) { if (o instanceof BSONTimestamp) { return ((BSONTimestamp) o).getTime(); } else if (o instanceof Symbol || o instanceof Code || o instanceof CodeWScope || o instanceof MinKey || o instanceof MaxKey || o instanceof ObjectId) { return o.toString(); } else if (o instanceof BasicBSONList) { List<Object> l = new ArrayList<Object>(); for (Object item : ((BasicBSONList) o)) { l.add(convert(item)); } return l; } else { return o; } }
protected void checkValue(Object value) { if (value instanceof MObject) return; if (value == null) return; if (value instanceof Number) return; if (value instanceof String) return; if (value instanceof Boolean) return; if (value instanceof Character) return; if (value instanceof ObjectId) return; if (value instanceof Date) return; if (value instanceof Pattern) return; if (value instanceof UUID) return; if (value instanceof MaxKey || value instanceof MinKey) return; if (value instanceof byte[]) return; if (value instanceof BSONTimestamp || value instanceof Symbol || value instanceof Code || value instanceof CodeWScope) return; throw new IllegalArgumentException(getClass().getSimpleName() + " can't store a " + value.getClass().getName()); }
@Override public void writeTimestamp(final BSONTimestamp value) { checkPreconditions("writeTimestamp", State.VALUE); buffer.write(BSONType.TIMESTAMP.getValue()); writeCurrentName(); buffer.writeInt(value.getInc()); buffer.writeInt(value.getTime()); setState(getNextState()); }
/** * Returns an <code>ObjectSerializer</code> that mostly conforms to the strict JSON format defined in * <a href="http://www.mongodb.org/display/DOCS/Mongo+Extended+JSON", but with a few differences to keep * compatibility with previous versions of the driver. Clients should generally prefer * <code>getStrict</code> in preference to this method. * * @return object serializer * @see #getStrict() */ public static ObjectSerializer getLegacy() { ClassMapBasedObjectSerializer serializer = addCommonSerializers(); serializer.addObjectSerializer(Date.class, new LegacyDateSerializer(serializer)); serializer.addObjectSerializer(BSONTimestamp.class, new LegacyBSONTimestampSerializer(serializer)); serializer.addObjectSerializer(Binary.class, new LegacyBinarySerializer()); serializer.addObjectSerializer(byte[].class, new LegacyBinarySerializer()); return serializer; }
/** * Returns an <code>ObjectSerializer</code> that conforms to the strict JSON format defined in * <a href="http://www.mongodb.org/display/DOCS/Mongo+Extended+JSON". * * @return object serializer */ public static ObjectSerializer getStrict() { ClassMapBasedObjectSerializer serializer = addCommonSerializers(); serializer.addObjectSerializer(Date.class, new DateSerializer(serializer)); serializer.addObjectSerializer(BSONTimestamp.class, new BSONTimestampSerializer(serializer)); serializer.addObjectSerializer(Binary.class, new BinarySerializer(serializer)); serializer.addObjectSerializer(byte[].class, new ByteArraySerializer(serializer)); return serializer; }
@Override public void serialize(Object obj, StringBuilder buf) { BSONTimestamp t = (BSONTimestamp) obj; BasicDBObject temp = new BasicDBObject(); temp.put("$ts", Integer.valueOf(t.getTime())); temp.put("$inc", Integer.valueOf(t.getInc())); serializer.serialize(temp, buf); }
@Override public void serialize(Object obj, StringBuilder buf) { BSONTimestamp t = (BSONTimestamp) obj; BasicDBObject temp = new BasicDBObject(); temp.put("t", Integer.valueOf(t.getTime())); temp.put("i", Integer.valueOf(t.getInc())); BasicDBObject timestampObj = new BasicDBObject(); timestampObj.put("$timestamp", temp); serializer.serialize(timestampObj, buf); }
@Override public Long convertFromMongoDbType(BSONTimestamp data) { int inc = data.getInc(); if (inc < 0 || inc >= 1000) { throw new RuntimeException( "Overflow occurs while converting BSONTimestamp into long: " + data); } return (long) data.getTime() * 1000 + inc; }
@Override public BSONTimestamp convertToMongoDbType(String data) { Matcher matcher = TIMESTAMP_PATTERN.matcher(data); if (!matcher.matches()) { throw new RuntimeException("Invalid BSONTimestamp " + data); } int time = Integer.parseInt(matcher.group(1)); int inc = Integer.parseInt(matcher.group(2)); return new BSONTimestamp(time, inc); }
private void updateOptime(BasicDBObject doc) { BSONTimestamp ts = (BSONTimestamp) doc.get("ts"); int optime = ts.getTime(), inc = ts.getInc(); // only checkpoint every 60 seconds if (!_optimeSet || (optime - _optime) >= 60) { _optime = optime; _inc = inc; _optimeSet = true; log.info("optime: " + _optime); saveOptime(); } }
public void gotTimestamp( String name , int time , int inc ){ _put( name , new BSONTimestamp( time , inc ) ); }
protected void _putObjectField( String name , Object val ){ if ( name.equals( "_transientFields" ) ) return; if ( DEBUG ) System.out.println( "\t put thing : " + name ); if ( name.contains( "\0" ) ) throw new IllegalArgumentException( "Document field names can't have a NULL character. (Bad Key: '" + name + "')" ); if ( name.equals( "$where") && val instanceof String ){ _put( CODE , name ); _putValueString( val.toString() ); return; } val = BSON.applyEncodingHooks( val ); if ( val == null ) putNull(name); else if ( val instanceof Date ) putDate( name , (Date)val ); else if ( val instanceof Number ) putNumber(name, (Number)val ); else if ( val instanceof Character ) putString(name, val.toString() ); else if ( val instanceof String ) putString(name, val.toString() ); else if ( val instanceof ObjectId ) putObjectId(name, (ObjectId)val ); else if ( val instanceof BSONObject ) putObject(name, (BSONObject)val ); else if ( val instanceof Boolean ) putBoolean(name, (Boolean)val ); else if ( val instanceof Pattern ) putPattern(name, (Pattern)val ); else if ( val instanceof Map ) putMap( name , (Map)val ); else if ( val instanceof Iterable) putIterable( name , (Iterable)val ); else if ( val instanceof byte[] ) putBinary( name , (byte[])val ); else if ( val instanceof Binary ) putBinary( name , (Binary)val ); else if ( val instanceof UUID ) putUUID( name , (UUID)val ); else if ( val.getClass().isArray() ) putArray( name , val ); else if (val instanceof Symbol) { putSymbol(name, (Symbol) val); } else if (val instanceof BSONTimestamp) { putTimestamp( name , (BSONTimestamp)val ); } else if (val instanceof CodeWScope) { putCodeWScope( name , (CodeWScope)val ); } else if (val instanceof Code) { putCode( name , (Code)val ); } else if (val instanceof DBRefBase) { BSONObject temp = new BasicBSONObject(); temp.put("$ref", ((DBRefBase)val).getRef()); temp.put("$id", ((DBRefBase)val).getId()); putObject( name, temp ); } else if ( val instanceof MinKey ) putMinKey( name ); else if ( val instanceof MaxKey ) putMaxKey( name ); else if ( putSpecial( name , val ) ){ // no-op } else { throw new IllegalArgumentException( "can't serialize " + val.getClass() ); } }
protected void putTimestamp(String name, BSONTimestamp ts ){ _put( TIMESTAMP , name ); _buf.writeInt( ts.getInc() ); _buf.writeInt( ts.getTime() ); }
/** * @deprecated This method is NOT a part of public API and will be dropped in 3.x versions. */ @Deprecated protected Object getElementValue( ElementRecord record ){ switch ( record.type ){ case BSON.EOO: case BSON.UNDEFINED: case BSON.NULL: return null; case BSON.MAXKEY: return new MaxKey(); case BSON.MINKEY: return new MinKey(); case BSON.BOOLEAN: return ( _input.get( record.valueOffset ) != 0 ); case BSON.NUMBER_INT: return _input.getInt( record.valueOffset ); case BSON.TIMESTAMP: int inc = _input.getInt( record.valueOffset ); int time = _input.getInt( record.valueOffset + 4 ); return new BSONTimestamp( time, inc ); case BSON.DATE: return new Date( _input.getLong( record.valueOffset ) ); case BSON.NUMBER_LONG: return _input.getLong( record.valueOffset ); case BSON.NUMBER: return Double.longBitsToDouble( _input.getLong( record.valueOffset ) ); case BSON.OID: return new ObjectId( _input.getIntBE( record.valueOffset ), _input.getIntBE( record.valueOffset + 4 ), _input.getIntBE( record.valueOffset + 8 ) ); case BSON.SYMBOL: return new Symbol( _input.getUTF8String( record.valueOffset ) ); case BSON.CODE: return new Code( _input.getUTF8String( record.valueOffset ) ); case BSON.STRING: return _input.getUTF8String( record.valueOffset ); case BSON.CODE_W_SCOPE: int strsize = _input.getInt( record.valueOffset + 4 ); String code = _input.getUTF8String( record.valueOffset + 4 ); BSONObject scope = (BSONObject) _callback.createObject( _input.array(), record.valueOffset + 4 + 4 + strsize ); return new CodeWScope( code, scope ); case BSON.REF: int csize = _input.getInt( record.valueOffset ); String ns = _input.getCString( record.valueOffset + 4 ); int oidOffset = record.valueOffset + csize + 4; ObjectId oid = new ObjectId( _input.getIntBE( oidOffset ), _input.getIntBE( oidOffset + 4 ), _input.getIntBE( oidOffset + 8 ) ); return _callback.createDBRef( ns, oid ); case BSON.OBJECT: return _callback.createObject( _input.array(), record.valueOffset ); case BSON.ARRAY: return _callback.createArray( _input.array(), record.valueOffset ); case BSON.BINARY: return readBinary( record.valueOffset ); case BSON.REGEX: int patternCStringSize = sizeCString( record.valueOffset ); String pattern = _input.getCString( record.valueOffset ); String flags = _input.getCString( record.valueOffset + patternCStringSize ); return Pattern.compile( pattern, BSON.regexFlags( flags ) ); default: throw new BSONException( "Invalid type " + record.type + " for field " + getElementFieldName( record.offset ) ); } }
/** * Gets the type byte for a given object. * @param o the object * @return the byte value associated with the type, or -1 if no type is matched */ @SuppressWarnings("deprecation") public static byte getType( Object o ){ if ( o == null ) return NULL; if ( o instanceof DBPointer ) return REF; if (o instanceof Integer || o instanceof Short || o instanceof Byte || o instanceof AtomicInteger) { return NUMBER_INT; } if (o instanceof Long || o instanceof AtomicLong) { return NUMBER_LONG; } if ( o instanceof Number ) return NUMBER; if ( o instanceof String ) return STRING; if ( o instanceof java.util.List ) return ARRAY; if ( o instanceof byte[] ) return BINARY; if ( o instanceof ObjectId ) return OID; if ( o instanceof Boolean ) return BOOLEAN; if ( o instanceof java.util.Date ) return DATE; if ( o instanceof BSONTimestamp ) return TIMESTAMP; if ( o instanceof java.util.regex.Pattern ) return REGEX; if ( o instanceof DBObject || o instanceof DBRefBase ) return OBJECT; if ( o instanceof Code ) return CODE; if ( o instanceof CodeWScope ) return CODE_W_SCOPE; return -1; }
public void setStartTimestamp(BSONTimestamp startTimestamp) { this.startTimestamp = startTimestamp; }
@Override public BSONTimestamp convertToMongoDbType(Long object) { return new BSONTimestamp((int) (object / 1000), (int) (object % 1000)); }
@Override public String convertFromMongoDbType(BSONTimestamp timestamp) { return "(" + timestamp.getTime() + ", " + timestamp.getInc() + ")"; }
@Test public void testTypes1() throws Exception { Schema schema = Types1.SCHEMA$; DBObject mongoObject = new BasicDBObject(ImmutableMap.of("x", 1.0, "y", 1.0)); String mongoString = JSON.serialize(mongoObject); String avroJson = "{\"objectId\": \"5401bf578de2a77380c5489a\", \"bsonTimestamp1\": \"(1409385948, 1)\", \"bsonTimestamp2\": 1409385948001, \"date1\": \"2014-08-31T08:09:34.033Z\", \"date2\": 1409472574033, \"mongoString\": \"" + mongoString.replace("\"", "\\\"") + "\"}"; Decoder decoder = DecoderFactory.get().jsonDecoder(schema, avroJson); SpecificDatumReader<Types1> reader = new SpecificDatumReader<Types1>(schema); Types1 types1 = reader.read(null, decoder); BasicDBObject object = new BasicDBObject(); object.put("_id", new ObjectId("5401bf578de2a77380c5489a")); object.put("bsonTimestamp1", new BSONTimestamp(1409385948, 1)); object.put("bsonTimestamp2", new BSONTimestamp(1409385948, 1)); object.put("date1", MongoDbTypeConverter.DATE_FORMAT.parse("2014-08-31T08:09:34.033Z")); object.put("date2", MongoDbTypeConverter.DATE_FORMAT.parse("2014-08-31T08:09:34.033Z")); object.put("mongoString", mongoObject); Types1 types2 = RecordConverter.toRecord(Types1.class, object); assertThat(types1.getObjectId().toString()).isEqualTo("5401bf578de2a77380c5489a"); assertThat(types2.getObjectId().toString()).isEqualTo("5401bf578de2a77380c5489a"); assertThat(types1.getBsonTimestamp1().toString()).isEqualTo("(1409385948, 1)"); assertThat(types2.getBsonTimestamp1().toString()).isEqualTo("(1409385948, 1)"); assertThat(types1.getBsonTimestamp2()).isEqualTo(1409385948001l); assertThat(types2.getBsonTimestamp2()).isEqualTo(1409385948001l); assertThat(types1.getDate1().toString()).isEqualTo("2014-08-31T08:09:34.033Z"); assertThat(types2.getDate1().toString()).isEqualTo("2014-08-31T08:09:34.033Z"); assertThat(types1.getDate2()).isEqualTo(1409472574033l); assertThat(types2.getDate2()).isEqualTo(1409472574033l); assertThat(types1.getMongoString().toString()).isEqualTo(mongoString); assertThat(types2.getMongoString().toString()).isEqualTo(mongoString); Document object1 = RecordConverter.toDocument(types1); Document object2 = RecordConverter.toDocument(types2); assertThat(object1.get("_id")).isEqualTo(new ObjectId("5401bf578de2a77380c5489a")); assertThat(object2.get("_id")).isEqualTo(new ObjectId("5401bf578de2a77380c5489a")); assertThat(object1.get("bsonTimestamp1")).isEqualTo(new BSONTimestamp(1409385948, 1)); assertThat(object2.get("bsonTimestamp1")).isEqualTo(new BSONTimestamp(1409385948, 1)); assertThat(object1.get("bsonTimestamp2")).isEqualTo(new BSONTimestamp(1409385948, 1)); assertThat(object2.get("bsonTimestamp2")).isEqualTo(new BSONTimestamp(1409385948, 1)); assertThat(object1.get("date1")).isEqualTo( MongoDbTypeConverter.DATE_FORMAT.parse("2014-08-31T08:09:34.033Z")); assertThat(object2.get("date1")).isEqualTo( MongoDbTypeConverter.DATE_FORMAT.parse("2014-08-31T08:09:34.033Z")); assertThat(object1.get("date2")).isEqualTo( MongoDbTypeConverter.DATE_FORMAT.parse("2014-08-31T08:09:34.033Z")); assertThat(object2.get("date2")).isEqualTo( MongoDbTypeConverter.DATE_FORMAT.parse("2014-08-31T08:09:34.033Z")); assertThat(object1.get("mongoString")).isEqualTo(mongoObject); assertThat(object2.get("mongoString")).isEqualTo(mongoObject); }
@Test public void testTypes2() throws Exception { Schema schema = Types2.SCHEMA$; DBObject mongoObject = new BasicDBObject(ImmutableMap.of("x", 1.0, "y", 1.0)); String mongoString = JSON.serialize(mongoObject); String avroJson = "{\"objectId\": \"5401bf578de2a77380c5489a\", \"bsonTimestamp1\": \"(1409385948, 1)\", \"bsonTimestamp2\": 1409385948001, \"date1\": \"2014-08-31T08:09:34.033Z\", \"date2\": 1409472574033, \"mongoString\": \"" + mongoString.replace("\"", "\\\"") + "\"}"; Decoder decoder = DecoderFactory.get().jsonDecoder(schema, avroJson); SpecificDatumReader<Types2> reader = new SpecificDatumReader<Types2>(schema); Types2 types1 = reader.read(null, decoder); BasicDBObject object = new BasicDBObject(); object.put("_id", new ObjectId("5401bf578de2a77380c5489a")); object.put("bsonTimestamp1", new BSONTimestamp(1409385948, 1)); object.put("bsonTimestamp2", new BSONTimestamp(1409385948, 1)); object.put("date1", MongoDbTypeConverter.DATE_FORMAT.parse("2014-08-31T08:09:34.033Z")); object.put("date2", MongoDbTypeConverter.DATE_FORMAT.parse("2014-08-31T08:09:34.033Z")); object.put("mongoString", mongoObject); Types2 types2 = RecordConverter.toRecord(Types2.class, object); assertThat(types1.getObjectId().toString()).isEqualTo("5401bf578de2a77380c5489a"); assertThat(types2.getObjectId().toString()).isEqualTo("5401bf578de2a77380c5489a"); assertThat(types1.getBsonTimestamp1().toString()).isEqualTo("(1409385948, 1)"); assertThat(types2.getBsonTimestamp1().toString()).isEqualTo("(1409385948, 1)"); assertThat(types1.getBsonTimestamp2()).isEqualTo(1409385948001l); assertThat(types2.getBsonTimestamp2()).isEqualTo(1409385948001l); assertThat(types1.getDate1().toString()).isEqualTo("2014-08-31T08:09:34.033Z"); assertThat(types2.getDate1().toString()).isEqualTo("2014-08-31T08:09:34.033Z"); assertThat(types1.getDate2()).isEqualTo(1409472574033l); assertThat(types2.getDate2()).isEqualTo(1409472574033l); assertThat(types1.getMongoString().toString()).isEqualTo(mongoString); assertThat(types2.getMongoString().toString()).isEqualTo(mongoString); Document object1 = RecordConverter.toDocument(types1); Document object2 = RecordConverter.toDocument(types2); assertThat(object1.get("_id")).isEqualTo(new ObjectId("5401bf578de2a77380c5489a")); assertThat(object2.get("_id")).isEqualTo(new ObjectId("5401bf578de2a77380c5489a")); assertThat(object1.get("bsonTimestamp1")).isEqualTo(new BSONTimestamp(1409385948, 1)); assertThat(object2.get("bsonTimestamp1")).isEqualTo(new BSONTimestamp(1409385948, 1)); assertThat(object1.get("bsonTimestamp2")).isEqualTo(new BSONTimestamp(1409385948, 1)); assertThat(object2.get("bsonTimestamp2")).isEqualTo(new BSONTimestamp(1409385948, 1)); assertThat(object1.get("date1")).isEqualTo( MongoDbTypeConverter.DATE_FORMAT.parse("2014-08-31T08:09:34.033Z")); assertThat(object2.get("date1")).isEqualTo( MongoDbTypeConverter.DATE_FORMAT.parse("2014-08-31T08:09:34.033Z")); assertThat(object1.get("date2")).isEqualTo( MongoDbTypeConverter.DATE_FORMAT.parse("2014-08-31T08:09:34.033Z")); assertThat(object2.get("date2")).isEqualTo( MongoDbTypeConverter.DATE_FORMAT.parse("2014-08-31T08:09:34.033Z")); assertThat(object1.get("mongoString")).isEqualTo(mongoObject); assertThat(object2.get("mongoString")).isEqualTo(mongoObject); }
public void setTimestamp(BSONTimestamp ts) { this._ts = ts; }
/** * Writes a BSON Timestamp to the writer. * * @param value The combined timestamp/increment value. */ public abstract void writeTimestamp(BSONTimestamp value);
/** * Writes a BSON Timestamp element to the writer. * * @param name The name of the element. * @param value The combined timestamp/increment value. */ public void writeTimestamp(final String name, final BSONTimestamp value) { writeName(name); writeTimestamp(value); }