public static IParser get( final Object obj ) throws IOException{ Schema.Type schemaType; try{ schemaType = genericUtil.induce( obj ).getType(); }catch( AvroTypeException e ){ return new AvroNullParser(); } switch( schemaType ){ case ARRAY: return new AvroArrayParser( (GenericArray)obj ); case MAP: return new AvroMapParser( (Map<Object,Object>)obj ); case RECORD: return new AvroRecordParser( (GenericRecord)obj ); case UNION : default: return new AvroNullParser(); } }
public static AvroValue getArray(AvroType type, Object value) { if(type.getType() != Type.TypeName.ARRAY) { throw new IllegalArgumentException("Can't create an array value with non-array schema: " + type.getType()); } return new AvroValue(value, type) { private final GenericArray<Object> array = (GenericArray<Object>)value; @Override public List<Object> arrayValue() { return array; } @Override public Value getElement(int index) { return type.getElementType().build(array.get(index)); } }; }
public void addGenericArray(K key, String fieldName, GenericArray<?> array) { if (isSuper( cassandraMapping.getFamily(fieldName) )) { int i= 0; for (Object itemValue: array) { // TODO: hack, do not store empty arrays if (itemValue instanceof GenericArray<?>) { if (((List<?>)itemValue).size() == 0) { continue; } } else if (itemValue instanceof Map<?,?>) { if (((Map<?, ?>)itemValue).size() == 0) { continue; } } addSubColumn(key, fieldName, i++, itemValue); } } else { addColumn(key, fieldName, array); } }
public GenericArray<ATableDescriptor> listTables() throws AIOError { try { HTableDescriptor[] tables = admin.listTables(); Schema atdSchema = Schema.createArray(ATableDescriptor.SCHEMA$); GenericData.Array<ATableDescriptor> result = null; result = new GenericData.Array<ATableDescriptor>(tables.length, atdSchema); for (HTableDescriptor table : tables) { result.add(AvroUtil.htdToATD(table)); } return result; } catch (IOException e) { AIOError ioe = new AIOError(); ioe.message = new Utf8(e.getMessage()); throw ioe; } }
@SuppressWarnings("unchecked") private void writeBlock() { if (logBlock != null) { LOG.debug("Writing block: {} {}", logBlock.get("second"), ((GenericArray<String>) logBlock.get("logLines")).size()); try { writer.append(logBlock); } catch (IOException e) { LOG.error("Error writing out record. Data lost.", e); } logBlock = null; linesInBlock = 0; } }
private static Element unwrapElement(GenericRecord record, Document doc) { String name = "" + record.get("name"); Element el = doc.createElement(name); @SuppressWarnings("unchecked") GenericArray<GenericRecord> attrArray = (GenericArray<GenericRecord>) record.get("attributes"); for (GenericRecord attrRecord : attrArray) el.setAttributeNode(unwrapAttr(attrRecord, doc)); @SuppressWarnings("unchecked") GenericArray<Object> childArray = (GenericArray<Object>) record.get("children"); for (Object childObj : childArray) { if (childObj instanceof GenericRecord) el.appendChild(unwrapElement((GenericRecord) childObj, doc)); if (childObj instanceof Utf8) el.appendChild(doc.createTextNode("" + childObj)); } return el; }
@Test() public void testInvalidArrayRecursiveType() throws Exception { Event.EventContext api = Event.EventContext.apiKey(apiKeys.writeKey()); byte[] bytes = mapper.writeValueAsBytes(ImmutableMap.of( "collection", "test", "api", api, "properties", ImmutableMap.of("test0", "test", "test1", ImmutableList.of("test", ImmutableMap.of("test", 2)), "test2", false))); Event event = mapper.readValue(bytes, Event.class); GenericArray test1 = event.getAttribute("test1"); assertEquals(test1.get(0), "test"); assertEquals(test1.get(1), "{\"test\":2}"); }
public static Object unionResolver(Object o) { if (o instanceof org.apache.avro.util.Utf8) { return o.toString(); } else if (o instanceof IndexedRecord) { return new AvroTupleWrapper<IndexedRecord>((IndexedRecord) o); } else if (o instanceof GenericArray) { return new AvroBagWrapper<GenericData.Record>( (GenericArray<GenericData.Record>) o); } else if (o instanceof Map) { return new AvroMapWrapper((Map<CharSequence, Object>) o); } else if (o instanceof GenericData.Fixed) { return new DataByteArray(((GenericData.Fixed) o).bytes()); } else if (o instanceof ByteBuffer) { return new DataByteArray(((ByteBuffer) o).array()); } else if (o instanceof GenericEnumSymbol) { return o.toString(); } else { return o; } }
private void updateRecord(CommonRecord record, GenericRecord delta) { List<Field> deltaFields = delta.getSchema().getFields(); for (Field deltaField : deltaFields) { String fieldName = deltaField.name(); Object rawDeltaField = delta.get(fieldName); if (LOG.isDebugEnabled()) { LOG.debug("Processing field \"{}\", current value: {}", fieldName, record.getField(fieldName) != null ? record .getField(fieldName).toString() : null); } if (AvroGenericUtils.isRecord(rawDeltaField)) { processRecordField(record, (GenericRecord) rawDeltaField, fieldName); } else if (AvroGenericUtils.isArray(rawDeltaField)) { processArrayField(record, (GenericArray) rawDeltaField, fieldName); } else if (AvroGenericUtils.isEnum(rawDeltaField)) { processEnumField(record, (GenericEnumSymbol) rawDeltaField, fieldName); } else if (AvroGenericUtils.isFixed(rawDeltaField)) { processFixedField(record, (GenericFixed) rawDeltaField, fieldName); } else { record.setField(fieldName, commonFactory.createCommonValue(rawDeltaField)); } } }
@Override public synchronized void processConfigurationData(ByteBuffer buffer, boolean fullResync) throws IOException { if (buffer != null) { if (schema == null) { throw new ConfigurationRuntimeException( "Can't process configuration update. Schema is null"); } GenericAvroConverter<GenericArray<GenericRecord>> converter = new GenericAvroConverter<>(schema); GenericArray<GenericRecord> deltaArray = converter.decodeBinary(buffer.array()); for (GenericRecord delta : deltaArray) { GenericRecord record = (GenericRecord) delta.get("delta"); int index = delta.getSchema().getField("delta").schema().getTypes().indexOf( record.getSchema()); for (GenericDeltaReceiver subscriber : onDeltaReceived) { subscriber.onDeltaReceived(index, record, fullResync); } } for (ConfigurationProcessedObserver callback : onProcessed) { callback.onConfigurationProcessed(); } } }
/** * Recursively removes UUIDs from the record. * * @param baseRecord The record containing UUID fields. */ public static void removeUuid(GenericRecord baseRecord) { Schema recordSchema = baseRecord.getSchema(); for (Schema.Field fieldSchema : recordSchema.getFields()) { if (baseRecord.get(fieldSchema.name()) != null) { Object field = baseRecord.get(fieldSchema.name()); if (field instanceof GenericFixed) { baseRecord.put(fieldSchema.name(), clearUuid((GenericFixed) field, fieldSchema)); } else if (field instanceof GenericRecord) { removeUuid((GenericRecord) field); } else if (field instanceof GenericArray) { GenericArray arrayField = (GenericArray) field; for (Object obj : arrayField) { if (obj instanceof GenericRecord) { removeUuid((GenericRecord) obj); } } } } } }
@Override public synchronized byte[] getData() throws IOException { if (deltaQueue != null && !deltaQueue.isEmpty()) { GenericArray deltaArray = new GenericData.Array(deltaQueue.size(), schema); while (!deltaQueue.isEmpty()) { GenericRecord deltaT = new GenericData.Record( schema.getElementType()); deltaT.put(DELTA, deltaQueue.poll()); deltaArray.add(deltaT); } GenericAvroConverter<GenericArray> converter = new GenericAvroConverter<>(schema); serializedData = converter.encode(deltaArray); if (LOG.isTraceEnabled()) { LOG.trace("Delta array: {}", deltaArray.toString()); } } return serializedData != null ? Arrays.copyOf(serializedData, serializedData.length) : null; }
/** * Fill delta without merge. * * @param delta the delta * @param root the root * @throws DeltaCalculatorException the delta calculator exception */ private void fillDeltaWithoutMerge(GenericRecord delta, GenericRecord root) throws DeltaCalculatorException { Schema rootSchema = root.getSchema(); for (Field field : rootSchema.getFields()) { Object value = root.get(field.name()); if (value instanceof List) { List<Object> values = (List<Object>) value; Schema arraySchema = getArraySchema(delta, field.name()); GenericArray deltaArray = new GenericData.Array(values.size(), arraySchema); for (Object item : values) { if (item instanceof GenericContainer) { GenericContainer record = (GenericContainer) item; addComplexItemToArray(record, deltaArray); } else { deltaArray.add(item); } } delta.put(field.name(), deltaArray); } else if (value instanceof GenericContainer) { processComplexField(delta, field.name(), (GenericContainer) value, null, null); } else { delta.put(field.name(), value); } } }
@SuppressWarnings("unchecked") protected Object cloneObject(Schema schema, Object toClone, Object cloned) { if(toClone == null) { return null; } switch(schema.getType()) { case MAP : Map<Utf8, Object> map = (Map<Utf8, Object>)newMap(cloned, 0); for(Map.Entry<Utf8, Object> entry: ((Map<Utf8, Object>)toClone).entrySet()) { map.put((Utf8)createString(entry.getKey().toString()) , cloneObject(schema.getValueType(), entry.getValue(), null)); } return map; case ARRAY : GenericArray<Object> array = (GenericArray<Object>) newArray(cloned, (int)((GenericArray<?>)toClone).size(), schema); for(Object element: (GenericArray<Object>)toClone) { array.add(cloneObject(schema.getElementType(), element, null)); } return array; case RECORD : return clone((Persistent)toClone, schema); case STRING : return createString(toClone.toString()); default : return toClone; //shallow copy is enough } }
@SuppressWarnings("unchecked") public void addGenericArray(K key, String fieldName, GenericArray array) { if (isSuper( cassandraMapping.getFamily(fieldName) )) { int i= 0; for (Object itemValue: array) { // TODO: hack, do not store empty arrays if (itemValue instanceof GenericArray<?>) { if (((GenericArray)itemValue).size() == 0) { continue; } } else if (itemValue instanceof StatefulHashMap<?,?>) { if (((StatefulHashMap)itemValue).size() == 0) { continue; } } addSubColumn(key, fieldName, i++, itemValue); } } else { addColumn(key, fieldName, array); } }
@Override public GenericArray<Object> jsonColumn(Value value) { if (!value.isArrayValue()) throw new RuntimeException("Support only array type json record"); List<Value> list = value.asArrayValue().list(); GenericArray<Object> array = new GenericData.Array<>(list.size(), avroSchema); for (Value val : list) { switch (val.getValueType()) { case STRING: array.add(elementConverter.stringColumn(val.asStringValue().toString())); break; case INTEGER: array.add(elementConverter.longColumn(val.asIntegerValue().toLong())); break; case FLOAT: array.add(elementConverter.doubleColumn(val.asFloatValue().toDouble())); break; case BOOLEAN: array.add(elementConverter.booleanColumn(val.asBooleanValue().getBoolean())); break; case ARRAY: array.add(elementConverter.jsonColumn(val.asArrayValue())); break; case MAP: array.add(elementConverter.jsonColumn(val.asMapValue())); break; default: throw new RuntimeException("Irregular Messagepack type"); } } return array; }
public static Schema getSchema(Object value) { if (value instanceof GenericArray) { return Schema.createArray( getElementSchema((GenericArray<?>)value) ); } else { return getSchema( getClass(value) ); } }
public void addStatefulHashMap(K key, String fieldName, Map<CharSequence,Object> map) { if (isSuper( cassandraMapping.getFamily(fieldName) )) { // as we don't know what has changed inside the map or If it's an empty map, then delete its content. deleteSubColumn(key, fieldName); // update if there is anything to update. if (!map.isEmpty()) { // If it's not empty, then update its content. for (CharSequence mapKey: map.keySet()) { // TODO: hack, do not store empty arrays Object mapValue = map.get(mapKey); if (mapValue instanceof GenericArray<?>) { if (((List<?>)mapValue).size() == 0) { continue; } } else if (mapValue instanceof Map<?,?>) { if (((Map<?, ?>)mapValue).size() == 0) { continue; } } addSubColumn(key, fieldName, mapKey.toString(), mapValue); } } } else { addColumn(key, fieldName, map); } }
public GenericArray<AResult> scannerGetRows(int scannerId, int numberOfRows) throws AIOError, AIllegalArgument { try { ResultScanner scanner = getScanner(scannerId); if (scanner == null) { AIllegalArgument aie = new AIllegalArgument(); aie.message = new Utf8("scanner ID is invalid: " + scannerId); throw aie; } return AvroUtil.resultsToAResults(scanner.next(numberOfRows)); } catch (IOException e) { AIOError ioe = new AIOError(); ioe.message = new Utf8(e.getMessage()); throw ioe; } }
static public GenericArray<AResult> resultsToAResults(Result[] results) { Schema s = Schema.createArray(AResult.SCHEMA$); GenericData.Array<AResult> aresults = null; if (results != null && results.length > 0) { aresults = new GenericData.Array<AResult>(results.length, s); for (Result result : results) { aresults.add(resultToAResult(result)); } } else { aresults = new GenericData.Array<AResult>(0, s); } return aresults; }
/** * Tests for creating, modifying, and deleting column families. * * @throws Exception */ @Test public void testFamilyAdminAndMetadata() throws Exception { AvroServer.HBaseImpl impl = new AvroServer.HBaseImpl(TEST_UTIL.getConfiguration()); ATableDescriptor tableA = new ATableDescriptor(); tableA.name = tableAname; AFamilyDescriptor familyA = new AFamilyDescriptor(); familyA.name = familyAname; Schema familyArraySchema = Schema.createArray(AFamilyDescriptor.SCHEMA$); GenericArray<AFamilyDescriptor> families = new GenericData.Array<AFamilyDescriptor>(1, familyArraySchema); families.add(familyA); tableA.families = families; impl.createTable(tableA); assertEquals(impl.describeTable(tableAname).families.size(), 1); impl.disableTable(tableAname); assertFalse(impl.isTableEnabled(tableAname)); familyA.maxVersions = 123456; impl.modifyFamily(tableAname, familyAname, familyA); assertEquals((int) impl.describeFamily(tableAname, familyAname).maxVersions, 123456); impl.deleteFamily(tableAname, familyAname); assertEquals(impl.describeTable(tableAname).families.size(), 0); impl.deleteTable(tableAname); }
private GenericRecord generatePersonRecord(Schema peopleSchema, String name, String email,String title, String homepage, String depiction, String twitter) { GenericRecord record = new GenericData.Record(peopleSchema); record.put("name", name); record.put("email", email); record.put("title", title); record.put("homepage", homepage); record.put("depiction", depiction); record.put("twitter", twitter); Schema userIdArraySchema = peopleSchema.getField("userids").schema(); Schema userIdSchema = userIdArraySchema.getElementType(); GenericArray<GenericRecord> useridrecords = new GenericData.Array<GenericRecord>(userIdArraySchema, new LinkedList<GenericRecord>()); Random r = new Random(); for(int j = 0; j < 2; j++) { StringBuilder idBuilder = new StringBuilder(); for(int i = 0; i < 8; i++) { idBuilder.append((char)(r.nextInt(26) + 'a')); } GenericRecord userId = new GenericData.Record(userIdSchema); userId.put("type", idNames[r.nextInt(idNames.length)]); userId.put("id", idBuilder.toString()); useridrecords.add(userId); } record.put("userids", useridrecords); return record; }
@SuppressWarnings("rawtypes") @Override protected void collapseSameType(GenericRecord obj) { for (Field f : obj.getSchema().getFields()) { Object value = obj.get(f.name()); if(value == null) { continue; } if (value instanceof GenericRecord) collapseSameType((GenericRecord)value); if (value instanceof GenericArray) { GenericArray array = (GenericArray)value; Set<Object> valuesHash = new HashSet<Object>(); boolean unmodified = true; for (int i = 0; i < array.size(); i++) { Object o = array.get(i); if (o instanceof GenericRecord) collapseSameType((GenericRecord) o); unmodified &= valuesHash.add(o); } if(!unmodified) { GenericArray<Object> newValues = new GenericData.Array<Object>(array.getSchema(), valuesHash); obj.put(f.name(), newValues); } } } }
@SuppressWarnings({ "rawtypes", "unchecked" }) private long getMemorySize(final IndexedRecord r) { int total = 0; final int bitsPerByte = 8; for (Field f : r.getSchema().getFields()) { switch (f.schema().getType()) { case BOOLEAN: case ENUM: case INT: total += Integer.SIZE << bitsPerByte; break; case DOUBLE: total += Double.SIZE << bitsPerByte; break; case FLOAT: total += Float.SIZE << bitsPerByte; break; case NULL: break; case STRING: total += ((String) r.get(f.pos())).length() * (Character.SIZE << bitsPerByte); break; case BYTES: total += ((Byte[]) r.get(f.pos())).length; break; case RECORD: total += new AvroTupleWrapper( (IndexedRecord) r.get(f.pos())).getMemorySize(); break; case ARRAY: total += new AvroBagWrapper( (GenericArray) r.get(f.pos())).getMemorySize(); break; } } return total; }
public void writeLine(long timestamp, String message) throws IOException { long ms = timestamp % 1000l; long second = timestamp / 1000l; if (logBlock != null && second != (Long) logBlock.get("second")) { LOG.debug("Flushing due to new second: old:{}, new:{}", (Long) logBlock.get("second"), second); writeBlock(); } if (logBlock == null) { logBlock = new GenericData.Record(logBlockSchema); logBlock.put("second", second); logBlock.put("createTime", System.currentTimeMillis()); logBlock.put("blockNumber", ++blockNumber); logBlock.put("logLines", new GenericData.Array<Record>( getMaxLinesPerRecord() / 8, messageWithMillisArraySchema)); } @SuppressWarnings("unchecked") GenericArray<Record> logLines = (GenericArray<Record>) logBlock .get("logLines"); Record record = new GenericData.Record(messageWithMillisSchema); record.put("ms", ms); record.put("eventId", 0); record.put("message", message); logLines.add(record); linesInBlock++; if (linesInBlock >= getMaxLinesPerRecord()) { LOG.debug("Hit max lines, flushing record."); writeBlock(); } }
public void writeLine(long timestamp, String message, int eventId, long createTime, long blockNumber) throws IOException { long ms = timestamp % 1000l; long second = timestamp / 1000l; if (logBlock != null) { if (second != (Long) logBlock.get("second")) { LOG.debug("Flushing due to change in second. old:{} new:{}", logBlock.get("second"), second); writeBlock(); } else if (createTime != (Long) logBlock.get("createTime")) { LOG.debug("Flushing due to change in createTime. old:{} new:{}", logBlock.get("createTime"), createTime); writeBlock(); } else if (blockNumber != (Long) logBlock.get("blockNumber")) { LOG.debug("Flushing due to change in blockNumber. old:{} new:{}", logBlock.get("blockNumber"), blockNumber); writeBlock(); } } if (logBlock == null) { logBlock = new GenericData.Record(logBlockSchema); logBlock.put("second", second); logBlock.put("createTime", createTime); logBlock.put("blockNumber", blockNumber); logBlock.put("logLines", new GenericData.Array<Record>( getMaxLinesPerRecord() / 8, messageWithMillisArraySchema)); } @SuppressWarnings("unchecked") GenericArray<Record> logLines = (GenericArray<Record>) logBlock .get("logLines"); Record record = new GenericData.Record(messageWithMillisSchema); record.put("ms", ms); record.put("eventId", eventId); record.put("message", message); logLines.add(record); }
