@ScalarFunction("parse_agent") @Description("Returns Map, which has keys such as 'category', 'name', 'os', 'version', 'vendor' and 'os_version'") @SqlType("map<varchar,varchar>") public Block parseAgent(@TypeParameter("map<varchar,varchar>") Type mapType, @SqlType(StandardTypes.VARCHAR) Slice slice) { String argument = slice.toStringUtf8(); Map<String, String> stringMap = Classifier.parse(argument); if (pageBuilder.isFull()) { pageBuilder.reset(); } BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(0); BlockBuilder singleMapBlockBuilder = blockBuilder.beginBlockEntry(); for (Map.Entry<String, String> entry : stringMap.entrySet()) { VARCHAR.writeSlice(singleMapBlockBuilder, Slices.utf8Slice(entry.getKey())); VARCHAR.writeSlice(singleMapBlockBuilder, Slices.utf8Slice(entry.getValue())); } blockBuilder.closeEntry(); pageBuilder.declarePosition(); return (Block) mapType.getObject(blockBuilder, blockBuilder.getPositionCount() - 1); }
private static void serializePrimitive(Type type, BlockBuilder builder, Object object) { requireNonNull(builder, "parent builder is null"); if (object == null) { builder.appendNull(); return; } if (BOOLEAN.equals(type)) { BOOLEAN.writeBoolean(builder, (Boolean) object); } else if (BIGINT.equals(type) || INTEGER.equals(type) || SMALLINT.equals(type) || TINYINT.equals(type) || REAL.equals(type) || DATE.equals(type) || TIMESTAMP.equals(type)) { type.writeLong(builder, getLongExpressedValue(object)); } else if (DOUBLE.equals(type)) { DOUBLE.writeDouble(builder, ((Number) object).doubleValue()); } else if (isVarcharType(type) || VARBINARY.equals(type) || isCharType(type)) { type.writeSlice(builder, getSliceExpressedValue(object, type)); } else { throw new UnsupportedOperationException("Unsupported primitive type: " + type); } }
@Test public void testStateDeserializer() throws Exception { String[] keys = {"loooooong string", "short string"}; double[] values = {3.14, 2.71}; MaxOrMinByStateSerializer serializer = new MaxOrMinByStateSerializer(DOUBLE, VARCHAR); BlockBuilder builder = new RowType(ImmutableList.of(VARCHAR, DOUBLE), Optional.empty()).createBlockBuilder(new BlockBuilderStatus(), 2); for (int i = 0; i < keys.length; i++) { serializer.serialize(makeState(keys[i], values[i]), builder); } Block serialized = builder.build(); for (int i = 0; i < keys.length; i++) { MaxOrMinByState deserialized = new MaxOrMinByStateFactory().createSingleState(); serializer.deserialize(serialized, i, deserialized); assertEquals(VARCHAR.getSlice(deserialized.getKey(), 0), Slices.utf8Slice(keys[i])); assertEquals(DOUBLE.getDouble(deserialized.getValue(), 0), values[i]); } }
public Block[] createAlternatingNullsBlock(List<Type> types, Block... sequenceBlocks) { Block[] alternatingNullsBlocks = new Block[sequenceBlocks.length]; for (int i = 0; i < sequenceBlocks.length; i++) { int positionCount = sequenceBlocks[i].getPositionCount(); Type type = types.get(i); BlockBuilder blockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), positionCount); for (int position = 0; position < positionCount; position++) { // append null blockBuilder.appendNull(); // append value type.appendTo(sequenceBlocks[i], position, blockBuilder); } alternatingNullsBlocks[i] = blockBuilder.build(); } return alternatingNullsBlocks; }
public static Block createTestBlock() { BlockBuilder blockBuilder = DATE.createBlockBuilder(new BlockBuilderStatus(), 15); DATE.writeLong(blockBuilder, 1111); DATE.writeLong(blockBuilder, 1111); DATE.writeLong(blockBuilder, 1111); DATE.writeLong(blockBuilder, 2222); DATE.writeLong(blockBuilder, 2222); DATE.writeLong(blockBuilder, 2222); DATE.writeLong(blockBuilder, 2222); DATE.writeLong(blockBuilder, 2222); DATE.writeLong(blockBuilder, 3333); DATE.writeLong(blockBuilder, 3333); DATE.writeLong(blockBuilder, 4444); return blockBuilder.build(); }
@Test public void testPrimitiveByteSerialization() { StateCompiler compiler = new StateCompiler(); AccumulatorStateFactory<ByteState> factory = compiler.generateStateFactory(ByteState.class); AccumulatorStateSerializer<ByteState> serializer = compiler.generateStateSerializer(ByteState.class); ByteState state = factory.createSingleState(); ByteState deserializedState = factory.createSingleState(); state.setByte((byte) 3); BlockBuilder builder = BIGINT.createBlockBuilder(new BlockBuilderStatus(), 1); serializer.serialize(state, builder); Block block = builder.build(); serializer.deserialize(block, 0, deserializedState); assertEquals(deserializedState.getByte(), state.getByte()); }
@Test public void testMapHistograms() throws Exception { MapType innerMapType = new MapType(VARCHAR, VARCHAR); MapType mapType = new MapType(innerMapType, BIGINT); InternalAggregationFunction aggregationFunction = metadata.getFunctionRegistry().getAggregateFunctionImplementation(new Signature(NAME, AGGREGATE, mapType.getTypeSignature().toString(), innerMapType.getTypeSignature().toString())); BlockBuilder builder = innerMapType.createBlockBuilder(new BlockBuilderStatus(), 3); innerMapType.writeObject(builder, mapBlockOf(VARCHAR, VARCHAR, ImmutableMap.of("a", "b"))); innerMapType.writeObject(builder, mapBlockOf(VARCHAR, VARCHAR, ImmutableMap.of("c", "d"))); innerMapType.writeObject(builder, mapBlockOf(VARCHAR, VARCHAR, ImmutableMap.of("e", "f"))); assertAggregation( aggregationFunction, 1.0, ImmutableMap.of(ImmutableMap.of("a", "b"), 1L, ImmutableMap.of("c", "d"), 1L, ImmutableMap.of("e", "f"), 1L), builder.build()); }
@OutputFunction("array<bigint>") public static void output(DigestAndPercentileArrayState state, BlockBuilder out) { QuantileDigest digest = state.getDigest(); List<Double> percentiles = state.getPercentiles(); if (percentiles == null || digest == null) { out.appendNull(); return; } BlockBuilder blockBuilder = out.beginBlockEntry(); for (int i = 0; i < percentiles.size(); i++) { Double percentile = percentiles.get(i); BIGINT.writeLong(blockBuilder, digest.getQuantile(percentile)); } out.closeEntry(); }
@Test public void testMassive() throws Exception { BlockBuilder inputBlockBuilder = BIGINT.createBlockBuilder(new BlockBuilderStatus(), 5000); TypedHistogram typedHistogram = new TypedHistogram(BIGINT, 1000); IntStream.range(1, 2000) .flatMap(i -> IntStream.iterate(i, IntUnaryOperator.identity()).limit(i)) .forEach(j -> BIGINT.writeLong(inputBlockBuilder, j)); Block inputBlock = inputBlockBuilder.build(); for (int i = 0; i < inputBlock.getPositionCount(); i++) { typedHistogram.add(i, inputBlock, 1); } Block outputBlock = typedHistogram.serialize(); for (int i = 0; i < outputBlock.getPositionCount(); i += 2) { assertEquals(BIGINT.getLong(outputBlock, i + 1), BIGINT.getLong(outputBlock, i)); } }
private BlockBuilder createBlockBuilderWithValues(long[][][] expectedValues) { BlockBuilder blockBuilder = new ArrayBlockBuilder(new ArrayBlockBuilder(BIGINT, new BlockBuilderStatus(), 100, 100), new BlockBuilderStatus(), 100); for (long[][] expectedValue : expectedValues) { if (expectedValue == null) { blockBuilder.appendNull(); } else { BlockBuilder intermediateBlockBuilder = new ArrayBlockBuilder(BIGINT, new BlockBuilderStatus(), 100, 100); for (int j = 0; j < expectedValue.length; j++) { if (expectedValue[j] == null) { intermediateBlockBuilder.appendNull(); } else { BlockBuilder innerMostBlockBuilder = BIGINT.createBlockBuilder(new BlockBuilderStatus(), expectedValue.length); for (long v : expectedValue[j]) { BIGINT.writeLong(innerMostBlockBuilder, v); } intermediateBlockBuilder.writeObject(innerMostBlockBuilder.build()).closeEntry(); } } blockBuilder.writeObject(intermediateBlockBuilder.build()).closeEntry(); } } return blockBuilder; }
@Override public void processRow(BlockBuilder output, int peerGroupStart, int peerGroupEnd, int frameStart, int frameEnd) { if (frameStart < 0) { // empty frame resetAccumulator(); } else if ((frameStart == currentStart) && (frameEnd >= currentEnd)) { // same or expanding frame accumulate(currentEnd + 1, frameEnd); currentEnd = frameEnd; } else { // different frame resetAccumulator(); accumulate(frameStart, frameEnd); currentStart = frameStart; currentEnd = frameEnd; } accumulator.evaluateFinal(output); }
public static Block createTestBlock() { BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(new BlockBuilderStatus(), 15); VARCHAR.writeString(blockBuilder, "apple"); VARCHAR.writeString(blockBuilder, "apple"); VARCHAR.writeString(blockBuilder, "apple"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "banana"); VARCHAR.writeString(blockBuilder, "cherry"); VARCHAR.writeString(blockBuilder, "cherry"); VARCHAR.writeString(blockBuilder, "date"); return blockBuilder.build(); }
public static void output(ArrayType outputType, MinMaxByNState state, BlockBuilder out) { TypedKeyValueHeap heap = state.getTypedKeyValueHeap(); if (heap == null || heap.isEmpty()) { out.appendNull(); return; } Type elementType = outputType.getElementType(); BlockBuilder arrayBlockBuilder = out.beginBlockEntry(); BlockBuilder reversedBlockBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), heap.getCapacity()); long startSize = heap.getEstimatedSize(); heap.popAll(reversedBlockBuilder); state.addMemoryUsage(heap.getEstimatedSize() - startSize); for (int i = reversedBlockBuilder.getPositionCount() - 1; i >= 0; i--) { elementType.appendTo(reversedBlockBuilder, i, arrayBlockBuilder); } out.closeEntry(); }
private void test(IntStream inputStream, BlockComparator comparator, PrimitiveIterator.OfInt outputIterator) { BlockBuilder blockBuilder = BIGINT.createBlockBuilder(new BlockBuilderStatus(), INPUT_SIZE); inputStream.forEach(x -> BIGINT.writeLong(blockBuilder, x)); TypedHeap heap = new TypedHeap(comparator, BIGINT, OUTPUT_SIZE); heap.addAll(blockBuilder); BlockBuilder resultBlockBuilder = BIGINT.createBlockBuilder(new BlockBuilderStatus(), OUTPUT_SIZE); heap.popAll(resultBlockBuilder); Block resultBlock = resultBlockBuilder.build(); assertEquals(resultBlock.getPositionCount(), OUTPUT_SIZE); for (int i = 0; i < OUTPUT_SIZE; i++) { assertEquals(BIGINT.getLong(resultBlock, i), outputIterator.nextInt()); } }
public static Block createArrayBigintBlock(Iterable<? extends Iterable<Long>> values) { ArrayType arrayType = new ArrayType(BIGINT); BlockBuilder builder = arrayType.createBlockBuilder(new BlockBuilderStatus(), 100); for (Iterable<Long> value : values) { if (value == null) { builder.appendNull(); } else { arrayType.writeObject(builder, createLongsBlock(value)); } } return builder.build(); }
@Override public void serialize(MaxOrMinByState state, BlockBuilder out) { Block keyState = state.getKey(); Block valueState = state.getValue(); checkState((keyState == null) == (valueState == null), "(keyState == null) != (valueState == null)"); if (keyState == null) { out.appendNull(); return; } BlockBuilder blockBuilder = out.beginBlockEntry(); keyType.appendTo(keyState, 0, blockBuilder); valueType.appendTo(valueState, 0, blockBuilder); out.closeEntry(); }
public static Block createTestBlock() { BlockBuilder blockBuilder = DOUBLE.createBlockBuilder(new BlockBuilderStatus(), 15); DOUBLE.writeDouble(blockBuilder, 11.11); DOUBLE.writeDouble(blockBuilder, 11.11); DOUBLE.writeDouble(blockBuilder, 11.11); DOUBLE.writeDouble(blockBuilder, 22.22); DOUBLE.writeDouble(blockBuilder, 22.22); DOUBLE.writeDouble(blockBuilder, 22.22); DOUBLE.writeDouble(blockBuilder, 22.22); DOUBLE.writeDouble(blockBuilder, 22.22); DOUBLE.writeDouble(blockBuilder, 33.33); DOUBLE.writeDouble(blockBuilder, 33.33); DOUBLE.writeDouble(blockBuilder, 44.44); return blockBuilder.build(); }
private static Block serializeObject(Type type, BlockBuilder builder, Object object) { if (!isStructuralType(type)) { serializePrimitive(type, builder, object); return null; } else if (isArrayType(type)) { return serializeList(type, builder, object); } else if (isMapType(type)) { return serializeMap(type, builder, object); } else if (isRowType(type)) { return serializeStruct(type, builder, object); } throw new RuntimeException("Unknown object type: " + type); }
private static Block serializeList(Type type, BlockBuilder builder, Object object) { List<?> list = (List) object; if (list == null) { requireNonNull(builder, "parent builder is null").appendNull(); return null; } List<Type> typeParameters = type.getTypeParameters(); checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter"); Type elementType = typeParameters.get(0); BlockBuilder currentBuilder; if (builder != null) { currentBuilder = builder.beginBlockEntry(); } else { currentBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), list.size()); } for (Object element : list) { serializeObject(elementType, currentBuilder, element); } if (builder != null) { builder.closeEntry(); return null; } else { Block resultBlock = currentBuilder.build(); return resultBlock; } }
private static Block serializeMap(Type type, BlockBuilder builder, Object object) { Map<?, ?> map = (Map) object; if (map == null) { requireNonNull(builder, "parent builder is null").appendNull(); return null; } List<Type> typeParameters = type.getTypeParameters(); checkArgument(typeParameters.size() == 2, "map must have exactly 2 type parameter"); Type keyType = typeParameters.get(0); Type valueType = typeParameters.get(1); BlockBuilder currentBuilder; if (builder != null) { currentBuilder = builder.beginBlockEntry(); } else { currentBuilder = new InterleavedBlockBuilder(typeParameters, new BlockBuilderStatus(), map.size()); } for (Map.Entry<?, ?> entry : map.entrySet()) { // Hive skips map entries with null keys if (entry.getKey() != null) { serializeObject(keyType, currentBuilder, entry.getKey()); serializeObject(valueType, currentBuilder, entry.getValue()); } } if (builder != null) { builder.closeEntry(); return null; } else { Block resultBlock = currentBuilder.build(); return resultBlock; } }
@Override public void appendTo(Block block, int position, BlockBuilder blockBuilder) { if (block.isNull(position)) { blockBuilder.appendNull(); } else { block.writeBytesTo(position, 0, block.getLength(position), blockBuilder); blockBuilder.closeEntry(); } }
@Override public void serialize(HyperLogLogState state, BlockBuilder out) { if (state.getHyperLogLog() == null) { out.appendNull(); } else { Slice slice = Slices.wrappedBuffer(HyperLogLog.toBytes(state.getHyperLogLog())); HyperLogLogType.HYPER_LOG_LOG.writeSlice(out, slice); } }
@SqlType("array(double)") public static Block geohash_decode(@SqlType(StandardTypes.VARCHAR) Slice geohash) { BlockBuilder blockBuilder = DOUBLE.createBlockBuilder(new BlockBuilderStatus(), 2); LatLong coordinates = GeoHash.decodeHash(geohash.toStringUtf8()); DOUBLE.writeDouble(blockBuilder, coordinates.getLat()); DOUBLE.writeDouble(blockBuilder, coordinates.getLon()); return blockBuilder.build(); }
public static Block nativeValueToBlock(Type type, Object object) { if (!Primitives.wrap(type.getJavaType()).isInstance(object)) { throw new IllegalArgumentException(String.format("Object '%s' does not match type %s", object, type.getJavaType())); } BlockBuilder blockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), 1); writeNativeValue(type, blockBuilder, object); return blockBuilder.build(); }
public static Block remove(MethodHandle equalsFunction, Type type, Block array, Object value) { int sizeAfterRemove = 0; List<Integer> positions = new ArrayList<>(); for (int i = 0; i < array.getPositionCount(); i++) { Object element = readNativeValue(type, array, i); try { if (element == null || !(boolean) equalsFunction.invoke(element, value)) { positions.add(i); sizeAfterRemove += array.getLength(i); } } catch (Throwable t) { Throwables.propagateIfInstanceOf(t, Error.class); Throwables.propagateIfInstanceOf(t, PrestoException.class); throw new PrestoException(INTERNAL_ERROR, t); } } if (array.getPositionCount() == positions.size()) { return array; } int entrySize = 0; if (!positions.isEmpty()) { entrySize = (int) Math.ceil(sizeAfterRemove / (double) positions.size()); } BlockBuilder blockBuilder = type.createBlockBuilder(new BlockBuilderStatus(), positions.size(), entrySize); for (int position : positions) { type.appendTo(array, position, blockBuilder); } return blockBuilder.build(); }
public static Block createStringsBlock(Iterable<String> values) { BlockBuilder builder = VARCHAR.createBlockBuilder(new BlockBuilderStatus(), 100); for (String value : values) { if (value == null) { builder.appendNull(); } else { VARCHAR.writeString(builder, value); } } return builder.build(); }
@UsedByGeneratedCode public static Block prependElement(Type elementType, Object value, Block block) { BlockBuilder blockBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), block.getPositionCount() + 1); elementType.writeObject(blockBuilder, value); for (int i = 0; i < block.getPositionCount(); i++) { elementType.appendTo(block, i, blockBuilder); } return blockBuilder.build(); }
private static void assertProjection( String expression, @Nullable Object expectedValue, Map<Symbol, Integer> symbolToInputMappings, Map<Symbol, Type> symbolTypes, int position, Block... blocks) { InterpretedProjectionFunction projectionFunction = new InterpretedProjectionFunction( createExpression(expression, METADATA, symbolTypes), symbolTypes, symbolToInputMappings, METADATA, SQL_PARSER, TEST_SESSION ); // create output Type type = projectionFunction.getType(); BlockBuilder builder = type.createBlockBuilder(new BlockBuilderStatus(), 1); // project projectionFunction.project(position, blocks, builder); // extract single value Object actualValue = BlockAssertions.getOnlyValue(type, builder.build()); assertEquals(actualValue, expectedValue); }
public static Block createTestBlock(Type mapType) { BlockBuilder blockBuilder = mapType.createBlockBuilder(new BlockBuilderStatus(), 2); mapType.writeObject(blockBuilder, mapBlockOf(BIGINT, VARCHAR, ImmutableMap.of(1, "hi"))); mapType.writeObject(blockBuilder, mapBlockOf(BIGINT, VARCHAR, ImmutableMap.of(1, "2", 2, "hello"))); return blockBuilder.build(); }
@Override public Block[] getSequenceBlocks(int start, int length) { BlockBuilder blockBuilder = BIGINT.createBlockBuilder(new BlockBuilderStatus(), length); for (int i = start; i < start + length; i++) { BIGINT.writeLong(blockBuilder, i); } return new Block[] {blockBuilder.build()}; }
@UsedByGeneratedCode public static Block prependElement(Type elementType, double value, Block block) { BlockBuilder blockBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), block.getPositionCount() + 1); elementType.writeDouble(blockBuilder, value); for (int i = 0; i < block.getPositionCount(); i++) { elementType.appendTo(block, i, blockBuilder); } return blockBuilder.build(); }
@Override public void processRow(BlockBuilder output, boolean newPeerGroup, int peerGroupCount, int currentPosition) { if (windowIndex.isNull(valueChannel, currentPosition)) { output.appendNull(); } else { long buckets = windowIndex.getLong(valueChannel, currentPosition); checkCondition(buckets > 0, INVALID_FUNCTION_ARGUMENT, "Buckets must be greater than 0"); BIGINT.writeLong(output, bucket(buckets, currentPosition) + 1); } }
protected void appendTo(PageBuilder pageBuilder, int outputChannelOffset) { BlockBuilder keyBlockBuilder = pageBuilder.getBlockBuilder(outputChannelOffset); BlockBuilder valueBlockBuilder = pageBuilder.getBlockBuilder(outputChannelOffset + 1); keyType.appendTo(block, position++, keyBlockBuilder); valueType.appendTo(block, position++, valueBlockBuilder); }
private static BlockBuilder createBlockBuilderWithValues(Slice[] expectedValues) { VariableWidthBlockBuilder blockBuilder = new VariableWidthBlockBuilder(new BlockBuilderStatus(), expectedValues.length, 32); for (Slice expectedValue : expectedValues) { if (expectedValue == null) { blockBuilder.appendNull(); } else { blockBuilder.writeBytes(expectedValue, 0, expectedValue.length()).closeEntry(); } } return blockBuilder; }
@Override public void processRow(BlockBuilder output, boolean newPeerGroup, int peerGroupCount, int currentPosition) { if (newPeerGroup) { count += peerGroupCount; } DOUBLE.writeDouble(output, ((double) count) / totalCount); }
public void nextVector(Type type, long items, BlockBuilder builder, boolean[] isNull) throws IOException { for (int i = 0; i < items; i++) { if (isNull[i]) { builder.appendNull(); } else { type.writeDouble(builder, next()); } } }
@Override public void project(int position, Block[] blocks, BlockBuilder output) { if (blocks[channelIndex].isNull(position)) { output.appendNull(); } else { BIGINT.writeLong(output, BIGINT.getLong(blocks[channelIndex], position) + 5); } }
public static Block toStackRepresentation(List<Type> parameterTypes, Object... values) { checkArgument(parameterTypes.size() == values.length, "parameterTypes.size(" + parameterTypes.size() + ") does not equal to values.length(" + values.length + ")"); BlockBuilder blockBuilder = new InterleavedBlockBuilder(parameterTypes, new BlockBuilderStatus(), 1024); for (int i = 0; i < values.length; i++) { appendToBlockBuilder(parameterTypes.get(i), values[i], blockBuilder); } return blockBuilder.build(); }
@UsedByGeneratedCode public static Block appendElement(Type elementType, Block block, Object value) { BlockBuilder blockBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), block.getPositionCount() + 1); for (int i = 0; i < block.getPositionCount(); i++) { elementType.appendTo(block, i, blockBuilder); } elementType.writeObject(blockBuilder, value); return blockBuilder.build(); }