@Test public void testNoSchemaFilter() throws Exception { // Create "orders" table in a different schema metadata.createTable(SESSION, tableMetadataBuilder(new SchemaTableName("other", "orders")) .column("orderkey", BIGINT) .build()); // Create another table that should not be selected metadata.createTable(SESSION, tableMetadataBuilder(new SchemaTableName("schema1", "foo")) .column("orderkey", BIGINT) .build()); TupleDomain<Integer> tupleDomain = TupleDomain.withColumnDomains( ImmutableMap.<Integer, Domain>builder() .put(1, Domain.singleValue(VARCHAR, utf8Slice("orders"))) .build()); MetadataDao metadataDao = dummyHandle.attach(MetadataDao.class); Set<Long> actual = ImmutableSet.copyOf(ShardMetadataRecordCursor.getTableIds(dbi, tupleDomain)); Set<Long> expected = ImmutableSet.of( metadataDao.getTableInformation("other", "orders").getTableId(), metadataDao.getTableInformation("test", "orders").getTableId()); assertEquals(actual, expected); }
@Test public void testNoTableFilter() throws Exception { // Create "orders" table in a different schema metadata.createTable(SESSION, tableMetadataBuilder(new SchemaTableName("test", "orders2")) .column("orderkey", BIGINT) .build()); // Create another table that should not be selected metadata.createTable(SESSION, tableMetadataBuilder(new SchemaTableName("schema1", "foo")) .column("orderkey", BIGINT) .build()); TupleDomain<Integer> tupleDomain = TupleDomain.withColumnDomains( ImmutableMap.<Integer, Domain>builder() .put(0, Domain.singleValue(VARCHAR, utf8Slice("test"))) .build()); MetadataDao metadataDao = dummyHandle.attach(MetadataDao.class); Set<Long> actual = ImmutableSet.copyOf(ShardMetadataRecordCursor.getTableIds(dbi, tupleDomain)); Set<Long> expected = ImmutableSet.of( metadataDao.getTableInformation("test", "orders").getTableId(), metadataDao.getTableInformation("test", "orders2").getTableId()); assertEquals(actual, expected); }
@Override public boolean matches(long numberOfRows, Map<Integer, ColumnStatistics> statisticsByColumnIndex) { ImmutableMap.Builder<C, Domain> domains = ImmutableMap.builder(); for (ColumnReference<C> columnReference : columnReferences) { ColumnStatistics columnStatistics = statisticsByColumnIndex.get(columnReference.getOrdinal()); Domain domain; if (columnStatistics == null) { // no stats for column domain = Domain.all(columnReference.getType()); } else { domain = getDomain(columnReference.getType(), numberOfRows, columnStatistics); } domains.put(columnReference.getColumn(), domain); } TupleDomain<C> stripeDomain = TupleDomain.withColumnDomains(domains.build()); return effectivePredicate.overlaps(stripeDomain); }
private static <F, T extends Comparable<T>> Domain createDomain(Type type, boolean hasNullValue, RangeStatistics<F> rangeStatistics, Function<F, T> function) { F min = rangeStatistics.getMin(); F max = rangeStatistics.getMax(); if (min != null && max != null) { return Domain.create(ValueSet.ofRanges(Range.range(type, function.apply(min), true, function.apply(max), true)), hasNullValue); } if (max != null) { return Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(type, function.apply(max))), hasNullValue); } if (min != null) { return Domain.create(ValueSet.ofRanges(Range.greaterThanOrEqual(type, function.apply(min))), hasNullValue); } return Domain.create(ValueSet.all(type), hasNullValue); }
public static Optional<String> stringFilter(TupleDomain<Integer> constraint, int index) { if (constraint.isNone()) { return Optional.empty(); } Domain domain = constraint.getDomains().get().get(index); if ((domain == null) || !domain.isSingleValue()) { return Optional.empty(); } Object value = domain.getSingleValue(); if (value instanceof Slice) { return Optional.of(((Slice) value).toStringUtf8()); } return Optional.empty(); }
private PlanNode rewriteFilterSource(FilterNode filterNode, PlanNode source, Symbol rowNumberSymbol, int upperBound) { ExtractionResult extractionResult = fromPredicate(metadata, session, filterNode.getPredicate(), types); TupleDomain<Symbol> tupleDomain = extractionResult.getTupleDomain(); if (!isEqualRange(tupleDomain, rowNumberSymbol, upperBound)) { return new FilterNode(filterNode.getId(), source, filterNode.getPredicate()); } // Remove the row number domain because it is absorbed into the node Map<Symbol, Domain> newDomains = tupleDomain.getDomains().get().entrySet().stream() .filter(entry -> !entry.getKey().equals(rowNumberSymbol)) .collect(toMap(Map.Entry::getKey, Map.Entry::getValue)); // Construct a new predicate TupleDomain<Symbol> newTupleDomain = TupleDomain.withColumnDomains(newDomains); Expression newPredicate = ExpressionUtils.combineConjuncts( extractionResult.getRemainingExpression(), toPredicate(newTupleDomain)); if (newPredicate.equals(BooleanLiteral.TRUE_LITERAL)) { return source; } return new FilterNode(filterNode.getId(), source, newPredicate); }
/** * Reduces the number of discrete components in the Domain if there are too many. */ public static Domain simplifyDomain(Domain domain) { ValueSet values = domain.getValues(); ValueSet simplifiedValueSet = values.getValuesProcessor().<Optional<ValueSet>>transform( ranges -> { if (ranges.getOrderedRanges().size() <= 32) { return Optional.empty(); } return Optional.of(ValueSet.ofRanges(ranges.getSpan())); }, discreteValues -> { if (discreteValues.getValues().size() <= 32) { return Optional.empty(); } return Optional.of(ValueSet.all(domain.getType())); }, allOrNone -> Optional.empty()) .orElse(values); return Domain.create(simplifiedValueSet, domain.isNullAllowed()); }
private static Expression toPredicate(Domain domain, QualifiedNameReference reference) { if (domain.getValues().isNone()) { return domain.isNullAllowed() ? new IsNullPredicate(reference) : FALSE_LITERAL; } if (domain.getValues().isAll()) { return domain.isNullAllowed() ? TRUE_LITERAL : new NotExpression(new IsNullPredicate(reference)); } List<Expression> disjuncts = new ArrayList<>(); disjuncts.addAll(domain.getValues().getValuesProcessor().transform( ranges -> extractDisjuncts(domain.getType(), ranges, reference), discreteValues -> extractDisjuncts(domain.getType(), discreteValues, reference), allOrNone -> { throw new IllegalStateException("Case should not be reachable"); })); // Add nullability disjuncts if (domain.isNullAllowed()) { disjuncts.add(new IsNullPredicate(reference)); } return combineDisjunctsWithDefault(disjuncts, TRUE_LITERAL); }
private static Domain extractOrderableDomain(ComparisonExpression.Type comparisonType, Type type, Object value, boolean complement) { checkArgument(value != null); switch (comparisonType) { case EQUAL: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.equal(type, value)), complement), false); case GREATER_THAN: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.greaterThan(type, value)), complement), false); case GREATER_THAN_OR_EQUAL: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.greaterThanOrEqual(type, value)), complement), false); case LESS_THAN: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.lessThan(type, value)), complement), false); case LESS_THAN_OR_EQUAL: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.lessThanOrEqual(type, value)), complement), false); case NOT_EQUAL: return Domain.create(complementIfNecessary(ValueSet.ofRanges(Range.lessThan(type, value), Range.greaterThan(type, value)), complement), false); case IS_DISTINCT_FROM: // Need to potential complement the whole domain for IS_DISTINCT_FROM since it is null-aware return complementIfNecessary(Domain.create(ValueSet.ofRanges(Range.lessThan(type, value), Range.greaterThan(type, value)), true), complement); default: throw new AssertionError("Unhandled type: " + comparisonType); } }
@Test public void testRoundTrip() throws Exception { TupleDomain<Symbol> tupleDomain = withColumnDomains(ImmutableMap.<Symbol, Domain>builder() .put(A, Domain.singleValue(BIGINT, 1L)) .put(B, Domain.onlyNull(DOUBLE)) .put(C, Domain.notNull(VARCHAR)) .put(D, Domain.singleValue(BOOLEAN, true)) .put(E, Domain.singleValue(BIGINT, 2L)) .put(F, Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(DOUBLE, 1.1), Range.equal(DOUBLE, 2.0), Range.range(DOUBLE, 3.0, false, 3.5, true)), true)) .put(G, Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(VARCHAR, utf8Slice("2013-01-01")), Range.greaterThan(VARCHAR, utf8Slice("2013-10-01"))), false)) .put(H, Domain.singleValue(TIMESTAMP, TIMESTAMP_VALUE)) .put(I, Domain.singleValue(DATE, DATE_VALUE)) .put(J, Domain.singleValue(COLOR, COLOR_VALUE_1)) .put(K, Domain.notNull(HYPER_LOG_LOG)) .build()); ExtractionResult result = fromPredicate(toPredicate(tupleDomain)); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), tupleDomain); }
@Test public void testToPredicateAllIgnored() throws Exception { TupleDomain<Symbol> tupleDomain = withColumnDomains(ImmutableMap.<Symbol, Domain>builder() .put(A, Domain.singleValue(BIGINT, 1L)) .put(B, Domain.onlyNull(DOUBLE)) .put(C, Domain.notNull(VARCHAR)) .put(D, Domain.all(BOOLEAN)) .build()); ExtractionResult result = fromPredicate(toPredicate(tupleDomain)); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.<Symbol, Domain>builder() .put(A, Domain.singleValue(BIGINT, 1L)) .put(B, Domain.onlyNull(DOUBLE)) .put(C, Domain.notNull(VARCHAR)) .build())); }
@Test public void testFromAndPredicate() throws Exception { Expression originalPredicate = and( and(greaterThan(A, longLiteral(1L)), unprocessableExpression1(A)), and(lessThan(A, longLiteral(5L)), unprocessableExpression2(A))); ExtractionResult result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), and(unprocessableExpression1(A), unprocessableExpression2(A))); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.range(BIGINT, 1L, false, 5L, false)), false)))); // Test complements originalPredicate = not(and( and(greaterThan(A, longLiteral(1L)), unprocessableExpression1(A)), and(lessThan(A, longLiteral(5L)), unprocessableExpression2(A)))); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), originalPredicate); assertTrue(result.getTupleDomain().isAll()); originalPredicate = not(and( not(and(greaterThan(A, longLiteral(1L)), unprocessableExpression1(A))), not(and(lessThan(A, longLiteral(5L)), unprocessableExpression2(A))))); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), originalPredicate); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.notNull(BIGINT)))); }
@Test public void testFromNotPredicate() throws Exception { Expression originalPredicate = not(and(equal(A, longLiteral(1L)), unprocessableExpression1(A))); ExtractionResult result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), originalPredicate); assertTrue(result.getTupleDomain().isAll()); originalPredicate = not(unprocessableExpression1(A)); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), originalPredicate); assertTrue(result.getTupleDomain().isAll()); originalPredicate = not(TRUE_LITERAL); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertTrue(result.getTupleDomain().isNone()); originalPredicate = not(equal(A, longLiteral(1L))); result = fromPredicate(originalPredicate); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 1L), Range.greaterThan(BIGINT, 1L)), false)))); }
@Test public void testFromIsNullPredicate() throws Exception { Expression originalExpression = isNull(A); ExtractionResult result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.onlyNull(BIGINT)))); originalExpression = isNull(K); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(K, Domain.onlyNull(HYPER_LOG_LOG)))); originalExpression = not(isNull(A)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.notNull(BIGINT)))); originalExpression = not(isNull(K)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(K, Domain.notNull(HYPER_LOG_LOG)))); }
@Test public void testFromIsNotNullPredicate() throws Exception { Expression originalExpression = isNotNull(A); ExtractionResult result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.notNull(BIGINT)))); originalExpression = isNotNull(K); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(K, Domain.notNull(HYPER_LOG_LOG)))); originalExpression = not(isNotNull(A)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.onlyNull(BIGINT)))); originalExpression = not(isNotNull(K)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(K, Domain.onlyNull(HYPER_LOG_LOG)))); }
@Override public boolean matches(long numberOfRows, Map<Integer, Statistics<?>> statisticsByColumnIndex) { if (numberOfRows == 0) { return false; } ImmutableMap.Builder<C, Domain> domains = ImmutableMap.builder(); for (ColumnReference<C> columnReference : columnReferences) { Statistics<?> statistics = statisticsByColumnIndex.get(columnReference.getOrdinal()); Domain domain = getDomain(columnReference.getType(), numberOfRows, statistics); if (domain != null) { domains.put(columnReference.getColumn(), domain); } } TupleDomain<C> stripeDomain = TupleDomain.withColumnDomains(domains.build()); return effectivePredicate.overlaps(stripeDomain); }
@Override public boolean matches(Map<Integer, ParquetDictionaryDescriptor> dictionariesByColumnIndex) { ImmutableMap.Builder<C, Domain> domains = ImmutableMap.builder(); for (ColumnReference<C> columnReference : columnReferences) { ParquetDictionaryDescriptor dictionaryDescriptor = dictionariesByColumnIndex.get(columnReference.getOrdinal()); Domain domain = getDomain(columnReference.getType(), dictionaryDescriptor); if (domain != null) { domains.put(columnReference.getColumn(), domain); } } TupleDomain<C> stripeDomain = TupleDomain.withColumnDomains(domains.build()); return effectivePredicate.overlaps(stripeDomain); }
private static <F, T extends Comparable<T>> Domain createDomain(Type type, boolean hasNullValue, ParquetRangeStatistics<F> rangeStatistics, Function<F, T> function) { F min = rangeStatistics.getMin(); F max = rangeStatistics.getMax(); if (min != null && max != null) { return Domain.create(ValueSet.ofRanges(Range.range(type, function.apply(min), true, function.apply(max), true)), hasNullValue); } if (max != null) { return Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(type, function.apply(max))), hasNullValue); } if (min != null) { return Domain.create(ValueSet.ofRanges(Range.greaterThanOrEqual(type, function.apply(min))), hasNullValue); } return Domain.create(ValueSet.all(type), hasNullValue); }
private static TupleDomain<HiveColumnHandle> toCompactTupleDomain(TupleDomain<ColumnHandle> effectivePredicate, int threshold) { checkArgument(effectivePredicate.getDomains().isPresent()); ImmutableMap.Builder<HiveColumnHandle, Domain> builder = ImmutableMap.builder(); for (Map.Entry<ColumnHandle, Domain> entry : effectivePredicate.getDomains().get().entrySet()) { HiveColumnHandle hiveColumnHandle = checkType(entry.getKey(), HiveColumnHandle.class, "ConnectorColumnHandle"); ValueSet values = entry.getValue().getValues(); ValueSet compactValueSet = values.getValuesProcessor().<Optional<ValueSet>>transform( ranges -> ranges.getRangeCount() > threshold ? Optional.of(ValueSet.ofRanges(ranges.getSpan())) : Optional.empty(), discreteValues -> discreteValues.getValues().size() > threshold ? Optional.of(ValueSet.all(values.getType())) : Optional.empty(), allOrNone -> Optional.empty()) .orElse(values); builder.put(hiveColumnHandle, Domain.create(compactValueSet, entry.getValue().isNullAllowed())); } return TupleDomain.withColumnDomains(builder.build()); }
private Optional<Map<ColumnHandle, NullableValue>> parseValuesAndFilterPartition(String partitionName, List<HiveColumnHandle> partitionColumns, TupleDomain<ColumnHandle> predicate) { checkArgument(predicate.getDomains().isPresent()); List<String> partitionValues = extractPartitionKeyValues(partitionName); Map<ColumnHandle, Domain> domains = predicate.getDomains().get(); ImmutableMap.Builder<ColumnHandle, NullableValue> builder = ImmutableMap.builder(); for (int i = 0; i < partitionColumns.size(); i++) { HiveColumnHandle column = partitionColumns.get(i); NullableValue parsedValue = parsePartitionValue(partitionName, partitionValues.get(i), column.getHiveType(), timeZone); Domain allowedDomain = domains.get(column); if (allowedDomain != null && !allowedDomain.includesNullableValue(parsedValue.getValue())) { return Optional.empty(); } builder.put(column, parsedValue); } return Optional.of(builder.build()); }
@Test public void testGetPartitionSplitsTableOfflinePartition() throws Exception { ConnectorSession session = newSession(); ConnectorTableHandle tableHandle = getTableHandle(tableOfflinePartition); assertNotNull(tableHandle); ColumnHandle dsColumn = metadata.getColumnHandles(session, tableHandle).get("ds"); assertNotNull(dsColumn); Domain domain = Domain.singleValue(VARCHAR, utf8Slice("2012-12-30")); TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of(dsColumn, domain)); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(tupleDomain, bindings -> true), Optional.empty()); try { getSplitCount(splitManager.getSplits(session, getOnlyElement(tableLayoutResults).getTableLayout().getHandle())); fail("Expected PartitionOfflineException"); } catch (PartitionOfflineException e) { assertEquals(e.getTableName(), tableOfflinePartition); assertEquals(e.getPartition(), "ds=2012-12-30"); } }
private static String getWhereClause( TupleDomain<Integer> tupleDomain, List<String> columnNames, List<Type> types, Set<Integer> uuidColumnIndexes, List<ValueBuffer> bindValues) { if (tupleDomain.isNone()) { return ""; } ImmutableList.Builder<String> conjunctsBuilder = ImmutableList.builder(); Map<Integer, Domain> domainMap = tupleDomain.getDomains().get(); for (Map.Entry<Integer, Domain> entry : domainMap.entrySet()) { int index = entry.getKey(); String columnName = columnNames.get(index); Type type = types.get(index); conjunctsBuilder.add(toPredicate(index, columnName, type, entry.getValue(), uuidColumnIndexes, bindValues)); } List<String> conjuncts = conjunctsBuilder.build(); if (conjuncts.isEmpty()) { return ""; } StringBuilder where = new StringBuilder("WHERE "); return Joiner.on(" AND\n").appendTo(where, conjuncts).toString(); }
@VisibleForTesting static Iterator<Long> getTableIds(IDBI dbi, TupleDomain<Integer> tupleDomain) { Map<Integer, Domain> domains = tupleDomain.getDomains().get(); Domain schemaNameDomain = domains.get(getColumnIndex(SHARD_METADATA, SCHEMA_NAME)); Domain tableNameDomain = domains.get(getColumnIndex(SHARD_METADATA, TABLE_NAME)); StringBuilder sql = new StringBuilder("SELECT table_id FROM tables "); if (schemaNameDomain != null || tableNameDomain != null) { sql.append("WHERE "); List<String> predicates = new ArrayList<>(); if (tableNameDomain != null && tableNameDomain.isSingleValue()) { predicates.add(format("table_name = '%s'", getStringValue(tableNameDomain.getSingleValue()))); } if (schemaNameDomain != null && schemaNameDomain.isSingleValue()) { predicates.add(format("schema_name = '%s'", getStringValue(schemaNameDomain.getSingleValue()))); } sql.append(Joiner.on(" AND ").join(predicates)); } ImmutableList.Builder<Long> tableIds = ImmutableList.builder(); try (Connection connection = dbi.open().getConnection(); Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery(sql.toString())) { while (resultSet.next()) { tableIds.add(resultSet.getLong("table_id")); } } catch (SQLException e) { throw metadataError(e); } return tableIds.build().iterator(); }
private List<String> toConjuncts(List<JdbcColumnHandle> columns, TupleDomain<ColumnHandle> tupleDomain) { ImmutableList.Builder<String> builder = ImmutableList.builder(); for (JdbcColumnHandle column : columns) { Type type = column.getColumnType(); if (type.equals(BigintType.BIGINT) || type.equals(DoubleType.DOUBLE) || type.equals(BooleanType.BOOLEAN)) { Domain domain = tupleDomain.getDomains().get().get(column); if (domain != null) { builder.add(toPredicate(column.getColumnName(), domain)); } } } return builder.build(); }
private void printConstraint(int indent, ColumnHandle column, TupleDomain<ColumnHandle> constraint) { checkArgument(!constraint.isNone()); Map<ColumnHandle, Domain> domains = constraint.getDomains().get(); if (!constraint.isAll() && domains.containsKey(column)) { print(indent, ":: %s", formatDomain(simplifyDomain(domains.get(column)))); } }
private static TupleDomain<ColumnHandle> spanTupleDomain(TupleDomain<ColumnHandle> tupleDomain) { if (tupleDomain.isNone()) { return tupleDomain; } // Simplify domains if they get too complex Map<ColumnHandle, Domain> spannedDomains = Maps.transformValues(tupleDomain.getDomains().get(), DomainUtils::simplifyDomain); return TupleDomain.withColumnDomains(spannedDomains); }
private static boolean isEqualRange(TupleDomain<Symbol> tupleDomain, Symbol symbol, long upperBound) { if (tupleDomain.isNone()) { return false; } Domain domain = tupleDomain.getDomains().get().get(symbol); return domain.getValues().equals(ValueSet.ofRanges(Range.lessThanOrEqual(domain.getType(), upperBound))); }
private static OptionalInt extractUpperBound(TupleDomain<Symbol> tupleDomain, Symbol symbol) { if (tupleDomain.isNone()) { return OptionalInt.empty(); } Domain rowNumberDomain = tupleDomain.getDomains().get().get(symbol); if (rowNumberDomain == null) { return OptionalInt.empty(); } ValueSet values = rowNumberDomain.getValues(); if (values.isAll() || values.isNone() || values.getRanges().getRangeCount() <= 0) { return OptionalInt.empty(); } Range span = values.getRanges().getSpan(); if (span.getHigh().isUpperUnbounded()) { return OptionalInt.empty(); } verify(rowNumberDomain.getType().equals(BIGINT)); long upperBound = (Long) span.getHigh().getValue(); if (span.getHigh().getBound() == BELOW) { upperBound--; } if (upperBound > Integer.MAX_VALUE) { return OptionalInt.empty(); } return OptionalInt.of(Ints.checkedCast(upperBound)); }
public static Expression toPredicate(TupleDomain<Symbol> tupleDomain) { if (tupleDomain.isNone()) { return FALSE_LITERAL; } ImmutableList.Builder<Expression> conjunctBuilder = ImmutableList.builder(); for (Map.Entry<Symbol, Domain> entry : tupleDomain.getDomains().get().entrySet()) { Symbol symbol = entry.getKey(); QualifiedNameReference reference = new QualifiedNameReference(symbol.toQualifiedName()); conjunctBuilder.add(toPredicate(entry.getValue(), reference)); } return combineConjuncts(conjunctBuilder.build()); }
private static Domain extractEquatableDomain(ComparisonExpression.Type comparisonType, Type type, Object value, boolean complement) { checkArgument(value != null); switch (comparisonType) { case EQUAL: return Domain.create(complementIfNecessary(ValueSet.of(type, value), complement), false); case NOT_EQUAL: return Domain.create(complementIfNecessary(ValueSet.of(type, value).complement(), complement), false); case IS_DISTINCT_FROM: // Need to potential complement the whole domain for IS_DISTINCT_FROM since it is null-aware return complementIfNecessary(Domain.create(ValueSet.of(type, value).complement(), true), complement); default: throw new AssertionError("Unhandled type: " + comparisonType); } }
@Override protected ExtractionResult visitIsNullPredicate(IsNullPredicate node, Boolean complement) { if (!(node.getValue() instanceof QualifiedNameReference)) { return super.visitIsNullPredicate(node, complement); } Symbol symbol = Symbol.fromQualifiedName(((QualifiedNameReference) node.getValue()).getName()); Type columnType = checkedTypeLookup(symbol); Domain domain = complementIfNecessary(Domain.onlyNull(columnType), complement); return new ExtractionResult( TupleDomain.withColumnDomains(ImmutableMap.of(symbol, domain)), TRUE_LITERAL); }
@Override protected ExtractionResult visitIsNotNullPredicate(IsNotNullPredicate node, Boolean complement) { if (!(node.getValue() instanceof QualifiedNameReference)) { return super.visitIsNotNullPredicate(node, complement); } Symbol symbol = Symbol.fromQualifiedName(((QualifiedNameReference) node.getValue()).getName()); Type columnType = checkedTypeLookup(symbol); Domain domain = complementIfNecessary(Domain.notNull(columnType), complement); return new ExtractionResult( TupleDomain.withColumnDomains(ImmutableMap.of(symbol, domain)), TRUE_LITERAL); }
@Test public void testToPredicateNone() throws Exception { TupleDomain<Symbol> tupleDomain = withColumnDomains(ImmutableMap.<Symbol, Domain>builder() .put(A, Domain.singleValue(BIGINT, 1L)) .put(B, Domain.onlyNull(DOUBLE)) .put(C, Domain.notNull(VARCHAR)) .put(D, Domain.none(BOOLEAN)) .build()); assertEquals(toPredicate(tupleDomain), FALSE_LITERAL); }
@Test public void testFromBetweenPredicate() throws Exception { Expression originalExpression = between(A, longLiteral(1L), longLiteral(2L)); ExtractionResult result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.range(BIGINT, 1L, true, 2L, true)), false)))); originalExpression = between(A, longLiteral(1L), doubleLiteral(2.1)); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.range(BIGINT, 1L, true, 2L, true)), false)))); originalExpression = between(A, longLiteral(1L), nullLiteral()); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertTrue(result.getTupleDomain().isNone()); // Test complements originalExpression = not(between(A, longLiteral(1L), longLiteral(2L))); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 1L), Range.greaterThan(BIGINT, 2L)), false)))); originalExpression = not(between(A, longLiteral(1L), doubleLiteral(2.1))); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 1L), Range.greaterThan(BIGINT, 2L)), false)))); originalExpression = not(between(A, longLiteral(1L), nullLiteral())); result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(A, Domain.create(ValueSet.ofRanges(Range.lessThan(BIGINT, 1L)), false)))); }
@Test public void testExpressionConstantFolding() throws Exception { Expression originalExpression = comparison(GREATER_THAN, reference(L), function("from_hex", stringLiteral("123456"))); ExtractionResult result = fromPredicate(originalExpression); assertEquals(result.getRemainingExpression(), TRUE_LITERAL); Slice value = Slices.wrappedBuffer(BaseEncoding.base16().decode("123456")); assertEquals(result.getTupleDomain(), withColumnDomains(ImmutableMap.of(L, Domain.create(ValueSet.ofRanges(Range.greaterThan(VARBINARY, value)), false)))); Expression expression = toPredicate(result.getTupleDomain()); assertEquals(expression, comparison(GREATER_THAN, reference(L), varbinaryLiteral(value))); }
private List<String> getFilteredPartitionNames(HiveMetastore metastore, SchemaTableName tableName, List<HiveColumnHandle> partitionKeys, TupleDomain<ColumnHandle> effectivePredicate) { checkArgument(effectivePredicate.getDomains().isPresent()); List<String> filter = new ArrayList<>(); for (HiveColumnHandle partitionKey : partitionKeys) { Domain domain = effectivePredicate.getDomains().get().get(partitionKey); if (domain != null && domain.isNullableSingleValue()) { Object value = domain.getNullableSingleValue(); if (value == null) { filter.add(HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION); } else if (value instanceof Slice) { filter.add(((Slice) value).toStringUtf8()); } else if ((value instanceof Boolean) || (value instanceof Double) || (value instanceof Long)) { if (assumeCanonicalPartitionKeys) { filter.add(value.toString()); } else { // Hive treats '0', 'false', and 'False' the same. However, the metastore differentiates between these. filter.add(PARTITION_VALUE_WILDCARD); } } else { throw new PrestoException(NOT_SUPPORTED, "Only Boolean, Double and Long partition keys are supported"); } } else { filter.add(PARTITION_VALUE_WILDCARD); } } // fetch the partition names return metastore.getPartitionNamesByParts(tableName.getSchemaName(), tableName.getTableName(), filter) .orElseThrow(() -> new TableNotFoundException(tableName)); }
@Test public void testGetPartitionsWithBindings() throws Exception { ConnectorTableHandle tableHandle = getTableHandle(tablePartitionFormat); List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(newSession(), tableHandle, new Constraint<>(TupleDomain.withColumnDomains(ImmutableMap.of(intColumn, Domain.singleValue(BIGINT, 5L))), bindings -> true), Optional.empty()); assertExpectedTableLayout(getOnlyElement(tableLayoutResults).getTableLayout(), tableLayout); }
public static ShardPredicate create(TupleDomain<RaptorColumnHandle> tupleDomain) { StringJoiner predicate = new StringJoiner(" AND ").setEmptyValue("true"); ImmutableList.Builder<JDBCType> types = ImmutableList.builder(); ImmutableList.Builder<Object> values = ImmutableList.builder(); for (Entry<RaptorColumnHandle, Domain> entry : tupleDomain.getDomains().get().entrySet()) { Domain domain = entry.getValue(); if (domain.isNullAllowed() || domain.isAll()) { continue; } RaptorColumnHandle handle = entry.getKey(); Type type = handle.getColumnType(); JDBCType jdbcType = jdbcType(type); if (jdbcType == null) { continue; } if (handle.isShardUuid()) { // TODO: support multiple shard UUIDs if (domain.isSingleValue()) { predicate.add("shard_uuid = ?"); types.add(jdbcType(type)); Slice uuidSlice = checkType(entry.getValue().getSingleValue(), Slice.class, "value"); values.add(uuidStringToBytes(uuidSlice)); } continue; } if (!domain.getType().isOrderable()) { continue; } Ranges ranges = domain.getValues().getRanges(); // TODO: support multiple ranges if (ranges.getRangeCount() != 1) { continue; } Range range = getOnlyElement(ranges.getOrderedRanges()); Object minValue = null; Object maxValue = null; if (range.isSingleValue()) { minValue = range.getSingleValue(); maxValue = range.getSingleValue(); } else { if (!range.getLow().isLowerUnbounded()) { minValue = range.getLow().getValue(); } if (!range.getHigh().isUpperUnbounded()) { maxValue = range.getHigh().getValue(); } } String min = minColumn(handle.getColumnId()); String max = maxColumn(handle.getColumnId()); if (minValue != null) { predicate.add(format("(%s >= ? OR %s IS NULL)", max, max)); types.add(jdbcType); values.add(minValue); } if (maxValue != null) { predicate.add(format("(%s <= ? OR %s IS NULL)", min, min)); types.add(jdbcType); values.add(maxValue); } } return new ShardPredicate(predicate.toString(), types.build(), values.build()); }
@Test public void testSimple() throws Exception { ShardManager shardManager = createShardManager(dbi); // Add shards to the table long tableId = 1; UUID uuid1 = UUID.randomUUID(); UUID uuid2 = UUID.randomUUID(); UUID uuid3 = UUID.randomUUID(); ShardInfo shardInfo1 = new ShardInfo(uuid1, ImmutableSet.of("node1"), ImmutableList.of(), 1, 10, 100); ShardInfo shardInfo2 = new ShardInfo(uuid2, ImmutableSet.of("node2"), ImmutableList.of(), 2, 20, 200); ShardInfo shardInfo3 = new ShardInfo(uuid3, ImmutableSet.of("node3"), ImmutableList.of(), 3, 30, 300); List<ShardInfo> shards = ImmutableList.of(shardInfo1, shardInfo2, shardInfo3); long transactionId = shardManager.beginTransaction(); shardManager.commitShards( transactionId, tableId, ImmutableList.of( new ColumnInfo(1, BIGINT), new ColumnInfo(2, DATE)), shards, Optional.empty()); Slice schema = utf8Slice(DEFAULT_TEST_ORDERS.getSchemaName()); Slice table = utf8Slice(DEFAULT_TEST_ORDERS.getTableName()); DateTime date1 = DateTime.parse("2015-01-01T00:00"); DateTime date2 = DateTime.parse("2015-01-02T00:00"); TupleDomain<Integer> tupleDomain = TupleDomain.withColumnDomains( ImmutableMap.<Integer, Domain>builder() .put(0, Domain.singleValue(VARCHAR, schema)) .put(1, Domain.create(ValueSet.ofRanges(lessThanOrEqual(VARCHAR, table)), true)) .put(6, Domain.create(ValueSet.ofRanges(lessThanOrEqual(BIGINT, date1.getMillis()), greaterThan(BIGINT, date2.getMillis())), true)) .put(7, Domain.create(ValueSet.ofRanges(lessThanOrEqual(BIGINT, date1.getMillis()), greaterThan(BIGINT, date2.getMillis())), true)) .build()); List<MaterializedRow> actual; try (RecordCursor cursor = new ShardMetadataSystemTable(dbi).cursor(null, SESSION, tupleDomain)) { actual = getMaterializedResults(cursor, SHARD_METADATA.getColumns()); } assertEquals(actual.size(), 3); List<MaterializedRow> expected = ImmutableList.of( new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid1.toString()), 100, 10, 1), new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid2.toString()), 200, 20, 2), new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid3.toString()), 300, 30, 3)); assertEquals(actual, expected); }
private static Domain createDomain(Range first, Range... ranges) { return Domain.create(ValueSet.ofRanges(first, ranges), false); }