public T processJsonArray(JsonParser jsonParser) throws IOException { int currentIndex = 0; while (true) { JsonToken token = jsonParser.nextToken(); if (token == null) { throw new JsonParseException("Unexpected end of array", jsonParser.getCurrentLocation()); } if (token == END_ARRAY) { // Index out of bounds if (exceptionOnOutOfBounds) { throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Index out of bounds"); } return null; } if (currentIndex == index) { break; } currentIndex++; jsonParser.skipChildren(); // Skip nested structure if currently at the start of one } return delegate.extract(jsonParser); }
@Override public boolean advanceNextPosition() { try { if (closed || !recordReader.hasNext()) { close(); return false; } row = (OrcStruct) recordReader.next(row); // reset loaded flags // partition keys are already loaded, but everything else is not System.arraycopy(isPartitionColumn, 0, loaded, 0, isPartitionColumn.length); return true; } catch (IOException | RuntimeException e) { closeWithSuppression(e); throw new PrestoException(HIVE_CURSOR_ERROR, e); } }
public WindowFunctionSupplier getWindowFunctionImplementation(Signature signature) { checkArgument(signature.getKind() == WINDOW || signature.getKind() == AGGREGATE, "%s is not a window function", signature); checkArgument(signature.getTypeParameterRequirements().isEmpty(), "%s has unbound type parameters", signature); Iterable<SqlFunction> candidates = functions.get(QualifiedName.of(signature.getName())); // search for exact match for (SqlFunction operator : candidates) { Type returnType = typeManager.getType(signature.getReturnType()); List<Type> argumentTypes = resolveTypes(signature.getArgumentTypes(), typeManager); Map<String, Type> boundTypeParameters = operator.getSignature().bindTypeParameters(returnType, argumentTypes, false, typeManager); if (boundTypeParameters != null) { try { return specializedWindowCache.getUnchecked(new SpecializedFunctionKey(operator, boundTypeParameters, signature.getArgumentTypes().size())); } catch (UncheckedExecutionException e) { throw Throwables.propagate(e.getCause()); } } } throw new PrestoException(FUNCTION_IMPLEMENTATION_MISSING, format("%s not found", signature)); }
private Set<HivePrivilege> getPrivileges(String user, HiveObjectRef objectReference) { ImmutableSet.Builder<HivePrivilege> privileges = ImmutableSet.builder(); try (HiveMetastoreClient client = clientProvider.createMetastoreClient()) { PrincipalPrivilegeSet privilegeSet = client.getPrivilegeSet(objectReference, user, null); if (privilegeSet != null) { Map<String, List<PrivilegeGrantInfo>> userPrivileges = privilegeSet.getUserPrivileges(); if (userPrivileges != null) { privileges.addAll(toGrants(userPrivileges.get(user))); } for (List<PrivilegeGrantInfo> rolePrivileges : privilegeSet.getRolePrivileges().values()) { privileges.addAll(toGrants(rolePrivileges)); } // We do not add the group permissions as Hive does not seem to process these } } catch (TException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } return privileges.build(); }
public static Block toArray(Type arrayType, ConnectorSession connectorSession, Slice json) { try { List<?> array = (List<?>) stackRepresentationToObject(connectorSession, json, arrayType); if (array == null) { return null; } Type elementType = ((ArrayType) arrayType).getElementType(); BlockBuilder blockBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), array.size()); for (Object element : array) { appendToBlockBuilder(elementType, element, blockBuilder); } return blockBuilder.build(); } catch (RuntimeException e) { throw new PrestoException(INVALID_CAST_ARGUMENT, "Value cannot be cast to " + arrayType, e); } }
private int loadNodeId(String nodeIdentifier) { Integer id = dao.getNodeId(nodeIdentifier); if (id != null) { return id; } // creating a node is idempotent runIgnoringConstraintViolation(() -> dao.insertNode(nodeIdentifier)); id = dao.getNodeId(nodeIdentifier); if (id == null) { throw new PrestoException(INTERNAL_ERROR, "node does not exist after insert"); } return id; }
@Override public final void load(LazyBlock lazyBlock) { if (loaded) { return; } checkState(batchId == expectedBatchId); try { Block block = recordReader.readBlock(type, columnIndex); lazyBlock.setBlock(block); } catch (IOException e) { if (e instanceof OrcCorruptionException) { throw new PrestoException(HIVE_BAD_DATA, e); } throw new PrestoException(HIVE_CURSOR_ERROR, e); } loaded = true; }
@UsedByGeneratedCode public static Object subscript(MethodHandle keyEqualsMethod, Type keyType, Type valueType, Block map, Object key) { for (int position = 0; position < map.getPositionCount(); position += 2) { try { if ((boolean) keyEqualsMethod.invokeExact(keyType.getObject(map, position), key)) { return readNativeValue(valueType, map, position + 1); // position + 1: value position } } catch (Throwable t) { Throwables.propagateIfInstanceOf(t, Error.class); Throwables.propagateIfInstanceOf(t, PrestoException.class); throw new PrestoException(INTERNAL_ERROR, t); } } return null; }
@Override public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle) { HiveTableHandle handle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); SchemaTableName tableName = schemaTableName(tableHandle); if (!allowDropTable) { throw new PrestoException(PERMISSION_DENIED, "DROP TABLE is disabled in this Hive catalog"); } Optional<Table> target = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!target.isPresent()) { throw new TableNotFoundException(tableName); } Table table = target.get(); if (!session.getUser().equals(table.getOwner())) { throw new PrestoException(PERMISSION_DENIED, format("Unable to drop table '%s': owner of the table is different from session user", table)); } metastore.dropTable(handle.getSchemaName(), handle.getTableName()); }
@PreDestroy public void stop() { boolean queryCancelled = false; for (QueryExecution queryExecution : queries.values()) { QueryInfo queryInfo = queryExecution.getQueryInfo(); if (queryInfo.getState().isDone()) { continue; } log.info("Server shutting down. Query %s has been cancelled", queryExecution.getQueryInfo().getQueryId()); queryExecution.fail(new PrestoException(SERVER_SHUTTING_DOWN, "Server is shutting down. Query " + queryInfo.getQueryId() + " has been cancelled")); queryCancelled = true; } if (queryCancelled) { try { TimeUnit.SECONDS.sleep(5); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } } queryManagementExecutor.shutdownNow(); queryExecutor.shutdownNow(); }
private void writeShard(UUID shardUuid) { if (backupStore.isPresent() && !backupExists(shardUuid)) { throw new PrestoException(RAPTOR_ERROR, "Backup does not exist after write"); } File stagingFile = storageService.getStagingFile(shardUuid); File storageFile = storageService.getStorageFile(shardUuid); storageService.createParents(storageFile); try { Files.move(stagingFile.toPath(), storageFile.toPath(), ATOMIC_MOVE); } catch (IOException e) { throw new PrestoException(RAPTOR_ERROR, "Failed to move shard file", e); } }
@UsedByGeneratedCode public static long arrayPosition(Type type, MethodHandle equalMethodHandle, Block array, Slice element) { int size = array.getPositionCount(); for (int i = 0; i < size; i++) { if (!array.isNull(i)) { Slice arrayValue = type.getSlice(array, i); try { if ((boolean) equalMethodHandle.invokeExact(arrayValue, element)) { return i + 1; // result is 1-based (instead of 0) } } catch (Throwable t) { Throwables.propagateIfInstanceOf(t, Error.class); Throwables.propagateIfInstanceOf(t, PrestoException.class); throw new PrestoException(INTERNAL_ERROR, t); } } } return 0; }
private List<ColumnInfo> getColumnInfo(OrcReader reader) { // TODO: These should be stored as proper metadata. // XXX: Relying on ORC types will not work when more Presto types are supported. List<String> names = reader.getColumnNames(); Type rowType = getType(reader.getFooter().getTypes(), 0); if (names.size() != rowType.getTypeParameters().size()) { throw new PrestoException(RAPTOR_ERROR, "Column names and types do not match"); } ImmutableList.Builder<ColumnInfo> list = ImmutableList.builder(); for (int i = 0; i < names.size(); i++) { list.add(new ColumnInfo(Long.parseLong(names.get(i)), rowType.getTypeParameters().get(i))); } return list.build(); }
public void appendRow(Row row) { List<Object> columns = row.getColumns(); checkArgument(columns.size() == columnTypes.size()); for (int channel = 0; channel < columns.size(); channel++) { tableInspector.setStructFieldData(orcRow, structFields.get(channel), columns.get(channel)); } try { recordWriter.write(serializer.serialize(orcRow, tableInspector)); } catch (IOException e) { throw new PrestoException(RAPTOR_ERROR, "Failed to write record", e); } rowCount++; uncompressedSize += row.getSizeInBytes(); }
private RelationPlan createTableCreationPlan(Analysis analysis) { QualifiedObjectName destination = analysis.getCreateTableDestination().get(); RelationPlan plan = createRelationPlan(analysis); TableMetadata tableMetadata = createTableMetadata(destination, getOutputTableColumns(plan), analysis.getCreateTableProperties(), plan.getSampleWeight().isPresent()); if (plan.getSampleWeight().isPresent() && !metadata.canCreateSampledTables(session, destination.getCatalogName())) { throw new PrestoException(NOT_SUPPORTED, "Cannot write sampled data to a store that doesn't support sampling"); } return createTableWriterPlan( analysis, plan, new CreateName(destination.getCatalogName(), tableMetadata), tableMetadata.getVisibleColumnNames()); }
@Override public final void load(LazyBlock lazyBlock) { if (loaded) { return; } checkState(batchId == expectedBatchId); try { Block block = recordReader.readBlock(type, columnIndex); lazyBlock.setBlock(block); } catch (IOException e) { throw new PrestoException(RAPTOR_ERROR, e); } loaded = true; }
@Test public void testAssignRandomNodeWhenBackupAvailable() throws InterruptedException, URISyntaxException { InMemoryNodeManager nodeManager = new InMemoryNodeManager(); RaptorConnectorId connectorId = new RaptorConnectorId("raptor"); NodeSupplier nodeSupplier = new RaptorNodeSupplier(nodeManager, connectorId); PrestoNode node = new PrestoNode(UUID.randomUUID().toString(), new URI("http://127.0.0.1/"), NodeVersion.UNKNOWN); nodeManager.addNode(connectorId.toString(), node); RaptorSplitManager raptorSplitManagerWithBackup = new RaptorSplitManager(connectorId, nodeSupplier, shardManager, true); deleteShardNodes(); ConnectorTableLayoutResult layout = getOnlyElement(metadata.getTableLayouts(SESSION, tableHandle, Constraint.alwaysTrue(), Optional.empty())); ConnectorSplitSource partitionSplit = getSplits(raptorSplitManagerWithBackup, layout); List<ConnectorSplit> batch = getFutureValue(partitionSplit.getNextBatch(1), PrestoException.class); assertEquals(getOnlyElement(getOnlyElement(batch).getAddresses()), node.getHostAndPort()); }
private String getFormattedSql(CreateView statement) { Query query = statement.getQuery(); String sql = formatSql(query); // verify round-trip Statement parsed; try { parsed = sqlParser.createStatement(sql); } catch (ParsingException e) { throw new PrestoException(INTERNAL_ERROR, "Formatted query does not parse: " + query); } if (!query.equals(parsed)) { throw new PrestoException(INTERNAL_ERROR, "Query does not round-trip: " + query); } return sql; }
@Test public void testRollback() { long tableId = createTable("test"); List<ColumnInfo> columns = ImmutableList.of(new ColumnInfo(1, BIGINT)); List<ShardInfo> shards = ImmutableList.of(shardInfo(UUID.randomUUID(), "node1")); shardManager.createTable(tableId, columns); long transactionId = shardManager.beginTransaction(); shardManager.rollbackTransaction(transactionId); try { shardManager.commitShards(transactionId, tableId, columns, shards, Optional.empty()); fail("expected exception"); } catch (PrestoException e) { assertEquals(e.getErrorCode(), TRANSACTION_CONFLICT.toErrorCode()); } }
@Test public void testTransactionAbort() throws Exception { // start table creation long transactionId = 1; ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(SESSION, getOrdersTable()); // transaction is in progress assertTrue(transactionExists(transactionId)); assertNull(transactionSuccessful(transactionId)); // force transaction to abort shardManager.rollbackTransaction(transactionId); assertTrue(transactionExists(transactionId)); assertFalse(transactionSuccessful(transactionId)); // commit table creation try { metadata.finishCreateTable(SESSION, outputHandle, ImmutableList.of()); fail("expected exception"); } catch (PrestoException e) { assertEquals(e.getErrorCode(), TRANSACTION_CONFLICT.toErrorCode()); } }
@Override public boolean advanceNextPosition() { try { if (closed || !recordReader.next(key, value)) { close(); return false; } // reset loaded flags // partition keys are already loaded, but everything else is not System.arraycopy(isPartitionColumn, 0, loaded, 0, isPartitionColumn.length); return true; } catch (IOException | RuntimeException e) { closeWithSuppression(e); throw new PrestoException(HIVE_CURSOR_ERROR, e); } }
@Override public void renameTable(ConnectorSession session, ConnectorTableHandle tableHandle, SchemaTableName newTableName) { if (!allowRenameTable) { throw new PrestoException(PERMISSION_DENIED, "Renaming tables is disabled in this Hive catalog"); } HiveTableHandle handle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); SchemaTableName tableName = schemaTableName(tableHandle); Optional<Table> source = metastore.getTable(handle.getSchemaName(), handle.getTableName()); if (!source.isPresent()) { throw new TableNotFoundException(tableName); } Table table = source.get(); table.setDbName(newTableName.getSchemaName()); table.setTableName(newTableName.getTableName()); metastore.alterTable(handle.getSchemaName(), handle.getTableName(), table); }
private static ScalarFunctionImplementation specializeArrayJoin(Map<String, Type> types, FunctionRegistry functionRegistry, List<Boolean> nullableArguments, Signature signature, MethodHandle methodHandle) { Type type = types.get("T"); if (type instanceof UnknownType) { return new ScalarFunctionImplementation(false, nullableArguments, methodHandle.bindTo(null).bindTo(type), true); } else { try { ScalarFunctionImplementation castFunction = functionRegistry.getScalarFunctionImplementation(internalOperator(CAST.name(), VARCHAR_TYPE_SIGNATURE, ImmutableList.of(type.getTypeSignature()))); return new ScalarFunctionImplementation(false, nullableArguments, methodHandle.bindTo(castFunction.getMethodHandle()).bindTo(type), true); } catch (PrestoException e) { throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Input type %s not supported", type), e); } } }
public static boolean equals(MethodHandle equalsFunction, Type type, Block leftArray, Block rightArray) { if (leftArray.getPositionCount() != rightArray.getPositionCount()) { return false; } for (int i = 0; i < leftArray.getPositionCount(); i++) { checkElementNotNull(leftArray.isNull(i), ARRAY_NULL_ELEMENT_MSG); checkElementNotNull(rightArray.isNull(i), ARRAY_NULL_ELEMENT_MSG); Object leftElement = readNativeValue(type, leftArray, i); Object rightElement = readNativeValue(type, rightArray, i); try { if (!(boolean) equalsFunction.invoke(leftElement, rightElement)) { return false; } } catch (Throwable t) { Throwables.propagateIfInstanceOf(t, Error.class); Throwables.propagateIfInstanceOf(t, PrestoException.class); throw new PrestoException(INTERNAL_ERROR, t); } } return true; }
@Override public long getLong() { if (isNull()) { return 0L; } switch (fieldType) { case BYTE: return value.get(); case SHORT: return value.getShort(); case INT: return value.getInt(); case LONG: return value.getLong(); default: throw new PrestoException(KINESIS_CONVERSION_NOT_SUPPORTED, format("conversion %s to long not supported", fieldType)); } }
@Override public void commitCreateTable(JdbcOutputTableHandle handle, Collection<Slice> fragments) { StringBuilder sql = new StringBuilder() .append("ALTER TABLE ") .append(quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTemporaryTableName())) .append(" RENAME TO ") .append(quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTableName())); try (Connection connection = getConnection(handle)) { execute(connection, sql.toString()); } catch (SQLException e) { throw new PrestoException(JDBC_ERROR, e); } }
public static void input(MethodHandle methodHandle, NullableLongState state, long value) { if (state.isNull()) { state.setNull(false); state.setLong(value); return; } try { if ((boolean) methodHandle.invokeExact(value, state.getLong())) { state.setLong(value); } } catch (Throwable t) { Throwables.propagateIfInstanceOf(t, Error.class); Throwables.propagateIfInstanceOf(t, PrestoException.class); throw new PrestoException(INTERNAL_ERROR, t); } }
private void handleFailure(Throwable t) { // Can not delegate to other callback while holding a lock on this checkNotHoldsLock(); requestsFailed.incrementAndGet(); requestsCompleted.incrementAndGet(); if (t instanceof PrestoException) { clientCallback.clientFailed(HttpPageBufferClient.this, t); } synchronized (HttpPageBufferClient.this) { increaseErrorDelay(); future = null; lastUpdate = DateTime.now(); } clientCallback.requestComplete(HttpPageBufferClient.this); }
public static void input(KeyValuePairsState state, Block key, Block value, int position) { KeyValuePairs pairs = state.get(); if (pairs == null) { pairs = new KeyValuePairs(state.getKeyType(), state.getValueType(), true); state.set(pairs); } long startSize = pairs.estimatedInMemorySize(); try { pairs.add(key, value, position, position); } catch (ExceededMemoryLimitException e) { throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("The result of map_agg may not exceed %s", e.getMaxMemory())); } state.addMemoryUsage(pairs.estimatedInMemorySize() - startSize); }
@Override public long getLong() { if (isNull()) { return 0L; } switch (fieldType) { case BYTE: return value.get(); case SHORT: return value.getShort(); case INT: return value.getInt(); case LONG: return value.getLong(); default: throw new PrestoException(DECODER_CONVERSION_NOT_SUPPORTED, format("conversion %s to long not supported", fieldType)); } }
public static void input(MethodHandle methodHandle, NullableDoubleState state, double value) { if (state.isNull()) { state.setNull(false); state.setDouble(value); return; } try { if ((boolean) methodHandle.invokeExact(value, state.getDouble())) { state.setDouble(value); } } catch (Throwable t) { Throwables.propagateIfInstanceOf(t, Error.class); Throwables.propagateIfInstanceOf(t, PrestoException.class); throw new PrestoException(INTERNAL_ERROR, t); } }
private static DateTime parseDateTimeHelper(DateTimeFormatter formatter, String datetimeString) { try { return formatter.parseDateTime(datetimeString); } catch (IllegalArgumentException e) { throw new PrestoException(INVALID_FUNCTION_ARGUMENT, e); } }
public static long timestampPartitionKey(String value, DateTimeZone zone, String name) { try { return parseHiveTimestamp(value, zone); } catch (IllegalArgumentException e) { throw new PrestoException(HIVE_INVALID_PARTITION_VALUE, format("Invalid partition value '%s' for TIMESTAMP partition key: %s", value, name)); } }
public static MethodHandle methodHandle(Class<?> clazz, String name, Class<?>... parameterTypes) { try { return MethodHandles.lookup().unreflect(clazz.getMethod(name, parameterTypes)); } catch (IllegalAccessException | NoSuchMethodException e) { throw new PrestoException(INTERNAL_ERROR, e); } }
public static Field field(Class<?> clazz, String name) { try { return clazz.getField(name); } catch (NoSuchFieldException e) { throw new PrestoException(INTERNAL_ERROR, e); } }
@Override public void renameColumn(ConnectorSession session, ConnectorTableHandle tableHandle, ColumnHandle source, String target) { if (!allowRenameColumn) { throw new PrestoException(PERMISSION_DENIED, "Renaming columns is disabled in this Hive catalog"); } HiveTableHandle hiveTableHandle = checkType(tableHandle, HiveTableHandle.class, "tableHandle"); HiveColumnHandle sourceHandle = checkType(source, HiveColumnHandle.class, "columnHandle"); Optional<Table> tableMetadata = metastore.getTable(hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName()); if (!tableMetadata.isPresent()) { throw new TableNotFoundException(hiveTableHandle.getSchemaTableName()); } Table table = tableMetadata.get(); StorageDescriptor sd = table.getSd(); ImmutableList.Builder<FieldSchema> columns = ImmutableList.builder(); for (FieldSchema fieldSchema : sd.getCols()) { if (fieldSchema.getName().equals(sourceHandle.getName())) { columns.add(new FieldSchema(target, fieldSchema.getType(), fieldSchema.getComment())); } else { columns.add(fieldSchema); } } sd.setCols(columns.build()); table.setSd(sd); metastore.alterTable(hiveTableHandle.getSchemaName(), hiveTableHandle.getTableName(), table); }
@ScalarOperator(DIVIDE) @SqlType(StandardTypes.DOUBLE) public static double divide(@SqlType(StandardTypes.DOUBLE) double left, @SqlType(StandardTypes.DOUBLE) double right) { try { return left / right; } catch (ArithmeticException e) { throw new PrestoException(DIVISION_BY_ZERO, e); } }
@ScalarOperator(OperatorType.CAST) @SqlType(StandardTypes.JSON) public static Slice castVarcharToJson(@SqlType(StandardTypes.VARCHAR) Slice slice) throws IOException { // TEMPORARY: added to ease migrating user away from cast between json and varchar throw new PrestoException(NOT_SUPPORTED, "`CAST (varcharValue as JSON)` is removed. Use `JSON_PARSE(varcharValue)`."); }
private static int hexDigitCharToInt(byte b) { if (b >= '0' && b <= '9') { return b - '0'; } else if (b >= 'a' && b <= 'f') { return b - 'a' + 10; } else if (b >= 'A' && b <= 'F') { return b - 'A' + 10; } throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "invalid hex character: " + (char) b); }
@Override public void restoreShard(UUID uuid, File target) { try { store.restoreShard(uuid, target); } catch (UncheckedTimeoutException e) { throw new PrestoException(RAPTOR_BACKUP_TIMEOUT, "Shard restore timed out"); } }