@Override public Optional<TableHandle> getTableHandle(Session session, QualifiedObjectName table) { requireNonNull(table, "table is null"); ConnectorEntry entry = getConnectorFor(session, table); if (entry != null) { ConnectorMetadata metadata = entry.getMetadata(session); ConnectorTableHandle tableHandle = metadata.getTableHandle(session.toConnectorSession(entry.getCatalog()), table.asSchemaTableName()); if (tableHandle != null) { return Optional.of(new TableHandle(entry.getConnectorId(), tableHandle)); } } return Optional.empty(); }
@Override public List<TableLayoutResult> getLayouts(Session session, TableHandle table, Constraint<ColumnHandle> constraint, Optional<Set<ColumnHandle>> desiredColumns) { if (constraint.getSummary().isNone()) { return ImmutableList.of(); } TupleDomain<ColumnHandle> summary = constraint.getSummary(); String connectorId = table.getConnectorId(); ConnectorTableHandle connectorTable = table.getConnectorHandle(); Predicate<Map<ColumnHandle, NullableValue>> predicate = constraint.predicate(); ConnectorEntry entry = getConnectorMetadata(connectorId); ConnectorMetadata metadata = entry.getMetadata(session); ConnectorTransactionHandle transaction = entry.getTransactionHandle(session); ConnectorSession connectorSession = session.toConnectorSession(entry.getCatalog()); List<ConnectorTableLayoutResult> layouts = metadata.getTableLayouts(connectorSession, connectorTable, new Constraint<>(summary, predicate::test), desiredColumns); return layouts.stream() .map(layout -> new TableLayoutResult(fromConnectorLayout(connectorId, transaction, layout.getTableLayout()), layout.getUnenforcedConstraint())) .collect(toImmutableList()); }
@Override public List<QualifiedObjectName> listTables(Session session, QualifiedTablePrefix prefix) { requireNonNull(prefix, "prefix is null"); String schemaNameOrNull = prefix.getSchemaName().orElse(null); Set<QualifiedObjectName> tables = new LinkedHashSet<>(); for (ConnectorEntry entry : allConnectorsFor(prefix.getCatalogName())) { ConnectorMetadata metadata = entry.getMetadata(session); ConnectorSession connectorSession = session.toConnectorSession(entry.getCatalog()); for (QualifiedObjectName tableName : transform(metadata.listTables(connectorSession, schemaNameOrNull), convertFromSchemaTableName(prefix.getCatalogName()))) { tables.add(tableName); } } return ImmutableList.copyOf(tables); }
@Override public void renameTable(Session session, TableHandle tableHandle, QualifiedObjectName newTableName) { String catalogName = newTableName.getCatalogName(); ConnectorEntry target = connectorsByCatalog.get(catalogName); if (target == null) { throw new PrestoException(NOT_FOUND, format("Target catalog '%s' does not exist", catalogName)); } if (!tableHandle.getConnectorId().equals(target.getConnectorId())) { throw new PrestoException(SYNTAX_ERROR, "Cannot rename tables across catalogs"); } ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadataForWrite(session); metadata.renameTable(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), newTableName.asSchemaTableName()); }
@Override public List<QualifiedObjectName> listViews(Session session, QualifiedTablePrefix prefix) { requireNonNull(prefix, "prefix is null"); String schemaNameOrNull = prefix.getSchemaName().orElse(null); Set<QualifiedObjectName> views = new LinkedHashSet<>(); for (ConnectorEntry entry : allConnectorsFor(prefix.getCatalogName())) { ConnectorMetadata metadata = entry.getMetadata(session); ConnectorSession connectorSession = session.toConnectorSession(entry.getCatalog()); for (QualifiedObjectName tableName : transform(metadata.listViews(connectorSession, schemaNameOrNull), convertFromSchemaTableName(prefix.getCatalogName()))) { views.add(tableName); } } return ImmutableList.copyOf(views); }
@Override public Map<QualifiedObjectName, ViewDefinition> getViews(Session session, QualifiedTablePrefix prefix) { requireNonNull(prefix, "prefix is null"); SchemaTablePrefix tablePrefix = prefix.asSchemaTablePrefix(); Map<QualifiedObjectName, ViewDefinition> views = new LinkedHashMap<>(); for (ConnectorEntry connectorEntry : allConnectorsFor(prefix.getCatalogName())) { ConnectorMetadata metadata = connectorEntry.getMetadata(session); ConnectorSession connectorSession = session.toConnectorSession(connectorEntry.getCatalog()); for (Entry<SchemaTableName, ConnectorViewDefinition> entry : metadata.getViews(connectorSession, tablePrefix).entrySet()) { QualifiedObjectName viewName = new QualifiedObjectName( prefix.getCatalogName(), entry.getKey().getSchemaName(), entry.getKey().getTableName()); views.put(viewName, deserializeView(entry.getValue().getViewData())); } } return ImmutableMap.copyOf(views); }
@Override public Optional<ViewDefinition> getView(Session session, QualifiedObjectName viewName) { ConnectorEntry entry = getConnectorFor(session, viewName); if (entry != null) { ConnectorMetadata metadata = entry.getMetadata(session); Map<SchemaTableName, ConnectorViewDefinition> views = metadata.getViews( session.toConnectorSession(entry.getCatalog()), viewName.asSchemaTableName().toSchemaTablePrefix()); ConnectorViewDefinition view = views.get(viewName.asSchemaTableName()); if (view != null) { return Optional.of(deserializeView(view.getViewData())); } } return Optional.empty(); }
private static void registerBogusConnector(TransactionManager transactionManager, String connectorId) { transactionManager.addConnector(connectorId, new Connector() { @Override public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) { // Just return something return new ConnectorTransactionHandle() {}; } @Override public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle) { throw new UnsupportedOperationException(); } @Override public ConnectorSplitManager getSplitManager() { throw new UnsupportedOperationException(); } }); }
/** * Guaranteed to be called at most once per transaction. The returned metadata will only be accessed * in a single threaded context. * * @param transactionHandle transaction handle */ @Override public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle) { HDFSMetadata metadata = transactions.get(transactionHandle); checkArgument(metadata != null, "no such transaction: %s", transactionHandle); return hdfsMetadataFactory.create(); }
@Override public Connector create(String connectorId, Map<String, String> properties) { return new Connector() { @Override public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) { checkConnectorSupports(READ_COMMITTED, isolationLevel); return JmxTransactionHandle.INSTANCE; } @Override public ConnectorMetadata getMetadata(ConnectorTransactionHandle transactionHandle) { return new JmxMetadata(connectorId, mbeanServer); } @Override public ConnectorSplitManager getSplitManager() { return new JmxSplitManager(connectorId, nodeManager); } @Override public ConnectorRecordSetProvider getRecordSetProvider() { return new JmxRecordSetProvider(mbeanServer, nodeManager.getCurrentNode().getNodeIdentifier()); } }; }
@Override public List<String> listSchemaNames(Session session, String catalogName) { checkCatalogName(catalogName); ImmutableSet.Builder<String> schemaNames = ImmutableSet.builder(); for (ConnectorEntry entry : allConnectorsFor(catalogName)) { ConnectorMetadata metadata = entry.getMetadata(session); schemaNames.addAll(metadata.listSchemaNames(session.toConnectorSession(entry.getCatalog()))); } return ImmutableList.copyOf(schemaNames.build()); }
@Override public TableLayout getLayout(Session session, TableLayoutHandle handle) { String connectorId = handle.getConnectorId(); ConnectorEntry entry = getConnectorMetadata(connectorId); ConnectorMetadata metadata = entry.getMetadata(session); ConnectorTransactionHandle transaction = entry.getTransactionHandle(session); return fromConnectorLayout(connectorId, transaction, metadata.getTableLayout(session.toConnectorSession(entry.getCatalog()), handle.getConnectorHandle())); }
@Override public TableMetadata getTableMetadata(Session session, TableHandle tableHandle) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle()); return new TableMetadata(tableHandle.getConnectorId(), tableMetadata); }
@Override public Map<String, ColumnHandle> getColumnHandles(Session session, TableHandle tableHandle) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); return metadata.getColumnHandles(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle()); }
@Override public ColumnMetadata getColumnMetadata(Session session, TableHandle tableHandle, ColumnHandle columnHandle) { requireNonNull(tableHandle, "tableHandle is null"); requireNonNull(columnHandle, "columnHandle is null"); ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); return metadata.getColumnMetadata(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), columnHandle); }
@Override public Optional<ColumnHandle> getSampleWeightColumnHandle(Session session, TableHandle tableHandle) { requireNonNull(tableHandle, "tableHandle is null"); ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); ColumnHandle handle = metadata.getSampleWeightColumnHandle(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle()); return Optional.ofNullable(handle); }
@Override public boolean canCreateSampledTables(Session session, String catalogName) { ConnectorEntry entry = connectorsByCatalog.get(catalogName); checkArgument(entry != null, "Catalog %s does not exist", catalogName); ConnectorMetadata metadata = entry.getMetadata(session); return metadata.canCreateSampledTables(session.toConnectorSession(entry.getCatalog())); }
@Override public void createTable(Session session, String catalogName, TableMetadata tableMetadata) { ConnectorEntry entry = connectorsByCatalog.get(catalogName); checkArgument(entry != null, "Catalog %s does not exist", catalogName); ConnectorMetadata metadata = entry.getMetadataForWrite(session); metadata.createTable(session.toConnectorSession(entry.getCatalog()), tableMetadata.getMetadata()); }
@Override public void renameColumn(Session session, TableHandle tableHandle, ColumnHandle source, String target) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadataForWrite(session); metadata.renameColumn(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), source, target.toLowerCase(ENGLISH)); }
@Override public void addColumn(Session session, TableHandle tableHandle, ColumnMetadata column) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadataForWrite(session); metadata.addColumn(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), column); }
@Override public void dropTable(Session session, TableHandle tableHandle) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadataForWrite(session); metadata.dropTable(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle()); }
@Override public OutputTableHandle beginCreateTable(Session session, String catalogName, TableMetadata tableMetadata) { ConnectorEntry entry = connectorsByCatalog.get(catalogName); checkArgument(entry != null, "Catalog %s does not exist", catalogName); ConnectorMetadata metadata = entry.getMetadataForWrite(session); ConnectorTransactionHandle transactionHandle = entry.getTransactionHandle(session); ConnectorSession connectorSession = session.toConnectorSession(entry.getCatalog()); ConnectorOutputTableHandle handle = metadata.beginCreateTable(connectorSession, tableMetadata.getMetadata()); return new OutputTableHandle(entry.getConnectorId(), transactionHandle, handle); }
@Override public void finishCreateTable(Session session, OutputTableHandle tableHandle, Collection<Slice> fragments) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); metadata.finishCreateTable(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), fragments); }
@Override public InsertTableHandle beginInsert(Session session, TableHandle tableHandle) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadataForWrite(session); ConnectorTransactionHandle transactionHandle = entry.getTransactionHandle(session); ConnectorInsertTableHandle handle = metadata.beginInsert(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle()); return new InsertTableHandle(tableHandle.getConnectorId(), transactionHandle, handle); }
@Override public void finishInsert(Session session, InsertTableHandle tableHandle, Collection<Slice> fragments) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); metadata.finishInsert(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), fragments); }
@Override public ColumnHandle getUpdateRowIdColumnHandle(Session session, TableHandle tableHandle) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); return metadata.getUpdateRowIdColumnHandle(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle()); }
@Override public boolean supportsMetadataDelete(Session session, TableHandle tableHandle, TableLayoutHandle tableLayoutHandle) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); return metadata.supportsMetadataDelete( session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), tableLayoutHandle.getConnectorHandle()); }
@Override public OptionalLong metadataDelete(Session session, TableHandle tableHandle, TableLayoutHandle tableLayoutHandle) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadataForWrite(session); return metadata.metadataDelete(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), tableLayoutHandle.getConnectorHandle()); }
@Override public TableHandle beginDelete(Session session, TableHandle tableHandle) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadataForWrite(session); ConnectorTableHandle newHandle = metadata.beginDelete(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle()); return new TableHandle(tableHandle.getConnectorId(), newHandle); }
@Override public void finishDelete(Session session, TableHandle tableHandle, Collection<Slice> fragments) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); metadata.finishDelete(session.toConnectorSession(entry.getCatalog()), tableHandle.getConnectorHandle(), fragments); }
@Override public void createView(Session session, QualifiedObjectName viewName, String viewData, boolean replace) { ConnectorEntry entry = connectorsByCatalog.get(viewName.getCatalogName()); checkArgument(entry != null, "Catalog %s does not exist", viewName.getCatalogName()); ConnectorMetadata metadata = entry.getMetadataForWrite(session); metadata.createView(session.toConnectorSession(entry.getCatalog()), viewName.asSchemaTableName(), viewData, replace); }
@Override public void dropView(Session session, QualifiedObjectName viewName) { ConnectorEntry entry = connectorsByCatalog.get(viewName.getCatalogName()); checkArgument(entry != null, "Catalog %s does not exist", viewName.getCatalogName()); ConnectorMetadata metadata = entry.getMetadataForWrite(session); metadata.dropView(session.toConnectorSession(entry.getCatalog()), viewName.asSchemaTableName()); }
@Override public Optional<ResolvedIndex> resolveIndex(Session session, TableHandle tableHandle, Set<ColumnHandle> indexableColumns, Set<ColumnHandle> outputColumns, TupleDomain<ColumnHandle> tupleDomain) { ConnectorEntry entry = lookupConnectorFor(tableHandle); ConnectorMetadata metadata = entry.getMetadata(session); ConnectorTransactionHandle transaction = entry.getTransactionHandle(session); ConnectorSession connectorSession = session.toConnectorSession(entry.getCatalog()); Optional<ConnectorResolvedIndex> resolvedIndex = metadata.resolveIndex(connectorSession, tableHandle.getConnectorHandle(), indexableColumns, outputColumns, tupleDomain); return resolvedIndex.map(resolved -> new ResolvedIndex(tableHandle.getConnectorId(), transaction, resolved)); }
public ConnectorMetadata getMetadataForWrite(Session session) { TransactionId transactionId = session.getRequiredTransactionId(); ConnectorMetadata metadata = transactionManager.getMetadata(transactionId, connectorId); transactionManager.checkConnectorWrite(transactionId, connectorId); return metadata; }
@Test public void testTransactionWorkflow() throws Exception { try (IdleCheckExecutor executor = new IdleCheckExecutor()) { TransactionManager transactionManager = TransactionManager.create(new TransactionManagerConfig(), executor.getExecutor(), finishingExecutor); Connector c1 = new TpchConnectorFactory(new InMemoryNodeManager()).create("c1", ImmutableMap.of()); transactionManager.addConnector("c1", c1); TransactionId transactionId = transactionManager.beginTransaction(false); Assert.assertEquals(transactionManager.getAllTransactionInfos().size(), 1); TransactionInfo transactionInfo = transactionManager.getTransactionInfo(transactionId); Assert.assertFalse(transactionInfo.isAutoCommitContext()); Assert.assertTrue(transactionInfo.getConnectorIds().isEmpty()); Assert.assertFalse(transactionInfo.getWrittenConnectorId().isPresent()); ConnectorMetadata metadata = transactionManager.getMetadata(transactionId, "c1"); metadata.listSchemaNames(TEST_SESSION.toConnectorSession("c1")); transactionInfo = transactionManager.getTransactionInfo(transactionId); Assert.assertEquals(transactionInfo.getConnectorIds(), ImmutableList.of("c1")); Assert.assertFalse(transactionInfo.getWrittenConnectorId().isPresent()); transactionManager.asyncCommit(transactionId).join(); Assert.assertTrue(transactionManager.getAllTransactionInfos().isEmpty()); } }
@Test public void testAbortedTransactionWorkflow() throws Exception { try (IdleCheckExecutor executor = new IdleCheckExecutor()) { TransactionManager transactionManager = TransactionManager.create(new TransactionManagerConfig(), executor.getExecutor(), finishingExecutor); Connector c1 = new TpchConnectorFactory(new InMemoryNodeManager()).create("c1", ImmutableMap.of()); transactionManager.addConnector("c1", c1); TransactionId transactionId = transactionManager.beginTransaction(false); Assert.assertEquals(transactionManager.getAllTransactionInfos().size(), 1); TransactionInfo transactionInfo = transactionManager.getTransactionInfo(transactionId); Assert.assertFalse(transactionInfo.isAutoCommitContext()); Assert.assertTrue(transactionInfo.getConnectorIds().isEmpty()); Assert.assertFalse(transactionInfo.getWrittenConnectorId().isPresent()); ConnectorMetadata metadata = transactionManager.getMetadata(transactionId, "c1"); metadata.listSchemaNames(TEST_SESSION.toConnectorSession("c1")); transactionInfo = transactionManager.getTransactionInfo(transactionId); Assert.assertEquals(transactionInfo.getConnectorIds(), ImmutableList.of("c1")); Assert.assertFalse(transactionInfo.getWrittenConnectorId().isPresent()); transactionManager.asyncAbort(transactionId).join(); Assert.assertTrue(transactionManager.getAllTransactionInfos().isEmpty()); } }
@Override public Connector create(String connectorId, Map<String, String> properties) { int splitsPerNode = getSplitsPerNode(properties); return new Connector() { @Override public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) { return TpchTransactionHandle.INSTANCE; } @Override public ConnectorMetadata getMetadata(ConnectorTransactionHandle transaction) { return new TpchMetadata(connectorId); } @Override public ConnectorSplitManager getSplitManager() { return new TpchSplitManager(connectorId, nodeManager, splitsPerNode); } @Override public ConnectorRecordSetProvider getRecordSetProvider() { return new TpchRecordSetProvider(); } }; }
@Override public Connector create(String connectorId, Map<String, String> properties) { requireNonNull(properties, "properties is null"); int splitsPerNode = getSplitsPerNode(properties); return new Connector() { @Override public ConnectorTransactionHandle beginTransaction(IsolationLevel isolationLevel, boolean readOnly) { return TpchTransactionHandle.INSTANCE; } @Override public ConnectorMetadata getMetadata(ConnectorTransactionHandle transaction) { return new SampledTpchMetadata(connectorId); } @Override public ConnectorSplitManager getSplitManager() { return new TpchSplitManager(connectorId, nodeManager, splitsPerNode); } @Override public ConnectorRecordSetProvider getRecordSetProvider() { return new SampledTpchRecordSetProvider(connectorId, sampleWeight); } }; }
@Parameters({ "kinesis.awsAccessKey", "kinesis.awsSecretKey" }) @Test public void testSpinUp(String awsAccessKey, String awsSecretKey) { ConnectorFactory factory = testConnectorExists(); // Important: this has to be created before we setup the injector in the factory: assertNotNull(factory.getHandleResolver()); Connector c = factory.create("kinesis.test-connector", ImmutableMap.<String, String>builder() .put("kinesis.hide-internal-columns", "false") .put("kinesis.access-key", TestUtils.noneToBlank(awsAccessKey)) .put("kinesis.secret-key", TestUtils.noneToBlank(awsSecretKey)) .build(), new TestingConnectorContext() {}); assertNotNull(c); // Verify that the key objects have been created on the connector assertNotNull(c.getRecordSetProvider()); assertNotNull(c.getSplitManager()); ConnectorMetadata md = c.getMetadata(KinesisTransactionHandle.INSTANCE); assertNotNull(md); ConnectorTransactionHandle handle = c.beginTransaction(READ_COMMITTED, true); assertTrue(handle != null && handle instanceof KinesisTransactionHandle); }
@Override public ConnectorMetadata getMetadata(ConnectorTransactionHandle transaction) { return new RestMetadata(rest); }