@Test public void replicaInSourceMetastore() throws Exception { helper.createUnpartitionedTable(toUri(sourceWarehouseUri, DATABASE, SOURCE_UNPARTITIONED_TABLE)); LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_UNPARTITIONED_TABLE)); exit.expectSystemExitWithStatus(0); File config = dataFolder.getFile("unpartitioned-single-table-same-metastore.yml"); CircusTrainRunner runner = CircusTrainRunner .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation) .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(), sourceCatalog.driverClassName()) .replicaMetaStore(sourceCatalog.getThriftConnectionUri()) // Override only this value .build(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { Table hiveTable = sourceCatalog.client().getTable(DATABASE, TARGET_UNPARTITIONED_TABLE); assertThat(hiveTable.getDbName(), is(DATABASE)); assertThat(hiveTable.getTableName(), is(TARGET_UNPARTITIONED_TABLE)); assertThat(isExternalTable(hiveTable), is(true)); assertThat(hiveTable.getSd().getCols(), is(DATA_COLUMNS)); } }); runner.run(config.getAbsolutePath()); }
protected void assertCommandLine(int exitCode, Assertion assertion, boolean includeConf, String... tokens) throws Exception { String[] commandLine = commandLine(includeConf, tokens); printCWD(); System.out.println(">>> " + Constants.command() + StringUtils.arrayToDelimitedString(commandLine, " ")); System.out.println(""); systemOutRule.clearLog(); exit.expectSystemExitWithStatus(exitCode); exit.checkAssertionAfterwards(assertion); try { Main.main(commandLine); } catch (Throwable t) { Chalk.setColorEnabled(true); throw t; } }
@Test public void testNonExistentConfigFile() throws Exception { exit.expectSystemExitWithStatus(-1); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { assertEquals("Unexpected output log", "Could not find config XML file " + "'src/main/resources/non_existent_config.xml'." + EOL, systemOut.getLog()); assertEquals("Unexpected system error log", "", systemErr.getLog()); } }); Main.main("-c", "src/main/resources/non_existent_config.xml", getPath("InputMain.java")); }
@Test public void testNonExistentClass() throws Exception { exit.expectSystemExitWithStatus(-2); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { final String expectedExceptionMessage = errorCounterOneMessage.getMessage() + EOL; assertEquals("Unexpected output log", expectedExceptionMessage, systemOut.getLog()); final String cause = "com.puppycrawl.tools.checkstyle.api.CheckstyleException:" + " cannot initialize module TreeWalker - "; assertTrue("Unexpected system error log", systemErr.getLog().startsWith(cause)); } }); Main.main("-c", getPath("InputMainConfig-non-existent-classname.xml"), getPath("InputMain.java")); }
@Test public void testExistingTargetFileXmlOutput() throws Exception { exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws IOException { final String expectedPath = getFilePath("InputMain.java"); final ResourceBundle compilationProperties = ResourceBundle.getBundle("checkstylecompilation", Locale.ROOT); final String version = compilationProperties .getString("checkstyle.compile.version"); assertEquals("Unexpected output log", "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + EOL + "<checkstyle version=\"" + version + "\">" + EOL + "<file name=\"" + expectedPath + "\">" + EOL + "</file>" + EOL + "</checkstyle>" + EOL, systemOut.getLog()); assertEquals("Unexpected system error log", "", systemErr.getLog()); } }); Main.main("-c", getPath("InputMainConfig-classname.xml"), "-f", "xml", getPath("InputMain.java")); }
@Test public void testExistentTargetFilePlainOutputToNonExistentFile() throws Exception { exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { assertEquals("Unexpected output log", "", systemOut.getLog()); assertEquals("Unexpected system error log", "", systemErr.getLog()); } }); Main.main("-c", getPath("InputMainConfig-classname.xml"), "-f", "plain", "-o", temporaryFolder.getRoot() + "/output.txt", getPath("InputMain.java")); }
@Test public void testExistingTargetFilePlainOutputToFile() throws Exception { final File file = temporaryFolder.newFile("file.output"); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { assertEquals("Unexpected output log", "", systemOut.getLog()); assertEquals("Unexpected system error log", "", systemErr.getLog()); } }); Main.main("-c", getPath("InputMainConfig-classname.xml"), "-f", "plain", "-o", file.getCanonicalPath(), getPath("InputMain.java")); }
@Test public void testExistingTargetFilePlainOutputProperties() throws Exception { mockStatic(Closeables.class); doNothing().when(Closeables.class); Closeables.closeQuietly(any(InputStream.class)); //exit.expectSystemExitWithStatus(0); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { assertEquals("Unexpected output log", auditStartMessage.getMessage() + EOL + auditFinishMessage.getMessage() + EOL, systemOut.getLog()); assertEquals("Unexpected system error log", "", systemErr.getLog()); } }); Main.main("-c", getPath("InputMainConfig-classname-prop.xml"), "-p", getPath("InputMainMycheckstyle.properties"), getPath("InputMain.java")); verifyStatic(times(1)); Closeables.closeQuietly(any(InputStream.class)); }
@Test public void testExistingTargetFilePlainOutputNonexistentProperties() throws Exception { exit.expectSystemExitWithStatus(-1); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { assertEquals("Unexpected output log", "Could not find file 'nonexistent.properties'." + System7.lineSeparator(), systemOut.getLog()); assertEquals("Unexpected system error log", "", systemErr.getLog()); } }); Main.main("-c", getPath("InputMainConfig-classname-prop.xml"), "-p", "nonexistent.properties", getPath("InputMain.java")); }
@Test public void testExistingIncorrectConfigFile() throws Exception { exit.expectSystemExitWithStatus(-2); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { final String output = errorCounterOneMessage.getMessage() + EOL; assertEquals("Unexpected output log", output, systemOut.getLog()); final String errorOutput = "com.puppycrawl.tools.checkstyle.api." + "CheckstyleException: unable to parse configuration stream - "; assertTrue("Unexpected system error log", systemErr.getLog().startsWith(errorOutput)); } }); Main.main("-c", getPath("InputMainConfig-Incorrect.xml"), getPath("InputMain.java")); }
@Test public void testExistingIncorrectChildrenInConfigFile() throws Exception { exit.expectSystemExitWithStatus(-2); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { final String output = errorCounterOneMessage.getMessage() + EOL; assertEquals("Unexpected output log", output, systemOut.getLog()); final String errorOutput = "com.puppycrawl.tools.checkstyle.api." + "CheckstyleException: cannot initialize module RegexpSingleline" + " - RegexpSingleline is not allowed as a child in RegexpSingleline"; assertTrue("Unexpected system error log", systemErr.getLog().startsWith(errorOutput)); } }); Main.main("-c", getPath("InputMainConfig-incorrectChildren.xml"), getPath("InputMain.java")); }
@Test public void testExistingIncorrectChildrenInConfigFile2() throws Exception { exit.expectSystemExitWithStatus(-2); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { final String output = errorCounterOneMessage.getMessage() + EOL; assertEquals("Unexpected output log", output, systemOut.getLog()); final String errorOutput = "com.puppycrawl.tools.checkstyle.api." + "CheckstyleException: cannot initialize module TreeWalker" + " - JavadocVariable is not allowed as a child in JavadocMethod"; assertTrue("Unexpected system error log", systemErr.getLog().startsWith(errorOutput)); } }); Main.main("-c", getPath("InputMainConfig-incorrectChildren2.xml"), getPath("InputMain.java")); }
@Test public void testFileReferenceDuringException() throws Exception { exit.expectSystemExitWithStatus(-2); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { final String expectedExceptionMessage = auditStartMessage.getMessage() + EOL + errorCounterOneMessage.getMessage() + EOL; assertEquals("Unexpected output log", expectedExceptionMessage, systemOut.getLog()); final String exceptionFirstLine = "com.puppycrawl.tools.checkstyle.api." + "CheckstyleException: Exception was thrown while processing " + new File(getNonCompilablePath("InputMainIncorrectClass.java")).getPath() + EOL; assertTrue("Unexpected system error log", systemErr.getLog().startsWith(exceptionFirstLine)); } }); // We put xml as source to cause parse exception Main.main("-c", getPath("InputMainConfig-classname.xml"), getNonCompilablePath("InputMainIncorrectClass.java")); }
@Test public void testPrintTreeJavadocOption() throws Exception { final String expected = new String(Files7.readAllBytes(Paths.get( getPath("InputMainExpectedInputJavadocComment.txt"))), StandardCharsets.UTF_8) .replaceAll("\\\\r\\\\n", "\\\\n"); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { assertEquals("Unexpected output log", expected, systemOut.getLog().replaceAll("\\\\r\\\\n", "\\\\n")); assertEquals("Unexpected system error log", "", systemErr.getLog()); } }); Main.main("-j", getPath("InputMainJavadocComment.javadoc")); }
@Test public void testPrintFullTreeOption() throws Exception { final String expected = new String(Files7.readAllBytes(Paths.get( getPath("InputMainExpectedInputAstTreeStringPrinterJavadoc.txt"))), StandardCharsets.UTF_8).replaceAll("\\\\r\\\\n", "\\\\n"); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { assertEquals("Unexpected output log", expected, systemOut.getLog().replaceAll("\\\\r\\\\n", "\\\\n")); assertEquals("Unexpected system error log", "", systemErr.getLog()); } }); Main.main("-J", getPath("InputMainAstTreeStringPrinterJavadoc.java")); }
@Test public void testConflictingOptionsTvsO() throws Exception { final File file = temporaryFolder.newFile("file.output"); exit.expectSystemExitWithStatus(-1); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { assertEquals("Unexpected output log", "Option '-t' cannot be used with other options." + System7.lineSeparator(), systemOut.getLog()); assertEquals("Unexpected system error log", "", systemErr.getLog()); } }); Main.main("-o", file.getCanonicalPath(), "-t", getPath("")); }
@Test public void testCustomRootModule() throws Exception { TestRootModuleChecker.reset(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { assertEquals("Unexpected output log", "", systemOut.getLog()); assertEquals("Unexpected system error log", "", systemErr.getLog()); assertTrue("Invalid Checker state", TestRootModuleChecker.isProcessed()); } }); Main.main("-c", getPath("InputMainConfig-custom-root-module.xml"), getPath("InputMain.java")); assertTrue("RootModule should be destroyed", TestRootModuleChecker.isDestroyed()); }
@Test public void testExecuteIgnoredModule() throws Exception { exit.expectSystemExitWithStatus(-2); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { final String expectedExceptionMessage = errorCounterOneMessage.getMessage() + EOL; assertEquals("Unexpected output log", expectedExceptionMessage, systemOut.getLog()); final String cause = "com.puppycrawl.tools.checkstyle.api.CheckstyleException:" + " cannot initialize module TreeWalker - "; assertTrue("Unexpected system error log", systemErr.getLog().startsWith(cause)); } }); Main.main("-c", getPath("InputMainConfig-non-existent-classname-ignore.xml"), "-executeIgnoredModules", getPath("InputMain.java")); }
@Test public void testCheckerThreadsNumber() throws Exception { TestRootModuleChecker.reset(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { assertEquals("Unexpected output log", "", systemOut.getLog()); assertEquals("Unexpected system error log", "", systemErr.getLog()); assertTrue("Invalid checker state", TestRootModuleChecker.isProcessed()); final DefaultConfiguration config = (DefaultConfiguration) TestRootModuleChecker.getConfig(); final ThreadModeSettings multiThreadModeSettings = config.getThreadModeSettings(); assertEquals("Invalid checker thread number", 4, multiThreadModeSettings.getCheckerThreadsNumber()); assertEquals("Invalid checker thread number", 1, multiThreadModeSettings.getTreeWalkerThreadsNumber()); } }); Main.main("-C", "4", "-c", getPath("InputMainConfig-custom-root-module.xml"), getPath("InputMain.java")); }
@Test public void testTreeWalkerThreadsNumber() throws Exception { TestRootModuleChecker.reset(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() { assertEquals("Unexpected output log", "", systemOut.getLog()); assertEquals("Unexpected system error log", "", systemErr.getLog()); assertTrue("Invalid checker state", TestRootModuleChecker.isProcessed()); final DefaultConfiguration config = (DefaultConfiguration) TestRootModuleChecker.getConfig(); final ThreadModeSettings multiThreadModeSettings = config.getThreadModeSettings(); assertEquals("Invalid checker thread number", 1, multiThreadModeSettings.getCheckerThreadsNumber()); assertEquals("Invalid checker thread number", 4, multiThreadModeSettings.getTreeWalkerThreadsNumber()); } }); Main.main("-W", "4", "-c", getPath("InputMainConfig-custom-root-module.xml"), getPath("InputMain.java")); }
@Test @Ignore("Bug report at https://www.pivotaltracker.com/story/show/107563624") public void verifyBDocWithWarning() throws IOException { exit.expectSystemExitWithStatus(0); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { assertThat(sout.getLog(), containsString("The signer's certificate is not supported by SSCD!")); } }); String[] params = new String[]{"-in", "testFiles/invalid-containers/warning.asice", "-verify", "-warnings"}; copyFile(new File("testFiles/yaml-configurations/digidoc4j_ForBDocWarningTest.yaml"), new File("digidoc4j.yaml")); DigiDoc4J.main(params); }
@Test public void testHooks() throws Exception { exit.expectSystemExit(); exit.checkAssertionAfterwards(new Assertion() { public void checkAssertion() { assertEquals("", out.getLog()); } }); int worklogCounter = 0; bootstrap(); up(); assertChangelogIntact(); assertWorklogRowCount(worklogCounter += 3); pending(); assertWorklogRowCount(++worklogCounter); down(); assertWorklogRowCount(++worklogCounter); versionDown(); assertWorklogRowCount(++worklogCounter); versionUp(); assertWorklogRowCount(++worklogCounter); out.clearLog(); System.exit(0); }
@Test public void testIgnoreMissingClassesByRegExCouldNotLoad() throws IOException, CannotCompileException { exit.expectSystemExitWithStatus(1); exit.checkAssertionAfterwards(new Assertion() { public void checkAssertion() { String errLogTrimmed = errLog.getLog().trim(); assertThat(errLogTrimmed, containsString("E: Could not load 'NotExistingSuperclass'".trim())); } }); ClassPool cp = new ClassPool(true); CtClass ctClassSuperclass = CtClassBuilder.create().name("NotExistingSuperclass").addToClassPool(cp); CtConstructorBuilder.create().addToClass(ctClassSuperclass); CtClass ctClass = CtClassBuilder.create().name("Test").withSuperclass(ctClassSuperclass).addToClassPool(cp); Path oldPath = Paths.get(System.getProperty("user.dir"), "target", JApiCmpTest.class.getSimpleName() + "_old.jar"); createJarFile(oldPath, ctClass); Path newPath = Paths.get(System.getProperty("user.dir"), "target", JApiCmpTest.class.getSimpleName() + "_new.jar"); createJarFile(newPath, ctClass); JApiCmp.main(new String[]{"-n", newPath.toString(), "-o", oldPath.toString()}); }
@Test public void testIgnoreMissingClassesByRegExMissingAreIgnore() throws IOException, CannotCompileException { exit.checkAssertionAfterwards(new Assertion() { public void checkAssertion() { String outLog = JApiCmpTest.this.outLog.getLog().trim(); assertThat(outLog, containsString("Comparing".trim())); assertThat(outLog, containsString("WARNING: You have ignored certain classes".trim())); } }); ClassPool cp = new ClassPool(true); CtClass ctClassSuperclass = CtClassBuilder.create().name("NotExistingSuperclass").addToClassPool(cp); CtConstructorBuilder.create().addToClass(ctClassSuperclass); CtClass ctClass = CtClassBuilder.create().name("Test").withSuperclass(ctClassSuperclass).addToClassPool(cp); Path oldPath = Paths.get(System.getProperty("user.dir"), "target", JApiCmpTest.class.getSimpleName() + "_old.jar"); createJarFile(oldPath, ctClass); Path newPath = Paths.get(System.getProperty("user.dir"), "target", JApiCmpTest.class.getSimpleName() + "_new.jar"); createJarFile(newPath, ctClass); JApiCmp.main(new String[]{"-n", newPath.toString(), "-o", oldPath.toString(), CliParser.IGNORE_MISSING_CLASSES_BY_REGEX, ".*Superc.*"}); }
@Test public void testInvalidEnvProxySettings() { exit.expectSystemExitWithStatus(1); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { assertThat(System.getProperty("http.proxyHost")).isEqualTo("localhost"); assertThat(System.getProperty("http.proxyPort")).isEqualTo("80"); assertThat(System.getProperty("https.proxyHost")).isNull(); assertThat(System.getProperty("https.proxyPort")).isNull(); assertThat(systemErrRule.getLog()) .isEqualToIgnoringCase( "invalid https proxy configuration: no protocol: 10.0.0.1:8443\n"); } }); Denominator.DenominatorCommand.setProtocolProxyFromEnv("http", "http://localhost"); Denominator.DenominatorCommand.setProtocolProxyFromEnv("https", "10.0.0.1:8443"); }
@Test public void singleYmlFile() throws Exception { exit.expectSystemExitWithStatus(0); File ymlFile = temp.newFile("test-application.yml"); List<String> lines = ImmutableList .<String> builder() .add("source-catalog:") .add(" name: source") .add(" configuration-properties:") .add(" " + ConfVars.METASTOREURIS.varname + ": " + hive.getThriftConnectionUri()) .add("replica-catalog:") .add(" name: replica") .add(" hive-metastore-uris: " + hive.getThriftConnectionUri()) .add("table-replications:") .add(" -") .add(" source-table:") .add(" database-name: " + DATABASE) .add(" table-name: source_" + TABLE) .add(" replica-table:") .add(" table-name: replica_" + TABLE) .add(" table-location: " + temp.newFolder("replica")) .build(); Files.asCharSink(ymlFile, UTF_8).writeLines(lines); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { assertTrue(hive.client().tableExists(DATABASE, "replica_" + TABLE)); } }); CircusTrain.main(new String[] { "--config=" + ymlFile.getAbsolutePath() }); }
@Test public void singleYmlFileWithUserExtension() throws Exception { TestLocomotiveListener.testBean = null; exit.expectSystemExitWithStatus(0); File ymlFile = temp.newFile("test-application.yml"); List<String> lines = ImmutableList .<String> builder() .add("source-catalog:") .add(" name: source") .add(" configuration-properties:") .add(" " + ConfVars.METASTOREURIS.varname + ": " + hive.getThriftConnectionUri()) .add("replica-catalog:") .add(" name: replica") .add(" hive-metastore-uris: " + hive.getThriftConnectionUri()) .add("table-replications:") .add(" -") .add(" source-table:") .add(" database-name: " + DATABASE) .add(" table-name: source_" + TABLE) .add(" replica-table:") .add(" table-name: replica_" + TABLE) .add(" table-location: " + temp.newFolder("replica")) .add("extension-packages: com.hotels.test.extension") .add("testExtensionConfig: foo") .build(); Files.asCharSink(ymlFile, UTF_8).writeLines(lines); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { assertThat(TestLocomotiveListener.testBean.getValue(), is("foo")); } }); CircusTrain.main(new String[] { "--config=" + ymlFile.getAbsolutePath() }); }
@Test public void twoYmlFiles() throws Exception { exit.expectSystemExitWithStatus(0); File ymlFile1 = temp.newFile("test-application1.yml"); File ymlFile2 = temp.newFile("test-application2.yml"); List<String> lines = ImmutableList .<String> builder() .add("source-catalog:") .add(" name: source") .add(" configuration-properties:") .add(" " + ConfVars.METASTOREURIS.varname + ": " + hive.getThriftConnectionUri()) .add("replica-catalog:") .add(" name: replica") .add(" hive-metastore-uris: " + hive.getThriftConnectionUri()) .build(); Files.asCharSink(ymlFile1, UTF_8).writeLines(lines); lines = ImmutableList .<String> builder() .add("table-replications:") .add(" -") .add(" source-table:") .add(" database-name: " + DATABASE) .add(" table-name: source_" + TABLE) .add(" replica-table:") .add(" table-name: replica_" + TABLE) .add(" table-location: " + temp.newFolder("replica")) .build(); Files.asCharSink(ymlFile2, UTF_8).writeLines(lines); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { assertTrue(hive.client().tableExists(DATABASE, "replica_" + TABLE)); } }); CircusTrain.main(new String[] { "--config=" + ymlFile1.getAbsolutePath() + "," + ymlFile2.getAbsolutePath() }); }
@Test public void partitionedTableWithNoPartitionsMirror() throws Exception { final URI sourceTableLocation = toUri("s3a://source/", DATABASE, SOURCE_PARTITIONED_TABLE); TestUtils.createPartitionedTable(sourceCatalog.client(), DATABASE, SOURCE_PARTITIONED_TABLE, sourceTableLocation); exit.expectSystemExitWithStatus(0); File config = dataFolder.getFile("partitioned-single-table-with-no-partitions-mirror.yml"); CircusTrainRunner runner = CircusTrainRunner .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation) .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(), sourceCatalog.driverClassName()) .replicaMetaStore(replicaCatalog.getThriftConnectionUri()) .copierOption(S3S3CopierOptions.Keys.S3_ENDPOINT_URI.keyName(), s3Proxy.getProxyUrl()) .sourceConfigurationProperty(ENDPOINT, s3Proxy.getProxyUrl()) .replicaConfigurationProperty(ENDPOINT, s3Proxy.getProxyUrl()) .replicaConfigurationProperty(ACCESS_KEY, s3Proxy.getAccessKey()) .replicaConfigurationProperty(SECRET_KEY, s3Proxy.getSecretKey()) .build(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { // Assert location Table hiveTable = replicaCatalog.client().getTable(DATABASE, TARGET_PARTITIONED_TABLE); assertThat(hiveTable.getSd().getLocation(), is(sourceTableLocation.toString())); assertThat(hiveTable.getParameters().get(REPLICATION_EVENT.parameterName()), startsWith("ctp-")); assertThat(hiveTable.getSd().getCols(), is(DATA_COLUMNS)); // Assert partitions List<Partition> partitions = replicaCatalog.client().listPartitions(DATABASE, TARGET_PARTITIONED_TABLE, (short) -1); assertThat(partitions.size(), is(0)); } }); runner.run(config.getAbsolutePath()); }
@Test public void unpartitionedTable() throws Exception { final URI sourceTableUri = toUri(sourceWarehouseUri, DATABASE, SOURCE_UNPARTITIONED_TABLE); helper.createUnpartitionedTable(sourceTableUri); exit.expectSystemExitWithStatus(0); File config = dataFolder.getFile("unpartitioned-single-table-hdfs-s3-replication.yml"); CircusTrainRunner runner = CircusTrainRunner .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation) .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(), sourceCatalog.driverClassName()) .replicaMetaStore(replicaCatalog.getThriftConnectionUri()) .copierOption(S3MapReduceCpOptionsParser.S3_ENDPOINT_URI, s3Proxy.getProxyUrl()) .replicaConfigurationProperty(ENDPOINT, s3Proxy.getProxyUrl()) .replicaConfigurationProperty(ACCESS_KEY, s3Proxy.getAccessKey()) .replicaConfigurationProperty(SECRET_KEY, s3Proxy.getSecretKey()) .build(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { // Assert location Table hiveTable = replicaCatalog.client().getTable(DATABASE, TARGET_UNPARTITIONED_TABLE); String eventId = hiveTable.getParameters().get(REPLICATION_EVENT.parameterName()); URI replicaLocation = toUri("s3a://replica/", DATABASE, TARGET_UNPARTITIONED_TABLE + "/" + eventId); assertThat(hiveTable.getSd().getLocation(), is(replicaLocation.toString())); // Assert copied files File dataFile = new File(sourceTableUri.getPath(), PART_00000); String fileKeyRegex = String.format("%s/%s/ctt-\\d{8}t\\d{6}.\\d{3}z-\\w{8}/%s", DATABASE, TARGET_UNPARTITIONED_TABLE, PART_00000); List<S3ObjectSummary> replicaFiles = TestUtils.listObjects(s3Client, "replica"); assertThat(replicaFiles.size(), is(1)); for (S3ObjectSummary objectSummary : replicaFiles) { assertThat(objectSummary.getSize(), is(dataFile.length())); assertThat(objectSummary.getKey().matches(fileKeyRegex), is(true)); } } }); runner.run(config.getAbsolutePath()); }
@Test public void partitionedTableWithNoPartitions() throws Exception { final URI sourceTableUri = toUri(sourceWarehouseUri, DATABASE, SOURCE_PARTITIONED_TABLE); TestUtils.createPartitionedTable(sourceCatalog.client(), DATABASE, SOURCE_PARTITIONED_TABLE, sourceTableUri); exit.expectSystemExitWithStatus(0); File config = dataFolder.getFile("partitioned-single-table-with-no-partitions.yml"); CircusTrainRunner runner = CircusTrainRunner .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation) .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(), sourceCatalog.driverClassName()) .replicaMetaStore(replicaCatalog.getThriftConnectionUri()) .copierOption(S3MapReduceCpOptionsParser.S3_ENDPOINT_URI, s3Proxy.getProxyUrl()) .replicaConfigurationProperty(ENDPOINT, s3Proxy.getProxyUrl()) .replicaConfigurationProperty(ACCESS_KEY, s3Proxy.getAccessKey()) .replicaConfigurationProperty(SECRET_KEY, s3Proxy.getSecretKey()) .build(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { // Assert location Table hiveTable = replicaCatalog.client().getTable(DATABASE, TARGET_PARTITIONED_TABLE); String eventId = hiveTable.getParameters().get(REPLICATION_EVENT.parameterName()); URI replicaLocation = toUri("s3a://replica/", DATABASE, TARGET_PARTITIONED_TABLE); assertThat(hiveTable.getSd().getLocation(), is(replicaLocation.toString())); assertThat(eventId, startsWith("ctp-")); // Assert partitions List<Partition> partitions = replicaCatalog.client().listPartitions(DATABASE, TARGET_PARTITIONED_TABLE, (short) -1); assertThat(partitions.size(), is(0)); // Assert table directory List<S3ObjectSummary> replicaFiles = TestUtils.listObjects(s3Client, "replica"); assertThat(replicaFiles.size(), is(0)); } }); runner.run(config.getAbsolutePath()); }
@Test public void partitionedTableHousekeepingEnabledNoAudit() throws Exception { helper.createPartitionedTable(toUri(sourceWarehouseUri, DATABASE, SOURCE_PARTITIONED_TABLE)); LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_PARTITIONED_TABLE)); exit.expectSystemExitWithStatus(0); File config = dataFolder.getFile("partitioned-single-table-no-housekeeping.yml"); CircusTrainRunner runner = CircusTrainRunner .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation) .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(), sourceCatalog.driverClassName()) .replicaMetaStore(replicaCatalog.getThriftConnectionUri()) .build(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { String jdbcUrl = housekeepingDbJdbcUrl(); try (Connection conn = getConnection(jdbcUrl, HOUSEKEEPING_DB_USER, HOUSEKEEPING_DB_PASSWD)) { List<LegacyReplicaPath> cleanUpPaths = TestUtils.getCleanUpPaths(conn, "SELECT * FROM circus_train.legacy_replica_path"); assertThat(cleanUpPaths.size(), is(0)); try { getCleanUpPaths(conn, "SELECT * FROM circus_train.legacy_replica_path_aud"); } catch (SQLException e) { assertThat(e.getMessage().startsWith("Table \"LEGACY_REPLICA_PATH_AUD\" not found;"), is(true)); } } } }); runner.run(config.getAbsolutePath()); }
@Test public void partitionedTableHousekeepingEnabledWithAudit() throws Exception { helper.createPartitionedTable(toUri(sourceWarehouseUri, DATABASE, SOURCE_PARTITIONED_TABLE)); LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_PARTITIONED_TABLE)); exit.expectSystemExitWithStatus(0); File config = dataFolder.getFile("partitioned-single-table-with-housekeeping.yml"); CircusTrainRunner runner = CircusTrainRunner .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation) .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(), sourceCatalog.driverClassName()) .replicaMetaStore(replicaCatalog.getThriftConnectionUri()) .build(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { // Assert deleted path String jdbcUrl = housekeepingDbJdbcUrl(); try (Connection conn = DriverManager.getConnection(jdbcUrl, HOUSEKEEPING_DB_USER, HOUSEKEEPING_DB_PASSWD)) { List<LegacyReplicaPath> cleanUpPaths = getCleanUpPaths(conn, "SELECT * FROM circus_train.legacy_replica_path"); assertThat(cleanUpPaths.size(), is(0)); List<LegacyReplicaPath> cleanUpPathsAudit = getCleanUpPaths(conn, "SELECT * FROM circus_train.legacy_replica_path_aud"); assertThat(cleanUpPathsAudit.size(), is(1)); assertThat(cleanUpPathsAudit.get(0).getEventId(), is("event-124")); assertThat(cleanUpPathsAudit.get(0).getPathEventId(), is("event-123")); assertThat(cleanUpPathsAudit.get(0).getPath(), is("file:/foo/bar/event-123/deleteme")); } } }); runner.run(config.getAbsolutePath()); }
@Test public void unpartitionedTableHousekeepingEnabledWithAudit() throws Exception { helper.createUnpartitionedTable(toUri(sourceWarehouseUri, DATABASE, SOURCE_UNPARTITIONED_TABLE)); LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_UNPARTITIONED_TABLE)); exit.expectSystemExitWithStatus(0); File config = dataFolder.getFile("unpartitioned-single-table-with-housekeeping.yml"); CircusTrainRunner runner = CircusTrainRunner .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation) .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(), sourceCatalog.driverClassName()) .replicaMetaStore(replicaCatalog.getThriftConnectionUri()) .build(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { // Assert deleted path String jdbcUrl = housekeepingDbJdbcUrl(); try (Connection conn = DriverManager.getConnection(jdbcUrl, HOUSEKEEPING_DB_USER, HOUSEKEEPING_DB_PASSWD)) { List<LegacyReplicaPath> cleanUpPaths = getCleanUpPaths(conn, "SELECT * FROM circus_train.legacy_replica_path"); assertThat(cleanUpPaths.size(), is(0)); List<LegacyReplicaPath> cleanUpPathsAudit = getCleanUpPaths(conn, "SELECT * FROM circus_train.legacy_replica_path_aud"); assertThat(cleanUpPathsAudit.size(), is(1)); assertThat(cleanUpPathsAudit.get(0).getEventId(), is("event-124")); assertThat(cleanUpPathsAudit.get(0).getPathEventId(), is("event-123")); assertThat(cleanUpPathsAudit.get(0).getPath(), is("file:/foo/bar/event-123/deleteme")); } } }); runner.run(config.getAbsolutePath()); }
@Test public void unpartitionedTableHousekeepingEnabledNoAudit() throws Exception { helper.createUnpartitionedTable(toUri(sourceWarehouseUri, DATABASE, SOURCE_UNPARTITIONED_TABLE)); LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_UNPARTITIONED_TABLE)); exit.expectSystemExitWithStatus(0); File config = dataFolder.getFile("unpartitioned-single-table-no-housekeeping.yml"); CircusTrainRunner runner = CircusTrainRunner .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation) .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(), sourceCatalog.driverClassName()) .replicaMetaStore(replicaCatalog.getThriftConnectionUri()) .build(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { // Assert deleted path String jdbcUrl = housekeepingDbJdbcUrl(); try (Connection conn = DriverManager.getConnection(jdbcUrl, HOUSEKEEPING_DB_USER, HOUSEKEEPING_DB_PASSWD)) { List<LegacyReplicaPath> cleanUpPaths = getCleanUpPaths(conn, "SELECT * FROM circus_train.legacy_replica_path"); assertThat(cleanUpPaths.size(), is(0)); try { getCleanUpPaths(conn, "SELECT * FROM circus_train.legacy_replica_path_aud"); } catch (SQLException e) { assertThat(e.getMessage().startsWith("Table \"LEGACY_REPLICA_PATH_AUD\" not found;"), is(true)); } } } }); runner.run(config.getAbsolutePath()); }
@Test public void unpartitionedTableMetadataMirror() throws Exception { helper.createManagedUnpartitionedTable(toUri(sourceWarehouseUri, DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE)); LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE)); // adjusting the sourceTable, mimicking the change we want to update Table sourceTable = sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE); sourceTable.putToParameters("paramToUpdate", "updated"); sourceCatalog.client().alter_table(sourceTable.getDbName(), sourceTable.getTableName(), sourceTable); exit.expectSystemExitWithStatus(0); File config = dataFolder.getFile("unpartitioned-single-table-mirror.yml"); CircusTrainRunner runner = CircusTrainRunner .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation) .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(), sourceCatalog.driverClassName()) .replicaMetaStore(replicaCatalog.getThriftConnectionUri()) .build(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { Table hiveTable = replicaCatalog.client().getTable(DATABASE, TARGET_UNPARTITIONED_MANAGED_TABLE); assertThat(hiveTable.getDbName(), is(DATABASE)); assertThat(hiveTable.getTableName(), is(TARGET_UNPARTITIONED_MANAGED_TABLE)); // MIRRORED table should be set to EXTERNAL assertThat(isExternalTable(hiveTable), is(true)); assertThat(hiveTable.getParameters().get("paramToUpdate"), is("updated")); assertThat(hiveTable.getSd().getCols(), is(DATA_COLUMNS)); File sameAsSourceLocation = new File(sourceWarehouseUri, DATABASE + "/" + SOURCE_MANAGED_UNPARTITIONED_TABLE); assertThat(hiveTable.getSd().getLocation() + "/", is(sameAsSourceLocation.toURI().toString())); } }); runner.run(config.getAbsolutePath()); }
@Test public void partitionedTableUrlEncodedPaths() throws Exception { helper.createTableWithEncodedPartition(toUri(sourceWarehouseUri, DATABASE, SOURCE_ENCODED_TABLE)); LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_ENCODED_TABLE)); exit.expectSystemExitWithStatus(0); File config = dataFolder.getFile("partitioned-single-table-no-housekeeping-url-encoded-paths.yml"); CircusTrainRunner runner = CircusTrainRunner .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation) .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(), sourceCatalog.driverClassName()) .replicaMetaStore(replicaCatalog.getThriftConnectionUri()) .build(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { Table hiveTable = replicaCatalog.client().getTable(DATABASE, SOURCE_ENCODED_TABLE); assertThat(hiveTable.getDbName(), is(DATABASE)); assertThat(hiveTable.getTableName(), is(SOURCE_ENCODED_TABLE)); List<Partition> partitions = replicaCatalog.client().listPartitions(DATABASE, SOURCE_ENCODED_TABLE, (short) 10); assertThat(partitions.size(), is(1)); assertThat(partitions.get(0).getSd().getLocation(), endsWith("continent=Europe/country=U%25K")); assertThat(partitions.get(0).getSd().getLocation(), startsWith(replicaWarehouseUri.toURI().toString())); Path copiedPartition = new Path(partitions.get(0).getSd().getLocation()); assertTrue(FileSystem.get(replicaCatalog.conf()).exists(copiedPartition)); } }); runner.run(config.getAbsolutePath()); }
@Test public void partitionedTableHousekeepingEnabledNoAuditPartialReplication() throws Exception { helper.createPartitionedTable(toUri(sourceWarehouseUri, DATABASE, SOURCE_PARTITIONED_TABLE)); LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_PARTITIONED_TABLE)); exit.expectSystemExitWithStatus(-2); File config = dataFolder.getFile("partitioned-single-table-no-housekeeping-partial-replication.yml"); CircusTrainRunner runner = CircusTrainRunner .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation) .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(), sourceCatalog.driverClassName()) .replicaMetaStore(replicaCatalog.getThriftConnectionUri()) .build(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { String jdbcUrl = housekeepingDbJdbcUrl(); try (Connection conn = getConnection(jdbcUrl, HOUSEKEEPING_DB_USER, HOUSEKEEPING_DB_PASSWD)) { List<LegacyReplicaPath> cleanUpPaths = TestUtils.getCleanUpPaths(conn, "SELECT * FROM circus_train.legacy_replica_path"); assertThat(cleanUpPaths.size(), is(0)); try { getCleanUpPaths(conn, "SELECT * FROM circus_train.legacy_replica_path_aud"); } catch (SQLException e) { assertThat(e.getMessage().startsWith("Table \"LEGACY_REPLICA_PATH_AUD\" not found;"), is(true)); } } } }); runner.run(config.getAbsolutePath()); }
@Test public void unpartitionedTableReplicateAvroSchema() throws Exception { helper.createManagedUnpartitionedTable(toUri(sourceWarehouseUri, DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE)); LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE)); java.nio.file.Path sourceAvroSchemaPath = Paths.get(sourceWarehouseUri.toString() + "/avro-schema-file.test"); Files.createDirectories(sourceAvroSchemaPath); String avroSchemaBaseUrl = sourceAvroSchemaPath.toString(); Table sourceTable = sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE); sourceTable.putToParameters("avro.schema.url", avroSchemaBaseUrl); sourceCatalog.client().alter_table(sourceTable.getDbName(), sourceTable.getTableName(), sourceTable); exit.expectSystemExitWithStatus(0); File config = dataFolder.getFile("unpartitioned-single-table-avro-schema.yml"); CircusTrainRunner runner = CircusTrainRunner .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation) .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(), sourceCatalog.driverClassName()) .replicaMetaStore(replicaCatalog.getThriftConnectionUri()) .build(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { Table replicaHiveTable = replicaCatalog.client().getTable(DATABASE, TARGET_UNPARTITIONED_MANAGED_TABLE); String expectedReplicaSchemaUrl = replicaWarehouseUri.toURI().toString() + "ct_database/"; String transformedAvroUrl = replicaHiveTable.getParameters().get("avro.schema.url"); assertThat(transformedAvroUrl, startsWith(expectedReplicaSchemaUrl)); } }); runner.run(config.getAbsolutePath()); }
@Test public void unpartitionedTableReplicateAvroSchemaOverride() throws Exception { helper.createManagedUnpartitionedTable(toUri(sourceWarehouseUri, DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE)); LOG.info(">>>> Table {} ", sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE)); java.nio.file.Path sourceAvroSchemaPath = Paths.get(sourceWarehouseUri.toString() + "/avro-schema-file.test"); Files.createDirectories(sourceAvroSchemaPath); String avroSchemaBaseUrl = sourceAvroSchemaPath.toString(); Table sourceTable = sourceCatalog.client().getTable(DATABASE, SOURCE_MANAGED_UNPARTITIONED_TABLE); sourceTable.putToParameters("avro.schema.url", avroSchemaBaseUrl); sourceCatalog.client().alter_table(sourceTable.getDbName(), sourceTable.getTableName(), sourceTable); exit.expectSystemExitWithStatus(0); File config = dataFolder.getFile("unpartitioned-single-table-avro-schema-override.yml"); CircusTrainRunner runner = CircusTrainRunner .builder(DATABASE, sourceWarehouseUri, replicaWarehouseUri, housekeepingDbLocation) .sourceMetaStore(sourceCatalog.getThriftConnectionUri(), sourceCatalog.connectionURL(), sourceCatalog.driverClassName()) .replicaMetaStore(replicaCatalog.getThriftConnectionUri()) .build(); exit.checkAssertionAfterwards(new Assertion() { @Override public void checkAssertion() throws Exception { Table replicaHiveTable = replicaCatalog.client().getTable(DATABASE, TARGET_UNPARTITIONED_MANAGED_TABLE); String expectedReplicaSchemaUrl = replicaWarehouseUri.toURI().toString() + "ct_database-override/"; String transformedAvroUrl = replicaHiveTable.getParameters().get("avro.schema.url"); assertThat(transformedAvroUrl, startsWith(expectedReplicaSchemaUrl)); } }); runner.run(config.getAbsolutePath()); }