/** * Start archiving table for given hfile cleaner * @param tableName table to archive * @param cleaner cleaner to check to make sure change propagated * @return underlying {@link LongTermArchivingHFileCleaner} that is managing archiving * @throws IOException on failure * @throws KeeperException on failure */ private List<BaseHFileCleanerDelegate> turnOnArchiving(String tableName, HFileCleaner cleaner) throws IOException, KeeperException { // turn on hfile retention LOG.debug("----Starting archiving for table:" + tableName); archivingClient.enableHFileBackupAsync(Bytes.toBytes(tableName)); assertTrue("Archving didn't get turned on", archivingClient.getArchivingEnabled(tableName)); // wait for the archiver to get the notification List<BaseHFileCleanerDelegate> cleaners = cleaner.getDelegatesForTesting(); LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0); while (!delegate.archiveTracker.keepHFiles(STRING_TABLE_NAME)) { // spin until propagation - should be fast } return cleaners; }
/** * Spy on the {@link LongTermArchivingHFileCleaner} to ensure we can catch when the cleaner has * seen all the files * @return a {@link CountDownLatch} to wait on that releases when the cleaner has been called at * least the expected number of times. */ private CountDownLatch setupCleanerWatching(LongTermArchivingHFileCleaner cleaner, List<BaseHFileCleanerDelegate> cleaners, final int expected) { // replace the cleaner with one that we can can check BaseHFileCleanerDelegate delegateSpy = Mockito.spy(cleaner); final int[] counter = new int[] { 0 }; final CountDownLatch finished = new CountDownLatch(1); Mockito.doAnswer(new Answer<Iterable<FileStatus>>() { @Override public Iterable<FileStatus> answer(InvocationOnMock invocation) throws Throwable { counter[0]++; LOG.debug(counter[0] + "/ " + expected + ") Wrapping call to getDeletableFiles for files: " + invocation.getArguments()[0]); @SuppressWarnings("unchecked") Iterable<FileStatus> ret = (Iterable<FileStatus>) invocation.callRealMethod(); if (counter[0] >= expected) finished.countDown(); return ret; } }).when(delegateSpy).getDeletableFiles(Mockito.anyListOf(FileStatus.class)); cleaners.set(0, delegateSpy); return finished; }
/** * Spy on the {@link LongTermArchivingHFileCleaner} to ensure we can catch when the cleaner has * seen all the files * @return a {@link CountDownLatch} to wait on that releases when the cleaner has been called at * least the expected number of times. */ private CountDownLatch setupCleanerWatching(LongTermArchivingHFileCleaner cleaner, List<BaseHFileCleanerDelegate> cleaners, final int expected) { // replace the cleaner with one that we can can check BaseHFileCleanerDelegate delegateSpy = Mockito.spy(cleaner); final int[] counter = new int[] { 0 }; final CountDownLatch finished = new CountDownLatch(1); Mockito.doAnswer(new Answer<Iterable<FileStatus>>() { @Override public Iterable<FileStatus> answer(InvocationOnMock invocation) throws Throwable { counter[0]++; LOG.debug(counter[0] + "/ " + expected + ") Wrapping call to getDeletableFiles for files: " + invocation.getArgument(0)); @SuppressWarnings("unchecked") Iterable<FileStatus> ret = (Iterable<FileStatus>) invocation.callRealMethod(); if (counter[0] >= expected) finished.countDown(); return ret; } }).when(delegateSpy).getDeletableFiles(Mockito.anyListOf(FileStatus.class)); cleaners.set(0, delegateSpy); return finished; }
/** * Spy on the {@link LongTermArchivingHFileCleaner} to ensure we can catch when the cleaner has * seen all the files * @return a {@link CountDownLatch} to wait on that releases when the cleaner has been called at * least the expected number of times. */ private CountDownLatch setupCleanerWatching(LongTermArchivingHFileCleaner cleaner, List<BaseHFileCleanerDelegate> cleaners, final int expected) { // replace the cleaner with one that we can can check BaseHFileCleanerDelegate delegateSpy = Mockito.spy(cleaner); final int[] counter = new int[] { 0 }; final CountDownLatch finished = new CountDownLatch(1); Mockito.doAnswer(new Answer<Boolean>() { @Override public Boolean answer(InvocationOnMock invocation) throws Throwable { counter[0]++; LOG.debug(counter[0] + "/ " + expected + ") Wrapping call to isFileDeletable for file: " + invocation.getArguments()[0]); Boolean ret = (Boolean) invocation.callRealMethod(); if (counter[0] >= expected) finished.countDown(); return ret; } }).when(delegateSpy).isFileDeletable(Mockito.any(Path.class)); cleaners.set(0, delegateSpy); return finished; }
@Test (timeout=300000) public void testArchivingOnSingleTable() throws Exception { createArchiveDirectory(); FileSystem fs = UTIL.getTestFileSystem(); Path archiveDir = getArchiveDir(); Path tableDir = getTableDir(STRING_TABLE_NAME); toCleanup.add(archiveDir); toCleanup.add(tableDir); Configuration conf = UTIL.getConfiguration(); // setup the delegate Stoppable stop = new StoppableImplementation(); HFileCleaner cleaner = setupAndCreateCleaner(conf, fs, archiveDir, stop); List<BaseHFileCleanerDelegate> cleaners = turnOnArchiving(STRING_TABLE_NAME, cleaner); final LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0); // create the region HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM); Region region = UTIL.createTestRegion(STRING_TABLE_NAME, hcd); loadFlushAndCompact(region, TEST_FAM); // get the current hfiles in the archive directory List<Path> files = getAllFiles(fs, archiveDir); if (files == null) { FSUtils.logFileSystemState(fs, UTIL.getDataTestDir(), LOG); throw new RuntimeException("Didn't archive any files!"); } CountDownLatch finished = setupCleanerWatching(delegate, cleaners, files.size()); runCleaner(cleaner, finished, stop); // know the cleaner ran, so now check all the files again to make sure they are still there List<Path> archivedFiles = getAllFiles(fs, archiveDir); assertEquals("Archived files changed after running archive cleaner.", files, archivedFiles); // but we still have the archive directory assertTrue(fs.exists(HFileArchiveUtil.getArchivePath(UTIL.getConfiguration()))); }
@Test (timeout=300000) public void testArchivingOnSingleTable() throws Exception { createArchiveDirectory(); FileSystem fs = UTIL.getTestFileSystem(); Path archiveDir = getArchiveDir(); Path tableDir = getTableDir(STRING_TABLE_NAME); toCleanup.add(archiveDir); toCleanup.add(tableDir); Configuration conf = UTIL.getConfiguration(); // setup the delegate Stoppable stop = new StoppableImplementation(); HFileCleaner cleaner = setupAndCreateCleaner(conf, fs, archiveDir, stop); List<BaseHFileCleanerDelegate> cleaners = turnOnArchiving(STRING_TABLE_NAME, cleaner); final LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0); // create the region HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM); HRegion region = UTIL.createTestRegion(STRING_TABLE_NAME, hcd); loadFlushAndCompact(region, TEST_FAM); // get the current hfiles in the archive directory List<Path> files = getAllFiles(fs, archiveDir); if (files == null) { FSUtils.logFileSystemState(fs, UTIL.getDataTestDir(), LOG); throw new RuntimeException("Didn't archive any files!"); } CountDownLatch finished = setupCleanerWatching(delegate, cleaners, files.size()); runCleaner(cleaner, finished, stop); // know the cleaner ran, so now check all the files again to make sure they are still there List<Path> archivedFiles = getAllFiles(fs, archiveDir); assertEquals("Archived files changed after running archive cleaner.", files, archivedFiles); // but we still have the archive directory assertTrue(fs.exists(HFileArchiveUtil.getArchivePath(UTIL.getConfiguration()))); }
@Test public void testArchivingOnSingleTable() throws Exception { createArchiveDirectory(); FileSystem fs = UTIL.getTestFileSystem(); Path archiveDir = getArchiveDir(); Path tableDir = getTableDir(STRING_TABLE_NAME); toCleanup.add(archiveDir); toCleanup.add(tableDir); Configuration conf = UTIL.getConfiguration(); // setup the delegate Stoppable stop = new StoppableImplementation(); HFileCleaner cleaner = setupAndCreateCleaner(conf, fs, archiveDir, stop); List<BaseHFileCleanerDelegate> cleaners = turnOnArchiving(STRING_TABLE_NAME, cleaner); final LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0); // create the region HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM); HRegion region = UTIL.createTestRegion(STRING_TABLE_NAME, hcd); loadFlushAndCompact(region, TEST_FAM); // get the current hfiles in the archive directory List<Path> files = getAllFiles(fs, archiveDir); if (files == null) { FSUtils.logFileSystemState(fs, UTIL.getDataTestDir(), LOG); throw new RuntimeException("Didn't archive any files!"); } CountDownLatch finished = setupCleanerWatching(delegate, cleaners, files.size()); runCleaner(cleaner, finished, stop); // know the cleaner ran, so now check all the files again to make sure they are still there List<Path> archivedFiles = getAllFiles(fs, archiveDir); assertEquals("Archived files changed after running archive cleaner.", files, archivedFiles); // but we still have the archive directory assertTrue(fs.exists(HFileArchiveUtil.getArchivePath(UTIL.getConfiguration()))); }
@Test (timeout=300000) public void testArchivingOnSingleTable() throws Exception { createArchiveDirectory(); FileSystem fs = UTIL.getTestFileSystem(); Path archiveDir = getArchiveDir(); Path tableDir = getTableDir(STRING_TABLE_NAME); toCleanup.add(archiveDir); toCleanup.add(tableDir); Configuration conf = UTIL.getConfiguration(); // setup the delegate Stoppable stop = new StoppableImplementation(); HFileCleaner cleaner = setupAndCreateCleaner(conf, fs, archiveDir, stop); List<BaseHFileCleanerDelegate> cleaners = turnOnArchiving(STRING_TABLE_NAME, cleaner); final LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0); // create the region ColumnFamilyDescriptor hcd = ColumnFamilyDescriptorBuilder.of(TEST_FAM); HRegion region = UTIL.createTestRegion(STRING_TABLE_NAME, hcd); List<HRegion> regions = new ArrayList<>(); regions.add(region); Mockito.doReturn(regions).when(rss).getRegions(); final CompactedHFilesDischarger compactionCleaner = new CompactedHFilesDischarger(100, stop, rss, false); loadFlushAndCompact(region, TEST_FAM); compactionCleaner.chore(); // get the current hfiles in the archive directory List<Path> files = getAllFiles(fs, archiveDir); if (files == null) { FSUtils.logFileSystemState(fs, UTIL.getDataTestDir(), LOG); throw new RuntimeException("Didn't archive any files!"); } CountDownLatch finished = setupCleanerWatching(delegate, cleaners, files.size()); runCleaner(cleaner, finished, stop); // know the cleaner ran, so now check all the files again to make sure they are still there List<Path> archivedFiles = getAllFiles(fs, archiveDir); assertEquals("Archived files changed after running archive cleaner.", files, archivedFiles); // but we still have the archive directory assertTrue(fs.exists(HFileArchiveUtil.getArchivePath(UTIL.getConfiguration()))); }