@Test public void testSingleStripeDropDeletes() throws Exception { Configuration conf = HBaseConfiguration.create(); StripeCompactionPolicy policy = createPolicy(conf); // Verify the deletes can be dropped if there are no L0 files. Long[][] stripes = new Long[][] { new Long[] { 3L, 2L, 2L, 2L }, new Long[] { 6L } }; StripeInformationProvider si = createStripesWithSizes(0, 0, stripes); verifySingleStripeCompaction(policy, si, 0, true); // But cannot be dropped if there are. si = createStripesWithSizes(2, 2, stripes); verifySingleStripeCompaction(policy, si, 0, false); // Unless there are enough to cause L0 compaction. si = createStripesWithSizes(6, 2, stripes); ConcatenatedLists<StoreFile> sfs = new ConcatenatedLists<StoreFile>(); sfs.addSublist(si.getLevel0Files()); sfs.addSublist(si.getStripes().get(0)); verifyCompaction( policy, si, sfs, si.getStartRow(0), si.getEndRow(0), si.getStripeBoundaries()); // If we cannot actually compact all files in some stripe, L0 is chosen. si = createStripesWithSizes(6, 2, new Long[][] { new Long[] { 10L, 1L, 1L, 1L, 1L }, new Long[] { 12L } }); verifyCompaction(policy, si, si.getLevel0Files(), null, null, si.getStripeBoundaries()); }
private byte[] getSplitPointFromAllFiles() throws IOException { ConcatenatedLists<StoreFile> sfs = new ConcatenatedLists<StoreFile>(); sfs.addSublist(state.level0Files); sfs.addAllSublists(state.stripeFiles); if (sfs.isEmpty()) return null; return StoreUtils.getLargestFile(sfs).getFileSplitPoint(this.kvComparator); }
@Test public void testSingleStripeDropDeletes() throws Exception { Configuration conf = HBaseConfiguration.create(); StripeCompactionPolicy policy = createPolicy(conf); // Verify the deletes can be dropped if there are no L0 files. Long[][] stripes = new Long[][] { new Long[] { 3L, 2L, 2L, 2L }, new Long[] { 6L } }; StripeInformationProvider si = createStripesWithSizes(0, 0, stripes); verifySingleStripeCompaction(policy, si, 0, true); // But cannot be dropped if there are. si = createStripesWithSizes(2, 2, stripes); verifySingleStripeCompaction(policy, si, 0, false); // Unless there are enough to cause L0 compaction. si = createStripesWithSizes(6, 2, stripes); ConcatenatedLists<StoreFile> sfs = new ConcatenatedLists<StoreFile>(); sfs.addSublist(si.getLevel0Files()); sfs.addSublist(si.getStripes().get(0)); verifyCompaction( policy, si, sfs, si.getStartRow(0), si.getEndRow(0), si.getStripeBoundaries()); // If we cannot actually compact all files in some stripe, L0 is chosen. si = createStripesWithSizes(6, 2, new Long[][] { new Long[] { 10L, 1L, 1L, 1L, 1L }, new Long[] { 12L } }); verifyCompaction(policy, si, si.getLevel0Files(), null, null, si.getStripeBoundaries()); // even if L0 has no file // if all files of stripe aren't selected, delete must not be dropped. stripes = new Long[][] { new Long[] { 100L, 3L, 2L, 2L, 2L }, new Long[] { 6L } }; si = createStripesWithSizes(0, 0, stripes); List<StoreFile> compact_file = new ArrayList<StoreFile>(); Iterator<StoreFile> iter = si.getStripes().get(0).listIterator(1); while (iter.hasNext()) { compact_file.add(iter.next()); } verifyCompaction(policy, si, compact_file, false, 1, null, si.getStartRow(0), si.getEndRow(0), true); }
@Test public void testSingleStripeDropDeletes() throws Exception { Configuration conf = HBaseConfiguration.create(); // Test depends on this not being set to pass. Default breaks test. TODO: Revisit. conf.unset("hbase.hstore.compaction.min.size"); StripeCompactionPolicy policy = createPolicy(conf); // Verify the deletes can be dropped if there are no L0 files. Long[][] stripes = new Long[][] { new Long[] { 3L, 2L, 2L, 2L }, new Long[] { 6L } }; StripeInformationProvider si = createStripesWithSizes(0, 0, stripes); verifySingleStripeCompaction(policy, si, 0, true); // But cannot be dropped if there are. si = createStripesWithSizes(2, 2, stripes); verifySingleStripeCompaction(policy, si, 0, false); // Unless there are enough to cause L0 compaction. si = createStripesWithSizes(6, 2, stripes); ConcatenatedLists<HStoreFile> sfs = new ConcatenatedLists<>(); sfs.addSublist(si.getLevel0Files()); sfs.addSublist(si.getStripes().get(0)); verifyCompaction( policy, si, sfs, si.getStartRow(0), si.getEndRow(0), si.getStripeBoundaries()); // If we cannot actually compact all files in some stripe, L0 is chosen. si = createStripesWithSizes(6, 2, new Long[][] { new Long[] { 10L, 1L, 1L, 1L, 1L }, new Long[] { 12L } }); verifyCompaction(policy, si, si.getLevel0Files(), null, null, si.getStripeBoundaries()); // even if L0 has no file // if all files of stripe aren't selected, delete must not be dropped. stripes = new Long[][] { new Long[] { 100L, 3L, 2L, 2L, 2L }, new Long[] { 6L } }; si = createStripesWithSizes(0, 0, stripes); List<HStoreFile> compactFile = new ArrayList<>(); Iterator<HStoreFile> iter = si.getStripes().get(0).listIterator(1); while (iter.hasNext()) { compactFile.add(iter.next()); } verifyCompaction(policy, si, compactFile, false, 1, null, si.getStartRow(0), si.getEndRow(0), true); }
private StripeCompactionRequest selectExpiredMergeCompaction( StripeInformationProvider si, boolean canDropDeletesNoL0) { long cfTtl = this.storeConfigInfo.getStoreFileTtl(); if (cfTtl == Long.MAX_VALUE) { return null; // minversion might be set, cannot delete old files } long timestampCutoff = EnvironmentEdgeManager.currentTime() - cfTtl; // Merge the longest sequence of stripes where all files have expired, if any. int start = -1, bestStart = -1, length = 0, bestLength = 0; ArrayList<ImmutableList<StoreFile>> stripes = si.getStripes(); OUTER: for (int i = 0; i < stripes.size(); ++i) { for (StoreFile storeFile : stripes.get(i)) { if (storeFile.getReader().getMaxTimestamp() < timestampCutoff) continue; // Found non-expired file, this stripe has to stay. if (length > bestLength) { bestStart = start; bestLength = length; } start = -1; length = 0; continue OUTER; } if (start == -1) { start = i; } ++length; } if (length > bestLength) { bestStart = start; bestLength = length; } if (bestLength == 0) return null; if (bestLength == 1) { // This is currently inefficient. If only one stripe expired, we will rewrite some // entire stripe just to delete some expired files because we rely on metadata and it // cannot simply be updated in an old file. When we either determine stripe dynamically // or move metadata to manifest, we can just drop the "expired stripes". if (bestStart == (stripes.size() - 1)) return null; ++bestLength; } LOG.debug("Merging " + bestLength + " stripes to delete expired store files"); int endIndex = bestStart + bestLength - 1; ConcatenatedLists<StoreFile> sfs = new ConcatenatedLists<StoreFile>(); sfs.addAllSublists(stripes.subList(bestStart, endIndex + 1)); SplitStripeCompactionRequest result = new SplitStripeCompactionRequest(sfs, si.getStartRow(bestStart), si.getEndRow(endIndex), 1, Long.MAX_VALUE); if (canDropDeletesNoL0) { result.setMajorRangeFull(); } return result; }
private StripeCompactionRequest selectExpiredMergeCompaction( StripeInformationProvider si, boolean canDropDeletesNoL0) { long cfTtl = this.storeConfigInfo.getStoreFileTtl(); if (cfTtl == Long.MAX_VALUE) { return null; // minversion might be set, cannot delete old files } long timestampCutoff = EnvironmentEdgeManager.currentTimeMillis() - cfTtl; // Merge the longest sequence of stripes where all files have expired, if any. int start = -1, bestStart = -1, length = 0, bestLength = 0; ArrayList<ImmutableList<StoreFile>> stripes = si.getStripes(); OUTER: for (int i = 0; i < stripes.size(); ++i) { for (StoreFile storeFile : stripes.get(i)) { if (storeFile.getReader().getMaxTimestamp() < timestampCutoff) continue; // Found non-expired file, this stripe has to stay. if (length > bestLength) { bestStart = start; bestLength = length; } start = -1; length = 0; continue OUTER; } if (start == -1) { start = i; } ++length; } if (length > bestLength) { bestStart = start; bestLength = length; } if (bestLength == 0) return null; if (bestLength == 1) { // This is currently inefficient. If only one stripe expired, we will rewrite some // entire stripe just to delete some expired files because we rely on metadata and it // cannot simply be updated in an old file. When we either determine stripe dynamically // or move metadata to manifest, we can just drop the "expired stripes". if (bestStart == (stripes.size() - 1)) return null; ++bestLength; } LOG.debug("Merging " + bestLength + " stripes to delete expired store files"); int endIndex = bestStart + bestLength - 1; ConcatenatedLists<StoreFile> sfs = new ConcatenatedLists<StoreFile>(); sfs.addAllSublists(stripes.subList(bestStart, endIndex + 1)); SplitStripeCompactionRequest result = new SplitStripeCompactionRequest(sfs, si.getStartRow(bestStart), si.getEndRow(endIndex), 1, Long.MAX_VALUE); if (canDropDeletesNoL0) { result.setMajorRangeFull(); } return result; }
private StripeCompactionRequest selectExpiredMergeCompaction( StripeInformationProvider si, boolean canDropDeletesNoL0) { long cfTtl = this.storeConfigInfo.getStoreFileTtl(); if (cfTtl == Long.MAX_VALUE) { return null; // minversion might be set, cannot delete old files } long timestampCutoff = EnvironmentEdgeManager.currentTime() - cfTtl; // Merge the longest sequence of stripes where all files have expired, if any. int start = -1, bestStart = -1, length = 0, bestLength = 0; ArrayList<ImmutableList<HStoreFile>> stripes = si.getStripes(); OUTER: for (int i = 0; i < stripes.size(); ++i) { for (HStoreFile storeFile : stripes.get(i)) { if (storeFile.getReader().getMaxTimestamp() < timestampCutoff) continue; // Found non-expired file, this stripe has to stay. if (length > bestLength) { bestStart = start; bestLength = length; } start = -1; length = 0; continue OUTER; } if (start == -1) { start = i; } ++length; } if (length > bestLength) { bestStart = start; bestLength = length; } if (bestLength == 0) return null; if (bestLength == 1) { // This is currently inefficient. If only one stripe expired, we will rewrite some // entire stripe just to delete some expired files because we rely on metadata and it // cannot simply be updated in an old file. When we either determine stripe dynamically // or move metadata to manifest, we can just drop the "expired stripes". if (bestStart == (stripes.size() - 1)) return null; ++bestLength; } LOG.debug("Merging " + bestLength + " stripes to delete expired store files"); int endIndex = bestStart + bestLength - 1; ConcatenatedLists<HStoreFile> sfs = new ConcatenatedLists<>(); sfs.addAllSublists(stripes.subList(bestStart, endIndex + 1)); SplitStripeCompactionRequest result = new SplitStripeCompactionRequest(sfs, si.getStartRow(bestStart), si.getEndRow(endIndex), 1, Long.MAX_VALUE); if (canDropDeletesNoL0) { result.setMajorRangeFull(); } return result; }
private Optional<byte[]> getSplitPointFromAllFiles() throws IOException { ConcatenatedLists<HStoreFile> sfs = new ConcatenatedLists<>(); sfs.addSublist(state.level0Files); sfs.addAllSublists(state.stripeFiles); return StoreUtils.getSplitPoint(sfs, cellComparator); }
