@Test public void testSortExtract(){ ManualEnvironmentEdge mee = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(mee); mee.setValue(1); DeadServer d = new DeadServer(); d.add(hostname123); mee.incValue(1); d.add(hostname1234); mee.incValue(1); d.add(hostname12345); List<Pair<ServerName, Long>> copy = d.copyDeadServersSince(2L); Assert.assertEquals(2, copy.size()); Assert.assertEquals(hostname1234, copy.get(0).getFirst()); Assert.assertEquals(new Long(2L), copy.get(0).getSecond()); Assert.assertEquals(hostname12345, copy.get(1).getFirst()); Assert.assertEquals(new Long(3L), copy.get(1).getSecond()); EnvironmentEdgeManager.reset(); }
@Test public void testIncrementTimestampsAreMonotonic() throws IOException { HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1); ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(edge); edge.setValue(10); Increment inc = new Increment(row); inc.setDurability(Durability.SKIP_WAL); inc.addColumn(fam1, qual1, 1L); region.increment(inc); Result result = region.get(new Get(row)); Cell c = result.getColumnLatestCell(fam1, qual1); assertNotNull(c); assertEquals(c.getTimestamp(), 10L); edge.setValue(1); // clock goes back region.increment(inc); result = region.get(new Get(row)); c = result.getColumnLatestCell(fam1, qual1); assertEquals(c.getTimestamp(), 10L); assertEquals(Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength()), 2L); }
@BeforeClass public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, REFRESH_TIME); TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10); TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); TEST_UTIL.startMiniCluster(1); TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME); QuotaCache.TEST_FORCE_REFRESH = true; tables = new Table[TABLE_NAMES.length]; for (int i = 0; i < TABLE_NAMES.length; ++i) { tables[i] = TEST_UTIL.createTable(TABLE_NAMES[i], FAMILY); } envEdge = new ManualEnvironmentEdge(); envEdge.setValue(EnvironmentEdgeManager.currentTime()); EnvironmentEdgeManagerTestHelper.injectEdge(envEdge); }
@Test public void testIncrementTimestampsAreMonotonic() throws IOException { HRegion region = initHRegion(tableName, method, CONF, fam1); ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(edge); edge.setValue(10); Increment inc = new Increment(row); inc.setDurability(Durability.SKIP_WAL); inc.addColumn(fam1, qual1, 1L); region.increment(inc); Result result = region.get(new Get(row)); Cell c = result.getColumnLatestCell(fam1, qual1); assertNotNull(c); assertEquals(10L, c.getTimestamp()); edge.setValue(1); // clock goes back region.increment(inc); result = region.get(new Get(row)); c = result.getColumnLatestCell(fam1, qual1); assertEquals(11L, c.getTimestamp()); assertEquals(2L, Bytes.toLong(c.getValueArray(), c.getValueOffset(), c.getValueLength())); }
protected ArrayList<HStoreFile> sfCreate(long[] minTimestamps, long[] maxTimestamps, long[] sizes) throws IOException { ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(timeMachine); // Has to be > 0 and < now. timeMachine.setValue(1); ArrayList<Long> ageInDisk = new ArrayList<>(); for (int i = 0; i < sizes.length; i++) { ageInDisk.add(0L); } ArrayList<HStoreFile> ret = Lists.newArrayList(); for (int i = 0; i < sizes.length; i++) { MockHStoreFile msf = new MockHStoreFile(TEST_UTIL, TEST_FILE, sizes[i], ageInDisk.get(i), false, i); msf.setTimeRangeTracker(TimeRangeTracker.create(TimeRangeTracker.Type.SYNC, minTimestamps[i], maxTimestamps[i])); ret.add(msf); } return ret; }
protected void compactEquals(long now, ArrayList<HStoreFile> candidates, long[] expectedFileSizes, long[] expectedBoundaries, boolean isMajor, boolean toCompact) throws IOException { ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(timeMachine); timeMachine.setValue(now); DateTieredCompactionRequest request; DateTieredCompactionPolicy policy = (DateTieredCompactionPolicy) store.storeEngine.getCompactionPolicy(); if (isMajor) { for (HStoreFile file : candidates) { ((MockHStoreFile) file).setIsMajor(true); } assertEquals(toCompact, policy.shouldPerformMajorCompaction(candidates)); request = (DateTieredCompactionRequest) policy.selectMajorCompaction(candidates); } else { assertEquals(toCompact, policy.needsCompaction(candidates, ImmutableList.of())); request = (DateTieredCompactionRequest) policy.selectMinorCompaction(candidates, false, false); } List<HStoreFile> actual = Lists.newArrayList(request.getFiles()); assertEquals(Arrays.toString(expectedFileSizes), Arrays.toString(getSizes(actual))); assertEquals(Arrays.toString(expectedBoundaries), Arrays.toString(request.getBoundaries().toArray())); }
@Test public void testAppendTimestampsAreMonotonic() throws IOException { HRegion region = initHRegion(tableName, name.getMethodName(), CONF, fam1); ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(edge); edge.setValue(10); Append a = new Append(row); a.setDurability(Durability.SKIP_WAL); a.add(fam1, qual1, qual1); region.append(a); Result result = region.get(new Get(row)); Cell c = result.getColumnLatestCell(fam1, qual1); assertNotNull(c); assertEquals(c.getTimestamp(), 10L); edge.setValue(1); // clock goes back region.append(a); result = region.get(new Get(row)); c = result.getColumnLatestCell(fam1, qual1); assertEquals(c.getTimestamp(), 10L); byte[] expected = new byte[qual1.length*2]; System.arraycopy(qual1, 0, expected, 0, qual1.length); System.arraycopy(qual1, 0, expected, qual1.length, qual1.length); assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), expected, 0, expected.length)); }
@SuppressWarnings("unchecked") @Test public void testMergeExpiredFiles() throws Exception { ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); long now = defaultTtl + 2; edge.setValue(now); EnvironmentEdgeManager.injectEdge(edge); try { StoreFile expiredFile = createFile(), notExpiredFile = createFile(); when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1); when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1); List<StoreFile> expired = Lists.newArrayList(expiredFile, expiredFile); List<StoreFile> notExpired = Lists.newArrayList(notExpiredFile, notExpiredFile); List<StoreFile> mixed = Lists.newArrayList(expiredFile, notExpiredFile); StripeCompactionPolicy policy = createPolicy(HBaseConfiguration.create(), defaultSplitSize, defaultSplitCount, defaultInitialCount, true); // Merge expired if there are eligible stripes. StripeCompactionPolicy.StripeInformationProvider si = createStripesWithFiles(expired, expired, expired); verifyWholeStripesCompaction(policy, si, 0, 2, null, 1, Long.MAX_VALUE, false); // Don't merge if nothing expired. si = createStripesWithFiles(notExpired, notExpired, notExpired); assertNull(policy.selectCompaction(si, al(), false)); // Merge one expired stripe with next. si = createStripesWithFiles(notExpired, expired, notExpired); verifyWholeStripesCompaction(policy, si, 1, 2, null, 1, Long.MAX_VALUE, false); // Merge the biggest run out of multiple options. // Merge one expired stripe with next. si = createStripesWithFiles(notExpired, expired, notExpired, expired, expired, notExpired); verifyWholeStripesCompaction(policy, si, 3, 4, null, 1, Long.MAX_VALUE, false); // Stripe with a subset of expired files is not merged. si = createStripesWithFiles(expired, expired, notExpired, expired, mixed); verifyWholeStripesCompaction(policy, si, 0, 1, null, 1, Long.MAX_VALUE, false); } finally { EnvironmentEdgeManager.reset(); } }
@SuppressWarnings("unchecked") @Test public void testMergeExpiredStripes() throws Exception { // HBASE-11397 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); long now = defaultTtl + 2; edge.setValue(now); EnvironmentEdgeManager.injectEdge(edge); try { StoreFile expiredFile = createFile(), notExpiredFile = createFile(); when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1); when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1); List<StoreFile> expired = Lists.newArrayList(expiredFile, expiredFile); List<StoreFile> notExpired = Lists.newArrayList(notExpiredFile, notExpiredFile); StripeCompactionPolicy policy = createPolicy(HBaseConfiguration.create(), defaultSplitSize, defaultSplitCount, defaultInitialCount, true); // Merge all three expired stripes into one. StripeCompactionPolicy.StripeInformationProvider si = createStripesWithFiles(expired, expired, expired); verifyMergeCompatcion(policy, si, 0, 2); // Merge two adjacent expired stripes into one. si = createStripesWithFiles(notExpired, expired, notExpired, expired, expired, notExpired); verifyMergeCompatcion(policy, si, 3, 4); } finally { EnvironmentEdgeManager.reset(); } }
@Test public void testCleanup() throws Exception { ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(edge); try { ServerNonceManager nm = createManager(6); ScheduledChore cleanup = nm.createCleanupScheduledChore(Mockito.mock(Stoppable.class)); edge.setValue(1); assertTrue(nm.startOperation(NO_NONCE, 1, createStoppable())); assertTrue(nm.startOperation(NO_NONCE, 2, createStoppable())); assertTrue(nm.startOperation(NO_NONCE, 3, createStoppable())); edge.setValue(2); nm.endOperation(NO_NONCE, 1, true); edge.setValue(4); nm.endOperation(NO_NONCE, 2, true); edge.setValue(9); cleanup.choreForTesting(); // Nonce 1 has been cleaned up. assertTrue(nm.startOperation(NO_NONCE, 1, createStoppable())); // Nonce 2 has not been cleaned up. assertFalse(nm.startOperation(NO_NONCE, 2, createStoppable())); // Nonce 3 was active and active ops should never be cleaned up; try to end and start. nm.endOperation(NO_NONCE, 3, false); assertTrue(nm.startOperation(NO_NONCE, 3, createStoppable())); edge.setValue(11); cleanup.choreForTesting(); // Now, nonce 2 has been cleaned up. assertTrue(nm.startOperation(NO_NONCE, 2, createStoppable())); } finally { EnvironmentEdgeManager.reset(); } }
@Test public void testWalNonces() throws Exception { ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(edge); try { ServerNonceManager nm = createManager(6); ScheduledChore cleanup = nm.createCleanupScheduledChore(Mockito.mock(Stoppable.class)); // Add nonces from WAL, including dups. edge.setValue(12); nm.reportOperationFromWal(NO_NONCE, 1, 8); nm.reportOperationFromWal(NO_NONCE, 2, 2); nm.reportOperationFromWal(NO_NONCE, 3, 5); nm.reportOperationFromWal(NO_NONCE, 3, 6); // WAL nonces should prevent cross-server conflicts. assertFalse(nm.startOperation(NO_NONCE, 1, createStoppable())); // Make sure we ignore very old nonces, but not borderline old nonces. assertTrue(nm.startOperation(NO_NONCE, 2, createStoppable())); assertFalse(nm.startOperation(NO_NONCE, 3, createStoppable())); // Make sure grace period is counted from recovery time. edge.setValue(17); cleanup.choreForTesting(); assertFalse(nm.startOperation(NO_NONCE, 1, createStoppable())); assertFalse(nm.startOperation(NO_NONCE, 3, createStoppable())); edge.setValue(19); cleanup.choreForTesting(); assertTrue(nm.startOperation(NO_NONCE, 1, createStoppable())); assertTrue(nm.startOperation(NO_NONCE, 3, createStoppable())); } finally { EnvironmentEdgeManager.reset(); } }
@Test public void testRenameAndSetModifyTime() throws Exception { assertTrue(FSUtils.isHDFS(conf)); FileSystem fs = FileSystem.get(conf); Path testDir = TEST_UTIL.getDataTestDir("testArchiveFile"); String file = UUID.randomUUID().toString(); Path p = new Path(testDir, file); FSDataOutputStream out = fs.create(p); out.close(); assertTrue("The created file should be present", FSUtils.isExists(fs, p)); long expect = System.currentTimeMillis() + 1000; assertFalse(expect == fs.getFileStatus(p).getModificationTime()); ManualEnvironmentEdge mockEnv = new ManualEnvironmentEdge(); mockEnv.setValue(expect); EnvironmentEdgeManager.injectEdge(mockEnv); String dstFile = UUID.randomUUID().toString(); Path dst = new Path(testDir , dstFile); assertTrue(HBaseFileSystem.renameAndSetModifyTime(fs, p, dst)); assertFalse("The moved file should not be present", FSUtils.isExists(fs, p)); assertTrue("The dst file should be present", FSUtils.isExists(fs, dst)); assertEquals(expect, fs.getFileStatus(dst).getModificationTime()); }
public void testIncrementColumnValue_BumpSnapshot() throws IOException { ManualEnvironmentEdge mee = new ManualEnvironmentEdge(); EnvironmentEdgeManagerTestHelper.injectEdge(mee); this.region = initHRegion(tableName, getName(), conf, fam1); try { long value = 42L; long incr = 44L; // first put something in kvset, then snapshot it. Put put = new Put(row); put.add(fam1, qual1, Bytes.toBytes(value)); region.put(put); // get the store in question: Store s = region.getStore(fam1); s.snapshot(); //bam // now increment: long newVal = region.incrementColumnValue(row, fam1, qual1, incr, false); assertEquals(value+incr, newVal); // get both versions: Get get = new Get(row); get.setMaxVersions(); get.addColumn(fam1,qual1); Result r = region.get(get, null); assertEquals(2, r.size()); KeyValue first = r.raw()[0]; KeyValue second = r.raw()[1]; assertTrue("ICV failed to upgrade timestamp", first.getTimestamp() != second.getTimestamp()); } finally { HRegion.closeHRegion(this.region); this.region = null; } }
@Test public void testCleanup() throws Exception { ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(edge); try { ServerNonceManager nm = createManager(6); Chore cleanup = nm.createCleanupChore(Mockito.mock(Stoppable.class)); edge.setValue(1); assertTrue(nm.startOperation(NO_NONCE, 1, createStoppable())); assertTrue(nm.startOperation(NO_NONCE, 2, createStoppable())); assertTrue(nm.startOperation(NO_NONCE, 3, createStoppable())); edge.setValue(2); nm.endOperation(NO_NONCE, 1, true); edge.setValue(4); nm.endOperation(NO_NONCE, 2, true); edge.setValue(9); cleanup.choreForTesting(); // Nonce 1 has been cleaned up. assertTrue(nm.startOperation(NO_NONCE, 1, createStoppable())); // Nonce 2 has not been cleaned up. assertFalse(nm.startOperation(NO_NONCE, 2, createStoppable())); // Nonce 3 was active and active ops should never be cleaned up; try to end and start. nm.endOperation(NO_NONCE, 3, false); assertTrue(nm.startOperation(NO_NONCE, 3, createStoppable())); edge.setValue(11); cleanup.choreForTesting(); // Now, nonce 2 has been cleaned up. assertTrue(nm.startOperation(NO_NONCE, 2, createStoppable())); } finally { EnvironmentEdgeManager.reset(); } }
@Test public void testWalNonces() throws Exception { ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(edge); try { ServerNonceManager nm = createManager(6); Chore cleanup = nm.createCleanupChore(Mockito.mock(Stoppable.class)); // Add nonces from WAL, including dups. edge.setValue(12); nm.reportOperationFromWal(NO_NONCE, 1, 8); nm.reportOperationFromWal(NO_NONCE, 2, 2); nm.reportOperationFromWal(NO_NONCE, 3, 5); nm.reportOperationFromWal(NO_NONCE, 3, 6); // WAL nonces should prevent cross-server conflicts. assertFalse(nm.startOperation(NO_NONCE, 1, createStoppable())); // Make sure we ignore very old nonces, but not borderline old nonces. assertTrue(nm.startOperation(NO_NONCE, 2, createStoppable())); assertFalse(nm.startOperation(NO_NONCE, 3, createStoppable())); // Make sure grace period is counted from recovery time. edge.setValue(17); cleanup.choreForTesting(); assertFalse(nm.startOperation(NO_NONCE, 1, createStoppable())); assertFalse(nm.startOperation(NO_NONCE, 3, createStoppable())); edge.setValue(19); cleanup.choreForTesting(); assertTrue(nm.startOperation(NO_NONCE, 1, createStoppable())); assertTrue(nm.startOperation(NO_NONCE, 3, createStoppable())); } finally { EnvironmentEdgeManager.reset(); } }
@Test public void testErrorBackoffTimeCalculation() throws Exception { // TODO: This test would seem to presume hardcoded RETRY_BACKOFF which it should not. final long ANY_PAUSE = 100; ServerName location = ServerName.valueOf("127.0.0.1", 1, 0); ServerName diffLocation = ServerName.valueOf("127.0.0.1", 2, 0); ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(timeMachine); try { long largeAmountOfTime = ANY_PAUSE * 1000; ConnectionImplementation.ServerErrorTracker tracker = new ConnectionImplementation.ServerErrorTracker(largeAmountOfTime, 100); // The default backoff is 0. assertEquals(0, tracker.calculateBackoffTime(location, ANY_PAUSE)); // Check some backoff values from HConstants sequence. tracker.reportServerError(location); assertEqualsWithJitter(ANY_PAUSE * HConstants.RETRY_BACKOFF[0], tracker.calculateBackoffTime(location, ANY_PAUSE)); tracker.reportServerError(location); tracker.reportServerError(location); tracker.reportServerError(location); assertEqualsWithJitter(ANY_PAUSE * HConstants.RETRY_BACKOFF[3], tracker.calculateBackoffTime(location, ANY_PAUSE)); // All of this shouldn't affect backoff for different location. assertEquals(0, tracker.calculateBackoffTime(diffLocation, ANY_PAUSE)); tracker.reportServerError(diffLocation); assertEqualsWithJitter(ANY_PAUSE * HConstants.RETRY_BACKOFF[0], tracker.calculateBackoffTime(diffLocation, ANY_PAUSE)); // Check with different base. assertEqualsWithJitter(ANY_PAUSE * 2 * HConstants.RETRY_BACKOFF[3], tracker.calculateBackoffTime(location, ANY_PAUSE * 2)); } finally { EnvironmentEdgeManager.reset(); } }
@Test public void testUnconfiguredLimiters() throws InterruptedException { ManualEnvironmentEdge testEdge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(testEdge); long limit = Long.MAX_VALUE; // For unconfigured limiters, it is supposed to use as much as possible RateLimiter avgLimiter = new AverageIntervalRateLimiter(); RateLimiter fixLimiter = new FixedIntervalRateLimiter(); assertEquals(limit, avgLimiter.getAvailable()); assertEquals(limit, fixLimiter.getAvailable()); assertTrue(avgLimiter.canExecute(limit)); avgLimiter.consume(limit); assertTrue(fixLimiter.canExecute(limit)); fixLimiter.consume(limit); // Make sure that available is Long.MAX_VALUE assertTrue(limit == avgLimiter.getAvailable()); assertTrue(limit == fixLimiter.getAvailable()); // after 100 millseconds, it should be able to execute limit as well testEdge.incValue(100); assertTrue(avgLimiter.canExecute(limit)); avgLimiter.consume(limit); assertTrue(fixLimiter.canExecute(limit)); fixLimiter.consume(limit); // Make sure that available is Long.MAX_VALUE assertTrue(limit == avgLimiter.getAvailable()); assertTrue(limit == fixLimiter.getAvailable()); EnvironmentEdgeManager.reset(); }
@Test public void testAppendTimestampsAreMonotonic() throws IOException { HRegion region = initHRegion(tableName, method, CONF, fam1); ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(edge); edge.setValue(10); Append a = new Append(row); a.setDurability(Durability.SKIP_WAL); a.addColumn(fam1, qual1, qual1); region.append(a); Result result = region.get(new Get(row)); Cell c = result.getColumnLatestCell(fam1, qual1); assertNotNull(c); assertEquals(10L, c.getTimestamp()); edge.setValue(1); // clock goes back region.append(a); result = region.get(new Get(row)); c = result.getColumnLatestCell(fam1, qual1); assertEquals(11L, c.getTimestamp()); byte[] expected = new byte[qual1.length*2]; System.arraycopy(qual1, 0, expected, 0, qual1.length); System.arraycopy(qual1, 0, expected, qual1.length, qual1.length); assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), expected, 0, expected.length)); }
@Test public void testCheckAndMutateTimestampsAreMonotonic() throws IOException { HRegion region = initHRegion(tableName, method, CONF, fam1); ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(edge); edge.setValue(10); Put p = new Put(row); p.setDurability(Durability.SKIP_WAL); p.addColumn(fam1, qual1, qual1); region.put(p); Result result = region.get(new Get(row)); Cell c = result.getColumnLatestCell(fam1, qual1); assertNotNull(c); assertEquals(10L, c.getTimestamp()); edge.setValue(1); // clock goes back p = new Put(row); p.setDurability(Durability.SKIP_WAL); p.addColumn(fam1, qual1, qual2); region.checkAndMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(qual1), p, false); result = region.get(new Get(row)); c = result.getColumnLatestCell(fam1, qual1); assertEquals(10L, c.getTimestamp()); assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0, qual2.length)); }
@Test public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException { HRegion region = initHRegion(tableName, method, CONF, fam1); ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); EnvironmentEdgeManager.injectEdge(edge); edge.setValue(10); Put p = new Put(row); p.setDurability(Durability.SKIP_WAL); p.addColumn(fam1, qual1, qual1); region.put(p); Result result = region.get(new Get(row)); Cell c = result.getColumnLatestCell(fam1, qual1); assertNotNull(c); assertEquals(10L, c.getTimestamp()); edge.setValue(1); // clock goes back p = new Put(row); p.setDurability(Durability.SKIP_WAL); p.addColumn(fam1, qual1, qual2); RowMutations rm = new RowMutations(row); rm.add(p); assertTrue(region.checkAndRowMutate(row, fam1, qual1, CompareOperator.EQUAL, new BinaryComparator(qual1), rm)); result = region.get(new Get(row)); c = result.getColumnLatestCell(fam1, qual1); assertEquals(10L, c.getTimestamp()); LOG.info("c value " + Bytes.toStringBinary(c.getValueArray(), c.getValueOffset(), c.getValueLength())); assertTrue(Bytes.equals(c.getValueArray(), c.getValueOffset(), c.getValueLength(), qual2, 0, qual2.length)); }
@SuppressWarnings("unchecked") @Test public void testMergeExpiredFiles() throws Exception { ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); long now = defaultTtl + 2; edge.setValue(now); EnvironmentEdgeManager.injectEdge(edge); try { HStoreFile expiredFile = createFile(), notExpiredFile = createFile(); when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1); when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1); List<HStoreFile> expired = Lists.newArrayList(expiredFile, expiredFile); List<HStoreFile> notExpired = Lists.newArrayList(notExpiredFile, notExpiredFile); List<HStoreFile> mixed = Lists.newArrayList(expiredFile, notExpiredFile); StripeCompactionPolicy policy = createPolicy(HBaseConfiguration.create(), defaultSplitSize, defaultSplitCount, defaultInitialCount, true); // Merge expired if there are eligible stripes. StripeCompactionPolicy.StripeInformationProvider si = createStripesWithFiles(expired, expired, expired); verifyWholeStripesCompaction(policy, si, 0, 2, null, 1, Long.MAX_VALUE, false); // Don't merge if nothing expired. si = createStripesWithFiles(notExpired, notExpired, notExpired); assertNull(policy.selectCompaction(si, al(), false)); // Merge one expired stripe with next. si = createStripesWithFiles(notExpired, expired, notExpired); verifyWholeStripesCompaction(policy, si, 1, 2, null, 1, Long.MAX_VALUE, false); // Merge the biggest run out of multiple options. // Merge one expired stripe with next. si = createStripesWithFiles(notExpired, expired, notExpired, expired, expired, notExpired); verifyWholeStripesCompaction(policy, si, 3, 4, null, 1, Long.MAX_VALUE, false); // Stripe with a subset of expired files is not merged. si = createStripesWithFiles(expired, expired, notExpired, expired, mixed); verifyWholeStripesCompaction(policy, si, 0, 1, null, 1, Long.MAX_VALUE, false); } finally { EnvironmentEdgeManager.reset(); } }
@SuppressWarnings("unchecked") @Test public void testMergeExpiredStripes() throws Exception { // HBASE-11397 ManualEnvironmentEdge edge = new ManualEnvironmentEdge(); long now = defaultTtl + 2; edge.setValue(now); EnvironmentEdgeManager.injectEdge(edge); try { HStoreFile expiredFile = createFile(), notExpiredFile = createFile(); when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1); when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1); List<HStoreFile> expired = Lists.newArrayList(expiredFile, expiredFile); List<HStoreFile> notExpired = Lists.newArrayList(notExpiredFile, notExpiredFile); StripeCompactionPolicy policy = createPolicy(HBaseConfiguration.create(), defaultSplitSize, defaultSplitCount, defaultInitialCount, true); // Merge all three expired stripes into one. StripeCompactionPolicy.StripeInformationProvider si = createStripesWithFiles(expired, expired, expired); verifyMergeCompatcion(policy, si, 0, 2); // Merge two adjacent expired stripes into one. si = createStripesWithFiles(notExpired, expired, notExpired, expired, expired, notExpired); verifyMergeCompatcion(policy, si, 3, 4); } finally { EnvironmentEdgeManager.reset(); } }