/** * CatalogJanitor.scan() should not clean parent regions if their own * parents are still referencing them. This ensures that grandfather regions * do not point to deleted parent regions. */ @Test public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception { HBaseTestingUtility htu = new HBaseTestingUtility(); setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents"); Server server = new MockServer(htu); MasterServices services = new MockMasterServices(server); final HTableDescriptor htd = createHTableDescriptor(); // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc. // Parent HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), new byte[0], true); // Sleep a second else the encoded name on these regions comes out // same for all with same start key and made in same second. Thread.sleep(1001); // Daughter a HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"), true); Thread.sleep(1001); // Make daughters of daughter a; splitaa and splitab. HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), false); HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false); // Daughter b HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"), new byte[0]); Thread.sleep(1001); final Map<HRegionInfo, Result> splitParents = new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator()); splitParents.put(parent, createResult(parent, splita, splitb)); splita.setOffline(true); //simulate that splita goes offline when it is split splitParents.put(splita, createResult(splita, splitaa,splitab)); final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>(); CatalogJanitor janitor = spy(new CatalogJanitor(server, services)); doReturn(new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>( 10, mergedRegions, splitParents)).when(janitor) .getMergedRegionsAndSplitParents(); //create ref from splita to parent Path splitaRef = createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false); //parent and A should not be removed assertEquals(0, janitor.scan()); //now delete the ref FileSystem fs = FileSystem.get(htu.getConfiguration()); assertTrue(fs.delete(splitaRef, true)); //now, both parent, and splita can be deleted assertEquals(2, janitor.scan()); services.stop("test finished"); janitor.cancel(true); }
/** * CatalogJanitor.scan() should not clean parent regions if their own * parents are still referencing them. This ensures that grandfather regions * do not point to deleted parent regions. */ @Test public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception { HBaseTestingUtility htu = new HBaseTestingUtility(); setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents"); Server server = new MockServer(htu); MasterServices services = new MockMasterServices(server); final HTableDescriptor htd = createHTableDescriptor(); // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc. // Parent HRegionInfo parent = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), new byte[0], true); // Sleep a second else the encoded name on these regions comes out // same for all with same start key and made in same second. Thread.sleep(1001); // Daughter a HRegionInfo splita = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"), true); Thread.sleep(1001); // Make daughters of daughter a; splitaa and splitab. HRegionInfo splitaa = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), false); HRegionInfo splitab = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false); // Daughter b HRegionInfo splitb = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), new byte[0]); Thread.sleep(1001); final Map<HRegionInfo, Result> splitParents = new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator()); splitParents.put(parent, makeResultFromHRegionInfo(parent, splita, splitb)); splita.setOffline(true);//simulate that splita goes offline when it is split splitParents.put(splita, makeResultFromHRegionInfo(splita, splitaa, splitab)); CatalogJanitor janitor = spy(new CatalogJanitor(server, services)); doReturn(new Pair<Integer, Map<HRegionInfo, Result>>( 10, splitParents)).when(janitor).getSplitParents(); //create ref from splita to parent Path splitaRef = createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false); //parent and A should not be removed assertEquals(0, janitor.scan()); //now delete the ref FileSystem fs = FileSystem.get(htu.getConfiguration()); assertTrue(fs.delete(splitaRef, true)); //now, both parent, and splita can be deleted assertEquals(2, janitor.scan()); services.stop("test finished"); janitor.join(); }
@Test public void testSplitParentFirstComparator() { SplitParentFirstComparator comp = new SplitParentFirstComparator(); final HTableDescriptor htd = createHTableDescriptor(); /* Region splits: * * rootRegion --- firstRegion --- firstRegiona * | |- firstRegionb * | * |- lastRegion --- lastRegiona --- lastRegionaa * | |- lastRegionab * |- lastRegionb * * rootRegion : [] - [] * firstRegion : [] - bbb * lastRegion : bbb - [] * firstRegiona : [] - aaa * firstRegionb : aaa - bbb * lastRegiona : bbb - ddd * lastRegionb : ddd - [] */ // root region HRegionInfo rootRegion = new HRegionInfo(htd.getName(), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, true); HRegionInfo firstRegion = new HRegionInfo(htd.getName(), HConstants.EMPTY_START_ROW, Bytes.toBytes("bbb"), true); HRegionInfo lastRegion = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"), HConstants.EMPTY_END_ROW, true); assertTrue(comp.compare(rootRegion, rootRegion) == 0); assertTrue(comp.compare(firstRegion, firstRegion) == 0); assertTrue(comp.compare(lastRegion, lastRegion) == 0); assertTrue(comp.compare(rootRegion, firstRegion) < 0); assertTrue(comp.compare(rootRegion, lastRegion) < 0); assertTrue(comp.compare(firstRegion, lastRegion) < 0); //first region split into a, b HRegionInfo firstRegiona = new HRegionInfo(htd.getName(), HConstants.EMPTY_START_ROW, Bytes.toBytes("aaa"), true); HRegionInfo firstRegionb = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), true); //last region split into a, b HRegionInfo lastRegiona = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"), Bytes.toBytes("ddd"), true); HRegionInfo lastRegionb = new HRegionInfo(htd.getName(), Bytes.toBytes("ddd"), HConstants.EMPTY_END_ROW, true); assertTrue(comp.compare(firstRegiona, firstRegiona) == 0); assertTrue(comp.compare(firstRegionb, firstRegionb) == 0); assertTrue(comp.compare(rootRegion, firstRegiona) < 0); assertTrue(comp.compare(rootRegion, firstRegionb) < 0); assertTrue(comp.compare(firstRegion, firstRegiona) < 0); assertTrue(comp.compare(firstRegion, firstRegionb) < 0); assertTrue(comp.compare(firstRegiona, firstRegionb) < 0); assertTrue(comp.compare(lastRegiona, lastRegiona) == 0); assertTrue(comp.compare(lastRegionb, lastRegionb) == 0); assertTrue(comp.compare(rootRegion, lastRegiona) < 0); assertTrue(comp.compare(rootRegion, lastRegionb) < 0); assertTrue(comp.compare(lastRegion, lastRegiona) < 0); assertTrue(comp.compare(lastRegion, lastRegionb) < 0); assertTrue(comp.compare(lastRegiona, lastRegionb) < 0); assertTrue(comp.compare(firstRegiona, lastRegiona) < 0); assertTrue(comp.compare(firstRegiona, lastRegionb) < 0); assertTrue(comp.compare(firstRegionb, lastRegiona) < 0); assertTrue(comp.compare(firstRegionb, lastRegionb) < 0); HRegionInfo lastRegionaa = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false); HRegionInfo lastRegionab = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), Bytes.toBytes("ddd"), false); assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0); assertTrue(comp.compare(lastRegiona, lastRegionab) < 0); assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0); }
/** * CatalogJanitor.scan() should not clean parent regions if their own * parents are still referencing them. This ensures that grandfather regions * do not point to deleted parent regions. */ @Test public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception { HBaseTestingUtility htu = new HBaseTestingUtility(); setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents"); Server server = new MockServer(htu); MasterServices services = new MockMasterServices(server); final HTableDescriptor htd = createHTableDescriptor(); // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc. // Parent HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), new byte[0], true); // Sleep a second else the encoded name on these regions comes out // same for all with same start key and made in same second. Thread.sleep(1001); // Daughter a HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"), true); Thread.sleep(1001); // Make daughters of daughter a; splitaa and splitab. HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), false); HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false); // Daughter b HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"), new byte[0]); Thread.sleep(1001); final Map<HRegionInfo, Result> splitParents = new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator()); splitParents.put(parent, createResult(parent, splita, splitb)); splita.setOffline(true); //simulate that splita goes offline when it is split splitParents.put(splita, createResult(splita, splitaa,splitab)); final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>(); CatalogJanitor janitor = spy(new CatalogJanitor(server, services)); doReturn(new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>( 10, mergedRegions, splitParents)).when(janitor) .getMergedRegionsAndSplitParents(); //create ref from splita to parent Path splitaRef = createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false); //parent and A should not be removed assertEquals(0, janitor.scan()); //now delete the ref FileSystem fs = FileSystem.get(htu.getConfiguration()); assertTrue(fs.delete(splitaRef, true)); //now, both parent, and splita can be deleted assertEquals(2, janitor.scan()); services.stop("test finished"); janitor.join(); }
/** * CatalogJanitor.scan() should not clean parent regions if their own * parents are still referencing them. This ensures that grandparent regions * do not point to deleted parent regions. */ @Test public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception { TableDescriptor td = createTableDescriptorForCurrentMethod(); // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc. // Parent HRegionInfo parent = new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), HConstants.EMPTY_BYTE_ARRAY, true); // Sleep a second else the encoded name on these regions comes out // same for all with same start key and made in same second. Thread.sleep(1001); // Daughter a HRegionInfo splita = new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"), true); Thread.sleep(1001); // Make daughters of daughter a; splitaa and splitab. HRegionInfo splitaa = new HRegionInfo(td.getTableName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), false); HRegionInfo splitab = new HRegionInfo(td.getTableName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false); // Daughter b HRegionInfo splitb = new HRegionInfo(td.getTableName(), Bytes.toBytes("ccc"), HConstants.EMPTY_BYTE_ARRAY); Thread.sleep(1001); // Parent has daughters splita and splitb. Splita has daughters splitaa and splitab. final Map<HRegionInfo, Result> splitParents = new TreeMap<>(new SplitParentFirstComparator()); splitParents.put(parent, createResult(parent, splita, splitb)); splita.setOffline(true); //simulate that splita goes offline when it is split splitParents.put(splita, createResult(splita, splitaa, splitab)); final Map<HRegionInfo, Result> mergedRegions = new TreeMap<>(); CatalogJanitor spy = spy(this.janitor); doReturn(new Triple<>(10, mergedRegions, splitParents)).when(spy). getMergedRegionsAndSplitParents(); // Create ref from splita to parent LOG.info("parent=" + parent.getShortNameToLog() + ", splita=" + splita.getShortNameToLog()); Path splitaRef = createReferences(this.masterServices, td, parent, splita, Bytes.toBytes("ccc"), false); LOG.info("Created reference " + splitaRef); // Parent and splita should not be removed because a reference from splita to parent. assertEquals(0, spy.scan()); // Now delete the ref FileSystem fs = FileSystem.get(HTU.getConfiguration()); assertTrue(fs.delete(splitaRef, true)); //now, both parent, and splita can be deleted assertEquals(2, spy.scan()); }
/** * CatalogJanitor.scan() should not clean parent regions if their own * parents are still referencing them. This ensures that grandfather regions * do not point to deleted parent regions. */ @Test public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception { HBaseTestingUtility htu = new HBaseTestingUtility(); setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents"); Server server = new MockServer(htu); MasterServices services = new MockMasterServices(server); final HTableDescriptor htd = createHTableDescriptor(); // Create regions: aaa->{lastEndKey}, aaa->ccc, aaa->bbb, bbb->ccc, etc. // Parent HRegionInfo parent = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), new byte[0], true); // Sleep a second else the encoded name on these regions comes out // same for all with same start key and made in same second. Thread.sleep(1001); // Daughter a HRegionInfo splita = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("ccc"), true); Thread.sleep(1001); // Make daughters of daughter a; splitaa and splitab. HRegionInfo splitaa = new HRegionInfo(htd.getName(), Bytes.toBytes("aaa"), Bytes.toBytes("bbb"), false); HRegionInfo splitab = new HRegionInfo(htd.getName(), Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), false); // Daughter b HRegionInfo splitb = new HRegionInfo(htd.getName(), Bytes.toBytes("ccc"), new byte[0]); Thread.sleep(1001); final Map<HRegionInfo, Result> splitParents = new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator()); splitParents.put(parent, createResult(parent, splita, splitb)); splita.setOffline(true); //simulate that splita goes offline when it is split splitParents.put(splita, createResult(splita, splitaa,splitab)); CatalogJanitor janitor = spy(new CatalogJanitor(server, services)); doReturn(new Pair<Integer, Map<HRegionInfo, Result>>( 10, splitParents)).when(janitor).getSplitParents(); //create ref from splita to parent Path splitaRef = createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false); //parent and A should not be removed assertEquals(0, janitor.scan()); //now delete the ref FileSystem fs = FileSystem.get(htu.getConfiguration()); assertTrue(fs.delete(splitaRef, true)); //now, both parent, and splita can be deleted assertEquals(2, janitor.scan()); services.stop("test finished"); janitor.join(); }