@Override protected void setup(Context context) throws IOException, InterruptedException { String tableNameToIndex = context.getConfiguration().get(TableIndexer.TABLE_NAME_TO_INDEX); HTable hTable = null; try { hTable = new HTable(context.getConfiguration(), tableNameToIndex); this.startKeys = hTable.getStartKeys(); byte[] indexBytes = hTable.getTableDescriptor().getValue(Constants.INDEX_SPEC_KEY); if (indexBytes != null) { TableIndices tableIndices = new TableIndices(); tableIndices.readFields(indexBytes); this.indices = tableIndices.getIndices(); } } finally { if (hTable != null) hTable.close(); } }
/** * Handles initializing this class with objects specific to it (i.e., the parser). Common * initialization that might be leveraged by a subsclass is done in <code>doSetup</code>. Hence a * subclass may choose to override this method and call <code>doSetup</code> as well before * handling it's own custom params. * @param context */ @Override protected void setup(Context context) throws IOException { doSetup(context); Configuration conf = context.getConfiguration(); parser = new TsvParser(conf.get(ImportTsv.COLUMNS_CONF_KEY), separator); if (parser.getRowKeyColumnIndex() == -1) { throw new RuntimeException("No row key column specified"); } String tableName = conf.get(TableInputFormat.INPUT_TABLE); HTable hTable = null; try { hTable = new HTable(conf, tableName); this.startKeys = hTable.getStartKeys(); byte[] indexBytes = hTable.getTableDescriptor().getValue(Constants.INDEX_SPEC_KEY); if (indexBytes != null) { TableIndices tableIndices = new TableIndices(); tableIndices.readFields(indexBytes); this.indices = tableIndices.getIndices(); } } finally { if (hTable != null) hTable.close(); } }
@Test(timeout = 180000) public void testCreateIndexTableWhenIndexTableAlreadyExist() throws Exception { HTableDescriptor iHtd = TestUtils.createIndexedHTableDescriptor("testCreateIndexTableWhenIndexTableAlreadyExist", "cf", "index_name", "cf", "cq"); admin.createTable(iHtd); admin.disableTable("testCreateIndexTableWhenIndexTableAlreadyExist"); admin.deleteTable("testCreateIndexTableWhenIndexTableAlreadyExist"); admin.createTable(iHtd); assertTrue("Table is not created.", admin.isTableAvailable("testCreateIndexTableWhenIndexTableAlreadyExist")); String indexTableName = "testCreateIndexTableWhenIndexTableAlreadyExist" + Constants.INDEX_TABLE_SUFFIX; assertTrue("Index table is not created.", admin.isTableAvailable(indexTableName)); }
@Test(timeout = 180000) public void testIndexTableCreationAfterMasterRestart() throws Exception { HTableDescriptor iHtd = TestUtils.createIndexedHTableDescriptor("testIndexTableCreationAfterMasterRestart", "cf", "index_name", "cf", "cq"); admin.createTable(iHtd); admin.disableTable("testIndexTableCreationAfterMasterRestart" + Constants.INDEX_TABLE_SUFFIX); admin.deleteTable("testIndexTableCreationAfterMasterRestart" + Constants.INDEX_TABLE_SUFFIX); MiniHBaseCluster cluster = UTIL.getHBaseCluster(); cluster.abortMaster(0); cluster.waitOnMaster(0); // start up a new master cluster.startMaster(); assertTrue(cluster.waitForActiveAndReadyMaster()); String indexTableName = "testIndexTableCreationAfterMasterRestart" + Constants.INDEX_TABLE_SUFFIX; assertTrue("Index tables is not created.", admin.isTableAvailable(indexTableName)); }
@Test(timeout = 180000) public void testPreCreateShouldNotBeSuccessfulIfIndicesAreNotSameAtBothTypeAndLength() throws IOException, KeeperException, InterruptedException { String userTableName = "testNotConsisIndex4"; HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); HColumnDescriptor hcd = new HColumnDescriptor("col"); IndexSpecification iSpec1 = new IndexSpecification("Index1"); iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); iSpec1.addIndexColumn(hcd, "q2", ValueType.String, 10); ihtd.addFamily(hcd); IndexSpecification iSpec2 = new IndexSpecification("Index2"); iSpec2.addIndexColumn(hcd, "q1", ValueType.Int, 10); iSpec2.addIndexColumn(hcd, "q2", ValueType.String, 7); TableIndices indices = new TableIndices(); indices.addIndex(iSpec1); indices.addIndex(iSpec2); ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); boolean returnVal = false; try { admin.createTable(ihtd); fail("IOException should be thrown"); } catch (IOException e) { returnVal = true; } Assert.assertTrue(returnVal); }
@Test(timeout = 180000) public void testPreCreateShouldBeSuccessfulIfIndicesAreSame() throws IOException, KeeperException, InterruptedException { String userTableName = "testConsistIndex"; HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); HColumnDescriptor hcd = new HColumnDescriptor("col"); IndexSpecification iSpec1 = new IndexSpecification("Index1"); iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); ihtd.addFamily(hcd); IndexSpecification iSpec2 = new IndexSpecification("Index2"); iSpec2.addIndexColumn(hcd, "q1", ValueType.String, 10); TableIndices indices = new TableIndices(); indices.addIndex(iSpec1); indices.addIndex(iSpec2); ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); try { admin.createTable(ihtd); } catch (IOException e) { fail("Exception should not be thrown"); } }
private void prepare() throws IOException { basedir = new Path(DIR + "TestIndexDelete"); Configuration conf = TEST_UTIL.getConfiguration(); // Prepare the 'employee' table region HTableDescriptor desc = new HTableDescriptor(TableName.valueOf("employee")); desc.addFamily(new HColumnDescriptor(CF_EMP).setMaxVersions(3)); desc.addFamily(new HColumnDescriptor(CF_DEPT).setMaxVersions(3)); HRegionInfo info = new HRegionInfo(desc.getTableName(), START_KEY.getBytes(), END_KEY.getBytes(), false); userRegion = HRegion.createHRegion(info, basedir, conf, desc); // Prepare the 'employee_idx' index table region HTableDescriptor idxDesc = new HTableDescriptor(TableName.valueOf("employee_idx")); idxDesc.addFamily(new HColumnDescriptor(Constants.IDX_COL_FAMILY)); HRegionInfo idxInfo = new HRegionInfo(idxDesc.getTableName(), START_KEY.getBytes(), END_KEY.getBytes(), false); indexRegion = HRegion.createHRegion(idxInfo, basedir, conf, idxDesc); }
@Test public void testSingleIndexExpressionWithOneEqualsExpression() throws Exception { String indexName = "idx1"; SingleIndexExpression singleIndexExpression = new SingleIndexExpression(indexName); byte[] value = "1".getBytes(); Column column = new Column(FAMILY1, QUALIFIER1); EqualsExpression equalsExpression = new EqualsExpression(column, value); singleIndexExpression.addEqualsExpression(equalsExpression); Scan scan = new Scan(); scan.setAttribute(Constants.INDEX_EXPRESSION, IndexUtils.toBytes(singleIndexExpression)); Filter filter = new SingleColumnValueFilter(FAMILY1, QUALIFIER1, CompareOp.EQUAL, value); scan.setFilter(filter); ScanFilterEvaluator evaluator = new ScanFilterEvaluator(); List<IndexSpecification> indices = new ArrayList<IndexSpecification>(); IndexSpecification index = new IndexSpecification(indexName); HColumnDescriptor colDesc = new HColumnDescriptor(FAMILY1); index.addIndexColumn(colDesc, COL1, ValueType.String, 10); indices.add(index); HRegion region = initHRegion(tableName.getBytes(), null, null, "testSingleIndexExpressionWithOneEqualsExpression", TEST_UTIL.getConfiguration(), FAMILY1); IndexRegionScanner scanner = evaluator.evaluate(scan, indices, new byte[0], region, tableName); // TODO add assertions }
@Test public void testNoIndexExpression() throws Exception { IndexExpression exp = new NoIndexExpression(); Scan scan = new Scan(); scan.setAttribute(Constants.INDEX_EXPRESSION, IndexUtils.toBytes(exp)); byte[] value1 = Bytes.toBytes("asdf"); scan.setFilter(new SingleColumnValueFilter(FAMILY1, QUALIFIER1, CompareOp.EQUAL, value1)); List<IndexSpecification> indices = new ArrayList<IndexSpecification>(); IndexSpecification is1 = new IndexSpecification("idx1"); HColumnDescriptor colDesc = new HColumnDescriptor(FAMILY1); is1.addIndexColumn(colDesc, COL1, ValueType.String, 15); indices.add(is1); ScanFilterEvaluator evaluator = new ScanFilterEvaluator(); HRegion region = initHRegion(tableName.getBytes(), null, null, "testNoIndexExpression", TEST_UTIL.getConfiguration(), FAMILY1); IndexRegionScanner scanner = evaluator.evaluate(scan, indices, new byte[0], region, tableName); assertNull(scanner); }
private byte[] generateStartKey(List<HRegionInfo> regionsOfTable) { byte[] startKey = regionsOfTable.get(0).getStartKey(); byte[] startRow = new byte[startKey.length + Constants.DEF_MAX_INDEX_NAME_LENGTH + 10 + "row999".getBytes().length]; System.arraycopy(startKey, 0, startRow, 0, startKey.length); System.arraycopy("ScanIndex".getBytes(), 0, startRow, startKey.length, "ScanIndex".length()); byte[] arr = new byte[18 - "ScanIndex".length()]; byte e[] = new byte[10]; System.arraycopy(arr, 0, startRow, startKey.length + "ScanIndex".length(), arr.length); System.arraycopy(e, 0, startRow, startKey.length + Constants.DEF_MAX_INDEX_NAME_LENGTH, 10); System.arraycopy("idxCat".getBytes(), 0, startRow, startKey.length + Constants.DEF_MAX_INDEX_NAME_LENGTH, "idxCat".getBytes().length); System.arraycopy("row99".getBytes(), 0, startRow, startKey.length + Constants.DEF_MAX_INDEX_NAME_LENGTH + 10, "row99".getBytes().length); System.out.println("constructed rowkey for indexed table " + Bytes.toString(startRow)); return startRow; }
@Override public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s) throws IOException { HRegionServer rs = (HRegionServer) c.getEnvironment().getRegionServerServices(); if (!store.getTableName().contains(Constants.INDEX_TABLE_SUFFIX)) { // Not an index table return null; } long smallestReadPoint = c.getEnvironment().getRegion().getSmallestReadPoint(); String actualTableName = IndexUtils.getActualTableNameFromIndexTableName(store.getTableName()); TTLStoreScanner ttlStoreScanner = new TTLStoreScanner(store, smallestReadPoint, earliestPutTs, scanType, scanners, new TTLExpiryChecker(), actualTableName, rs); return ttlStoreScanner; }
@Test(timeout = 180000) public void testCreateIndexTableWhenIndexTableAlreadyExist() throws Exception { HBaseAdmin admin = UTIL.getHBaseAdmin(); MiniHBaseCluster cluster = UTIL.getHBaseCluster(); HMaster master = cluster.getMaster(); IndexedHTableDescriptor iHtd = createIndexedHTableDescriptor("testCreateIndexTableWhenIndexTableAlreadyExist", "cf", "index_name", "cf", "cq"); admin.createTable(iHtd); admin.disableTable("testCreateIndexTableWhenIndexTableAlreadyExist"); admin.deleteTable("testCreateIndexTableWhenIndexTableAlreadyExist"); admin.createTable(iHtd); assertTrue("Table is not created.", admin.isTableAvailable("testCreateIndexTableWhenIndexTableAlreadyExist")); String indexTableName = "testCreateIndexTableWhenIndexTableAlreadyExist" + Constants.INDEX_TABLE_SUFFIX; assertTrue("Index table is not created.", admin.isTableAvailable(indexTableName)); }
@Test(timeout = 180000) public void testCreateIndexTableWithOutIndexDetails() throws Exception { HBaseAdmin admin = UTIL.getHBaseAdmin(); MiniHBaseCluster cluster = UTIL.getHBaseCluster(); HMaster master = cluster.getMaster(); IndexedHTableDescriptor iHtd = new IndexedHTableDescriptor("testCreateIndexTableWithOutIndexDetails"); admin.createTable(iHtd); assertTrue("Table is not created.", admin.isTableAvailable("testCreateIndexTableWithOutIndexDetails")); String indexTableName = "testCreateIndexTableWithOutIndexDetails" + Constants.INDEX_TABLE_SUFFIX; assertTrue("Index tables is not created.", admin.isTableAvailable(indexTableName)); }
@Test(timeout = 180000) public void testIndexTableCreationAfterMasterRestart() throws Exception { HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); IndexedHTableDescriptor iHtd = createIndexedHTableDescriptor("testIndexTableCreationAfterMasterRestart", "cf", "index_name", "cf", "cq"); admin.createTable(iHtd); admin.disableTable("testIndexTableCreationAfterMasterRestart" + Constants.INDEX_TABLE_SUFFIX); admin.deleteTable("testIndexTableCreationAfterMasterRestart" + Constants.INDEX_TABLE_SUFFIX); MiniHBaseCluster cluster = UTIL.getHBaseCluster(); cluster.abortMaster(0); cluster.waitOnMaster(0); HMaster master = cluster.startMaster().getMaster(); cluster.waitForActiveAndReadyMaster(); String indexTableName = "testIndexTableCreationAfterMasterRestart" + Constants.INDEX_TABLE_SUFFIX; assertTrue("Index tables is not created.", admin.isTableAvailable(indexTableName)); }
@Test(timeout = 180000) public void testIndexTableCreationAlongWithNormalTablesAfterMasterRestart() throws Exception { HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration()); HTableDescriptor htd = new HTableDescriptor("testIndexTableCreationAlongWithNormalTablesAfterMasterRestart"); admin.createTable(htd); MiniHBaseCluster cluster = UTIL.getHBaseCluster(); cluster.abortMaster(0); cluster.waitOnMaster(0); HMaster master = cluster.startMaster().getMaster(); cluster.waitForActiveAndReadyMaster(); boolean tableExist = MetaReader.tableExists(master.getCatalogTracker(), "testIndexTableCreationAlongWithNormalTablesAfterMasterRestart" + Constants.INDEX_TABLE_SUFFIX); assertFalse("Index table should be not created after master start up.", tableExist); }
private void prepare() throws IOException { basedir = new Path(DIR + "TestIndexDelete"); Configuration conf = TEST_UTIL.getConfiguration(); // Prepare the 'employee' table region HTableDescriptor desc = new HTableDescriptor("employee"); desc.addFamily(new HColumnDescriptor(CF_EMP)); desc.addFamily(new HColumnDescriptor(CF_DEPT)); HRegionInfo info = new HRegionInfo(desc.getName(), START_KEY.getBytes(), END_KEY.getBytes(), false); userRegion = HRegion.createHRegion(info, basedir, conf, desc); // Prepare the 'employee_idx' index table region HTableDescriptor idxDesc = new HTableDescriptor("employee_idx"); idxDesc.addFamily(new HColumnDescriptor(Constants.IDX_COL_FAMILY)); HRegionInfo idxInfo = new HRegionInfo(idxDesc.getName(), START_KEY.getBytes(), END_KEY.getBytes(), false); indexRegion = HRegion.createHRegion(idxInfo, basedir, conf, idxDesc); }
private void checkEndsWithIndexSuffix(TableName tableName) throws IOException { if (IndexUtils.isIndexTable(tableName)) { String message = "User table name should not be ends with " + Constants.INDEX_TABLE_SUFFIX + '.'; LOG.error(message); throw new DoNotRetryIOException(new IllegalArgumentException(message)); } }
@Override public void preCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx, HTableDescriptor desc, HRegionInfo[] regions) throws IOException { if (desc.getValue(Constants.INDEX_SPEC_KEY) != null) { LoadBalancer balancer = ctx.getEnvironment().getMasterServices().getAssignmentManager().getBalancer(); if (balancer instanceof SecIndexLoadBalancer) { ((SecIndexLoadBalancer) balancer).addIndexedTable(desc.getTableName()); } } }
@Override public void postOpen(ObserverContext<RegionCoprocessorEnvironment> contx) { HTableDescriptor tableDesc = contx.getEnvironment().getRegion().getTableDesc(); RegionServerServices rss = contx.getEnvironment().getRegionServerServices(); TableName tableName = tableDesc.getTableName(); if (isNotIndexedTableDescriptor(tableDesc)) { return; } LOG.trace("Entering postOpen for the table " + tableName); this.indexManager.incrementRegionCount(tableName.getNameAsString()); List<IndexSpecification> list = indexManager.getIndicesForTable(tableName.getNameAsString()); if (null != list) { LOG.trace("Index Manager already contains an entry for the table " + ". Hence returning from postOpen"); return; } byte[] indexBytes = tableDesc.getValue(Constants.INDEX_SPEC_KEY); TableIndices tableIndices = new TableIndices(); try { tableIndices.readFields(indexBytes); } catch (IOException e) { rss.abort("Some unidentified scenario while reading from the " + "table descriptor . Aborting RegionServer", e); } list = tableIndices.getIndices(); if (list != null && list.size() > 0) { indexManager.addIndexForTable(tableName.getNameAsString(), list); LOG.trace("Added index Specification in the Manager for the " + tableName); } LOG.trace("Exiting postOpen for the table " + tableName); }
@Override public void preSplitAfterPONR(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException { RegionCoprocessorEnvironment environment = e.getEnvironment(); HRegionServer rs = (HRegionServer) environment.getRegionServerServices(); HRegion region = environment.getRegion(); String userTableName = region.getTableDesc().getNameAsString(); String indexTableName = IndexUtils.getIndexTableName(userTableName); if (IndexUtils.isIndexTable(userTableName)) { return; } LOG.trace("Entering postSplit for the table " + userTableName + " for the region " + region.getRegionInfo()); SplitTransaction splitTransaction = null; if (region.getTableDesc().getValue(Constants.INDEX_SPEC_KEY) != null) { try { SplitInfo splitInfo = splitThreadLocal.get(); if (splitInfo == null) return; splitTransaction = splitInfo.getSplitTransaction(); PairOfSameType<HRegion> daughters = splitInfo.getDaughters(); if (splitTransaction != null && daughters != null) { splitTransaction.stepsAfterPONR(rs, rs, daughters); LOG.info("Daughter regions are opened and split transaction finished for zknodes for index table " + indexTableName + " for the region " + region.getRegionInfo()); } } catch (Exception ex) { String msg = "Splitting of index region has failed in stepsAfterPONR stage so aborting the server"; LOG.error(msg, ex); rs.abort(msg); } } }
public static Delete prepareIndexDelete(Delete userDelete, IndexSpecification index, byte[] indexRegionStartKey) throws IOException { ByteArrayBuilder indexRow = IndexUtils.getIndexRowKeyHeader(index, indexRegionStartKey, userDelete.getRow()); boolean update = false; for (ColumnQualifier cq : index.getIndexColumns()) { Cell kvFound = null; for (Entry<byte[], List<Cell>> entry : userDelete.getFamilyCellMap().entrySet()) { for (Cell cell : entry.getValue()) { Cell kv = KeyValueUtil.ensureKeyValue(cell); if (Bytes.equals(cq.getColumnFamily(), kv.getFamily()) && Bytes.equals(cq.getQualifier(), kv.getQualifier())) { kvFound = kv; update = true; break; } } } if (kvFound == null) { indexRow.position(indexRow.position() + cq.getMaxValueLength()); } else { IndexUtils.updateRowKeyForKV(cq, kvFound, indexRow); } } if (update) { // Append the actual row key at the end of the index row key. indexRow.put(userDelete.getRow()); Delete idxDelete = new Delete(indexRow.array()); idxDelete.deleteColumn(Constants.IDX_COL_FAMILY, Constants.IDX_COL_QUAL, userDelete.getTimeStamp()); idxDelete.setDurability(Durability.SKIP_WAL); return idxDelete; } return null; }
@Test(timeout = 180000) public void testCreateTableWithIndexTableSuffix() throws Exception { HTableDescriptor htd = TestUtils.createIndexedHTableDescriptor("testCreateTableWithIndexTableSuffix" + Constants.INDEX_TABLE_SUFFIX, "cf", "index_name", "cf", "cq"); try { admin.createTable(htd); fail("User table should not ends with " + Constants.INDEX_TABLE_SUFFIX); } catch (IOException e) { } }
@Test(timeout = 180000) public void testCreateIndexTableWithOutIndexDetails() throws Exception { HTableDescriptor iHtd = new HTableDescriptor(TableName.valueOf("testCreateIndexTableWithOutIndexDetails")); TableIndices indices = new TableIndices(); iHtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); admin.createTable(iHtd); assertTrue("Table is not created.", admin.isTableAvailable("testCreateIndexTableWithOutIndexDetails")); String indexTableName = "testCreateIndexTableWithOutIndexDetails" + Constants.INDEX_TABLE_SUFFIX; assertTrue("Index tables is not created.", admin.isTableAvailable(indexTableName)); }
@Test(timeout = 180000) public void testPreCreateShouldNotBeSuccessfulIfIndicesAreNotSame() throws IOException, KeeperException, InterruptedException { ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); String userTableName = "testNotConsisIndex1"; HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); HColumnDescriptor hcd = new HColumnDescriptor("col"); IndexSpecification iSpec1 = new IndexSpecification("Index1"); iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); ihtd.addFamily(hcd); TableIndices indices = new TableIndices(); indices.addIndex(iSpec1); IndexSpecification iSpec2 = new IndexSpecification("Index2"); iSpec2.addIndexColumn(hcd, "q1", ValueType.Int, 10); indices.addIndex(iSpec2); ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); boolean returnVal = false; try { admin.createTable(ihtd); fail("Exception should be thrown"); } catch (IOException e) { returnVal = true; } Assert.assertTrue(returnVal); ZKAssign.blockUntilNoRIT(zkw); }
@Test(timeout = 180000) public void testPreCreateShouldNotBeSuccessfulIfIndicesAreNotSameAtLength() throws IOException, KeeperException, InterruptedException { ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); String userTableName = "testNotConsisIndex2"; HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); HColumnDescriptor hcd = new HColumnDescriptor("col"); IndexSpecification iSpec1 = new IndexSpecification("Index1"); iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); iSpec1.addIndexColumn(hcd, "q2", ValueType.String, 4); ihtd.addFamily(hcd); TableIndices indices = new TableIndices(); indices.addIndex(iSpec1); IndexSpecification iSpec2 = new IndexSpecification("Index2"); iSpec2.addIndexColumn(hcd, "q3", ValueType.String, 10); iSpec2.addIndexColumn(hcd, "q2", ValueType.String, 10); indices.addIndex(iSpec2); ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); boolean returnVal = false; try { admin.createTable(ihtd); fail("Exception should be thrown"); } catch (IOException e) { returnVal = true; } Assert.assertTrue(returnVal); ZKAssign.blockUntilNoRIT(zkw); }
@Test(timeout = 180000) public void testPreCreateShouldNotBeSuccessfulIfIndicesAreNotSameAtType() throws IOException, KeeperException, InterruptedException { ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); String userTableName = "testNotConsisIndex3"; HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); HColumnDescriptor hcd = new HColumnDescriptor("col"); IndexSpecification iSpec1 = new IndexSpecification("Index1"); iSpec1.addIndexColumn(hcd, "q1", ValueType.String, 10); iSpec1.addIndexColumn(hcd, "q2", ValueType.String, 10); ihtd.addFamily(hcd); IndexSpecification iSpec2 = new IndexSpecification("Index2"); iSpec2.addIndexColumn(hcd, "q1", ValueType.Int, 10); iSpec2.addIndexColumn(hcd, "q3", ValueType.String, 10); TableIndices indices = new TableIndices(); indices.addIndex(iSpec1); indices.addIndex(iSpec2); ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); boolean returnVal = false; try { admin.createTable(ihtd); fail("Exception should be thrown"); } catch (IOException e) { returnVal = true; } Assert.assertTrue(returnVal); ZKAssign.blockUntilNoRIT(zkw); }
@Test(timeout = 180000) public void testCreateIndexTableFromExistingTable() throws Exception { String tableName = "testCreateIndexTableFromExistingTable"; HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f1"))); htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f2"))); byte[][] split = new byte[][] { Bytes.toBytes("A"), Bytes.toBytes("B") }; admin.createTable(htd, split); TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName)); HTable table = new HTable(admin.getConfiguration(), tableName); Put p = new Put(Bytes.toBytes("row1")); p.add(Bytes.toBytes("f1"), Bytes.toBytes("q1"), Bytes.toBytes("2")); p.add(Bytes.toBytes("f2"), Bytes.toBytes("q2"), Bytes.toBytes("3")); table.put(p); table.flushCommits(); admin.flush(tableName); UTIL.getConfiguration().set("table.columns.index", "IDX1=>f1:[q1->Int&10],[q2],[q3];f2:[q1->String&15],[q2->Int&15]#IDX2=>f1:[q5]"); IndexUtils.createIndexTable(tableName, UTIL.getConfiguration(), null); List<HRegionInfo> regionsOfTable = UTIL.getHBaseCluster().getMaster().getAssignmentManager() .getRegionStates().getRegionsOfTable(indexTableName); while (regionsOfTable.size() != 3) { Thread.sleep(500); regionsOfTable = UTIL.getHBaseCluster().getMaster().getAssignmentManager() .getRegionStates().getRegionsOfTable(indexTableName); } HTableDescriptor indexedTableDesc = admin.getTableDescriptor(TableName.valueOf(tableName)); byte[] value = indexedTableDesc.getValue(Constants.INDEX_SPEC_KEY); TableIndices indices = new TableIndices(); indices.readFields(value); assertEquals(indices.getIndices().size(), 2); Scan s = new Scan(); ResultScanner scanner = table.getScanner(s); Result[] next = scanner.next(10); List<Cell> cf1 = next[0].getColumnCells(Bytes.toBytes("f1"), Bytes.toBytes("q1")); List<Cell> cf2 = next[0].getColumnCells(Bytes.toBytes("f2"), Bytes.toBytes("q2")); assertTrue(cf1.size() > 0 && cf2.size() > 0); }
@Test(timeout = 180000) public void testShouldRetainTheExistingCFsInHTD() throws Exception { String tableName = "testShouldRetainTheExistingCFsInHTD"; HBaseTestingUtility.getZooKeeperWatcher(UTIL); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f1"))); htd.addFamily(new HColumnDescriptor(Bytes.toBytes("f2"))); admin.createTable(htd); TableName indexTableName = TableName.valueOf(IndexUtils.getIndexTableName(tableName)); HTable table = new HTable(admin.getConfiguration(), tableName); Put p = new Put(Bytes.toBytes("row1")); p.add(Bytes.toBytes("f1"), Bytes.toBytes("q1"), Bytes.toBytes("2")); p.add(Bytes.toBytes("f2"), Bytes.toBytes("q2"), Bytes.toBytes("3")); table.put(p); table.flushCommits(); admin.flush(tableName); UTIL.getConfiguration().set("table.columns.index", "IDX1=>f1:[q1->Int&10],[q2],[q3];"); IndexUtils.createIndexTable(tableName, UTIL.getConfiguration(), null); List<HRegionInfo> regionsOfTable = UTIL.getHBaseCluster().getMaster().getAssignmentManager() .getRegionStates().getRegionsOfTable(indexTableName); while (regionsOfTable.size() != 1) { Thread.sleep(500); regionsOfTable = UTIL.getHBaseCluster().getMaster().getAssignmentManager() .getRegionStates().getRegionsOfTable(indexTableName); } HTableDescriptor indexedTableDesc = admin.getTableDescriptor(TableName.valueOf(tableName)); byte[] value = indexedTableDesc.getValue(Constants.INDEX_SPEC_KEY); TableIndices indices = new TableIndices(); indices.readFields(value); assertEquals(indices.getIndices().size(), 1); Scan s = new Scan(); ResultScanner scanner = table.getScanner(s); Result[] next = scanner.next(10); List<Cell> cf1 = next[0].getColumnCells(Bytes.toBytes("f1"), Bytes.toBytes("q1")); List<Cell> cf2 = next[0].getColumnCells(Bytes.toBytes("f2"), Bytes.toBytes("q2")); assertTrue(cf1.size() > 0 && cf2.size() > 0); }
@Test(timeout = 180000) public void testShouldRetrieveIndexPutsOnRSRestart() throws IOException, KeeperException, InterruptedException { Configuration conf = UTIL.getConfiguration(); String userTableName = "testPutContainingTheIndexedColumn"; HTableDescriptor ihtd = TestUtils.createIndexedHTableDescriptor(userTableName, "col", "Index1", "col", "ql"); admin.createTable(ihtd); HTable table = new HTable(conf, userTableName); // test put with the indexed column Put p = new Put("row1".getBytes()); p.add("col".getBytes(), "ql".getBytes(), "myValue".getBytes()); table.put(p); // Thread.sleep(2000); int i = countNumberOfRows(userTableName); Assert.assertEquals(1, i); i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); Assert.assertEquals(1, i); HRegionServer regionServer = UTIL.getHBaseCluster().getRegionServer(0); HMaster master = UTIL.getHBaseCluster().getMaster(); regionServer.abort("Aborting region server"); while (master.getServerManager().areDeadServersInProgress()) { Thread.sleep(1000); } UTIL.getHBaseCluster().startRegionServer(); i = countNumberOfRows(userTableName); Assert.assertEquals(1, i); i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); Assert.assertEquals(1, i); }
@Test(timeout = 180000) public void testPutWithAndWithoutTheIndexedColumn() throws IOException, KeeperException, InterruptedException { Configuration conf = UTIL.getConfiguration(); String userTableName = "testPutContainingTheIndexedColumn"; HTableDescriptor ihtd = TestUtils.createIndexedHTableDescriptor(userTableName, "col", "Index1", "col", "ql"); admin.createTable(ihtd); HTable table = new HTable(conf, userTableName); // test put with the indexed column Put p = new Put("row1".getBytes()); p.add("col".getBytes(), "ql".getBytes(), "myValue".getBytes()); table.put(p); int i = countNumberOfRows(userTableName); Assert.assertEquals(1, i); i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); Assert.assertEquals(1, i); // Test put without the indexed column Put p1 = new Put("row2".getBytes()); p1.add("col".getBytes(), "ql1".getBytes(), "myValue".getBytes()); table.put(p1); i = countNumberOfRows(userTableName); Assert.assertEquals(2, i); i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); Assert.assertEquals(1, i); }
@Test(timeout = 180000) public void testMultipleIndicesOnUniqueColumns() throws IOException, KeeperException, InterruptedException { Configuration conf = UTIL.getConfiguration(); String userTableName = "testMultipleIndicesOnUniqueColumns"; HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); HColumnDescriptor hcd = new HColumnDescriptor("col"); IndexSpecification iSpec1 = new IndexSpecification("Index1"); IndexSpecification iSpec2 = new IndexSpecification("Index2"); iSpec1.addIndexColumn(hcd, "ql1", ValueType.String, 10); iSpec2.addIndexColumn(hcd, "ql2", ValueType.String, 10); ihtd.addFamily(hcd); TableIndices indices = new TableIndices(); indices.addIndex(iSpec1); indices.addIndex(iSpec2); ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); admin.createTable(ihtd); HTable table = new HTable(conf, userTableName); Put p = new Put("row1".getBytes()); p.add("col".getBytes(), "ql3".getBytes(), "myValue".getBytes()); p.add("col".getBytes(), "ql4".getBytes(), "myValue".getBytes()); table.put(p); int i = countNumberOfRows(userTableName); Assert.assertEquals(1, i); i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); Assert.assertEquals(0, i); p = new Put("row2".getBytes()); p.add("col".getBytes(), "ql1".getBytes(), "myValue".getBytes()); p.add("col".getBytes(), "ql2".getBytes(), "myValue".getBytes()); table.put(p); i = countNumberOfRows(userTableName); Assert.assertEquals(2, i); i = countNumberOfRows(userTableName + Constants.INDEX_TABLE_SUFFIX); Assert.assertEquals(2, i); }
@Test(timeout = 180000) public void testIndexPutRowkeyWithAllTheValues() throws IOException { String DIR = UTIL.getDataTestDir("TestStore").toString(); Path basedir = new Path(DIR + "TestIndexPut"); // Path logdir = new Path(DIR+"TestIndexPut"+"/logs"); FileSystem fs = UTIL.getTestFileSystem(); Configuration conf = UTIL.getConfiguration(); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestIndexPut")); HRegionInfo info = new HRegionInfo(htd.getTableName(), "A".getBytes(), "B".getBytes(), false); HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); IndexSpecification spec = new IndexSpecification("testSpec"); spec.addIndexColumn(new HColumnDescriptor("cf1"), "ql1", ValueType.String, 10); spec.addIndexColumn(new HColumnDescriptor("cf2"), "ql1", ValueType.String, 10); // Scenario where both the indexed cols are there in the put byte[] rowKey = "Arow1".getBytes(); Put p = new Put(rowKey); long time = 1234567; p.add("cf1".getBytes(), "ql1".getBytes(), time, "testvalue1".getBytes()); p.add("cf2".getBytes(), "ql1".getBytes(), time + 10, "testvalue1".getBytes()); Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); Assert.assertEquals(region.getStartKey().length + 1 + Constants.DEF_MAX_INDEX_NAME_LENGTH + 2 * 10 + rowKey.length, indexPut.getRow().length); Assert.assertEquals(time + 10, indexPut.get(Constants.IDX_COL_FAMILY, "".getBytes()).get(0) .getTimestamp()); }
@Test(timeout = 180000) public void testIndexPutWithOnlyOneValue() throws IOException { String DIR = UTIL.getDataTestDir("TestStore").toString(); Path basedir = new Path(DIR + "TestIndexPut"); // Path logdir = new Path(DIR+"TestIndexPut"+"/logs"); FileSystem fs = UTIL.getTestFileSystem(); Configuration conf = UTIL.getConfiguration(); HTableDescriptor htd = new HTableDescriptor("TestIndexPut"); HRegionInfo info = new HRegionInfo(htd.getTableName(), "A".getBytes(), "B".getBytes(), false); HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); IndexSpecification spec = new IndexSpecification("testSpec"); spec.addIndexColumn(new HColumnDescriptor("cf1"), "ql1", ValueType.String, 10); spec.addIndexColumn(new HColumnDescriptor("cf2"), "ql1", ValueType.String, 10); byte[] rowKey = "Arow1".getBytes(); Put p = new Put(rowKey); long time = 1234567; p.add("cf1".getBytes(), "ql1".getBytes(), time, "testvalue1".getBytes()); Put indexPut = IndexUtils.prepareIndexPut(p, spec, region); Assert.assertEquals(region.getStartKey().length + 1 + Constants.DEF_MAX_INDEX_NAME_LENGTH + 2 * 10 + rowKey.length, indexPut.getRow().length); Assert.assertEquals(time, indexPut.get(Constants.IDX_COL_FAMILY, "".getBytes()).get(0) .getTimestamp()); // asserting pad........this has to be hardcoded. byte[] pad = new byte[10]; System.arraycopy(indexPut.getRow(), region.getStartKey().length + 1 + Constants.DEF_MAX_INDEX_NAME_LENGTH + 10, pad, 0, 10); Assert.assertTrue(Bytes.equals(pad, new byte[10])); }
@Test(timeout = 180000) public void testIndexTableValue() throws IOException { String DIR = UTIL.getDataTestDir("TestStore").toString(); Path basedir = new Path(DIR + "TestIndexPut"); // Path logdir = new Path(DIR+"TestIndexPut"+"/logs"); FileSystem fs = UTIL.getTestFileSystem(); Configuration conf = UTIL.getConfiguration(); HTableDescriptor htd = new HTableDescriptor("TestIndexPut"); HRegionInfo info = new HRegionInfo(htd.getTableName(), "ABC".getBytes(), "BBB".getBytes(), false); HLog hlog = UTIL.getMiniHBaseCluster().getRegionServer(0).getWAL(); HRegion region = new HRegion(basedir, hlog, fs, conf, info, htd, null); IndexSpecification spec = new IndexSpecification("index"); spec.addIndexColumn(new HColumnDescriptor("cf1"), "ql1", ValueType.String, 10); spec.addIndexColumn(new HColumnDescriptor("cf2"), "ql1", ValueType.String, 10); byte[] rowKey = "Arow1".getBytes(); Put p1 = new Put(rowKey); long time = 1234567; p1.add("cf1".getBytes(), "ql1".getBytes(), time, "testcase".getBytes()); p1.add("cf2".getBytes(), "ql1".getBytes(), time, "value".getBytes()); Put indexPut1 = IndexUtils.prepareIndexPut(p1, spec, region); List<Cell> kvs = indexPut1.get(Constants.IDX_COL_FAMILY, "".getBytes()); Cell kv = null; if (null != kvs) { kv = kvs.get(0); } byte[] val = kv.getValue(); byte[] startKeyLengthInBytes = new byte[2]; System.arraycopy(val, 0, startKeyLengthInBytes, 0, startKeyLengthInBytes.length); int startkeylen = (int) (Bytes.toShort(startKeyLengthInBytes)); Assert.assertEquals(3, startkeylen); byte[] rowKeyOffset = new byte[2]; System.arraycopy(val, startKeyLengthInBytes.length, rowKeyOffset, 0, rowKeyOffset.length); int rowKeyOffsetInt = Bytes.toShort(rowKeyOffset); Assert.assertEquals(42, rowKeyOffsetInt); }
@Test(timeout = 180000) public void testPutWithValueLengthMoreThanMaxValueLength() throws IOException, KeeperException, InterruptedException { Configuration conf = UTIL.getConfiguration(); String userTableName = "testPutWithValueLengthMoreThanMaxValueLength"; HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); HColumnDescriptor hcd = new HColumnDescriptor("col"); IndexSpecification iSpec1 = new IndexSpecification("Index1"); iSpec1.addIndexColumn(hcd, "ql1", ValueType.String, 10); ihtd.addFamily(hcd); TableIndices indices = new TableIndices(); indices.addIndex(iSpec1); ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); admin.createTable(ihtd); HTable table = new HTable(conf, userTableName); table.setAutoFlush(false, false); List<Put> putList = new ArrayList<Put>(3); putList.add(new Put("row1".getBytes()).add("col".getBytes(), "ql1".getBytes(), "valueLengthMoreThanMaxValueLength".getBytes())); putList.add(new Put("row2".getBytes()).add("col".getBytes(), "ql1".getBytes(), "myValue".getBytes())); putList.add(new Put("row3".getBytes()).add("col".getBytes(), "ql1".getBytes(), "myValue".getBytes())); table.put(putList); try { table.flushCommits(); } catch (RetriesExhaustedWithDetailsException e) { // nothing to do. } Assert.assertEquals(1, table.getWriteBuffer().size()); }
@Test(timeout = 180000) public void testIndexManagerCleanUp() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.setBoolean("hbase.use.secondary.index", true); String userTableName = "testIndexManagerCleanUp"; HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); HColumnDescriptor hcd = new HColumnDescriptor("col1"); ihtd.addFamily(hcd); IndexSpecification iSpec = new IndexSpecification("Index1"); iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); TableIndices indices = new TableIndices(); indices.addIndex(iSpec); ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); byte[][] splits = new byte[10][]; char c = 'A'; for (int i = 0; i < 10; i++) { byte[] b = { (byte) c }; splits[i] = b; c++; } admin.createTable(ihtd, splits); IndexManager instance = IndexManager.getInstance(); int regionCount = instance.getTableRegionCount(userTableName); Assert.assertEquals(11, regionCount); admin.disableTable(Bytes.toBytes(userTableName)); regionCount = instance.getTableRegionCount(userTableName); Assert.assertEquals(0, regionCount); admin.enableTable(userTableName); regionCount = instance.getTableRegionCount(userTableName); Assert.assertEquals(11, regionCount); }
@Test(timeout = 180000) public void testIndexManagerWithSplitTransactions() throws Exception { Configuration conf = UTIL.getConfiguration(); ZooKeeperWatcher zkw = UTIL.getZooKeeperWatcher(UTIL); conf.setBoolean("hbase.use.secondary.index", true); String userTableName = "testIndexManagerWithSplitTransactions"; HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); HColumnDescriptor hcd = new HColumnDescriptor("col1"); ihtd.addFamily(hcd); IndexSpecification iSpec = new IndexSpecification("Index1"); iSpec.addIndexColumn(hcd, "ql", ValueType.String, 10); TableIndices indices = new TableIndices(); indices.addIndex(iSpec); ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); admin.createTable(ihtd); IndexManager manager = IndexManager.getInstance(); int count = manager.getTableRegionCount(userTableName); Assert.assertEquals(1, count); HTable table = new HTable(conf, userTableName); Put p = null; for (int i = 0; i < 10; i++) { p = new Put(Bytes.toBytes("row" + i)); p.add(Bytes.toBytes("col1"), Bytes.toBytes("ql"), Bytes.toBytes("test_val")); table.put(p); } admin.split(userTableName, "row5"); Threads.sleep(10000); ZKAssign.blockUntilNoRIT(zkw); UTIL.waitUntilAllRegionsAssigned(TableName.valueOf(userTableName)); count = manager.getTableRegionCount(userTableName); Assert.assertEquals(2, count); }