@BeforeClass public static void setUpBeforeClass() throws Exception { SecureTestUtil.enableSecurity(conf); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, AccessController.class.getName() + "," + IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, AccessController.class.getName() + "," + SecureBulkLoadEndpoint.class.getName() + "," + IndexRegionObserver.class.getName()); conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, AccessController.class.getName()); conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); conf.setInt("hbase.regionserver.lease.period", 10 * 60 * 1000); conf.setInt("hbase.rpc.timeout", 10 * 60 * 1000); conf.setBoolean("hbase.use.secondary.index", true); TEST_UTIL.startMiniCluster(2); TEST_UTIL.waitTableAvailable(AccessControlLists.ACL_TABLE_NAME, 5000); // create a set of test users SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); TEST_USER = User.createUserForTesting(conf, "testUser", new String[0]); }
@Override public boolean preWALWrite(ObserverContext<WALCoprocessorEnvironment> ctx, HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { TableName tableName = info.getTable(); if (IndexUtils.isCatalogOrSystemTable(tableName) || IndexUtils.isIndexTable(tableName)) { return true; } List<IndexSpecification> indices = indexManager.getIndicesForTable(tableName.getNameAsString()); if (indices != null && !indices.isEmpty()) { LOG.trace("Entering preWALWrite for the table " + tableName); String indexTableName = IndexUtils.getIndexTableName(tableName); IndexEdits iEdits = IndexRegionObserver.threadLocal.get(); WALEdit indexWALEdit = iEdits.getWALEdit(); // This size will be 0 when none of the Mutations to the user table to be indexed. // or write to WAL is disabled for the Mutations if (indexWALEdit.getKeyValues().size() == 0) { return true; } LOG.trace("Adding indexWALEdits into WAL for table " + tableName); HRegion indexRegion = iEdits.getRegion(); // TS in all KVs within WALEdit will be the same. So considering the 1st one. Long time = indexWALEdit.getKeyValues().get(0).getTimestamp(); indexRegion.getLog().appendNoSync(indexRegion.getRegionInfo(), TableName.valueOf(indexTableName), indexWALEdit, logKey.getClusterIds(), time, indexRegion.getTableDesc(), indexRegion.getSequenceId(), true, HConstants.NO_NONCE, HConstants.NO_NONCE); LOG.trace("Exiting preWALWrite for the table " + tableName); } return true; }
@BeforeClass public static void setupBeforeClass() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); conf.setBoolean("hbase.hregion.scan.loadColumnFamiliesOnDemand", false); conf.setBoolean("hbase.use.secondary.index", true); conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, LoadBalancer.class); UTIL.startMiniCluster(1); admin = new IndexAdmin(conf); }
@Before public void setUp() throws Exception { IndexRegionObserver.setIndexedFlowUsed(false); IndexRegionObserver.setSeekpointAdded(false); IndexRegionObserver.setSeekPoints(null); IndexRegionObserver.setIsTestingEnabled(true); IndexRegionObserver.addSeekPoints(null); }
@BeforeClass public static void setUpBeforeClass() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); conf.setBoolean("hbase.use.secondary.index", true); conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, LoadBalancer.class); UTIL.startMiniCluster(2); admin = new IndexAdmin(conf); }
@BeforeClass public static void setupBeforeClass() throws Exception { final int NUM_MASTERS = 1; final int NUM_RS = 4; Configuration conf = UTIL.getConfiguration(); conf.setBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO,true); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); conf.setBoolean("hbase.use.secondary.index", true); conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, LoadBalancer.class); UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); admin = new IndexAdmin(UTIL.getConfiguration()); }
@BeforeClass public static void setupBeforeClass() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); conf.setBoolean("hbase.use.secondary.index", true); conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, LoadBalancer.class); UTIL.startMiniCluster(1); }
@BeforeClass public static void setupBeforeClass() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.setBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO,true); conf.setBoolean("hbase.use.secondary.index", true); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); conf.set("index.data.block.encoding.algo", "PREFIX"); conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, LoadBalancer.class); UTIL.startMiniCluster(1); admin = new IndexAdmin(conf); }
@BeforeClass public static void setupBeforeClass() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); conf.setBoolean("hbase.use.secondary.index", true); conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, LoadBalancer.class); UTIL.startMiniCluster(1); admin = new IndexAdmin(conf); }
@BeforeClass public static void setupBeforeClass() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); conf.setBoolean("hbase.use.secondary.index", true); conf.set("index.data.block.encoding.algo", "PREFIX"); conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, SecIndexLoadBalancer.class, LoadBalancer.class); UTIL.startMiniCluster(1); }
@Override public boolean preWALWrite(ObserverContext<WALCoprocessorEnvironment> ctx, HRegionInfo info, HLogKey logKey, WALEdit logEdit) throws IOException { String tableNameStr = info.getTableNameAsString(); if (IndexUtils.isCatalogTable(info.getTableName()) || IndexUtils.isIndexTable(tableNameStr)) { return true; } List<IndexSpecification> indices = indexManager.getIndicesForTable(tableNameStr); if (indices != null && !indices.isEmpty()) { LOG.trace("Entering preWALWrite for the table " + tableNameStr); String indexTableName = IndexUtils.getIndexTableName(tableNameStr); IndexEdits iEdits = IndexRegionObserver.threadLocal.get(); WALEdit indexWALEdit = iEdits.getWALEdit(); // This size will be 0 when none of the Mutations to the user table to be indexed. // or write to WAL is disabled for the Mutations if (indexWALEdit.getKeyValues().size() == 0) { return true; } LOG.trace("Adding indexWALEdits into WAL for table " + tableNameStr); HRegion indexRegion = iEdits.getRegion(); // TS in all KVs within WALEdit will be the same. So considering the 1st one. Long time = indexWALEdit.getKeyValues().get(0).getTimestamp(); ctx.getEnvironment() .getWAL() .appendNoSync(indexRegion.getRegionInfo(), Bytes.toBytes(indexTableName), indexWALEdit, logKey.getClusterId(), time, indexRegion.getTableDesc()); LOG.trace("Exiting preWALWrite for the table " + tableNameStr); } return true; }
@BeforeClass public static void setupBeforeClass() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); conf.setInt("hbase.regionserver.lease.period", 10 * 60 * 1000); conf.setInt("hbase.rpc.timeout", 10 * 60 * 1000); UTIL.startMiniCluster(1); }
@BeforeClass public static void setUpBeforeClass() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); conf.setBoolean("hbase.use.secondary.index", true); UTIL.startMiniCluster(2); }
@BeforeClass public static void setupBeforeClass() throws Exception { final int NUM_MASTERS = 1; final int NUM_RS = 4; Configuration conf = UTIL.getConfiguration(); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); conf.setBoolean("hbase.use.secondary.index", true); UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); }
@BeforeClass public static void setupBeforeClass() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); conf.setBoolean("hbase.use.secondary.index", true); UTIL.startMiniCluster(1); }
@BeforeClass public static void setupBeforeClass() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.setBoolean("hbase.use.secondary.index", true); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); conf.set("index.data.block.encoding.algo", "PREFIX"); UTIL.startMiniCluster(1); }
@BeforeClass public static void setupBeforeClass() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); UTIL.startMiniCluster(1); }
@BeforeClass public static void setupBeforeClass() throws Exception { Configuration conf = UTIL.getConfiguration(); conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName()); conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, IndexRegionObserver.class.getName()); conf.set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY, IndexWALObserver.class.getName()); conf.setBoolean("hbase.use.secondary.index", true); conf.set("index.data.block.encoding.algo", "PREFIX"); UTIL.startMiniCluster(1); }
@After public void tearDown() throws Exception { IndexRegionObserver.setIsTestingEnabled(false); }
@Test(timeout = 180000) public void testSeparatorPartition() throws Exception { Configuration conf = UTIL.getConfiguration(); String userTableName = "testSeparatorPartition"; HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); HColumnDescriptor hcd = new HColumnDescriptor("cf1"); ihtd.addFamily(hcd); ValuePartition vp = new SeparatorPartition("_", 3); IndexSpecification iSpec = new IndexSpecification("idx1"); iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); TableIndices indices = new TableIndices(); indices.addIndex(iSpec); ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); admin.createTable(ihtd); HTable table = new HTable(conf, "testSeparatorPartition"); byte[] value1 = "2ndFloor_solitaire_huawei_bangalore_karnataka".getBytes(); Put p = new Put("row".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), value1); table.put(p); p = new Put("row2".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "7thFloor_solitaire_huawei_bangalore_karnataka".getBytes()); table.put(p); p = new Put("row3".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "rrr_sss_hhh_bangalore_karnataka".getBytes()); table.put(p); Scan scan = new Scan(); scan.setCaching(1); scan.setFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), CompareOp.EQUAL, "huawei".getBytes(), vp)); int i = 0; ResultScanner scanner = table.getScanner(scan); List<Result> testRes = new ArrayList<Result>(); Result[] result = scanner.next(1); while (result != null && result.length > 0) { testRes.add(result[0]); i++; result = scanner.next(1); } assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); assertTrue("Seekpoints should get added by index scanner", IndexRegionObserver.getSeekpointAdded()); assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver .getMultipleSeekPoints().size()); assertEquals("Overall result should have only 2 rows", 2, testRes.size()); }
@Test(timeout = 180000) public void testSpatialPartition() throws Exception { Configuration conf = UTIL.getConfiguration(); String userTableName = "testSpatialPartition"; HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); HColumnDescriptor hcd = new HColumnDescriptor("cf1"); ihtd.addFamily(hcd); ValuePartition vp = new SpatialPartition(2, 3); IndexSpecification iSpec = new IndexSpecification("idx1"); iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); TableIndices indices = new TableIndices(); indices.addIndex(iSpec); ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); admin.createTable(ihtd); HTable table = new HTable(conf, "testSpatialPartition"); byte[] value1 = "helloworld".getBytes(); Put p = new Put("row".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), value1); table.put(p); p = new Put("row2".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "spatial".getBytes()); table.put(p); p = new Put("row3".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "partition".getBytes()); table.put(p); Scan scan = new Scan(); scan.setFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), CompareOp.EQUAL, "rti".getBytes(), vp)); int i = 0; ResultScanner scanner = table.getScanner(scan); List<Result> testRes = new ArrayList<Result>(); Result[] result = scanner.next(1); while (result != null && result.length > 0) { testRes.add(result[0]); i++; result = scanner.next(1); } assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); assertTrue("Seekpoints should get added by index scanner", IndexRegionObserver.getSeekpointAdded()); assertEquals("It should get 1 seek point from index scanner.", 1, IndexRegionObserver .getMultipleSeekPoints().size()); assertEquals("Overall result should have only 1 rows", 1, testRes.size()); }
@Test(timeout = 180000) public void testSingleColumnValuePartitionFilterBySettingAsAttributeToScan() throws Exception { Configuration conf = UTIL.getConfiguration(); String userTableName = "testSingleColumnValuePartitionFilterBySettingAsAttributeToScan"; HTableDescriptor ihtd = new HTableDescriptor(TableName.valueOf(userTableName)); TableIndices indices = new TableIndices(); HColumnDescriptor hcd = new HColumnDescriptor("cf1"); ihtd.addFamily(hcd); ValuePartition vp = new SeparatorPartition("_", 3); IndexSpecification iSpec = new IndexSpecification("idx1"); iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); indices.addIndex(iSpec); ihtd.setValue(Constants.INDEX_SPEC_KEY, indices.toByteArray()); admin.createTable(ihtd); HTable table = new HTable(conf, userTableName); byte[] value1 = "2ndFloor_solitaire_huawei_bangalore_karnataka".getBytes(); Put p = new Put("row".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), value1); table.put(p); p = new Put("row2".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "7thFloor_solitaire_huawei_bangalore_karnataka".getBytes()); table.put(p); p = new Put("row3".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "rrr_sss_hhh_bangalore_karnataka".getBytes()); table.put(p); Scan scan = new Scan(); SingleIndexExpression singleIndexExpression = new SingleIndexExpression("idx1"); byte[] value = "huawei".getBytes(); Column column = new Column("cf1".getBytes(), "cq".getBytes(), vp); EqualsExpression equalsExpression = new EqualsExpression(column, value); singleIndexExpression.addEqualsExpression(equalsExpression); scan.setAttribute(Constants.INDEX_EXPRESSION, IndexUtils.toBytes(singleIndexExpression)); scan.setFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), CompareOp.EQUAL, "huawei".getBytes(), vp)); int i = 0; ResultScanner scanner = table.getScanner(scan); List<Result> testRes = new ArrayList<Result>(); Result[] result = scanner.next(1); while (result != null && result.length > 0) { testRes.add(result[0]); i++; result = scanner.next(1); } assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); assertTrue("Seekpoints should get added by index scanner", IndexRegionObserver.getSeekpointAdded()); assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver .getMultipleSeekPoints().size()); assertTrue("Overall result should have only 2 rows", testRes.size() == 2); }
@Test(timeout = 180000) public void testSeparatorPartition() throws Exception { HBaseAdmin admin = UTIL.getHBaseAdmin(); Configuration conf = UTIL.getConfiguration(); String userTableName = "testSeparatorPartition"; IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); HColumnDescriptor hcd = new HColumnDescriptor("cf1"); ihtd.addFamily(hcd); ValuePartition vp = new SeparatorPartition("_", 3); IndexSpecification iSpec = new IndexSpecification("idx1"); iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); ihtd.addIndex(iSpec); admin.createTable(ihtd); HTable table = new HTable(conf, "testSeparatorPartition"); byte[] value1 = "2ndFloor_solitaire_huawei_bangalore_karnataka".getBytes(); Put p = new Put("row".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), value1); table.put(p); p = new Put("row2".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "7thFloor_solitaire_huawei_bangalore_karnataka".getBytes()); table.put(p); p = new Put("row3".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "rrr_sss_hhh_bangalore_karnataka".getBytes()); table.put(p); Scan scan = new Scan(); scan.setFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), CompareOp.EQUAL, "huawei".getBytes(), vp)); int i = 0; ResultScanner scanner = table.getScanner(scan); List<Result> testRes = new ArrayList<Result>(); Result[] result = scanner.next(1); while (result != null && result.length > 0) { testRes.add(result[0]); i++; result = scanner.next(1); } Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); Assert.assertTrue("Seekpoints should get added by index scanner", IndexRegionObserver.getSeekpointAdded()); Assert.assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver .getMultipleSeekPoints().size()); Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 2); }
@Test(timeout = 180000) public void testSpatialPartition() throws Exception { HBaseAdmin admin = UTIL.getHBaseAdmin(); Configuration conf = UTIL.getConfiguration(); String userTableName = "testSpatialPartition"; IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); HColumnDescriptor hcd = new HColumnDescriptor("cf1"); ihtd.addFamily(hcd); ValuePartition vp = new SpatialPartition(2, 3); IndexSpecification iSpec = new IndexSpecification("idx1"); iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); ihtd.addIndex(iSpec); admin.createTable(ihtd); HTable table = new HTable(conf, "testSpatialPartition"); byte[] value1 = "helloworld".getBytes(); Put p = new Put("row".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), value1); table.put(p); p = new Put("row2".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "spatial".getBytes()); table.put(p); p = new Put("row3".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "partition".getBytes()); table.put(p); Scan scan = new Scan(); scan.setFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), CompareOp.LESS_OR_EQUAL, "rti".getBytes(), vp)); int i = 0; ResultScanner scanner = table.getScanner(scan); List<Result> testRes = new ArrayList<Result>(); Result[] result = scanner.next(1); while (result != null && result.length > 0) { testRes.add(result[0]); i++; result = scanner.next(1); } Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); Assert.assertTrue("Seekpoints should get added by index scanner", IndexRegionObserver.getSeekpointAdded()); Assert.assertEquals("It should get two seek points from index scanner.", 3, IndexRegionObserver .getMultipleSeekPoints().size()); Assert.assertTrue("Overall result should have only 1 rows", testRes.size() == 3); }
@Test(timeout = 180000) public void testSpatialPartitionIfMulitplePartsOfValueAreIndexedByDifferentIndicesOnSameColumn() throws Exception { HBaseAdmin admin = UTIL.getHBaseAdmin(); Configuration conf = UTIL.getConfiguration(); String userTableName = "testSpatialPartitionIfMulitplePartsOfValueAreIndexedByDifferentIndicesOnSameColumn"; IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); HColumnDescriptor hcd = new HColumnDescriptor("cf1"); ihtd.addFamily(hcd); ValuePartition vp = new SpatialPartition(2, 3); IndexSpecification iSpec = new IndexSpecification("idx1"); iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); ihtd.addIndex(iSpec); ValuePartition vp2 = new SpatialPartition(5, 2); iSpec = new IndexSpecification("idx2"); iSpec.addIndexColumn(hcd, "cq", vp2, ValueType.String, 200); ihtd.addIndex(iSpec); admin.createTable(ihtd); HTable table = new HTable(conf, userTableName); byte[] value1 = "helloworldmultiple".getBytes(); Put p = new Put("row".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), value1); table.put(p); p = new Put("row2".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "spatialmultiple".getBytes()); table.put(p); p = new Put("row3".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "partitionmultiple".getBytes()); table.put(p); FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), CompareOp.EQUAL, "rti".getBytes(), vp)); masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), CompareOp.GREATER_OR_EQUAL, "ti".getBytes(), vp2)); Scan scan = new Scan(); scan.setFilter(masterFilter); int i = 0; ResultScanner scanner = table.getScanner(scan); List<Result> testRes = new ArrayList<Result>(); Result[] result = scanner.next(1); while (result != null && result.length > 0) { testRes.add(result[0]); i++; result = scanner.next(1); } Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); Assert.assertTrue("Seekpoints should get added by index scanner", IndexRegionObserver.getSeekpointAdded()); Assert.assertEquals("It should get two seek points from index scanner.", 1, IndexRegionObserver .getMultipleSeekPoints().size()); Assert.assertTrue("Overall result should have only 1 rows", testRes.size() == 1); masterFilter = new FilterList(Operator.MUST_PASS_ONE); masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), CompareOp.LESS, "rti".getBytes(), vp)); masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), CompareOp.GREATER, "ti".getBytes(), vp2)); scan = new Scan(); scan.setFilter(masterFilter); i = 0; scanner = table.getScanner(scan); testRes = new ArrayList<Result>(); result = scanner.next(1); while (result != null && result.length > 0) { testRes.add(result[0]); i++; result = scanner.next(1); } Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); Assert.assertTrue("Seekpoints should get added by index scanner", IndexRegionObserver.getSeekpointAdded()); Assert.assertEquals("It should get two seek points from index scanner.", 3, IndexRegionObserver .getMultipleSeekPoints().size()); Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 2); }
@Test(timeout = 180000) public void testSeparatorPartitionIfMulitplePartsOfValueAreIndexedByDifferentIndicesOnSameColumn() throws Exception { HBaseAdmin admin = UTIL.getHBaseAdmin(); Configuration conf = UTIL.getConfiguration(); String userTableName = "testSeparatorPartitionIfMulitplePartsOfValueAreIndexedByDifferentIndicesOnSameColumn"; IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); HColumnDescriptor hcd = new HColumnDescriptor("cf1"); ihtd.addFamily(hcd); ValuePartition vp = new SeparatorPartition("--", 3); IndexSpecification iSpec = new IndexSpecification("idx1"); iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); ihtd.addIndex(iSpec); ValuePartition vp2 = new SeparatorPartition("--", 2); iSpec = new IndexSpecification("idx2"); iSpec.addIndexColumn(hcd, "cq", vp2, ValueType.String, 200); ihtd.addIndex(iSpec); admin.createTable(ihtd); HTable table = new HTable(conf, userTableName); byte[] value1 = "hello--world--multiple--1".getBytes(); Put p = new Put("row".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), value1); table.put(p); p = new Put("row2".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "spatial--partition--multiple".getBytes()); table.put(p); p = new Put("row3".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "partition--by--separator--multiple".getBytes()); table.put(p); FilterList masterFilter = new FilterList(Operator.MUST_PASS_ALL); masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), CompareOp.EQUAL, "multiple".getBytes(), vp)); masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), CompareOp.GREATER_OR_EQUAL, "by".getBytes(), vp2)); Scan scan = new Scan(); scan.setFilter(masterFilter); int i = 0; ResultScanner scanner = table.getScanner(scan); List<Result> testRes = new ArrayList<Result>(); Result[] result = scanner.next(1); while (result != null && result.length > 0) { testRes.add(result[0]); i++; result = scanner.next(1); } Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); Assert.assertTrue("Seekpoints should get added by index scanner", IndexRegionObserver.getSeekpointAdded()); Assert.assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver .getMultipleSeekPoints().size()); Assert.assertTrue("Overall result should have only 1 rows", testRes.size() == 2); masterFilter = new FilterList(Operator.MUST_PASS_ONE); masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), CompareOp.GREATER_OR_EQUAL, "person".getBytes(), vp)); masterFilter.addFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), CompareOp.LESS, "multiple".getBytes(), vp2)); scan = new Scan(); scan.setFilter(masterFilter); i = 0; scanner = table.getScanner(scan); testRes = new ArrayList<Result>(); result = scanner.next(1); while (result != null && result.length > 0) { testRes.add(result[0]); i++; result = scanner.next(1); } Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); Assert.assertTrue("Seekpoints should get added by index scanner", IndexRegionObserver.getSeekpointAdded()); Assert.assertEquals("It should get two seek points from index scanner.", 3, IndexRegionObserver .getMultipleSeekPoints().size()); Assert.assertTrue("Overall result should have only 1 rows", testRes.size() == 1); }
@Test(timeout = 180000) public void testSingleColumnValuePartitionFilterBySettingAsAttributeToScan() throws Exception { HBaseAdmin admin = UTIL.getHBaseAdmin(); Configuration conf = UTIL.getConfiguration(); String userTableName = "testSingleColumnValuePartitionFilterBySettingAsAttributeToScan"; IndexedHTableDescriptor ihtd = new IndexedHTableDescriptor(userTableName); HColumnDescriptor hcd = new HColumnDescriptor("cf1"); ihtd.addFamily(hcd); ValuePartition vp = new SeparatorPartition("_", 3); IndexSpecification iSpec = new IndexSpecification("idx1"); iSpec.addIndexColumn(hcd, "cq", vp, ValueType.String, 200); ihtd.addIndex(iSpec); admin.createTable(ihtd); HTable table = new HTable(conf, userTableName); byte[] value1 = "2ndFloor_solitaire_huawei_bangalore_karnataka".getBytes(); Put p = new Put("row".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), value1); table.put(p); p = new Put("row2".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "7thFloor_solitaire_huawei_bangalore_karnataka".getBytes()); table.put(p); p = new Put("row3".getBytes()); p.add("cf1".getBytes(), "cq".getBytes(), "rrr_sss_hhh_bangalore_karnataka".getBytes()); table.put(p); Scan scan = new Scan(); SingleIndexExpression singleIndexExpression = new SingleIndexExpression("idx1"); byte[] value = "huawei".getBytes(); Column column = new Column("cf1".getBytes(), "cq".getBytes(), vp); EqualsExpression equalsExpression = new EqualsExpression(column, value); singleIndexExpression.addEqualsExpression(equalsExpression); scan.setAttribute(Constants.INDEX_EXPRESSION, IndexUtils.toBytes(singleIndexExpression)); scan.setFilter(new SingleColumnValuePartitionFilter(hcd.getName(), "cq".getBytes(), CompareOp.EQUAL, "huawei".getBytes(), vp)); int i = 0; ResultScanner scanner = table.getScanner(scan); List<Result> testRes = new ArrayList<Result>(); Result[] result = scanner.next(1); while (result != null && result.length > 0) { testRes.add(result[0]); i++; result = scanner.next(1); } Assert.assertTrue("Index flow should get used.", IndexRegionObserver.getIndexedFlowUsed()); Assert.assertTrue("Seekpoints should get added by index scanner", IndexRegionObserver.getSeekpointAdded()); Assert.assertEquals("It should get two seek points from index scanner.", 2, IndexRegionObserver .getMultipleSeekPoints().size()); Assert.assertTrue("Overall result should have only 2 rows", testRes.size() == 2); }