@Test public void testFirstKeyFilter() throws IOException { // Initialize int numCols = 5; String columnValue = "includeThisValue"; Table table = getConnection().getTable(TABLE_NAME); byte[] rowKey = dataHelper.randomData("testRow-"); Put put = new Put(rowKey); for (int i = 0; i < numCols; ++i) { put.addColumn(COLUMN_FAMILY, dataHelper.randomData(""), Bytes.toBytes(columnValue)); } table.put(put); // Filter for results Filter filter = new FirstKeyOnlyFilter(); Get get = new Get(rowKey).setFilter(filter); Result result = table.get(get); Assert.assertEquals("Should only return 1 keyvalue", 1, result.size()); table.close(); }
/** * Gets the whole set of keys in a table * @return The list of keys (byte[]) as objects */ public List<Object> getKeys() { List<Object> keys = new ArrayList<Object>(); try { Scan scan = new Scan(); scan.setFilter(new FirstKeyOnlyFilter()); ResultScanner scanner = table.getScanner(scan); for (Result rr : scanner) { byte[] key = rr.getRow(); keys.add(key); } return keys; } catch (IOException e) { e.printStackTrace(); return keys; } }
public static boolean isReallyEmptyRegion(HConnection connection, String tableName, HRegionInfo regionInfo) throws IOException { boolean emptyRegion = false; // verify really empty region by scanning records try (HTableInterface table = connection.getTable(tableName)) { Scan scan = new Scan(regionInfo.getStartKey(), regionInfo.getEndKey()); FilterList filterList = new FilterList(); filterList.addFilter(new KeyOnlyFilter()); filterList.addFilter(new FirstKeyOnlyFilter()); scan.setFilter(filterList); scan.setCacheBlocks(false); scan.setSmall(true); scan.setCaching(1); try (ResultScanner scanner = table.getScanner(scan)) { if (scanner.next() == null) emptyRegion = true; } } return emptyRegion; }
@Override public void run() { try (HTableInterface table = connection.getTable(tableName.getBytes())) { // Do not use Get not to increase read request count metric. // Use Scan. Scan scan = new Scan("".getBytes(), "".getBytes()); FilterList filterList = new FilterList(); filterList.addFilter(new KeyOnlyFilter()); filterList.addFilter(new FirstKeyOnlyFilter()); scan.setFilter(filterList); //noinspection EmptyTryBlock try(ResultScanner ignored = table.getScanner(scan)) { } return; } catch (IOException ignore) { } clean(tableName); }
/** * Tries to scan a row from passed region * @param admin * @param region * @throws IOException */ private void isSuccessfulScan(Admin admin, RegionInfo region) throws IOException { Scan scan = new Scan(region.getStartKey()); scan.setBatch(1); scan.setCaching(1); scan.setFilter(new FirstKeyOnlyFilter()); try { Table table = admin.getConnection().getTable(region.getTable()); try { ResultScanner scanner = table.getScanner(scan); try { scanner.next(); } finally { scanner.close(); } } finally { table.close(); } } catch (IOException e) { LOG.error("Could not scan region:" + region.getEncodedName(), e); throw e; } }
public static void count() { long begin = System.currentTimeMillis(); AggregationClient ac = new AggregationClient(conf); Scan scan = new Scan(); // scan.setStartRow(Bytes.toBytes("3")); // scan.addColumn(Bytes.toBytes("fal"), Bytes.toBytes("val")); scan.addFamily(Bytes.toBytes("fal")); scan.setFilter(new FirstKeyOnlyFilter()); long rowCount = 0; try { rowCount = ac.rowCount(Bytes.toBytes("test3"), new LongColumnInterpreter(), scan); // rowCount = ac.max(Bytes.toBytes("test"), new LongColumnInterpreter(), scan); System.out.println(rowCount); } catch (Throwable e) { e.printStackTrace(); } long end = System.currentTimeMillis(); System.out.println(end - begin); }
private LoopRowKeysStrategy getAllDataRowKeys() throws IOException { LoopRowKeysStrategy strategy = null; HTable table = null; ResultScanner rs = null; try { table = new HTable(this.getConf(), this.vertexTableName); Scan scan = new Scan(); scan.setFilter(new FirstKeyOnlyFilter()); rs = table.getScanner(scan); strategy = new GetAllRowKeysStrategy(table, rs); } catch (IOException e) { LOG.error("getSampleDataRowKey failed", e); throw e; } return strategy; }
@Override protected Collection<SchemaPath> transformColumns(Collection<SchemaPath> columns) { Set<SchemaPath> transformed = Sets.newLinkedHashSet(); rowKeyOnly = true; if (!isStarQuery()) { for (SchemaPath column : columns) { if (column.getRootSegment().getPath().equalsIgnoreCase(ROW_KEY)) { transformed.add(ROW_KEY_PATH); continue; } rowKeyOnly = false; NameSegment root = column.getRootSegment(); byte[] family = root.getPath().getBytes(); transformed.add(SchemaPath.getSimplePath(root.getPath())); PathSegment child = root.getChild(); if (child != null && child.isNamed()) { byte[] qualifier = child.getNameSegment().getPath().getBytes(); hbaseScan.addColumn(family, qualifier); } else { hbaseScan.addFamily(family); } } /* if only the row key was requested, add a FirstKeyOnlyFilter to the scan * to fetch only one KV from each row. If a filter is already part of this * scan, add the FirstKeyOnlyFilter as the LAST filter of a MUST_PASS_ALL * FilterList. */ if (rowKeyOnly) { hbaseScan.setFilter( HBaseUtils.andFilterAtIndex(hbaseScan.getFilter(), HBaseUtils.LAST_FILTER, new FirstKeyOnlyFilter())); } } else { rowKeyOnly = false; transformed.add(ROW_KEY_PATH); } return transformed; }
@Test public void testFilterAllRecords() throws IOException { Scan scan = new Scan(); scan.setBatch(1); scan.setCaching(1); // Filter out any records scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new InclusiveStopFilter(new byte[0]))); Table table = TEST_UTIL.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME); ResultScanner s = table.getScanner(scan); assertNull(s.next()); table.close(); }
@Test public void testReverseScanWithoutPadding() throws Exception { byte[] row1 = Bytes.toBytes("a"); byte[] row2 = Bytes.toBytes("ab"); byte[] row3 = Bytes.toBytes("b"); Put put1 = new Put(row1); put1.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); Put put2 = new Put(row2); put2.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); Put put3 = new Put(row3); put3.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); region.put(put1); region.put(put2); region.put(put3); region.flush(true); Scan scan = new Scan(); scan.setCacheBlocks(false); scan.setReversed(true); scan.setFilter(new FirstKeyOnlyFilter()); scan.addFamily(cfName); RegionScanner scanner = region.getScanner(scan); List<Cell> res = new ArrayList<Cell>(); int count = 1; while (scanner.next(res)) { count++; } assertEquals(Bytes.toString(res.get(0).getRowArray(), res.get(0).getRowOffset(), res.get(0) .getRowLength()), "b"); assertEquals(Bytes.toString(res.get(1).getRowArray(), res.get(1).getRowOffset(), res.get(1) .getRowLength()), "ab"); assertEquals(Bytes.toString(res.get(2).getRowArray(), res.get(2).getRowOffset(), res.get(2) .getRowLength()), "a"); assertEquals(3, count); }
@Test public void testReverseScanWithPadding() throws Exception { byte[] terminator = new byte[] { -1 }; byte[] row1 = Bytes.add(invert(Bytes.toBytes("a")), terminator); byte[] row2 = Bytes.add(invert(Bytes.toBytes("ab")), terminator); byte[] row3 = Bytes.add(invert(Bytes.toBytes("b")), terminator); Put put1 = new Put(row1); put1.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); Put put2 = new Put(row2); put2.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); Put put3 = new Put(row3); put3.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); region.put(put1); region.put(put2); region.put(put3); region.flush(true); Scan scan = new Scan(); scan.setCacheBlocks(false); scan.setReversed(true); scan.setFilter(new FirstKeyOnlyFilter()); scan.addFamily(cfName); RegionScanner scanner = region.getScanner(scan); List<Cell> res = new ArrayList<Cell>(); int count = 1; while (scanner.next(res)) { count++; } assertEquals(3, count); }
/** * Test partial Result re-assembly in the presence of different filters. The Results from the * partial scanner should match the Results returned from a scanner that receives all of the * results in one RPC to the server. The partial scanner is tested with a variety of different * result sizes (all of which are less than the size necessary to fetch an entire row) * @throws Exception */ @Test public void testPartialResultsWithColumnFilter() throws Exception { testPartialResultsWithColumnFilter(new FirstKeyOnlyFilter()); testPartialResultsWithColumnFilter(new ColumnPrefixFilter(Bytes.toBytes("testQualifier5"))); testPartialResultsWithColumnFilter(new ColumnRangeFilter(Bytes.toBytes("testQualifer1"), true, Bytes.toBytes("testQualifier7"), true)); Set<byte[]> qualifiers = new LinkedHashSet<>(); qualifiers.add(Bytes.toBytes("testQualifier5")); testPartialResultsWithColumnFilter(new FirstKeyValueMatchingQualifiersFilter(qualifiers)); }
@Test public void testFirstKeyOnlyFilter() throws Exception { Scan s = new Scan(); s.setFilter(new FirstKeyOnlyFilter()); // Expected KVs, the first KV from each of the remaining 6 rows KeyValue [] kvs = { new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]) }; verifyScanFull(s, kvs); }
/** * Returns a count of the rows in the region where this coprocessor is loaded. */ @Override public void getRowCount(RpcController controller, ExampleProtos.CountRequest request, RpcCallback<ExampleProtos.CountResponse> done) { Scan scan = new Scan(); scan.setFilter(new FirstKeyOnlyFilter()); ExampleProtos.CountResponse response = null; InternalScanner scanner = null; try { scanner = env.getRegion().getScanner(scan); List<Cell> results = new ArrayList<Cell>(); boolean hasMore = false; byte[] lastRow = null; long count = 0; do { hasMore = scanner.next(results); for (Cell kv : results) { byte[] currentRow = CellUtil.cloneRow(kv); if (lastRow == null || !Bytes.equals(lastRow, currentRow)) { lastRow = currentRow; count++; } } results.clear(); } while (hasMore); response = ExampleProtos.CountResponse.newBuilder() .setCount(count).build(); } catch (IOException ioe) { ResponseConverter.setControllerException(controller, ioe); } finally { if (scanner != null) { try { scanner.close(); } catch (IOException ignored) {} } } done.run(response); }
List<WorkflowInstanceExecutionData> executionData(WorkflowId workflowId, String offset, int limit) throws IOException { try (final Table eventsTable = connection.getTable(EVENTS_TABLE_NAME)) { final Scan scan = new Scan() .setRowPrefixFilter(Bytes.toBytes(workflowId.toKey() + '#')) .setFilter(new FirstKeyOnlyFilter()); if (!Strings.isNullOrEmpty(offset)) { final WorkflowInstance offsetInstance = WorkflowInstance.create(workflowId, offset); scan.setStartRow(Bytes.toBytes(offsetInstance.toKey() + '#')); } final Set<WorkflowInstance> workflowInstancesSet = Sets.newHashSet(); try (ResultScanner scanner = eventsTable.getScanner(scan)) { Result result = scanner.next(); while (result != null) { final String key = new String(result.getRow()); final int lastHash = key.lastIndexOf('#'); final WorkflowInstance wfi = WorkflowInstance.parseKey(key.substring(0, lastHash)); workflowInstancesSet.add(wfi); if (workflowInstancesSet.size() == limit) { break; } result = scanner.next(); } } return executionData(workflowInstancesSet); } }
List<WorkflowInstanceExecutionData> executionData(WorkflowId workflowId, String start, String stop) throws IOException { try (final Table eventsTable = connection.getTable(EVENTS_TABLE_NAME)) { final Scan scan = new Scan() .setRowPrefixFilter(Bytes.toBytes(workflowId.toKey() + '#')) .setFilter(new FirstKeyOnlyFilter()); final WorkflowInstance startRow = WorkflowInstance.create(workflowId, start); scan.setStartRow(Bytes.toBytes(startRow.toKey() + '#')); if (!Strings.isNullOrEmpty(stop)) { final WorkflowInstance stopRow = WorkflowInstance.create(workflowId, stop); scan.setStopRow(Bytes.toBytes(stopRow.toKey() + '#')); } final Set<WorkflowInstance> workflowInstancesSet = Sets.newHashSet(); try (ResultScanner scanner = eventsTable.getScanner(scan)) { Result result = scanner.next(); while (result != null) { final String key = new String(result.getRow()); final int lastHash = key.lastIndexOf('#'); final WorkflowInstance wfi = WorkflowInstance.parseKey(key.substring(0, lastHash)); workflowInstancesSet.add(wfi); result = scanner.next(); } } return executionData(workflowInstancesSet); } }
public void run(){ finished=false; running=true; try { HTable table=new HTable(conf,tableName); Scan scan=new Scan(); scan.setCacheBlocks(false); scan.setMaxVersions(1); scan.setCaching(1000); scan.setStartRow(region.getStartKey()); scan.setStopRow(region.getEndKey()); FilterList flist=new FilterList(); flist.addFilter(new KeyOnlyFilter()); flist.addFilter(new FirstKeyOnlyFilter()); scan.setFilter(flist); ResultScanner rs=table.getScanner(scan); while((rs.next())!=null){ count++; } } catch (IOException e) { e.printStackTrace(); }finally{ finished=true; running=false; } }
@Override public <T, S> long getRowNum(ColumnInterpreter<T, S> ci, Scan scan) throws IOException { long counter = 0l; List<KeyValue> results = new ArrayList<KeyValue>(); byte[] colFamily = scan.getFamilies()[0]; byte[] qualifier = scan.getFamilyMap().get(colFamily).pollFirst(); if (scan.getFilter() == null && qualifier == null) scan.setFilter(new FirstKeyOnlyFilter()); InternalScanner scanner = ((RegionCoprocessorEnvironment) getEnvironment()) .getRegion().getScanner(scan); try { boolean hasMoreRows = false; do { hasMoreRows = scanner.next(results); if (results.size() > 0) { counter++; } results.clear(); } while (hasMoreRows); } finally { scanner.close(); } log.info("Row counter from this region is " + ((RegionCoprocessorEnvironment) getEnvironment()).getRegion() .getRegionNameAsString() + ": " + counter); return counter; }
@Override public RowFilter adapt(FilterAdapterContext context, FirstKeyOnlyFilter filter) throws IOException { return RowFilter.newBuilder() .setCellsPerRowLimitFilter(1) .build(); }
@Test public void onlyTheFirstKeyFromEachRowIsEmitted() throws IOException { RowFilter adaptedFilter = adapter.adapt( new FilterAdapterContext(new Scan()), new FirstKeyOnlyFilter()); Assert.assertEquals( RowFilter.newBuilder() .setCellsPerRowLimitFilter(1) .build(), adaptedFilter); }
@Test(timeout = 180000) public void testScanMultipleIdxWithDifferentFiltersShouldBeSuccessfulAndShouldNotGoWithIndexedFlow() throws Exception { Configuration conf = UTIL.getConfiguration(); String userTableName = "testScanWithMultIndexedDiffFilters"; putMulIndex(userTableName); HTable table = new HTable(conf, userTableName); int i = 0; Scan s = new Scan(); FilterList filterList = new FilterList(); // check for combination of cat in q1 and dog in q1 Filter filter1 = new RowFilter(CompareOp.LESS_OR_EQUAL, new BinaryComparator("row5".getBytes())); Filter filter2 = new FirstKeyOnlyFilter(); filterList.addFilter(filter1); filterList.addFilter(filter2); s.setFilter(filterList); ResultScanner scanner = table.getScanner(s); for (Result result : scanner) { i++; } Assert.assertEquals( "Should match for 5 rows in multiple index with diff column family successfully ", 5, i); Assert.assertFalse("Seek points should not be added ", IndexRegionObserver.getSeekpointAdded()); Assert.assertFalse("Indexed table should not be used ", IndexRegionObserver.getIndexedFlowUsed()); }
@Test public void deleteMutiple() throws Exception { String TABLE_NAME = "student"; HTableInterface table = htablePool.getTable(TABLE_NAME);// TableNotFoundException // filterList FilterList filterList = new FilterList( FilterList.Operator.MUST_PASS_ALL); FirstKeyOnlyFilter firstKeyFilter = new FirstKeyOnlyFilter(); filterList.addFilter(firstKeyFilter); Scan scan = new Scan(); scan.setCaching(200); // scan.setFilter(filterList); long beg = System.currentTimeMillis(); int count = 100; ResultScanner scanner = table.getScanner(scan); int rowCount = 0; for (Result result : scanner) { byte[] row = result.getRow(); Delete delete = new Delete(row); table.delete(delete); // rowCount++; if (rowCount >= count) { break; } } scanner.close();// 一定要關閉 System.out.println("rowCount: " + rowCount); long end = System.currentTimeMillis(); System.out.println((end - beg) + " at mills."); }
/** * 只傳回第一個keyValue * * @throws Exception */ @Test // rowCount: 8321 // 1152 at mills. public void firstKeyOnlyFilter() throws Exception { String TABLE_NAME = "student"; // HTableInterface table = htablePool.getTable(TABLE_NAME);// TableNotFoundException // filterList FilterList filterList = new FilterList( FilterList.Operator.MUST_PASS_ALL); // filter FirstKeyOnlyFilter firstKeyFilter = new FirstKeyOnlyFilter(); filterList.addFilter(firstKeyFilter); Scan scan = new Scan(); scan.setCaching(200); // scan.setFilter(filterList); // long beg = System.currentTimeMillis(); ResultScanner scanner = table.getScanner(scan); int rowCount = 0; for (Result result : scanner) { // printlnResult(result); // rowCount++; // if (rowCount > 9) { break; } } scanner.close();// 一定要關閉 long end = System.currentTimeMillis(); System.out.println((end - beg) + " at mills."); System.out.println("rowCount: " + rowCount); }
@Test // 7487 at mills. // 7461 at mills. // 7492 at mills. public void rowCountByKeyFilter() throws Exception { String TABLE_NAME = "student"; // long beg = System.currentTimeMillis(); HTableInterface table = htablePool.getTable(TABLE_NAME);// TableNotFoundException // filterList FilterList filterList = new FilterList( FilterList.Operator.MUST_PASS_ALL); // filter FirstKeyOnlyFilter firstKeyFilter = new FirstKeyOnlyFilter(); filterList.addFilter(firstKeyFilter); // filter KeyOnlyFilter keyOnlyFilter = new KeyOnlyFilter(); filterList.addFilter(keyOnlyFilter); int rowCount = 0; Scan scan = new Scan(); // scan.setCaching(200); scan.setFilter(filterList); ResultScanner scanner = table.getScanner(scan); for (Result result : scanner) { // printlnResult(result); rowCount++; } scanner.close();// 一定要關閉 long end = System.currentTimeMillis(); System.out.println((end - beg) + " at mills."); System.out.println("rowCount: " + rowCount); }
protected Job createSubmittableJob(Configuration conf, String[] args) throws IOException { String[] tableNames = args[0].split(":"); StringBuilder sb = new StringBuilder(); for (int i = 1; i < args.length; i++) { sb.append(args[i]); sb.append(" "); } Job job = new Job(conf, "MultiThemisTableRowCounter_" + args[0]); job.setJarByClass(RowCounter.class); Scan scan = new Scan(); scan.setCacheBlocks(false); scan.setFilter(new FirstKeyOnlyFilter()); if (sb.length() > 0) { for (String columnName : sb.toString().trim().split(" ")) { String family = StringUtils.substringBefore(columnName, ":"); String qualifier = StringUtils.substringAfter(columnName, ":"); if (StringUtils.isBlank(qualifier)) { throw new IOException("must specify qualifier to read themis table"); } else { scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier)); } } } List<Scan> scans = new ArrayList<Scan>(); for (String tableName : tableNames) { Scan thisScan = new Scan(scan); thisScan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(tableName)); scans.add(thisScan); } job.setOutputFormatClass(NullOutputFormat.class); ThemisTableMapReduceUtil.initTableMapperJob(scans, RowCounterMapper.class, ImmutableBytesWritable.class, Result.class, job); job.setNumReduceTasks(0); return job; }
@Test public void testFilterAllRecords() throws IOException { Scan scan = new Scan(); scan.setBatch(1); scan.setCaching(1); // Filter out any records scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new InclusiveStopFilter(new byte[0]))); try (Table table = TEST_UTIL.getConnection().getTable(TableName.NAMESPACE_TABLE_NAME)) { try (ResultScanner s = table.getScanner(scan)) { assertNull(s.next()); } } }
@Test public void testReverseScanWithoutPadding() throws Exception { byte[] row1 = Bytes.toBytes("a"); byte[] row2 = Bytes.toBytes("ab"); byte[] row3 = Bytes.toBytes("b"); Put put1 = new Put(row1); put1.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); Put put2 = new Put(row2); put2.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); Put put3 = new Put(row3); put3.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); region.put(put1); region.put(put2); region.put(put3); region.flush(true); Scan scan = new Scan(); scan.setCacheBlocks(false); scan.setReversed(true); scan.setFilter(new FirstKeyOnlyFilter()); scan.addFamily(cfName); RegionScanner scanner = region.getScanner(scan); List<Cell> res = new ArrayList<>(); int count = 1; while (scanner.next(res)) { count++; } assertEquals("b", Bytes.toString(res.get(0).getRowArray(), res.get(0).getRowOffset(), res.get(0).getRowLength())); assertEquals("ab", Bytes.toString(res.get(1).getRowArray(), res.get(1).getRowOffset(), res.get(1).getRowLength())); assertEquals("a", Bytes.toString(res.get(2).getRowArray(), res.get(2).getRowOffset(), res.get(2).getRowLength())); assertEquals(3, count); }
