@Override public RowFilter adapt(FilterAdapterContext context, TimestampsFilter filter) { Interleave.Builder interleaveBuilder = RowFilter.Interleave.newBuilder(); for (long timestamp : filter.getTimestamps()) { // HBase TimestampsFilters are of the form: [N, M], however; bigtable // uses [N, M) to express timestamp ranges. In order to express an HBase // single point timestamp [M, M], we need to specify [M, M+1) to bigtable. long bigtableStartTimestamp = BigtableConstants.BIGTABLE_TIMEUNIT.convert( timestamp, BigtableConstants.HBASE_TIMEUNIT); long bigtableEndTimestamp = BigtableConstants.BIGTABLE_TIMEUNIT.convert( timestamp + 1, BigtableConstants.HBASE_TIMEUNIT); interleaveBuilder.addFilters( RowFilter.newBuilder() .setTimestampRangeFilter( TimestampRange.newBuilder() .setStartTimestampMicros(bigtableStartTimestamp) .setEndTimestampMicros(bigtableEndTimestamp))); } return RowFilter.newBuilder().setInterleave(interleaveBuilder).build(); }
@Test public void timestampFiltersAreAdapted() { // Timestamps are sorted by the filter min -> max TimestampsFilter filter = new TimestampsFilter(ImmutableList.of(1L, 10L, 20L)); RowFilter rowFilter = filterAdapter.adapt(emptyScanContext, filter); Assert.assertEquals(3, rowFilter.getInterleave().getFiltersCount()); Assert.assertEquals( 10000L, rowFilter .getInterleave() .getFilters(1) .getTimestampRangeFilter() .getStartTimestampMicros()); Assert.assertEquals( 11000L, rowFilter .getInterleave() .getFilters(1) .getTimestampRangeFilter() .getEndTimestampMicros()); }
@Test public void testGetSeek() throws IOException { StoreFileScanner.instrument(); prepareRegion(); Get g = new Get(RK_BYTES); final TimestampsFilter timestampsFilter = new TimestampsFilter(ImmutableList.of(5L), true); g.setFilter(timestampsFilter); final long initialSeekCount = StoreFileScanner.getSeekCount(); region.get(g); final long finalSeekCount = StoreFileScanner.getSeekCount(); /* Make sure there's more than one. Aka one seek to get to the row, and one to get to the time. */ assertTrue(finalSeekCount >= initialSeekCount + 3 ); }
/** * Uses the TimestampFilter on a Get to request a specified list of * versions for the row/column specified by rowIdx & colIdx. * */ private Cell[] getNVersions(Table ht, byte[] cf, int rowIdx, int colIdx, List<Long> versions) throws IOException { byte row[] = Bytes.toBytes("row:" + rowIdx); byte column[] = Bytes.toBytes("column:" + colIdx); Filter filter = new TimestampsFilter(versions); Get get = new Get(row); get.addColumn(cf, column); get.setFilter(filter); get.setMaxVersions(); Result result = ht.get(get); return result.rawCells(); }
/** * Uses the TimestampFilter on a Scan to request a specified list of * versions for the rows from startRowIdx to endRowIdx (both inclusive). */ private Result[] scanNVersions(Table ht, byte[] cf, int startRowIdx, int endRowIdx, List<Long> versions) throws IOException { byte startRow[] = Bytes.toBytes("row:" + startRowIdx); byte endRow[] = Bytes.toBytes("row:" + endRowIdx + 1); // exclusive Filter filter = new TimestampsFilter(versions); Scan scan = new Scan(startRow, endRow); scan.setFilter(filter); scan.setMaxVersions(); ResultScanner scanner = ht.getScanner(scan); return scanner.next(endRowIdx - startRowIdx + 1); }
/** * Uses the TimestampFilter on a Get to request a specified list of * versions for the row/column specified by rowIdx & colIdx. * */ private KeyValue[] getNVersions(HTable ht, byte[] cf, int rowIdx, int colIdx, List<Long> versions) throws IOException { byte row[] = Bytes.toBytes("row:" + rowIdx); byte column[] = Bytes.toBytes("column:" + colIdx); Filter filter = new TimestampsFilter(versions); Get get = new Get(row); get.addColumn(cf, column); get.setFilter(filter); get.setMaxVersions(); Result result = ht.get(get); return result.raw(); }
/** * Uses the TimestampFilter on a Scan to request a specified list of * versions for the rows from startRowIdx to endRowIdx (both inclusive). */ private Result[] scanNVersions(HTable ht, byte[] cf, int startRowIdx, int endRowIdx, List<Long> versions) throws IOException { byte startRow[] = Bytes.toBytes("row:" + startRowIdx); byte endRow[] = Bytes.toBytes("row:" + endRowIdx + 1); // exclusive Filter filter = new TimestampsFilter(versions); Scan scan = new Scan(startRow, endRow); scan.setFilter(filter); scan.setMaxVersions(); ResultScanner scanner = ht.getScanner(scan); return scanner.next(endRowIdx - startRowIdx + 1); }
@Test public void testTimestampsFilter() throws IOException { // Initialize int numCols = 5; String goodValue = "includeThisValue"; Table table = getConnection().getTable(TABLE_NAME); byte[] rowKey = dataHelper.randomData("testRow-"); Put put = new Put(rowKey); for (int i = 0; i < numCols; ++i) { put.addColumn(COLUMN_FAMILY, dataHelper.randomData(""), i, Bytes.toBytes(goodValue)); } table.put(put); // Filter for results Filter filter = new TimestampsFilter(ImmutableList.<Long>of(0L, 1L)); Get get = new Get(rowKey).setFilter(filter); Result result = table.get(get); Cell[] cells = result.rawCells(); Assert.assertEquals("Should have two cells, timestamps 0 and 1.", 2, cells.length); // Since the qualifiers are random, ignore the order of the returned cells. long[] timestamps = new long[]{cells[0].getTimestamp(), cells[1].getTimestamp()}; Arrays.sort(timestamps); Assert.assertArrayEquals(new long[]{0L, 1L}, timestamps); table.close(); }
/** * Uses the TimestampFilter on a Get to request a specified list of * versions for the row/column specified by rowIdx & colIdx. * */ private Cell[] getNVersions(HTable ht, byte[] cf, int rowIdx, int colIdx, List<Long> versions) throws IOException { byte row[] = Bytes.toBytes("row:" + rowIdx); byte column[] = Bytes.toBytes("column:" + colIdx); Filter filter = new TimestampsFilter(versions); Get get = new Get(row); get.addColumn(cf, column); get.setFilter(filter); get.setMaxVersions(); Result result = ht.get(get); return result.rawCells(); }
@Test public void testGetDoesntSeekWithNoHint() throws IOException { StoreFileScanner.instrument(); prepareRegion(); Get g = new Get(RK_BYTES); g.setFilter(new TimestampsFilter(ImmutableList.of(5L))); final long initialSeekCount = StoreFileScanner.getSeekCount(); region.get(g); final long finalSeekCount = StoreFileScanner.getSeekCount(); assertTrue(finalSeekCount >= initialSeekCount ); assertTrue(finalSeekCount < initialSeekCount + 3); }
void addFilterByMapping( FilterList fl, CompareFilter.CompareOp comp, Class<?> comparatorClass, Object comparator, Mapping.TupleMapping tupleMapping ) throws NoSuchMethodException, InstantiationException, IllegalAccessException, java.lang.reflect.InvocationTargetException { switch ( tupleMapping ) { case KEY: { addFilter( RowFilter.class, fl, comp, comparatorClass, comparator ); return; } case FAMILY: { addFilter( FamilyFilter.class, fl, comp, comparatorClass, comparator ); return; } case COLUMN: { //TODO Check if ColumnPrefixFilter works faster and suit more addFilter( QualifierFilter.class, fl, comp, comparatorClass, comparator ); return; } case VALUE: { addFilter( ValueFilter.class, fl, comp, comparatorClass, comparator ); return; } case TIMESTAMP: { addFilter( TimestampsFilter.class, fl, comp, comparatorClass, comparator ); // Constructor<TimestampsFilter> columnFilterConstructor = // TimestampsFilter.class.getConstructor( CompareFilter.CompareOp.class, comparatorClass ); // TimestampsFilter scf = columnFilterConstructor.newInstance( comp, comparator ); // fl.addFilter( scf ); return; } } }
@Test public void testMultiColumns() throws Exception { byte [] TABLE = Bytes.toBytes("testTimestampsFilterMultiColumns"); byte [] FAMILY = Bytes.toBytes("event_log"); byte [][] FAMILIES = new byte[][] { FAMILY }; // create table; set versions to max... Table ht = TEST_UTIL.createTable(TableName.valueOf(TABLE), FAMILIES, Integer.MAX_VALUE); Put p = new Put(Bytes.toBytes("row")); p.add(FAMILY, Bytes.toBytes("column0"), 3, Bytes.toBytes("value0-3")); p.add(FAMILY, Bytes.toBytes("column1"), 3, Bytes.toBytes("value1-3")); p.add(FAMILY, Bytes.toBytes("column2"), 1, Bytes.toBytes("value2-1")); p.add(FAMILY, Bytes.toBytes("column2"), 2, Bytes.toBytes("value2-2")); p.add(FAMILY, Bytes.toBytes("column2"), 3, Bytes.toBytes("value2-3")); p.add(FAMILY, Bytes.toBytes("column3"), 2, Bytes.toBytes("value3-2")); p.add(FAMILY, Bytes.toBytes("column4"), 1, Bytes.toBytes("value4-1")); p.add(FAMILY, Bytes.toBytes("column4"), 2, Bytes.toBytes("value4-2")); p.add(FAMILY, Bytes.toBytes("column4"), 3, Bytes.toBytes("value4-3")); ht.put(p); ArrayList<Long> timestamps = new ArrayList<Long>(); timestamps.add(new Long(3)); TimestampsFilter filter = new TimestampsFilter(timestamps); Get g = new Get(Bytes.toBytes("row")); g.setFilter(filter); g.setMaxVersions(); g.addColumn(FAMILY, Bytes.toBytes("column2")); g.addColumn(FAMILY, Bytes.toBytes("column4")); Result result = ht.get(g); for (Cell kv : result.listCells()) { System.out.println("found row " + Bytes.toString(CellUtil.cloneRow(kv)) + ", column " + Bytes.toString(CellUtil.cloneQualifier(kv)) + ", value " + Bytes.toString(CellUtil.cloneValue(kv))); } assertEquals(result.listCells().size(), 2); assertTrue(CellUtil.matchingValue(result.listCells().get(0), Bytes.toBytes("value2-3"))); assertTrue(CellUtil.matchingValue(result.listCells().get(1), Bytes.toBytes("value4-3"))); ht.close(); }
@Test public void testMultiColumns() throws Exception { byte [] TABLE = Bytes.toBytes("testTimestampsFilterMultiColumns"); byte [] FAMILY = Bytes.toBytes("event_log"); byte [][] FAMILIES = new byte[][] { FAMILY }; KeyValue kvs[]; // create table; set versions to max... HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE); Put p = new Put(Bytes.toBytes("row")); p.add(FAMILY, Bytes.toBytes("column0"), 3, Bytes.toBytes("value0-3")); p.add(FAMILY, Bytes.toBytes("column1"), 3, Bytes.toBytes("value1-3")); p.add(FAMILY, Bytes.toBytes("column2"), 1, Bytes.toBytes("value2-1")); p.add(FAMILY, Bytes.toBytes("column2"), 2, Bytes.toBytes("value2-2")); p.add(FAMILY, Bytes.toBytes("column2"), 3, Bytes.toBytes("value2-3")); p.add(FAMILY, Bytes.toBytes("column3"), 2, Bytes.toBytes("value3-2")); p.add(FAMILY, Bytes.toBytes("column4"), 1, Bytes.toBytes("value4-1")); p.add(FAMILY, Bytes.toBytes("column4"), 2, Bytes.toBytes("value4-2")); p.add(FAMILY, Bytes.toBytes("column4"), 3, Bytes.toBytes("value4-3")); ht.put(p); ArrayList timestamps = new ArrayList(); timestamps.add(new Long(3)); TimestampsFilter filter = new TimestampsFilter(timestamps); Get g = new Get(Bytes.toBytes("row")); g.setFilter(filter); g.setMaxVersions(); g.addColumn(FAMILY, Bytes.toBytes("column2")); g.addColumn(FAMILY, Bytes.toBytes("column4")); Result result = ht.get(g); for (KeyValue kv : result.list()) { System.out.println("found row " + Bytes.toString(kv.getRow()) + ", column " + Bytes.toString(kv.getQualifier()) + ", value " + Bytes.toString(kv.getValue())); } assertEquals(result.list().size(), 2); assertEquals(Bytes.toString(result.list().get(0).getValue()), "value2-3"); assertEquals(Bytes.toString(result.list().get(1).getValue()), "value4-3"); ht.close(); }
/** * Verify that basic filters still behave correctly with * minimum versions enabled. */ public void testFilters() throws Exception { HTableDescriptor htd = createTableDescriptor(getName(), 2, 1000, 1, false); HRegion region = createNewHRegion(htd, null, null); final byte [] c1 = COLUMNS[1]; // 2s in the past long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000; try { Put p = new Put(T1, ts-3); p.add(c0, c0, T0); p.add(c1, c1, T0); region.put(p); p = new Put(T1, ts-2); p.add(c0, c0, T1); p.add(c1, c1, T1); region.put(p); p = new Put(T1, ts-1); p.add(c0, c0, T2); p.add(c1, c1, T2); region.put(p); p = new Put(T1, ts); p.add(c0, c0, T3); p.add(c1, c1, T3); region.put(p); List<Long> tss = new ArrayList<Long>(); tss.add(ts-1); tss.add(ts-2); Get g = new Get(T1); g.addColumn(c1,c1); g.setFilter(new TimestampsFilter(tss)); g.setMaxVersions(); Result r = region.get(g, null); checkResult(r, c1, T2,T1); g = new Get(T1); g.addColumn(c0,c0); g.setFilter(new TimestampsFilter(tss)); g.setMaxVersions(); r = region.get(g, null); checkResult(r, c0, T2,T1); // now flush/compact region.flushcache(); region.compactStores(true); g = new Get(T1); g.addColumn(c1,c1); g.setFilter(new TimestampsFilter(tss)); g.setMaxVersions(); r = region.get(g, null); checkResult(r, c1, T2); g = new Get(T1); g.addColumn(c0,c0); g.setFilter(new TimestampsFilter(tss)); g.setMaxVersions(); r = region.get(g, null); checkResult(r, c0, T2); } finally { region.close(); region.getLog().closeAndDelete(); } }
@Override public FilterSupportStatus isFilterSupported( FilterAdapterContext context, TimestampsFilter filter) { return FilterSupportStatus.SUPPORTED; }
/** * Create a new FilterAdapter */ public static FilterAdapter buildAdapter() { FilterAdapter adapter = new FilterAdapter(); adapter.addFilterAdapter( ColumnPrefixFilter.class, new ColumnPrefixFilterAdapter()); adapter.addFilterAdapter( ColumnRangeFilter.class, new ColumnRangeFilterAdapter()); adapter.addFilterAdapter( KeyOnlyFilter.class, new KeyOnlyFilterAdapter()); adapter.addFilterAdapter( MultipleColumnPrefixFilter.class, new MultipleColumnPrefixFilterAdapter()); adapter.addFilterAdapter( TimestampsFilter.class, new TimestampsFilterAdapter()); ValueFilterAdapter valueFilterAdapter = new ValueFilterAdapter(); adapter.addFilterAdapter( ValueFilter.class, valueFilterAdapter); SingleColumnValueFilterAdapter scvfa = new SingleColumnValueFilterAdapter(valueFilterAdapter); adapter.addFilterAdapter( SingleColumnValueFilter.class, scvfa); adapter.addFilterAdapter( SingleColumnValueExcludeFilter.class, new SingleColumnValueExcludeFilterAdapter(scvfa)); adapter.addFilterAdapter( ColumnPaginationFilter.class, new ColumnPaginationFilterAdapter()); adapter.addFilterAdapter( FirstKeyOnlyFilter.class, new FirstKeyOnlyFilterAdapter()); adapter.addFilterAdapter( ColumnCountGetFilter.class, new ColumnCountGetFilterAdapter()); adapter.addFilterAdapter( RandomRowFilter.class, new RandomRowFilterAdapter()); adapter.addFilterAdapter( PrefixFilter.class, new PrefixFilterAdapter()); adapter.addFilterAdapter( QualifierFilter.class, new QualifierFilterAdapter()); // Passing the FilterAdapter in to the FilterListAdapter is a bit // unfortunate, but makes adapting the FilterList's subfilters simpler. FilterListAdapter filterListAdapter = new FilterListAdapter(adapter); // FilterList implements UnsupportedStatusCollector so it should // be used when possible (third parameter to addFilterAdapter()). adapter.addFilterAdapter( FilterList.class, filterListAdapter, filterListAdapter); return adapter; }
/** * Verify that basic filters still behave correctly with * minimum versions enabled. */ @Test public void testFilters() throws Exception { HTableDescriptor htd = hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE); HRegion region = hbu.createLocalHRegion(htd, null, null); final byte [] c1 = COLUMNS[1]; // 2s in the past long ts = EnvironmentEdgeManager.currentTime() - 2000; try { Put p = new Put(T1, ts-3); p.add(c0, c0, T0); p.add(c1, c1, T0); region.put(p); p = new Put(T1, ts-2); p.add(c0, c0, T1); p.add(c1, c1, T1); region.put(p); p = new Put(T1, ts-1); p.add(c0, c0, T2); p.add(c1, c1, T2); region.put(p); p = new Put(T1, ts); p.add(c0, c0, T3); p.add(c1, c1, T3); region.put(p); List<Long> tss = new ArrayList<Long>(); tss.add(ts-1); tss.add(ts-2); Get g = new Get(T1); g.addColumn(c1,c1); g.setFilter(new TimestampsFilter(tss)); g.setMaxVersions(); Result r = region.get(g); checkResult(r, c1, T2,T1); g = new Get(T1); g.addColumn(c0,c0); g.setFilter(new TimestampsFilter(tss)); g.setMaxVersions(); r = region.get(g); checkResult(r, c0, T2,T1); // now flush/compact region.flushcache(); region.compactStores(true); g = new Get(T1); g.addColumn(c1,c1); g.setFilter(new TimestampsFilter(tss)); g.setMaxVersions(); r = region.get(g); checkResult(r, c1, T2); g = new Get(T1); g.addColumn(c0,c0); g.setFilter(new TimestampsFilter(tss)); g.setMaxVersions(); r = region.get(g); checkResult(r, c0, T2); } finally { HRegion.closeHRegion(region); } }
@Test public void testMultiColumns() throws Exception { byte [] TABLE = Bytes.toBytes("testTimestampsFilterMultiColumns"); byte [] FAMILY = Bytes.toBytes("event_log"); byte [][] FAMILIES = new byte[][] { FAMILY }; // create table; set versions to max... HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE); Put p = new Put(Bytes.toBytes("row")); p.add(FAMILY, Bytes.toBytes("column0"), 3, Bytes.toBytes("value0-3")); p.add(FAMILY, Bytes.toBytes("column1"), 3, Bytes.toBytes("value1-3")); p.add(FAMILY, Bytes.toBytes("column2"), 1, Bytes.toBytes("value2-1")); p.add(FAMILY, Bytes.toBytes("column2"), 2, Bytes.toBytes("value2-2")); p.add(FAMILY, Bytes.toBytes("column2"), 3, Bytes.toBytes("value2-3")); p.add(FAMILY, Bytes.toBytes("column3"), 2, Bytes.toBytes("value3-2")); p.add(FAMILY, Bytes.toBytes("column4"), 1, Bytes.toBytes("value4-1")); p.add(FAMILY, Bytes.toBytes("column4"), 2, Bytes.toBytes("value4-2")); p.add(FAMILY, Bytes.toBytes("column4"), 3, Bytes.toBytes("value4-3")); ht.put(p); ArrayList<Long> timestamps = new ArrayList<Long>(); timestamps.add(new Long(3)); TimestampsFilter filter = new TimestampsFilter(timestamps); Get g = new Get(Bytes.toBytes("row")); g.setFilter(filter); g.setMaxVersions(); g.addColumn(FAMILY, Bytes.toBytes("column2")); g.addColumn(FAMILY, Bytes.toBytes("column4")); Result result = ht.get(g); for (Cell kv : result.listCells()) { System.out.println("found row " + Bytes.toString(CellUtil.cloneRow(kv)) + ", column " + Bytes.toString(CellUtil.cloneQualifier(kv)) + ", value " + Bytes.toString(CellUtil.cloneValue(kv))); } assertEquals(result.listCells().size(), 2); assertTrue(CellUtil.matchingValue(result.listCells().get(0), Bytes.toBytes("value2-3"))); assertTrue(CellUtil.matchingValue(result.listCells().get(1), Bytes.toBytes("value4-3"))); ht.close(); }
