@Test public void testSmallScan() throws Exception { TableName TABLE = TableName.valueOf("testSmallScan"); int numRows = 10; byte[][] ROWS = HTestConst.makeNAscii(ROW, numRows); int numQualifiers = 10; byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, numQualifiers); Table ht = TEST_UTIL.createTable(TABLE, FAMILY); Put put; List<Put> puts = new ArrayList<Put>(); for (int row = 0; row < ROWS.length; row++) { put = new Put(ROWS[row]); for (int qual = 0; qual < QUALIFIERS.length; qual++) { KeyValue kv = new KeyValue(ROWS[row], FAMILY, QUALIFIERS[qual], VALUE); put.add(kv); } puts.add(put); } ht.put(puts); int expectedRows = numRows; int expectedCols = numRows * numQualifiers; // Test normal and reversed testSmallScan(ht, true, expectedRows, expectedCols); testSmallScan(ht, false, expectedRows, expectedCols); }
public void testProcessBasedCluster() throws Exception { ProcessBasedLocalHBaseCluster cluster = new ProcessBasedLocalHBaseCluster( TEST_UTIL.getConfiguration(), 2, 3); cluster.startMiniDFS(); cluster.startHBase(); try { TEST_UTIL.createRandomTable(HTestConst.DEFAULT_TABLE_STR, HTestConst.DEFAULT_CF_STR_SET, HColumnDescriptor.DEFAULT_VERSIONS, COLS_PER_ROW, FLUSHES, NUM_REGIONS, ROWS_PER_FLUSH); Table table = new HTable(TEST_UTIL.getConfiguration(), HTestConst.DEFAULT_TABLE); ResultScanner scanner = table.getScanner(HTestConst.DEFAULT_CF_BYTES); Result result; int rows = 0; int cols = 0; while ((result = scanner.next()) != null) { ++rows; cols += result.getFamilyMap(HTestConst.DEFAULT_CF_BYTES).size(); } LOG.info("Read " + rows + " rows, " + cols + " columns"); scanner.close(); table.close(); // These numbers are deterministic, seeded by table name. assertEquals(19, rows); assertEquals(35, cols); } catch (Exception ex) { LOG.error(ex); throw ex; } finally { cluster.shutdown(); } }
public void testProcessBasedCluster() throws Exception { ProcessBasedLocalHBaseCluster cluster = new ProcessBasedLocalHBaseCluster( TEST_UTIL.getConfiguration(), 2, 3); cluster.startMiniDFS(); cluster.startHBase(); try { TEST_UTIL.createRandomTable(HTestConst.DEFAULT_TABLE_STR, HTestConst.DEFAULT_CF_STR_SET, HColumnDescriptor.DEFAULT_VERSIONS, COLS_PER_ROW, FLUSHES, NUM_REGIONS, ROWS_PER_FLUSH); HTable table = new HTable(TEST_UTIL.getConfiguration(), HTestConst.DEFAULT_TABLE_BYTES); ResultScanner scanner = table.getScanner(HTestConst.DEFAULT_CF_BYTES); Result result; int rows = 0; int cols = 0; while ((result = scanner.next()) != null) { ++rows; cols += result.getFamilyMap(HTestConst.DEFAULT_CF_BYTES).size(); } LOG.info("Read " + rows + " rows, " + cols + " columns"); scanner.close(); table.close(); // These numbers are deterministic, seeded by table name. assertEquals(19, rows); assertEquals(35, cols); } catch (Exception ex) { LOG.error(ex); throw ex; } finally { cluster.shutdown(); } }
@Test public void testSmallScan() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); int numRows = 10; byte[][] ROWS = HTestConst.makeNAscii(ROW, numRows); int numQualifiers = 10; byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, numQualifiers); Table ht = TEST_UTIL.createTable(tableName, FAMILY); Put put; List<Put> puts = new ArrayList<>(); for (int row = 0; row < ROWS.length; row++) { put = new Put(ROWS[row]); for (int qual = 0; qual < QUALIFIERS.length; qual++) { KeyValue kv = new KeyValue(ROWS[row], FAMILY, QUALIFIERS[qual], VALUE); put.add(kv); } puts.add(put); } ht.put(puts); int expectedRows = numRows; int expectedCols = numRows * numQualifiers; // Test normal and reversed testSmallScan(ht, true, expectedRows, expectedCols); testSmallScan(ht, false, expectedRows, expectedCols); }
@Test public void testMaxResultSizeIsSetToDefault() throws Exception { TableName TABLE = TableName.valueOf("testMaxResultSizeIsSetToDefault"); Table ht = TEST_UTIL.createTable(TABLE, FAMILY); // The max result size we expect the scan to use by default. long expectedMaxResultSize = TEST_UTIL.getConfiguration().getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE); int numRows = 5; byte[][] ROWS = HTestConst.makeNAscii(ROW, numRows); int numQualifiers = 10; byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, numQualifiers); // Specify the cell size such that a single row will be larger than the default // value of maxResultSize. This means that Scan RPCs should return at most a single // result back to the client. int cellSize = (int) (expectedMaxResultSize / (numQualifiers - 1)); byte[] cellValue = Bytes.createMaxByteArray(cellSize); Put put; List<Put> puts = new ArrayList<Put>(); for (int row = 0; row < ROWS.length; row++) { put = new Put(ROWS[row]); for (int qual = 0; qual < QUALIFIERS.length; qual++) { KeyValue kv = new KeyValue(ROWS[row], FAMILY, QUALIFIERS[qual], cellValue); put.add(kv); } puts.add(put); } ht.put(puts); // Create a scan with the default configuration. Scan scan = new Scan(); ResultScanner scanner = ht.getScanner(scan); assertTrue(scanner instanceof ClientScanner); ClientScanner clientScanner = (ClientScanner) scanner; // Call next to issue a single RPC to the server scanner.next(); // The scanner should have, at most, a single result in its cache. If there more results exists // in the cache it means that more than the expected max result size was fetched. assertTrue("The cache contains: " + clientScanner.getCacheSize() + " results", clientScanner.getCacheSize() <= 1); }
/** * Test from client side for scan with maxResultPerCF set * * @throws Exception */ @Test public void testScanLimitAndOffset() throws Exception { //byte [] TABLE = HTestConst.DEFAULT_TABLE_BYTES; byte [][] ROWS = HTestConst.makeNAscii(HTestConst.DEFAULT_ROW_BYTES, 2); byte [][] FAMILIES = HTestConst.makeNAscii(HTestConst.DEFAULT_CF_BYTES, 3); byte [][] QUALIFIERS = HTestConst.makeNAscii(HTestConst.DEFAULT_QUALIFIER_BYTES, 10); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(HTestConst.DEFAULT_TABLE_BYTES)); HRegionInfo info = new HRegionInfo(HTestConst.DEFAULT_TABLE, null, null, false); for (byte[] family : FAMILIES) { HColumnDescriptor hcd = new HColumnDescriptor(family); htd.addFamily(hcd); } HRegion region = HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(), htd); try { Put put; Scan scan; Result result; boolean toLog = true; List<Cell> kvListExp = new ArrayList<Cell>(); int storeOffset = 1; int storeLimit = 3; for (int r = 0; r < ROWS.length; r++) { put = new Put(ROWS[r]); for (int c = 0; c < FAMILIES.length; c++) { for (int q = 0; q < QUALIFIERS.length; q++) { KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1, HTestConst.DEFAULT_VALUE_BYTES); put.add(kv); if (storeOffset <= q && q < storeOffset + storeLimit) { kvListExp.add(kv); } } } region.put(put); } scan = new Scan(); scan.setRowOffsetPerColumnFamily(storeOffset); scan.setMaxResultsPerColumnFamily(storeLimit); RegionScanner scanner = region.getScanner(scan); List<Cell> kvListScan = new ArrayList<Cell>(); List<Cell> results = new ArrayList<Cell>(); while (scanner.next(results) || !results.isEmpty()) { kvListScan.addAll(results); results.clear(); } result = Result.create(kvListScan); TestScannersFromClientSide.verifyResult(result, kvListExp, toLog, "Testing scan with storeOffset and storeLimit"); } finally { region.close(); } }
@Test public void testMaxResultSizeIsSetToDefault() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); Table ht = TEST_UTIL.createTable(tableName, FAMILY); // The max result size we expect the scan to use by default. long expectedMaxResultSize = TEST_UTIL.getConfiguration().getLong(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE); int numRows = 5; byte[][] ROWS = HTestConst.makeNAscii(ROW, numRows); int numQualifiers = 10; byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, numQualifiers); // Specify the cell size such that a single row will be larger than the default // value of maxResultSize. This means that Scan RPCs should return at most a single // result back to the client. int cellSize = (int) (expectedMaxResultSize / (numQualifiers - 1)); byte[] cellValue = Bytes.createMaxByteArray(cellSize); Put put; List<Put> puts = new ArrayList<>(); for (int row = 0; row < ROWS.length; row++) { put = new Put(ROWS[row]); for (int qual = 0; qual < QUALIFIERS.length; qual++) { KeyValue kv = new KeyValue(ROWS[row], FAMILY, QUALIFIERS[qual], cellValue); put.add(kv); } puts.add(put); } ht.put(puts); // Create a scan with the default configuration. Scan scan = new Scan(); ResultScanner scanner = ht.getScanner(scan); assertTrue(scanner instanceof ClientScanner); ClientScanner clientScanner = (ClientScanner) scanner; // Call next to issue a single RPC to the server scanner.next(); // The scanner should have, at most, a single result in its cache. If there more results exists // in the cache it means that more than the expected max result size was fetched. assertTrue("The cache contains: " + clientScanner.getCacheSize() + " results", clientScanner.getCacheSize() <= 1); }
/** * Test from client side for scan with maxResultPerCF set * * @throws Exception */ @Test public void testScanLimitAndOffset() throws Exception { //byte [] TABLE = HTestConst.DEFAULT_TABLE_BYTES; byte [][] ROWS = HTestConst.makeNAscii(HTestConst.DEFAULT_ROW_BYTES, 2); byte [][] FAMILIES = HTestConst.makeNAscii(HTestConst.DEFAULT_CF_BYTES, 3); byte [][] QUALIFIERS = HTestConst.makeNAscii(HTestConst.DEFAULT_QUALIFIER_BYTES, 10); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(HTestConst.DEFAULT_TABLE_BYTES)); HRegionInfo info = new HRegionInfo(HTestConst.DEFAULT_TABLE, null, null, false); for (byte[] family : FAMILIES) { HColumnDescriptor hcd = new HColumnDescriptor(family); htd.addFamily(hcd); } HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(), htd); try { Put put; Scan scan; Result result; boolean toLog = true; List<Cell> kvListExp = new ArrayList<>(); int storeOffset = 1; int storeLimit = 3; for (int r = 0; r < ROWS.length; r++) { put = new Put(ROWS[r]); for (int c = 0; c < FAMILIES.length; c++) { for (int q = 0; q < QUALIFIERS.length; q++) { KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1, HTestConst.DEFAULT_VALUE_BYTES); put.add(kv); if (storeOffset <= q && q < storeOffset + storeLimit) { kvListExp.add(kv); } } } region.put(put); } scan = new Scan(); scan.setRowOffsetPerColumnFamily(storeOffset); scan.setMaxResultsPerColumnFamily(storeLimit); RegionScanner scanner = region.getScanner(scan); List<Cell> kvListScan = new ArrayList<>(); List<Cell> results = new ArrayList<>(); while (scanner.next(results) || !results.isEmpty()) { kvListScan.addAll(results); results.clear(); } result = Result.create(kvListScan); TestScannersFromClientSide.verifyResult(result, kvListExp, toLog, "Testing scan with storeOffset and storeLimit"); } finally { HBaseTestingUtility.closeRegionAndWAL(region); } }
/** * Test from client side for scan with maxResultPerCF set * * @throws Exception */ @Test public void testScanLimitAndOffset() throws Exception { //byte [] TABLE = HTestConst.DEFAULT_TABLE_BYTES; byte [][] ROWS = HTestConst.makeNAscii(HTestConst.DEFAULT_ROW_BYTES, 2); byte [][] FAMILIES = HTestConst.makeNAscii(HTestConst.DEFAULT_CF_BYTES, 3); byte [][] QUALIFIERS = HTestConst.makeNAscii(HTestConst.DEFAULT_QUALIFIER_BYTES, 10); HTableDescriptor htd = new HTableDescriptor(HTestConst.DEFAULT_TABLE_BYTES); HRegionInfo info = new HRegionInfo(HTestConst.DEFAULT_TABLE_BYTES, null, null, false); for (byte[] family : FAMILIES) { HColumnDescriptor hcd = new HColumnDescriptor(family); htd.addFamily(hcd); } HRegion region = HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(), TEST_UTIL.getConfiguration(), htd); try { Put put; Scan scan; Result result; boolean toLog = true; List<KeyValue> kvListExp = new ArrayList<KeyValue>(); int storeOffset = 1; int storeLimit = 3; for (int r = 0; r < ROWS.length; r++) { put = new Put(ROWS[r]); for (int c = 0; c < FAMILIES.length; c++) { for (int q = 0; q < QUALIFIERS.length; q++) { KeyValue kv = new KeyValue(ROWS[r], FAMILIES[c], QUALIFIERS[q], 1, HTestConst.DEFAULT_VALUE_BYTES); put.add(kv); if (storeOffset <= q && q < storeOffset + storeLimit) { kvListExp.add(kv); } } } region.put(put); } scan = new Scan(); scan.setRowOffsetPerColumnFamily(storeOffset); scan.setMaxResultsPerColumnFamily(storeLimit); RegionScanner scanner = region.getScanner(scan); List<KeyValue> kvListScan = new ArrayList<KeyValue>(); List<KeyValue> results = new ArrayList<KeyValue>(); while (scanner.next(results) || !results.isEmpty()) { kvListScan.addAll(results); results.clear(); } result = new Result(kvListScan); TestScannersFromClientSide.verifyResult(result, kvListExp, toLog, "Testing scan with storeOffset and storeLimit"); } finally { region.close(); } }