@Override public void deleteAllTs(ByteBuffer tableName, ByteBuffer row, ByteBuffer column, long timestamp, Map<ByteBuffer, ByteBuffer> attributes) throws IOError { Table table = null; try { table = getTable(tableName); Delete delete = new Delete(getBytes(row)); addAttributes(delete, attributes); byte [][] famAndQf = KeyValue.parseColumn(getBytes(column)); if (famAndQf.length == 1) { delete.deleteFamily(famAndQf[0], timestamp); } else { delete.deleteColumns(famAndQf[0], famAndQf[1], timestamp); } table.delete(delete); } catch (IOException e) { LOG.warn(e.getMessage(), e); throw new IOError(Throwables.getStackTraceAsString(e)); } finally { closeTable(table); } }
/** * show data */ public static void getAllRecord(String tableName) { try { Table table = connection.getTable(TableName.valueOf(tableName)); Scan s = new Scan(); ResultScanner ss = table.getScanner(s); for (Result r : ss) { for (KeyValue kv : r.raw()) { System.out.print(new String(kv.getRow()) + " "); System.out.print(new String(kv.getFamily()) + ":"); System.out.print(new String(kv.getQualifier()) + " "); System.out.print(kv.getTimestamp() + " "); System.out.println(new String(kv.getValue())); } } } catch (IOException e) { e.printStackTrace(); } }
/** * Test seeking while file is encoded. */ @Test public void testSeekToBlockWithDecreasingCommonPrefix() throws IOException { List<KeyValue> sampleKv = new ArrayList<KeyValue>(); KeyValue kv1 = new KeyValue(Bytes.toBytes("row10aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), Bytes.toBytes("val")); sampleKv.add(kv1); KeyValue kv2 = new KeyValue(Bytes.toBytes("row10aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q2"), Bytes.toBytes("val")); sampleKv.add(kv2); KeyValue kv3 = new KeyValue(Bytes.toBytes("row10aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q3"), Bytes.toBytes("val")); sampleKv.add(kv3); KeyValue kv4 = new KeyValue(Bytes.toBytes("row11baa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), Bytes.toBytes("val")); sampleKv.add(kv4); KeyValue toSeek = KeyValueUtil.createLastOnRow(kv3.getRowArray(), kv3.getRowOffset(), kv3.getRowLength(), null, 0, 0, null, 0, 0); seekToTheKey(kv3, sampleKv, toSeek); }
public static Range[] fromFilter(SingleColumnValueFilter filter) { if (!(filter.getComparator() instanceof BinaryComparator)) { return new Range[0]; } byte[] column = KeyValue.makeColumn(filter.getFamily(), filter.getQualifier()); CompareOp compareOp = filter.getOperator(); byte[] value = filter.getComparator().getValue(); if (compareOp == CompareOp.NOT_EQUAL) { return new Range[] { new Range(column, null, CompareOp.NO_OP, value, CompareOp.LESS), new Range(column, value, CompareOp.GREATER, null, CompareOp.NO_OP) }; } else { switch (compareOp) { case EQUAL: case GREATER_OR_EQUAL: case GREATER: return new Range[] { new Range(column, value, compareOp, null, CompareOp.NO_OP) }; case LESS: case LESS_OR_EQUAL: return new Range[] { new Range(column, null, CompareOp.NO_OP, value, compareOp) }; default: return new Range[0]; } } }
/** * Getting all records a row from an existing SS tables * @method getAllRecord * @inputParameters hbaseBtable Name used * @return type: no return type as its a void method * **/ @SuppressWarnings({ "deprecation", "resource" }) public static void getAllRecord(String myHbaseBtableName) { ResultScanner hbaseBSs = null; try { HTable hbaseBtable = new HTable(hbaseBconf, myHbaseBtableName); Scan hbaseBScan = new Scan(); hbaseBSs = hbaseBtable.getScanner(hbaseBScan); for (Result r : hbaseBSs) { for (KeyValue hbaseBkv : r.raw()) { System.out.print(new String(hbaseBkv.getRow()) + " "); System.out.print(new String(hbaseBkv.getFamily()) + ":"); System.out.print(new String(hbaseBkv.getQualifier()) + " "); System.out.print(hbaseBkv.getTimestamp() + " "); System.out.println(new String(hbaseBkv.getValue())); } } } catch (IOException eio) { eip.printStackTrace(); } finally { if (hbaseBSs != null) hbaseBSs.close(); // closing the ss hbaseBtable } }
@Test public void testSeekToBlockWithDiffFamilyAndQualifer() throws IOException { List<KeyValue> sampleKv = new ArrayList<KeyValue>(); KeyValue kv1 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("fam1"), Bytes.toBytes("q1"), Bytes.toBytes("val")); sampleKv.add(kv1); KeyValue kv2 = new KeyValue(Bytes.toBytes("aab"), Bytes.toBytes("fam1"), Bytes.toBytes("q1"), Bytes.toBytes("val")); sampleKv.add(kv2); KeyValue kv4 = new KeyValue(Bytes.toBytes("aac"), Bytes.toBytes("fam1"), Bytes.toBytes("q1"), Bytes.toBytes("val")); sampleKv.add(kv4); KeyValue kv5 = new KeyValue(Bytes.toBytes("aac"), Bytes.toBytes("fam1"), Bytes.toBytes("q2"), Bytes.toBytes("val")); sampleKv.add(kv5); KeyValue toSeek = new KeyValue(Bytes.toBytes("aac"), Bytes.toBytes("fam2"), Bytes.toBytes("q2"), Bytes.toBytes("val")); seekToTheKey(kv5, sampleKv, toSeek); }
@Test public void testSeekToBlockWithDiffQualiferOnSameRowButDescendingInSize() throws IOException { List<KeyValue> sampleKv = new ArrayList<KeyValue>(); KeyValue kv1 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qual1"), Bytes.toBytes("val")); sampleKv.add(kv1); KeyValue kv2 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qual2"), Bytes.toBytes("val")); sampleKv.add(kv2); KeyValue kv4 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qual3"), Bytes.toBytes("val")); sampleKv.add(kv4); KeyValue kv5 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qual4"), Bytes.toBytes("val")); sampleKv.add(kv5); KeyValue kv6 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qz"), Bytes.toBytes("val")); sampleKv.add(kv6); KeyValue toSeek = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("qz"), Bytes.toBytes("val")); seekToTheKey(kv6, sampleKv, toSeek); }
public static List<Cell> recoverClusteringResult(List<Cell> cells, byte[] family, byte[] qualifier) { if (cells == null || cells.size() == 0) return cells; byte[][] indexColumn = IndexPutParser.parseIndexRowKey(cells.get(0).getRow()); List<Cell> list = new ArrayList<>(cells.size() + 1); for (Cell cell : cells) { byte[] tag = cell.getTagsArray(); if (tag != null && tag.length > KeyValue.MAX_TAGS_LENGTH) tag = null; KeyValue kv = new KeyValue(indexColumn[0], CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell), cell.getTimestamp(), KeyValue.Type.codeToType(cell.getTypeByte()), CellUtil.cloneValue(cell), tag); list.add(kv); } list.add(new KeyValue(indexColumn[0], family, qualifier, indexColumn[1])); Collections.sort(list, KeyValue.COMPARATOR); return list; }
/** * Insert a mix of puts and deletes * @throws Exception */ @Test public void testMixedPutDelete() throws Exception { List<WALEntry> entries = new ArrayList<WALEntry>(BATCH_SIZE/2); List<Cell> cells = new ArrayList<Cell>(); for(int i = 0; i < BATCH_SIZE/2; i++) { entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells)); } SINK.replicateEntries(entries, CellUtil.createCellScanner(cells)); entries = new ArrayList<WALEntry>(BATCH_SIZE); cells = new ArrayList<Cell>(); for(int i = 0; i < BATCH_SIZE; i++) { entries.add(createEntry(TABLE_NAME1, i, i % 2 != 0 ? KeyValue.Type.Put: KeyValue.Type.DeleteColumn, cells)); } SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator())); Scan scan = new Scan(); ResultScanner scanRes = table1.getScanner(scan); assertEquals(BATCH_SIZE/2, scanRes.next(BATCH_SIZE).length); }
private static int insertData(TableName tableName, String column, double prob) throws IOException { byte[] k = new byte[3]; byte[][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column)); List<Put> puts = new ArrayList<>(); for (int i = 0; i < 9; i++) { Put put = new Put(Bytes.toBytes("row" + i)); put.setDurability(Durability.SKIP_WAL); put.add(famAndQf[0], famAndQf[1], k); put.setCellVisibility(new CellVisibility("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET)); puts.add(put); } try (Table table = new HTable(TEST_UTIL.getConfiguration(), tableName)) { table.put(puts); } return puts.size(); }
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags) throws IOException { int size; HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, block.getHFileContext()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(block.getDummyHeaderForVersion()); DataOutputStream dos = new DataOutputStream(baos); blockEncoder.startBlockEncoding(context, dos); for (KeyValue kv : kvs) { blockEncoder.encode(kv, context, dos); } BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream(); baos.writeTo(stream); blockEncoder.endBlockEncoding(context, dos, stream.getBuffer(), BlockType.DATA); byte[] encodedBytes = baos.toByteArray(); size = encodedBytes.length - block.getDummyHeaderForVersion().length; return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), block.getHFileContext()); }
@Test public void testCreateKey() { CompoundBloomFilterBase cbfb = new CompoundBloomFilterBase(); byte[] row = "myRow".getBytes(); byte[] qualifier = "myQualifier".getBytes(); byte[] rowKey = cbfb.createBloomKey(row, 0, row.length, row, 0, 0); byte[] rowColKey = cbfb.createBloomKey(row, 0, row.length, qualifier, 0, qualifier.length); KeyValue rowKV = KeyValue.createKeyValueFromKey(rowKey); KeyValue rowColKV = KeyValue.createKeyValueFromKey(rowColKey); assertEquals(rowKV.getTimestamp(), rowColKV.getTimestamp()); assertEquals(Bytes.toStringBinary(rowKV.getRow()), Bytes.toStringBinary(rowColKV.getRow())); assertEquals(0, rowKV.getQualifier().length); }
private void writeStoreFile(final StoreFile.Writer writer) throws IOException { byte[] fam = Bytes.toBytes("f"); byte[] qualifier = Bytes.toBytes("q"); long now = System.currentTimeMillis(); byte[] b = Bytes.toBytes("k1"); Tag t1 = new Tag((byte) 1, "tag1"); Tag t2 = new Tag((byte) 2, "tag2"); Tag t3 = new Tag((byte) 3, "tag3"); try { writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t1 })); b = Bytes.toBytes("k3"); writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t2, t1 })); b = Bytes.toBytes("k4"); writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t3 })); b = Bytes.toBytes("k5"); writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t3 })); } finally { writer.close(); } }
/** * @param c * @param kv Presume first on row: i.e. empty column, maximum timestamp and * a type of Type.Maximum * @param ttl Time to live in ms for this Store * @param metaregion True if this is hbase:meta or -ROOT- region. */ GetClosestRowBeforeTracker(final KVComparator c, final KeyValue kv, final long ttl, final boolean metaregion) { super(); this.metaregion = metaregion; this.targetkey = kv; // If we are in a metaregion, then our table name is the prefix on the // targetkey. this.rowoffset = kv.getRowOffset(); int l = -1; if (metaregion) { l = KeyValue.getDelimiter(kv.getRowArray(), rowoffset, kv.getRowLength(), HConstants.DELIMITER) - this.rowoffset; } this.tablenamePlusDelimiterLength = metaregion? l + 1: -1; this.now = System.currentTimeMillis(); this.oldestUnexpiredTs = now - ttl; this.kvcomparator = c; KeyValue.RowOnlyComparator rc = new KeyValue.RowOnlyComparator(this.kvcomparator); this.deletes = new TreeMap<KeyValue, NavigableSet<KeyValue>>(rc); }
@Override public void write(DataOutput out) throws IOException { LOG.warn("WALEdit is being serialized to writable - only expected in test code"); out.writeInt(VERSION_2); out.writeInt(cells.size()); // We interleave the two lists for code simplicity for (Cell cell : cells) { // This is not used in any of the core code flows so it is just fine to convert to KV KeyValue kv = KeyValueUtil.ensureKeyValue(cell); if (compressionContext != null) { KeyValueCompression.writeKV(out, kv, compressionContext); } else{ KeyValue.write(kv, out); } } if (scopes == null) { out.writeInt(0); } else { out.writeInt(scopes.size()); for (byte[] key : scopes.keySet()) { Bytes.writeByteArray(out, key); out.writeInt(scopes.get(key)); } } }
/** * Write random values to the writer assuming a table created using * {@link #FAMILIES} as column family descriptors */ private void writeRandomKeyValues(RecordWriter<ImmutableBytesWritable, KeyValue> writer, TaskAttemptContext context, Set<byte[]> families, int numRows) throws IOException, InterruptedException { byte keyBytes[] = new byte[Bytes.SIZEOF_INT]; int valLength = 10; byte valBytes[] = new byte[valLength]; int taskId = context.getTaskAttemptID().getTaskID().getId(); assert taskId < Byte.MAX_VALUE : "Unit tests dont support > 127 tasks!"; final byte [] qualifier = Bytes.toBytes("data"); Random random = new Random(); for (int i = 0; i < numRows; i++) { Bytes.putInt(keyBytes, 0, i); random.nextBytes(valBytes); ImmutableBytesWritable key = new ImmutableBytesWritable(keyBytes); for (byte[] family : families) { KeyValue kv = new KeyValue(keyBytes, family, qualifier, valBytes); writer.write(key, kv); } } }
LCIndexQueryProcessor createLCIndexProcessor(IndexTableRelation indexTableRelation, Map<byte[], Store> storeMap, Scan scan, long readPt, KeyValue.KVComparator comparator) { try { ScanRange.ScanRangeList rangeList = ScanRange.ScanRangeList.getScanRangeList(scan); checkSingleScanFamily(scan, rangeList); // calculate the primary range for scan ScanRange primaryRange = selectTheBestRange(indexTableRelation, storeMap, rangeList, scan.getId()); if (primaryRange != null) { printRanges(primaryRange, rangeList); return new LCIndexQueryProcessor(scan, indexTableRelation, primaryRange, rangeList, (HStore) storeMap.get(primaryRange.getFamily()), scan.getFamilyMap().get(primaryRange.getFamily()), readPt, comparator); } } catch (IOException e) { e.printStackTrace(); } return null; }
/** * Constructor. * @param c Comparator */ public DefaultMemStore(final Configuration conf, final KeyValue.KVComparator c) { this.conf = conf; this.comparator = c; this.cellSet = new CellSkipListSet(c); this.snapshot = new CellSkipListSet(c); timeRangeTracker = new TimeRangeTracker(); snapshotTimeRangeTracker = new TimeRangeTracker(); this.size = new AtomicLong(DEEP_OVERHEAD); this.snapshotSize = 0; if (conf.getBoolean(USEMSLAB_KEY, USEMSLAB_DEFAULT)) { String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); this.allocator = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] { Configuration.class }, new Object[] { conf }); } else { this.allocator = null; } }
public void testDeleteVersionMaskingMultiplePuts() throws IOException { long now = System.currentTimeMillis(); KeyValue [] kvs1 = new KeyValue[] { KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Delete, "dont-care") }; KeyValue [] kvs2 = new KeyValue[] { KeyValueTestUtil.create("R1", "cf", "a", now-500, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "a", now-100, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "a", now, KeyValue.Type.Put, "dont-care") }; List<KeyValueScanner> scanners = scanFixture(kvs1, kvs2); StoreScanner scan = new StoreScanner(new Scan(Bytes.toBytes("R1")), scanInfo, scanType, getCols("a"), scanners); List<Cell> results = new ArrayList<Cell>(); // the two put at ts=now will be masked by the 1 delete, and // since the scan default returns 1 version we'll return the newest // key, which is kvs[2], now-100. assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs2[1], results.get(0)); }
@Test public void testOne() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); MessageCodec cmc = new MessageCodec(); Codec.Encoder encoder = cmc.getEncoder(dos); final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("v")); encoder.write(kv); encoder.flush(); dos.close(); long offset = cos.getCount(); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = cmc.getDecoder(dis); assertTrue(decoder.advance()); // First read should pull in the KV assertFalse(decoder.advance()); // Second read should trip over the end-of-stream marker and return false dis.close(); assertEquals(offset, cis.getCount()); }
/** * query record */ public static void getOneRecord(String tableName, String rowKey) throws IOException { Table table = connection.getTable(TableName.valueOf(tableName)); Get get = new Get(rowKey.getBytes()); Result rs = table.get(get); for (KeyValue kv : rs.raw()) { System.out.print(new String(kv.getRow()) + " "); System.out.print(new String(kv.getFamily()) + ":"); System.out.print(new String(kv.getQualifier()) + " "); System.out.print(kv.getTimestamp() + " "); System.out.println(new String(kv.getValue())); } }
/** * Creates a half file reader for a hfile referred to by an hfilelink. * @param fs fileystem to read from * @param p path to hfile * @param in {@link FSDataInputStreamWrapper} * @param size Full size of the hfile file * @param cacheConf * @param r original reference file (contains top or bottom) * @param conf Configuration * @throws IOException */ public HalfStoreFileReader(final FileSystem fs, final Path p, final FSDataInputStreamWrapper in, long size, final CacheConfig cacheConf, final Reference r, final Configuration conf) throws IOException { super(fs, p, in, size, cacheConf, conf); // This is not actual midkey for this half-file; its just border // around which we split top and bottom. Have to look in files to find // actual last and first keys for bottom and top halves. Half-files don't // have an actual midkey themselves. No midkey is how we indicate file is // not splittable. this.splitkey = r.getSplitKey(); this.splitCell = new KeyValue.KeyOnlyKeyValue(this.splitkey, 0, this.splitkey.length); // Is it top or bottom half? this.top = Reference.isTopFileRegion(r.getFileRegion()); }
/** * Test GroupingTableMapper class */ @Test public void testGroupingTableMapper() throws Exception { GroupingTableMapper mapper = new GroupingTableMapper(); Configuration configuration = new Configuration(); configuration.set(GroupingTableMapper.GROUP_COLUMNS, "family1:clm family2:clm"); mapper.setConf(configuration); Result result = mock(Result.class); @SuppressWarnings("unchecked") Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Result>.Context context = mock(Mapper.Context.class); context.write(any(ImmutableBytesWritable.class), any(Result.class)); List<Cell> keyValue = new ArrayList<Cell>(); byte[] row = {}; keyValue.add(new KeyValue(row, Bytes.toBytes("family2"), Bytes.toBytes("clm"), Bytes .toBytes("value1"))); keyValue.add(new KeyValue(row, Bytes.toBytes("family1"), Bytes.toBytes("clm"), Bytes .toBytes("value2"))); when(result.listCells()).thenReturn(keyValue); mapper.map(null, result, context); // template data byte[][] data = { Bytes.toBytes("value1"), Bytes.toBytes("value2") }; ImmutableBytesWritable ibw = mapper.createGroupKey(data); verify(context).write(ibw, result); }
@Test public void testForwardScanner() { int counter = -1; while (searcher.advance()) { ++counter; KeyValue inputKv = rows.getInputs().get(counter); KeyValue outputKv = KeyValueUtil.copyToNewKeyValue(searcher.current()); assertKeyAndValueEqual(inputKv, outputKv); } // assert same number of cells Assert.assertEquals(rows.getInputs().size(), counter + 1); }
private boolean internalQueueTincrement(TIncrement inc) throws TException { byte[][] famAndQf = KeyValue.parseColumn(inc.getColumn()); if (famAndQf.length != 2) return false; return internalQueueIncrement(inc.getTable(), inc.getRow(), famAndQf[0], famAndQf[1], inc.getAmmount()); }
/** * Convert a line of TSV text into an HBase table row after transforming the * values by multiplying them by 3. */ @Override public void map(LongWritable offset, Text value, Context context) throws IOException { byte[] family = Bytes.toBytes("FAM"); final byte[][] qualifiers = { Bytes.toBytes("A"), Bytes.toBytes("B") }; // do some basic line parsing byte[] lineBytes = value.getBytes(); String[] valueTokens = new String(lineBytes, "UTF-8").split("\u001b"); // create the rowKey and Put ImmutableBytesWritable rowKey = new ImmutableBytesWritable(Bytes.toBytes(valueTokens[0])); Put put = new Put(rowKey.copyBytes()); put.setDurability(Durability.SKIP_WAL); //The value should look like this: VALUE1 or VALUE2. Let's multiply //the integer by 3 for(int i = 1; i < valueTokens.length; i++) { String prefix = valueTokens[i].substring(0, "VALUE".length()); String suffix = valueTokens[i].substring("VALUE".length()); String newValue = prefix + Integer.parseInt(suffix) * 3; KeyValue kv = new KeyValue(rowKey.copyBytes(), family, qualifiers[i-1], Bytes.toBytes(newValue)); put.add(kv); } try { context.write(rowKey, put); } catch (InterruptedException e) { e.printStackTrace(); } }
public void verifyInvocationResults(Integer[] selectQualifiers, Integer[] expectedQualifiers) throws Exception { Get get = new Get(ROW_BYTES); for (int i = 0; i < selectQualifiers.length; i++) { get.addColumn(FAMILY_NAME_BYTES, Bytes.toBytes(QUALIFIER_PREFIX + selectQualifiers[i])); } get.setFilter(new InvocationRecordFilter()); List<KeyValue> expectedValues = new ArrayList<KeyValue>(); for (int i = 0; i < expectedQualifiers.length; i++) { expectedValues.add(new KeyValue(ROW_BYTES, FAMILY_NAME_BYTES, Bytes .toBytes(QUALIFIER_PREFIX + expectedQualifiers[i]), expectedQualifiers[i], Bytes.toBytes(VALUE_PREFIX + expectedQualifiers[i]))); } Scan scan = new Scan(get); List<Cell> actualValues = new ArrayList<Cell>(); List<Cell> temp = new ArrayList<Cell>(); InternalScanner scanner = this.region.getScanner(scan); while (scanner.next(temp)) { actualValues.addAll(temp); temp.clear(); } actualValues.addAll(temp); Assert.assertTrue("Actual values " + actualValues + " differ from the expected values:" + expectedValues, expectedValues.equals(actualValues)); }
public void testBasicGetValue() throws Exception { KeyValue [] kvs = genKVs(row, family, value, 1, 100); Arrays.sort(kvs, KeyValue.COMPARATOR); Result r = Result.create(kvs); for (int i = 0; i < 100; ++i) { final byte[] qf = Bytes.toBytes(i); assertByteEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf)); assertTrue(r.containsColumn(family, qf)); } }
/** * This method is deprecated and you should override Cell getNextKeyHint(Cell) instead. */ @Override @Deprecated public KeyValue getNextKeyHint(KeyValue currentKV) throws IOException { // This will never get called. return KeyValueUtil.ensureKeyValue(this.filter.getNextCellHint((Cell)currentKV)); }
@Test public void testSmallScan() throws Exception { TableName TABLE = TableName.valueOf("testSmallScan"); int numRows = 10; byte[][] ROWS = HTestConst.makeNAscii(ROW, numRows); int numQualifiers = 10; byte[][] QUALIFIERS = HTestConst.makeNAscii(QUALIFIER, numQualifiers); Table ht = TEST_UTIL.createTable(TABLE, FAMILY); Put put; List<Put> puts = new ArrayList<Put>(); for (int row = 0; row < ROWS.length; row++) { put = new Put(ROWS[row]); for (int qual = 0; qual < QUALIFIERS.length; qual++) { KeyValue kv = new KeyValue(ROWS[row], FAMILY, QUALIFIERS[qual], VALUE); put.add(kv); } puts.add(put); } ht.put(puts); int expectedRows = numRows; int expectedCols = numRows * numQualifiers; // Test normal and reversed testSmallScan(ht, true, expectedRows, expectedCols); testSmallScan(ht, false, expectedRows, expectedCols); }
/** * Add the specified column and value, with the specified timestamp as * its version to this Put operation. * @param family family name * @param qualifier column qualifier * @param ts version timestamp * @param value column value * @return this */ public Put addColumn(byte [] family, byte [] qualifier, long ts, byte [] value) { if (ts < 0) { throw new IllegalArgumentException("Timestamp cannot be negative. ts=" + ts); } List<Cell> list = getCellList(family); KeyValue kv = createPutKeyValue(family, qualifier, ts, value); list.add(kv); familyMap.put(CellUtil.cloneFamily(kv), list); return this; }
public void testDeleteFamily() throws IOException { KeyValue [] kvs = new KeyValue[] { KeyValueTestUtil.create("R1", "cf", "a", 100, KeyValue.Type.DeleteFamily, "dont-care"), KeyValueTestUtil.create("R1", "cf", "b", 11, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "c", 11, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "e", 11, KeyValue.Type.DeleteColumn, "dont-care"), KeyValueTestUtil.create("R1", "cf", "f", 11, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "g", 11, KeyValue.Type.Delete, "dont-care"), KeyValueTestUtil.create("R1", "cf", "h", 11, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R1", "cf", "i", 11, KeyValue.Type.Put, "dont-care"), KeyValueTestUtil.create("R2", "cf", "a", 11, KeyValue.Type.Put, "dont-care"), }; List<KeyValueScanner> scanners = scanFixture(kvs); StoreScanner scan = new StoreScanner( new Scan().setMaxVersions(Integer.MAX_VALUE), scanInfo, scanType, null, scanners); List<Cell> results = new ArrayList<Cell>(); assertEquals(true, scan.next(results)); assertEquals(0, results.size()); assertEquals(true, scan.next(results)); assertEquals(1, results.size()); assertEquals(kvs[kvs.length-1], results.get(0)); assertEquals(false, scan.next(results)); }
public static KeyValue.Type generateKeyType(Random rand) { if (rand.nextBoolean()) { // Let's make half of KVs puts. return KeyValue.Type.Put; } else { KeyValue.Type keyType = KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)]; if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) { throw new RuntimeException("Generated an invalid key type: " + keyType + ". " + "Probably the layout of KeyValue.Type has changed."); } return keyType; } }
/** * Sanity check that we can move logs around while we are reading * from them. Should this test fail, ReplicationSource would have a hard * time reading logs that are being archived. * @throws Exception */ @Test public void testLogMoving() throws Exception{ Path logPath = new Path(logDir, "log"); if (!FS.exists(logDir)) FS.mkdirs(logDir); if (!FS.exists(oldLogDir)) FS.mkdirs(oldLogDir); WALProvider.Writer writer = WALFactory.createWALWriter(FS, logPath, TEST_UTIL.getConfiguration()); for(int i = 0; i < 3; i++) { byte[] b = Bytes.toBytes(Integer.toString(i)); KeyValue kv = new KeyValue(b,b,b); WALEdit edit = new WALEdit(); edit.add(kv); WALKey key = new WALKey(b, TableName.valueOf(b), 0, 0, HConstants.DEFAULT_CLUSTER_ID); writer.append(new WAL.Entry(key, edit)); writer.sync(); } writer.close(); WAL.Reader reader = WALFactory.createReader(FS, logPath, TEST_UTIL.getConfiguration()); WAL.Entry entry = reader.next(); assertNotNull(entry); Path oldLogPath = new Path(oldLogDir, "log"); FS.rename(logPath, oldLogPath); entry = reader.next(); assertNotNull(entry); entry = reader.next(); entry = reader.next(); assertNull(entry); reader.close(); }
/** * probably not needed since testReverseScannerWithJitter() below is more thorough */ @Test public void testReverseScanner() { searcher.positionAfterLastCell(); int counter = -1; while (searcher.previous()) { ++counter; int oppositeIndex = rows.getInputs().size() - counter - 1; KeyValue inputKv = rows.getInputs().get(oppositeIndex); KeyValue outputKv = KeyValueUtil.copyToNewKeyValue(searcher.current()); assertKeyAndValueEqual(inputKv, outputKv); } Assert.assertEquals(rows.getInputs().size(), counter + 1); }
public CollectionBackedScanner(KeyValue.KVComparator comparator, Cell... array) { this.comparator = comparator; List<Cell> tmp = new ArrayList<Cell>(array.length); Collections.addAll(tmp, array); Collections.sort(tmp, comparator); data = tmp; init(); }
/** * Set column for this index. * * @param indexColumn */ public void setIndexColumn(byte[] indexColumn) { this.indexColumn = indexColumn; byte[][] fq = KeyValue.parseColumn(this.indexColumn); this.family = fq[0]; this.qualifier = fq[1]; this.indexId = Bytes.add(this.family, Bytes.toBytes("_"), this.qualifier); }
/** * Delete the specified version of the specified column. * @param family family name * @param qualifier column qualifier * @param timestamp version timestamp * @return this for invocation chaining */ public Delete addColumn(byte [] family, byte [] qualifier, long timestamp) { if (timestamp < 0) { throw new IllegalArgumentException("Timestamp cannot be negative. ts=" + timestamp); } List<Cell> list = familyMap.get(family); if(list == null) { list = new ArrayList<Cell>(); } KeyValue kv = new KeyValue(this.row, family, qualifier, timestamp, KeyValue.Type.Delete); list.add(kv); familyMap.put(family, list); return this; }
private boolean seekToScanner(final HFileScanner scanner, final KeyValue firstOnRow, final KeyValue firstKV) throws IOException { KeyValue kv = firstOnRow; // If firstOnRow < firstKV, set to firstKV if (this.comparator.compareRows(firstKV, firstOnRow) == 0) kv = firstKV; int result = scanner.seekTo(kv); return result != -1; }
@Test public void testThree() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); MessageCodec cmc = new MessageCodec(); Codec.Encoder encoder = cmc.getEncoder(dos); final KeyValue kv1 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("1"), Bytes.toBytes("1")); final KeyValue kv2 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("2"), Bytes.toBytes("2")); final KeyValue kv3 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("3"), Bytes.toBytes("3")); encoder.write(kv1); encoder.write(kv2); encoder.write(kv3); encoder.flush(); dos.close(); long offset = cos.getCount(); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = cmc.getDecoder(dis); assertTrue(decoder.advance()); Cell c = decoder.current(); assertTrue(CellComparator.equals(c, kv1)); assertTrue(decoder.advance()); c = decoder.current(); assertTrue(CellComparator.equals(c, kv2)); assertTrue(decoder.advance()); c = decoder.current(); assertTrue(CellComparator.equals(c, kv3)); assertFalse(decoder.advance()); dis.close(); assertEquals(offset, cis.getCount()); }