public static List<Cell> recoverClusteringResult(List<Cell> cells, byte[] family, byte[] qualifier) { if (cells == null || cells.size() == 0) return cells; byte[][] indexColumn = IndexPutParser.parseIndexRowKey(cells.get(0).getRow()); List<Cell> list = new ArrayList<>(cells.size() + 1); for (Cell cell : cells) { byte[] tag = cell.getTagsArray(); if (tag != null && tag.length > KeyValue.MAX_TAGS_LENGTH) tag = null; KeyValue kv = new KeyValue(indexColumn[0], CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell), cell.getTimestamp(), KeyValue.Type.codeToType(cell.getTypeByte()), CellUtil.cloneValue(cell), tag); list.add(kv); } list.add(new KeyValue(indexColumn[0], family, qualifier, indexColumn[1])); Collections.sort(list, KeyValue.COMPARATOR); return list; }
/** * Creates a {@link TResult} (Thrift) from a {@link Result} (HBase). * * @param in the <code>Result</code> to convert * * @return converted result, returns an empty result if the input is <code>null</code> */ public static TResult resultFromHBase(Result in) { Cell[] raw = in.rawCells(); TResult out = new TResult(); byte[] row = in.getRow(); if (row != null) { out.setRow(in.getRow()); } List<TColumnValue> columnValues = new ArrayList<TColumnValue>(); for (Cell kv : raw) { TColumnValue col = new TColumnValue(); col.setFamily(CellUtil.cloneFamily(kv)); col.setQualifier(CellUtil.cloneQualifier(kv)); col.setTimestamp(kv.getTimestamp()); col.setValue(CellUtil.cloneValue(kv)); if (kv.getTagsLength() > 0) { col.setTags(CellUtil.getTagArray(kv)); } columnValues.add(col); } out.setColumnValues(columnValues); return out; }
protected int binarySearch(final Cell [] kvs, final byte [] family, final byte [] qualifier) { Cell searchTerm = KeyValueUtil.createFirstOnRow(CellUtil.cloneRow(kvs[0]), family, qualifier); // pos === ( -(insertion point) - 1) int pos = Arrays.binarySearch(kvs, searchTerm, KeyValue.COMPARATOR); // never will exact match if (pos < 0) { pos = (pos+1) * -1; // pos is now insertion point } if (pos == kvs.length) { return -1; // doesn't exist } return pos; }
/** * Extracts and partitions the visibility tags and nonVisibility Tags * * @param cell - the cell for which we would extract and partition the * visibility and non visibility tags * @param visTags * - all the visibilty tags of type TagType.VISIBILITY_TAG_TYPE would * be added to this list * @param nonVisTags - all the non visibility tags would be added to this list * @return - the serailization format of the tag. Can be null if no tags are found or * if there is no visibility tag found */ public static Byte extractAndPartitionTags(Cell cell, List<Tag> visTags, List<Tag> nonVisTags) { Byte serializationFormat = null; if (cell.getTagsLength() > 0) { Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) { serializationFormat = tag.getBuffer()[tag.getTagOffset()]; } else if (tag.getType() == VISIBILITY_TAG_TYPE) { visTags.add(tag); } else { // ignore string encoded visibility expressions, will be added in replication handling nonVisTags.add(tag); } } } return serializationFormat; }
private void verifyData(Region newReg, int startRow, int numRows, byte[] qf, byte[]... families) throws IOException { for (int i = startRow; i < startRow + numRows; i++) { byte[] row = Bytes.toBytes("" + i); Get get = new Get(row); for (byte[] family : families) { get.addColumn(family, qf); } Result result = newReg.get(get); Cell[] raw = result.rawCells(); assertEquals(families.length, result.size()); for (int j = 0; j < families.length; j++) { assertTrue(CellUtil.matchingRow(raw[j], row)); assertTrue(CellUtil.matchingFamily(raw[j], families[j])); assertTrue(CellUtil.matchingQualifier(raw[j], qf)); } } }
private static Map<String, Object> toStringMap(Cell cell) { Map<String, Object> stringMap = new HashMap<String, Object>(); stringMap.put("row", Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())); stringMap.put("family", Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength())); stringMap.put("qualifier", Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength())); stringMap.put("timestamp", cell.getTimestamp()); stringMap.put("vlen", cell.getValueLength()); if (cell.getTagsLength() > 0) { List<String> tagsString = new ArrayList<String>(); Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); tagsString.add((tag.getType()) + ":" + Bytes.toStringBinary(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength())); } stringMap.put("tag", tagsString); } return stringMap; }
@Override public long append(HTableDescriptor htd, HRegionInfo info, WALKey key, WALEdit edits, boolean inMemstore) { if (!this.listeners.isEmpty()) { final long start = System.nanoTime(); long len = 0; for (Cell cell : edits.getCells()) { len += CellUtil.estimatedSerializedSizeOf(cell); } final long elapsed = (System.nanoTime() - start)/1000000l; for (WALActionsListener listener : this.listeners) { listener.postAppend(len, elapsed); } } return -1; }
private int countDeleteMarkers(Region region) throws IOException { Scan s = new Scan(); s.setRaw(true); // use max versions from the store(s) s.setMaxVersions(region.getStores().iterator().next().getScanInfo().getMaxVersions()); InternalScanner scan = region.getScanner(s); List<Cell> kvs = new ArrayList<Cell>(); int res = 0; boolean hasMore; do { hasMore = scan.next(kvs); for (Cell kv : kvs) { if(CellUtil.isDelete(kv)) res++; } kvs.clear(); } while (hasMore); scan.close(); return res; }
@Override public Result next() throws IOException { values.clear(); scanner.nextRaw(values); if (values.isEmpty()) { //we are done return null; } Result result = Result.create(values); if (this.scanMetrics != null) { long resultSize = 0; for (Cell cell : values) { resultSize += CellUtil.estimatedSerializedSizeOf(cell); } this.scanMetrics.countOfBytesInResults.addAndGet(resultSize); } return result; }
public synchronized NavigableSet<NamespaceDescriptor> list() throws IOException { NavigableSet<NamespaceDescriptor> ret = Sets.newTreeSet(NamespaceDescriptor.NAMESPACE_DESCRIPTOR_COMPARATOR); ResultScanner scanner = getNamespaceTable().getScanner(HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES); try { for(Result r : scanner) { byte[] val = CellUtil.cloneValue(r.getColumnLatestCell( HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES, HTableDescriptor.NAMESPACE_COL_DESC_BYTES)); ret.add(ProtobufUtil.toNamespaceDescriptor( HBaseProtos.NamespaceDescriptor.parseFrom(val))); } } finally { scanner.close(); } return ret; }
/** * Tests the the {@link WhileMatchFilter} works in combination with a * {@link Filter} that uses the * {@link Filter#filterRowKey(byte[], int, int)} method. * * See HBASE-2258. * * @throws Exception */ @Test public void testWhileMatchFilterWithFilterRowKey() throws Exception { Scan s = new Scan(); String prefix = "testRowOne"; WhileMatchFilter filter = new WhileMatchFilter(new PrefixFilter(Bytes.toBytes(prefix))); s.setFilter(filter); InternalScanner scanner = this.region.getScanner(s); while (true) { ArrayList<Cell> values = new ArrayList<Cell>(); boolean isMoreResults = scanner.next(values); if (!isMoreResults || !Bytes.toString(CellUtil.cloneRow(values.get(0))).startsWith(prefix)) { assertTrue("The WhileMatchFilter should now filter all remaining", filter.filterAllRemaining()); } if (!isMoreResults) { break; } } }
@Override public boolean backwardSeek(Cell seekKey) throws IOException { if (current == null) { return false; } heap.add(current); current = null; KeyValueScanner scanner; while ((scanner = heap.poll()) != null) { Cell topKey = scanner.peek(); if ((CellUtil.matchingRow(seekKey, topKey) && comparator .getComparator().compare(seekKey, topKey) <= 0) || comparator.getComparator().compareRows(seekKey, topKey) > 0) { heap.add(scanner); current = pollRealKV(); return current != null; } if (!scanner.backwardSeek(seekKey)) { scanner.close(); } else { heap.add(scanner); } } return false; }
FSWALEntry(final long sequence, final WALKey key, final WALEdit edit, final HTableDescriptor htd, final HRegionInfo hri, final boolean inMemstore) { super(key, edit); this.inMemstore = inMemstore; this.htd = htd; this.hri = hri; this.sequence = sequence; if (inMemstore) { // construct familyNames here to reduce the work of log sinker. ArrayList<Cell> cells = this.getEdit().getCells(); if (CollectionUtils.isEmpty(cells)) { this.familyNames = Collections.<byte[]> emptySet(); } else { Set<byte[]> familySet = Sets.newTreeSet(Bytes.BYTES_COMPARATOR); for (Cell cell : cells) { if (!CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) { familySet.add(CellUtil.cloneFamily(cell)); } } this.familyNames = Collections.unmodifiableSet(familySet); } } else { this.familyNames = Collections.<byte[]> emptySet(); } }
/** * Here is where a WAL edit gets its sequenceid. * @return The sequenceid we stamped on this edit. * @throws IOException */ long stampRegionSequenceId() throws IOException { long regionSequenceId = WALKey.NO_SEQUENCE_ID; MultiVersionConcurrencyControl mvcc = getKey().getMvcc(); MultiVersionConcurrencyControl.WriteEntry we = null; if (mvcc != null) { we = mvcc.begin(); regionSequenceId = we.getWriteNumber(); } if (!this.getEdit().isReplay() && inMemstore) { for (Cell c:getEdit().getCells()) { CellUtil.setSequenceId(c, regionSequenceId); } } // This has to stay in this order WALKey key = getKey(); key.setLogSeqNum(regionSequenceId); key.setWriteEntry(we); return regionSequenceId; }
@Ignore @Test public void testRpcScheduler() throws IOException, InterruptedException { PriorityFunction qosFunction = mock(PriorityFunction.class); Abortable abortable = new AbortServer(); RpcScheduler scheduler = new SimpleRpcScheduler(CONF, 2, 0, 0, qosFunction, abortable, 0); RpcServer rpcServer = new TestRpcServer(scheduler); RpcClientImpl client = new RpcClientImpl(CONF, HConstants.CLUSTER_ID_DEFAULT); try { rpcServer.start(); MethodDescriptor md = SERVICE.getDescriptorForType().findMethodByName("echo"); EchoRequestProto param = EchoRequestProto.newBuilder().setMessage("hello").build(); PayloadCarryingRpcController controller = new PayloadCarryingRpcController(CellUtil.createCellScanner(ImmutableList.of(CELL))); InetSocketAddress address = rpcServer.getListenerAddress(); if (address == null) { throw new IOException("Listener channel is closed"); } client.call(controller, md, param, md.getOutputType().toProto(), User.getCurrent(), address, new MetricsConnection.CallStats()); } catch (Throwable e) { assert(abortable.isAborted() == true); } finally { rpcServer.stop(); } }
public static Result recoverClusteringResult(Result raw, byte[] family, byte[] qualifier) { if (raw == null) return null; byte[][] indexColumn = IndexPutParser.parseIndexRowKey(raw.getRow()); List<KeyValue> list = new ArrayList<>(raw.listCells().size() + 1); for (Cell cell : raw.listCells()) { byte[] tag = cell.getTagsArray(); if (tag != null && tag.length > KeyValue.MAX_TAGS_LENGTH) tag = null; KeyValue kv = new KeyValue(indexColumn[0], CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell), cell.getTimestamp(), KeyValue.Type.codeToType(cell.getTypeByte()), CellUtil.cloneValue(cell), tag); list.add(kv); } list.add(new KeyValue(indexColumn[0], family, qualifier, indexColumn[1])); Collections.sort(list, KeyValue.COMPARATOR); return new Result(list); }
static void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families) throws IOException { for (int i = startRow; i < startRow + numRows; i++) { byte[] row = Bytes.toBytes("" + i); Get get = new Get(row); for (byte[] family : families) { get.addColumn(family, qf); } Result result = newReg.get(get); Cell[] raw = result.rawCells(); assertEquals(families.length, result.size()); for (int j = 0; j < families.length; j++) { assertTrue(CellUtil.matchingRow(raw[j], row)); assertTrue(CellUtil.matchingFamily(raw[j], families[j])); assertTrue(CellUtil.matchingQualifier(raw[j], qf)); } } }
/** * Method to account for the size of retained cells and retained data blocks. * @return an object that represents the last referenced block from this response. */ Object addSize(RpcCallContext context, Result r, Object lastBlock) { if (context != null && !r.isEmpty()) { for (Cell c : r.rawCells()) { context.incrementResponseCellSize(CellUtil.estimatedHeapSizeOf(c)); // We're using the last block being the same as the current block as // a proxy for pointing to a new block. This won't be exact. // If there are multiple gets that bounce back and forth // Then it's possible that this will over count the size of // referenced blocks. However it's better to over count and // use two RPC's than to OOME the RegionServer. byte[] valueArray = c.getValueArray(); if (valueArray != lastBlock) { context.incrementResponseBlockSize(valueArray.length); lastBlock = valueArray; } } } return lastBlock; }
/** Clean up the current data block */ private void finishBlock() throws IOException { if (!fsBlockWriter.isWriting() || fsBlockWriter.blockSizeWritten() == 0) return; // Update the first data block offset for scanning. if (firstDataBlockOffset == -1) { firstDataBlockOffset = outputStream.getPos(); } // Update the last data block offset lastDataBlockOffset = outputStream.getPos(); fsBlockWriter.writeHeaderAndData(outputStream); int onDiskSize = fsBlockWriter.getOnDiskSizeWithHeader(); Cell indexEntry = CellComparator.getMidpoint(this.comparator, lastCellOfPreviousBlock, firstCellInBlock); dataBlockIndexWriter.addEntry(CellUtil.getCellKeySerializedAsKeyValueKey(indexEntry), lastDataBlockOffset, onDiskSize); totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader(); if (cacheConf.shouldCacheDataOnWrite()) { doCacheOnWrite(lastDataBlockOffset); } }
/** * Add last bits of metadata to file info before it is written out. */ protected void finishFileInfo() throws IOException { if (lastCell != null) { // Make a copy. The copy is stuffed into our fileinfo map. Needs a clean // byte buffer. Won't take a tuple. byte [] lastKey = CellUtil.getCellKeySerializedAsKeyValueKey(this.lastCell); fileInfo.append(FileInfo.LASTKEY, lastKey, false); } // Average key length. int avgKeyLen = entryCount == 0 ? 0 : (int) (totalKeyLength / entryCount); fileInfo.append(FileInfo.AVG_KEY_LEN, Bytes.toBytes(avgKeyLen), false); // Average value length. int avgValueLen = entryCount == 0 ? 0 : (int) (totalValueLength / entryCount); fileInfo.append(FileInfo.AVG_VALUE_LEN, Bytes.toBytes(avgValueLen), false); fileInfo.append(FileInfo.CREATE_TIME_TS, Bytes.toBytes(hFileContext.getFileCreateTime()), false); }
/** * Insert a mix of puts and deletes * @throws Exception */ @Test public void testMixedPutDelete() throws Exception { List<WALEntry> entries = new ArrayList<WALEntry>(BATCH_SIZE/2); List<Cell> cells = new ArrayList<Cell>(); for(int i = 0; i < BATCH_SIZE/2; i++) { entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells)); } SINK.replicateEntries(entries, CellUtil.createCellScanner(cells)); entries = new ArrayList<WALEntry>(BATCH_SIZE); cells = new ArrayList<Cell>(); for(int i = 0; i < BATCH_SIZE; i++) { entries.add(createEntry(TABLE_NAME1, i, i % 2 != 0 ? KeyValue.Type.Put: KeyValue.Type.DeleteColumn, cells)); } SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator())); Scan scan = new Scan(); ResultScanner scanRes = table1.getScanner(scan); assertEquals(BATCH_SIZE/2, scanRes.next(BATCH_SIZE).length); }
private int getNumberOfRows(String keyPrefix, String value, Table ht) throws Exception { ResultScanner resultScanner = buildScanner(keyPrefix, value, ht); Iterator<Result> scanner = resultScanner.iterator(); int numberOfResults = 0; while (scanner.hasNext()) { Result result = scanner.next(); System.out.println("Got back key: " + Bytes.toString(result.getRow())); for (Cell kv : result.rawCells()) { System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(CellUtil.cloneValue(kv))); } numberOfResults++; } return numberOfResults; }
private void assertKey(Cell key, byte [] row, byte [] family, byte [] qualifier, byte [] value) throws Exception { assertTrue("Expected row [" + Bytes.toString(row) + "] " + "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]", equals(row, CellUtil.cloneRow(key))); assertTrue("Expected family [" + Bytes.toString(family) + "] " + "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]", equals(family, CellUtil.cloneFamily(key))); assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]", equals(qualifier, CellUtil.cloneQualifier(key))); assertTrue("Expected value [" + Bytes.toString(value) + "] " + "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]", equals(value, CellUtil.cloneValue(key))); }
/** * Compare only the bytes within the window of the current token * @param key * @return return -1 if key is lessThan (before) this, 0 if equal, and 1 if key is after */ protected int compareToCurrentToken(Cell key) { int startIndex = rowLength - currentRowNode.getTokenLength(); int endIndexExclusive = startIndex + currentRowNode.getTokenLength(); for (int i = startIndex; i < endIndexExclusive; ++i) { if (i >= key.getRowLength()) {// key was shorter, so it's first return -1; } byte keyByte = CellUtil.getRowByte(key, i); byte thisByte = rowBuffer[i]; if (keyByte == thisByte) { continue; } return UnsignedBytes.compare(keyByte, thisByte); } if (!currentRowNode.hasOccurrences() && rowLength >= key.getRowLength()) { // key was shorter return -1; } return 0; }
private void assertSingleResult(Result result, byte [] row, byte [] family, byte [] qualifier, long ts, byte [] value) throws Exception { assertTrue("Expected row [" + Bytes.toString(row) + "] " + "Got row [" + Bytes.toString(result.getRow()) +"]", equals(row, result.getRow())); assertTrue("Expected a single key but result contains " + result.size(), result.size() == 1); Cell kv = result.rawCells()[0]; assertTrue("Expected family [" + Bytes.toString(family) + "] " + "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]", equals(family, CellUtil.cloneFamily(kv))); assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]", equals(qualifier, CellUtil.cloneQualifier(kv))); assertTrue("Expected ts [" + ts + "] " + "Got ts [" + kv.getTimestamp() + "]", ts == kv.getTimestamp()); assertTrue("Expected value [" + Bytes.toString(value) + "] " + "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]", equals(value, CellUtil.cloneValue(kv))); }
public static void printPut(Put put) { StringBuilder sb = new StringBuilder(); sb.append("{").append(Bytes.toStringBinary(put.getRow())); for (Map.Entry<byte[], List<Cell>> entry : put.getFamilyCellMap().entrySet()) { byte[] family = entry.getKey(); for (Cell cell : entry.getValue()) { sb.append("[").append(Bytes.toString(family)).append(":") .append(Bytes.toString(CellUtil.cloneQualifier(cell))).append("]=") .append(Bytes.toStringBinary(CellUtil.cloneValue(cell))); } } System.out.println(sb.toString()); }
/** * Add the specified KeyValue to this Put operation. Operation assumes that * the passed KeyValue is immutable and its backing array will not be modified * for the duration of this Put. * @param kv individual KeyValue * @return this * @throws java.io.IOException e */ public Put add(Cell kv) throws IOException{ byte [] family = CellUtil.cloneFamily(kv); List<Cell> list = getCellList(family); //Checking that the row of the kv is the same as the put int res = Bytes.compareTo(this.row, 0, row.length, kv.getRowArray(), kv.getRowOffset(), kv.getRowLength()); if (res != 0) { throw new WrongRowIOException("The row in " + kv.toString() + " doesn't match the original one " + Bytes.toStringBinary(this.row)); } list.add(kv); familyMap.put(family, list); return this; }
public static TDelete deleteFromHBase(Delete in) { TDelete out = new TDelete(ByteBuffer.wrap(in.getRow())); List<TColumn> columns = new ArrayList<TColumn>(); long rowTimestamp = in.getTimeStamp(); if (rowTimestamp != HConstants.LATEST_TIMESTAMP) { out.setTimestamp(rowTimestamp); } // Map<family, List<KeyValue>> for (Map.Entry<byte[], List<org.apache.hadoop.hbase.Cell>> familyEntry: in.getFamilyCellMap().entrySet()) { TColumn column = new TColumn(ByteBuffer.wrap(familyEntry.getKey())); for (org.apache.hadoop.hbase.Cell cell: familyEntry.getValue()) { byte[] family = CellUtil.cloneFamily(cell); byte[] qualifier = CellUtil.cloneQualifier(cell); long timestamp = cell.getTimestamp(); if (family != null) { column.setFamily(family); } if (qualifier != null) { column.setQualifier(qualifier); } if (timestamp != HConstants.LATEST_TIMESTAMP) { column.setTimestamp(timestamp); } } columns.add(column); } out.setColumns(columns); return out; }
/** * This utility method creates a list of Thrift TCell "struct" based on * an Hbase Cell array. The empty list is returned if the input is null. * @param in Hbase Cell array * @return Thrift TCell array */ static public List<TCell> cellFromHBase(Cell[] in) { List<TCell> list = null; if (in != null) { list = new ArrayList<TCell>(in.length); for (int i = 0; i < in.length; i++) { list.add(new TCell(ByteBuffer.wrap(CellUtil.cloneValue(in[i])), in[i].getTimestamp())); } } else { list = new ArrayList<TCell>(0); } return list; }
/** * Add the specified column and value, with the specified timestamp as * its version to this Put operation. * @param family family name * @param qualifier column qualifier * @param ts version timestamp * @param value column value * @return this */ public Put addColumn(byte [] family, byte [] qualifier, long ts, byte [] value) { if (ts < 0) { throw new IllegalArgumentException("Timestamp cannot be negative. ts=" + ts); } List<Cell> list = getCellList(family); KeyValue kv = createPutKeyValue(family, qualifier, ts, value); list.add(kv); familyMap.put(CellUtil.cloneFamily(kv), list); return this; }
/** * Returns a count of the rows in the region where this coprocessor is loaded. */ @Override public void getRowCount(RpcController controller, ExampleProtos.CountRequest request, RpcCallback<ExampleProtos.CountResponse> done) { Scan scan = new Scan(); scan.setFilter(new FirstKeyOnlyFilter()); ExampleProtos.CountResponse response = null; InternalScanner scanner = null; try { scanner = env.getRegion().getScanner(scan); List<Cell> results = new ArrayList<Cell>(); boolean hasMore = false; byte[] lastRow = null; long count = 0; do { hasMore = scanner.next(results); for (Cell kv : results) { byte[] currentRow = CellUtil.cloneRow(kv); if (lastRow == null || !Bytes.equals(lastRow, currentRow)) { lastRow = currentRow; count++; } } results.clear(); } while (hasMore); response = ExampleProtos.CountResponse.newBuilder() .setCount(count).build(); } catch (IOException ioe) { ResponseConverter.setControllerException(controller, ioe); } finally { if (scanner != null) { try { scanner.close(); } catch (IOException ignored) {} } } done.run(response); }
@Override public Cell postMutationBeforeWAL(ObserverContext<RegionCoprocessorEnvironment> ctx, MutationType opType, Mutation mutation, Cell oldCell, Cell newCell) throws IOException { List<Tag> tags = Lists.newArrayList(); CellVisibility cellVisibility = null; try { cellVisibility = mutation.getCellVisibility(); } catch (DeserializationException e) { throw new IOException(e); } if (cellVisibility == null) { return newCell; } // Prepend new visibility tags to a new list of tags for the cell // Don't check user auths for labels with Mutations when the user is super user boolean authCheck = authorizationEnabled && checkAuths && !(isSystemOrSuperUser()); tags.addAll(this.visibilityLabelService.createVisibilityExpTags(cellVisibility.getExpression(), true, authCheck)); // Save an object allocation where we can if (newCell.getTagsLength() > 0) { // Carry forward all other tags Iterator<Tag> tagsItr = CellUtil.tagsIterator(newCell.getTagsArray(), newCell.getTagsOffset(), newCell.getTagsLength()); while (tagsItr.hasNext()) { Tag tag = tagsItr.next(); if (tag.getType() != TagType.VISIBILITY_TAG_TYPE && tag.getType() != TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) { tags.add(tag); } } } Cell rewriteCell = new TagRewriteCell(newCell, Tag.fromList(tags)); return rewriteCell; }
public static boolean isVisibilityTagsPresent(Cell cell) { if (cell.getTagsLength() == 0) { return false; } Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); if (tag.getType() == VISIBILITY_TAG_TYPE) { return true; } } return false; }
@Override public ReturnCode filterKeyValue(Cell v) throws IOException { try { Thread.sleep(SERVER_TIME_LIMIT + 10); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } return Bytes.equals(CellUtil.cloneRow(v), ROWS[NUM_ROWS - 1]) ? ReturnCode.INCLUDE : ReturnCode.SKIP; }
/** * Get total size of raw cells * @param result * @return Total size. */ public static long getTotalSizeOfCells(Result result) { long size = 0; if (result.isEmpty()) { return size; } for (Cell c : result.rawCells()) { size += CellUtil.estimatedHeapSizeOf(c); } return size; }
private void filterCellByStore(Entry logEntry) { Map<byte[], Long> maxSeqIdInStores = regionMaxSeqIdInStores.get(Bytes.toString(logEntry.getKey().getEncodedRegionName())); if (maxSeqIdInStores == null || maxSeqIdInStores.isEmpty()) { return; } // Create the array list for the cells that aren't filtered. // We make the assumption that most cells will be kept. ArrayList<Cell> keptCells = new ArrayList<Cell>(logEntry.getEdit().getCells().size()); for (Cell cell : logEntry.getEdit().getCells()) { if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) { keptCells.add(cell); } else { byte[] family = CellUtil.cloneFamily(cell); Long maxSeqId = maxSeqIdInStores.get(family); // Do not skip cell even if maxSeqId is null. Maybe we are in a rolling upgrade, // or the master was crashed before and we can not get the information. if (maxSeqId == null || maxSeqId.longValue() < logEntry.getKey().getLogSeqNum()) { keptCells.add(cell); } } } // Anything in the keptCells array list is still live. // So rather than removing the cells from the array list // which would be an O(n^2) operation, we just replace the list logEntry.getEdit().setCells(keptCells); }
@GET @Produces(MIMETYPE_BINARY) public Response getBinary(final @Context UriInfo uriInfo) { if (LOG.isDebugEnabled()) { LOG.debug("GET " + uriInfo.getAbsolutePath() + " as "+ MIMETYPE_BINARY); } servlet.getMetrics().incrementRequests(1); // doesn't make sense to use a non specific coordinate as this can only // return a single cell if (!rowspec.hasColumns() || rowspec.getColumns().length > 1) { servlet.getMetrics().incrementFailedGetRequests(1); return Response.status(Response.Status.BAD_REQUEST).type(MIMETYPE_TEXT) .entity("Bad request: Either 0 or more than 1 columns specified." + CRLF).build(); } MultivaluedMap<String, String> params = uriInfo.getQueryParameters(); try { ResultGenerator generator = ResultGenerator.fromRowSpec(tableResource.getName(), rowspec, null, !params.containsKey(NOCACHE_PARAM_NAME)); if (!generator.hasNext()) { servlet.getMetrics().incrementFailedGetRequests(1); return Response.status(Response.Status.NOT_FOUND) .type(MIMETYPE_TEXT).entity("Not found" + CRLF) .build(); } Cell value = generator.next(); ResponseBuilder response = Response.ok(CellUtil.cloneValue(value)); response.header("X-Timestamp", value.getTimestamp()); servlet.getMetrics().incrementSucessfulGetRequests(1); return response.build(); } catch (Exception e) { servlet.getMetrics().incrementFailedGetRequests(1); return processException(e); } }
public static List<InputSplit> getSplits(Scan scan, SnapshotManifest manifest, List<HRegionInfo> regionManifests, Path restoreDir, Configuration conf) throws IOException { // load table descriptor HTableDescriptor htd = manifest.getTableDescriptor(); Path tableDir = FSUtils.getTableDir(restoreDir, htd.getTableName()); List<InputSplit> splits = new ArrayList<InputSplit>(); for (HRegionInfo hri : regionManifests) { // load region descriptor if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), hri.getStartKey(), hri.getEndKey())) { // compute HDFS locations from snapshot files (which will get the locations for // referred hfiles) List<String> hosts = getBestLocations(conf, HRegion.computeHDFSBlocksDistribution(conf, htd, hri, tableDir)); int len = Math.min(3, hosts.size()); hosts = hosts.subList(0, len); splits.add(new InputSplit(htd, hri, hosts, scan, restoreDir)); } } return splits; }
private List<Cell> filterExpectedResults(Set<String> qualSet, byte[] startRow, byte[] endRow, int maxVersions) { final List<Cell> filteredKVs = new ArrayList<Cell>(); final Map<String, Integer> verCount = new HashMap<String, Integer>(); for (Cell kv : expectedKVs) { if (startRow.length > 0 && Bytes.compareTo(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), startRow, 0, startRow.length) < 0) { continue; } // In this unit test the end row is always inclusive. if (endRow.length > 0 && Bytes.compareTo(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), endRow, 0, endRow.length) > 0) { continue; } if (!qualSet.isEmpty() && (!CellUtil.matchingFamily(kv, FAMILY_BYTES) || !qualSet.contains(Bytes.toString(CellUtil.cloneQualifier(kv))))) { continue; } final String rowColStr = Bytes.toStringBinary(CellUtil.cloneRow(kv)) + "/" + Bytes.toStringBinary(CellUtil.cloneFamily(kv)) + ":" + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)); final Integer curNumVer = verCount.get(rowColStr); final int newNumVer = curNumVer != null ? (curNumVer + 1) : 1; if (newNumVer <= maxVersions) { filteredKVs.add(kv); verCount.put(rowColStr, newNumVer); } } return filteredKVs; }
protected Cell parseCell() throws IOException { CellProtos.Cell pbcell = CellProtos.Cell.parseDelimitedFrom(this.in); return CellUtil.createCell(pbcell.getRow().toByteArray(), pbcell.getFamily().toByteArray(), pbcell.getQualifier().toByteArray(), pbcell.getTimestamp(), (byte)pbcell.getCellType().getNumber(), pbcell.getValue().toByteArray()); }