/** * Checks whether cell contains any tag with type as VISIBILITY_TAG_TYPE. This * tag type is reserved and should not be explicitly set by user. There are * two versions of this method one that accepts pair and other without pair. * In case of preAppend and preIncrement the additional operations are not * needed like checking for STRING_VIS_TAG_TYPE and hence the API without pair * could be used. * * @param cell * @return true or false * @throws IOException */ private boolean checkForReservedVisibilityTagPresence(Cell cell) throws IOException { // Bypass this check when the operation is done by a system/super user. // This is done because, while Replication, the Cells coming to the peer // cluster with reserved // typed tags and this is fine and should get added to the peer cluster // table if (isSystemOrSuperUser()) { return true; } if (cell.getTagsLength() > 0) { Iterator<Tag> tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagsItr.hasNext()) { if (RESERVED_VIS_TAG_TYPES.contains(tagsItr.next().getType())) { return false; } } } return true; }
/** * Extract the visibility tags of the given Cell into the given List * @param cell - the cell * @param tags - the array that will be populated if visibility tags are present * @return The visibility tags serialization format */ public static Byte extractVisibilityTags(Cell cell, List<Tag> tags) { Byte serializationFormat = null; if (cell.getTagsLength() > 0) { Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) { serializationFormat = tag.getBuffer()[tag.getTagOffset()]; } else if (tag.getType() == VISIBILITY_TAG_TYPE) { tags.add(tag); } } } return serializationFormat; }
/** * Extracts and partitions the visibility tags and nonVisibility Tags * * @param cell - the cell for which we would extract and partition the * visibility and non visibility tags * @param visTags * - all the visibilty tags of type TagType.VISIBILITY_TAG_TYPE would * be added to this list * @param nonVisTags - all the non visibility tags would be added to this list * @return - the serailization format of the tag. Can be null if no tags are found or * if there is no visibility tag found */ public static Byte extractAndPartitionTags(Cell cell, List<Tag> visTags, List<Tag> nonVisTags) { Byte serializationFormat = null; if (cell.getTagsLength() > 0) { Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) { serializationFormat = tag.getBuffer()[tag.getTagOffset()]; } else if (tag.getType() == VISIBILITY_TAG_TYPE) { visTags.add(tag); } else { // ignore string encoded visibility expressions, will be added in replication handling nonVisTags.add(tag); } } } return serializationFormat; }
@Override public boolean matchVisibility(List<Tag> putVisTags, Byte putTagsFormat, List<Tag> deleteVisTags, Byte deleteTagsFormat) throws IOException { if ((deleteTagsFormat != null && deleteTagsFormat == SORTED_ORDINAL_SERIALIZATION_FORMAT) && (putTagsFormat == null || putTagsFormat == SORTED_ORDINAL_SERIALIZATION_FORMAT)) { if (putVisTags.size() == 0) { // Early out if there are no tags in the cell return false; } if (putTagsFormat == null) { return matchUnSortedVisibilityTags(putVisTags, deleteVisTags); } else { return matchOrdinalSortedVisibilityTags(putVisTags, deleteVisTags); } } throw new IOException("Unexpected tag format passed for comparison, deleteTagsFormat : " + deleteTagsFormat + ", putTagsFormat : " + putTagsFormat); }
/** * @param putVisTags Visibility tags in Put Mutation * @param deleteVisTags Visibility tags in Delete Mutation * @return true when all the visibility tags in Put matches with visibility tags in Delete. * This is used when both the set of tags are sorted based on the label ordinal. */ private static boolean matchOrdinalSortedVisibilityTags(List<Tag> putVisTags, List<Tag> deleteVisTags) { boolean matchFound = false; // If the size does not match. Definitely we are not comparing the equal tags. if ((deleteVisTags.size()) == putVisTags.size()) { for (Tag tag : deleteVisTags) { matchFound = false; for (Tag givenTag : putVisTags) { if (Bytes.equals(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength(), givenTag.getBuffer(), givenTag.getTagOffset(), givenTag.getTagLength())) { matchFound = true; break; } } if (!matchFound) break; } } return matchFound; }
private static void addCellPermissions(final byte[] perms, Map<byte[], List<Cell>> familyMap) { // Iterate over the entries in the familyMap, replacing the cells therein // with new cells including the ACL data for (Map.Entry<byte[], List<Cell>> e: familyMap.entrySet()) { List<Cell> newCells = Lists.newArrayList(); for (Cell cell: e.getValue()) { // Prepend the supplied perms in a new ACL tag to an update list of tags for the cell List<Tag> tags = Lists.newArrayList(new Tag(AccessControlLists.ACL_TAG_TYPE, perms)); if (cell.getTagsLength() > 0) { Iterator<Tag> tagIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagIterator.hasNext()) { tags.add(tagIterator.next()); } } newCells.add(new TagRewriteCell(cell, Tag.fromList(tags))); } // This is supposed to be safe, won't CME e.setValue(newCells); } }
private static Map<String, Object> toStringMap(Cell cell) { Map<String, Object> stringMap = new HashMap<String, Object>(); stringMap.put("row", Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())); stringMap.put("family", Bytes.toStringBinary(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength())); stringMap.put("qualifier", Bytes.toStringBinary(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength())); stringMap.put("timestamp", cell.getTimestamp()); stringMap.put("vlen", cell.getValueLength()); if (cell.getTagsLength() > 0) { List<String> tagsString = new ArrayList<String>(); Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagsIterator.hasNext()) { Tag tag = tagsIterator.next(); tagsString.add((tag.getType()) + ":" + Bytes.toStringBinary(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength())); } stringMap.put("tag", tagsString); } return stringMap; }
@Override public List<Tag> createVisibilityExpTags(String visExpression, boolean withSerializationFormat, boolean checkAuths) throws IOException { ExpressionNode node = null; try { node = this.expressionParser.parse(visExpression); } catch (ParseException e) { throw new IOException(e); } node = this.expressionExpander.expand(node); List<Tag> tags = new ArrayList<Tag>(); if (withSerializationFormat) { tags.add(STRING_SERIALIZATION_FORMAT_TAG); } if (node instanceof NonLeafExpressionNode && ((NonLeafExpressionNode) node).getOperator() == Operator.OR) { for (ExpressionNode child : ((NonLeafExpressionNode) node).getChildExps()) { tags.add(createTag(child)); } } else { tags.add(createTag(node)); } return tags; }
private static boolean checkForMatchingVisibilityTagsWithSortedOrder(List<Tag> putVisTags, List<Tag> deleteVisTags) { boolean matchFound = false; // If the size does not match. Definitely we are not comparing the equal // tags. if ((deleteVisTags.size()) == putVisTags.size()) { for (Tag tag : deleteVisTags) { matchFound = false; for (Tag givenTag : putVisTags) { if (Bytes.equals(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength(), givenTag.getBuffer(), givenTag.getTagOffset(), givenTag.getTagLength())) { matchFound = true; break; } } if (!matchFound) break; } } return matchFound; }
protected static void doAssert(byte[] row, String visTag) throws Exception { if (VisibilityReplicationEndPointForTest.lastEntries == null) { return; // first call } Assert.assertEquals(1, VisibilityReplicationEndPointForTest.lastEntries.size()); List<Cell> cells = VisibilityReplicationEndPointForTest.lastEntries.get(0).getEdit().getCells(); Assert.assertEquals(4, cells.size()); boolean tagFound = false; for (Cell cell : cells) { if ((Bytes.equals(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), row, 0, row.length))) { List<Tag> tags = Tag .asList(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); for (Tag tag : tags) { if (tag.getType() == TagType.STRING_VIS_TAG_TYPE) { assertEquals(visTag, Bytes.toString(tag.getValue())); tagFound = true; break; } } } } assertTrue(tagFound); }
/** * This verifies that each cell has a tag that is equal to its rowkey name. For this to work * the hbase instance must have HConstants.RPC_CODEC_CONF_KEY set to * KeyValueCodecWithTags.class.getCanonicalName()); * @param table table containing tagged cells * @throws IOException if problems reading table */ public static void verifyTags(Table table) throws IOException { ResultScanner s = table.getScanner(new Scan()); for (Result r : s) { for (Cell c : r.listCells()) { byte[] ta = c.getTagsArray(); int toff = c.getTagsOffset(); int tlen = c.getTagsLength(); Tag t = Tag.getTag(ta, toff, tlen, TagType.ACL_TAG_TYPE); if (t == null) { fail(c.toString() + " has null tag"); continue; } byte[] tval = t.getValue(); assertArrayEquals(c.toString() + " has tag" + Bytes.toString(tval), r.getRow(), tval); } } }
private void writeStoreFile(final StoreFile.Writer writer) throws IOException { byte[] fam = Bytes.toBytes("f"); byte[] qualifier = Bytes.toBytes("q"); long now = System.currentTimeMillis(); byte[] b = Bytes.toBytes("k1"); Tag t1 = new Tag((byte) 1, "tag1"); Tag t2 = new Tag((byte) 2, "tag2"); Tag t3 = new Tag((byte) 3, "tag3"); try { writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t1 })); b = Bytes.toBytes("k3"); writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t2, t1 })); b = Bytes.toBytes("k4"); writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t3 })); b = Bytes.toBytes("k5"); writer.append(new KeyValue(b, fam, qualifier, now, b, new Tag[] { t3 })); } finally { writer.close(); } }
@Override public boolean postScannerNext(ObserverContext<RegionCoprocessorEnvironment> e, InternalScanner s, List<Result> results, int limit, boolean hasMore) throws IOException { if (checkTagPresence) { if (results.size() > 0) { // Check tag presence in the 1st cell in 1st Result Result result = results.get(0); CellScanner cellScanner = result.cellScanner(); if (cellScanner.advance()) { Cell cell = cellScanner.current(); tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); } } } return hasMore; }
@Test public void testKVWithTags() throws Exception { CompressionContext ctx = new CompressionContext(LRUDictionary.class, false, false); DataOutputBuffer buf = new DataOutputBuffer(BUF_SIZE); KeyValueCompression.writeKV(buf, createKV(1), ctx); KeyValueCompression.writeKV(buf, createKV(0), ctx); KeyValueCompression.writeKV(buf, createKV(2), ctx); ctx.clear(); DataInputStream in = new DataInputStream(new ByteArrayInputStream( buf.getData(), 0, buf.getLength())); KeyValue readBack = KeyValueCompression.readKV(in, ctx); List<Tag> tags = readBack.getTags(); assertEquals(1, tags.size()); }
/** * Test data block encoding of empty KeyValue. * * @throws IOException * On test failure. */ @Test public void testEmptyKeyValues() throws IOException { List<KeyValue> kvList = new ArrayList<KeyValue>(); byte[] row = new byte[0]; byte[] family = new byte[0]; byte[] qualifier = new byte[0]; byte[] value = new byte[0]; if (!includesTags) { kvList.add(new KeyValue(row, family, qualifier, 0l, value)); kvList.add(new KeyValue(row, family, qualifier, 0l, value)); } else { byte[] metaValue1 = Bytes.toBytes("metaValue1"); byte[] metaValue2 = Bytes.toBytes("metaValue2"); kvList.add(new KeyValue(row, family, qualifier, 0l, value, new Tag[] { new Tag((byte) 1, metaValue1) })); kvList.add(new KeyValue(row, family, qualifier, 0l, value, new Tag[] { new Tag((byte) 1, metaValue2) })); } testEncodersOnDataset(kvList, includesMemstoreTS, includesTags); }
/** * Test KeyValues with negative timestamp. * * @throws IOException * On test failure. */ @Test public void testNegativeTimestamps() throws IOException { List<KeyValue> kvList = new ArrayList<KeyValue>(); byte[] row = new byte[0]; byte[] family = new byte[0]; byte[] qualifier = new byte[0]; byte[] value = new byte[0]; if (includesTags) { byte[] metaValue1 = Bytes.toBytes("metaValue1"); byte[] metaValue2 = Bytes.toBytes("metaValue2"); kvList.add(new KeyValue(row, family, qualifier, 0l, value, new Tag[] { new Tag((byte) 1, metaValue1) })); kvList.add(new KeyValue(row, family, qualifier, 0l, value, new Tag[] { new Tag((byte) 1, metaValue2) })); } else { kvList.add(new KeyValue(row, family, qualifier, -1l, Type.Put, value)); kvList.add(new KeyValue(row, family, qualifier, -2l, Type.Put, value)); } testEncodersOnDataset(kvList, includesMemstoreTS, includesTags); }
@Test public void testZeroByte() throws IOException { List<KeyValue> kvList = new ArrayList<KeyValue>(); byte[] row = Bytes.toBytes("abcd"); byte[] family = new byte[] { 'f' }; byte[] qualifier0 = new byte[] { 'b' }; byte[] qualifier1 = new byte[] { 'c' }; byte[] value0 = new byte[] { 'd' }; byte[] value1 = new byte[] { 0x00 }; if (includesTags) { kvList.add(new KeyValue(row, family, qualifier0, 0, value0, new Tag[] { new Tag((byte) 1, "value1") })); kvList.add(new KeyValue(row, family, qualifier1, 0, value1, new Tag[] { new Tag((byte) 1, "value1") })); } else { kvList.add(new KeyValue(row, family, qualifier0, 0, Type.Put, value0)); kvList.add(new KeyValue(row, family, qualifier1, 0, Type.Put, value1)); } testEncodersOnDataset(kvList, includesMemstoreTS, includesTags); }
private static Map<String, Object> cellToStringMap(Cell c) { Map<String, Object> stringMap = new HashMap<String, Object>(); stringMap.put("qualifier", Bytes.toStringBinary(c.getQualifierArray(), c.getQualifierOffset(), c.getQualifierLength())); stringMap.put("timestamp", c.getTimestamp()); stringMap.put("vlen", c.getValueLength()); List<Tag> tags = Tag.asList(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength()); if (tags != null) { List<String> tagsString = new ArrayList<String>(); for (Tag t : tags) { tagsString.add((t.getType()) + ":" + Bytes.toStringBinary(t.getValue())); } stringMap.put("tag", tagsString); } return stringMap; }
@Override void testRow(final int i) throws IOException { byte[] row = getRandomRow(this.rand, this.totalRows); Put put = new Put(row); byte[] value = generateData(this.rand, ROW_LENGTH); if (useTags) { byte[] tag = generateData(this.rand, TAG_LENGTH); Tag[] tags = new Tag[noOfTags]; for (int n = 0; n < noOfTags; n++) { Tag t = new Tag((byte) n, tag); tags[n] = t; } KeyValue kv = new KeyValue(row, FAMILY_NAME, QUALIFIER_NAME, HConstants.LATEST_TIMESTAMP, value, tags); put.add(kv); } else { put.add(FAMILY_NAME, QUALIFIER_NAME, value); } put.setDurability(writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL); mutator.mutate(put); }
@Override void testRow(final int i) throws IOException { byte[] row = format(i); Put put = new Put(row); byte[] value = generateData(this.rand, ROW_LENGTH); if (useTags) { byte[] tag = generateData(this.rand, TAG_LENGTH); Tag[] tags = new Tag[noOfTags]; for (int n = 0; n < noOfTags; n++) { Tag t = new Tag((byte) n, tag); tags[n] = t; } KeyValue kv = new KeyValue(row, FAMILY_NAME, QUALIFIER_NAME, HConstants.LATEST_TIMESTAMP, value, tags); put.add(kv); } else { put.add(FAMILY_NAME, QUALIFIER_NAME, value); } put.setDurability(writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL); mutator.mutate(put); }
public Pair<Cell, Cell> getCells(final Element element, final Pair<byte[], byte[]> row, final byte[] cq, final CellCreator kvCreator) throws SerialisationException { final long ts = getTimestamp(element.getProperties()); final byte[] value = getValue(element); final String visibilityStr = Bytes.toString(getColumnVisibility(element)); byte[] tags = null; if (!visibilityStr.isEmpty()) { try { tags = Tag.fromList(kvCreator.getVisibilityExpressionResolver().createVisibilityExpTags(visibilityStr)); } catch (final IOException e) { throw new RuntimeException("Unable to create visibility tags", e); } } final Cell cell = CellUtil.createCell(row.getFirst(), HBaseStoreConstants.getColFam(), cq, ts, KeyValue.Type.Maximum, value, tags); final Pair<Cell, Cell> cells = new Pair<>(cell); if (null != row.getSecond()) { cells.setSecond(CellUtil.createCell(row.getSecond(), HBaseStoreConstants.getColFam(), cq, ts, KeyValue.Type.Maximum, value, tags)); } return cells; }
private void checkForReservedTagPresence(User user, Mutation m) throws IOException { // Superusers are allowed to store cells unconditionally. if (superusers.contains(user.getShortName())) { return; } // We already checked (prePut vs preBatchMutation) if (m.getAttribute(TAG_CHECK_PASSED) != null) { return; } for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) { Cell cell = cellScanner.current(); if (cell.getTagsLength() > 0) { Iterator<Tag> tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength()); while (tagsItr.hasNext()) { if (tagsItr.next().getType() == AccessControlLists.ACL_TAG_TYPE) { throw new AccessDeniedException("Mutation contains cell with reserved type tag"); } } } } m.setAttribute(TAG_CHECK_PASSED, TRUE); }
@Override public List<Tag> createVisibilityExpTags(String visExpression) throws IOException { VisibilityLabelOrdinalProvider provider = new VisibilityLabelOrdinalProvider() { @Override public int getLabelOrdinal(String label) { return labels.get(label); } @Override public String getLabel(int ordinal) { // Unused throw new UnsupportedOperationException( "getLabel should not be used in VisibilityExpressionResolver"); } }; return VisibilityUtils.createVisibilityExpTags(visExpression, true, false, null, provider); }
@Test public void testReservedCellTags() throws Exception { AccessTestAction putWithReservedTag = new AccessTestAction() { @Override public Object run() throws Exception { Table t = new HTable(conf, TEST_TABLE.getTableName()); try { KeyValue kv = new KeyValue(TEST_ROW, TEST_FAMILY, TEST_QUALIFIER, HConstants.LATEST_TIMESTAMP, HConstants.EMPTY_BYTE_ARRAY, new Tag[] { new Tag(AccessControlLists.ACL_TAG_TYPE, ProtobufUtil.toUsersAndPermissions(USER_OWNER.getShortName(), new Permission(Permission.Action.READ)).toByteArray()) }); t.put(new Put(TEST_ROW).add(kv)); } finally { t.close(); } return null; } }; // Current user is superuser verifyAllowed(putWithReservedTag, User.getCurrent()); // No other user should be allowed verifyDenied(putWithReservedTag, USER_OWNER, USER_ADMIN, USER_CREATE, USER_RW, USER_RO); }
private int writeSomeRecords(Writer writer, int start, int n, boolean useTags) throws IOException { String value = "value"; KeyValue kv; for (int i = start; i < (start + n); i++) { String key = String.format(localFormatter, Integer.valueOf(i)); if (useTags) { Tag t = new Tag((byte) 1, "myTag1"); Tag[] tags = new Tag[1]; tags[0] = t; kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value + key), tags); writer.append(kv); } else { kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), Bytes.toBytes(value + key)); writer.append(kv); } } return (start + n); }