public static Increment incrementFromThrift(TIncrement in) throws IOException { Increment out = new Increment(in.getRow()); for (TColumnIncrement column : in.getColumns()) { out.addColumn(column.getFamily(), column.getQualifier(), column.getAmount()); } if (in.isSetAttributes()) { addAttributes(out,in.getAttributes()); } if (in.isSetDurability()) { out.setDurability(durabilityFromThrift(in.getDurability())); } if(in.getCellVisibility() != null) { out.setCellVisibility(new CellVisibility(in.getCellVisibility().getExpression())); } return out; }
public static Append appendFromThrift(TAppend append) throws IOException { Append out = new Append(append.getRow()); for (TColumnValue column : append.getColumns()) { out.add(column.getFamily(), column.getQualifier(), column.getValue()); } if (append.isSetAttributes()) { addAttributes(out, append.getAttributes()); } if (append.isSetDurability()) { out.setDurability(durabilityFromThrift(append.getDurability())); } if(append.getCellVisibility() != null) { out.setCellVisibility(new CellVisibility(append.getCellVisibility().getExpression())); } return out; }
@Override protected void map(NullWritable key, NullWritable value, Context context) throws IOException, InterruptedException { String suffix = "/" + shortTaskId; int BLOCK_SIZE = (int) (recordsToWrite / 100); for (long i = 0; i < recordsToWrite;) { for (long idx = 0; idx < BLOCK_SIZE && i < recordsToWrite; idx++, i++) { int expIdx = rand.nextInt(BLOCK_SIZE) % VISIBILITY_EXPS_COUNT; String exp = VISIBILITY_EXPS[expIdx]; byte[] row = Bytes.add(Bytes.toBytes(i), Bytes.toBytes(suffix), Bytes.toBytes(exp)); Put p = new Put(row); p.add(TEST_FAMILY, TEST_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY); p.setCellVisibility(new CellVisibility(exp)); getCounter(expIdx).increment(1); mutator.mutate(p); if (i % 100 == 0) { context.setStatus("Written " + i + "/" + recordsToWrite + " records"); context.progress(); } } // End of block, flush all of them before we start writing anything // pointing to these! mutator.flush(); } }
private static int insertData(TableName tableName, String column, double prob) throws IOException { byte[] k = new byte[3]; byte[][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column)); List<Put> puts = new ArrayList<>(); for (int i = 0; i < 9; i++) { Put put = new Put(Bytes.toBytes("row" + i)); put.setDurability(Durability.SKIP_WAL); put.add(famAndQf[0], famAndQf[1], k); put.setCellVisibility(new CellVisibility("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET)); puts.add(put); } try (Table table = new HTable(TEST_UTIL.getConfiguration(), tableName)) { table.put(puts); } return puts.size(); }
public Pair<Put, Put> getPuts(final Element element, final Pair<byte[], byte[]> row, final byte[] cq) throws SerialisationException { final long ts = getTimestamp(element); final byte[] value = getValue(element); final String visibilityStr = Bytes.toString(getColumnVisibility(element)); final CellVisibility cellVisibility = visibilityStr.isEmpty() ? null : new CellVisibility(visibilityStr); final Put put = new Put(row.getFirst()); put.addColumn(HBaseStoreConstants.getColFam(), cq, ts, value); if (null != cellVisibility) { put.setCellVisibility(cellVisibility); } final Pair<Put, Put> puts = new Pair<>(put); if (null != row.getSecond()) { final Put put2 = new Put(row.getSecond()); put2.addColumn(HBaseStoreConstants.getColFam(), cq, value); if (null != cellVisibility) { put2.setCellVisibility(cellVisibility); } puts.setSecond(put2); } return puts; }
protected KeyValue createPuts(byte[] lineBytes, ImportTsv.TsvParser.ParsedLine parsed, Put put, int i) throws BadTsvLineException, IOException { KeyValue kv = null; if (hfileOutPath == null) { kv = new KeyValue(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength(), parser.getFamily(i), 0, parser.getFamily(i).length, parser.getQualifier(i), 0, parser.getQualifier(i).length, ts, KeyValue.Type.Put, lineBytes, parsed.getColumnOffset(i), parsed.getColumnLength(i)); if (cellVisibilityExpr != null) { // We won't be validating the expression here. The Visibility CP will do // the validation put.setCellVisibility(new CellVisibility(cellVisibilityExpr)); } } else { kv = labelExpander.createKVFromCellVisibilityExpr( parsed.getRowKeyOffset(), parsed.getRowKeyLength(), parser.getFamily(i), 0, parser.getFamily(i).length, parser.getQualifier(i), 0, parser.getQualifier(i).length, ts, KeyValue.Type.Put, lineBytes, parsed.getColumnOffset(i), parsed.getColumnLength(i), cellVisibilityExpr); } put.add(kv); return kv; }
private static int insertData(String tableName, String column, double prob) throws IOException { Random rng = new Random(); int count = 0; HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); byte[] k = new byte[3]; byte[][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(column)); for (int i = 0; i < 9; i++) { Put put = new Put(Bytes.toBytes("row" + i)); put.setDurability(Durability.SKIP_WAL); put.add(famAndQf[0], famAndQf[1], k); put.setCellVisibility(new CellVisibility("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET)); table.put(put); count++; } table.flushCommits(); return count; }
@Override protected void map(NullWritable key, NullWritable value, Context context) throws IOException, InterruptedException { String suffix = "/" + shortTaskId; int BLOCK_SIZE = (int) (recordsToWrite / 100); for (long i = 0; i < recordsToWrite;) { for (long idx = 0; idx < BLOCK_SIZE && i < recordsToWrite; idx++, i++) { int expIdx = rand.nextInt(BLOCK_SIZE) % VISIBILITY_EXPS_COUNT; String exp = VISIBILITY_EXPS[expIdx]; byte[] row = Bytes.add(Bytes.toBytes(i), Bytes.toBytes(suffix), Bytes.toBytes(exp)); Put p = new Put(row); p.add(TEST_FAMILY, TEST_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY); p.setCellVisibility(new CellVisibility(exp)); getCounter(expIdx).increment(1); table.put(p); if (i % 100 == 0) { context.setStatus("Written " + i + "/" + recordsToWrite + " records"); context.progress(); } } // End of block, flush all of them before we start writing anything // pointing to these! table.flushCommits(); } }
public static Append appendFromThrift(TAppend append) throws IOException { Append out = new Append(append.getRow()); for (TColumnValue column : append.getColumns()) { out.addColumn(column.getFamily(), column.getQualifier(), column.getValue()); } if (append.isSetAttributes()) { addAttributes(out, append.getAttributes()); } if (append.isSetDurability()) { out.setDurability(durabilityFromThrift(append.getDurability())); } if(append.getCellVisibility() != null) { out.setCellVisibility(new CellVisibility(append.getCellVisibility().getExpression())); } return out; }
protected void populatePut(byte[] lineBytes, ImportTsv.TsvParser.ParsedLine parsed, Put put, int i) throws BadTsvLineException, IOException { Cell cell = null; if (hfileOutPath == null) { cell = new KeyValue(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength(), parser.getFamily(i), 0, parser.getFamily(i).length, parser.getQualifier(i), 0, parser.getQualifier(i).length, ts, KeyValue.Type.Put, lineBytes, parsed.getColumnOffset(i), parsed.getColumnLength(i)); if (cellVisibilityExpr != null) { // We won't be validating the expression here. The Visibility CP will do // the validation put.setCellVisibility(new CellVisibility(cellVisibilityExpr)); } if (ttl > 0) { put.setTTL(ttl); } } else { // Creating the KV which needs to be directly written to HFiles. Using the Facade // KVCreator for creation of kvs. cell = this.kvCreator.create(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength(), parser.getFamily(i), 0, parser.getFamily(i).length, parser.getQualifier(i), 0, parser.getQualifier(i).length, ts, lineBytes, parsed.getColumnOffset(i), parsed.getColumnLength(i), tags); } put.add(cell); }
@Override protected void map(NullWritable key, NullWritable value, Context context) throws IOException, InterruptedException { String suffix = "/" + shortTaskId; int BLOCK_SIZE = (int) (recordsToWrite / 100); for (long i = 0; i < recordsToWrite;) { for (long idx = 0; idx < BLOCK_SIZE && i < recordsToWrite; idx++, i++) { int expIdx = rand.nextInt(BLOCK_SIZE) % VISIBILITY_EXPS_COUNT; String exp = VISIBILITY_EXPS[expIdx]; byte[] row = Bytes.add(Bytes.toBytes(i), Bytes.toBytes(suffix), Bytes.toBytes(exp)); Put p = new Put(row); p.addColumn(TEST_FAMILY, TEST_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY); p.setCellVisibility(new CellVisibility(exp)); getCounter(expIdx).increment(1); mutator.mutate(p); if (i % 100 == 0) { context.setStatus("Written " + i + "/" + recordsToWrite + " records"); context.progress(); } } // End of block, flush all of them before we start writing anything // pointing to these! mutator.flush(); } }
@Override protected void processKV(ImmutableBytesWritable key, Result result, org.apache.hadoop.mapreduce.Mapper.Context context, Put put, org.apache.hadoop.hbase.client.Delete delete) throws IOException, InterruptedException { String visibilityExps = split[index * 2] + OR + split[(index * 2) + 1]; for (Cell kv : result.rawCells()) { // skip if we filter it out if (kv == null) continue; // Create deletes here if (delete == null) { delete = new Delete(key.get()); } delete.setCellVisibility(new CellVisibility(visibilityExps)); delete.addFamily(CellUtil.cloneFamily(kv)); } if (delete != null) { context.write(key, delete); } }
private static int insertData(TableName tableName, String column, double prob) throws IOException { byte[] k = new byte[3]; byte[][] famAndQf = CellUtil.parseColumn(Bytes.toBytes(column)); List<Put> puts = new ArrayList<>(9); for (int i = 0; i < 9; i++) { Put put = new Put(Bytes.toBytes("row" + i)); put.setDurability(Durability.SKIP_WAL); put.addColumn(famAndQf[0], famAndQf[1], k); put.setCellVisibility(new CellVisibility("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET)); puts.add(put); } try (Table table = TEST_UTIL.getConnection().getTable(tableName)) { table.put(puts); } return puts.size(); }
@Override protected RowMutations buildJoinMutation( final byte[] secondaryIndexRowId, final byte[] adapterId, final byte[] indexedAttributeFieldId, final byte[] primaryIndexId, final byte[] primaryIndexRowId, final byte[] attributeVisibility ) throws IOException { final RowMutations m = new RowMutations( secondaryIndexRowId); final Put p = new Put( secondaryIndexRowId); p.setCellVisibility(new CellVisibility( StringUtils.stringFromBinary(attributeVisibility))); p.addColumn( SecondaryIndexUtils.constructColumnFamily( adapterId, indexedAttributeFieldId), SecondaryIndexUtils.constructColumnQualifier( primaryIndexId, primaryIndexRowId), EMPTY_VALUE); m.add(p); return m; }
@Test(expected=UnsupportedOperationException.class) public void testDeleteWithCellVisibility() throws IOException { clean(); Delete delete = new Delete(Bytes.toBytes(rowPrefix)); delete.setCellVisibility(new CellVisibility("abc")); table.delete(delete); }
@Override protected void persist(org.apache.hadoop.mapreduce.Mapper.Context output, long count, byte[][] prev, byte[][] current, byte[] id) throws IOException { String visibilityExps = ""; String[] split = labels.split(COMMA); for (int i = 0; i < current.length; i++) { for (int j = 0; j < DEFAULT_TABLES_COUNT; j++) { Put put = new Put(current[i]); put.add(FAMILY_NAME, COLUMN_PREV, prev == null ? NO_KEY : prev[i]); if (count >= 0) { put.add(FAMILY_NAME, COLUMN_COUNT, Bytes.toBytes(count + i)); } if (id != null) { put.add(FAMILY_NAME, COLUMN_CLIENT, id); } visibilityExps = split[j * 2] + OR + split[(j * 2) + 1]; put.setCellVisibility(new CellVisibility(visibilityExps)); tables[j].mutate(put); try { Thread.sleep(1); } catch (InterruptedException e) { throw new IOException(); } } if (i % 1000 == 0) { // Tickle progress every so often else maprunner will think us hung output.progress(); } } }
/** * Convert a protocol buffer CellVisibility bytes to a client CellVisibility * * @param protoBytes * @return the converted client CellVisibility * @throws DeserializationException */ public static CellVisibility toCellVisibility(byte[] protoBytes) throws DeserializationException { if (protoBytes == null) return null; ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder(); ClientProtos.CellVisibility proto = null; try { ProtobufUtil.mergeFrom(builder, protoBytes); proto = builder.build(); } catch (IOException e) { throw new DeserializationException(e); } return toCellVisibility(proto); }
protected void populatePut(byte[] lineBytes, ImportTsv.TsvParser.ParsedLine parsed, Put put, int i) throws BadTsvLineException, IOException { Cell cell = null; if (hfileOutPath == null) { cell = new KeyValue(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength(), parser.getFamily(i), 0, parser.getFamily(i).length, parser.getQualifier(i), 0, parser.getQualifier(i).length, ts, KeyValue.Type.Put, lineBytes, parsed.getColumnOffset(i), parsed.getColumnLength(i)); if (cellVisibilityExpr != null) { // We won't be validating the expression here. The Visibility CP will do // the validation put.setCellVisibility(new CellVisibility(cellVisibilityExpr)); } if (ttl > 0) { put.setTTL(ttl); } } else { // Creating the KV which needs to be directly written to HFiles. Using the Facade // KVCreator for creation of kvs. List<Tag> tags = new ArrayList<Tag>(); if (cellVisibilityExpr != null) { tags.addAll(kvCreator.getVisibilityExpressionResolver() .createVisibilityExpTags(cellVisibilityExpr)); } // Add TTL directly to the KV so we can vary them when packing more than one KV // into puts if (ttl > 0) { tags.add(new Tag(TagType.TTL_TAG_TYPE, Bytes.toBytes(ttl))); } cell = this.kvCreator.create(lineBytes, parsed.getRowKeyOffset(), parsed.getRowKeyLength(), parser.getFamily(i), 0, parser.getFamily(i).length, parser.getQualifier(i), 0, parser.getQualifier(i).length, ts, lineBytes, parsed.getColumnOffset(i), parsed.getColumnLength(i), tags); } put.add(cell); }
/** * Convert a protocol buffer CellVisibility bytes to a client CellVisibility * * @param protoBytes * @return the converted client CellVisibility * @throws DeserializationException */ public static CellVisibility toCellVisibility(byte[] protoBytes) throws DeserializationException { if (protoBytes == null) return null; ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder(); ClientProtos.CellVisibility proto = null; try { proto = builder.mergeFrom(protoBytes).build(); } catch (InvalidProtocolBufferException e) { throw new DeserializationException(e); } return toCellVisibility(proto); }
/** * Creates a {@link Put} (HBase) from a {@link TPut} (Thrift) * * @param in the <code>TPut</code> to convert * * @return converted <code>Put</code> */ public static Put putFromThrift(TPut in) { Put out; if (in.isSetTimestamp()) { out = new Put(in.getRow(), in.getTimestamp()); } else { out = new Put(in.getRow()); } if (in.isSetDurability()) { out.setDurability(durabilityFromThrift(in.getDurability())); } for (TColumnValue columnValue : in.getColumnValues()) { if (columnValue.isSetTimestamp()) { out.addImmutable( columnValue.getFamily(), columnValue.getQualifier(), columnValue.getTimestamp(), columnValue.getValue()); } else { out.addImmutable( columnValue.getFamily(), columnValue.getQualifier(), columnValue.getValue()); } } if (in.isSetAttributes()) { addAttributes(out,in.getAttributes()); } if (in.getCellVisibility() != null) { out.setCellVisibility(new CellVisibility(in.getCellVisibility().getExpression())); } return out; }
@Override protected void persist(org.apache.hadoop.mapreduce.Mapper.Context output, long count, byte[][] prev, byte[][] current, byte[] id) throws IOException { String visibilityExps = ""; String[] split = labels.split(COMMA); for (int i = 0; i < current.length; i++) { for (int j = 0; j < DEFAULT_TABLES_COUNT; j++) { Put put = new Put(current[i]); byte[] value = prev == null ? NO_KEY : prev[i]; put.addColumn(FAMILY_NAME, COLUMN_PREV, value); if (count >= 0) { put.addColumn(FAMILY_NAME, COLUMN_COUNT, Bytes.toBytes(count + i)); } if (id != null) { put.addColumn(FAMILY_NAME, COLUMN_CLIENT, id); } visibilityExps = split[j * 2] + OR + split[(j * 2) + 1]; put.setCellVisibility(new CellVisibility(visibilityExps)); tables[j].mutate(put); try { Thread.sleep(1); } catch (InterruptedException e) { throw new IOException(); } } if (i % 1000 == 0) { // Tickle progress every so often else maprunner will think us hung output.progress(); } } }
/** * Convert a protocol buffer CellVisibility bytes to a client CellVisibility * * @param protoBytes * @return the converted client CellVisibility * @throws DeserializationException */ private static CellVisibility toCellVisibility(byte[] protoBytes) throws DeserializationException { if (protoBytes == null) return null; ClientProtos.CellVisibility.Builder builder = ClientProtos.CellVisibility.newBuilder(); ClientProtos.CellVisibility proto = null; try { ProtobufUtil.mergeFrom(builder, protoBytes); proto = builder.build(); } catch (IOException e) { throw new DeserializationException(e); } return toCellVisibility(proto); }
@Override protected RowMutations buildMutation( final byte[] secondaryIndexRowId, final byte[] adapterId, final byte[] indexedAttributeFieldId, final byte[] dataId, final byte[] fieldId, final byte[] fieldValue, final byte[] fieldVisibility ) throws IOException { final RowMutations m = new RowMutations( secondaryIndexRowId); final Put p = new Put( secondaryIndexRowId); p.setCellVisibility(new CellVisibility( StringUtils.stringFromBinary(fieldVisibility))); p.addColumn( SecondaryIndexUtils.constructColumnFamily( adapterId, indexedAttributeFieldId), SecondaryIndexUtils.constructColumnQualifier( fieldId, dataId), fieldValue); m.add(p); return m; }
@Override protected void addPutToKv(Put put, Cell kv) throws IOException { String visibilityExps = split[index * 2] + OR + split[(index * 2) + 1]; put.setCellVisibility(new CellVisibility(visibilityExps)); super.addPutToKv(put, kv); }
@Override public Put setCellVisibility(CellVisibility expression) { return (Put) super.setCellVisibility(expression); }
@Override public Delete setCellVisibility(CellVisibility expression) { return (Delete) super.setCellVisibility(expression); }
/** * Sets the visibility expression associated with cells in this Mutation. * It is illegal to set <code>CellVisibility</code> on <code>Delete</code> mutation. * @param expression */ public Mutation setCellVisibility(CellVisibility expression) { this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY, ProtobufUtil .toCellVisibility(expression).toByteArray()); return this; }
/** * @return CellVisibility associated with cells in this Mutation. * @throws DeserializationException */ public CellVisibility getCellVisibility() throws DeserializationException { byte[] cellVisibilityBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY); if (cellVisibilityBytes == null) return null; return ProtobufUtil.toCellVisibility(cellVisibilityBytes); }
