private Job doVerify(Configuration conf, HTableDescriptor htd, String... auths) throws IOException, InterruptedException, ClassNotFoundException { Path outputDir = getTestDir(TEST_NAME, "verify-output"); Job job = new Job(conf); job.setJarByClass(this.getClass()); job.setJobName(TEST_NAME + " Verification for " + htd.getTableName()); setJobScannerConf(job); Scan scan = new Scan(); scan.setAuthorizations(new Authorizations(auths)); TableMapReduceUtil.initTableMapperJob(htd.getTableName().getNameAsString(), scan, VerifyMapper.class, NullWritable.class, NullWritable.class, job); TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING); TableMapReduceUtil.setScannerCaching(job, scannerCaching); job.setNumReduceTasks(0); FileOutputFormat.setOutputPath(job, outputDir); assertTrue(job.waitForCompletion(true)); return job; }
@Test public void testSetAuthorizations() { Scan scan = new Scan(); try { scan.setAuthorizations(new Authorizations("\u002b|\u0029")); scan.setAuthorizations(new Authorizations("A", "B", "0123", "A0", "1A1", "_a")); scan.setAuthorizations(new Authorizations("A|B")); scan.setAuthorizations(new Authorizations("A&B")); scan.setAuthorizations(new Authorizations("!B")); scan.setAuthorizations(new Authorizations("A", "(A)")); scan.setAuthorizations(new Authorizations("A", "{A")); scan.setAuthorizations(new Authorizations(" ")); scan.setAuthorizations(new Authorizations(":B")); scan.setAuthorizations(new Authorizations("-B")); scan.setAuthorizations(new Authorizations(".B")); scan.setAuthorizations(new Authorizations("/B")); } catch (IllegalArgumentException e) { fail("should not throw exception"); } }
private Job doVerify(Configuration conf, HTableDescriptor htd, String... auths) throws IOException, InterruptedException, ClassNotFoundException { Path outputDir = getTestDir(TEST_NAME, "verify-output"); Job job = new Job(conf); job.setJarByClass(this.getClass()); job.setJobName(TEST_NAME + " Verification for " + htd.getTableName()); setJobScannerConf(job); Scan scan = new Scan(); scan.setAuthorizations(new Authorizations(auths)); TableMapReduceUtil.initTableMapperJob(htd.getTableName().getNameAsString(), scan, VerifyMapper.class, NullWritable.class, NullWritable.class, job); TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(), AbstractHBaseTool.class); int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING); TableMapReduceUtil.setScannerCaching(job, scannerCaching); job.setNumReduceTasks(0); FileOutputFormat.setOutputPath(job, outputDir); assertTrue(job.waitForCompletion(true)); return job; }
public ResultScanner getScannedResults( final Scan scanner, final String tableName, final String... authorizations ) throws IOException { if (authorizations != null) { scanner.setAuthorizations(new Authorizations( authorizations)); } final Table table = conn.getTable(getTableName(getQualifiedTableName(tableName))); final ResultScanner results = table.getScanner(scanner); table.close(); return results; }
@Test(expected=UnsupportedOperationException.class) public void testScanRowWithAuthorization() throws IOException { clean(); String row = rowPrefix + 0; putRow(row + 0, 1000); putRow(row + 3, 1000); Scan scan = new Scan(); scan.setAuthorizations(new Authorizations()); scan.setStartRow(Bytes.toBytes(row + 0)); scan.setStopRow(Bytes.toBytes(row + 3)); ResultScanner scanResult = table.getScanner(scan); scanResult.next(); }
private int doVerify(Path outputDir, int numReducers) throws IOException, InterruptedException, ClassNotFoundException { job = new Job(getConf()); job.setJobName("Link Verifier"); job.setNumReduceTasks(numReducers); job.setJarByClass(getClass()); setJobScannerConf(job); Scan scan = new Scan(); scan.addColumn(FAMILY_NAME, COLUMN_PREV); scan.setCaching(10000); scan.setCacheBlocks(false); String[] split = labels.split(COMMA); scan.setAuthorizations(new Authorizations(split[this.labelIndex * 2], split[(this.labelIndex * 2) + 1])); TableMapReduceUtil.initTableMapperJob(tableName.getName(), scan, VerifyMapper.class, BytesWritable.class, BytesWritable.class, job); TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); job.getConfiguration().setBoolean("mapreduce.map.speculative", false); job.setReducerClass(VerifyReducer.class); job.setOutputFormatClass(TextOutputFormat.class); TextOutputFormat.setOutputPath(job, outputDir); boolean success = job.waitForCompletion(true); return success ? 0 : 1; }
/** * Convert a protocol buffer Authorizations bytes to a client Authorizations * * @param protoBytes * @return the converted client Authorizations * @throws DeserializationException */ public static Authorizations toAuthorizations(byte[] protoBytes) throws DeserializationException { if (protoBytes == null) return null; ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder(); ClientProtos.Authorizations proto = null; try { ProtobufUtil.mergeFrom(builder, protoBytes); proto = builder.build(); } catch (IOException e) { throw new DeserializationException(e); } return toAuthorizations(proto); }
/** * Create a protocol buffer Authorizations based on a client Authorizations. * * @param authorizations * @return a protocol buffer Authorizations */ public static ClientProtos.Authorizations toAuthorizations(Authorizations authorizations) { ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder(); for (String label : authorizations.getLabels()) { builder.addLabel(label); } return builder.build(); }
public static void deleteAllRows(final HBaseStore store, final String... auths) throws StoreException { final Connection connection = store.getConnection(); try { if (connection.getAdmin().tableExists(store.getTableName())) { connection.getAdmin().flush(store.getTableName()); final Table table = connection.getTable(store.getTableName()); final Scan scan = new Scan(); scan.setAuthorizations(new Authorizations(auths)); try (ResultScanner scanner = table.getScanner(scan)) { final List<Delete> deletes = new ArrayList<>(); for (final Result result : scanner) { deletes.add(new Delete(result.getRow())); } table.delete(deletes); connection.getAdmin().flush(store.getTableName()); } try (ResultScanner scanner = table.getScanner(scan)) { if (scanner.iterator().hasNext()) { throw new StoreException("Some rows in table " + store.getTableName() + " failed to delete"); } } } } catch (final IOException e) { throw new StoreException("Failed to delete all rows in table " + store.getTableName(), e); } }
public HBaseRetriever(final HBaseStore store, final OP operation, final User user, final Iterable<? extends ElementId> ids, final boolean includeMatchedVertex, final Class<?>... extraProcessors) throws StoreException { this.serialisation = new ElementSerialisation(store.getSchema()); this.rowRangeFactory = new RowRangeFactory(serialisation); this.validator = new ElementValidator(operation.getView()); this.store = store; this.operation = operation; this.ids = ids; if (null != user && null != user.getDataAuths()) { this.authorisations = new Authorizations( user.getDataAuths().toArray(new String[user.getDataAuths().size()])); } else { this.authorisations = new Authorizations(); } this.includeMatchedVertex = includeMatchedVertex; if (null != extraProcessors && extraProcessors.length > 0) { this.extraProcessors = StringUtil.toCsv(extraProcessors); } else { this.extraProcessors = null; } }
/** * Convert a protocol buffer Authorizations bytes to a client Authorizations * * @param protoBytes * @return the converted client Authorizations * @throws DeserializationException */ public static Authorizations toAuthorizations(byte[] protoBytes) throws DeserializationException { if (protoBytes == null) return null; ClientProtos.Authorizations.Builder builder = ClientProtos.Authorizations.newBuilder(); ClientProtos.Authorizations proto = null; try { proto = builder.mergeFrom(protoBytes).build(); } catch (InvalidProtocolBufferException e) { throw new DeserializationException(e); } return toAuthorizations(proto); }
private void createLabels() throws IOException { // This scan should be done by user with global_admin previliges.. Ensure // that it works HTable visibilityLabelsTable = null; ResultScanner scanner = null; try { labels = new HashMap<String, Integer>(); visibilityLabelsTable = new HTable(conf, LABELS_TABLE_NAME.getName()); Scan scan = new Scan(); scan.setAuthorizations(new Authorizations(VisibilityUtils.SYSTEM_LABEL)); scan.addColumn(LABELS_TABLE_FAMILY, LABEL_QUALIFIER); scanner = visibilityLabelsTable.getScanner(scan); while (true) { Result next = scanner.next(); if (next == null) { break; } byte[] row = next.getRow(); byte[] value = next.getValue(LABELS_TABLE_FAMILY, LABEL_QUALIFIER); labels.put(Bytes.toString(value), Bytes.toInt(row)); } } finally { try { if (scanner != null) { scanner.close(); } } finally { if (visibilityLabelsTable != null) { visibilityLabelsTable.close(); } } } }
public int runCopier(String outputDir) throws Exception { Job job = null; Scan scan = null; job = new Job(getConf()); job.setJobName("Data copier"); job.getConfiguration().setInt("INDEX", labelIndex); job.getConfiguration().set("LABELS", labels); job.setJarByClass(getClass()); scan = new Scan(); scan.setCacheBlocks(false); scan.setRaw(true); String[] split = labels.split(COMMA); scan.setAuthorizations(new Authorizations(split[this.labelIndex * 2], split[(this.labelIndex * 2) + 1])); if (delete) { LOG.info("Running deletes"); } else { LOG.info("Running copiers"); } if (delete) { TableMapReduceUtil.initTableMapperJob(tableName.getNameAsString(), scan, VisibilityDeleteImport.class, null, null, job); } else { TableMapReduceUtil.initTableMapperJob(tableName.getNameAsString(), scan, VisibilityImport.class, null, null, job); } job.getConfiguration().setBoolean("mapreduce.map.speculative", false); job.getConfiguration().setBoolean("mapreduce.reduce.speculative", false); TableMapReduceUtil.initTableReducerJob(COMMON_TABLE_NAME, null, job, null, null, null, null); TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); TableMapReduceUtil.initCredentials(job); job.setNumReduceTasks(0); boolean success = job.waitForCompletion(true); return success ? 0 : 1; }
@Test public void TestGetRowFromGetCopyConstructor() throws Exception { Get get = new Get(ROW); get.setFilter(null); get.setAuthorizations(new Authorizations("foo")); get.setACL("u", new Permission(Permission.Action.READ)); get.setConsistency(Consistency.TIMELINE); get.setReplicaId(2); get.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED); get.setCheckExistenceOnly(true); get.setTimeRange(3, 4); get.setMaxVersions(11); get.setMaxResultsPerColumnFamily(10); get.setRowOffsetPerColumnFamily(11); get.setCacheBlocks(true); Get copyGet = new Get(get); assertEquals(0, Bytes.compareTo(get.getRow(), copyGet.getRow())); // from OperationWithAttributes assertEquals(get.getId(), copyGet.getId()); // from Query class assertEquals(get.getFilter(), copyGet.getFilter()); assertTrue(get.getAuthorizations().toString().equals(copyGet.getAuthorizations().toString())); assertTrue(Bytes.equals(get.getACL(), copyGet.getACL())); assertEquals(get.getConsistency(), copyGet.getConsistency()); assertEquals(get.getReplicaId(), copyGet.getReplicaId()); assertEquals(get.getIsolationLevel(), copyGet.getIsolationLevel()); // from Get class assertEquals(get.isCheckExistenceOnly(), copyGet.isCheckExistenceOnly()); assertTrue(get.getTimeRange().equals(copyGet.getTimeRange())); assertEquals(get.getMaxVersions(), copyGet.getMaxVersions()); assertEquals(get.getMaxResultsPerColumnFamily(), copyGet.getMaxResultsPerColumnFamily()); assertEquals(get.getRowOffsetPerColumnFamily(), copyGet.getRowOffsetPerColumnFamily()); assertEquals(get.getCacheBlocks(), copyGet.getCacheBlocks()); assertEquals(get.getId(), copyGet.getId()); }
/** * Creates a {@link Get} (HBase) from a {@link TGet} (Thrift). * * This ignores any timestamps set on {@link TColumn} objects. * * @param in the <code>TGet</code> to convert * * @return <code>Get</code> object * * @throws IOException if an invalid time range or max version parameter is given */ public static Get getFromThrift(TGet in) throws IOException { Get out = new Get(in.getRow()); // Timestamp overwrites time range if both are set if (in.isSetTimestamp()) { out.setTimeStamp(in.getTimestamp()); } else if (in.isSetTimeRange()) { out.setTimeRange(in.getTimeRange().getMinStamp(), in.getTimeRange().getMaxStamp()); } if (in.isSetMaxVersions()) { out.setMaxVersions(in.getMaxVersions()); } if (in.isSetFilterString()) { ParseFilter parseFilter = new ParseFilter(); out.setFilter(parseFilter.parseFilterString(in.getFilterString())); } if (in.isSetAttributes()) { addAttributes(out,in.getAttributes()); } if (in.isSetAuthorizations()) { out.setAuthorizations(new Authorizations(in.getAuthorizations().getLabels())); } if (!in.isSetColumns()) { return out; } for (TColumn column : in.getColumns()) { if (column.isSetQualifier()) { out.addColumn(column.getFamily(), column.getQualifier()); } else { out.addFamily(column.getFamily()); } } return out; }
public static Scan scanFromThrift(TScan in) throws IOException { Scan out = new Scan(); if (in.isSetStartRow()) out.setStartRow(in.getStartRow()); if (in.isSetStopRow()) out.setStopRow(in.getStopRow()); if (in.isSetCaching()) out.setCaching(in.getCaching()); if (in.isSetMaxVersions()) { out.setMaxVersions(in.getMaxVersions()); } if (in.isSetColumns()) { for (TColumn column : in.getColumns()) { if (column.isSetQualifier()) { out.addColumn(column.getFamily(), column.getQualifier()); } else { out.addFamily(column.getFamily()); } } } TTimeRange timeRange = in.getTimeRange(); if (timeRange != null && timeRange.isSetMinStamp() && timeRange.isSetMaxStamp()) { out.setTimeRange(timeRange.getMinStamp(), timeRange.getMaxStamp()); } if (in.isSetBatchSize()) { out.setBatch(in.getBatchSize()); } if (in.isSetFilterString()) { ParseFilter parseFilter = new ParseFilter(); out.setFilter(parseFilter.parseFilterString(in.getFilterString())); } if (in.isSetAttributes()) { addAttributes(out,in.getAttributes()); } if (in.isSetAuthorizations()) { out.setAuthorizations(new Authorizations(in.getAuthorizations().getLabels())); } if (in.isSetReversed()) { out.setReversed(in.isReversed()); } return out; }
/** * Sets the authorizations to be used by this Query * @param authorizations */ public Query setAuthorizations(Authorizations authorizations) { this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY, ProtobufUtil .toAuthorizations(authorizations).toByteArray()); return this; }
/** * @return The authorizations this Query is associated with. * @throws DeserializationException */ public Authorizations getAuthorizations() throws DeserializationException { byte[] authorizationsBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY); if (authorizationsBytes == null) return null; return ProtobufUtil.toAuthorizations(authorizationsBytes); }
@Override public Scan setAuthorizations(Authorizations authorizations) { return (Scan) super.setAuthorizations(authorizations); }
@Override public Get setAuthorizations(Authorizations authorizations) { return (Get) super.setAuthorizations(authorizations); }
@Test public void TestGetRowFromGetCopyConstructor() throws Exception { Get get = new Get(ROW); get.setFilter(null); get.setAuthorizations(new Authorizations("foo")); get.setACL("u", new Permission(Permission.Action.READ)); get.setConsistency(Consistency.TIMELINE); get.setReplicaId(2); get.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED); get.setCheckExistenceOnly(true); get.setClosestRowBefore(true); get.setTimeRange(3, 4); get.setMaxVersions(11); get.setMaxResultsPerColumnFamily(10); get.setRowOffsetPerColumnFamily(11); get.setCacheBlocks(true); Get copyGet = new Get(get); assertEquals(0, Bytes.compareTo(get.getRow(), copyGet.getRow())); // from OperationWithAttributes assertEquals(get.getId(), copyGet.getId()); // from Query class assertEquals(get.getFilter(), copyGet.getFilter()); assertTrue(get.getAuthorizations().toString().equals(copyGet.getAuthorizations().toString())); assertTrue(Bytes.equals(get.getACL(), copyGet.getACL())); assertEquals(get.getConsistency(), copyGet.getConsistency()); assertEquals(get.getReplicaId(), copyGet.getReplicaId()); assertEquals(get.getIsolationLevel(), copyGet.getIsolationLevel()); // from Get class assertEquals(get.isCheckExistenceOnly(), copyGet.isCheckExistenceOnly()); assertEquals(get.isClosestRowBefore(), copyGet.isClosestRowBefore()); assertTrue(get.getTimeRange().equals(copyGet.getTimeRange())); assertEquals(get.isClosestRowBefore(), copyGet.isClosestRowBefore()); assertEquals(get.getMaxVersions(), copyGet.getMaxVersions()); assertEquals(get.getMaxResultsPerColumnFamily(), copyGet.getMaxResultsPerColumnFamily()); assertEquals(get.getRowOffsetPerColumnFamily(), copyGet.getRowOffsetPerColumnFamily()); assertEquals(get.getCacheBlocks(), copyGet.getCacheBlocks()); assertEquals(get.getId(), copyGet.getId()); }
/** * @param scan the scan specification * @throws Exception */ public static ScannerModel fromScan(Scan scan) throws Exception { ScannerModel model = new ScannerModel(); model.setStartRow(scan.getStartRow()); model.setEndRow(scan.getStopRow()); Map<byte [], NavigableSet<byte []>> families = scan.getFamilyMap(); if (families != null) { for (Map.Entry<byte [], NavigableSet<byte []>> entry : families.entrySet()) { if (entry.getValue() != null) { for (byte[] qualifier: entry.getValue()) { model.addColumn(Bytes.add(entry.getKey(), COLUMN_DIVIDER, qualifier)); } } else { model.addColumn(entry.getKey()); } } } model.setStartTime(scan.getTimeRange().getMin()); model.setEndTime(scan.getTimeRange().getMax()); int caching = scan.getCaching(); if (caching > 0) { model.setCaching(caching); } int batch = scan.getBatch(); if (batch > 0) { model.setBatch(batch); } int maxVersions = scan.getMaxVersions(); if (maxVersions > 0) { model.setMaxVersions(maxVersions); } Filter filter = scan.getFilter(); if (filter != null) { model.setFilter(stringifyFilter(filter)); } // Add the visbility labels if found in the attributes Authorizations authorizations = scan.getAuthorizations(); if (authorizations != null) { List<String> labels = authorizations.getLabels(); for (String label : labels) { model.addLabel(label); } } return model; }
public Authorizations getAuthorizations() throws DeserializationException { return scan.getAuthorizations(); }
public AnyScan setAuthorizations(Authorizations authorizations) { scan.setAuthorizations(authorizations); return this; }
public Authorizations getAuthorizations() throws DeserializationException { return get.getAuthorizations(); }
public AnyGet setAuthorizations(Authorizations authorizations) { get.setAuthorizations(authorizations); return this; }