private void refreshNodes(List<ZKUtil.NodeAndData> nodes) throws IOException { for (ZKUtil.NodeAndData n : nodes) { if (n.isEmpty()) continue; String path = n.getNode(); String namespace = ZKUtil.getNodeName(path); byte[] nodeData = n.getData(); if (LOG.isDebugEnabled()) { LOG.debug("Updating namespace cache from node "+namespace+" with data: "+ Bytes.toStringBinary(nodeData)); } NamespaceDescriptor ns = ProtobufUtil.toNamespaceDescriptor( HBaseProtos.NamespaceDescriptor.parseFrom(nodeData)); cache.put(ns.getName(), ns); } }
public static boolean hasMinimumVersion(HBaseProtos.VersionInfo versionInfo, int major, int minor) { if (versionInfo != null) { try { String[] components = versionInfo.getVersion().split("\\."); int clientMajor = components.length > 0 ? Integer.parseInt(components[0]) : 0; if (clientMajor != major) { return clientMajor > major; } int clientMinor = components.length > 1 ? Integer.parseInt(components[1]) : 0; return clientMinor >= minor; } catch (NumberFormatException e) { return false; } } return false; }
@Override public void deserializeStateData(final InputStream stream) throws IOException { super.deserializeStateData(stream); MasterProcedureProtos.CreateTableStateData state = MasterProcedureProtos.CreateTableStateData.parseDelimitedFrom(stream); user = MasterProcedureUtil.toUserInfo(state.getUserInfo()); hTableDescriptor = HTableDescriptor.convert(state.getTableSchema()); if (state.getRegionInfoCount() == 0) { newRegions = null; } else { newRegions = new ArrayList<HRegionInfo>(state.getRegionInfoCount()); for (HBaseProtos.RegionInfo hri: state.getRegionInfoList()) { newRegions.add(HRegionInfo.convert(hri)); } } }
@Override public void deserializeStateData(final InputStream stream) throws IOException { super.deserializeStateData(stream); MasterProcedureProtos.TruncateTableStateData state = MasterProcedureProtos.TruncateTableStateData.parseDelimitedFrom(stream); user = MasterProcedureUtil.toUserInfo(state.getUserInfo()); if (state.hasTableSchema()) { hTableDescriptor = HTableDescriptor.convert(state.getTableSchema()); tableName = hTableDescriptor.getTableName(); } else { tableName = ProtobufUtil.toTableName(state.getTableName()); } preserveSplits = state.getPreserveSplits(); if (state.getRegionInfoCount() == 0) { regions = null; } else { regions = new ArrayList<HRegionInfo>(state.getRegionInfoCount()); for (HBaseProtos.RegionInfo hri: state.getRegionInfoList()) { regions.add(HRegionInfo.convert(hri)); } } }
@Override public void deserializeStateData(final InputStream stream) throws IOException { super.deserializeStateData(stream); MasterProcedureProtos.DeleteTableStateData state = MasterProcedureProtos.DeleteTableStateData.parseDelimitedFrom(stream); user = MasterProcedureUtil.toUserInfo(state.getUserInfo()); tableName = ProtobufUtil.toTableName(state.getTableName()); if (state.getRegionInfoCount() == 0) { regions = null; } else { regions = new ArrayList<HRegionInfo>(state.getRegionInfoCount()); for (HBaseProtos.RegionInfo hri: state.getRegionInfoList()) { regions.add(HRegionInfo.convert(hri)); } } }
public synchronized NavigableSet<NamespaceDescriptor> list() throws IOException { NavigableSet<NamespaceDescriptor> ret = Sets.newTreeSet(NamespaceDescriptor.NAMESPACE_DESCRIPTOR_COMPARATOR); ResultScanner scanner = getNamespaceTable().getScanner(HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES); try { for(Result r : scanner) { byte[] val = CellUtil.cloneValue(r.getColumnLatestCell( HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES, HTableDescriptor.NAMESPACE_COL_DESC_BYTES)); ret.add(ProtobufUtil.toNamespaceDescriptor( HBaseProtos.NamespaceDescriptor.parseFrom(val))); } } finally { scanner.close(); } return ret; }
@Override protected int doWork() throws Exception { Connection connection = null; Admin admin = null; try { connection = ConnectionFactory.createConnection(getConf()); admin = connection.getAdmin(); HBaseProtos.SnapshotDescription.Type type = HBaseProtos.SnapshotDescription.Type.FLUSH; if (snapshotType != null) { type = HBaseProtos.SnapshotDescription.Type.valueOf(snapshotName.toUpperCase()); } admin.snapshot(snapshotName, TableName.valueOf(tableName), type); } catch (Exception e) { return -1; } finally { if (admin != null) { admin.close(); } if (connection != null) { connection.close(); } } return 0; }
private ClusterStatusProtos.ServerLoad createServerLoadProto() { HBaseProtos.RegionSpecifier rSpecOne = HBaseProtos.RegionSpecifier.newBuilder() .setType(HBaseProtos.RegionSpecifier.RegionSpecifierType.ENCODED_REGION_NAME) .setValue(ByteString.copyFromUtf8("ASDFGQWERT")).build(); HBaseProtos.RegionSpecifier rSpecTwo = HBaseProtos.RegionSpecifier.newBuilder() .setType(HBaseProtos.RegionSpecifier.RegionSpecifierType.ENCODED_REGION_NAME) .setValue(ByteString.copyFromUtf8("QWERTYUIOP")).build(); ClusterStatusProtos.RegionLoad rlOne = ClusterStatusProtos.RegionLoad.newBuilder().setRegionSpecifier(rSpecOne).setStores(10) .setStorefiles(101).setStoreUncompressedSizeMB(106).setStorefileSizeMB(520) .setStorefileIndexSizeMB(42).setRootIndexSizeKB(201).setReadRequestsCount(Integer.MAX_VALUE).setWriteRequestsCount(Integer.MAX_VALUE).build(); ClusterStatusProtos.RegionLoad rlTwo = ClusterStatusProtos.RegionLoad.newBuilder().setRegionSpecifier(rSpecTwo).setStores(3) .setStorefiles(13).setStoreUncompressedSizeMB(23).setStorefileSizeMB(300) .setStorefileIndexSizeMB(40).setRootIndexSizeKB(303).setReadRequestsCount(Integer.MAX_VALUE).setWriteRequestsCount(Integer.MAX_VALUE).build(); ClusterStatusProtos.ServerLoad sl = ClusterStatusProtos.ServerLoad.newBuilder().addRegionLoads(rlOne). addRegionLoads(rlTwo).build(); return sl; }
public static ScanMetrics toScanMetrics(final byte[] bytes) { Parser<MapReduceProtos.ScanMetrics> parser = MapReduceProtos.ScanMetrics.PARSER; MapReduceProtos.ScanMetrics pScanMetrics = null; try { pScanMetrics = parser.parseFrom(bytes); } catch (InvalidProtocolBufferException e) { //Ignored there are just no key values to add. } ScanMetrics scanMetrics = new ScanMetrics(); if (pScanMetrics != null) { for (HBaseProtos.NameInt64Pair pair : pScanMetrics.getMetricsList()) { if (pair.hasName() && pair.hasValue()) { scanMetrics.setCounter(pair.getName(), pair.getValue()); } } } return scanMetrics; }
/** * Convert a protocol buffer TimeUnit to a client TimeUnit * @param proto * @return the converted client TimeUnit */ public static TimeUnit toTimeUnit(final HBaseProtos.TimeUnit proto) { switch (proto) { case NANOSECONDS: return TimeUnit.NANOSECONDS; case MICROSECONDS: return TimeUnit.MICROSECONDS; case MILLISECONDS: return TimeUnit.MILLISECONDS; case SECONDS: return TimeUnit.SECONDS; case MINUTES: return TimeUnit.MINUTES; case HOURS: return TimeUnit.HOURS; case DAYS: return TimeUnit.DAYS; default: throw new RuntimeException("Invalid TimeUnit " + proto); } }
/** * Convert a client TimeUnit to a protocol buffer TimeUnit * @param timeUnit * @return the converted protocol buffer TimeUnit */ public static HBaseProtos.TimeUnit toProtoTimeUnit(final TimeUnit timeUnit) { switch (timeUnit) { case NANOSECONDS: return HBaseProtos.TimeUnit.NANOSECONDS; case MICROSECONDS: return HBaseProtos.TimeUnit.MICROSECONDS; case MILLISECONDS: return HBaseProtos.TimeUnit.MILLISECONDS; case SECONDS: return HBaseProtos.TimeUnit.SECONDS; case MINUTES: return HBaseProtos.TimeUnit.MINUTES; case HOURS: return HBaseProtos.TimeUnit.HOURS; case DAYS: return HBaseProtos.TimeUnit.DAYS; default: throw new RuntimeException("Invalid TimeUnit " + timeUnit); } }
/** * List available namespace descriptors * @return List of descriptors * @throws IOException */ @Override public NamespaceDescriptor[] listNamespaceDescriptors() throws IOException { return executeCallable(new MasterCallable<NamespaceDescriptor[]>(getConnection()) { @Override public NamespaceDescriptor[] call(int callTimeout) throws Exception { PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setCallTimeout(callTimeout); List<HBaseProtos.NamespaceDescriptor> list = master.listNamespaceDescriptors(controller, ListNamespaceDescriptorsRequest.newBuilder().build()) .getNamespaceDescriptorList(); NamespaceDescriptor[] res = new NamespaceDescriptor[list.size()]; for(int i = 0; i < list.size(); i++) { res[i] = ProtobufUtil.toNamespaceDescriptor(list.get(i)); } return res; } }); }
/** * Get list of table names by namespace * @param name namespace name * @return The list of table names in the namespace * @throws IOException */ @Override public TableName[] listTableNamesByNamespace(final String name) throws IOException { return executeCallable(new MasterCallable<TableName[]>(getConnection()) { @Override public TableName[] call(int callTimeout) throws Exception { PayloadCarryingRpcController controller = rpcControllerFactory.newController(); controller.setCallTimeout(callTimeout); List<HBaseProtos.TableName> tableNames = master.listTableNamesByNamespace(controller, ListTableNamesByNamespaceRequest. newBuilder().setNamespaceName(name).build()) .getTableNameList(); TableName[] result = new TableName[tableNames.size()]; for (int i = 0; i < tableNames.size(); i++) { result[i] = ProtobufUtil.toTableName(tableNames.get(i)); } return result; } }); }
/** * Check to make sure that the description of the snapshot requested is valid * @param snapshot description of the snapshot * @throws IllegalArgumentException if the name of the snapshot or the name of the table to * snapshot are not valid names. */ public static void assertSnapshotRequestIsValid(HBaseProtos.SnapshotDescription snapshot) throws IllegalArgumentException { // make sure the snapshot name is valid TableName.isLegalTableQualifierName(Bytes.toBytes(snapshot.getName()), true); if(snapshot.hasTable()) { // make sure the table name is valid, this will implicitly check validity TableName tableName = TableName.valueOf(snapshot.getTable()); if (tableName.isSystemTable()) { throw new IllegalArgumentException("System table snapshots are not allowed"); } } }
FilterProtos.SingleColumnValueFilter convert() { FilterProtos.SingleColumnValueFilter.Builder builder = FilterProtos.SingleColumnValueFilter.newBuilder(); if (this.columnFamily != null) { builder.setColumnFamily(ByteStringer.wrap(this.columnFamily)); } if (this.columnQualifier != null) { builder.setColumnQualifier(ByteStringer.wrap(this.columnQualifier)); } HBaseProtos.CompareType compareOp = CompareType.valueOf(this.compareOp.name()); builder.setCompareOp(compareOp); builder.setComparator(ProtobufUtil.toComparator(this.comparator)); builder.setFilterIfMissing(this.filterIfMissing); builder.setLatestVersionOnly(this.latestVersionOnly); return builder.build(); }
@Test public void testExclude() throws Exception { List<HBaseProtos.SnapshotDescription> snapshotDescriptions; String[] argsParam; SnapshotArgs args; Snapshot app; // create table String tableName2 = createAdditionalTable(tableName + "2"); // with table list argsParam = new String[]{"localhost", ".*", "--exclude=" + tableName, "--test"}; args = new SnapshotArgs(argsParam); app = new Snapshot(admin, args); // create snapshot app.run(); snapshotDescriptions = listSnapshots(tableName + ".*"); assertEquals(1, snapshotDescriptions.size()); assertEquals(tableName2, snapshotDescriptions.get(0).getTable()); }
private ClusterStatusProtos.ServerLoad createServerLoadProto() { HBaseProtos.RegionSpecifier rSpecOne = HBaseProtos.RegionSpecifier.newBuilder() .setType(HBaseProtos.RegionSpecifier.RegionSpecifierType.ENCODED_REGION_NAME) .setValue(ByteString.copyFromUtf8("ASDFGQWERT")).build(); HBaseProtos.RegionSpecifier rSpecTwo = HBaseProtos.RegionSpecifier.newBuilder() .setType(HBaseProtos.RegionSpecifier.RegionSpecifierType.ENCODED_REGION_NAME) .setValue(ByteString.copyFromUtf8("QWERTYUIOP")).build(); ClusterStatusProtos.RegionLoad rlOne = ClusterStatusProtos.RegionLoad.newBuilder().setRegionSpecifier(rSpecOne).setStores(10) .setStorefiles(101).setStoreUncompressedSizeMB(106).setStorefileSizeMB(520) .setStorefileIndexSizeMB(42).setRootIndexSizeKB(201).build(); ClusterStatusProtos.RegionLoad rlTwo = ClusterStatusProtos.RegionLoad.newBuilder().setRegionSpecifier(rSpecTwo).setStores(3) .setStorefiles(13).setStoreUncompressedSizeMB(23).setStorefileSizeMB(300) .setStorefileIndexSizeMB(40).setRootIndexSizeKB(303).build(); ClusterStatusProtos.ServerLoad sl = ClusterStatusProtos.ServerLoad.newBuilder().addRegionLoads(rlOne). addRegionLoads(rlTwo).build(); return sl; }
@Test public void testDuplicatedName() throws Exception { List<HBaseProtos.SnapshotDescription> snapshotDescriptions; String[] argsParam; SnapshotArgs args; Snapshot app; argsParam = new String[]{"localhost", ".*", "--test"}; args = new SnapshotArgs(argsParam); app = new Snapshot(admin, args); String snapshotName = app.getPrefix(tableName) + "test"; // create snapshot first app.snapshot(null, tableName, snapshotName); snapshotDescriptions = listSnapshots(tableName + ".*"); assertEquals(1, snapshotDescriptions.size()); // create snapshot again app.snapshot(null, tableName, snapshotName); snapshotDescriptions = listSnapshots(tableName + ".*"); assertEquals(1, snapshotDescriptions.size()); }
/** * List available namespace descriptors * @return List of descriptors * @throws IOException */ @Override public NamespaceDescriptor[] listNamespaceDescriptors() throws IOException { return executeCallable(new MasterCallable<NamespaceDescriptor[]>(getConnection()) { @Override public NamespaceDescriptor[] call(int callTimeout) throws Exception { List<HBaseProtos.NamespaceDescriptor> list = master.listNamespaceDescriptors(null, ListNamespaceDescriptorsRequest.newBuilder(). build()).getNamespaceDescriptorList(); NamespaceDescriptor[] res = new NamespaceDescriptor[list.size()]; for(int i = 0; i < list.size(); i++) { res[i] = ProtobufUtil.toNamespaceDescriptor(list.get(i)); } return res; } }); }
/** * Get list of table names by namespace * @param name namespace name * @return The list of table names in the namespace * @throws IOException */ @Override public TableName[] listTableNamesByNamespace(final String name) throws IOException { return executeCallable(new MasterCallable<TableName[]>(getConnection()) { @Override public TableName[] call(int callTimeout) throws Exception { List<HBaseProtos.TableName> tableNames = master.listTableNamesByNamespace(null, ListTableNamesByNamespaceRequest. newBuilder().setNamespaceName(name).build()) .getTableNameList(); TableName[] result = new TableName[tableNames.size()]; for (int i = 0; i < tableNames.size(); i++) { result[i] = ProtobufUtil.toTableName(tableNames.get(i)); } return result; } }); }
@Test public void testAfterSuccess() throws Exception { List<HBaseProtos.SnapshotDescription> snapshotDescriptions; // all tables, keep unlimited String[] argsParam = {"localhost", ".*", "--test", "--" + Args.OPTION_AFTER_SUCCESS + "=" + AlertSenderTest.ALERT_SCRIPT}; SnapshotArgs args = new SnapshotArgs(argsParam); Snapshot app = new Snapshot(admin, args); int sendCountBefore = AlertSender.getSendCount(); // create snapshot app.run(); snapshotDescriptions = listSnapshots(tableName + ".*"); assertEquals(1, snapshotDescriptions.size()); Assert.assertEquals(sendCountBefore + 1, AlertSender.getSendCount()); }
@Test public void testExcludeRegexList() throws Exception { List<HBaseProtos.SnapshotDescription> snapshotDescriptions; String[] argsParam; SnapshotArgs args; Snapshot app; // create table String tableName2 = createAdditionalTable(tableName + "2"); createAdditionalTable(tableName + "21"); // with table list argsParam = new String[]{"localhost", ".*", "--exclude=" + tableName2 + ".*", "--test"}; args = new SnapshotArgs(argsParam); app = new Snapshot(admin, args); // create snapshot app.run(); snapshotDescriptions = listSnapshots(tableName + ".*"); assertEquals(1, snapshotDescriptions.size()); assertEquals(tableName, snapshotDescriptions.get(0).getTable()); }
@Override public void write(DataOutput out) throws IOException { MapReduceProtos.TableSnapshotRegionSplit.Builder builder = MapReduceProtos.TableSnapshotRegionSplit.newBuilder() .setRegion(HBaseProtos.RegionSpecifier.newBuilder() .setType(HBaseProtos.RegionSpecifier.RegionSpecifierType.ENCODED_REGION_NAME) .setValue(HBaseZeroCopyByteString.wrap(Bytes.toBytes(regionName))).build()); for (String location : locations) { builder.addLocations(location); } MapReduceProtos.TableSnapshotRegionSplit split = builder.build(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); split.writeTo(baos); baos.close(); byte[] buf = baos.toByteArray(); out.writeInt(buf.length); out.write(buf); }
/** * Get list of table names by namespace * @param name namespace name * @return The list of table names in the namespace * @throws IOException */ public TableName[] listTableNamesByNamespace(final String name) throws IOException { return executeCallable(new MasterCallable<TableName[]>(getConnection()) { @Override public TableName[] call() throws Exception { List<HBaseProtos.TableName> tableNames = master.listTableNamesByNamespace(null, ListTableNamesByNamespaceRequest. newBuilder().setNamespaceName(name).build()) .getTableNameList(); TableName[] result = new TableName[tableNames.size()]; for (int i = 0; i < tableNames.size(); i++) { result[i] = ProtobufUtil.toTableName(tableNames.get(i)); } return result; } }); }
/** * Build up the znode content. * @param sn What to put into the znode. * @return The content of the meta-region-server znode */ static byte [] toByteArray(final ServerName sn) { // ZNode content is a pb message preceded by some pb magic. HBaseProtos.ServerName pbsn = HBaseProtos.ServerName.newBuilder() .setHostName(sn.getHostname()) .setPort(sn.getPort()) .setStartCode(sn.getStartcode()) .build(); ZooKeeperProtos.MetaRegionServer pbrsr = ZooKeeperProtos.MetaRegionServer.newBuilder() .setServer(pbsn) .setRpcVersion(HConstants.RPC_CURRENT_VERSION) .build(); return ProtobufUtil.prependPBMagic(pbrsr.toByteArray()); }
@Test public void testNamespace() throws Exception { List<HBaseProtos.SnapshotDescription> snapshotDescriptions; String[] argsParam; SnapshotArgs args; Snapshot app; String fullTableName = null; try { // create table with namespace fullTableName = TEST_NAMESPACE + ":" + TestBase.tableName; createTable(fullTableName); // table with namespace argsParam = new String[]{"localhost", fullTableName}; args = new SnapshotArgs(argsParam); app = new Snapshot(admin, args); // create snapshot app.run(); snapshotDescriptions = admin.listSnapshots(app.getPrefix(fullTableName) + ".*"); assertEquals(1, snapshotDescriptions.size()); } finally { dropTable(fullTableName); } }
public FilterProtos.SingleColumnValueFilter convert() { FilterProtos.SingleColumnValueFilter.Builder builder = FilterProtos.SingleColumnValueFilter.newBuilder(); if (this.columnFamily != null) { builder.setColumnFamily(HBaseZeroCopyByteString.wrap(this.columnFamily)); } if (this.columnQualifier != null) { builder.setColumnQualifier(HBaseZeroCopyByteString.wrap(this.columnQualifier)); } HBaseProtos.CompareType compareOp = CompareType.valueOf(this.compareOp.name()); builder.setCompareOp(compareOp); builder.setComparator(ProtobufUtil.toComparator(this.comparator)); builder.setFilterIfMissing(this.filterIfMissing); builder.setLatestVersionOnly(this.latestVersionOnly); return builder.build(); }
public org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALKey.Builder getBuilder( WALCellCodec.ByteStringCompressor compressor) throws IOException { org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALKey.Builder builder = org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALKey.newBuilder(); if (compressionContext == null) { builder.setEncodedRegionName(ByteStringer.wrap(this.encodedRegionName)); builder.setTableName(ByteStringer.wrap(this.tablename.getName())); } else { builder.setEncodedRegionName(compressor.compress(this.encodedRegionName, compressionContext.regionDict)); builder.setTableName(compressor.compress(this.tablename.getName(), compressionContext.tableDict)); } builder.setLogSequenceNumber(this.logSeqNum); builder.setWriteTime(writeTime); if (this.origLogSeqNum > 0) { builder.setOrigSequenceNumber(this.origLogSeqNum); } if (this.nonce != HConstants.NO_NONCE) { builder.setNonce(nonce); } if (this.nonceGroup != HConstants.NO_NONCE) { builder.setNonceGroup(nonceGroup); } HBaseProtos.UUID.Builder uuidBuilder = HBaseProtos.UUID.newBuilder(); for (UUID clusterId : clusterIds) { uuidBuilder.setLeastSigBits(clusterId.getLeastSignificantBits()); uuidBuilder.setMostSigBits(clusterId.getMostSignificantBits()); builder.addClusterIds(uuidBuilder.build()); } if (scopes != null) { for (Map.Entry<byte[], Integer> e : scopes.entrySet()) { ByteString family = (compressionContext == null) ? ByteStringer.wrap(e.getKey()) : compressor.compress(e.getKey(), compressionContext.familyDict); builder.addScopes(FamilyScope.newBuilder() .setFamily(family).setScopeType(ScopeType.valueOf(e.getValue()))); } } return builder; }
/** * @return This instance serialized into a byte array * @see #parseFrom(byte[]) */ public byte [] toByteArray() { // First create a pb ServerName. Then create a ByteString w/ the TaskState // bytes in it. Finally create a SplitLogTaskState passing in the two // pbs just created. HBaseProtos.ServerName snpb = ProtobufUtil.toServerName(this.originServer); ZooKeeperProtos.SplitLogTask slts = ZooKeeperProtos.SplitLogTask.newBuilder().setServerName(snpb).setState(this.state). setMode(this.mode).build(); return ProtobufUtil.prependPBMagic(slts.toByteArray()); }
/** * @param favoredNodes The PB'ed bytes of favored nodes * @return the array of {@link ServerName} for the byte array of favored nodes. * @throws InvalidProtocolBufferException */ public static ServerName[] getFavoredNodesList(byte[] favoredNodes) throws InvalidProtocolBufferException { FavoredNodes f = FavoredNodes.parseFrom(favoredNodes); List<HBaseProtos.ServerName> protoNodes = f.getFavoredNodeList(); ServerName[] servers = new ServerName[protoNodes.size()]; int i = 0; for (HBaseProtos.ServerName node : protoNodes) { servers[i++] = ProtobufUtil.toServerName(node); } return servers; }