/** * sourceToString * @return a string contains sourceReplicationLoad information */ public String sourceToString() { if (this.sourceMetricsList == null) return null; StringBuilder sb = new StringBuilder(); for (ClusterStatusProtos.ReplicationLoadSource rls : this.replicationLoadSourceList) { sb = Strings.appendKeyValue(sb, "\n PeerID", rls.getPeerID()); sb = Strings.appendKeyValue(sb, "AgeOfLastShippedOp", rls.getAgeOfLastShippedOp()); sb = Strings.appendKeyValue(sb, "SizeOfLogQueue", rls.getSizeOfLogQueue()); sb = Strings.appendKeyValue(sb, "TimeStampsOfLastShippedOp", (new Date(rls.getTimeStampOfLastShippedOp()).toString())); sb = Strings.appendKeyValue(sb, "Replication Lag", rls.getReplicationLag()); } return sb.toString(); }
/** * @deprecated mistakenly made public in 0.98.7. scope will change to package-private */ @Deprecated public String reverseDNS(InetAddress ipAddress) throws NamingException, UnknownHostException { String hostName = this.reverseDNSCacheMap.get(ipAddress); if (hostName == null) { String ipAddressString = null; try { ipAddressString = DNS.reverseDns(ipAddress, null); } catch (Exception e) { // We can use InetAddress in case the jndi failed to pull up the reverse DNS entry from the // name service. Also, in case of ipv6, we need to use the InetAddress since resolving // reverse DNS using jndi doesn't work well with ipv6 addresses. ipAddressString = InetAddress.getByName(ipAddress.getHostAddress()).getHostName(); } if (ipAddressString == null) throw new UnknownHostException("No host found for " + ipAddress); hostName = Strings.domainNamePointerToHostName(ipAddressString); this.reverseDNSCacheMap.put(ipAddress, hostName); } return hostName; }
protected static String toStringWithPadding(final KeyValue kv, final int maxRowLength, int maxFamilyLength, int maxQualifierLength, int maxTimestampLength, boolean includeMeta) { String leadingLengths = ""; String familyLength = kv.getFamilyLength() + " "; if (includeMeta) { leadingLengths += Strings.padFront(kv.getKeyLength() + "", '0', 4); leadingLengths += " "; leadingLengths += Strings.padFront(kv.getValueLength() + "", '0', 4); leadingLengths += " "; leadingLengths += Strings.padFront(kv.getRowLength() + "", '0', 2); leadingLengths += " "; } int spacesAfterRow = maxRowLength - getRowString(kv).length() + 2; int spacesAfterFamily = maxFamilyLength - getFamilyString(kv).length() + 2; int spacesAfterQualifier = maxQualifierLength - getQualifierString(kv).length() + 1; int spacesAfterTimestamp = maxTimestampLength - Long.valueOf(kv.getTimestamp()).toString().length() + 1; return leadingLengths + getRowString(kv) + Strings.repeat(' ', spacesAfterRow) + familyLength + getFamilyString(kv) + Strings.repeat(' ', spacesAfterFamily) + getQualifierString(kv) + Strings.repeat(' ', spacesAfterQualifier) + getTimestampString(kv) + Strings.repeat(' ', spacesAfterTimestamp) + getTypeString(kv) + " " + getValueString(kv); }
private static String reverseDNS(InetAddress ipAddress) throws NamingException, UnknownHostException { String hostName = reverseDNSCacheMap.get(ipAddress); if (hostName == null) { String ipAddressString = null; try { ipAddressString = DNS.reverseDns(ipAddress, null); } catch (Exception e) { // We can use InetAddress in case the jndi failed to pull up the reverse DNS entry from the // name service. Also, in case of ipv6, we need to use the InetAddress since resolving // reverse DNS using jndi doesn't work well with ipv6 addresses. ipAddressString = InetAddress.getByName(ipAddress.getHostAddress()).getHostName(); } if (ipAddressString == null) { throw new UnknownHostException("No host found for " + ipAddress); } hostName = Strings.domainNamePointerToHostName(ipAddressString); reverseDNSCacheMap.put(ipAddress, hostName); } return hostName; }
private StringBuilder appendHistogram(StringBuilder sb, MetricsHistogram histogram) { sb = Strings.appendKeyValue(sb, histogram.getName() + "Mean", StringUtils.limitDecimalTo2(histogram.getMean())); sb = Strings.appendKeyValue(sb, histogram.getName() + "Count", StringUtils.limitDecimalTo2(histogram.getCount())); final Snapshot s = histogram.getSnapshot(); sb = Strings.appendKeyValue(sb, histogram.getName() + "Median", StringUtils.limitDecimalTo2(s.getMedian())); sb = Strings.appendKeyValue(sb, histogram.getName() + "75th", StringUtils.limitDecimalTo2(s.get75thPercentile())); sb = Strings.appendKeyValue(sb, histogram.getName() + "95th", StringUtils.limitDecimalTo2(s.get95thPercentile())); sb = Strings.appendKeyValue(sb, histogram.getName() + "99th", StringUtils.limitDecimalTo2(s.get99thPercentile())); sb = Strings.appendKeyValue(sb, histogram.getName() + "999th", StringUtils.limitDecimalTo2(s.get999thPercentile())); return sb; }
public TokenServer(Configuration conf) throws IOException { this.conf = conf; this.startcode = EnvironmentEdgeManager.currentTime(); // Server to handle client requests. String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost("default", "default")); int port = 0; // Creation of an ISA will force a resolve. InetSocketAddress initialIsa = new InetSocketAddress(hostname, port); if (initialIsa.getAddress() == null) { throw new IllegalArgumentException("Failed resolve of " + initialIsa); } final List<BlockingServiceAndInterface> sai = new ArrayList<BlockingServiceAndInterface>(1); BlockingService service = AuthenticationProtos.AuthenticationService.newReflectiveBlockingService(this); sai.add(new BlockingServiceAndInterface(service, AuthenticationProtos.AuthenticationService.BlockingInterface.class)); this.rpcServer = new RpcServer(this, "tokenServer", sai, initialIsa, conf, new FifoRpcScheduler(conf, 1)); this.isa = this.rpcServer.getListenerAddress(); this.sleeper = new Sleeper(1000, this); }
public TokenServer(Configuration conf) throws IOException { this.conf = conf; this.startcode = EnvironmentEdgeManager.currentTimeMillis(); // Server to handle client requests. String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost("default", "default")); int port = 0; // Creation of an ISA will force a resolve. InetSocketAddress initialIsa = new InetSocketAddress(hostname, port); if (initialIsa.getAddress() == null) { throw new IllegalArgumentException("Failed resolve of " + initialIsa); } final List<BlockingServiceAndInterface> sai = new ArrayList<BlockingServiceAndInterface>(1); BlockingService service = AuthenticationProtos.AuthenticationService.newReflectiveBlockingService(this); sai.add(new BlockingServiceAndInterface(service, AuthenticationProtos.AuthenticationService.BlockingInterface.class)); this.rpcServer = new RpcServer(this, "tokenServer", sai, initialIsa, conf, new FifoRpcScheduler(conf, 1)); this.isa = this.rpcServer.getListenerAddress(); this.sleeper = new Sleeper(1000, this); }
String reverseDNS(InetAddress ipAddress) throws UnknownHostException { String hostName = this.reverseDNSCacheMap.get(ipAddress); if (hostName == null) { String ipAddressString = null; try { ipAddressString = DNS.reverseDns(ipAddress, null); } catch (Exception e) { // We can use InetAddress in case the jndi failed to pull up the reverse DNS entry from the // name service. Also, in case of ipv6, we need to use the InetAddress since resolving // reverse DNS using jndi doesn't work well with ipv6 addresses. ipAddressString = InetAddress.getByName(ipAddress.getHostAddress()).getHostName(); } if (ipAddressString == null) throw new UnknownHostException("No host found for " + ipAddress); hostName = Strings.domainNamePointerToHostName(ipAddressString); this.reverseDNSCacheMap.put(ipAddress, hostName); } return hostName; }
private static Pair<FilterHolder, Class<? extends ServletContainer>> loginServerPrincipal( UserProvider userProvider, Configuration conf) throws Exception { Class<? extends ServletContainer> containerClass = ServletContainer.class; if (userProvider.isHadoopSecurityEnabled() && userProvider.isHBaseSecurityEnabled()) { String machineName = Strings.domainNamePointerToHostName( DNS.getDefaultHost(conf.get(REST_DNS_INTERFACE, "default"), conf.get(REST_DNS_NAMESERVER, "default"))); String keytabFilename = conf.get(REST_KEYTAB_FILE); Preconditions.checkArgument(keytabFilename != null && !keytabFilename.isEmpty(), REST_KEYTAB_FILE + " should be set if security is enabled"); String principalConfig = conf.get(REST_KERBEROS_PRINCIPAL); Preconditions.checkArgument(principalConfig != null && !principalConfig.isEmpty(), REST_KERBEROS_PRINCIPAL + " should be set if security is enabled"); userProvider.login(REST_KEYTAB_FILE, REST_KERBEROS_PRINCIPAL, machineName); if (conf.get(REST_AUTHENTICATION_TYPE) != null) { containerClass = RESTServletContainer.class; FilterHolder authFilter = new FilterHolder(); authFilter.setClassName(AuthFilter.class.getName()); authFilter.setName("AuthenticationFilter"); return new Pair<>(authFilter,containerClass); } } return new Pair<>(null, containerClass); }
protected static String toStringWithPadding(final KeyValue kv, final int maxRowLength, int maxFamilyLength, int maxQualifierLength, int maxTimestampLength, boolean includeMeta) { String leadingLengths = ""; String familyLength = kv.getFamilyLength() + " "; if (includeMeta) { leadingLengths += Strings.padFront(kv.getKeyLength() + "", '0', 4); leadingLengths += " "; leadingLengths += Strings.padFront(kv.getValueLength() + "", '0', 4); leadingLengths += " "; leadingLengths += Strings.padFront(kv.getRowLength() + "", '0', 2); leadingLengths += " "; } int spacesAfterRow = maxRowLength - getRowString(kv).length() + 2; int spacesAfterFamily = maxFamilyLength - getFamilyString(kv).length() + 2; int spacesAfterQualifier = maxQualifierLength - getQualifierString(kv).length() + 1; int spacesAfterTimestamp = maxTimestampLength - Long.valueOf(kv.getTimestamp()).toString().length() + 1; return leadingLengths + getRowString(kv) + StringUtils.repeat(' ', spacesAfterRow) + familyLength + getFamilyString(kv) + StringUtils.repeat(' ', spacesAfterFamily) + getQualifierString(kv) + StringUtils.repeat(' ', spacesAfterQualifier) + getTimestampString(kv) + StringUtils.repeat(' ', spacesAfterTimestamp) + getTypeString(kv) + " " + getValueString(kv); }
public TokenServer(Configuration conf) throws IOException { this.conf = conf; this.startcode = EnvironmentEdgeManager.currentTimeMillis(); // Server to handle client requests. String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost("default", "default")); int port = 0; // Creation of an ISA will force a resolve. InetSocketAddress initialIsa = new InetSocketAddress(hostname, port); if (initialIsa.getAddress() == null) { throw new IllegalArgumentException("Failed resolve of " + initialIsa); } final List<BlockingServiceAndInterface> sai = new ArrayList<BlockingServiceAndInterface>(1); BlockingService service = AuthenticationProtos.AuthenticationService.newReflectiveBlockingService(this); sai.add(new BlockingServiceAndInterface(service, AuthenticationProtos.AuthenticationService.BlockingInterface.class)); this.rpcServer = new RpcServer(this, "tokenServer", sai, initialIsa, 3, 1, conf, HConstants.QOS_THRESHOLD); this.isa = this.rpcServer.getListenerAddress(); this.sleeper = new Sleeper(1000, this); }
private void doPuts(HRegion region) throws IOException{ LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(MIN_VALUE_SIZE, MAX_VALUE_SIZE); for (int i = 0; i < NUM_ROWS; ++i) { byte[] key = LoadTestKVGenerator.md5PrefixedKey(i).getBytes(); for (int j = 0; j < NUM_COLS_PER_ROW; ++j) { Put put = new Put(key); byte[] col = Bytes.toBytes(String.valueOf(j)); byte[] value = dataGenerator.generateRandomSizeValue(key, col); put.add(CF_BYTES, col, value); if(VERBOSE){ KeyValue kvPut = new KeyValue(key, CF_BYTES, col, value); System.err.println(Strings.padFront(i+"", ' ', 4)+" "+kvPut); } region.put(put); } if (i % NUM_ROWS_PER_FLUSH == 0) { region.flushcache(); } } }
/** * Start up or shuts down the Thrift server, depending on the arguments. * @param args */ void doMain(final String[] args) throws Exception { processOptions(args); // login the server principal (if using secure Hadoop) if (User.isSecurityEnabled() && User.isHBaseSecurityEnabled(conf)) { String machineName = Strings.domainNamePointerToHostName( DNS.getDefaultHost(conf.get("hbase.thrift.dns.interface", "default"), conf.get("hbase.thrift.dns.nameserver", "default"))); User.login(conf, "hbase.thrift.keytab.file", "hbase.thrift.kerberos.principal", machineName); } serverRunner = new ThriftServerRunner(conf); // Put up info server. int port = conf.getInt("hbase.thrift.info.port", 9095); if (port >= 0) { conf.setLong("startcode", System.currentTimeMillis()); String a = conf.get("hbase.thrift.info.bindAddress", "0.0.0.0"); infoServer = new InfoServer("thrift", a, port, false, conf); infoServer.setAttribute("hbase.conf", conf); infoServer.start(); } serverRunner.run(); }
public TokenServer(Configuration conf) throws IOException { this.conf = conf; this.startcode = EnvironmentEdgeManager.currentTimeMillis(); // Server to handle client requests. String hostname = Strings.domainNamePointerToHostName( DNS.getDefaultHost("default", "default")); int port = 0; // Creation of an ISA will force a resolve. InetSocketAddress initialIsa = new InetSocketAddress(hostname, port); if (initialIsa.getAddress() == null) { throw new IllegalArgumentException("Failed resolve of " + initialIsa); } this.rpcServer = HBaseServerRPC.getServer(TokenServer.class, this, new Class<?>[]{AuthenticationProtos.AuthenticationService.Interface.class}, initialIsa.getHostName(), // BindAddress is IP we got for this server. initialIsa.getPort(), 3, // handlers 1, // meta handlers (not used) true, this.conf, HConstants.QOS_THRESHOLD); // Set our address. this.isa = this.rpcServer.getListenerAddress(); this.sleeper = new Sleeper(1000, this); }
public ThriftServerRunner(Configuration conf) throws IOException { UserProvider userProvider = UserProvider.instantiate(conf); // login the server principal (if using secure Hadoop) securityEnabled = userProvider.isHadoopSecurityEnabled() && userProvider.isHBaseSecurityEnabled(); if (securityEnabled) { host = Strings.domainNamePointerToHostName(DNS.getDefaultHost( conf.get("hbase.thrift.dns.interface", "default"), conf.get("hbase.thrift.dns.nameserver", "default"))); userProvider.login("hbase.thrift.keytab.file", "hbase.thrift.kerberos.principal", host); } this.conf = HBaseConfiguration.create(conf); this.listenPort = conf.getInt(PORT_CONF_KEY, DEFAULT_LISTEN_PORT); this.metrics = new ThriftMetrics(conf, ThriftMetrics.ThriftServerType.ONE); this.hbaseHandler = new HBaseHandler(conf, userProvider); this.hbaseHandler.initMetrics(metrics); this.handler = HbaseHandlerMetricsProxy.newInstance( hbaseHandler, metrics, conf); this.realUser = userProvider.getCurrent().getUGI(); qop = conf.get(THRIFT_QOP_KEY); doAsEnabled = conf.getBoolean(THRIFT_SUPPORT_PROXYUSER, false); if (qop != null) { if (!qop.equals("auth") && !qop.equals("auth-int") && !qop.equals("auth-conf")) { throw new IOException("Invalid " + THRIFT_QOP_KEY + ": " + qop + ", it must be 'auth', 'auth-int', or 'auth-conf'"); } if (!securityEnabled) { throw new IOException("Thrift server must" + " run in secure mode to support authentication"); } } }
/** * sinkToString * @return a string contains sinkReplicationLoad information */ public String sinkToString() { if (this.replicationLoadSink == null) return null; StringBuilder sb = new StringBuilder(); sb = Strings.appendKeyValue(sb, "AgeOfLastAppliedOp", this.replicationLoadSink.getAgeOfLastAppliedOp()); sb = Strings.appendKeyValue(sb, "TimeStampsOfLastAppliedOp", (new Date(this.replicationLoadSink.getTimeStampsOfLastAppliedOp()).toString())); return sb.toString(); }
public static String getHostname(Configuration conf, boolean isMaster) throws UnknownHostException { String hostname = conf.get(isMaster? HRegionServer.MASTER_HOSTNAME_KEY : HRegionServer.RS_HOSTNAME_KEY); if (hostname == null || hostname.isEmpty()) { String masterOrRS = isMaster ? "master" : "regionserver"; return Strings.domainNamePointerToHostName(DNS.getDefaultHost( conf.get("hbase." + masterOrRS + ".dns.interface", "default"), conf.get("hbase." + masterOrRS + ".dns.nameserver", "default"))); } else { LOG.info("hostname is configured to be " + hostname); return hostname; } }
public TokenServer(Configuration conf) throws IOException { this.conf = conf; this.startcode = EnvironmentEdgeManager.currentTime(); // Server to handle client requests. String hostname = Strings.domainNamePointerToHostName(DNS.getDefaultHost("default", "default")); int port = 0; // Creation of an ISA will force a resolve. InetSocketAddress initialIsa = new InetSocketAddress(hostname, port); if (initialIsa.getAddress() == null) { throw new IllegalArgumentException("Failed resolve of " + initialIsa); } final List<BlockingServiceAndInterface> sai = new ArrayList<BlockingServiceAndInterface>(1); BlockingService service = AuthenticationProtos.AuthenticationService.newReflectiveBlockingService(this); sai.add(new BlockingServiceAndInterface(service, AuthenticationProtos.AuthenticationService.BlockingInterface.class)); this.rpcServer = new RpcServer(this, "tokenServer", sai, initialIsa, conf, new FifoRpcScheduler(conf, 1)); InetSocketAddress address = rpcServer.getListenerAddress(); if (address == null) { throw new IOException("Listener channel is closed"); } this.isa = address; this.sleeper = new Sleeper(1000, this); }
private void doPuts(Region region) throws IOException{ LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(MIN_VALUE_SIZE, MAX_VALUE_SIZE); for (int i = 0; i < NUM_ROWS; ++i) { byte[] key = LoadTestKVGenerator.md5PrefixedKey(i).getBytes(); for (int j = 0; j < NUM_COLS_PER_ROW; ++j) { Put put = new Put(key); put.setDurability(Durability.ASYNC_WAL); byte[] col = Bytes.toBytes(String.valueOf(j)); byte[] value = dataGenerator.generateRandomSizeValue(key, col); if (includeTags) { Tag[] tag = new Tag[1]; tag[0] = new Tag((byte) 1, "Visibility"); KeyValue kv = new KeyValue(key, CF_BYTES, col, HConstants.LATEST_TIMESTAMP, value, tag); put.add(kv); } else { put.add(CF_BYTES, col, value); } if(VERBOSE){ KeyValue kvPut = new KeyValue(key, CF_BYTES, col, value); System.err.println(Strings.padFront(i+"", ' ', 4)+" "+kvPut); } region.put(put); } if (i % NUM_ROWS_PER_FLUSH == 0) { region.flush(true); } } }
/** * Returns the configuration to be used by the authentication filter * to initialize the authentication handler. * * This filter retrieves all HBase configurations and passes those started * with REST_PREFIX to the authentication handler. It is useful to support * plugging different authentication handlers. */ @Override protected Properties getConfiguration( String configPrefix, FilterConfig filterConfig) throws ServletException { Properties props = super.getConfiguration(configPrefix, filterConfig); //setting the cookie path to root '/' so it is used for all resources. props.setProperty(AuthenticationFilter.COOKIE_PATH, "/"); Configuration conf = HBaseConfiguration.create(); for (Map.Entry<String, String> entry : conf) { String name = entry.getKey(); if (name.startsWith(REST_PREFIX)) { String value = entry.getValue(); if(name.equals(REST_AUTHENTICATION_PRINCIPAL)) { try { String machineName = Strings.domainNamePointerToHostName( DNS.getDefaultHost(conf.get(REST_DNS_INTERFACE, "default"), conf.get(REST_DNS_NAMESERVER, "default"))); value = SecurityUtil.getServerPrincipal(value, machineName); } catch (IOException ie) { throw new ServletException("Failed to retrieve server principal", ie); } } LOG.debug("Setting property " + name + "=" + value); name = name.substring(REST_PREFIX_LEN); props.setProperty(name, value); } } return props; }
/****************** standard methods ************************/ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append(Strings.padFront(builderNode.getOutputArrayOffset() + "", ' ', 3) + ","); sb.append("["); sb.append(Bytes.toString(token)); sb.append("]->"); sb.append(parentStartPosition); return sb.toString(); }
public String getPaddedTokenAndOccurrenceString() { StringBuilder sb = new StringBuilder(); sb.append(getBnlIndicator(true)); sb.append(Strings.padFront(numOccurrences + "", ' ', 3)); sb.append(Strings.padFront(nodeDepth + "", ' ', 3)); if (outputArrayOffset >= 0) { sb.append(Strings.padFront(outputArrayOffset + "", ' ', 3)); } sb.append(" "); for (int i = 0; i < tokenStartOffset; ++i) { sb.append(" "); } sb.append(Bytes.toString(token.deepCopyToNewArray()).replaceAll(" ", "_")); return sb.toString(); }
