public static void main(String[] args) throws Exception { final Configuration conf = HBaseConfiguration.create(); final ChoreService choreService = new ChoreService("CANARY_TOOL"); final ScheduledChore authChore = AuthUtil.getAuthChore(conf); if (authChore != null) { choreService.scheduleChore(authChore); } // loading the generic options to conf new GenericOptionsParser(conf, args); int numThreads = conf.getInt("hbase.canary.threads.num", MAX_THREADS_NUM); LOG.info("Number of exection threads " + numThreads); ExecutorService executor = new ScheduledThreadPoolExecutor(numThreads); Class<? extends Sink> sinkClass = conf.getClass("hbase.canary.sink.class", RegionServerStdOutSink.class, Sink.class); Sink sink = ReflectionUtils.newInstance(sinkClass); int exitCode = ToolRunner.run(conf, new Canary(executor, sink), args); choreService.shutdown(); executor.shutdown(); System.exit(exitCode); }
/** Get an array of FilterConfiguration specified in the conf */ private static FilterInitializer[] getFilterInitializers(Configuration conf) { if (conf == null) { return null; } Class<?>[] classes = conf.getClasses(FILTER_INITIALIZERS_PROPERTY); if (classes == null) { return null; } FilterInitializer[] initializers = new FilterInitializer[classes.length]; for(int i = 0; i < classes.length; i++) { initializers[i] = (FilterInitializer)ReflectionUtils.newInstance(classes[i]); } return initializers; }
/** * Constructor. * @param c Comparator */ public DefaultMemStore(final Configuration conf, final KeyValue.KVComparator c) { this.conf = conf; this.comparator = c; this.cellSet = new CellSkipListSet(c); this.snapshot = new CellSkipListSet(c); timeRangeTracker = new TimeRangeTracker(); snapshotTimeRangeTracker = new TimeRangeTracker(); this.size = new AtomicLong(DEEP_OVERHEAD); this.snapshotSize = 0; if (conf.getBoolean(USEMSLAB_KEY, USEMSLAB_DEFAULT)) { String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); this.allocator = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] { Configuration.class }, new Object[] { conf }); } else { this.allocator = null; } }
public static MonkeyFactory getFactory(String factoryName) { MonkeyFactory fact = FACTORIES.get(factoryName); if (fact == null && factoryName != null && !factoryName.isEmpty()) { Class klass = null; try { klass = Class.forName(factoryName); if (klass != null) { fact = (MonkeyFactory) ReflectionUtils.newInstance(klass); } } catch (Exception e) { LOG.error("Error trying to create " + factoryName + " could not load it by class name"); return null; } } return fact; }
public static RpcRetryingCallerFactory instantiate(Configuration configuration, RetryingCallerInterceptor interceptor, ServerStatisticTracker stats) { String clazzName = RpcRetryingCallerFactory.class.getName(); String rpcCallerFactoryClazz = configuration.get(RpcRetryingCallerFactory.CUSTOM_CALLER_CONF_KEY, clazzName); RpcRetryingCallerFactory factory; if (rpcCallerFactoryClazz.equals(clazzName)) { factory = new RpcRetryingCallerFactory(configuration, interceptor); } else { factory = ReflectionUtils.instantiateWithCustomCtor( rpcCallerFactoryClazz, new Class[] { Configuration.class }, new Object[] { configuration }); } // setting for backwards compat with existing caller factories, rather than in the ctor factory.setStatisticTracker(stats); return factory; }
public static RpcControllerFactory instantiate(Configuration configuration) { String rpcControllerFactoryClazz = configuration.get(CUSTOM_CONTROLLER_CONF_KEY, RpcControllerFactory.class.getName()); try { return ReflectionUtils.instantiateWithCustomCtor(rpcControllerFactoryClazz, new Class[] { Configuration.class }, new Object[] { configuration }); } catch (UnsupportedOperationException | NoClassDefFoundError ex) { // HBASE-14960: In case the RPCController is in a non-HBase jar (Phoenix), but the application // is a pure HBase application, we want to fallback to the default one. String msg = "Cannot load configured \"" + CUSTOM_CONTROLLER_CONF_KEY + "\" (" + rpcControllerFactoryClazz + ") from hbase-site.xml, falling back to use " + "default RpcControllerFactory"; if (LOG.isDebugEnabled()) { LOG.warn(msg, ex); // if DEBUG enabled, we want the exception, but still log in WARN level } else { LOG.warn(msg); } return new RpcControllerFactory(configuration); } }
/** * Constructor. * @param c Comparator */ public DefaultMemStore(final Configuration conf, final KeyValue.KVComparator c) { this.conf = conf; this.comparator = c; this.cellSet = new CellSkipListSet(c); this.snapshot = new CellSkipListSet(c); timeRangeTracker = new TimeRangeTracker(); snapshotTimeRangeTracker = new TimeRangeTracker(); this.size = new AtomicLong(DEEP_OVERHEAD); this.snapshotSize = 0; if (conf.getBoolean(USEMSLAB_KEY, USEMSLAB_DEFAULT)) { String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); this.allocator = ReflectionUtils.instantiateWithCustomCtor(className, new Class[]{Configuration.class}, new Object[]{conf}); } else { this.allocator = null; } }
public static <R, S, P extends Message, Q extends Message, T extends Message> CompletableFuture<R> median(AsyncTable<AdvancedScanResultConsumer> table, ColumnInterpreter<R, S, P, Q, T> ci, Scan scan) { CompletableFuture<R> future = new CompletableFuture<>(); sumByRegion(table, ci, scan).whenComplete((sumByRegion, error) -> { if (error != null) { future.completeExceptionally(error); } else if (sumByRegion.isEmpty()) { future.completeExceptionally(new NoSuchElementException()); } else { findMedian(future, table, ci, ReflectionUtils.newInstance(scan.getClass(), scan), sumByRegion); } }); return future; }
public static void main(String[] args) throws Exception { final Configuration conf = HBaseConfiguration.create(); // loading the generic options to conf new GenericOptionsParser(conf, args); int numThreads = conf.getInt("hbase.canary.threads.num", MAX_THREADS_NUM); LOG.info("Number of execution threads " + numThreads); ExecutorService executor = new ScheduledThreadPoolExecutor(numThreads); Class<? extends Sink> sinkClass = conf.getClass("hbase.canary.sink.class", RegionServerStdOutSink.class, Sink.class); Sink sink = ReflectionUtils.newInstance(sinkClass); int exitCode = ToolRunner.run(conf, new Canary(executor, sink), args); executor.shutdown(); System.exit(exitCode); }
/** * Returns a brand new instance of the FileSystem. It does not use * the FileSystem.Cache. In newer versions of HDFS, we can directly * invoke FileSystem.newInstance(Configuration). * * @param conf Configuration * @return A new instance of the filesystem */ private static FileSystem newInstanceFileSystem(Configuration conf) throws IOException { URI uri = FileSystem.getDefaultUri(conf); FileSystem fs = null; Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null); if (clazz != null) { // This will be true for Hadoop 1.0, or 0.20. fs = (FileSystem) org.apache.hadoop.util.ReflectionUtils.newInstance(clazz, conf); fs.initialize(uri, conf); } else { // For Hadoop 2.0, we have to go through FileSystem for the filesystem // implementation to be loaded by the service loader in case it has not // been loaded yet. Configuration clone = new Configuration(conf); clone.setBoolean("fs." + uri.getScheme() + ".impl.disable.cache", true); fs = FileSystem.get(uri, clone); } if (fs == null) { throw new IOException("No FileSystem for scheme: " + uri.getScheme()); } return fs; }
public static RpcServer createRpcServer(final Server server, final String name, final List<BlockingServiceAndInterface> services, final InetSocketAddress bindAddress, Configuration conf, RpcScheduler scheduler, boolean reservoirEnabled) throws IOException { String rpcServerClass = conf.get(CUSTOM_RPC_SERVER_IMPL_CONF_KEY, NettyRpcServer.class.getName()); StringBuilder servicesList = new StringBuilder(); for (BlockingServiceAndInterface s: services) { ServiceDescriptor sd = s.getBlockingService().getDescriptorForType(); if (sd == null) continue; // Can be null for certain tests like TestTokenAuthentication if (servicesList.length() > 0) servicesList.append(", "); servicesList.append(sd.getFullName()); } LOG.info("Creating " + rpcServerClass + " hosting " + servicesList); return ReflectionUtils.instantiateWithCustomCtor(rpcServerClass, new Class[] { Server.class, String.class, List.class, InetSocketAddress.class, Configuration.class, RpcScheduler.class, boolean.class }, new Object[] { server, name, services, bindAddress, conf, scheduler, reservoirEnabled }); }
/** * Create a new AsyncConnection instance using the passed {@code conf} and {@code user}. * AsyncConnection encapsulates all housekeeping for a connection to the cluster. All tables and * interfaces created from returned connection share zookeeper connection, meta cache, and * connections to region servers and masters. * <p> * The caller is responsible for calling {@link AsyncConnection#close()} on the returned * connection instance. * <p> * Usually you should only create one AsyncConnection instance in your code and use it everywhere * as it is thread safe. * @param conf configuration * @param user the user the asynchronous connection is for * @return AsyncConnection object wrapped by CompletableFuture * @throws IOException */ public static CompletableFuture<AsyncConnection> createAsyncConnection(Configuration conf, User user) { CompletableFuture<AsyncConnection> future = new CompletableFuture<>(); AsyncRegistry registry = AsyncRegistryFactory.getRegistry(conf); registry.getClusterId().whenComplete((clusterId, error) -> { if (error != null) { future.completeExceptionally(error); return; } if (clusterId == null) { future.completeExceptionally(new IOException("clusterid came back null")); return; } Class<? extends AsyncConnection> clazz = conf.getClass(HBASE_CLIENT_ASYNC_CONNECTION_IMPL, AsyncConnectionImpl.class, AsyncConnection.class); try { future.complete(ReflectionUtils.newInstance(clazz, conf, registry, clusterId, user)); } catch (Exception e) { future.completeExceptionally(e); } }); return future; }
/** * Constructor. * @param c Comparator */ public DefaultMemStore(final Configuration conf, final KeyValue.KVComparator c) { this.conf = conf; this.comparator = c; this.kvset = new KeyValueSkipListSet(c); this.snapshot = new KeyValueSkipListSet(c); timeRangeTracker = new TimeRangeTracker(); snapshotTimeRangeTracker = new TimeRangeTracker(); this.size = new AtomicLong(DEEP_OVERHEAD); this.snapshotSize = 0; if (conf.getBoolean(USEMSLAB_KEY, USEMSLAB_DEFAULT)) { String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); this.allocator = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] { Configuration.class }, new Object[] { conf }); } else { this.allocator = null; } }
@Override public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(), request, response)) { return; } response.setContentType("text/plain; charset=UTF-8"); try (PrintStream out = new PrintStream( response.getOutputStream(), false, "UTF-8")) { Threads.printThreadInfo(out, ""); out.flush(); } ReflectionUtils.logThreadInfo(LOG, "jsp requested", 1); }
@Override public T newChannel() { try { return ReflectionUtils.instantiateWithCustomCtor(clazz.getName(), new Class[] { InternetProtocolFamily.class }, new Object[] { family }); } catch (Throwable t) { throw new ChannelException("Unable to create Channel from class " + clazz, t); } }
/** * Create the StoreEngine configured for the given Store. * @param store The store. An unfortunate dependency needed due to it * being passed to coprocessors via the compactor. * @param conf Store configuration. * @param kvComparator KVComparator for storeFileManager. * @return StoreEngine to use. */ public static StoreEngine<?, ?, ?, ?> create( Store store, Configuration conf, KVComparator kvComparator) throws IOException { String className = conf.get(STORE_ENGINE_CLASS_KEY, DEFAULT_STORE_ENGINE_CLASS.getName()); try { StoreEngine<?,?,?,?> se = ReflectionUtils.instantiateWithCustomCtor( className, new Class[] { }, new Object[] { }); se.createComponentsOnce(conf, store, kvComparator); return se; } catch (Exception e) { throw new IOException("Unable to load configured store engine '" + className + "'", e); } }
/** * Create a merge transaction * @param a region a to merge * @param b region b to merge * @param forcible if false, we will only merge adjacent regions * @return transaction instance */ public RegionMergeTransactionImpl create(final Region a, final Region b, final boolean forcible) { // The implementation class must extend RegionMergeTransactionImpl, not only // implement the RegionMergeTransaction interface like you might expect, // because various places such as AssignmentManager use static methods // from RegionMergeTransactionImpl. Whatever we use for implementation must // be compatible, so it's safest to require ? extends RegionMergeTransactionImpl. // If not compatible we will throw a runtime exception from here. return ReflectionUtils.instantiateWithCustomCtor( conf.getClass(MERGE_TRANSACTION_IMPL_KEY, RegionMergeTransactionImpl.class, RegionMergeTransactionImpl.class).getName(), new Class[] { Region.class, Region.class, boolean.class }, new Object[] { a, b, forcible }); }
/** * Creates a snapshot of the current memstore. * Snapshot must be cleared by call to {@link #clearSnapshot(long)} */ @Override public MemStoreSnapshot snapshot() { // If snapshot currently has entries, then flusher failed or didn't call // cleanup. Log a warning. if (!this.snapshot.isEmpty()) { LOG.warn("Snapshot called again without clearing previous. " + "Doing nothing. Another ongoing flush or did we fail last attempt?"); } else { this.snapshotId = EnvironmentEdgeManager.currentTime(); this.snapshotSize = keySize(); if (!this.cellSet.isEmpty()) { this.snapshot = this.cellSet; this.cellSet = new CellSkipListSet(this.comparator); this.snapshotTimeRangeTracker = this.timeRangeTracker; this.timeRangeTracker = new TimeRangeTracker(); // Reset heap to not include any keys this.size.set(DEEP_OVERHEAD); this.snapshotAllocator = this.allocator; // Reset allocator so we get a fresh buffer for the new memstore if (allocator != null) { String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); this.allocator = ReflectionUtils.instantiateWithCustomCtor(className, new Class[] { Configuration.class }, new Object[] { conf }); } timeOfOldestEdit = Long.MAX_VALUE; } } MemStoreSnapshot memStoreSnapshot = new MemStoreSnapshot(this.snapshotId, snapshot.size(), this.snapshotSize, this.snapshotTimeRangeTracker, new CollectionBackedScanner(snapshot, this.comparator), this.tagsPresent); this.tagsPresent = false; return memStoreSnapshot; }
/** * Create a split transaction * @param r the region to split * @param splitrow the split point in the keyspace * @return transaction instance */ public SplitTransaction create(final Region r, final byte [] splitrow) { return ReflectionUtils.instantiateWithCustomCtor( // The implementation class must extend SplitTransactionImpl, not only // implement the SplitTransaction interface like you might expect, // because various places such as AssignmentManager use static methods // from SplitTransactionImpl. Whatever we use for implementation must // be compatible, so it's safest to require ? extends SplitTransactionImpl. // If not compatible we will throw a runtime exception from here. conf.getClass(SPLIT_TRANSACTION_IMPL_KEY, SplitTransactionImpl.class, SplitTransactionImpl.class).getName(), new Class[] { Region.class, byte[].class }, new Object[] { r, splitrow }); }
/** * Test the perf of a CompactionPolicy with settings. * @param cpClass The compaction policy to test * @param inMmax The maximum number of file to compact * @param inMin The min number of files to compact * @param inRatio The ratio that files must be under to be compacted. */ public PerfTestCompactionPolicies( final Class<? extends CompactionPolicy> cpClass, final Class<? extends StoreFileListGenerator> fileGenClass, final int inMmax, final int inMin, final float inRatio) throws IllegalAccessException, InstantiationException { super(PerfTestCompactionPolicies.class); this.fileGenClass = fileGenClass; this.max = inMmax; this.min = inMin; this.ratio = inRatio; // Hide lots of logging so the system out is usable as a tab delimited file. org.apache.log4j.Logger.getLogger(CompactionConfiguration.class). setLevel(org.apache.log4j.Level.ERROR); org.apache.log4j.Logger.getLogger(RatioBasedCompactionPolicy.class). setLevel(org.apache.log4j.Level.ERROR); org.apache.log4j.Logger.getLogger(cpClass).setLevel(org.apache.log4j.Level.ERROR); Configuration configuration = HBaseConfiguration.create(); // Make sure that this doesn't include every file. configuration.setInt("hbase.hstore.compaction.max", max); configuration.setInt("hbase.hstore.compaction.min", min); configuration.setFloat("hbase.hstore.compaction.ratio", ratio); store = createMockStore(); this.cp = ReflectionUtils.instantiateWithCustomCtor(cpClass.getName(), new Class[] {Configuration.class, StoreConfigInformation.class }, new Object[] {configuration, store }); this.generator = fileGenClass.newInstance(); // Used for making paths }
public static ClientBackoffPolicy create(Configuration conf) { // create the backoff policy String className = conf.get(ClientBackoffPolicy.BACKOFF_POLICY_CLASS, NoBackoffPolicy.class .getName()); return ReflectionUtils.instantiateWithCustomCtor(className, new Class<?>[] { Configuration.class }, new Object[] { conf }); }
/** * Creates a new RpcClient by the class defined in the configuration or falls back to * RpcClientImpl * @param conf configuration * @param clusterId the cluster id * @param localAddr client socket bind address. * @param metrics the connection metrics * @return newly created RpcClient */ public static RpcClient createClient(Configuration conf, String clusterId, SocketAddress localAddr, MetricsConnection metrics) { String rpcClientClass = conf.get(CUSTOM_RPC_CLIENT_IMPL_CONF_KEY, RpcClientImpl.class.getName()); return ReflectionUtils.instantiateWithCustomCtor( rpcClientClass, new Class[] { Configuration.class, String.class, SocketAddress.class, MetricsConnection.class }, new Object[] { conf, clusterId, localAddr, metrics } ); }
/** * Creates a snapshot of the current memstore. * Snapshot must be cleared by call to {@link #clearSnapshot(long)} */ @Override public MemStoreSnapshot snapshot() { // If snapshot currently has entries, then flusher failed or didn't call // cleanup. Log a warning. if (!this.snapshot.isEmpty()) { LOG.warn("Snapshot called again without clearing previous. " + "Doing nothing. Another ongoing flush or did we fail last attempt?"); } else { this.snapshotId = EnvironmentEdgeManager.currentTime(); this.snapshotSize = keySize(); if (!this.cellSet.isEmpty()) { this.snapshot = this.cellSet; this.cellSet = new CellSkipListSet(this.comparator); this.snapshotTimeRangeTracker = this.timeRangeTracker; this.timeRangeTracker = new TimeRangeTracker(); // Reset heap to not include any keys this.size.set(DEEP_OVERHEAD); this.snapshotAllocator = this.allocator; // Reset allocator so we get a fresh buffer for the new memstore if (allocator != null) { String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); this.allocator = ReflectionUtils.instantiateWithCustomCtor(className, new Class[]{Configuration.class}, new Object[]{conf}); } timeOfOldestEdit = Long.MAX_VALUE; } } return new MemStoreSnapshot(this.snapshotId, snapshot.size(), this.snapshotSize, this.snapshotTimeRangeTracker, new CollectionBackedScanner(snapshot, this.comparator)); }
public static RpcControllerFactory instantiate(Configuration configuration) { String rpcControllerFactoryClazz = configuration.get(CUSTOM_CONTROLLER_CONF_KEY, RpcControllerFactory.class.getName()); return ReflectionUtils.instantiateWithCustomCtor(rpcControllerFactoryClazz, new Class[] { Configuration.class }, new Object[] { configuration }); }
/** * Creates a new RpcClient by the class defined in the configuration or falls back to * RpcClientImpl * @param conf configuration * @param clusterId the cluster id * @param localAddr client socket bind address. * @return newly created RpcClient */ public static RpcClient createClient(Configuration conf, String clusterId, SocketAddress localAddr) { String rpcClientClass = conf.get(CUSTOM_RPC_CLIENT_IMPL_CONF_KEY, RpcClientImpl.class.getName()); return ReflectionUtils.instantiateWithCustomCtor( rpcClientClass, new Class[] { Configuration.class, String.class, SocketAddress.class }, new Object[] { conf, clusterId, localAddr } ); }
public static RpcRetryingCallerFactory instantiate(Configuration configuration) { String rpcCallerFactoryClazz = configuration.get(RpcRetryingCallerFactory.CUSTOM_CALLER_CONF_KEY, RpcRetryingCallerFactory.class.getName()); return ReflectionUtils.instantiateWithCustomCtor(rpcCallerFactoryClazz, new Class[] { Configuration.class }, new Object[] { configuration }); }
/** * Get the storage policy of the source path (directory/file). * @param path The source path (directory/file). * @return Storage policy name, or {@code null} if not using {@link DistributedFileSystem} or * exception thrown when trying to get policy */ @Nullable public String getStoragePolicyName(Path path) { try { Object blockStoragePolicySpi = ReflectionUtils.invokeMethod(this.fs, "getStoragePolicy", path); return (String) ReflectionUtils.invokeMethod(blockStoragePolicySpi, "getName"); } catch (Exception e) { // Maybe fail because of using old HDFS version, try the old way if (LOG.isTraceEnabled()) { LOG.trace("Failed to get policy directly", e); } return getStoragePolicyForOldHDFSVersion(path); } }
