private static Runnable suppressHdfsShutdownHook(final FileSystem fs) { try { // This introspection has been updated to work for hadoop 0.20, 0.21 and for // cloudera 0.20. 0.21 and cloudera 0.20 both have hadoop-4829. With the // latter in place, things are a little messy in that there are now two // instances of the data member clientFinalizer; an uninstalled one in // FileSystem and one in the innner class named Cache that actually gets // registered as a shutdown hook. If the latter is present, then we are // on 0.21 or cloudera patched 0.20. Runnable hdfsClientFinalizer = null; // Look into the FileSystem#Cache class for clientFinalizer Class<?> [] classes = FileSystem.class.getDeclaredClasses(); Class<?> cache = null; for (Class<?> c: classes) { if (c.getSimpleName().equals("Cache")) { cache = c; break; } } if (cache == null) { throw new RuntimeException( "This should not happen. Could not find the cache class in FileSystem."); } Field field = null; try { field = cache.getDeclaredField(CLIENT_FINALIZER_DATA_METHOD); } catch (NoSuchFieldException e) { // We can get here if the Cache class does not have a clientFinalizer // instance: i.e. we're running on straight 0.20 w/o hadoop-4829. } if (field != null) { field.setAccessible(true); Field cacheField = FileSystem.class.getDeclaredField("CACHE"); cacheField.setAccessible(true); Object cacheInstance = cacheField.get(fs); hdfsClientFinalizer = (Runnable)field.get(cacheInstance); } else { // Then we didnt' find clientFinalizer in Cache. Presume clean 0.20 hadoop. field = FileSystem.class.getDeclaredField(CLIENT_FINALIZER_DATA_METHOD); field.setAccessible(true); hdfsClientFinalizer = (Runnable)field.get(null); } if (hdfsClientFinalizer == null) { throw new RuntimeException("Client finalizer is null, can't suppress!"); } synchronized (fsShutdownHooks) { if (!fsShutdownHooks.containsKey(hdfsClientFinalizer) && !ShutdownHookManager.deleteShutdownHook(hdfsClientFinalizer)) { throw new RuntimeException("Failed suppression of fs shutdown hook: " + hdfsClientFinalizer); } Integer refs = fsShutdownHooks.get(hdfsClientFinalizer); fsShutdownHooks.put(hdfsClientFinalizer, refs == null ? 1 : refs + 1); } return hdfsClientFinalizer; } catch (NoSuchFieldException nsfe) { LOG.fatal("Couldn't find field 'clientFinalizer' in FileSystem!", nsfe); throw new RuntimeException("Failed to suppress HDFS shutdown hook"); } catch (IllegalAccessException iae) { LOG.fatal("Couldn't access field 'clientFinalizer' in FileSystem!", iae); throw new RuntimeException("Failed to suppress HDFS shutdown hook"); } }
private static Runnable suppressHdfsShutdownHook(final FileSystem fs) { try { // This introspection has been updated to work for hadoop 0.20, 0.21 and for // cloudera 0.20. 0.21 and cloudera 0.20 both have hadoop-4829. With the // latter in place, things are a little messy in that there are now two // instances of the data member clientFinalizer; an uninstalled one in // FileSystem and one in the innner class named Cache that actually gets // registered as a shutdown hook. If the latter is present, then we are // on 0.21 or cloudera patched 0.20. Runnable hdfsClientFinalizer = null; // Look into the FileSystem#Cache class for clientFinalizer Class<?> [] classes = FileSystem.class.getDeclaredClasses(); Class<?> cache = null; for (Class<?> c: classes) { if (c.getSimpleName().equals("Cache")) { cache = c; break; } } Field field = null; try { field = cache.getDeclaredField(CLIENT_FINALIZER_DATA_METHOD); } catch (NoSuchFieldException e) { // We can get here if the Cache class does not have a clientFinalizer // instance: i.e. we're running on straight 0.20 w/o hadoop-4829. } if (field != null) { field.setAccessible(true); Field cacheField = FileSystem.class.getDeclaredField("CACHE"); cacheField.setAccessible(true); Object cacheInstance = cacheField.get(fs); hdfsClientFinalizer = (Runnable)field.get(cacheInstance); } else { // Then we didnt' find clientFinalizer in Cache. Presume clean 0.20 hadoop. field = FileSystem.class.getDeclaredField(CLIENT_FINALIZER_DATA_METHOD); field.setAccessible(true); hdfsClientFinalizer = (Runnable)field.get(null); } if (hdfsClientFinalizer == null) { throw new RuntimeException("Client finalizer is null, can't suppress!"); } if (!fsShutdownHooks.containsKey(hdfsClientFinalizer) && !ShutdownHookManager.deleteShutdownHook(hdfsClientFinalizer)) { throw new RuntimeException("Failed suppression of fs shutdown hook: " + hdfsClientFinalizer); } synchronized (fsShutdownHooks) { Integer refs = fsShutdownHooks.get(hdfsClientFinalizer); fsShutdownHooks.put(hdfsClientFinalizer, refs == null ? 1 : refs + 1); } return hdfsClientFinalizer; } catch (NoSuchFieldException nsfe) { LOG.fatal("Couldn't find field 'clientFinalizer' in FileSystem!", nsfe); throw new RuntimeException("Failed to suppress HDFS shutdown hook"); } catch (IllegalAccessException iae) { LOG.fatal("Couldn't access field 'clientFinalizer' in FileSystem!", iae); throw new RuntimeException("Failed to suppress HDFS shutdown hook"); } }
private static Runnable suppressHdfsShutdownHook(final FileSystem fs) { try { // This introspection has been updated to work for hadoop 0.20, 0.21 and for // cloudera 0.20. 0.21 and cloudera 0.20 both have hadoop-4829. With the // latter in place, things are a little messy in that there are now two // instances of the data member clientFinalizer; an uninstalled one in // FileSystem and one in the innner class named Cache that actually gets // registered as a shutdown hook. If the latter is present, then we are // on 0.21 or cloudera patched 0.20. Runnable hdfsClientFinalizer = null; // Look into the FileSystem#Cache class for clientFinalizer Class<?> [] classes = FileSystem.class.getDeclaredClasses(); Class<?> cache = null; for (Class<?> c: classes) { if (c.getSimpleName().equals("Cache")) { cache = c; break; } } if (cache == null) { throw new RuntimeException( "This should not happen. Could not find the cache class in FileSystem."); } Field field = null; try { field = cache.getDeclaredField(CLIENT_FINALIZER_DATA_METHOD); } catch (NoSuchFieldException e) { // We can get here if the Cache class does not have a clientFinalizer // instance: i.e. we're running on straight 0.20 w/o hadoop-4829. } if (field != null) { field.setAccessible(true); Field cacheField = FileSystem.class.getDeclaredField("CACHE"); cacheField.setAccessible(true); Object cacheInstance = cacheField.get(fs); hdfsClientFinalizer = (Runnable)field.get(cacheInstance); } else { // Then we didnt' find clientFinalizer in Cache. Presume clean 0.20 hadoop. field = FileSystem.class.getDeclaredField(CLIENT_FINALIZER_DATA_METHOD); field.setAccessible(true); hdfsClientFinalizer = (Runnable)field.get(null); } if (hdfsClientFinalizer == null) { throw new RuntimeException("Client finalizer is null, can't suppress!"); } synchronized (fsShutdownHooks) { boolean isFSCacheDisabled = fs.getConf().getBoolean("fs.hdfs.impl.disable.cache", false); if (!isFSCacheDisabled && !fsShutdownHooks.containsKey(hdfsClientFinalizer) && !ShutdownHookManager.deleteShutdownHook(hdfsClientFinalizer)) { throw new RuntimeException( "Failed suppression of fs shutdown hook: " + hdfsClientFinalizer); } Integer refs = fsShutdownHooks.get(hdfsClientFinalizer); fsShutdownHooks.put(hdfsClientFinalizer, refs == null ? 1 : refs + 1); } return hdfsClientFinalizer; } catch (NoSuchFieldException nsfe) { LOG.error(HBaseMarkers.FATAL, "Couldn't find field 'clientFinalizer' in FileSystem!", nsfe); throw new RuntimeException("Failed to suppress HDFS shutdown hook"); } catch (IllegalAccessException iae) { LOG.error(HBaseMarkers.FATAL, "Couldn't access field 'clientFinalizer' in FileSystem!", iae); throw new RuntimeException("Failed to suppress HDFS shutdown hook"); } }
private static Runnable suppressHdfsShutdownHook(final FileSystem fs) { try { // This introspection has been updated to work for hadoop 0.20, 0.21 and for // cloudera 0.20. 0.21 and cloudera 0.20 both have hadoop-4829. With the // latter in place, things are a little messy in that there are now two // instances of the data member clientFinalizer; an uninstalled one in // FileSystem and one in the innner class named Cache that actually gets // registered as a shutdown hook. If the latter is present, then we are // on 0.21 or cloudera patched 0.20. Runnable hdfsClientFinalizer = null; // Look into the FileSystem#Cache class for clientFinalizer Class<?> [] classes = FileSystem.class.getDeclaredClasses(); Class<?> cache = null; for (Class<?> c: classes) { if (c.getSimpleName().equals("Cache")) { cache = c; break; } } if (cache == null) { throw new RuntimeException( "This should not happen. Could not find the cache class in FileSystem."); } Field field = null; try { field = cache.getDeclaredField(CLIENT_FINALIZER_DATA_METHOD); } catch (NoSuchFieldException e) { // We can get here if the Cache class does not have a clientFinalizer // instance: i.e. we're running on straight 0.20 w/o hadoop-4829. } if (field != null) { field.setAccessible(true); Field cacheField = FileSystem.class.getDeclaredField("CACHE"); cacheField.setAccessible(true); Object cacheInstance = cacheField.get(fs); hdfsClientFinalizer = (Runnable)field.get(cacheInstance); } else { // Then we didnt' find clientFinalizer in Cache. Presume clean 0.20 hadoop. field = FileSystem.class.getDeclaredField(CLIENT_FINALIZER_DATA_METHOD); field.setAccessible(true); hdfsClientFinalizer = (Runnable)field.get(null); } if (hdfsClientFinalizer == null) { throw new RuntimeException("Client finalizer is null, can't suppress!"); } if (!fsShutdownHooks.containsKey(hdfsClientFinalizer) && !ShutdownHookManager.deleteShutdownHook(hdfsClientFinalizer)) { throw new RuntimeException("Failed suppression of fs shutdown hook: " + hdfsClientFinalizer); } synchronized (fsShutdownHooks) { Integer refs = fsShutdownHooks.get(hdfsClientFinalizer); fsShutdownHooks.put(hdfsClientFinalizer, refs == null ? 1 : refs + 1); } return hdfsClientFinalizer; } catch (NoSuchFieldException nsfe) { LOG.fatal("Couldn't find field 'clientFinalizer' in FileSystem!", nsfe); throw new RuntimeException("Failed to suppress HDFS shutdown hook"); } catch (IllegalAccessException iae) { LOG.fatal("Couldn't access field 'clientFinalizer' in FileSystem!", iae); throw new RuntimeException("Failed to suppress HDFS shutdown hook"); } }
/** * Install a shutdown hook that calls stop on the passed Stoppable * and then thread joins against the passed <code>threadToJoin</code>. * When this thread completes, it then runs the hdfs thread (This install * removes the hdfs shutdown hook keeping a handle on it to run it after * <code>threadToJoin</code> has stopped). * * <p>To suppress all shutdown hook handling -- both the running of the * regionserver hook and of the hdfs hook code -- set * {@link ShutdownHook#RUN_SHUTDOWN_HOOK} in {@link Configuration} to * <code>false</code>. * This configuration value is checked when the hook code runs. * @param conf * @param fs Instance of Filesystem used by the RegionServer * @param stop Installed shutdown hook will call stop against this passed * <code>Stoppable</code> instance. * @param threadToJoin After calling stop on <code>stop</code> will then * join this thread. */ public static void install(final Configuration conf, final FileSystem fs, final Stoppable stop, final Thread threadToJoin) { Runnable fsShutdownHook = suppressHdfsShutdownHook(fs); Thread t = new ShutdownHookThread(conf, stop, threadToJoin, fsShutdownHook); ShutdownHookManager.affixShutdownHook(t, 0); LOG.debug("Installed shutdown hook thread: " + t.getName()); }
/** * Install a shutdown hook that calls stop on the passed Stoppable * and then thread joins against the passed <code>threadToJoin</code>. * When this thread completes, it then runs the hdfs thread (This install * removes the hdfs shutdown hook keeping a handle on it to run it after * <code>threadToJoin</code> has stopped). * * <p>To suppress all shutdown hook handling -- both the running of the * regionserver hook and of the hdfs hook code -- set * {@link ShutdownHook#RUN_SHUTDOWN_HOOK} in {@link Configuration} to * <code>false</code>. * This configuration value is checked when the hook code runs. * @param conf * @param fs Instance of Filesystem used by the RegionServer * @param stop Installed shutdown hook will call stop against this passed * <code>Stoppable</code> instance. * @param threadToJoin After calling stop on <code>stop</code> will then * join this thread. */ public static void install(final Configuration conf, final FileSystem fs, final Stoppable stop, final Thread threadToJoin) { Runnable fsShutdownHook = suppressHdfsShutdownHook(fs); Thread t = new ShutdownHookThread(conf, stop, threadToJoin, fsShutdownHook); ShutdownHookManager.affixShutdownHook(t, 0); LOG.info("Installed shutdown hook thread: " + t.getName()); }