public DomainSocketFactory(Conf conf) { final String feature; if (conf.isShortCircuitLocalReads() && (!conf.isUseLegacyBlockReaderLocal())) { feature = "The short-circuit local reads feature"; } else if (conf.isDomainSocketDataTraffic()) { feature = "UNIX domain socket data traffic"; } else { feature = null; } if (feature == null) { PerformanceAdvisory.LOG.debug( "Both short-circuit local reads and UNIX domain socket are disabled."); } else { if (conf.getDomainSocketPath().isEmpty()) { throw new HadoopIllegalArgumentException(feature + " is enabled but " + DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY + " is not set."); } else if (DomainSocket.getLoadingFailureReason() != null) { LOG.warn(feature + " cannot be used because " + DomainSocket.getLoadingFailureReason()); } else { LOG.debug(feature + " is enabled."); } } }
public DomainSocketFactory(ShortCircuitConf conf) { final String feature; if (conf.isShortCircuitLocalReads() && (!conf.isUseLegacyBlockReaderLocal())) { feature = "The short-circuit local reads feature"; } else if (conf.isDomainSocketDataTraffic()) { feature = "UNIX domain socket data traffic"; } else { feature = null; } if (feature == null) { PerformanceAdvisory.LOG.debug( "Both short-circuit local reads and UNIX domain socket are disabled."); } else { if (conf.getDomainSocketPath().isEmpty()) { throw new HadoopIllegalArgumentException(feature + " is enabled but " + HdfsClientConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY + " is not set."); } else if (DomainSocket.getLoadingFailureReason() != null) { LOG.warn(feature + " cannot be used because " + DomainSocket.getLoadingFailureReason()); } else { LOG.debug(feature + " is enabled."); } } }
public JniBasedUnixGroupsMappingWithFallback() { if (NativeCodeLoader.isNativeCodeLoaded()) { this.impl = new JniBasedUnixGroupsMapping(); } else { PerformanceAdvisory.LOG.debug("Falling back to shell based"); this.impl = new ShellBasedUnixGroupsMapping(); } if (LOG.isDebugEnabled()){ LOG.debug("Group mapping impl=" + impl.getClass().getName()); } }
/** * Get crypto codec for specified algorithm/mode/padding. * * @param conf * the configuration * @param cipherSuite * algorithm/mode/padding * @return CryptoCodec the codec object. Null value will be returned if no * crypto codec classes with cipher suite configured. */ public static CryptoCodec getInstance(Configuration conf, CipherSuite cipherSuite) { List<Class<? extends CryptoCodec>> klasses = getCodecClasses( conf, cipherSuite); if (klasses == null) { return null; } CryptoCodec codec = null; for (Class<? extends CryptoCodec> klass : klasses) { try { CryptoCodec c = ReflectionUtils.newInstance(klass, conf); if (c.getCipherSuite().getName().equals(cipherSuite.getName())) { if (codec == null) { PerformanceAdvisory.LOG.debug("Using crypto codec {}.", klass.getName()); codec = c; } } else { PerformanceAdvisory.LOG.debug( "Crypto codec {} doesn't meet the cipher suite {}.", klass.getName(), cipherSuite.getName()); } } catch (Exception e) { PerformanceAdvisory.LOG.debug("Crypto codec {} is not available.", klass.getName()); } } return codec; }
public OpensslSecureRandom() { if (!nativeEnabled) { PerformanceAdvisory.LOG.debug("Build does not support openssl, " + "falling back to Java SecureRandom."); fallback = new java.security.SecureRandom(); } }
/** * Get {@link BlockReaderLocalLegacy} for short circuited local reads. * This block reader implements the path-based style of local reads * first introduced in HDFS-2246. */ private BlockReader getLegacyBlockReaderLocal() throws IOException { LOG.trace("{}: trying to construct BlockReaderLocalLegacy", this); if (!DFSUtilClient.isLocalAddress(inetSocketAddress)) { LOG.trace("{}: can't construct BlockReaderLocalLegacy because the address" + "{} is not local", this, inetSocketAddress); return null; } if (clientContext.getDisableLegacyBlockReaderLocal()) { PerformanceAdvisory.LOG.debug("{}: can't construct " + "BlockReaderLocalLegacy because " + "disableLegacyBlockReaderLocal is set.", this); return null; } IOException ioe; try { return BlockReaderLocalLegacy.newBlockReader(conf, userGroupInformation, configuration, fileName, block, token, datanode, startOffset, length, storageType, tracer); } catch (RemoteException remoteException) { ioe = remoteException.unwrapRemoteException( InvalidToken.class, AccessControlException.class); } catch (IOException e) { ioe = e; } if ((!(ioe instanceof AccessControlException)) && isSecurityException(ioe)) { // Handle security exceptions. // We do not handle AccessControlException here, since // BlockReaderLocalLegacy#newBlockReader uses that exception to indicate // that the user is not in dfs.block.local-path-access.user, a condition // which requires us to disable legacy SCR. throw ioe; } LOG.warn(this + ": error creating legacy BlockReaderLocal. " + "Disabling legacy local reads.", ioe); clientContext.setDisableLegacyBlockReaderLocal(); return null; }
private BlockReader getBlockReaderLocal() throws InvalidToken { LOG.trace("{}: trying to construct a BlockReaderLocal for short-circuit " + " reads.", this); if (pathInfo == null) { pathInfo = clientContext.getDomainSocketFactory() .getPathInfo(inetSocketAddress, conf.getShortCircuitConf()); } if (!pathInfo.getPathState().getUsableForShortCircuit()) { PerformanceAdvisory.LOG.debug("{}: {} is not usable for short circuit; " + "giving up on BlockReaderLocal.", this, pathInfo); return null; } ShortCircuitCache cache = clientContext.getShortCircuitCache(); ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId()); ShortCircuitReplicaInfo info = cache.fetchOrCreate(key, this); InvalidToken exc = info.getInvalidTokenException(); if (exc != null) { LOG.trace("{}: got InvalidToken exception while trying to construct " + "BlockReaderLocal via {}", this, pathInfo.getPath()); throw exc; } if (info.getReplica() == null) { PerformanceAdvisory.LOG.debug("{}: failed to get " + "ShortCircuitReplica. Cannot construct " + "BlockReaderLocal via {}", this, pathInfo.getPath()); return null; } return new BlockReaderLocal.Builder(conf.getShortCircuitConf()). setFilename(fileName). setBlock(block). setStartOffset(startOffset). setShortCircuitReplica(info.getReplica()). setVerifyChecksum(verifyChecksum). setCachingStrategy(cachingStrategy). setStorageType(storageType). setTracer(tracer). build(); }
/** * Get {@link BlockReaderLocalLegacy} for short circuited local reads. * This block reader implements the path-based style of local reads * first introduced in HDFS-2246. */ private BlockReader getLegacyBlockReaderLocal() throws IOException { if (LOG.isTraceEnabled()) { LOG.trace(this + ": trying to construct BlockReaderLocalLegacy"); } if (!DFSClient.isLocalAddress(inetSocketAddress)) { if (LOG.isTraceEnabled()) { LOG.trace(this + ": can't construct BlockReaderLocalLegacy because " + "the address " + inetSocketAddress + " is not local"); } return null; } if (clientContext.getDisableLegacyBlockReaderLocal()) { PerformanceAdvisory.LOG.debug(this + ": can't construct " + "BlockReaderLocalLegacy because " + "disableLegacyBlockReaderLocal is set."); return null; } IOException ioe = null; try { return BlockReaderLocalLegacy.newBlockReader(conf, userGroupInformation, configuration, fileName, block, token, datanode, startOffset, length, storageType); } catch (RemoteException remoteException) { ioe = remoteException.unwrapRemoteException( InvalidToken.class, AccessControlException.class); } catch (IOException e) { ioe = e; } if ((!(ioe instanceof AccessControlException)) && isSecurityException(ioe)) { // Handle security exceptions. // We do not handle AccessControlException here, since // BlockReaderLocalLegacy#newBlockReader uses that exception to indicate // that the user is not in dfs.block.local-path-access.user, a condition // which requires us to disable legacy SCR. throw ioe; } LOG.warn(this + ": error creating legacy BlockReaderLocal. " + "Disabling legacy local reads.", ioe); clientContext.setDisableLegacyBlockReaderLocal(); return null; }
private BlockReader getBlockReaderLocal() throws InvalidToken { if (LOG.isTraceEnabled()) { LOG.trace(this + ": trying to construct a BlockReaderLocal " + "for short-circuit reads."); } if (pathInfo == null) { pathInfo = clientContext.getDomainSocketFactory(). getPathInfo(inetSocketAddress, conf); } if (!pathInfo.getPathState().getUsableForShortCircuit()) { PerformanceAdvisory.LOG.debug(this + ": " + pathInfo + " is not " + "usable for short circuit; giving up on BlockReaderLocal."); return null; } ShortCircuitCache cache = clientContext.getShortCircuitCache(); ExtendedBlockId key = new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId()); ShortCircuitReplicaInfo info = cache.fetchOrCreate(key, this); InvalidToken exc = info.getInvalidTokenException(); if (exc != null) { if (LOG.isTraceEnabled()) { LOG.trace(this + ": got InvalidToken exception while trying to " + "construct BlockReaderLocal via " + pathInfo.getPath()); } throw exc; } if (info.getReplica() == null) { if (LOG.isTraceEnabled()) { PerformanceAdvisory.LOG.debug(this + ": failed to get " + "ShortCircuitReplica. Cannot construct " + "BlockReaderLocal via " + pathInfo.getPath()); } return null; } return new BlockReaderLocal.Builder(conf). setFilename(fileName). setBlock(block). setStartOffset(startOffset). setShortCircuitReplica(info.getReplica()). setVerifyChecksum(verifyChecksum). setCachingStrategy(cachingStrategy). setStorageType(storageType). build(); }
/** * Get a RemoteBlockReader that communicates over a UNIX domain socket. * * @return The new BlockReader, or null if we failed to create the block * reader. * * @throws InvalidToken If the block token was invalid. * Potentially other security-related execptions. */ private BlockReader getRemoteBlockReaderFromDomain() throws IOException { if (pathInfo == null) { pathInfo = clientContext.getDomainSocketFactory(). getPathInfo(inetSocketAddress, conf); } if (!pathInfo.getPathState().getUsableForDataTransfer()) { PerformanceAdvisory.LOG.debug(this + ": not trying to create a " + "remote block reader because the UNIX domain socket at " + pathInfo + " is not usable."); return null; } if (LOG.isTraceEnabled()) { LOG.trace(this + ": trying to create a remote block reader from the " + "UNIX domain socket at " + pathInfo.getPath()); } while (true) { BlockReaderPeer curPeer = nextDomainPeer(); if (curPeer == null) break; if (curPeer.fromCache) remainingCacheTries--; DomainPeer peer = (DomainPeer)curPeer.peer; BlockReader blockReader = null; try { blockReader = getRemoteBlockReader(peer); return blockReader; } catch (IOException ioe) { IOUtils.cleanup(LOG, peer); if (isSecurityException(ioe)) { if (LOG.isTraceEnabled()) { LOG.trace(this + ": got security exception while constructing " + "a remote block reader from the unix domain socket at " + pathInfo.getPath(), ioe); } throw ioe; } if (curPeer.fromCache) { // Handle an I/O error we got when using a cached peer. These are // considered less serious, because the underlying socket may be stale. if (LOG.isDebugEnabled()) { LOG.debug("Closed potentially stale domain peer " + peer, ioe); } } else { // Handle an I/O error we got when using a newly created domain peer. // We temporarily disable the domain socket path for a few minutes in // this case, to prevent wasting more time on it. LOG.warn("I/O error constructing remote block reader. Disabling " + "domain socket " + peer.getDomainSocket(), ioe); clientContext.getDomainSocketFactory() .disableDomainSocketPath(pathInfo.getPath()); return null; } } finally { if (blockReader == null) { IOUtils.cleanup(LOG, peer); } } } return null; }
/** * Get a RemoteBlockReader that communicates over a UNIX domain socket. * * @return The new BlockReader, or null if we failed to create the block * reader. * * @throws InvalidToken If the block token was invalid. * Potentially other security-related execptions. */ private BlockReader getRemoteBlockReaderFromDomain() throws IOException { if (pathInfo == null) { pathInfo = clientContext.getDomainSocketFactory() .getPathInfo(inetSocketAddress, conf.getShortCircuitConf()); } if (!pathInfo.getPathState().getUsableForDataTransfer()) { PerformanceAdvisory.LOG.debug("{}: not trying to create a " + "remote block reader because the UNIX domain socket at {}" + " is not usable.", this, pathInfo); return null; } LOG.trace("{}: trying to create a remote block reader from the UNIX domain " + "socket at {}", this, pathInfo.getPath()); while (true) { BlockReaderPeer curPeer = nextDomainPeer(); if (curPeer == null) break; if (curPeer.fromCache) remainingCacheTries--; DomainPeer peer = (DomainPeer)curPeer.peer; BlockReader blockReader = null; try { blockReader = getRemoteBlockReader(peer); return blockReader; } catch (IOException ioe) { IOUtilsClient.cleanup(LOG, peer); if (isSecurityException(ioe)) { LOG.trace("{}: got security exception while constructing a remote " + " block reader from the unix domain socket at {}", this, pathInfo.getPath(), ioe); throw ioe; } if (curPeer.fromCache) { // Handle an I/O error we got when using a cached peer. These are // considered less serious because the underlying socket may be stale. LOG.debug("Closed potentially stale domain peer {}", peer, ioe); } else { // Handle an I/O error we got when using a newly created domain peer. // We temporarily disable the domain socket path for a few minutes in // this case, to prevent wasting more time on it. LOG.warn("I/O error constructing remote block reader. Disabling " + "domain socket " + peer.getDomainSocket(), ioe); clientContext.getDomainSocketFactory() .disableDomainSocketPath(pathInfo.getPath()); return null; } } finally { if (blockReader == null) { IOUtilsClient.cleanup(LOG, peer); } } } return null; }
