@Override public StartContainersResponse startContainers( StartContainersRequest requests) throws YarnException { StartContainersResponse response = recordFactory.newRecordInstance(StartContainersResponse.class); for (StartContainerRequest request : requests.getStartContainerRequests()) { Token containerToken = request.getContainerToken(); ContainerTokenIdentifier tokenId = null; try { tokenId = newContainerTokenIdentifier(containerToken); } catch (IOException e) { throw RPCUtil.getRemoteException(e); } ContainerStatus status = recordFactory.newRecordInstance(ContainerStatus.class); status.setState(ContainerState.RUNNING); status.setContainerId(tokenId.getContainerID()); status.setExitStatus(0); statuses.add(status); } return response; }
private void startContainer(final YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken, org.apache.hadoop.yarn.api.records.Token containerToken, NodeId nodeId, String user) throws Exception { ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); StartContainerRequest scRequest = StartContainerRequest.newInstance(context,containerToken); List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); list.add(scRequest); StartContainersRequest allRequests = StartContainersRequest.newInstance(list); ContainerManagementProtocol proxy = null; try { proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user); StartContainersResponse response = proxy.startContainers(allRequests); for(SerializedException ex : response.getFailedRequests().values()){ parseAndThrowException(ex.deSerialize()); } } finally { if (proxy != null) { rpc.stopProxy(proxy, conf); } } }
private StartContainersResponse startContainer(Context context, final ContainerManagerImpl cm, ContainerId cid, ContainerLaunchContext clc, LogAggregationContext logAggregationContext) throws Exception { UserGroupInformation user = UserGroupInformation.createRemoteUser( cid.getApplicationAttemptId().toString()); StartContainerRequest scReq = StartContainerRequest.newInstance( clc, TestContainerManager.createContainerToken(cid, 0, context.getNodeId(), user.getShortUserName(), context.getContainerTokenSecretManager(), logAggregationContext)); final List<StartContainerRequest> scReqList = new ArrayList<StartContainerRequest>(); scReqList.add(scReq); NMTokenIdentifier nmToken = new NMTokenIdentifier( cid.getApplicationAttemptId(), context.getNodeId(), user.getShortUserName(), context.getNMTokenSecretManager().getCurrentKey().getKeyId()); user.addTokenIdentifier(nmToken); return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() { @Override public StartContainersResponse run() throws Exception { return cm.startContainers( StartContainersRequest.newInstance(scReqList)); } }); }
private StartContainersResponse startContainer(Context context, final ContainerManagerImpl cm, ContainerId cid, ContainerLaunchContext clc, LogAggregationContext logAggregationContext) throws Exception { UserGroupInformation user = UserGroupInformation.createRemoteUser( cid.getApplicationAttemptId().toString(), false); StartContainerRequest scReq = StartContainerRequest.newInstance( clc, TestContainerManager.createContainerToken(cid, 0, context.getNodeId(), user.getShortUserName(), context.getContainerTokenSecretManager(), logAggregationContext, user.getShortUserName() + "Folder")); final List<StartContainerRequest> scReqList = new ArrayList<StartContainerRequest>(); scReqList.add(scReq); NMTokenIdentifier nmToken = new NMTokenIdentifier( cid.getApplicationAttemptId(), context.getNodeId(), user.getShortUserName(), context.getNMTokenSecretManager().getCurrentKey().getKeyId()); user.addTokenIdentifier(nmToken); return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() { @Override public StartContainersResponse run() throws Exception { return cm.startContainers( StartContainersRequest.newInstance(scReqList)); } }); }
@Test(timeout = 20000) public void testNMProxyRPCRetry() throws Exception { conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, 1000); conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 100); StartContainersRequest allRequests = Records.newRecord(StartContainersRequest.class); Configuration newConf = new YarnConfiguration(conf); newConf.setInt( CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 100); newConf.setInt(CommonConfigurationKeysPublic. IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 100); // connect to some dummy address so that it can trigger // connection failure and RPC level retires. newConf.set(YarnConfiguration.NM_ADDRESS, "1234"); ContainerManagementProtocol proxy = getNMProxy(newConf); try { proxy.startContainers(allRequests); Assert.fail("should get socket exception"); } catch (IOException e) { // socket exception should be thrown immediately, without RPC retries. Assert.assertTrue(e instanceof java.net.SocketException); } }
private void setupCryptoMaterial(StartContainersRequest request, String user) throws IOException { Path kStorePath = Paths.get(HopsSSLSocketFactory.LOCALIZED_KEYSTORE_FILE_NAME); Path tStorePath = Paths.get(HopsSSLSocketFactory.LOCALIZED_TRUSTSTORE_FILE_NAME); Path passwdPath = Paths.get(HopsSSLSocketFactory.LOCALIZED_PASSWD_FILE_NAME); byte[] keyStoreBin = Files.readAllBytes(kStorePath); ByteBuffer kStore = ByteBuffer.wrap(keyStoreBin); ByteBuffer tStore = ByteBuffer.wrap(Files.readAllBytes(tStorePath)); String password = readCryptoMaterialPassword(passwdPath.toFile()); request.setKeyStore(kStore); request.setKeyStorePassword(password); request.setTrustStore(tStore); request.setTrustStorePassword(password); }
@Override public StartContainersResponse startContainers(StartContainersRequest requests) throws YarnException, IOException { StartContainersRequestProto requestProto = ((StartContainersRequestPBImpl) requests).getProto(); try { return new StartContainersResponsePBImpl(proxy.startContainers(null, requestProto)); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); return null; } }
@Override public StartContainersResponse startContainers( StartContainersRequest requests) throws YarnException, IOException { try { // make the thread sleep to look like its not going to respond Thread.sleep(10000); } catch (Exception e) { LOG.error(e); throw new YarnException(e); } throw new YarnException("Shouldn't happen!!"); }
private void launch() throws IOException, YarnException { connect(); ContainerId masterContainerID = masterContainer.getId(); ApplicationSubmissionContext applicationContext = application.getSubmissionContext(); LOG.info("Setting up container " + masterContainer + " for AM " + application.getAppAttemptId()); ContainerLaunchContext launchContext = createAMContainerLaunchContext(applicationContext, masterContainerID); StartContainerRequest scRequest = StartContainerRequest.newInstance(launchContext, masterContainer.getContainerToken()); List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); list.add(scRequest); StartContainersRequest allRequests = StartContainersRequest.newInstance(list); StartContainersResponse response = containerMgrProxy.startContainers(allRequests); if (response.getFailedRequests() != null && response.getFailedRequests().containsKey(masterContainerID)) { Throwable t = response.getFailedRequests().get(masterContainerID).deSerialize(); parseAndThrowException(t); } else { LOG.info("Done launching container " + masterContainer + " for AM " + application.getAppAttemptId()); } }
@Override public StartContainersResponse startContainers(StartContainersRequest request) throws YarnException { containerTokens = request.getStartContainerRequests().get(0).getContainerLaunchContext().getTokens(); return StartContainersResponse.newInstance(null, null, null); }
@Override public StartContainersResponse startContainers(StartContainersRequest requests) throws YarnException { StartContainerRequest request = requests.getStartContainerRequests().get(0); LOG.info("Container started by MyContainerManager: " + request); launched = true; Map<String, String> env = request.getContainerLaunchContext().getEnvironment(); Token containerToken = request.getContainerToken(); ContainerTokenIdentifier tokenId = null; try { tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken); } catch (IOException e) { throw RPCUtil.getRemoteException(e); } ContainerId containerId = tokenId.getContainerID(); containerIdAtContainerManager = containerId.toString(); attemptIdAtContainerManager = containerId.getApplicationAttemptId().toString(); nmHostAtContainerManager = tokenId.getNmHostAddress(); submitTimeAtContainerManager = Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV)); maxAppAttempts = Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV)); return StartContainersResponse.newInstance( new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(), new HashMap<ContainerId, SerializedException>()); }
@Override public StartContainersResponse startContainers(StartContainersRequest request) throws YarnException, IOException { ContainersResponse startContainersResp = containerManager .startContainers(request.getStartContainerRequests()); return StartContainersResponse.newInstance( startContainersResp.getServicesMetaData(), startContainersResp.getSucceededContainers(), startContainersResp.getFailedContainers()); }
@Override public StartContainersResponse startContainers( StartContainersRequest requests) throws YarnException, IOException { Exception e = new Exception("Dummy function", new Exception( "Dummy function cause")); throw new YarnException(e); }
@Test(timeout = 20000) public void testNMProxyRetry() throws Exception { conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, 10000); conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 100); StartContainersRequest allRequests = Records.newRecord(StartContainersRequest.class); ContainerManagementProtocol proxy = getNMProxy(); retryCount = 0; shouldThrowNMNotYetReadyException = false; proxy.startContainers(allRequests); Assert.assertEquals(5, retryCount); retryCount = 0; shouldThrowNMNotYetReadyException = false; proxy.stopContainers(Records.newRecord(StopContainersRequest.class)); Assert.assertEquals(5, retryCount); retryCount = 0; shouldThrowNMNotYetReadyException = false; proxy.getContainerStatuses(Records .newRecord(GetContainerStatusesRequest.class)); Assert.assertEquals(5, retryCount); retryCount = 0; shouldThrowNMNotYetReadyException = true; proxy.startContainers(allRequests); Assert.assertEquals(5, retryCount); }
@Test(timeout = 20000, expected = IOException.class) public void testShouldNotRetryForeverForNonNetworkExceptionsOnNMConnections() throws Exception { conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, -1); StartContainersRequest allRequests = Records.newRecord(StartContainersRequest.class); ContainerManagementProtocol proxy = getNMProxy(); shouldThrowNMNotYetReadyException = false; retryCount = 0; proxy.startContainers(allRequests); }
@Override public StartContainersResponse startContainers(StartContainersRequest requests) throws YarnException { StartContainerRequest request = requests.getStartContainerRequests().get(0); LOG.info("Container started by MyContainerManager: " + request); launched = true; Map<String, String> env = request.getContainerLaunchContext().getEnvironment(); Token containerToken = request.getContainerToken(); ContainerTokenIdentifier tokenId = null; try { tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken); } catch (IOException e) { throw RPCUtil.getRemoteException(e); } ContainerId containerId = tokenId.getContainerID(); containerIdAtContainerManager = containerId.toString(); attemptIdAtContainerManager = containerId.getApplicationAttemptId().toString(); nmHostAtContainerManager = tokenId.getNmHostAddress(); submitTimeAtContainerManager = Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV)); maxAppAttempts = YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS; return StartContainersResponse.newInstance( new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(), new HashMap<ContainerId, SerializedException>()); }
