@Override public StartContainersResponse startContainers( StartContainersRequest requests) throws YarnException { StartContainersResponse response = recordFactory.newRecordInstance(StartContainersResponse.class); for (StartContainerRequest request : requests.getStartContainerRequests()) { Token containerToken = request.getContainerToken(); ContainerTokenIdentifier tokenId = null; try { tokenId = newContainerTokenIdentifier(containerToken); } catch (IOException e) { throw RPCUtil.getRemoteException(e); } ContainerStatus status = recordFactory.newRecordInstance(ContainerStatus.class); status.setState(ContainerState.RUNNING); status.setContainerId(tokenId.getContainerID()); status.setExitStatus(0); statuses.add(status); } return response; }
private void startContainer(final YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken, org.apache.hadoop.yarn.api.records.Token containerToken, NodeId nodeId, String user) throws Exception { ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); StartContainerRequest scRequest = StartContainerRequest.newInstance(context,containerToken); List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); list.add(scRequest); StartContainersRequest allRequests = StartContainersRequest.newInstance(list); ContainerManagementProtocol proxy = null; try { proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user); StartContainersResponse response = proxy.startContainers(allRequests); for(SerializedException ex : response.getFailedRequests().values()){ parseAndThrowException(ex.deSerialize()); } } finally { if (proxy != null) { rpc.stopProxy(proxy, conf); } } }
@Override public void storeContainer(ContainerId containerId, StartContainerRequest startRequest) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("storeContainer: containerId= " + containerId + ", startRequest= " + startRequest); } String key = CONTAINERS_KEY_PREFIX + containerId.toString() + CONTAINER_REQUEST_KEY_SUFFIX; try { db.put(bytes(key), ((StartContainerRequestPBImpl) startRequest).getProto().toByteArray()); } catch (DBException e) { throw new IOException(e); } }
private StartContainersResponse startContainer(Context context, final ContainerManagerImpl cm, ContainerId cid, ContainerLaunchContext clc, LogAggregationContext logAggregationContext) throws Exception { UserGroupInformation user = UserGroupInformation.createRemoteUser( cid.getApplicationAttemptId().toString()); StartContainerRequest scReq = StartContainerRequest.newInstance( clc, TestContainerManager.createContainerToken(cid, 0, context.getNodeId(), user.getShortUserName(), context.getContainerTokenSecretManager(), logAggregationContext)); final List<StartContainerRequest> scReqList = new ArrayList<StartContainerRequest>(); scReqList.add(scReq); NMTokenIdentifier nmToken = new NMTokenIdentifier( cid.getApplicationAttemptId(), context.getNodeId(), user.getShortUserName(), context.getNMTokenSecretManager().getCurrentKey().getKeyId()); user.addTokenIdentifier(nmToken); return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() { @Override public StartContainersResponse run() throws Exception { return cm.startContainers( StartContainersRequest.newInstance(scReqList)); } }); }
@Override public Cancellable start(YarnContainerInfo containerInfo, YarnLaunchContext launchContext) { ContainerLaunchContext context = launchContext.getLaunchContext(); context.setUser(System.getProperty("user.name")); Container container = containerInfo.getContainer(); context.setContainerId(container.getId()); context.setResource(container.getResource()); StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class); startRequest.setContainerLaunchContext(context); ContainerManager manager = connectContainerManager(container); try { manager.startContainer(startRequest); return new ContainerTerminator(container, manager); } catch (YarnRemoteException e) { LOG.error("Error in launching process", e); throw Throwables.propagate(e); } }
private StartContainersResponse startContainer(Context context, final ContainerManagerImpl cm, ContainerId cid, ContainerLaunchContext clc, LogAggregationContext logAggregationContext) throws Exception { UserGroupInformation user = UserGroupInformation.createRemoteUser( cid.getApplicationAttemptId().toString(), false); StartContainerRequest scReq = StartContainerRequest.newInstance( clc, TestContainerManager.createContainerToken(cid, 0, context.getNodeId(), user.getShortUserName(), context.getContainerTokenSecretManager(), logAggregationContext, user.getShortUserName() + "Folder")); final List<StartContainerRequest> scReqList = new ArrayList<StartContainerRequest>(); scReqList.add(scReq); NMTokenIdentifier nmToken = new NMTokenIdentifier( cid.getApplicationAttemptId(), context.getNodeId(), user.getShortUserName(), context.getNMTokenSecretManager().getCurrentKey().getKeyId()); user.addTokenIdentifier(nmToken); return user.doAs(new PrivilegedExceptionAction<StartContainersResponse>() { @Override public StartContainersResponse run() throws Exception { return cm.startContainers( StartContainersRequest.newInstance(scReqList)); } }); }
/** * Connects to CM, sets up container launch context * for shell command and eventually dispatches the container * start request to the CM. */ public void run() { // Connect to ContainerManager connectToCM(); // configure the launcher for the Giraph task it will host StartContainerRequest startReq = Records.newRecord(StartContainerRequest.class); startReq.setContainerLaunchContext(buildContainerLaunchContext()); // request CM to start this container as spec'd in ContainerLaunchContext try { containerManager.startContainer(startReq); } catch (YarnRemoteException yre) { LOG.error("StartContainerRequest failed for containerId=" + container.getId(), yre); } }
private void addLocalRequestsToProto() { maybeInitBuilder(); builder.clearStartContainerRequest(); List<StartContainerRequestProto> protoList = new ArrayList<StartContainerRequestProto>(); for (StartContainerRequest r : this.requests) { protoList.add(convertToProtoFormat(r)); } builder.addAllStartContainerRequest(protoList); }
private void initLocalRequests() { StartContainersRequestProtoOrBuilder p = viaProto ? proto : builder; List<StartContainerRequestProto> requestList = p.getStartContainerRequestList(); this.requests = new ArrayList<StartContainerRequest>(); for (StartContainerRequestProto r : requestList) { this.requests.add(convertFromProtoFormat(r)); } }
@Override public void setStartContainerRequests(List<StartContainerRequest> requests) { maybeInitBuilder(); if (requests == null) { builder.clearStartContainerRequest(); } this.requests = requests; }
@Override public List<StartContainerRequest> getStartContainerRequests() { if (this.requests != null) { return this.requests; } initLocalRequests(); return this.requests; }
@SuppressWarnings("unchecked") private void recoverContainer(RecoveredContainerState rcs) throws IOException { StartContainerRequest req = rcs.getStartRequest(); ContainerLaunchContext launchContext = req.getContainerLaunchContext(); ContainerTokenIdentifier token = BuilderUtils.newContainerTokenIdentifier(req.getContainerToken()); ContainerId containerId = token.getContainerID(); ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId(); LOG.info("Recovering " + containerId + " in state " + rcs.getStatus() + " with exit code " + rcs.getExitCode()); if (context.getApplications().containsKey(appId)) { Credentials credentials = parseCredentials(launchContext); Container container = new ContainerImpl(getConfig(), dispatcher, context.getNMStateStore(), req.getContainerLaunchContext(), credentials, metrics, token, rcs.getStatus(), rcs.getExitCode(), rcs.getDiagnostics(), rcs.getKilled()); context.getContainers().put(containerId, container); dispatcher.getEventHandler().handle( new ApplicationContainerInitEvent(container)); } else { if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED) { LOG.warn(containerId + " has no corresponding application!"); } LOG.info("Adding " + containerId + " to recently stopped containers"); nodeStatusUpdater.addCompletedContainer(containerId); } }
@Override public synchronized void storeContainer(ContainerId containerId, StartContainerRequest startRequest) throws IOException { RecoveredContainerState rcs = new RecoveredContainerState(); rcs.startRequest = startRequest; containerStates.put(containerId, rcs); }
private void launch() throws IOException, YarnException { connect(); ContainerId masterContainerID = masterContainer.getId(); ApplicationSubmissionContext applicationContext = application.getSubmissionContext(); LOG.info("Setting up container " + masterContainer + " for AM " + application.getAppAttemptId()); ContainerLaunchContext launchContext = createAMContainerLaunchContext(applicationContext, masterContainerID); StartContainerRequest scRequest = StartContainerRequest.newInstance(launchContext, masterContainer.getContainerToken()); List<StartContainerRequest> list = new ArrayList<StartContainerRequest>(); list.add(scRequest); StartContainersRequest allRequests = StartContainersRequest.newInstance(list); StartContainersResponse response = containerMgrProxy.startContainers(allRequests); if (response.getFailedRequests() != null && response.getFailedRequests().containsKey(masterContainerID)) { Throwable t = response.getFailedRequests().get(masterContainerID).deSerialize(); parseAndThrowException(t); } else { LOG.info("Done launching container " + masterContainer + " for AM " + application.getAppAttemptId()); } }
@Override public StartContainersResponse startContainers(StartContainersRequest requests) throws YarnException { StartContainerRequest request = requests.getStartContainerRequests().get(0); LOG.info("Container started by MyContainerManager: " + request); launched = true; Map<String, String> env = request.getContainerLaunchContext().getEnvironment(); Token containerToken = request.getContainerToken(); ContainerTokenIdentifier tokenId = null; try { tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken); } catch (IOException e) { throw RPCUtil.getRemoteException(e); } ContainerId containerId = tokenId.getContainerID(); containerIdAtContainerManager = containerId.toString(); attemptIdAtContainerManager = containerId.getApplicationAttemptId().toString(); nmHostAtContainerManager = tokenId.getNmHostAddress(); submitTimeAtContainerManager = Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV)); maxAppAttempts = Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV)); return StartContainersResponse.newInstance( new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(), new HashMap<ContainerId, SerializedException>()); }
