@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { Assert.assertEquals("response ID mismatch", responseId, request.getResponseId()); ++responseId; org.apache.hadoop.yarn.api.records.Token yarnToken = null; if (amToken != null) { yarnToken = org.apache.hadoop.yarn.api.records.Token.newInstance( amToken.getIdentifier(), amToken.getKind().toString(), amToken.getPassword(), amToken.getService().toString()); } return AllocateResponse.newInstance(responseId, Collections.<ContainerStatus>emptyList(), Collections.<Container>emptyList(), Collections.<NodeReport>emptyList(), Resources.none(), null, 1, null, Collections.<NMToken>emptyList(), yarnToken, Collections.<ContainerResourceIncrease>emptyList(), Collections.<ContainerResourceDecrease>emptyList()); }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { lastAsk = request.getAskList(); for (ResourceRequest req : lastAsk) { if (ResourceRequest.ANY.equals(req.getResourceName())) { Priority priority = req.getPriority(); if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) { lastAnyAskMap = req.getNumContainers(); } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){ lastAnyAskReduce = req.getNumContainers(); } } } AllocateResponse response = AllocateResponse.newInstance( request.getResponseId(), containersToComplete, containersToAllocate, Collections.<NodeReport>emptyList(), Resource.newInstance(512000, 1024, 1024), null, 10, null, Collections.<NMToken>emptyList()); containersToComplete.clear(); containersToAllocate.clear(); return response; }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { HPCAllocateRequest allocateRequest = new HPCAllocateRequest(); allocateRequest.setAppProgress(request.getProgress()); allocateRequest.setContainersToBeReleased(request.getReleaseList()); allocateRequest.setResourceAsk(request.getAskList()); allocateRequest.setResourceBlacklistRequest(request .getResourceBlacklistRequest()); allocateRequest.setResponseID(request.getResponseId()); HPCAllocateResponse hpcAllocateResponse = applicationMaster .allocate(allocateRequest); AllocateResponse response = Records.newRecord(AllocateResponse.class); response.setAllocatedContainers(hpcAllocateResponse .getAllocatedContainers()); response.setNMTokens(hpcAllocateResponse.getNmTokens()); response.setCompletedContainersStatuses(hpcAllocateResponse.getCompletedContainers()); return response; }
protected AllocateResponse allocate(final int testAppId, final AllocateRequest request) throws Exception, YarnException, IOException { final ApplicationUserInfo ugi = getApplicationUserInfo(testAppId); return ugi.getUser().doAs( new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { AllocateResponse response = getAMRMProxyService().allocate(request); return response; } }); }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { Assert.assertEquals("response ID mismatch", responseId, request.getResponseId()); ++responseId; org.apache.hadoop.yarn.api.records.Token yarnToken = null; if (amToken != null) { yarnToken = org.apache.hadoop.yarn.api.records.Token.newInstance( amToken.getIdentifier(), amToken.getKind().toString(), amToken.getPassword(), amToken.getService().toString()); } AllocateResponse response = AllocateResponse.newInstance(responseId, Collections.<ContainerStatus>emptyList(), Collections.<Container>emptyList(), Collections.<NodeReport>emptyList(), Resources.none(), null, 1, null, Collections.<NMToken>emptyList(), yarnToken, Collections.<Container>emptyList(), Collections.<Container>emptyList()); response.setApplicationPriority(Priority.newInstance(0)); return response; }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { lastAsk = request.getAskList(); for (ResourceRequest req : lastAsk) { if (ResourceRequest.ANY.equals(req.getResourceName())) { Priority priority = req.getPriority(); if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) { lastAnyAskMap = req.getNumContainers(); } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){ lastAnyAskReduce = req.getNumContainers(); } } } AllocateResponse response = AllocateResponse.newInstance( request.getResponseId(), containersToComplete, containersToAllocate, Collections.<NodeReport>emptyList(), Resource.newInstance(512000, 1024), null, 10, null, Collections.<NMToken>emptyList()); // RM will always ensure that a default priority is sent to AM response.setApplicationPriority(Priority.newInstance(0)); containersToComplete.clear(); containersToAllocate.clear(); return response; }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { lastAsk = request.getAskList(); for (ResourceRequest req : lastAsk) { if (ResourceRequest.ANY.equals(req.getResourceName())) { Priority priority = req.getPriority(); if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) { lastAnyAskMap = req.getNumContainers(); } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){ lastAnyAskReduce = req.getNumContainers(); } } } AllocateResponse response = AllocateResponse.newInstance( request.getResponseId(), containersToComplete, containersToAllocate, Collections.<NodeReport>emptyList(), Resource.newInstance(512000, 1024), null, 10, null, Collections.<NMToken>emptyList()); containersToComplete.clear(); containersToAllocate.clear(); return response; }
public AllocateResponse allocate( List<ResourceRequest> resourceRequest, List<ContainerId> releases) throws Exception { final AllocateRequest req = AllocateRequest.newInstance(++responseId, 0F, resourceRequest, releases, null); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); Token<AMRMTokenIdentifier> token = context.getRMApps().get(attemptId.getApplicationId()) .getRMAppAttempt(attemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); try { return ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return amRMProtocol.allocate(req); } }); } catch (UndeclaredThrowableException e) { throw (Exception) e.getCause(); } }
public AllocateResponse allocate(AllocateRequest allocateRequest) throws Exception { final AllocateRequest req = allocateRequest; req.setResponseId(++responseId); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); Token<AMRMTokenIdentifier> token = context.getRMApps().get(attemptId.getApplicationId()) .getRMAppAttempt(attemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); try { return ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return amRMProtocol.allocate(req); } }); } catch (UndeclaredThrowableException e) { throw (Exception) e.getCause(); } }
@Test(timeout = 15000) public void testAllocateOnHA() throws YarnException, IOException, InterruptedException { AllocateRequest request = AllocateRequest.newInstance(0, 50f, new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>(), ResourceBlacklistRequest.newInstance(new ArrayList<String>(), new ArrayList<String>())); int nbTry = 0; AllocateResponse response = null; while (nbTry < 10) { try { response = amClient.allocate(request); break; } catch (IOException ex) { if (!(ex instanceof SecretManager.InvalidToken)) { throw ex; } } Thread.sleep(200); nbTry++; } Assert.assertEquals(response, this.cluster.createFakeAllocateResponse()); }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { Assert.assertEquals("response ID mismatch", responseId, request.getResponseId()); ++responseId; org.apache.hadoop.yarn.api.records.Token yarnToken = null; if (amToken != null) { yarnToken = org.apache.hadoop.yarn.api.records.Token.newInstance( amToken.getIdentifier(), amToken.getKind().toString(), amToken.getPassword(), amToken.getService().toString()); } AllocateResponse response = AllocateResponse.newInstance(responseId, Collections.<ContainerStatus>emptyList(), Collections.<Container>emptyList(), Collections.<NodeReport>emptyList(), Resources.none(), null, 1, null, Collections.<NMToken>emptyList(), yarnToken, Collections.<UpdatedContainer>emptyList()); response.setApplicationPriority(Priority.newInstance(0)); return response; }
private void setupHeartbeats() { long heartbeatInterval = config.getLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RM_AM_EXPIRY_INTERVAL_MS); System.err.println("Heartbeat interval: " + heartbeatInterval); heartbeatInterval = Math.min(heartbeatInterval, 1000); System.err.println("Heartbeat interval: " + heartbeatInterval); timer.schedule(new TimerTask() { @Override public void run() { AllocateRequest hb = Records.newRecord(AllocateRequest.class); populateAllocateRequest(hb); hb.setApplicationAttemptId(amrmc.getApplicationAttemptId()); hb.setProgress(0); try { AllocateResponse allocateResponse = amrmc.getAMRMProtocol().allocate(hb); List<Container> allocatedContainers = allocateResponse.getAMResponse().getAllocatedContainers(); List<ContainerStatus> completedContainers = allocateResponse.getAMResponse() .getCompletedContainersStatuses(); processAllocation(allocatedContainers, completedContainers); } catch (YarnRemoteException e) { e.printStackTrace(); } } }, 0, heartbeatInterval); }
/** * Compose and send the allocation request for our Giraph BSP worker/master * compute nodes. Right now the requested containers are identical, mirroring * Giraph's behavior when running on Hadoop MRv1. Giraph could use YARN * to set fine-grained capability to each container, including host choice. * @return The AM resource descriptor with our container allocations. */ private AMResponse sendAllocationRequest() { AllocateRequest allocRequest = Records.newRecord(AllocateRequest.class); try { List<ResourceRequest> containerList = buildResourceRequests(); allocRequest.addAllAsks(containerList); List<ContainerId> releasedContainers = Lists.newArrayListWithCapacity(0); allocRequest.setResponseId(lastResponseId.get()); allocRequest.setApplicationAttemptId(appAttemptId); allocRequest.addAllReleases(releasedContainers); allocRequest.setProgress(progress); AllocateResponse allocResponse = resourceManager.allocate(allocRequest); AMResponse amResponse = allocResponse.getAMResponse(); if (amResponse.getResponseId() != lastResponseId.get()) { lastResponseId.set(amResponse.getResponseId()); } checkForRebootFlag(amResponse); // now, make THIS our new HEARTBEAT object, but with ZERO new requests! initHeartbeatRequestObject(allocRequest); return amResponse; } catch (YarnRemoteException yre) { throw new IllegalStateException("Giraph Application Master could not " + "successfully allocate the specified containers from the RM.", yre); } }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { AllocateRequestProto requestProto = ((AllocateRequestPBImpl) request).getProto(); try { return new AllocateResponsePBImpl(proxy.allocate(null, requestProto)); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); return null; } }
private AllocateResponse allocate(ApplicationAttemptId attemptId, final AllocateRequest req) throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(attemptId.getApplicationId()) .getRMAppAttempt(attemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); return ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return amService.allocate(req); } }); }
private AllocateResponse allocate(final ApplicationAttemptId attemptId, final AllocateRequest req) throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps().get(attemptId.getApplicationId()) .getRMAppAttempt(attemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); return ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return amService.allocate(req); } }); }
public AllocateResponse allocate( List<ResourceRequest> resourceRequest, List<ContainerId> releases) throws Exception { final AllocateRequest req = AllocateRequest.newInstance(0, 0F, resourceRequest, releases, null); return allocate(req); }
public AllocateResponse allocate(AllocateRequest allocateRequest) throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser(attemptId.toString()); Token<AMRMTokenIdentifier> token = context.getRMApps().get(attemptId.getApplicationId()) .getRMAppAttempt(attemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); lastResponse = doAllocateAs(ugi, allocateRequest); return lastResponse; }
public AllocateResponse doAllocateAs(UserGroupInformation ugi, final AllocateRequest req) throws Exception { req.setResponseId(++responseId); try { return ugi.doAs(new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return amRMProtocol.allocate(req); } }); } catch (UndeclaredThrowableException e) { throw (Exception) e.getCause(); } }
@Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { resetStartFailoverFlag(true); // make sure failover has been triggered Assert.assertTrue(waittingForFailOver()); return createFakeAllocateResponse(); }
@Test(timeout = 15000) public void testAllocateOnHA() throws YarnException, IOException { AllocateRequest request = AllocateRequest.newInstance(0, 50f, new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>(), ResourceBlacklistRequest.newInstance(new ArrayList<String>(), new ArrayList<String>())); AllocateResponse response = amClient.allocate(request); Assert.assertEquals(response, this.cluster.createFakeAllocateResponse()); }
protected AllocateRequest createAllocateRequest(List<ResourceRequest> ask, List<ContainerId> toRelease) { AllocateRequest allocateRequest = recordFactory.newRecordInstance(AllocateRequest.class); allocateRequest.setResponseId(RESPONSE_ID ++); allocateRequest.setAskList(ask); allocateRequest.setReleaseList(toRelease); return allocateRequest; }
/** * send out request for AM container */ protected void requestAMContainer() throws YarnException, IOException, InterruptedException { List<ResourceRequest> ask = new ArrayList<ResourceRequest>(); ResourceRequest amRequest = createResourceRequest( BuilderUtils.newResource(MR_AM_CONTAINER_RESOURCE_MEMORY_MB, MR_AM_CONTAINER_RESOURCE_VCORES), ResourceRequest.ANY, 1, 1); ask.add(amRequest); LOG.debug(MessageFormat.format("Application {0} sends out allocate " + "request for its AM", appId)); final AllocateRequest request = this.createAllocateRequest(ask); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(appAttemptId.toString()); Token<AMRMTokenIdentifier> token = rm.getRMContext().getRMApps() .get(appAttemptId.getApplicationId()) .getRMAppAttempt(appAttemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); AllocateResponse response = ugi.doAs( new PrivilegedExceptionAction<AllocateResponse>() { @Override public AllocateResponse run() throws Exception { return rm.getApplicationMasterService().allocate(request); } }); if (response != null) { responseQueue.put(response); } }
/** * This is called by the AMs started on this node to send heart beat to RM. * This method does the initial authorization and then forwards the request to * the application instance specific pipeline, which is a chain of request * intercepter objects. One application request processing pipeline is created * per AM instance. */ @Override public AllocateResponse allocate(AllocateRequest request) throws YarnException, IOException { AMRMTokenIdentifier amrmTokenIdentifier = YarnServerSecurityUtils.authorizeRequest(); RequestInterceptorChainWrapper pipeline = getInterceptorChain(amrmTokenIdentifier); AllocateResponse allocateResponse = pipeline.getRootInterceptor().allocate(request); updateAMRMTokens(amrmTokenIdentifier, pipeline, allocateResponse); return allocateResponse; }
@Override public AllocateResponse allocate(final AllocateRequest request) throws YarnException, IOException { if (LOG.isDebugEnabled()) { LOG.debug("Forwarding allocate request to the real YARN RM"); } AllocateResponse allocateResponse = rmClient.allocate(request); if (allocateResponse.getAMRMToken() != null) { updateAMRMToken(allocateResponse.getAMRMToken()); } return allocateResponse; }
public AllocateResponse sendContainerResizingRequest( List<ContainerResourceChangeRequest> increaseRequests, List<ContainerResourceChangeRequest> decreaseRequests) throws Exception { final AllocateRequest req = AllocateRequest.newInstance(0, 0F, null, null, null, increaseRequests, decreaseRequests); return allocate(req); }
