@SuppressWarnings("deprecation") private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue, Set<String> tags, boolean unmanaged) { ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); ApplicationSubmissionContext submissionContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); submissionContext.setAMContainerSpec(amContainerSpec); submissionContext.setApplicationName(name); submissionContext.setQueue(queue); submissionContext.setApplicationId(appId); submissionContext.setResource(resource); submissionContext.setApplicationType(appType); submissionContext.setApplicationTags(tags); submissionContext.setUnmanagedAM(unmanaged); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); submitRequest.setApplicationSubmissionContext(submissionContext); return submitRequest; }
private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue) { ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); ApplicationSubmissionContext submissionContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); submissionContext.setAMContainerSpec(amContainerSpec); submissionContext.setApplicationName(name); submissionContext.setQueue(queue); submissionContext.setApplicationId(appId); submissionContext.setResource(resource); submissionContext.setApplicationType(appType); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); submitRequest.setApplicationSubmissionContext(submissionContext); return submitRequest; }
private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue, Set<String> tags, boolean unmanaged) { ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); ApplicationSubmissionContext submissionContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); submissionContext.setAMContainerSpec(amContainerSpec); submissionContext.setApplicationName(name); submissionContext.setQueue(queue); submissionContext.setApplicationId(appId); submissionContext.setResource(resource); submissionContext.setApplicationType(appType); submissionContext.setApplicationTags(tags); submissionContext.setUnmanagedAM(unmanaged); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); submitRequest.setApplicationSubmissionContext(submissionContext); return submitRequest; }
@Override public SubmitApplicationResponse submitApplication( SubmitApplicationRequest request) throws YarnException, IOException { SubmitApplicationRequestProto requestProto = ((SubmitApplicationRequestPBImpl) request).getProto(); try { return new SubmitApplicationResponsePBImpl(proxy.submitApplication(null, requestProto)); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); return null; } }
private ApplicationId submitAppAndGetAppId(String submitter, String queueName, boolean setupACLs) throws Exception { GetNewApplicationRequest newAppRequest = GetNewApplicationRequest.newInstance(); ApplicationClientProtocol submitterClient = getRMClientForUser(submitter); ApplicationId applicationId = submitterClient.getNewApplication(newAppRequest).getApplicationId(); Resource resource = BuilderUtils.newResource(1024, 1); Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs); ContainerLaunchContext amContainerSpec = ContainerLaunchContext.newInstance(null, null, null, null, null, acls); ApplicationSubmissionContext appSubmissionContext = ApplicationSubmissionContext.newInstance(applicationId, "applicationName", queueName, null, amContainerSpec, false, true, 1, resource, "applicationType"); appSubmissionContext.setApplicationId(applicationId); appSubmissionContext.setQueue(queueName); SubmitApplicationRequest submitRequest = SubmitApplicationRequest.newInstance(appSubmissionContext); submitterClient.submitApplication(submitRequest); resourceManager.waitForState(applicationId, RMAppState.ACCEPTED); return applicationId; }
@Test(timeout=20000) public void testAppSubmissionWithInvalidDelegationToken() throws Exception { Configuration conf = new Configuration(); conf.set( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); MockRM rm = new MockRM(conf) { @Override protected void doSecureLogin() throws IOException { // Skip the login. } }; ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes()); ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance( new HashMap<String, LocalResource>(), new HashMap<String, String>(), new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens, new HashMap<ApplicationAccessType, String>()); ApplicationSubmissionContext appSubContext = ApplicationSubmissionContext.newInstance( ApplicationId.newInstance(1234121, 0), "BOGUS", "default", Priority.UNDEFINED, amContainer, false, true, 1, Resource.newInstance(1024, 1, 1), "BOGUS"); SubmitApplicationRequest request = SubmitApplicationRequest.newInstance(appSubContext); try { rm.getClientRMService().submitApplication(request); fail("Error was excepted."); } catch (YarnException e) { Assert.assertTrue(e.getMessage().contains( "Bad header found in token storage")); } }
@SuppressWarnings("deprecation") private ApplicationId submitAppAndGetAppId(AccessControlList viewACL, AccessControlList modifyACL) throws Exception { SubmitApplicationRequest submitRequest = recordFactory .newRecordInstance(SubmitApplicationRequest.class); ApplicationSubmissionContext context = recordFactory .newRecordInstance(ApplicationSubmissionContext.class); ApplicationId applicationId = rmClient.getNewApplication( recordFactory.newRecordInstance(GetNewApplicationRequest.class)) .getApplicationId(); context.setApplicationId(applicationId); Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>(); acls.put(ApplicationAccessType.VIEW_APP, viewACL.getAclString()); acls.put(ApplicationAccessType.MODIFY_APP, modifyACL.getAclString()); ContainerLaunchContext amContainer = recordFactory .newRecordInstance(ContainerLaunchContext.class); Resource resource = BuilderUtils.newResource(1024, 1); context.setResource(resource); amContainer.setApplicationACLs(acls); context.setAMContainerSpec(amContainer); submitRequest.setApplicationSubmissionContext(context); rmClient.submitApplication(submitRequest); resourceManager.waitForState(applicationId, RMAppState.ACCEPTED); return applicationId; }
@SuppressWarnings("deprecation") public synchronized void submit() throws IOException, YarnException { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(this.applicationId); context.setQueue(this.queue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); context.setAMContainerSpec(amContainer); context.setResource(Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB)); SubmitApplicationRequest request = recordFactory .newRecordInstance(SubmitApplicationRequest.class); request.setApplicationSubmissionContext(context); final ResourceScheduler scheduler = resourceManager.getResourceScheduler(); resourceManager.getClientRMService().submitApplication(request); // Notify scheduler AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId, this.queue, "user"); scheduler.handle(addAppEvent); AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(this.applicationAttemptId, false); scheduler.handle(addAttemptEvent); }
@Override public SubmitApplicationResponse submitApplication( SubmitApplicationRequest request) throws YarnException { resetStartFailoverFlag(true); // make sure failover has been triggered Assert.assertTrue(waittingForFailOver()); return super.submitApplication(request); }
@Test(timeout=20000) public void testAppSubmissionWithInvalidDelegationToken() throws Exception { Configuration conf = new Configuration(); conf.set( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); MockRM rm = new MockRM(conf) { @Override protected void doSecureLogin() throws IOException { // Skip the login. } }; ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes()); ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance( new HashMap<String, LocalResource>(), new HashMap<String, String>(), new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens, new HashMap<ApplicationAccessType, String>()); ApplicationSubmissionContext appSubContext = ApplicationSubmissionContext.newInstance( ApplicationId.newInstance(1234121, 0), "BOGUS", "default", Priority.UNDEFINED, amContainer, false, true, 1, Resource.newInstance(1024, 1), "BOGUS"); SubmitApplicationRequest request = SubmitApplicationRequest.newInstance(appSubContext); try { rm.getClientRMService().submitApplication(request); fail("Error was excepted."); } catch (YarnException e) { Assert.assertTrue(e.getMessage().contains( "Bad header found in token storage")); } }
@SuppressWarnings("deprecation") public synchronized void submit() throws IOException, YarnException { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(this.applicationId); context.setQueue(this.queue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); context.setAMContainerSpec(amContainer); context.setResource(Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB)); SubmitApplicationRequest request = recordFactory .newRecordInstance(SubmitApplicationRequest.class); request.setApplicationSubmissionContext(context); final ResourceScheduler scheduler = resourceManager.getResourceScheduler(); resourceManager.getClientRMService().submitApplication(request); // Notify scheduler，同时提交app和appAttempt AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId, this.queue, "user"); scheduler.handle(addAppEvent); AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(this.applicationAttemptId, false); scheduler.handle(addAttemptEvent); }
private ApplicationId submitAppAndGetAppId(AccessControlList viewACL, AccessControlList modifyACL) throws Exception { SubmitApplicationRequest submitRequest = recordFactory .newRecordInstance(SubmitApplicationRequest.class); ApplicationSubmissionContext context = recordFactory .newRecordInstance(ApplicationSubmissionContext.class); ApplicationId applicationId = rmClient.getNewApplication( recordFactory.newRecordInstance(GetNewApplicationRequest.class)) .getApplicationId(); context.setApplicationId(applicationId); Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>(); acls.put(ApplicationAccessType.VIEW_APP, viewACL.getAclString()); acls.put(ApplicationAccessType.MODIFY_APP, modifyACL.getAclString()); ContainerLaunchContext amContainer = recordFactory .newRecordInstance(ContainerLaunchContext.class); Resource resource = BuilderUtils.newResource(1024, 1); context.setResource(resource); amContainer.setApplicationACLs(acls); context.setAMContainerSpec(amContainer); submitRequest.setApplicationSubmissionContext(context); rmClient.submitApplication(submitRequest); resourceManager.waitForState(applicationId, RMAppState.ACCEPTED); return applicationId; }
public synchronized void submit() throws IOException, YarnException { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(this.applicationId); context.setQueue(this.queue); SubmitApplicationRequest request = recordFactory .newRecordInstance(SubmitApplicationRequest.class); request.setApplicationSubmissionContext(context); resourceManager.getClientRMService().submitApplication(request); }
@Override public ApplicationId submitApplication(ApplicationSubmissionContext appContext) throws YarnException, IOException { ApplicationId applicationId = appContext.getApplicationId(); appContext.setApplicationId(applicationId); SubmitApplicationRequest request = Records.newRecord(SubmitApplicationRequest.class); request.setApplicationSubmissionContext(appContext); rmClient.submitApplication(request); int pollCount = 0; while (true) { YarnApplicationState state = getApplicationReport(applicationId).getYarnApplicationState(); if (!state.equals(YarnApplicationState.NEW) && !state.equals(YarnApplicationState.NEW_SAVING)) { break; } // Notify the client through the log every 10 poll, in case the client // is blocked here too long. if (++pollCount % 10 == 0) { LOG.info("Application submission is not finished, " + "submitted application " + applicationId + " is still in " + state); } try { Thread.sleep(statePollIntervalMillis); } catch (InterruptedException ie) { } } LOG.info("Submitted application " + applicationId + " to ResourceManager" + " at " + rmAddress); return applicationId; }
