@VisibleForTesting protected void initializeWebApp(Configuration conf) { webApp = new HsWebApp(history); InetSocketAddress bindAddress = MRWebAppUtil.getJHSWebBindAddress(conf); // NOTE: there should be a .at(InetSocketAddress) WebApps .$for("jobhistory", HistoryClientService.class, this, "ws") .with(conf) .withHttpSpnegoKeytabKey( JHAdminConfig.MR_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) .withHttpSpnegoPrincipalKey( JHAdminConfig.MR_WEBAPP_SPNEGO_USER_NAME_KEY) .at(NetUtils.getHostPortString(bindAddress)).start(webApp); String connectHost = MRWebAppUtil.getJHSWebappURLWithoutScheme(conf).split(":")[0]; MRWebAppUtil.setJHSWebappURLWithoutScheme(conf, connectHost + ":" + webApp.getListenerAddress().getPort()); }
@Override protected void serviceStart() throws Exception { super.serviceStart(); //need to do this because historyServer.init creates a new Configuration getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), MRWebAppUtil.getJHSWebappURLWithoutScheme(historyServer.getConfig())); LOG.info("MiniMRYARN ResourceManager address: " + getConfig().get(YarnConfiguration.RM_ADDRESS)); LOG.info("MiniMRYARN ResourceManager web address: " + WebAppUtils.getRMWebAppURLWithoutScheme(getConfig())); LOG.info("MiniMRYARN HistoryServer address: " + getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); LOG.info("MiniMRYARN HistoryServer web address: " + getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(), MRWebAppUtil.getJHSHttpPolicy() == HttpConfig.Policy.HTTPS_ONLY)); }
private void constructJobReport() { report = Records.newRecord(JobReport.class); report.setJobId(jobId); report.setJobState(JobState.valueOf(jobInfo.getJobStatus())); report.setSubmitTime(jobInfo.getSubmitTime()); report.setStartTime(jobInfo.getLaunchTime()); report.setFinishTime(jobInfo.getFinishTime()); report.setJobName(jobInfo.getJobname()); report.setUser(jobInfo.getUsername()); report.setMapProgress((float) getCompletedMaps() / getTotalMaps()); report.setReduceProgress((float) getCompletedReduces() / getTotalReduces()); report.setJobFile(getConfFile().toString()); String historyUrl = "N/A"; try { historyUrl = MRWebAppUtil.getApplicationWebURLOnJHSWithoutScheme(conf, jobId.getAppId()); } catch (UnknownHostException e) { //Ignore. } report.setTrackingUrl(historyUrl); report.setAMInfos(getAMInfos()); report.setIsUber(isUber()); }
private void initializeWebApp(Configuration conf) { webApp = new HsWebApp(history); InetSocketAddress bindAddress = MRWebAppUtil.getJHSWebBindAddress(conf); // NOTE: there should be a .at(InetSocketAddress) WebApps .$for("jobhistory", HistoryClientService.class, this, "ws") .with(conf) .withHttpSpnegoKeytabKey( JHAdminConfig.MR_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) .withHttpSpnegoPrincipalKey( JHAdminConfig.MR_WEBAPP_SPNEGO_USER_NAME_KEY) .at(NetUtils.getHostPortString(bindAddress)).start(webApp); MRWebAppUtil.setJHSWebappURLWithoutScheme(conf, NetUtils.getHostPortString(webApp.getListenerAddress())); }
@Override protected void serviceInit(Configuration conf) throws Exception { Configuration config = new YarnConfiguration(conf); config.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); // This is required for WebApps to use https if enabled. MRWebAppUtil.initialize(getConfig()); HttpConfig.setPolicy(MRWebAppUtil.getJHSHttpPolicy()); try { doSecureLogin(conf); } catch(IOException ie) { throw new YarnRuntimeException("History Server Failed to login", ie); } jobHistoryService = new JobHistory(); historyContext = (HistoryContext)jobHistoryService; this.jhsDTSecretManager = createJHSSecretManager(conf); clientService = new HistoryClientService(historyContext, this.jhsDTSecretManager); aggLogDelService = new AggregatedLogDeletionService(); addService(jobHistoryService); addService(clientService); addService(aggLogDelService); super.serviceInit(config); }
@VisibleForTesting protected void initializeWebApp(Configuration conf) { webApp = new HsWebApp(history); InetSocketAddress bindAddress = MRWebAppUtil.getJHSWebBindAddress(conf); // NOTE: there should be a .at(InetSocketAddress) WebApps .$for("jobhistory", HistoryClientService.class, this, "ws") .with(conf) .withHttpSpnegoKeytabKey( JHAdminConfig.MR_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) .withHttpSpnegoPrincipalKey( JHAdminConfig.MR_WEBAPP_SPNEGO_USER_NAME_KEY) .at(NetUtils.getHostPortString(bindAddress)).start(webApp); MRWebAppUtil.setJHSWebappURLWithoutScheme(conf, NetUtils.getHostPortString(webApp.getListenerAddress())); }
protected AppController(App app, Configuration conf, RequestContext ctx, String title) { super(ctx); this.app = app; set(APP_ID, app.context.getApplicationID().toString()); set(RM_WEB, JOINER.join(MRWebAppUtil.getYARNWebappScheme(), WebAppUtils.getResolvedRMWebAppURLWithoutScheme(conf, MRWebAppUtil.getYARNHttpPolicy()))); }
protected void register() { //Register InetSocketAddress serviceAddr = null; if (clientService != null ) { serviceAddr = clientService.getBindAddress(); } try { RegisterApplicationMasterRequest request = recordFactory.newRecordInstance(RegisterApplicationMasterRequest.class); if (serviceAddr != null) { request.setHost(serviceAddr.getHostName()); request.setRpcPort(serviceAddr.getPort()); request.setTrackingUrl(MRWebAppUtil .getAMWebappScheme(getConfig()) + serviceAddr.getHostName() + ":" + clientService.getHttpPort()); } RegisterApplicationMasterResponse response = scheduler.registerApplicationMaster(request); isApplicationMasterRegistered = true; maxContainerCapability = response.getMaximumResourceCapability(); this.context.getClusterInfo().setMaxContainerCapability( maxContainerCapability); if (UserGroupInformation.isSecurityEnabled()) { setClientToAMToken(response.getClientToAMTokenMasterKey()); } this.applicationACLs = response.getApplicationACLs(); LOG.info("maxContainerCapability: " + maxContainerCapability); String queue = response.getQueue(); LOG.info("queue: " + queue); job.setQueueName(queue); this.schedulerResourceTypes.addAll(response.getSchedulerResourceTypes()); } catch (Exception are) { LOG.error("Exception while registering", are); throw new YarnRuntimeException(are); } }
/** * Gets the URI to access the given application on MapReduce history server * @param id the ID for which a URI is returned * @return the tracking URI * @throws URISyntaxException */ @Override public URI getTrackingUri(ApplicationId id) throws URISyntaxException { String jobSuffix = id.toString().replaceFirst("^application_", "job_"); String historyServerAddress = MRWebAppUtil.getJHSWebappURLWithScheme(getConf()); return new URI(historyServerAddress + "/jobhistory/job/"+ jobSuffix); }
@Override protected void serviceInit(Configuration conf) throws Exception { Configuration config = new YarnConfiguration(conf); config.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); // This is required for WebApps to use https if enabled. MRWebAppUtil.initialize(getConfig()); try { doSecureLogin(conf); } catch(IOException ie) { throw new YarnRuntimeException("History Server Failed to login", ie); } jobHistoryService = new JobHistory(); historyContext = (HistoryContext)jobHistoryService; stateStore = createStateStore(conf); this.jhsDTSecretManager = createJHSSecretManager(conf, stateStore); clientService = createHistoryClientService(); aggLogDelService = new AggregatedLogDeletionService(); hsAdminServer = new HSAdminServer(aggLogDelService, jobHistoryService); addService(stateStore); addService(new HistoryServerSecretManagerService()); addService(jobHistoryService); addService(clientService); addService(aggLogDelService); addService(hsAdminServer); super.serviceInit(config); }
private void constructJobReport() { report = Records.newRecord(JobReport.class); report.setJobId(jobId); report.setJobState(JobState.valueOf(jobInfo.getJobStatus())); report.setSubmitTime(jobInfo.getSubmitTime()); report.setStartTime(jobInfo.getLaunchTime()); report.setFinishTime(jobInfo.getFinishTime()); report.setJobName(jobInfo.getJobname()); report.setUser(jobInfo.getUsername()); if ( getTotalMaps() == 0 ) { report.setMapProgress(1.0f); } else { report.setMapProgress((float) getCompletedMaps() / getTotalMaps()); } if ( getTotalReduces() == 0 ) { report.setReduceProgress(1.0f); } else { report.setReduceProgress((float) getCompletedReduces() / getTotalReduces()); } report.setJobFile(getConfFile().toString()); String historyUrl = "N/A"; try { historyUrl = MRWebAppUtil.getApplicationWebURLOnJHSWithoutScheme(conf, jobId.getAppId()); } catch (UnknownHostException e) { //Ignore. } report.setTrackingUrl(historyUrl); report.setAMInfos(getAMInfos()); report.setIsUber(isUber()); }
protected AppController(App app, Configuration conf, RequestContext ctx, String title) { super(ctx); this.app = app; set(APP_ID, app.context.getApplicationID().toString()); set(RM_WEB, JOINER.join(MRWebAppUtil.getYARNWebappScheme(), WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(conf, MRWebAppUtil.getYARNHttpPolicy()))); }
@Override public synchronized void serviceStart() throws Exception { try { if (!getConfig().getBoolean( JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, JHAdminConfig.DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS)) { String hostname = MiniYARNCluster.getHostname(); // pick free random ports. getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, hostname + ":0"); MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), hostname + ":0"); getConfig().set(JHAdminConfig.JHS_ADMIN_ADDRESS, hostname + ":0"); } historyServer = new JobHistoryServer(); historyServer.init(getConfig()); new Thread() { public void run() { historyServer.start(); jhsStarted = true; }; }.start(); while (!jhsStarted) { LOG.info("Waiting for HistoryServer to start..."); Thread.sleep(1500); } //TODO Add a timeout. State.STOPPED check ? if (historyServer.getServiceState() != STATE.STARTED) { throw new IOException("HistoryServer failed to start"); } super.serviceStart(); } catch (Throwable t) { throw new YarnRuntimeException(t); } }
@Override protected void serviceInit(Configuration conf) throws Exception { Configuration config = new YarnConfiguration(conf); config.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); // This is required for WebApps to use https if enabled. MRWebAppUtil.initialize(getConfig()); try { doSecureLogin(conf); } catch(IOException ie) { throw new YarnRuntimeException("History Server Failed to login", ie); } jobHistoryService = new JobHistory(); historyContext = (HistoryContext)jobHistoryService; stateStore = createStateStore(conf); this.jhsDTSecretManager = createJHSSecretManager(conf, stateStore); clientService = createHistoryClientService(); aggLogDelService = new AggregatedLogDeletionService(); hsAdminServer = new HSAdminServer(aggLogDelService, jobHistoryService); addService(stateStore); addService(new HistoryServerSecretManagerService()); addService(jobHistoryService); addService(clientService); addService(aggLogDelService); addService(hsAdminServer); DefaultMetricsSystem.initialize("JobHistoryServer"); JvmMetrics jm = JvmMetrics.initSingleton("JobHistoryServer", null); pauseMonitor = new JvmPauseMonitor(); addService(pauseMonitor); jm.setPauseMonitor(pauseMonitor); super.serviceInit(config); }
private void constructJobReport() { report = Records.newRecord(JobReport.class); report.setJobId(jobId); report.setJobState(JobState.valueOf(jobInfo.getJobStatus())); report.setSubmitTime(jobInfo.getSubmitTime()); report.setStartTime(jobInfo.getLaunchTime()); report.setFinishTime(jobInfo.getFinishTime()); report.setJobName(jobInfo.getJobname()); report.setUser(jobInfo.getUsername()); if ( getTotalMaps() == 0 ) { report.setMapProgress(1.0f); } else { report.setMapProgress((float) getCompletedMaps() / getTotalMaps()); } if ( getTotalReduces() == 0 ) { report.setReduceProgress(1.0f); } else { report.setReduceProgress((float) getCompletedReduces() / getTotalReduces()); } report.setJobFile(getConfFile().toString()); String historyUrl = "N/A"; try { historyUrl = MRWebAppUtil.getApplicationWebURLOnJHSWithScheme(conf, jobId.getAppId()); } catch (UnknownHostException e) { LOG.error("Problem determining local host: " + e.getMessage()); } report.setTrackingUrl(historyUrl); report.setAMInfos(getAMInfos()); report.setIsUber(isUber()); }
@Override protected void serviceInit(Configuration conf) throws Exception { Configuration config = new YarnConfiguration(conf); config.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); // This is required for WebApps to use https if enabled. MRWebAppUtil.initialize(getConfig()); try { doSecureLogin(conf); } catch(IOException ie) { throw new YarnRuntimeException("History Server Failed to login", ie); } jobHistoryService = new JobHistory(); historyContext = (HistoryContext)jobHistoryService; stateStore = createStateStore(conf); this.jhsDTSecretManager = createJHSSecretManager(conf, stateStore); clientService = createHistoryClientService(); aggLogDelService = new AggregatedLogDeletionService(); hsAdminServer = new HSAdminServer(aggLogDelService, jobHistoryService); killedHistoryService = new KilledHistoryService(); addService(stateStore); addService(new HistoryServerSecretManagerService()); addService(jobHistoryService); addService(clientService); addService(aggLogDelService); addService(hsAdminServer); addService(killedHistoryService); super.serviceInit(config); }
