/** * Connect to the default {@link JobTracker}. * @param conf the job configuration. * @throws IOException */ public void init(JobConf conf) throws IOException { setConf(conf); Limits.init(conf); String tracker = conf.get("mapred.job.tracker", "local"); tasklogtimeout = conf.getInt( TASKLOG_PULL_TIMEOUT_KEY, DEFAULT_TASKLOG_TIMEOUT); this.ugi = UserGroupInformation.getCurrentUser(); if ("local".equals(tracker)) { conf.setNumMapTasks(1); this.jobSubmitClient = new LocalJobRunner(conf); } else if (!HAUtil.isHAEnabled(conf, tracker)) { this.jobSubmitClient = createRPCProxy(JobTracker.getAddress(conf), conf); } else { this.jobSubmitClient = createRPCProxy(tracker, conf); } // Read progress monitor poll interval from config. Default is 1 second. this.progMonitorPollIntervalMillis = conf.getInt(PROGRESS_MONITOR_POLL_INTERVAL_KEY, DEFAULT_MONITOR_POLL_INTERVAL); if (this.progMonitorPollIntervalMillis < 1) { LOG.warn(PROGRESS_MONITOR_POLL_INTERVAL_KEY + " has been set to an invalid value; " + " replacing with " + DEFAULT_MONITOR_POLL_INTERVAL); this.progMonitorPollIntervalMillis = DEFAULT_MONITOR_POLL_INTERVAL; } }
private void testMaxCounters(final Counters counters) { LOG.info("counters max="+ Limits.getCountersMax()); for (int i = 0; i < Limits.getCountersMax(); ++i) { counters.findCounter("test", "test"+ i); } setExpected(counters); shouldThrow(LimitExceededException.class, new Runnable() { public void run() { counters.findCounter("test", "bad"); } }); checkExpected(counters); }
private void testMaxGroups(final Counters counters) { LOG.info("counter groups max="+ Limits.getGroupsMax()); for (int i = 0; i < Limits.getGroupsMax(); ++i) { // assuming COUNTERS_MAX > GROUPS_MAX counters.findCounter("test"+ i, "test"); } setExpected(counters); shouldThrow(LimitExceededException.class, new Runnable() { public void run() { counters.findCounter("bad", "test"); } }); checkExpected(counters); }
@Test public void testResetOnDeserialize() throws IOException { // Allow only one counterGroup Configuration conf = new Configuration(); conf.setInt(MRJobConfig.COUNTER_GROUPS_MAX_KEY, 1); Limits.init(conf); Counters countersWithOneGroup = new Counters(); countersWithOneGroup.findCounter("firstOf1Allowed", "First group"); boolean caughtExpectedException = false; try { countersWithOneGroup.findCounter("secondIsTooMany", "Second group"); } catch (LimitExceededException _) { caughtExpectedException = true; } assertTrue("Did not throw expected exception", caughtExpectedException); Counters countersWithZeroGroups = new Counters(); DataOutputBuffer out = new DataOutputBuffer(); countersWithZeroGroups.write(out); DataInputBuffer in = new DataInputBuffer(); in.reset(out.getData(), out.getLength()); countersWithOneGroup.readFields(in); // After reset one should be able to add a group countersWithOneGroup.findCounter("firstGroupAfterReset", "After reset " + "limit should be set back to zero"); }
/** * Constructs the HistoryViewer object * @param historyFile The fully qualified Path of the History File * @param conf The Configuration file * @param printAll Toggle to print all status to only killed/failed status * @throws IOException */ public HistoryViewer(String historyFile, Configuration conf, boolean printAll) throws IOException { this.printAll = printAll; String errorMsg = "Unable to initialize History Viewer"; try { Path jobFile = new Path(historyFile); fs = jobFile.getFileSystem(conf); String[] jobDetails = jobFile.getName().split("_"); if (jobDetails.length < 2) { // NOT a valid name System.err.println("Ignore unrecognized file: " + jobFile.getName()); throw new IOException(errorMsg); } final Path jobConfPath = new Path(jobFile.getParent(), jobDetails[0] + "_" + jobDetails[1] + "_" + jobDetails[2] + "_conf.xml"); final Configuration jobConf = new Configuration(conf); try { jobConf.addResource(fs.open(jobConfPath), jobConfPath.toString()); Limits.reset(jobConf); } catch (FileNotFoundException fnf) { if (LOG.isWarnEnabled()) { LOG.warn("Missing job conf in history", fnf); } } JobHistoryParser parser = new JobHistoryParser(fs, jobFile); job = parser.parse(); jobId = job.getJobId().toString(); } catch(Exception e) { throw new IOException(errorMsg, e); } }
/** * Connect to the default cluster * @param conf the job configuration. * @throws IOException */ public void init(JobConf conf) throws IOException { setConf(conf); Limits.init(conf); cluster = new Cluster(conf); clientUgi = UserGroupInformation.getCurrentUser(); }
private void testMaxCounters(final Counters counters) { LOG.info("counters max="+ Limits.COUNTERS_MAX); for (int i = 0; i < Limits.COUNTERS_MAX; ++i) { counters.findCounter("test", "test"+ i); } setExpected(counters); shouldThrow(LimitExceededException.class, new Runnable() { public void run() { counters.findCounter("test", "bad"); } }); checkExpected(counters); }
private void testMaxGroups(final Counters counters) { LOG.info("counter groups max="+ Limits.GROUPS_MAX); for (int i = 0; i < Limits.GROUPS_MAX; ++i) { // assuming COUNTERS_MAX > GROUPS_MAX counters.findCounter("test"+ i, "test"); } setExpected(counters); shouldThrow(LimitExceededException.class, new Runnable() { public void run() { counters.findCounter("bad", "test"); } }); checkExpected(counters); }
GenericGroup(String name, String displayName, Limits limits) { super(name, displayName, limits); }
@Override protected Group newGenericGroup(String name, String displayName, Limits limits) { return new Group(new GenericGroup(name, displayName, limits)); }
@Override protected CounterGroup newGenericGroup(String name, String displayName, Limits limits) { return new GenericGroup(name, displayName, limits); }
