public JobMetrics(Job job, String bytesReplicatedKey) { Builder<String, Long> builder = ImmutableMap.builder(); if (job != null) { Counters counters; try { counters = job.getCounters(); } catch (IOException e) { throw new CircusTrainException("Unable to get counters from job.", e); } if (counters != null) { for (CounterGroup group : counters) { for (Counter counter : group) { builder.put(DotJoiner.join(group.getName(), counter.getName()), counter.getValue()); } } } } metrics = builder.build(); Long bytesReplicatedValue = metrics.get(bytesReplicatedKey); if (bytesReplicatedValue != null) { bytesReplicated = bytesReplicatedValue; } else { bytesReplicated = 0L; } }
public JobTaskAttemptCounterInfo(TaskAttempt taskattempt) { this.id = MRApps.toString(taskattempt.getID()); total = taskattempt.getCounters(); taskAttemptCounterGroup = new ArrayList<TaskCounterGroupInfo>(); if (total != null) { for (CounterGroup g : total) { if (g != null) { TaskCounterGroupInfo cginfo = new TaskCounterGroupInfo(g.getName(), g); if (cginfo != null) { taskAttemptCounterGroup.add(cginfo); } } } } }
public JobCounterInfo(AppContext ctx, Job job) { getCounters(ctx, job); counterGroup = new ArrayList<CounterGroupInfo>(); this.id = MRApps.toString(job.getID()); if (total != null) { for (CounterGroup g : total) { if (g != null) { CounterGroup mg = map == null ? null : map.getGroup(g.getName()); CounterGroup rg = reduce == null ? null : reduce .getGroup(g.getName()); CounterGroupInfo cginfo = new CounterGroupInfo(g.getName(), g, mg, rg); counterGroup.add(cginfo); } } } }
@Private public JsonNode countersToJSON(Counters counters) { ObjectMapper mapper = new ObjectMapper(); ArrayNode nodes = mapper.createArrayNode(); if (counters != null) { for (CounterGroup counterGroup : counters) { ObjectNode groupNode = nodes.addObject(); groupNode.put("NAME", counterGroup.getName()); groupNode.put("DISPLAY_NAME", counterGroup.getDisplayName()); ArrayNode countersNode = groupNode.putArray("COUNTERS"); for (Counter counter : counterGroup) { ObjectNode counterNode = countersNode.addObject(); counterNode.put("NAME", counter.getName()); counterNode.put("DISPLAY_NAME", counter.getDisplayName()); counterNode.put("VALUE", counter.getValue()); } } } return nodes; }
static JhCounters toAvro(Counters counters, String name) { JhCounters result = new JhCounters(); result.name = new Utf8(name); result.groups = new ArrayList<JhCounterGroup>(0); if (counters == null) return result; for (CounterGroup group : counters) { JhCounterGroup g = new JhCounterGroup(); g.name = new Utf8(group.getName()); g.displayName = new Utf8(group.getDisplayName()); g.counts = new ArrayList<JhCounter>(group.size()); for (Counter counter : group) { JhCounter c = new JhCounter(); c.name = new Utf8(counter.getName()); c.displayName = new Utf8(counter.getDisplayName()); c.value = counter.getValue(); g.counts.add(c); } result.groups.add(g); } return result; }
public TaskCounterGroupInfo(String name, CounterGroup group) { this.counterGroupName = name; this.counter = new ArrayList<TaskCounterInfo>(); for (Counter c : group) { TaskCounterInfo cinfo = new TaskCounterInfo(c.getName(), c.getValue()); this.counter.add(cinfo); } }
static Counters fromAvro(JhCounters counters) { Counters result = new Counters(); if(counters != null) { for (JhCounterGroup g : counters.getGroups()) { CounterGroup group = result.addGroup(StringInterner.weakIntern(g.getName().toString()), StringInterner.weakIntern(g.getDisplayName().toString())); for (JhCounter c : g.getCounts()) { group.addCounter(StringInterner.weakIntern(c.getName().toString()), StringInterner.weakIntern(c.getDisplayName().toString()), c.getValue()); } } } return result; }
static JhCounters toAvro(Counters counters, String name) { JhCounters result = new JhCounters(); result.setName(new Utf8(name)); result.setGroups(new ArrayList<JhCounterGroup>(0)); if (counters == null) return result; for (CounterGroup group : counters) { JhCounterGroup g = new JhCounterGroup(); g.setName(new Utf8(group.getName())); g.setDisplayName(new Utf8(group.getDisplayName())); g.setCounts(new ArrayList<JhCounter>(group.size())); for (Counter counter : group) { JhCounter c = new JhCounter(); c.setName(new Utf8(counter.getName())); c.setDisplayName(new Utf8(counter.getDisplayName())); c.setValue(counter.getValue()); g.getCounts().add(c); } result.getGroups().add(g); } return result; }
/** * Create a {@link gobblin.metrics.GobblinMetrics} instance for this job run from the Hadoop counters. */ private void countersToMetrics(Optional<Counters> counters, GobblinMetrics metrics) { if (counters.isPresent()) { // Write job-level counters CounterGroup jobCounterGroup = counters.get().getGroup(MetricGroup.JOB.name()); for (Counter jobCounter : jobCounterGroup) { metrics.getCounter(jobCounter.getName()).inc(jobCounter.getValue()); } // Write task-level counters CounterGroup taskCounterGroup = counters.get().getGroup(MetricGroup.TASK.name()); for (Counter taskCounter : taskCounterGroup) { metrics.getCounter(taskCounter.getName()).inc(taskCounter.getValue()); } } }
private void dumpSummary(Job job, JoinPhaseConfig joinPhaseConfig) throws IOException { System.out.println("Join Input Matrix."); CounterGroup group = job.getCounters().getGroup("DynamicCounter"); for (String source : joinPhaseConfig.getJoinSpec().getSourceNames()) { System.out.print(String.format("%25s\t", source)); } if (group != null) { Iterator<Counter> iterator = group.iterator(); while (iterator.hasNext()) { Counter counter = iterator.next(); String displayName = counter.getDisplayName(); String[] split = displayName.replace("[", "").replace("[", "").split(","); for (String str : split) { if (str.trim().equals("1")) { System.out.print(String.format("%25s\t", "1")); } else { System.out.print(String.format("%25s\t", "-")); } } } } }
/** {@inheritDoc} */ @Override public Iterator<CounterGroup> iterator() { final Iterator<String> iter = getGroupNames().iterator(); return new Iterator<CounterGroup>() { @Override public boolean hasNext() { return iter.hasNext(); } @Override public CounterGroup next() { if (!hasNext()) throw new NoSuchElementException(); return new HadoopMapReduceCounterGroup(HadoopMapReduceCounters.this, iter.next()); } @Override public void remove() { throw new UnsupportedOperationException("not implemented"); } }; }
@Private public JsonNode countersToJSON(Counters counters) { ArrayNode nodes = FACTORY.arrayNode(); if (counters != null) { for (CounterGroup counterGroup : counters) { ObjectNode groupNode = nodes.addObject(); groupNode.put("NAME", counterGroup.getName()); groupNode.put("DISPLAY_NAME", counterGroup.getDisplayName()); ArrayNode countersNode = groupNode.putArray("COUNTERS"); for (Counter counter : counterGroup) { ObjectNode counterNode = countersNode.addObject(); counterNode.put("NAME", counter.getName()); counterNode.put("DISPLAY_NAME", counter.getDisplayName()); counterNode.put("VALUE", counter.getValue()); } } } return nodes; }
JhCounters(Counters counters, String name) { this.name = name; this.groups = new ArrayList<JhCounterGroup>(); if (counters == null) return; for (CounterGroup group : counters) { JhCounterGroup g = new JhCounterGroup(); g.name = group.getName(); g.displayName = group.getDisplayName(); g.counts = new ArrayList<JhCounter>(group.size()); for (Counter counter : group) { JhCounter c = new JhCounter(); c.name = counter.getName(); c.displayName = counter.getDisplayName(); c.value = counter.getValue(); g.counts.add(c); } this.groups.add(g); } }
/** * Create a {@link org.apache.gobblin.metrics.GobblinMetrics} instance for this job run from the Hadoop counters. */ @VisibleForTesting void countersToMetrics(GobblinMetrics metrics) throws IOException { Optional<Counters> counters = Optional.fromNullable(this.job.getCounters()); if (counters.isPresent()) { // Write job-level counters CounterGroup jobCounterGroup = counters.get().getGroup(MetricGroup.JOB.name()); for (Counter jobCounter : jobCounterGroup) { metrics.getCounter(jobCounter.getName()).inc(jobCounter.getValue()); } // Write task-level counters CounterGroup taskCounterGroup = counters.get().getGroup(MetricGroup.TASK.name()); for (Counter taskCounter : taskCounterGroup) { metrics.getCounter(taskCounter.getName()).inc(taskCounter.getValue()); } } }
