@Test public void testFileSystemGroupIteratorConcurrency() { Counters counters = new Counters(); // create 2 filesystem counter groups counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1); counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1); // Iterate over the counters in this group while updating counters in // the group Group group = counters.getGroup(FileSystemCounter.class.getName()); Iterator<Counter> iterator = group.iterator(); counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1); assertTrue(iterator.hasNext()); iterator.next(); counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1); assertTrue(iterator.hasNext()); iterator.next(); }
@SuppressWarnings("rawtypes") @Test public void testFrameworkCounter() { GroupFactory groupFactory = new GroupFactoryForTest(); FrameworkGroupFactory frameworkGroupFactory = groupFactory.newFrameworkGroupFactory(JobCounter.class); Group group = (Group) frameworkGroupFactory.newGroup("JobCounter"); FrameworkCounterGroup counterGroup = (FrameworkCounterGroup) group.getUnderlyingGroup(); org.apache.hadoop.mapreduce.Counter count1 = counterGroup.findCounter(JobCounter.NUM_FAILED_MAPS.toString()); Assert.assertNotNull(count1); // Verify no exception get thrown when finding an unknown counter org.apache.hadoop.mapreduce.Counter count2 = counterGroup.findCounter("Unknown"); Assert.assertNull(count2); }
/** * Converts Hadoop counters to a JSON representation. * * @param counters the Hadoop counters to convert * @return the JSON representation of the given counters * * @throws SerializationException if mapping the counters to JSON fails */ @VisibleForTesting static String toJson(Counters counters) throws SerializationException { ArrayNode countersJsonNode = JsonNodeFactory.instance.arrayNode(); ArrayNode groupsJsonNode = JsonNodeFactory.instance.arrayNode(); for (Group group: counters) { for (Counters.Counter counter: group) { ObjectNode counterJsonNode = JsonNodeFactory.instance.objectNode(); counterJsonNode.put("counterName", counter.getName()) .put("value", counter.getValue()); countersJsonNode.add(counterJsonNode); } ObjectNode groupJsonNode = JsonNodeFactory.instance.objectNode(); groupJsonNode.put("groupName", group.getDisplayName()) .put("counters", countersJsonNode); groupsJsonNode.add(groupJsonNode); } ObjectMapper mapper = new ObjectMapper(); try { return mapper.writeValueAsString(groupsJsonNode); } catch (JsonProcessingException e) { throw new SerializationException(e); } }
public static Object countersToJson(Counters counters) { Map<String, Object> jsonObj = new HashMap<String, Object>(); if (counters == null) { return jsonObj; } Collection<String> counterGroups = counters.getGroupNames(); for (String groupName : counterGroups) { Map<String, String> counterStats = new HashMap<String, String>(); Group group = counters.getGroup(groupName); Iterator<Counters.Counter> it = group.iterator(); while (it.hasNext()) { Counter counter = it.next(); counterStats.put(counter.getDisplayName(), String.valueOf(counter.getCounter())); } jsonObj.put(groupName, counterStats); } return jsonObj; }
public Counters.Counter getCounter(String group, String name) { Counters.Counter counter = null; if (counters != null) { counter = counters.findCounter(group, name); if (counter == null) { Group grp = counters.addGroup(group, group); counter = grp.addCounter(name, name, 10); } } return counter; }
@SuppressWarnings("deprecation") @Test public void testCounterIteratorConcurrency() { Counters counters = new Counters(); counters.incrCounter("group1", "counter1", 1); Iterator<Group> iterator = counters.iterator(); counters.incrCounter("group2", "counter2", 1); iterator.next(); }
@SuppressWarnings("deprecation") @Test public void testGroupIteratorConcurrency() { Counters counters = new Counters(); counters.incrCounter("group1", "counter1", 1); Group group = counters.getGroup("group1"); Iterator<Counter> iterator = group.iterator(); counters.incrCounter("group1", "counter2", 1); iterator.next(); }
@Test public void testFilesystemCounter() { GroupFactory groupFactory = new GroupFactoryForTest(); Group fsGroup = groupFactory.newFileSystemGroup(); org.apache.hadoop.mapreduce.Counter count1 = fsGroup.findCounter("ANY_BYTES_READ"); Assert.assertNotNull(count1); // Verify no exception get thrown when finding an unknown counter org.apache.hadoop.mapreduce.Counter count2 = fsGroup.findCounter("Unknown"); Assert.assertNull(count2); }
private void validateCounters() throws IOException { Counters counters = job.running_.getCounters(); assertNotNull("Counters", counters); Group group = counters.getGroup("UserCounters"); assertNotNull("Group", group); Counter counter = group.getCounterForName("InputLines"); assertNotNull("Counter", counter); assertEquals(3, counter.getCounter()); }