/** * Collect data for CloudWatch. * * @param stats * current statistics object. * @param account * currently used credentials object. * @param region * currently used aws region. */ public static void scanCloudWatch(AwsStats stats, AwsAccount account, Regions region) { LOG.debug("Scan for CloudWatch in region " + region.getName() + " in account " + account.getAccountId()); try { AmazonCloudWatchClient cw = new AmazonCloudWatchClient(account.getCredentials()); cw.setRegion(Region.getRegion(region)); int totalMetrics = 0; for (Metric m : cw.listMetrics().getMetrics()) { AwsResource res = new AwsResource(m.getMetricName(), account.getAccountId(), AwsResourceType.CloudWatch, region); stats.add(res); totalMetrics++; } LOG.info(totalMetrics + " CloudWatch metrics in region " + region.getName() + " in account " + account.getAccountId()); } catch (AmazonServiceException ase) { LOG.error("Exception of CloudWatch: " + ase.getMessage()); } catch (Exception ex) { LOG.error("Exception of CloudWatch: " + ex.getMessage()); } }
@Test public void testDimensions() throws Exception { new CloudWatchCollector( "---\nregion: reg\nmetrics:\n- aws_namespace: AWS/ELB\n aws_metric_name: RequestCount\n aws_dimensions:\n - AvailabilityZone\n - LoadBalancerName", client).register(registry); Mockito.when(client.listMetrics((ListMetricsRequest)argThat( new ListMetricsRequestMatcher().Namespace("AWS/ELB").MetricName("RequestCount").Dimensions("AvailabilityZone", "LoadBalancerName")))) .thenReturn(new ListMetricsResult().withMetrics( new Metric().withDimensions(new Dimension().withName("AvailabilityZone").withValue("a"), new Dimension().withName("LoadBalancerName").withValue("myLB")), new Metric().withDimensions(new Dimension().withName("AvailabilityZone").withValue("a"), new Dimension().withName("LoadBalancerName").withValue("myLB"), new Dimension().withName("ThisExtraDimensionIsIgnored").withValue("dummy")), new Metric().withDimensions(new Dimension().withName("AvailabilityZone").withValue("b"), new Dimension().withName("LoadBalancerName").withValue("myOtherLB")))); Mockito.when(client.getMetricStatistics((GetMetricStatisticsRequest)argThat( new GetMetricStatisticsRequestMatcher().Namespace("AWS/ELB").MetricName("RequestCount").Dimension("AvailabilityZone", "a").Dimension("LoadBalancerName", "myLB")))) .thenReturn(new GetMetricStatisticsResult().withDatapoints( new Datapoint().withTimestamp(new Date()).withAverage(2.0))); Mockito.when(client.getMetricStatistics((GetMetricStatisticsRequest)argThat( new GetMetricStatisticsRequestMatcher().Namespace("AWS/ELB").MetricName("RequestCount").Dimension("AvailabilityZone", "b").Dimension("LoadBalancerName", "myOtherLB")))) .thenReturn(new GetMetricStatisticsResult().withDatapoints( new Datapoint().withTimestamp(new Date()).withAverage(3.0))); assertEquals(2.0, registry.getSampleValue("aws_elb_request_count_average", new String[]{"job", "instance", "availability_zone", "load_balancer_name"}, new String[]{"aws_elb", "", "a", "myLB"}), .01); assertEquals(3.0, registry.getSampleValue("aws_elb_request_count_average", new String[]{"job", "instance", "availability_zone", "load_balancer_name"}, new String[]{"aws_elb", "", "b", "myOtherLB"}), .01); assertNull(registry.getSampleValue("aws_elb_request_count_average", new String[]{"job", "instance", "availability_zone", "load_balancer_name", "this_extra_dimension_is_ignored"}, new String[]{"aws_elb", "", "a", "myLB", "dummy"})); }
@Test public void testDimensionSelect() throws Exception { new CloudWatchCollector( "---\nregion: reg\nmetrics:\n- aws_namespace: AWS/ELB\n aws_metric_name: RequestCount\n aws_dimensions:\n - AvailabilityZone\n - LoadBalancerName\n aws_dimension_select:\n LoadBalancerName:\n - myLB", client).register(registry); Mockito.when(client.listMetrics((ListMetricsRequest)argThat( new ListMetricsRequestMatcher().Namespace("AWS/ELB").MetricName("RequestCount").Dimensions("AvailabilityZone", "LoadBalancerName")))) .thenReturn(new ListMetricsResult().withMetrics( new Metric().withDimensions(new Dimension().withName("AvailabilityZone").withValue("a"), new Dimension().withName("LoadBalancerName").withValue("myLB")), new Metric().withDimensions(new Dimension().withName("AvailabilityZone").withValue("b"), new Dimension().withName("LoadBalancerName").withValue("myLB")), new Metric().withDimensions(new Dimension().withName("AvailabilityZone").withValue("a"), new Dimension().withName("LoadBalancerName").withValue("myOtherLB")))); Mockito.when(client.getMetricStatistics((GetMetricStatisticsRequest)argThat( new GetMetricStatisticsRequestMatcher().Namespace("AWS/ELB").MetricName("RequestCount").Dimension("AvailabilityZone", "a").Dimension("LoadBalancerName", "myLB")))) .thenReturn(new GetMetricStatisticsResult().withDatapoints( new Datapoint().withTimestamp(new Date()).withAverage(2.0))); Mockito.when(client.getMetricStatistics((GetMetricStatisticsRequest)argThat( new GetMetricStatisticsRequestMatcher().Namespace("AWS/ELB").MetricName("RequestCount").Dimension("AvailabilityZone", "b").Dimension("LoadBalancerName", "myLB")))) .thenReturn(new GetMetricStatisticsResult().withDatapoints( new Datapoint().withTimestamp(new Date()).withAverage(2.0))); assertEquals(2.0, registry.getSampleValue("aws_elb_request_count_average", new String[]{"job", "instance", "availability_zone", "load_balancer_name"}, new String[]{"aws_elb", "", "a", "myLB"}), .01); assertEquals(2.0, registry.getSampleValue("aws_elb_request_count_average", new String[]{"job", "instance", "availability_zone", "load_balancer_name"}, new String[]{"aws_elb", "", "b", "myLB"}), .01); assertNull(registry.getSampleValue("aws_elb_request_count_average", new String[]{"job", "instance", "availability_zone", "load_balancer_name"}, new String[]{"aws_elb", "", "a", "myOtherLB"})); }
@Test public void testDimensionSelectRegex() throws Exception { new CloudWatchCollector( "---\nregion: reg\nmetrics:\n- aws_namespace: AWS/ELB\n aws_metric_name: RequestCount\n aws_dimensions:\n - AvailabilityZone\n - LoadBalancerName\n aws_dimension_select_regex:\n LoadBalancerName:\n - myLB(.*)", client).register(registry); Mockito.when(client.listMetrics((ListMetricsRequest) argThat( new ListMetricsRequestMatcher().Namespace("AWS/ELB").MetricName("RequestCount").Dimensions("AvailabilityZone", "LoadBalancerName")))) .thenReturn(new ListMetricsResult().withMetrics( new Metric().withDimensions(new Dimension().withName("AvailabilityZone").withValue("a"), new Dimension().withName("LoadBalancerName").withValue("myLB1")), new Metric().withDimensions(new Dimension().withName("AvailabilityZone").withValue("b"), new Dimension().withName("LoadBalancerName").withValue("myLB2")), new Metric().withDimensions(new Dimension().withName("AvailabilityZone").withValue("a"), new Dimension().withName("LoadBalancerName").withValue("myOtherLB")))); Mockito.when(client.getMetricStatistics((GetMetricStatisticsRequest) argThat( new GetMetricStatisticsRequestMatcher().Namespace("AWS/ELB").MetricName("RequestCount").Dimension("AvailabilityZone", "a").Dimension("LoadBalancerName", "myLB1")))) .thenReturn(new GetMetricStatisticsResult().withDatapoints( new Datapoint().withTimestamp(new Date()).withAverage(2.0))); Mockito.when(client.getMetricStatistics((GetMetricStatisticsRequest) argThat( new GetMetricStatisticsRequestMatcher().Namespace("AWS/ELB").MetricName("RequestCount").Dimension("AvailabilityZone", "b").Dimension("LoadBalancerName", "myLB2")))) .thenReturn(new GetMetricStatisticsResult().withDatapoints( new Datapoint().withTimestamp(new Date()).withAverage(2.0))); assertEquals(2.0, registry.getSampleValue("aws_elb_request_count_average", new String[]{"job", "instance", "availability_zone", "load_balancer_name"}, new String[]{"aws_elb", "", "a", "myLB1"}), .01); assertEquals(2.0, registry.getSampleValue("aws_elb_request_count_average", new String[]{"job", "instance", "availability_zone", "load_balancer_name"}, new String[]{"aws_elb", "", "b", "myLB2"}), .01); assertNull(registry.getSampleValue("aws_elb_request_count_average", new String[]{"job", "instance", "availability_zone", "load_balancer_name"}, new String[]{"aws_elb", "", "a", "myOtherLB"})); }
@Test public void testGetDimensionsUsesNextToken() throws Exception { new CloudWatchCollector( "---\nregion: reg\nmetrics:\n- aws_namespace: AWS/ELB\n aws_metric_name: RequestCount\n aws_dimensions:\n - AvailabilityZone\n - LoadBalancerName\n aws_dimension_select:\n LoadBalancerName:\n - myLB", client).register(registry); Mockito.when(client.listMetrics((ListMetricsRequest)argThat( new ListMetricsRequestMatcher().Namespace("AWS/ELB").MetricName("RequestCount").Dimensions("AvailabilityZone", "LoadBalancerName")))) .thenReturn(new ListMetricsResult().withNextToken("ABC")); Mockito.when(client.listMetrics((ListMetricsRequest)argThat( new ListMetricsRequestMatcher().Namespace("AWS/ELB").MetricName("RequestCount").Dimensions("AvailabilityZone", "LoadBalancerName").NextToken("ABC")))) .thenReturn(new ListMetricsResult().withMetrics( new Metric().withDimensions(new Dimension().withName("AvailabilityZone").withValue("a"), new Dimension().withName("LoadBalancerName").withValue("myLB")))); Mockito.when(client.getMetricStatistics((GetMetricStatisticsRequest)argThat( new GetMetricStatisticsRequestMatcher().Namespace("AWS/ELB").MetricName("RequestCount").Dimension("AvailabilityZone", "a").Dimension("LoadBalancerName", "myLB")))) .thenReturn(new GetMetricStatisticsResult().withDatapoints( new Datapoint().withTimestamp(new Date()).withAverage(2.0))); assertEquals(2.0, registry.getSampleValue("aws_elb_request_count_average", new String[]{"job", "instance", "availability_zone", "load_balancer_name"}, new String[]{"aws_elb", "", "a", "myLB"}), .01); }
public static void main(String[] args) { final String USAGE = "To run this example, supply a metric name and metric namespace\n" + "Ex: ListMetrics <metric-name> <metric-namespace>\n"; if (args.length != 2) { System.out.println(USAGE); System.exit(1); } String name = args[0]; String namespace = args[1]; final AmazonCloudWatch cw = AmazonCloudWatchClientBuilder.defaultClient(); ListMetricsRequest request = new ListMetricsRequest() .withMetricName(name) .withNamespace(namespace); boolean done = false; while(!done) { ListMetricsResult response = cw.listMetrics(request); for(Metric metric : response.getMetrics()) { System.out.printf( "Retrieved metric %s", metric.getMetricName()); } request.setNextToken(response.getNextToken()); if(response.getNextToken() == null) { done = true; } } }
public static List<Metric> listMetrics(String filter) { connect(); ListMetricsRequest req = new ListMetricsRequest(); if (filter != null) req.setMetricName(filter); ListMetricsResult res = client.listMetrics(req); return res.getMetrics(); }
private List<List<Dimension>> getDimensions(MetricRule rule, AmazonCloudWatchClient client) { List<List<Dimension>> dimensions = new ArrayList<List<Dimension>>(); if (rule.awsDimensions == null) { dimensions.add(new ArrayList<Dimension>()); return dimensions; } ListMetricsRequest request = new ListMetricsRequest(); request.setNamespace(rule.awsNamespace); request.setMetricName(rule.awsMetricName); List<DimensionFilter> dimensionFilters = new ArrayList<DimensionFilter>(); for (String dimension: rule.awsDimensions) { dimensionFilters.add(new DimensionFilter().withName(dimension)); } request.setDimensions(dimensionFilters); String nextToken = null; do { request.setNextToken(nextToken); ListMetricsResult result = client.listMetrics(request); cloudwatchRequests.inc(); for (Metric metric: result.getMetrics()) { if (metric.getDimensions().size() != dimensionFilters.size()) { // AWS returns all the metrics with dimensions beyond the ones we ask for, // so filter them out. continue; } if (useMetric(rule, metric)) { dimensions.add(metric.getDimensions()); } } nextToken = result.getNextToken(); } while (nextToken != null); return dimensions; }
/** * Check if a metric should be used according to `aws_dimension_select` or `aws_dimension_select_regex` */ private boolean useMetric(MetricRule rule, Metric metric) { if (rule.awsDimensionSelect == null && rule.awsDimensionSelectRegex == null) { return true; } if (rule.awsDimensionSelect != null && metricsIsInAwsDimensionSelect(rule, metric)) { return true; } if (rule.awsDimensionSelectRegex != null && metricIsInAwsDimensionSelectRegex(rule, metric)) { return true; } return false; }
/** * Check if a metric is matched in `aws_dimension_select` */ private boolean metricsIsInAwsDimensionSelect(MetricRule rule, Metric metric) { Set<String> dimensionSelectKeys = rule.awsDimensionSelect.keySet(); for (Dimension dimension : metric.getDimensions()) { String dimensionName = dimension.getName(); String dimensionValue = dimension.getValue(); if (dimensionSelectKeys.contains(dimensionName)) { List<String> allowedDimensionValues = rule.awsDimensionSelect.get(dimensionName); if (!allowedDimensionValues.contains(dimensionValue)) { return false; } } } return true; }
/** * Check if a metric is matched in `aws_dimension_select_regex` */ private boolean metricIsInAwsDimensionSelectRegex(MetricRule rule, Metric metric) { Set<String> dimensionSelectRegexKeys = rule.awsDimensionSelectRegex.keySet(); for (Dimension dimension : metric.getDimensions()) { String dimensionName = dimension.getName(); String dimensionValue = dimension.getValue(); if (dimensionSelectRegexKeys.contains(dimensionName)) { List<String> allowedDimensionValues = rule.awsDimensionSelectRegex.get(dimensionName); if (!regexListMatch(allowedDimensionValues, dimensionValue)) { return false; } } } return true; }
@Test public void testDynamoIndexDimensions() throws Exception { new CloudWatchCollector( "---\nregion: reg\nmetrics:\n- aws_namespace: AWS/DynamoDB\n aws_metric_name: ConsumedReadCapacityUnits\n aws_dimensions:\n - TableName\n - GlobalSecondaryIndexName\n- aws_namespace: AWS/DynamoDB\n aws_metric_name: OnlineIndexConsumedWriteCapacity\n aws_dimensions:\n - TableName\n - GlobalSecondaryIndexName\n- aws_namespace: AWS/DynamoDB\n aws_metric_name: ConsumedReadCapacityUnits\n aws_dimensions:\n - TableName", client).register(registry); Mockito.when(client.listMetrics((ListMetricsRequest)argThat( new ListMetricsRequestMatcher().Namespace("AWS/DynamoDB").MetricName("ConsumedReadCapacityUnits").Dimensions("TableName", "GlobalSecondaryIndexName")))) .thenReturn(new ListMetricsResult().withMetrics( new Metric().withDimensions(new Dimension().withName("TableName").withValue("myTable"), new Dimension().withName("GlobalSecondaryIndexName").withValue("myIndex")))); Mockito.when(client.listMetrics((ListMetricsRequest)argThat( new ListMetricsRequestMatcher().Namespace("AWS/DynamoDB").MetricName("OnlineIndexConsumedWriteCapacity").Dimensions("TableName", "GlobalSecondaryIndexName")))) .thenReturn(new ListMetricsResult().withMetrics( new Metric().withDimensions(new Dimension().withName("TableName").withValue("myTable"), new Dimension().withName("GlobalSecondaryIndexName").withValue("myIndex")))); Mockito.when(client.listMetrics((ListMetricsRequest)argThat( new ListMetricsRequestMatcher().Namespace("AWS/DynamoDB").MetricName("ConsumedReadCapacityUnits").Dimensions("TableName")))) .thenReturn(new ListMetricsResult().withMetrics( new Metric().withDimensions(new Dimension().withName("TableName").withValue("myTable")))); Mockito.when(client.getMetricStatistics((GetMetricStatisticsRequest)argThat( new GetMetricStatisticsRequestMatcher().Namespace("AWS/DynamoDB").MetricName("ConsumedReadCapacityUnits").Dimension("TableName", "myTable").Dimension("GlobalSecondaryIndexName", "myIndex")))) .thenReturn(new GetMetricStatisticsResult().withDatapoints( new Datapoint().withTimestamp(new Date()).withSum(1.0))); Mockito.when(client.getMetricStatistics((GetMetricStatisticsRequest)argThat( new GetMetricStatisticsRequestMatcher().Namespace("AWS/DynamoDB").MetricName("OnlineIndexConsumedWriteCapacity").Dimension("TableName", "myTable").Dimension("GlobalSecondaryIndexName", "myIndex")))) .thenReturn(new GetMetricStatisticsResult().withDatapoints( new Datapoint().withTimestamp(new Date()).withSum(2.0))); Mockito.when(client.getMetricStatistics((GetMetricStatisticsRequest)argThat( new GetMetricStatisticsRequestMatcher().Namespace("AWS/DynamoDB").MetricName("ConsumedReadCapacityUnits").Dimension("TableName", "myTable")))) .thenReturn(new GetMetricStatisticsResult().withDatapoints( new Datapoint().withTimestamp(new Date()).withSum(3.0))); assertEquals(1.0, registry.getSampleValue("aws_dynamodb_consumed_read_capacity_units_index_sum", new String[]{"job", "instance", "table_name", "global_secondary_index_name"}, new String[]{"aws_dynamodb", "", "myTable", "myIndex"}), .01); assertEquals(2.0, registry.getSampleValue("aws_dynamodb_online_index_consumed_write_capacity_sum", new String[]{"job", "instance", "table_name", "global_secondary_index_name"}, new String[]{"aws_dynamodb", "", "myTable", "myIndex"}), .01); assertEquals(3.0, registry.getSampleValue("aws_dynamodb_consumed_read_capacity_units_sum", new String[]{"job", "instance", "table_name"}, new String[]{"aws_dynamodb", "", "myTable"}), .01); }
@Test public void testKeyValueOperations() throws Exception { AmazonCloudWatchClient cwClient = provider.getClient(); Assume.assumeNotNull("AWS client not null", cwClient); List<Metric> staleMetrics = cwClient.listMetrics(new ListMetricsRequest().withNamespace(NAMESPACE)).getMetrics() .stream() // .filter(metric -> !metric.getMetricName().startsWith(CloudWatchIntegrationTest.class.getSimpleName()) || System.currentTimeMillis() - AWSUtils.toEpochMillis(metric.getMetricName()) > AWSUtils.TWO_WEEKS) // .collect(Collectors.toList()); if (staleMetrics.size() > 0) { Assert.fail("Found '" + CloudWatchIntegrationTest.class.getName() + "-*' metrics older than two weeks: " + staleMetrics); } WildFlyCamelContext camelctx = new WildFlyCamelContext(); camelctx.getNamingContext().bind("cwClient", cwClient); camelctx.addRoutes(new RouteBuilder() { @Override public void configure() throws Exception { from("direct:metrics").to("aws-cw://" + NAMESPACE + "?amazonCwClient=#cwClient"); } }); camelctx.start(); try { Map<String, Object> headers = new HashMap<>(); headers.put(CwConstants.METRIC_NAME, METRIC_NAME); headers.put(CwConstants.METRIC_DIMENSION_NAME, DIM_NAME); headers.put(CwConstants.METRIC_DIMENSION_VALUE, DIM_VALUE); ListMetricsRequest request = new ListMetricsRequest().withNamespace(NAMESPACE).withMetricName(METRIC_NAME) .withDimensions(new DimensionFilter().withName(DIM_NAME).withValue(DIM_VALUE)); List<Metric> metrics = Collections.emptyList(); ProducerTemplate producer = camelctx.createProducerTemplate(); for (int i = 100; i < 105 && metrics.size() == 0; i++) { producer.sendBodyAndHeaders("direct:metrics", new Double(i), headers); metrics = cwClient.listMetrics(request).getMetrics(); System.out.println("metrics #" + i + ": " + metrics); Thread.sleep(1000); } // It may take several minutes for the metric to show up // Assert.assertEquals(1, metrics.size()); } finally { camelctx.stop(); } }