Java 类com.amazonaws.services.autoscaling.model.Tag 实例源码

/***
 * Get list of {@link AutoScalingGroup}s for a given tag
 *
 * @param tag Tag to filter the auto scaling groups
 * @return List of {@link AutoScalingGroup}s qualifying the filter tag
 */
public List<AutoScalingGroup> getAutoScalingGroupsWithTag(Tag tag) {

  final AmazonAutoScaling autoScaling = getAmazonAutoScalingClient();

  final DescribeAutoScalingGroupsRequest describeAutoScalingGroupsRequest = new DescribeAutoScalingGroupsRequest();

  final List<AutoScalingGroup> allAutoScalingGroups = autoScaling
      .describeAutoScalingGroups(describeAutoScalingGroupsRequest)
      .getAutoScalingGroups();

  final List<AutoScalingGroup> filteredAutoScalingGroups = Lists.newArrayList();
  for (AutoScalingGroup autoScalingGroup : allAutoScalingGroups) {
    for (TagDescription tagDescription : autoScalingGroup.getTags()) {
      if (tagDescription.getKey().equalsIgnoreCase(tag.getKey()) &&
          tagDescription.getValue().equalsIgnoreCase(tag.getValue())) {
        filteredAutoScalingGroups.add(autoScalingGroup);
      }
    }
  }

  return filteredAutoScalingGroups;
}

public void setDeployMetadataTags(final String version, Properties properties) {
    List<Tag> tags = new ArrayList<>();
    tags.add(new Tag().withPropagateAtLaunch(true)
            .withResourceType(AUTO_SCALING_GROUP)
            .withKey(LATEST_REQUEST_TAG).withValue(version)
            .withResourceId(activeConfiguration.getAutoScalingGroupId()));
    tags.add(new Tag().withPropagateAtLaunch(true)
            .withResourceType(AUTO_SCALING_GROUP)
            .withKey(SCOPE_TAG).withValue(Boolean.toString(activeConfiguration.isTestScope()))
            .withResourceId(activeConfiguration.getAutoScalingGroupId()));

    if (!activeConfiguration.getAutoScalingProperties().isEmpty()) {
        tags.add(new Tag().withPropagateAtLaunch(true)
                .withResourceType(AUTO_SCALING_GROUP)
                .withKey(PROPERTIES_TAGS).withValue(activeConfiguration.getAutoScalingProperties().stream().map(key -> key + ":" + getProperty(key, properties)).collect(Collectors.joining(";")))
                .withResourceId(activeConfiguration.getAutoScalingGroupId())
        );
    }
    if (!activeConfiguration.getExclusions().isEmpty()) {
        tags.add(new Tag().withPropagateAtLaunch(true)
                .withResourceType(AUTO_SCALING_GROUP)
                .withKey(EXCLUSION_TAG).withValue(activeConfiguration.getExclusions().stream().map(e -> e.getGroupId() + ":" + e.getGroupId()).collect(Collectors.joining(";")))
                .withResourceId(activeConfiguration.getAutoScalingGroupId()));
    }
    awsAsClient.createOrUpdateTags(new CreateOrUpdateTagsRequest().withTags(tags));
}

@Override
public void execute(Context context) throws Exception {
    ASGroupHelper helper = new ASGroupHelper(context.env);
    helper.createLaunchConfig(resource);

    CreateAutoScalingGroupRequest request = new CreateAutoScalingGroupRequest()
        .withAutoScalingGroupName(context.env.name + "-" + resource.id)
        .withLaunchConfigurationName(resource.launchConfig.remoteLaunchConfig.getLaunchConfigurationName())
        .withDesiredCapacity(resource.desiredSize)
        .withMinSize(resource.minSize)
        .withMaxSize(resource.maxSize)
        .withDefaultCooldown(60)
        .withHealthCheckGracePeriod(300)    // give 5 mins for server and application startup
        .withTerminationPolicies(ASGroup.TERMINATE_POLICY_OLDEST_INSTANCE)      // always remove oldest instance, OldestLaunchConfiguration should not be used due to during deployment the old LaunchConfig can be deleted first, the ASG may fail to compare, and terminate unwanted instance
        .withTags(new Tag().withKey("cloud-manager:env").withValue(context.env.name).withPropagateAtLaunch(true),
            helper.nameTag(resource));

    if (resource.elb != null) {
        request.withHealthCheckType("ELB")
            .withLoadBalancerNames(resource.elb.remoteELB.getLoadBalancerName());
    } else {
        request.withHealthCheckType("EC2");
    }

    List<String> availabilityZones = Lists.newArrayList();
    StringBuilder subnetIds = new StringBuilder();
    int index = 0;
    for (Subnet remoteSubnet : resource.subnet.remoteSubnets) {
        if (index > 0) subnetIds.append(',');
        subnetIds.append(remoteSubnet.getSubnetId());
        availabilityZones.add(remoteSubnet.getAvailabilityZone());
        index++;
    }
    request.withAvailabilityZones(availabilityZones)
        .withVPCZoneIdentifier(subnetIds.toString());

    resource.remoteASGroup = AWS.as.createASGroup(request);
}

@RequestMapping(value = "/autoscalinggroup", method = RequestMethod.POST)
@ResponseBody
public void createAutoScalingGroup(@RequestParam("accessKey") String accessKey, @RequestParam("secretKey") String secretKey,
        @RequestBody Map<String, String> body) {
    awsec2Service.createAmazonAutoScalingGroup(
               awsCredentialsFactory.createSimpleAWSCredentials(accessKey, secretKey),
            body.get("autoScalingGroupName"),
               Integer.valueOf(body.get("maxSize")),
               Integer.valueOf(body.get("minSize")),
               Integer.valueOf(body.get("defaultCooldown")),
            body.get("launchConfigurationName"),
               Collections.<String> emptyList(),
               Collections.<Tag> emptyList());
}

public void updateTag(String asGroupName, Tag tag) {
    logger.info("update auto scaling group tag, asGroup={}, key={}, value={}", asGroupName, tag.getKey(), tag.getValue());
    tag.withResourceId(asGroupName).withResourceType("auto-scaling-group");
    autoScaling.createOrUpdateTags(new CreateOrUpdateTagsRequest().withTags(tag));
}

Tag nameTag(ASGroup asGroup) {
    return new Tag().withKey("Name").withValue(instanceName(asGroup)).withPropagateAtLaunch(true);
}

@VisibleForTesting
Optional<String> getReconnectableClusterId() throws IOException {
  // List ASGs with Tag of cluster name
  final Tag clusterNameTag = new Tag()
      .withKey(CLUSTER_NAME_ASG_TAG)
      .withValue(this.clusterName);
  final List<AutoScalingGroup> autoScalingGroups = this.awsSdkClient.getAutoScalingGroupsWithTag(clusterNameTag);

  // If no auto scaling group is found, we don't have an existing cluster to connect to
  if (autoScalingGroups.size() == 0) {
    return Optional.absent();
  }

  // If more than 0 auto scaling groups are found, validate the setup
  if (autoScalingGroups.size() != 2) {
    throw new IOException("Expected 2 auto scaling groups (1 each for master and workers) but found: " +
      autoScalingGroups.size());
  }

  // Retrieve cluster information from ASGs
  Optional<String> clusterId = Optional.absent();
  Optional<AutoScalingGroup> masterAsg = Optional.absent();
  Optional<AutoScalingGroup> workersAsg = Optional.absent();

  for (TagDescription tagDescription : autoScalingGroups.get(0).getTags()) {
    LOGGER.info("Found tag: " + tagDescription);
    if (tagDescription.getKey().equalsIgnoreCase(CLUSTER_ID_ASG_TAG)) {
      clusterId = Optional.of(tagDescription.getValue());
    }
    if (tagDescription.getKey().equalsIgnoreCase(ASG_TYPE_ASG_TAG)) {
      if (tagDescription.getValue().equalsIgnoreCase(ASG_TYPE_MASTER)) {
        masterAsg = Optional.of(autoScalingGroups.get(0));
        workersAsg = Optional.of(autoScalingGroups.get(1));
      } else {
        masterAsg = Optional.of(autoScalingGroups.get(1));
        workersAsg = Optional.of(autoScalingGroups.get(0));
      }
    }
  }

  if (!clusterId.isPresent()) {
    throw new IOException("Found 2 auto scaling group names for: " + this.clusterName +
        " but tags seem to be corrupted, hence could not determine cluster id");
  }

  if (!masterAsg.isPresent() || !workersAsg.isPresent()) {
    throw new IOException("Found 2 auto scaling group names for: " + this.clusterName +
    " but tags seem to be corrupted, hence could not determine master and workers ASG");
  }

  // Get Master and Workers launch config name and auto scaling group name
  this.masterAutoScalingGroupName = masterAsg.get().getAutoScalingGroupName();
  this.masterLaunchConfigName = masterAsg.get().getLaunchConfigurationName();
  this.workerAutoScalingGroupName = workersAsg.get().getAutoScalingGroupName();
  this.workerLaunchConfigName = workersAsg.get().getLaunchConfigurationName();

  LOGGER.info("Trying to find cluster master public ip");
  this.masterPublicIp = getMasterPublicIp();
  LOGGER.info("Master public ip: "+ this.masterPublicIp);

  return clusterId;
}

private String launchClusterMaster(String uuid, String keyName, String securityGroups,
    AvailabilityZone availabilityZone) {

  // Get cloud-init script to launch cluster master
  final String userData = CloudInitScriptBuilder.buildClusterMasterCommand(this.clusterName,
      this.nfsParentDir,
      this.sinkLogRootDir,
      this.awsConfDir,
      this.appWorkDir,
      this.masterS3ConfUri,
      this.masterS3ConfFiles,
      this.masterS3JarsUri,
      this.masterS3JarsFiles,
      this.masterJarsDir,
      this.masterJvmMemory,
      this.masterJvmArgs,
      this.gobblinVersion);

  // Create launch config for Cluster master
  this.masterLaunchConfigName = MASTER_LAUNCH_CONFIG_NAME_PREFIX + uuid;
  this.awsSdkClient.createLaunchConfig(this.masterLaunchConfigName,
      this.masterAmiId,
      this.masterInstanceType,
      keyName,
      securityGroups,
      Optional.<String>absent(),
      Optional.<String>absent(),
      Optional.<BlockDeviceMapping>absent(),
      Optional.<String>absent(),
      Optional.<InstanceMonitoring>absent(),
      userData);

  // Create ASG for Cluster master
  // TODO: Make size configurable when we have support multi-master
  this.masterAutoScalingGroupName = MASTER_ASG_NAME_PREFIX + uuid;
  final int minNumMasters = 1;
  final int maxNumMasters = 1;
  final int desiredNumMasters = 1;
  final Tag clusterNameTag = new Tag().withKey(CLUSTER_NAME_ASG_TAG).withValue(this.clusterName);
  final Tag clusterUuidTag = new Tag().withKey(CLUSTER_ID_ASG_TAG).withValue(uuid);
  final Tag asgTypeTag = new Tag().withKey(ASG_TYPE_ASG_TAG).withValue(ASG_TYPE_MASTER);
  this.awsSdkClient.createAutoScalingGroup(this.masterAutoScalingGroupName,
      this.masterLaunchConfigName,
      minNumMasters,
      maxNumMasters,
      desiredNumMasters,
      Optional.of(availabilityZone.getZoneName()),
      Optional.<Integer>absent(),
      Optional.<Integer>absent(),
      Optional.<String>absent(),
      Optional.<String>absent(),
      Optional.<String>absent(), Lists.newArrayList(clusterNameTag, clusterUuidTag, asgTypeTag));

  LOGGER.info("Waiting for cluster master to launch");
  this.masterPublicIp = getMasterPublicIp();
  LOGGER.info("Master public ip: "+ this.masterPublicIp);

  return uuid;
}

private void launchWorkUnitRunners(String uuid, String keyName,
    String securityGroups,
    AvailabilityZone availabilityZone) {

  // Get cloud-init script to launch cluster worker
  final String userData = CloudInitScriptBuilder.buildClusterWorkerCommand(this.clusterName,
      this.nfsParentDir,
      this.sinkLogRootDir,
      this.awsConfDir,
      this.appWorkDir,
      this.masterPublicIp,
      this.workerS3ConfUri,
      this.workerS3ConfFiles,
      this.workerS3JarsUri,
      this.workerS3JarsFiles,
      this.workerJarsDir,
      this.workerJvmMemory,
      this.workerJvmArgs,
      this.gobblinVersion);

  // Create launch config for Cluster worker
  this.workerLaunchConfigName = WORKERS_LAUNCH_CONFIG_PREFIX + uuid;
  this.awsSdkClient.createLaunchConfig(this.workerLaunchConfigName,
      this.workerAmiId,
      this.workerInstanceType,
      keyName,
      securityGroups,
      Optional.<String>absent(),
      Optional.<String>absent(),
      Optional.<BlockDeviceMapping>absent(),
      Optional.<String>absent(),
      Optional.<InstanceMonitoring>absent(),
      userData);

  // Create ASG for Cluster workers
  this.workerAutoScalingGroupName = WORKERS_ASG_NAME_PREFIX + uuid;
  final Tag clusterNameTag = new Tag().withKey(CLUSTER_NAME_ASG_TAG).withValue(this.clusterName);
  final Tag clusterUuidTag = new Tag().withKey(CLUSTER_ID_ASG_TAG).withValue(uuid);
  final Tag asgTypeTag = new Tag().withKey(ASG_TYPE_ASG_TAG).withValue(ASG_TYPE_WORKERS);
  this.awsSdkClient.createAutoScalingGroup(this.workerAutoScalingGroupName,
      this.workerLaunchConfigName,
      this.minWorkers,
      this.maxWorkers,
      this.desiredWorkers,
      Optional.of(availabilityZone.getZoneName()),
      Optional.<Integer>absent(),
      Optional.<Integer>absent(),
      Optional.<String>absent(),
      Optional.<String>absent(),
      Optional.<String>absent(),
      Lists.newArrayList(clusterNameTag, clusterUuidTag, asgTypeTag));
}

/***
 * Create and launch an {@link AmazonAutoScaling} group
 *
 * @param groupName Auto scaling group name
 * @param launchConfig Launch configuration string
 * @param minSize Minimum number of instances to maintain in auto scaling group
 * @param maxSize Maximum number of instances to scale up-to for load
 * @param desiredCapacity Desired number of instances to maintain in auto scaling group
 * @param availabilityZones Optional availability zones to make use of
 * @param cooldown Optional cooldown period before any scaling event (default is 300 secs)
 * @param healthCheckGracePeriod Optional grace period till which no health check is performed after bootup (default is 300 secs)
 * @param healthCheckType Optional health check type (default is EC2 instance check)
 * @param loadBalancer Optional load balancer to use
 * @param terminationPolicy Optional termination policies
 * @param tags Optional tags to set on auto scaling group (they are set to propagate to EC2 instances implicitly)
 */
public void createAutoScalingGroup(String groupName,
    String launchConfig,
    Integer minSize, Integer maxSize, Integer desiredCapacity,
    Optional<String> availabilityZones,
    Optional<Integer> cooldown,
    Optional<Integer> healthCheckGracePeriod,
    Optional<String> healthCheckType,
    Optional<String> loadBalancer,
    Optional<String> terminationPolicy,
    List<Tag> tags) {

  AmazonAutoScaling autoScaling = getAmazonAutoScalingClient();

  // Propagate ASG tags to EC2 instances launched under the ASG by default
  // (we want to ensure this, hence not configurable)
  final List<Tag> tagsWithPropagationSet = Lists.newArrayList();
  for (Tag tag : tags) {
    tagsWithPropagationSet.add(tag.withPropagateAtLaunch(true));
  }

  CreateAutoScalingGroupRequest createAutoScalingGroupRequest = new CreateAutoScalingGroupRequest()
      .withAutoScalingGroupName(groupName)
      .withLaunchConfigurationName(launchConfig)
      .withMinSize(minSize)
      .withMaxSize(maxSize)
      .withDesiredCapacity(desiredCapacity)
      .withTags(tagsWithPropagationSet);
  if (availabilityZones.isPresent()) {
    createAutoScalingGroupRequest = createAutoScalingGroupRequest
        .withAvailabilityZones(SPLITTER.splitToList(availabilityZones.get()));
  }
  if (cooldown.isPresent()) {
    createAutoScalingGroupRequest = createAutoScalingGroupRequest
        .withDefaultCooldown(cooldown.get());
  }
  if (healthCheckGracePeriod.isPresent()) {
    createAutoScalingGroupRequest = createAutoScalingGroupRequest
        .withHealthCheckGracePeriod(healthCheckGracePeriod.get());
  }
  if (healthCheckType.isPresent()) {
    createAutoScalingGroupRequest = createAutoScalingGroupRequest
        .withHealthCheckType(healthCheckType.get());
  }
  if (loadBalancer.isPresent()) {
    createAutoScalingGroupRequest = createAutoScalingGroupRequest
        .withLoadBalancerNames(SPLITTER.splitToList(loadBalancer.get()));
  }
  if (terminationPolicy.isPresent()) {
    createAutoScalingGroupRequest = createAutoScalingGroupRequest
        .withTerminationPolicies(SPLITTER.splitToList(terminationPolicy.get()));
  }

  autoScaling.createAutoScalingGroup(createAutoScalingGroupRequest);

  LOGGER.info("Created AutoScalingGroup: " + groupName);
}

@BeforeClass
public void setUp() throws Exception {

  // Mock AWS SDK calls
  MockitoAnnotations.initMocks(this);

  PowerMockito.whenNew(AWSSdkClient.class).withAnyArguments().thenReturn(awsSdkClient);

  Mockito.doNothing()
      .when(awsSdkClient)
      .createSecurityGroup(Mockito.anyString(), Mockito.anyString());
  Mockito.doReturn(Lists.<AvailabilityZone>newArrayList(availabilityZone))
      .when(awsSdkClient)
      .getAvailabilityZones();
  Mockito.doReturn("dummy")
      .when(awsSdkClient)
      .createKeyValuePair(Mockito.anyString());
  Mockito.doReturn(Lists.<AutoScalingGroup>newArrayList(masterASG, workerASG))
      .when(awsSdkClient)
      .getAutoScalingGroupsWithTag(Mockito.any(Tag.class));
  Mockito.doReturn(Lists.<Instance>newArrayList(instance))
      .when(awsSdkClient)
      .getInstancesForGroup(Mockito.anyString(), Mockito.anyString());
  Mockito.doReturn(Lists.<S3ObjectSummary>newArrayList())
      .when(awsSdkClient)
      .listS3Bucket(Mockito.anyString(), Mockito.anyString());
  Mockito.doNothing()
      .when(awsSdkClient)
      .addPermissionsToSecurityGroup(Mockito.any(String.class), Mockito.any(String.class), Mockito.any(String.class),
          Mockito.any(Integer.class), Mockito.any(Integer.class));
  Mockito.doNothing()
      .when(awsSdkClient)
      .createAutoScalingGroup(Mockito.any(String.class), Mockito.any(String.class), Mockito.any(Integer.class),
          Mockito.any(Integer.class), Mockito.any(Integer.class), Mockito.any(Optional.class),
          Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(Optional.class),
          Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(List.class));
  Mockito.doNothing()
      .when(awsSdkClient)
      .createLaunchConfig(Mockito.any(String.class), Mockito.any(String.class), Mockito.any(String.class),
          Mockito.any(String.class), Mockito.any(String.class), Mockito.any(Optional.class),
          Mockito.any(Optional.class), Mockito.any(Optional.class), Mockito.any(Optional.class),
          Mockito.any(Optional.class), Mockito.any(String.class));
  Mockito
      .doNothing()
      .when(awsSdkClient)
      .deleteAutoScalingGroup(Mockito.any(String.class), Mockito.any(boolean.class));
  Mockito
      .doNothing()
      .when(awsSdkClient)
      .deleteLaunchConfiguration(Mockito.any(String.class));
  Mockito.doNothing()
      .when(awsSdkClient)
      .addPermissionsToSecurityGroup(Mockito.any(String.class), Mockito.any(String.class), Mockito.any(String.class),
          Mockito.any(Integer.class), Mockito.any(Integer.class));

  // Local test Zookeeper
  final TestingServer testingZKServer = this.closer.register(new TestingServer(-1));
  LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString());
  this.curatorFramework = TestHelper.createZkClient(testingZKServer, this.closer);

  // Load configuration
  final URL url = GobblinAWSClusterLauncherTest.class.getClassLoader().getResource(
      GobblinAWSClusterLauncherTest.class.getSimpleName() + ".conf");
  Assert.assertNotNull(url, "Could not find resource " + url);
  this.config = ConfigFactory.parseURL(url)
      .withValue("gobblin.cluster.zk.connection.string",
                 ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString()))
      .resolve();
  this.helixClusterName = this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY);

  final String zkConnectionString = this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY);
  this.helixManager = HelixManagerFactory
      .getZKHelixManager(this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY),
          TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectionString);

  // Gobblin AWS Cluster Launcher to test
  this.gobblinAwsClusterLauncher = new GobblinAWSClusterLauncher(this.config);
}