Java 类com.amazonaws.services.ec2.model.DescribeSpotInstanceRequestsRequest 实例源码

/**
 * Checks whether the specified Spot instance request is visible, <em>i.e.</em> can
 * be located by a describe Spot instance request call.
 *
 * @param spotInstanceRequestId the Spot instance request ID
 * @return whether the Spot instance request is visible, <em>i.e.</em> can
 * be located by a describe Spot instance request call
 */
@VisibleForTesting
protected boolean spotInstanceRequestExists(String spotInstanceRequestId) {
  DescribeSpotInstanceRequestsResult result;
  try {
    result = client.describeSpotInstanceRequests(
        new DescribeSpotInstanceRequestsRequest()
            .withSpotInstanceRequestIds(spotInstanceRequestId)
    );
  } catch (AmazonServiceException e) {
    if ("InvalidSpotInstanceRequestID.NotFound".equals(e.getErrorCode())) {
      return false;
    }
    throw AWSExceptions.propagate(e);
  }

  for (SpotInstanceRequest spotInstanceRequest : result.getSpotInstanceRequests()) {
    if (spotInstanceRequestId.equals(spotInstanceRequest.getSpotInstanceRequestId())) {
      return true;
    }
  }
  return false;
}

public SpotState getSpotStatus() {
    if (spotRequestId == null)
        return SpotState.SPOT_REQUEST_NOT_FOUND;

    com.amazonaws.services.ec2.AmazonEC2 client = AmazonEC2.connect();

    DescribeSpotInstanceRequestsRequest spotInstanceReq = new DescribeSpotInstanceRequestsRequest();
    List<String> spotInstanceRequestIds = new ArrayList<String>();
    spotInstanceRequestIds.add(spotRequestId);
    spotInstanceReq.setSpotInstanceRequestIds(spotInstanceRequestIds);
    DescribeSpotInstanceRequestsResult res = client
            .describeSpotInstanceRequests(spotInstanceReq);

    List<SpotInstanceRequest> reqs = res.getSpotInstanceRequests();
    if (reqs.size() > 0) {
        SpotInstanceRequest req = reqs.get(0);
        id = req.getInstanceId();
        return SpotState.valueFromRequest(req);
    } else {
        return SpotState.SPOT_REQUEST_NOT_FOUND;
    }
}

private void checkForSpotRequest(Activity activity, AmazonEC2Client amazonEC2Client) {
    if (WAIT_FOR_SPOT_INSTANCES_STATUS_CODE.equals(activity.getStatusCode())) {
        Pattern pattern = Pattern.compile(SPOT_ID_PATTERN);
        Matcher matcher = pattern.matcher(activity.getStatusMessage());
        if (matcher.find()) {
            String spotId = matcher.group(0);
            DescribeSpotInstanceRequestsResult spotResult = amazonEC2Client.describeSpotInstanceRequests(
                    new DescribeSpotInstanceRequestsRequest().withSpotInstanceRequestIds(spotId));
            Optional<SpotInstanceRequest> request = spotResult.getSpotInstanceRequests().stream().findFirst();
            if (request.isPresent()) {
                if (LOW_SPOT_PRICE_STATUS_CODE.equals(request.get().getStatus().getCode())) {
                    throw new CloudConnectorException(request.get().getStatus().getMessage());
                }
            }

        }
    }
}

@Override
public void execute(AmazonEC2 client, Pool pool, DelegateExecution execution) {
    LOG.info(">> retrieving instance Ids from spot request Ids");

    @SuppressWarnings("unchecked")
    List<String> requestIds =
        (List<String>) execution.getVariable(ProcessVariables.SPOT_INSTANCE_REQUEST_IDS);
    DescribeSpotInstanceRequestsResult result = client.describeSpotInstanceRequests(
        new DescribeSpotInstanceRequestsRequest().withSpotInstanceRequestIds(requestIds));

    List<String> instanceIds = new ArrayList<String>();
    for (SpotInstanceRequest spotRequest : result.getSpotInstanceRequests()) {
        if (spotRequest.getInstanceId() != null) {
            instanceIds.add(spotRequest.getInstanceId());
        }
    }

    execution.setVariable(ProcessVariables.INSTANCE_IDS, instanceIds);
}

@Override
public void execute(AmazonEC2 client, Pool pool, DelegateExecution execution) {

    LOG.info(">> Checking if all spot requests match predicate {}", predicate);

    @SuppressWarnings("unchecked")
    List<String> requestIds = (List<String>) execution.getVariable(ProcessVariables.SPOT_INSTANCE_REQUEST_IDS);
    checkNotNull(requestIds, "process variable '{}' not found", ProcessVariables.SPOT_INSTANCE_REQUEST_IDS);

    DescribeSpotInstanceRequestsRequest describeRequest = new DescribeSpotInstanceRequestsRequest();
    describeRequest.setSpotInstanceRequestIds(requestIds);

    // Retrieve all of the requests we want to monitor.
    DescribeSpotInstanceRequestsResult describeResult = client.describeSpotInstanceRequests(describeRequest);
    List<SpotInstanceRequest> requests = describeResult.getSpotInstanceRequests();

    if (Iterables.all(requests, predicate)) {
        LOG.info(">> All {} requests match predicate {} ", requests, predicate);
        execution.setVariable(resultVariable, true);
    } else {
        LOG.info("<< Not all requests {} match predicate {}", requests, predicate);
        execution.setVariable(resultVariable, false);
    }
}

public void shutdownInstances() {
    AmazonEC2 ec2 = new AmazonEC2Client(getAwsCredentials());
    ec2.setEndpoint(getOrCry("ec2endpoint"));

    try {
        // cancel spot request, so no new instances will be launched
        DescribeSpotInstanceRequestsRequest describeRequest = new DescribeSpotInstanceRequestsRequest();
        DescribeSpotInstanceRequestsResult describeResult = ec2
                .describeSpotInstanceRequests(describeRequest);
        List<SpotInstanceRequest> describeResponses = describeResult
                .getSpotInstanceRequests();
        List<String> spotRequestIds = new ArrayList<String>();
        List<String> instanceIds = new ArrayList<String>();

        for (SpotInstanceRequest describeResponse : describeResponses) {
            spotRequestIds.add(describeResponse.getSpotInstanceRequestId());
            if ("active".equals(describeResponse.getState())) {
                instanceIds.add(describeResponse.getInstanceId());
            }
        }
        ec2.cancelSpotInstanceRequests(new CancelSpotInstanceRequestsRequest()
                .withSpotInstanceRequestIds(spotRequestIds));
        log.info("Cancelled spot request");

        if (instanceIds.size() > 0) {
            ec2.terminateInstances(new TerminateInstancesRequest(
                    instanceIds));
            log.info("Shut down " + instanceIds.size() + " instances");
        }

    } catch (Exception e) {
        log.warn("Failed to shutdown instances - ", e);
    }
}

public List<SpotInstanceOfferResult> getStatusOfSpotInstanceOffers(String... spotInstanceOfferIds)
{
    DescribeSpotInstanceRequestsResult describeSpotInstanceResult = ec2_.describeSpotInstanceRequests(new DescribeSpotInstanceRequestsRequest()
            .withSpotInstanceRequestIds(spotInstanceOfferIds));

    return toSpotInstanceResults(null, describeSpotInstanceResult.getSpotInstanceRequests());
}

public List<String> getActiveSpotInstanceId(
        List<String> spotInstanceRequestIds) {
    DescribeSpotInstanceRequestsRequest describeRequest = new DescribeSpotInstanceRequestsRequest();
    describeRequest.setSpotInstanceRequestIds(spotInstanceRequestIds);

    System.out
            .println("Checking to determine if Spot Bids have reached the active state...");

    List<String> instanceIds = new ArrayList<String>();

    try {
        DescribeSpotInstanceRequestsResult describeResult = ec2
                .describeSpotInstanceRequests(describeRequest);
        List<SpotInstanceRequest> describeResponses = describeResult
                .getSpotInstanceRequests();

        for (SpotInstanceRequest describeResponse : describeResponses) {
            System.out.println(" "
                    + describeResponse.getSpotInstanceRequestId()
                    + " is in the " + describeResponse.getState()
                    + " state.");

            if (describeResponse.getState().equals("open"))
                return null;

            if (describeResponse.getState().equals("active"))
                instanceIds.add(describeResponse.getInstanceId());
        }
    } catch (AmazonServiceException e) {
        System.out.println("Error when calling describeSpotInstances");
        System.out.println("Caught Exception: " + e.getMessage());
        System.out.println("Reponse Status Code: " + e.getStatusCode());
        System.out.println("Error Code: " + e.getErrorCode());
        System.out.println("Request ID: " + e.getRequestId());

        return null;
    }
    return instanceIds;
}

@Override
public SpotInstanceRequest call() throws AmazonClientException {
    DescribeSpotInstanceRequestsRequest request = new DescribeSpotInstanceRequestsRequest();
    request.withSpotInstanceRequestIds(this.spotInstanceRequestId);
    DescribeSpotInstanceRequestsResult result = getClient().getApi().describeSpotInstanceRequests(request);
    return Iterables.getOnlyElement(result.getSpotInstanceRequests());
}

@Override
public List<SpotInstanceRequest> call() throws AmazonClientException {
    DescribeSpotInstanceRequestsRequest request = new DescribeSpotInstanceRequestsRequest();
    if (this.spotRequestIds != null) {
        request.withSpotInstanceRequestIds(this.spotRequestIds);
    }
    if (this.filters != null) {
        request.withFilters(this.filters);
    }
    DescribeSpotInstanceRequestsResult result = getClient().getApi().describeSpotInstanceRequests(request);
    return result.getSpotInstanceRequests();
}

@Override
public List<AbstractResource<?>> describeSpotInstanceRequests(Account account, Region region, DateTime dt, Ec2Filter... filters) {
    AmazonEC2 ec2 = findClient(account, region);

    DescribeSpotInstanceRequestsRequest req = new DescribeSpotInstanceRequestsRequest();
    for (Ec2Filter filter : filters) {
        Filter f = new Filter().withName(filter.getName()).withValues(filter.getValues());
        req.withFilters(f);
    }

    log.debug("start describing spot instance requests for account:{} in region:{} via api", account.getId() + "=>" + account.getName(), region);
    DescribeSpotInstanceRequestsResult res = ec2.describeSpotInstanceRequests(req);

    return converter.toEc2SpotInstanceRequest(res.getSpotInstanceRequests(), account.getId(), region, dt);
}

@Override
public void execute(AmazonEC2 client, Pool pool, DelegateExecution execution) throws IOException {
    /* before sending a new request, we check to see if we already registered
       a launch group with the process ID, if yes, we don't re-send the request */
    final String businessKey = execution.getProcessBusinessKey();

    /* we timeout if requests have already been sent - the activity is being retried. */
    Optional<Object> alreadySent = Optional.fromNullable(
        execution.getVariable(ProcessVariables.SPOT_INSTANCE_REQUEST_IDS));

    if (alreadySent.isPresent()) {
        DescribeSpotInstanceRequestsRequest describeRequest = new DescribeSpotInstanceRequestsRequest()
            .withFilters(new Filter()
                .withName("launch-group").withValues(businessKey)
                .withName("state").withValues("open", "active"));

        Stopwatch stopwatch = new Stopwatch().start();
        while (stopwatch.elapsedTime(TimeUnit.MINUTES) < 2) {
            DescribeSpotInstanceRequestsResult result = client.describeSpotInstanceRequests(describeRequest);
            List<SpotInstanceRequest> pending = result.getSpotInstanceRequests();
            if (pending.size() > 0) {
                LOG.info("Not resending spot instance requests {} for businessKey: {}.", pending, businessKey);
                execution.setVariable(ProcessVariables.SPOT_INSTANCE_REQUEST_IDS,
                    collectSpotInstanceRequestIds(pending));
                return;
            }
            LOG.info("The describe call has not returned anything yet, waiting 20s and retrying.");
            Uninterruptibles.sleepUninterruptibly(20, TimeUnit.SECONDS);
        }
    }

    final RequestSpotInstancesRequest request = createSpotInstancesRequest(pool, execution);
    execution.setVariable(ProcessVariables.SPOT_REQUESTS_SENT, true);
    RequestSpotInstancesResult requestResult = client.requestSpotInstances(request);
    List<String> spotInstanceRequestIds = collectSpotInstanceRequestIds(requestResult.getSpotInstanceRequests());

    execution.setVariable(ProcessVariables.SPOT_INSTANCE_REQUEST_IDS, spotInstanceRequestIds);
}

@Test
public void testRunSpotInstances() throws Exception {
    ProcessVariablesCollector collector = new ProcessVariablesCollector();
    collector.install(execution);

    activity.execute(execution);

    @SuppressWarnings("unchecked")
    ArgumentCaptor<List<String>> argument = (ArgumentCaptor<List<String>>) 
            (Object) ArgumentCaptor.forClass(List.class);
    verify(execution).setVariable(eq(ProcessVariables.SPOT_INSTANCE_REQUEST_IDS), argument.capture());
    when(execution.getVariable(ProcessVariables.SPOT_INSTANCE_REQUEST_IDS)).thenReturn(argument.getValue());
    /* The timeout is needed because the describe calls don't return immediately. */
    // TODO: see if we can eliminate this after adding the process variables conditions
    Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MINUTES);

    // shouldn't do anything
    activity.execute(execution);

    Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MINUTES);

    DescribeSpotInstanceRequestsResult result = client.describeSpotInstanceRequests(
            new DescribeSpotInstanceRequestsRequest().withFilters(new Filter()
                .withName("launch-group").withValues(BUSINESS_KEY)));

    assertThat(result.getSpotInstanceRequests()).hasSize(1);
    /* we also need to sleep before the teardown */
    Uninterruptibles.sleepUninterruptibly(1, TimeUnit.MINUTES);
}

@Override
public DescribeSpotInstanceRequestsResult describeSpotInstanceRequests(DescribeSpotInstanceRequestsRequest describeSpotInstanceRequestsRequest) throws AmazonServiceException, AmazonClientException {
    throw new UnsupportedOperationException("Not supported in mock");
}

/**
 * Identifies reusable Spot instance requests orphaned by a previous call.
 *
 * @return the reusable Spot instance requests orphaned by a previous call
 */
@VisibleForTesting
protected Set<String> checkForOrphanedSpotInstanceRequests() {

  Set<String> orphanedSpotInstanceRequests = Sets.newHashSet();

  LOG.info(">> Checking for orphaned Spot instance requests");
  String idTagName = ec2TagHelper.getClouderaDirectorIdTagName();
  DescribeSpotInstanceRequestsRequest describeSpotInstanceRequestsRequest =
      new DescribeSpotInstanceRequestsRequest().withFilters(
          new Filter()
              .withName("tag:" + idTagName)
              .withValues(virtualInstanceIds));
  DescribeSpotInstanceRequestsResult describeSpotInstanceRequestsResult =
      client.describeSpotInstanceRequests(describeSpotInstanceRequestsRequest);
  for (SpotInstanceRequest existingSpotInstanceRequest :
      describeSpotInstanceRequestsResult.getSpotInstanceRequests()) {
    String spotInstanceRequestId = existingSpotInstanceRequest.getSpotInstanceRequestId();
    String virtualInstanceId = null;
    for (Tag tag : existingSpotInstanceRequest.getTags()) {
      if (idTagName.equals(tag.getKey())) {
        virtualInstanceId = tag.getValue();
      }
    }
    if (virtualInstanceId == null) {
      LOG.warn(">> Orphaned Spot instance request {} has no virtual instance id",
          spotInstanceRequestId);
    } else {
      SpotAllocationRecord spotAllocationRecord = getSpotAllocationRecord(virtualInstanceId);
      SpotInstanceState spotInstanceState =
          SpotInstanceState.fromValue(existingSpotInstanceRequest.getState());
      switch (spotInstanceState) {
        case Active:
          spotAllocationRecord.spotInstanceRequestId = spotInstanceRequestId;
          String ec2InstanceId = existingSpotInstanceRequest.getInstanceId();
          LOG.info(">> Reusing fulfilled orphaned Spot instance request {} / {} / {}",
              spotInstanceRequestId, virtualInstanceId, ec2InstanceId);
          if (spotAllocationRecord.ec2InstanceId == null) {
            spotAllocationRecord.ec2InstanceId = ec2InstanceId;
          }
          break;
        case Cancelled:
        case Closed:
        case Failed:
          break;
        default:
          if (existingSpotInstanceRequest.getValidUntil().getTime() > System.currentTimeMillis()) {
            LOG.info(">> Reusing pending orphaned Spot instance request {} / {}",
                spotInstanceRequestId, virtualInstanceId);
            spotAllocationRecord.spotInstanceRequestId = spotInstanceRequestId;
          }
          break;
      }
    }
  }

  return orphanedSpotInstanceRequests;
}

@Override
public Collection<PlannedNode> provision(Label label, int excessWorkload) {
    try {

        List<Node> nodes = Hudson.getInstance().getNodes();
        for (int i = 0, len = nodes.size(); i < len; i++) {
            if (!(nodes.get(i) instanceof EC2SpotSlave)) {
                continue;
            }
            EC2SpotSlave n = (EC2SpotSlave) nodes.get(i);
            //If the slave is online then it is already counted by Jenkins
            // We only want to count potential additional Spot instance slaves
            if (n.getComputer().isOffline() && label.matches(n.getAssignedLabels())) {
                DescribeSpotInstanceRequestsRequest dsir
                        = new DescribeSpotInstanceRequestsRequest().withSpotInstanceRequestIds(n.getSpotInstanceRequestId());

                for (SpotInstanceRequest sir : connect().describeSpotInstanceRequests(dsir).getSpotInstanceRequests()) {
                    // Count Spot requests that are open and still have a chance to be active
                    // A request can be active and not yet registered as a slave. We check above
                    // to ensure only unregistered slaves get counted
                    if (sir.getState().equals("open") || sir.getState().equals("active")) {
                        excessWorkload -= n.getNumExecutors();
                    }
                }
            }
        }
        LOGGER.log(Level.INFO, "Excess workload after pending Spot instances: " + excessWorkload);

        List<PlannedNode> r = new ArrayList<PlannedNode>();

        final SlaveTemplate t = getTemplate(label);
        int amiCap = t.getInstanceCap();

        while (excessWorkload > 0) {

            if (!addProvisionedSlave(t.ami, amiCap)) {
                break;
            }

            r.add(new PlannedNode(t.getDisplayName(),
                    Computer.threadPoolForRemoting.submit(new Callable<Node>() {
                        public Node call() throws Exception {
                            // TODO: record the output somewhere
                            try {
                                EC2AbstractSlave s = t.provision(new StreamTaskListener(System.out));
                                Hudson.getInstance().addNode(s);
                                // EC2 instances may have a long init script. If we declare
                                // the provisioning complete by returning without the connect
                                // operation, NodeProvisioner may decide that it still wants
                                // one more instance, because it sees that (1) all the slaves
                                // are offline (because it's still being launched) and
                                // (2) there's no capacity provisioned yet.
                                //
                                // deferring the completion of provisioning until the launch
                                // goes successful prevents this problem.
                                s.toComputer().connect(false).get();
                                return s;
                            } finally {
                                decrementAmiSlaveProvision(t.ami);
                            }
                        }
                    }), t.getNumExecutors()));

            excessWorkload -= t.getNumExecutors();

        }
        return r;
    } catch (AmazonClientException e) {
        LOGGER.log(Level.WARNING, "Failed to count the # of live instances on EC2", e);
        return Collections.emptyList();
    }
}