public boolean isInstanceReady(String instanceId) { LOGGER.debug("isInstanceReady('{}') entered", instanceId); DescribeInstanceStatusResult result = getEC2() .describeInstanceStatus( new DescribeInstanceStatusRequest() .withInstanceIds(instanceId)); List<InstanceStatus> statusList = result.getInstanceStatuses(); boolean instanceStatus = false; boolean systemStatus = false; for (InstanceStatus status : statusList) { LOGGER.debug(" InstanceState: {}", status.getInstanceState()); LOGGER.debug(" InstanceStatus: {}", status.getInstanceStatus() .getStatus()); LOGGER.debug(" SystemStatus: {}", status.getSystemStatus() .getStatus()); LOGGER.debug(" AvailabilityZone: {}", status.getAvailabilityZone()); instanceStatus = ("ok".equals(status.getInstanceStatus() .getStatus())); systemStatus = ("ok".equals(status.getSystemStatus().getStatus())); } LOGGER.debug("isInstanceReady('{}') left", instanceId); return instanceStatus && systemStatus; }
public void createDescribeInstanceStatusResult(String instanceId, String stateName, String instanceStatusName, String systemStatusName) { InstanceState state = new InstanceState().withName(stateName); InstanceStatusSummary instanceSummary = new InstanceStatusSummary() .withStatus(instanceStatusName); InstanceStatusSummary systemSummary = new InstanceStatusSummary() .withStatus(systemStatusName); InstanceStatus instanceStatus = new com.amazonaws.services.ec2.model.InstanceStatus() .withInstanceId(instanceId).withInstanceState(state) .withInstanceStatus(instanceSummary) .withSystemStatus(systemSummary); DescribeInstanceStatusResult instanceStatusResult = new DescribeInstanceStatusResult() .withInstanceStatuses(instanceStatus); doReturn(instanceStatusResult).when(ec2).describeInstanceStatus( any(DescribeInstanceStatusRequest.class)); }
public void addDescribeInstanceStatusResult(String instanceId, String stateName, String instanceStatusName, String systemStatusName) { InstanceState state = new InstanceState().withName(stateName); InstanceStatusSummary instanceSummary = new InstanceStatusSummary() .withStatus(instanceStatusName); InstanceStatusSummary systemSummary = new InstanceStatusSummary() .withStatus(systemStatusName); com.amazonaws.services.ec2.model.InstanceStatus instanceStatus = new com.amazonaws.services.ec2.model.InstanceStatus() .withInstanceId(instanceId).withInstanceState(state) .withInstanceStatus(instanceSummary) .withSystemStatus(systemSummary); DescribeInstanceStatusResult instanceStatusResult = new DescribeInstanceStatusResult() .withInstanceStatuses(instanceStatus); answerDescribeInstanceStatus.add(instanceStatusResult); }
@Test public void itShouldStartAnEC2InstanceFromAnAMI() throws ServiceBrokerException { when( ec2Client.runInstances(awsRqst(r -> r.getImageId().equals( "test_image")))).thenReturn(runInstanceResult); when(ec2Client.describeAddresses()).thenReturn( new DescribeAddressesResult().withAddresses(Collections .singleton(new Address().withPublicIp("10.10.10.10")))); when(ec2Client.describeInstanceStatus(any())).thenReturn( new DescribeInstanceStatusResult() .withInstanceStatuses(Collections .singleton(new InstanceStatus() .withInstanceState(new InstanceState() .withName("running"))))); when(hostUtils.waitForBoot(anyString(), anyInt())).thenReturn(true); assertThat(aws.startEC2Instance("test_image"), is(equalTo("test_instance"))); }
@Test public void testStartInstance() { final DescribeInstanceStatusRequest describeInstanceStatusRequest = new DescribeInstanceStatusRequest().withIncludeAllInstances(true).withInstanceIds(INSTANCE_ID); final DescribeInstanceStatusResult describeInstanceStatusResult = new DescribeInstanceStatusResult().withInstanceStatuses(new InstanceStatus().withInstanceState(new InstanceState().withName(InstanceStateName.Stopped))); final StartInstancesRequest startInstancesRequest = new StartInstancesRequest().withInstanceIds(INSTANCE_ID); final StartInstancesResult startInstancesResult = new StartInstancesResult().withStartingInstances(new InstanceStateChange().withCurrentState(new InstanceState().withName(InstanceStateName.Running))); Mockito.doReturn(describeInstanceStatusResult).when(amazonEC2Client).describeInstanceStatus(describeInstanceStatusRequest); Mockito.doReturn(startInstancesResult).when(amazonEC2Client).startInstances(startInstancesRequest); amazonEC2Service.startInstance(INSTANCE_ID); final InOrder inOrder = Mockito.inOrder(amazonEC2Client); inOrder.verify(amazonEC2Client).describeInstanceStatus(describeInstanceStatusRequest); inOrder.verify(amazonEC2Client).startInstances(startInstancesRequest); }
@Test public void testStopInstanceStopping() { final DescribeInstanceStatusRequest describeInstanceStatusRequest = new DescribeInstanceStatusRequest().withIncludeAllInstances(true).withInstanceIds(INSTANCE_ID); final DescribeInstanceStatusResult describeInstanceStatusResult = new DescribeInstanceStatusResult().withInstanceStatuses(new InstanceStatus().withInstanceState(new InstanceState().withName(InstanceStateName.Running))); final StopInstancesRequest stopInstancesRequest = new StopInstancesRequest().withInstanceIds(INSTANCE_ID); final StopInstancesResult stopInstancesResult = new StopInstancesResult().withStoppingInstances(new InstanceStateChange().withCurrentState(new InstanceState().withName(InstanceStateName.Stopping))); Mockito.doReturn(describeInstanceStatusResult).when(amazonEC2Client).describeInstanceStatus(describeInstanceStatusRequest); Mockito.doReturn(stopInstancesResult).when(amazonEC2Client).stopInstances(stopInstancesRequest); amazonEC2Service.stopInstance(INSTANCE_ID); final InOrder inOrder = Mockito.inOrder(amazonEC2Client); inOrder.verify(amazonEC2Client).describeInstanceStatus(describeInstanceStatusRequest); inOrder.verify(amazonEC2Client).stopInstances(stopInstancesRequest); }
@Test public void testStopInstanceStopped() { final DescribeInstanceStatusRequest describeInstanceStatusRequest = new DescribeInstanceStatusRequest().withIncludeAllInstances(true).withInstanceIds(INSTANCE_ID); final DescribeInstanceStatusResult describeInstanceStatusResult = new DescribeInstanceStatusResult().withInstanceStatuses(new InstanceStatus().withInstanceState(new InstanceState().withName(InstanceStateName.Running))); final StopInstancesRequest stopInstancesRequest = new StopInstancesRequest().withInstanceIds(INSTANCE_ID); final StopInstancesResult stopInstancesResult = new StopInstancesResult().withStoppingInstances(new InstanceStateChange().withCurrentState(new InstanceState().withName(InstanceStateName.Stopped))); Mockito.doReturn(describeInstanceStatusResult).when(amazonEC2Client).describeInstanceStatus(describeInstanceStatusRequest); Mockito.doReturn(stopInstancesResult).when(amazonEC2Client).stopInstances(stopInstancesRequest); amazonEC2Service.stopInstance(INSTANCE_ID); final InOrder inOrder = Mockito.inOrder(amazonEC2Client); inOrder.verify(amazonEC2Client).describeInstanceStatus(describeInstanceStatusRequest); inOrder.verify(amazonEC2Client).stopInstances(stopInstancesRequest); }
/** * This function is a wrapper to the ec2 describeInstanceStatus function. * * @param region * @return List of InstanceStatus */ public List<InstanceStatus> describeInstancesStatusAsync(Region region) throws Exception { Preconditions.checkNotNull(region); List<InstanceStatus> statusList = new ArrayList<>(); List<AvailabilityZone> zones = getAvailabilityZones(region); AmazonEC2Client client = getClient(region); ExecutorService executor = Executors.newFixedThreadPool(zones.size()); OperationStats op = new OperationStats("ec2InstanceStore", "describeInstancesStatusAsync"); try { List<Callable<List<InstanceStatus>>> retrieveFunction = new ArrayList<>(zones.size()); for (AvailabilityZone zone : zones) { retrieveFunction.add(new Callable<List<InstanceStatus>>() { @Override public List<InstanceStatus> call() throws Exception { return getInstancesStatusByZone(zone, client); } }); } List<Future<List<InstanceStatus>>> futures = executor.invokeAll(retrieveFunction); for (Future<List<InstanceStatus>> future : futures) { statusList.addAll(future.get()); } op.succeed(); } catch (Exception ex) { op.failed(); throw ex; } finally { executor.shutdown(); } return statusList; }
private List<InstanceStatus> getInstancesStatusByZone(AvailabilityZone zone, AmazonEC2Client client) { // Create this list to capture paginated async results from aws sdk List<InstanceStatus> statusList = new ArrayList<>(); // Create an initial request object DescribeInstanceStatusRequest statusRequest = new DescribeInstanceStatusRequest() .withMaxResults(1000) .withFilters(new Filter("availability-zone", Arrays.asList(zone.getZoneName()))) .withSdkClientExecutionTimeout(600 * 1000) .withSdkRequestTimeout(300 * 1000); // Make the request for instanceStatus DescribeInstanceStatusResult result = client.describeInstanceStatus(statusRequest); // Until more results are available we loop through this code while (result != null) { statusList.addAll(result.getInstanceStatuses()); if (result.getNextToken() != null) { statusRequest.setNextToken(result.getNextToken()); result = client.describeInstanceStatus(statusRequest); } else { result = null; } } // Return all statuses as a list of InstanceStatus objects return statusList; }
/** * Waits until the instance has entered a running state. * * @param ec2InstanceId the EC2 instance id * @return true if the instance has entered a running state, false if the instance is shutting down/terminated or * the function has timed out waiting for the instance to enter one of these two states. */ private boolean waitUntilInstanceHasStarted(String ec2InstanceId) throws InterruptedException { // TODO: Add a timeout to this loop. while (true) { try { DescribeInstanceStatusResult result = client.describeInstanceStatus( new DescribeInstanceStatusRequest() .withIncludeAllInstances(true) .withInstanceIds(ec2InstanceId) ); for (InstanceStatus status : result.getInstanceStatuses()) { InstanceStateName currentState = InstanceStateName.fromValue(status.getInstanceState().getName()); if (ec2InstanceId.equals(status.getInstanceId())) { if (currentState.equals(InstanceStateName.Terminated) || currentState.equals(InstanceStateName.ShuttingDown)) { LOG.error("Instance {} has unexpectedly terminated", ec2InstanceId); return false; } else if (!currentState.equals(InstanceStateName.Pending)) { return true; } } } } catch (AmazonServiceException e) { if (!INVALID_INSTANCE_ID_NOT_FOUND.equals(e.getErrorCode())) { AWSExceptions.propagate(e); } } TimeUnit.SECONDS.sleep(5); } }
public void waitUntilRunning(List<String> instanceIds) throws InterruptedException { int attempts = 0; while (true) { attempts++; Threads.sleepRoughly(Duration.ofSeconds(30)); List<InstanceStatus> statuses = ec2.describeInstanceStatus(new DescribeInstanceStatusRequest() .withInstanceIds(instanceIds)).getInstanceStatuses(); if (statuses.size() < instanceIds.size()) { logger.info("status is not synced, continue to wait"); continue; } for (InstanceStatus status : statuses) { logger.info("instance status {} => {}, checks => {}, {}", status.getInstanceId(), status.getInstanceState().getName(), status.getSystemStatus().getStatus(), status.getInstanceStatus().getStatus()); } boolean allOK = statuses.stream().allMatch(status -> "running".equalsIgnoreCase(status.getInstanceState().getName()) && "ok".equalsIgnoreCase(status.getSystemStatus().getStatus()) && "ok".equalsIgnoreCase(status.getInstanceStatus().getStatus())); if (allOK) { break; } else if (attempts > 20) { // roughly after 10 mins throw new Error("waited too long to get instance status, something is wrong, please check aws console"); } } }
private void waitUntilInstanceRunning(List<String> newInstanceIds) throws InterruptedException { while (true) { Threads.sleepRoughly(Duration.ofSeconds(30)); List<InstanceStatus> statuses = AWS.ec2.ec2.describeInstanceStatus(new DescribeInstanceStatusRequest() .withInstanceIds(newInstanceIds)).getInstanceStatuses(); for (InstanceStatus status : statuses) { logger.info("instance status {} => {}, checks => {}, {}", status.getInstanceId(), status.getInstanceState().getName(), status.getSystemStatus().getStatus(), status.getInstanceStatus().getStatus()); } boolean allOK = statuses.stream().allMatch(status -> "running".equalsIgnoreCase(status.getInstanceState().getName()) && "ok".equalsIgnoreCase(status.getSystemStatus().getStatus()) && "ok".equalsIgnoreCase(status.getInstanceStatus().getStatus()) ); if (allOK) { logger.info("all new instances are running"); break; } else { logger.info("continue to wait, not all new instances are running"); } } }
@Test(expected = AmazonClientException.class) public void testStartInstanceAlreadyRunning() { final DescribeInstanceStatusRequest describeInstanceStatusRequest = new DescribeInstanceStatusRequest().withIncludeAllInstances(true).withInstanceIds(INSTANCE_ID); final DescribeInstanceStatusResult describeInstanceStatusResult = new DescribeInstanceStatusResult().withInstanceStatuses(new InstanceStatus().withInstanceState(new InstanceState().withName(InstanceStateName.Running))); Mockito.doReturn(describeInstanceStatusResult).when(amazonEC2Client).describeInstanceStatus(describeInstanceStatusRequest); try { amazonEC2Service.startInstance(INSTANCE_ID); } finally { Mockito.verify(amazonEC2Client).describeInstanceStatus(describeInstanceStatusRequest); } }
@Test(expected = AmazonClientException.class) public void testStopInstanceAlreadyStopped() { final DescribeInstanceStatusRequest describeInstanceStatusRequest = new DescribeInstanceStatusRequest().withIncludeAllInstances(true).withInstanceIds(INSTANCE_ID); final DescribeInstanceStatusResult describeInstanceStatusResult = new DescribeInstanceStatusResult().withInstanceStatuses(new InstanceStatus().withInstanceState(new InstanceState().withName(InstanceStateName.Stopped))); Mockito.doReturn(describeInstanceStatusResult).when(amazonEC2Client).describeInstanceStatus(describeInstanceStatusRequest); try { amazonEC2Service.stopInstance(INSTANCE_ID); } finally { Mockito.verify(amazonEC2Client).describeInstanceStatus(describeInstanceStatusRequest); } }
@Test(expected = AmazonClientException.class) public void testGetInstanceStatusMultipleInstances() { final DescribeInstanceStatusRequest describeInstanceStatusRequest = new DescribeInstanceStatusRequest().withIncludeAllInstances(true).withInstanceIds(INSTANCE_ID); final DescribeInstanceStatusResult describeInstanceStatusResult = new DescribeInstanceStatusResult().withInstanceStatuses(new InstanceStatus().withInstanceState(new InstanceState().withName(InstanceStateName.Running)), new InstanceStatus().withInstanceState(new InstanceState().withName(InstanceStateName.Running))); Mockito.doReturn(describeInstanceStatusResult).when(amazonEC2Client).describeInstanceStatus(describeInstanceStatusRequest); try { amazonEC2Service.getInstanceStatus(INSTANCE_ID); } finally { Mockito.verify(amazonEC2Client).describeInstanceStatus(describeInstanceStatusRequest); } }
public InstanceStatus getInstanceStatus(AmazonEC2Client ec2Client, String instanceId) { List<String> instanceIds = new ArrayList<String>(); instanceIds.add(instanceId); DescribeInstanceStatusRequest statusReq = new DescribeInstanceStatusRequest(); statusReq.setInstanceIds(instanceIds); DescribeInstanceStatusResult result = ec2Client.describeInstanceStatus(statusReq); List<InstanceStatus> statuses = result.getInstanceStatuses(); if (statuses == null || statuses.size() == 0) { return null; } return statuses.iterator().next(); }
public List<String> dumpEc2InstanceStatus(AmazonEC2Client ec2Client, List<Instance> ec2Instances) { try { List<String> instanceIds = new ArrayList<String>(); for (Instance ec2node : ec2Instances) { instanceIds.add(ec2node.getInstanceId()); } DescribeInstanceStatusRequest statusReq = new DescribeInstanceStatusRequest(); statusReq.setInstanceIds(instanceIds); DescribeInstanceStatusResult result = ec2Client.describeInstanceStatus(statusReq); List<InstanceStatus> statuses = result.getInstanceStatuses(); List<String> impairedInstances = new ArrayList<String>(); StringBuffer sb = new StringBuffer("EC2 worker instance STATUS:\n"); for (InstanceStatus status : statuses) { sb.append("\tid:"+status.getInstanceId() + "\taz:" + status.getAvailabilityZone() + "\tstate:" + status.getInstanceState().getName() + "\tstatus:" + status.getInstanceStatus().getStatus() + "\tsystem_status: " + status.getSystemStatus().getStatus() + "\n"); if (status.getInstanceStatus().getStatus().equalsIgnoreCase("impaired")) { impairedInstances.add(status.getInstanceId()); } } logger.info(sb.toString()+"\n"); return impairedInstances; } catch(Exception e) { logger.error("Error getting instance state: " + e.getMessage(),e); return null; } }
private List<InstanceStatus> returnInstanceStatus(int start, int end) { List<InstanceStatus> instanceStatuses = new ArrayList<>(); for (int i = start; i < end; i++) { InstanceStatus instanceStatus = new InstanceStatus(); InstanceState instanceState = new InstanceState(); instanceState.setCode(16); instanceStatus.setInstanceState(instanceState); instanceStatus.setInstanceId(Integer.toString(i)); instanceStatuses.add(instanceStatus); } return instanceStatuses; }
public InstanceStatus getRaw() { return raw; }
public void setRaw(InstanceStatus raw) { this.raw = raw; }
private Response getAwsInstancesStatus(List<String> instanceIds) throws Exception { Map<String, Integer> statuses = new HashMap<>(); for (String instanceId : instanceIds) { int intStatus = 0; EsAwsStatus status = cmdbInstanceStore.getAwsStatus(instanceId); if (status == null) { // Instance Status not found in cmdbStore . Mark status as -1 logger.info("No status found for instanceId {}", instanceId); intStatus = -1; } else { AwsStatus awsStatus = status.getAwsStatus(); if (awsStatus != null) { List<String> codes = awsStatus.getCodes(); InstanceStatus raw = awsStatus.getRaw(); String instanceStatus = raw.getInstanceStatus().getStatus(); String systemStatus = raw.getSystemStatus().getStatus(); if (codes != null) { if (StringUtils.equalsIgnoreCase(instanceStatus, "initializing") || StringUtils.equalsIgnoreCase(systemStatus, "initializing")) { // codes are defined but one of the status is initializing logger.info("Aws Status requested. Instance is still initializing..."); intStatus = 0; } else { // codes are present and system or instance status is abnormal logger.warn("Unhealthy instance was found. {} with status {} and {}", instanceId, instanceStatus, systemStatus); intStatus = 1; } } } } // Add trimmed status of the instance to statuses Map statuses.put(instanceId, intStatus); } return Response.status(Response.Status.OK) .type(MediaType.APPLICATION_JSON) .entity(statuses) .build(); }
private Boolean checkInstancesStatus() { try { // Type cast cloudInstanceStore to use ec2 functions Ec2InstanceStore ec2Store = (Ec2InstanceStore) cloudInstanceStore; // Fetch aws status for all instances in the given region List<InstanceStatus> statuses = ec2Store.describeInstancesStatusAsync(region); if (statuses == null) { logger.warn("AWS did not return any InstanceStatus"); return false; } // Creates a reverse map of id to Status Map<String, InstanceStatus> idToInstanceStatus = new HashMap<>(statuses.size()); for (InstanceStatus status : statuses) { idToInstanceStatus.put(status.getInstanceId(), status); } // Iterate over all Running CmdbInstances & add status updated ones to list Iterator<EsAwsStatus> iterator = cmdbInstanceStore.getRunningAndTerminatedAwsStatus(region, 1); List<EsAwsStatus> updateInstanceList = getUpdatedInstanceList(iterator, idToInstanceStatus); logger.info("Number of instances found to update AwsStatus = {}", updateInstanceList.size()); // Log the count of healthy instances & unhealthy ones. logger.info("Total count: = {} and unhealthy nodes count with status" + " not ok = {}", total, badInstanceCount); // Update all instances healthy + non healthy in CMDB logger.info("Starting a bulk update for awsStatus tag..."); cmdbInstanceStore.bulkUpdateAwsStatus(updateInstanceList); // Reset the counters to 0 total = 0; badInstanceCount = 0; logger.info("Re initialized total to {} and badInstanceCount to {}", total, badInstanceCount); return true; } catch (Exception e) { e.printStackTrace(); } return false; }
/** * Returns a map from virtual instance IDs to instance state for the specified batch of virtual * instance IDs. * * @param virtualInstanceIds batch of virtual instance IDs * @return the map from instance IDs to instance state for the specified batch of virtual * instance IDs */ private Map<String, InstanceState> getBatchInstanceState(Collection<String> virtualInstanceIds) { Map<String, InstanceState> instanceStateByVirtualInstanceId = Maps.newHashMapWithExpectedSize(virtualInstanceIds.size()); BiMap<String, String> virtualInstanceIdsByEC2InstanceId = getEC2InstanceIdsByVirtualInstanceId(virtualInstanceIds).inverse(); int page = 0; LOG.info(">> Fetching page {}", page); if (virtualInstanceIdsByEC2InstanceId.isEmpty()) { // No EC2 instances are found, which means these id's are already terminated and gone. // In practice, this is possible when no instances were provisioned to begin with. // For example, when a deployment fails due to tagging error. return instanceStateByVirtualInstanceId; } DescribeInstanceStatusResult result = client.describeInstanceStatus( new DescribeInstanceStatusRequest() // Note that sending in an empty set will result in fetching _all_ instance Ids. // It requires you to send one or more EC2 Ids .withInstanceIds(virtualInstanceIdsByEC2InstanceId.keySet()) .withIncludeAllInstances(true) ); LOG.info("<< Result: {}", result); while (!result.getInstanceStatuses().isEmpty()) { for (InstanceStatus status : result.getInstanceStatuses()) { InstanceStateName currentState = InstanceStateName.fromValue(status.getInstanceState().getName()); String ec2InstanceId = status.getInstanceId(); String virtualInstanceId = virtualInstanceIdsByEC2InstanceId.get(ec2InstanceId); InstanceState instanceState = EC2InstanceState.fromInstanceStateName(currentState); instanceStateByVirtualInstanceId.put(virtualInstanceId, instanceState); } String nextToken = result.getNextToken(); if (nextToken != null) { page++; LOG.info(">> Fetching page {} using token {}", page, nextToken); result = client.describeInstanceStatus(new DescribeInstanceStatusRequest() .withNextToken(nextToken)); LOG.info("<< Result: {}", result); } else { break; } } return instanceStateByVirtualInstanceId; }
public void start() throws Exception { // seed start this.masterStartAt = new Date(); /** * START the source for our TOC... if specified */ if (this.sourceEc2StartStopInstanceId != null) { // already running? InstanceStatus alreadyRunning = ec2util.getInstanceStatus(this.ec2Client, this.sourceEc2StartStopInstanceId); if (alreadyRunning == null || !alreadyRunning.getInstanceState().getName().equalsIgnoreCase("running")) { this.ec2util.startInstance(this.ec2Client, this.sourceEc2StartStopInstanceId); } // check status boolean sourceEc2InstanceStarted = false; while(!sourceEc2InstanceStarted) { InstanceStatus status = ec2util.getInstanceStatus(this.ec2Client, this.sourceEc2StartStopInstanceId); if (status == null || !status.getInstanceState().getName().equalsIgnoreCase("running")) { sourceEc2InstanceStarted = false; logger.debug("Still waiting for source host instance ["+this.sourceEc2StartStopInstanceId+"] to be running...."); Thread.currentThread().sleep(2000); } else { logger.debug("Up and RUNNING! source host instance ["+this.sourceEc2StartStopInstanceId+"]"); sourceEc2InstanceStarted = true; break; } } if (this.sourceEc2PostStartCmd != null) { Thread.currentThread().sleep(30000); // run the 'master.source.host.post.start.cmd' command CmdResult postStartResult = new CommandExecutor().execute(CommandLine.parse(this.sourceEc2PostStartCmd), 3); if (postStartResult.getExitCode() > 0) { throw new Exception("Source host post start cmd failed: " + this.sourceEc2PostStartCmd); } } } if (props.getProperty("tocGenerator.source.dir") != null && !new File(props.getProperty("tocGenerator.source.dir")).exists()) { throw new Exception("'tocGenerator.source.dir' does not exist! " + props.getProperty("tocGenerator.source.dir")); } // initialize workers this.currentMode = CCMode.INITIALIZED; this.controlChannel.send(true, CCPayloadType.MASTER_CURRENT_MODE, this.currentMode); // spawn ec2 cluster if specified spawnEC2(); // start our own monitor this.masterMonitor = new Thread(this); this.masterMonitor.start(); // @see handlePayload(CCPayload) for what happens next as workers come online }
@Override protected Boolean doCall() { LOGGER.info("Checking status of Auto Scaling group '{}'", autoScalingGroupName); AmazonEC2Client amazonEC2Client = awsClient.createAccess(new AwsCredentialView(getAuthenticatedContext().getCloudCredential()), getAuthenticatedContext().getCloudContext().getLocation().getRegion().value()); List<String> instanceIds = cloudFormationStackUtil.getInstanceIds(autoScalingClient, autoScalingGroupName); if (instanceIds.size() < requiredInstances) { LOGGER.debug("Instances in AS group: {}, needed: {}", instanceIds.size(), requiredInstances); List<Activity> activities = getAutoScalingActivities(); if (latestActivity.isPresent()) { checkForSpotRequest(latestActivity.get(), amazonEC2Client); activities = activities.stream().filter(activity -> activity.getStartTime().after(latestActivity.get().getStartTime())) .collect(Collectors.toList()); } updateLatestActivity(activities); checkFailedActivities(activities); return false; } List<DescribeInstanceStatusResult> describeInstanceStatusResultList = new ArrayList<>(); List<List<String>> partitionedInstanceIdsList = Lists.partition(instanceIds, MAX_INSTANCE_ID_SIZE); for (List<String> partitionedInstanceIds : partitionedInstanceIdsList) { DescribeInstanceStatusRequest describeInstanceStatusRequest = new DescribeInstanceStatusRequest().withInstanceIds(partitionedInstanceIds); DescribeInstanceStatusResult describeResult = amazonEC2Client.describeInstanceStatus(describeInstanceStatusRequest); describeInstanceStatusResultList.add(describeResult); } List<InstanceStatus> instanceStatusList = describeInstanceStatusResultList.stream() .flatMap(describeInstanceStatusResult -> describeInstanceStatusResult.getInstanceStatuses().stream()) .collect(Collectors.toList()); if (instanceStatusList.size() < requiredInstances) { LOGGER.debug("Instances up: {}, needed: {}", instanceStatusList.size(), requiredInstances); return false; } for (InstanceStatus status : instanceStatusList) { if (INSTANCE_RUNNING != status.getInstanceState().getCode()) { LOGGER.debug("Instances are up but not all of them are in running state."); return false; } } return true; }
@Test public void successTest() throws Exception { int requiredInstances = 160; AuthenticatedContext authenticatedContext = mock(AuthenticatedContext.class); CloudContext cloudContext = mock(CloudContext.class); String regionName = "eu-west-1"; when(cloudContext.getLocation()).thenReturn(Location.location(Region.region(regionName))); when(authenticatedContext.getCloudContext()).thenReturn(cloudContext); CloudCredential cloudCredential = mock(CloudCredential.class); when(authenticatedContext.getCloudCredential()).thenReturn(cloudCredential); String asGroupName = "as-group"; AwsClient awsClient = mock(AwsClient.class); AmazonEC2Client amazonEC2Client = mock(AmazonEC2Client.class); when(awsClient.createAccess(any(AwsCredentialView.class), eq(regionName))).thenReturn(amazonEC2Client); DescribeInstanceStatusResult firstDescribeInstanceStatusResult = new DescribeInstanceStatusResult(); List<InstanceStatus> firstInstanceStatuses = returnInstanceStatus(0, 100); firstDescribeInstanceStatusResult.setInstanceStatuses(firstInstanceStatuses); DescribeInstanceStatusResult secondDescribeInstanceStatusResult = new DescribeInstanceStatusResult(); List<InstanceStatus> secondInstanceStatuses = returnInstanceStatus(100, 160); secondDescribeInstanceStatusResult.setInstanceStatuses(secondInstanceStatuses); when(amazonEC2Client.describeInstanceStatus(any(DescribeInstanceStatusRequest.class))) .thenReturn(firstDescribeInstanceStatusResult) .thenReturn(secondDescribeInstanceStatusResult); CloudFormationStackUtil cloudFormationStackUtil = mock(CloudFormationStackUtil.class); List<String> instancIds = new ArrayList<>(); for (int i = 0; i < requiredInstances; i++) { instancIds.add(Integer.toString(i)); } AmazonAutoScalingClient autoScalingClient = mock(AmazonAutoScalingClient.class); when(awsClient.createAutoScalingClient(any(AwsCredentialView.class), anyString())).thenReturn(autoScalingClient); when(autoScalingClient.describeScalingActivities(any(DescribeScalingActivitiesRequest.class))).thenReturn(new DescribeScalingActivitiesResult()); when(cloudFormationStackUtil.getInstanceIds(any(AmazonAutoScalingClient.class), eq(asGroupName))).thenReturn(instancIds); ASGroupStatusCheckerTask asGroupStatusCheckerTask = new ASGroupStatusCheckerTask(authenticatedContext, asGroupName, requiredInstances, awsClient, cloudFormationStackUtil); Boolean taskResult = asGroupStatusCheckerTask.call(); ArgumentCaptor<DescribeInstanceStatusRequest> instanceStatusRequestArgumentCaptor = ArgumentCaptor.forClass(DescribeInstanceStatusRequest.class); verify(amazonEC2Client, times(2)).describeInstanceStatus(instanceStatusRequestArgumentCaptor.capture()); List<DescribeInstanceStatusRequest> allValues = instanceStatusRequestArgumentCaptor.getAllValues(); assertEquals(100, allValues.get(0).getInstanceIds().size()); assertEquals(60, allValues.get(1).getInstanceIds().size()); assertTrue(taskResult); }
@Test public void failTest() throws Exception { int requiredInstances = 160; AuthenticatedContext authenticatedContext = mock(AuthenticatedContext.class); CloudContext cloudContext = mock(CloudContext.class); String regionName = "eu-west-1"; when(cloudContext.getLocation()).thenReturn(Location.location(Region.region(regionName))); when(authenticatedContext.getCloudContext()).thenReturn(cloudContext); CloudCredential cloudCredential = mock(CloudCredential.class); when(authenticatedContext.getCloudCredential()).thenReturn(cloudCredential); String asGroupName = "as-group"; AwsClient awsClient = mock(AwsClient.class); AmazonEC2Client amazonEC2Client = mock(AmazonEC2Client.class); when(awsClient.createAccess(any(AwsCredentialView.class), eq(regionName))).thenReturn(amazonEC2Client); DescribeInstanceStatusResult firstDescribeInstanceStatusResult = new DescribeInstanceStatusResult(); List<InstanceStatus> firstInstanceStatuses = returnInstanceStatus(0, 99); firstDescribeInstanceStatusResult.setInstanceStatuses(firstInstanceStatuses); DescribeInstanceStatusResult secondDescribeInstanceStatusResult = new DescribeInstanceStatusResult(); List<InstanceStatus> secondInstanceStatuses = returnInstanceStatus(100, 160); secondDescribeInstanceStatusResult.setInstanceStatuses(secondInstanceStatuses); when(amazonEC2Client.describeInstanceStatus(any(DescribeInstanceStatusRequest.class))) .thenReturn(firstDescribeInstanceStatusResult) .thenReturn(secondDescribeInstanceStatusResult); CloudFormationStackUtil cloudFormationStackUtil = mock(CloudFormationStackUtil.class); List<String> instancIds = new ArrayList<>(); for (int i = 0; i < requiredInstances; i++) { instancIds.add(Integer.toString(i)); } AmazonAutoScalingClient autoScalingClient = mock(AmazonAutoScalingClient.class); when(awsClient.createAutoScalingClient(any(AwsCredentialView.class), anyString())).thenReturn(autoScalingClient); when(autoScalingClient.describeScalingActivities(any(DescribeScalingActivitiesRequest.class))).thenReturn(new DescribeScalingActivitiesResult()); when(cloudFormationStackUtil.getInstanceIds(any(AmazonAutoScalingClient.class), eq(asGroupName))).thenReturn(instancIds); ASGroupStatusCheckerTask asGroupStatusCheckerTask = new ASGroupStatusCheckerTask(authenticatedContext, asGroupName, requiredInstances, awsClient, cloudFormationStackUtil); Boolean taskResult = asGroupStatusCheckerTask.call(); ArgumentCaptor<DescribeInstanceStatusRequest> instanceStatusRequestArgumentCaptor = ArgumentCaptor.forClass(DescribeInstanceStatusRequest.class); verify(amazonEC2Client, times(2)).describeInstanceStatus(instanceStatusRequestArgumentCaptor.capture()); List<DescribeInstanceStatusRequest> allValues = instanceStatusRequestArgumentCaptor.getAllValues(); assertEquals(100, allValues.get(0).getInstanceIds().size()); assertEquals(60, allValues.get(1).getInstanceIds().size()); assertFalse(taskResult); }
@Test public void testGetInstanceStatus() { final DescribeInstanceStatusRequest describeInstanceStatusRequest = new DescribeInstanceStatusRequest().withIncludeAllInstances(true).withInstanceIds(INSTANCE_ID); final DescribeInstanceStatusResult describeInstanceStatusResult = new DescribeInstanceStatusResult().withInstanceStatuses(new InstanceStatus().withInstanceState(new InstanceState().withName(InstanceStateName.Running))); Mockito.doReturn(describeInstanceStatusResult).when(amazonEC2Client).describeInstanceStatus(describeInstanceStatusRequest); Assertions.assertThat(amazonEC2Service.getInstanceStatus(INSTANCE_ID)).isEqualTo(InstanceStateName.Running); Mockito.verify(amazonEC2Client).describeInstanceStatus(describeInstanceStatusRequest); }
