/** * Returns the events for a named stack. * * @param stackId Stack ID. * @return Collection of events */ public List<StackEvent> getStackEvents(final String stackId) { final DescribeStackEventsRequest request = new DescribeStackEventsRequest().withStackName(stackId); try { final DescribeStackEventsResult result = cloudFormationClient.describeStackEvents(request); return result.getStackEvents(); } catch (final AmazonServiceException ase) { // Stack doesn't exist, just return with no status if (ase.getStatusCode() != 400) { throw ase; } } return Collections.emptyList(); }
@Test public void shouldReturnStatusDeletionFailDueToOtherException() throws WrongNumberOfStacksException, WrongStackStatus, InterruptedException { List<String> aborts = Arrays.asList(StackMonitor.DELETE_ABORTS); AmazonServiceException amazonServiceException = new AmazonServiceException("message"); amazonServiceException.setErrorCode("someOtherError"); EasyMock.expect(cfnRepository.waitForStatusToChangeFrom(stackName, StackStatus.DELETE_IN_PROGRESS, aborts)). andThrow(amazonServiceException); List<StackEvent> events = new LinkedList<StackEvent>(); EasyMock.expect(cfnRepository.getStackEvents(stackName)).andReturn(events); replayAll(); String result = monitor.waitForDeleteFinished(stackId); assertEquals(StackStatus.DELETE_FAILED.toString(), result); verifyAll(); }
@Test public void shouldMonitorMultiplePendingDeletesAndLeaveIndexCorrectOnFailure() throws WrongNumberOfStacksException, InterruptedException { List<String> aborts = Arrays.asList(StackMonitor.DELETE_ABORTS); StackStatus targetStatus = StackStatus.DELETE_COMPLETE; DeletionsPending pending = new DeletionsPending(); pending.add(3, new StackNameAndId("stackC", "id3")); pending.add(2, new StackNameAndId("stackB", "id2")); pending.add(1, new StackNameAndId("stackA", "id1")); EasyMock.expect(cfnRepository.waitForStatusToChangeFrom("stackC", StackStatus.DELETE_IN_PROGRESS, aborts)) .andReturn(targetStatus.toString()); EasyMock.expect(cfnRepository.waitForStatusToChangeFrom("stackB", StackStatus.DELETE_IN_PROGRESS, aborts)) .andReturn(StackStatus.DELETE_FAILED.toString()); List<StackEvent> events = new LinkedList<StackEvent>(); EasyMock.expect(cfnRepository.getStackEvents("stackB")).andReturn(events); replayAll(); monitor.waitForDeleteFinished(pending, this); assertEquals(2, lastDeltaIndex); verifyAll(); }
@Override protected Boolean doCall() { LOGGER.info("Checking if AWS CloudFormation stack '{}' reached status '{}'", cloudFormationStackName, successStatus); try { com.amazonaws.services.cloudformation.model.Stack cfStack = cfClient.describeStacks(describeStacksRequest).getStacks().get(0); List<StackEvent> stackEvents = cfClient.describeStackEvents(stackEventsRequest).getStackEvents(); return doCheck(cfStack, stackEvents); } catch (AmazonServiceException e) { return handleError(e); } }
protected boolean isSuccess(Stack cfStack, List<StackEvent> stackEvents) { if (!stackEvents.isEmpty() && cfStack != null) { StackStatus cfStackStatus = StackStatus.valueOf(cfStack.getStackStatus()); if (stackErrorStatuses.contains(cfStackStatus)) { throw new CloudConnectorException(getErrorMessage(errorStatus.toString(), getErrorCauseStatusReason(stackEvents, errorStatus))); } else if (cfStackStatus.equals(successStatus)) { return true; } } return false; }
private String getErrorCauseStatusReason(List<StackEvent> stackEvents, StackStatus errorStatus) { StackEvent cause = null; for (StackEvent event : stackEvents) { if (event.getResourceStatus().equals(errorStatus.toString())) { if (cause == null || cause.getTimestamp().getTime() > event.getTimestamp().getTime()) { cause = event; } } } return cause == null ? "unknown" : cause.getResourceStatusReason(); }
private void setRepoExcpetationsForFailure(List<String> aborts, StackStatus initialStatus, StackStatus failureStatus) throws WrongNumberOfStacksException, InterruptedException { EasyMock.expect(cfnRepository.waitForStatusToChangeFrom(stackName, initialStatus, aborts)). andReturn(failureStatus.toString()); List<StackEvent> events = new LinkedList<StackEvent>(); EasyMock.expect(cfnRepository.getStackEvents(stackName)).andReturn(events); }
/** * Blocking call that waits for a stack change to complete. Times out if waiting more than 30 minutes. * * @param endStatuses Status to end on * @return The final status */ public StackStatus waitForStatus(final String stackId, final HashSet<StackStatus> endStatuses) { DateTime now = DateTime.now(DateTimeZone.UTC).minusSeconds(10); DateTime timeoutDateTime = now.plusMinutes(90); Set<String> recordedStackEvents = Sets.newHashSet(); boolean isRunning = true; StackStatus stackStatus = null; while (isRunning) { try { TimeUnit.SECONDS.sleep(5); } catch (InterruptedException e) { logger.warn("Thread sleep interrupted. Continuing...", e); } stackStatus = getStackStatus(stackId); if (endStatuses.contains(stackStatus) || stackStatus == null) { isRunning = false; } if (stackStatus != null) { List<StackEvent> stackEvents = getStackEvents(stackId); stackEvents.sort((o1, o2) -> o1.getTimestamp().compareTo(o2.getTimestamp())); for (StackEvent stackEvent : stackEvents) { DateTime eventTime = new DateTime(stackEvent.getTimestamp()); if (!recordedStackEvents.contains(stackEvent.getEventId()) && now.isBefore(eventTime)) { logger.info( String.format("TS: %s, Status: %s, Type: %s, Reason: %s", Chalk.on(stackEvent.getTimestamp().toString()).yellow(), getStatusColor(stackEvent.getResourceStatus()), Chalk.on(stackEvent.getResourceType()).yellow(), Chalk.on(stackEvent.getResourceStatusReason()).yellow())); recordedStackEvents.add(stackEvent.getEventId()); } } } if (timeoutDateTime.isBeforeNow()) { logger.error("Timed out waiting for CloudFormation completion status."); isRunning = false; } } return stackStatus; }
@Override protected boolean doCheck(com.amazonaws.services.cloudformation.model.Stack cfStack, List<StackEvent> stackEvents) { return isSuccess(cfStack, stackEvents); }
@Override protected boolean doCheck(Stack cfStack, List<StackEvent> stackEvents) { return isSuccess(cfStack, stackEvents); }
private void logStackEvents(List<StackEvent> stackEvents) { for(StackEvent event : stackEvents) { logger.info(event.toString()); } }
private void printStackEvents() { final DescribeStackEventsRequest request = new DescribeStackEventsRequest(); request.withStackName(name); final DescribeStackEventsResult describeStackEvents = amazonClient .describeStackEvents(request); final List<StackEvent> stackEvents = describeStackEvents .getStackEvents(); Collections.reverse(stackEvents); logger.info("stack events:"); for (final StackEvent event : stackEvents) { final StringBuilder text = new StringBuilder(128); text.append("\n\t"); text.append("time="); text.append(event.getTimestamp()); text.append("\n\t"); text.append("id="); text.append(event.getEventId()); text.append("\n\t"); text.append("type="); text.append(event.getResourceType()); text.append("\n\t"); text.append("status="); text.append(event.getResourceStatus()); text.append("\n\t"); text.append("reason="); text.append(event.getResourceStatusReason()); logger.info("event {}", text); } }
abstract boolean doCheck(com.amazonaws.services.cloudformation.model.Stack cfStack, List<StackEvent> stackEvents);