void attachSnapshotTags(AmazonEC2Async client, String sourceSnapshotId, String snapshotId) { DescribeSnapshotsResult result = client .describeSnapshots(new DescribeSnapshotsRequest().withSnapshotIds(sourceSnapshotId)); List<Snapshot> snapshots = result.getSnapshots(); if (snapshots.size() != 1) { throw new RuntimeException("snapshot can not found. sourceSnapshotId[" + snapshotId + "]"); } List<Tag> sourceSnapshotTags = snapshots.get(0).getTags(); List<Tag> tags = new ArrayList<Tag>(); tags.addAll(sourceSnapshotTags); tags.add(new Tag("SourceSnapshotId", sourceSnapshotId)); tags.add(new Tag("BackupType", "copy-snapshot")); // overwrite CreateTagsRequest snapshotTagsRequest = new CreateTagsRequest().withResources(snapshotId); snapshotTagsRequest.setTags(tags); client.createTags(snapshotTagsRequest); }
@Test(expected = ServiceBrokerException.class) public void itShouldThrowWhenMoreThanOneAMIIsFoundToBeDeleted() throws ServiceBrokerException, ServiceInstanceBindingExistsException { when(ec2Client.describeSnapshots()) .thenReturn( new DescribeSnapshotsResult().withSnapshots(Arrays.asList( new Snapshot() .withDescription("Created by CreateImage(source_instance) for test_image from vol-e7526fac"), new Snapshot() .withDescription( "Created by CreateImage(source_instance) for test_image from vol-e7526fac") .withSnapshotId("test_snapshot"), new Snapshot() .withDescription("Created by CreateImage(source_instance) for ami-xx from vol-e7526fac"), new Snapshot()))); aws.deleteStorageArtifacts("test_image"); verify(ec2Client, never()).deleteSnapshot(any()); }
@Override protected Snapshot convertObject(SnapshotInfo from) { Snapshot to = new Snapshot(); to.setSnapshotId(from.getSnapshotId()); to.setVolumeId(from.getVolumeId()); to.setState(from.getStatus()); to.setStartTime(from.getStartTime().getTime()); to.setProgress(from.getProgress()); // 未実装 to.setOwnerId(null); to.setDescription(null); to.setVolumeSize(null); to.setOwnerAlias(null); to.setTags(null); return to; }
@Override @SuppressWarnings("rawtypes") protected boolean isEqual(AbstractResource newResource) { Snapshot oldSnapshot = this.getResource(); Ec2Snapshot newEc2Snapshot = (Ec2Snapshot) newResource; Snapshot newSnapshot = (Snapshot) newResource.getResource(); if (notEqual(oldSnapshot.getSnapshotId(), newSnapshot.getSnapshotId())) return false; if (notEqual(oldSnapshot.getVolumeId(), newSnapshot.getVolumeId())) return false; if (notEqual(oldSnapshot.getState(), newSnapshot.getState())) return false; if (notEqual(oldSnapshot.getStartTime(), newSnapshot.getStartTime())) return false; if (notEqual(oldSnapshot.getProgress(), newSnapshot.getProgress())) return false; if (notEqual(oldSnapshot.getOwnerId(), newSnapshot.getOwnerId())) return false; if (notEqual(oldSnapshot.getDescription(), newSnapshot.getDescription())) return false; if (notEqual(oldSnapshot.getVolumeSize(), newSnapshot.getVolumeSize())) return false; if (notEqual(oldSnapshot.getOwnerAlias(), newSnapshot.getOwnerAlias())) return false; if (notEqualCollection(oldSnapshot.getTags(), newSnapshot.getTags())) return false; if (notEqualCollection(this.getCreateVolumePermissions(), newEc2Snapshot.getCreateVolumePermissions())) return false; if (notEqualCollection(this.getProductCodes(), newEc2Snapshot.getProductCodes())) return false; return true; }
private void compareSnapshotState(Collection<Event> result, Ec2Snapshot oldResource, Ec2Snapshot newResource) { Snapshot newInstance = newResource.getResource(); Snapshot oldInstance = oldResource.getResource(); String newState = newInstance.getState(); String oldState = oldInstance.getState(); if (notEqual(newState, oldState)) { switch (newState) { case "pending": result.add(createEvent(oldResource, newResource, EventType.Ec2_Snapshot_Pending)); break; case "completed": result.add(createEvent(oldResource, newResource, EventType.Ec2_Snapshot_Created)); break; default: log.error("not handled instance state:" + newState); result.add(createEvent(oldResource, newResource, EventType.Unknown)); } } }
@Override public List<AbstractResource<?>> describeSnapshots(Account account, Region region, DateTime dt, Ec2Filter... filters) { AmazonEC2 ec2 = findClient(account, region); DescribeSnapshotsRequest req = new DescribeSnapshotsRequest(); for (Ec2Filter filter : filters) { Filter f = new Filter().withName(filter.getName()).withValues(filter.getValues()); req.withFilters(f); } log.debug("start describing snapshots for account:{} in region:{} via api", account.getId() + "=>" + account.getName(), region); DescribeSnapshotsResult res = ec2.describeSnapshots(req); List<List<CreateVolumePermission>> createVolumePermissions = new ArrayList<>(); List<List<ProductCode>> productCodes = new ArrayList<>(); for (Snapshot snapShot : res.getSnapshots()) { productCodes.add(findSnapshotProductCodes(account, region, snapShot.getSnapshotId())); createVolumePermissions.add(findCreateVolumePermissions(account, region, snapShot.getSnapshotId())); } return converter.toEc2Snapshots(res.getSnapshots(), createVolumePermissions, productCodes, account.getId(), region, dt); }
void pargeEbsSnapshot(AmazonEC2Async client, String volumeId, int generationCount, Context context) { LambdaLogger logger = context.getLogger(); logger.log("Parge snapshot start. VolumeId[" + volumeId + "] generationCount[" + generationCount + "]"); List<Filter> filters = new ArrayList<>(); filters.add(new Filter().withName("volume-id").withValues(volumeId)); filters.add(new Filter().withName("tag:VolumeId").withValues(volumeId)); filters.add(new Filter().withName("tag:BackupType").withValues("snapshot")); DescribeSnapshotsRequest snapshotRequest = new DescribeSnapshotsRequest().withFilters(filters); DescribeSnapshotsResult snapshotResult = client.describeSnapshots(snapshotRequest); List<Snapshot> snapshots = snapshotResult.getSnapshots(); Collections.sort(snapshots, new SnapshotComparator()); int snapshotSize = snapshots.size(); if (generationCount < snapshotSize) { for (int i = 0; i < snapshotSize - generationCount; i++) { Snapshot snapshot = snapshots.get(i); if (SnapshotState.Completed.toString().equals(snapshot.getState())) { String snapshotId = snapshot.getSnapshotId(); pargeSnapshot(client, snapshotId); logger.log("Parge EBS snapshot. snapshotId[" + snapshotId + "] volumeId[" + volumeId + "]"); } } } }
@Override public void handleRequest(InputStream is, OutputStream os, Context context) { LambdaLogger logger = context.getLogger(); String regionName = System.getenv("AWS_DEFAULT_REGION"); AmazonEC2Async client = RegionUtils.getRegion(regionName).createClient(AmazonEC2AsyncClient.class, new DefaultAWSCredentialsProviderChain(), cc); try { ObjectMapper om = new ObjectMapper(); DeregisterImageRequest event = om.readValue(is, DeregisterImageRequest.class); String imageId = event.getDetail().getRequestParameters().getImageId(); logger.log("Deregister AMI parge snapshot Start. ImageId[" + imageId + "]"); List<Snapshot> snapshots = describeSnapshot(client, imageId, context); if (snapshots.size() == 0) { logger.log("Target of snapshot there is nothing."); } else { snapshots.stream().forEach(s -> pargeSnapshot(client, s.getSnapshotId(), context)); } logger.log("[SUCCESS][DeregisterImage]has been completed successfully." + imageId); } catch (Exception e) { logger.log("[ERROR][DeregisterImage]An unexpected error has occurred. message[" + e.getMessage() + "]"); } finally { client.shutdown(); } }
private List<Snapshot> describeSnapshot(AmazonEC2Async client, String imageId, Context context) { Filter filter = new Filter().withName("description") .withValues("Created by CreateImage(*) for " + imageId + " from *"); DescribeSnapshotsRequest request = new DescribeSnapshotsRequest().withFilters(filter); DescribeSnapshotsResult result = client.describeSnapshots(request); return result.getSnapshots(); }
void pargeEbsSnapshot(AmazonEC2Async client, String sourceSnapshotId, String snapshotId, int generationCount, Context context) { LambdaLogger logger = context.getLogger(); logger.log( "Parge snapshot start. SnapshotId[" + sourceSnapshotId + "] generationCount[" + generationCount + "]"); // get volumeId from tags String volumeId = getVolumeIdFromTag(client, snapshotId); // describe filter tag VolumeId List<Filter> filters = new ArrayList<>(); filters.add(new Filter().withName("tag:VolumeId").withValues(volumeId)); filters.add(new Filter().withName("tag:BackupType").withValues("copy-snapshot")); DescribeSnapshotsRequest snapshotRequest = new DescribeSnapshotsRequest().withFilters(filters); DescribeSnapshotsResult snapshotResult = client.describeSnapshots(snapshotRequest); // snapshot作成開始日でソートします。(古い→新しい) List<Snapshot> snapshots = snapshotResult.getSnapshots(); Collections.sort(snapshots, new SnapshotComparator()); // 世代管理保持数 < snapshotの数の場合、対象をpargeします。 int snapshotSize = snapshots.size(); if (generationCount < snapshotSize) { for (int i = 0; i < snapshotSize - generationCount; i++) { Snapshot snapshot = snapshots.get(i); // (念のため)snapshotのステータスが完了しているものだけをparge対象とする。 if (SnapshotState.Completed.toString().equals(snapshot.getState())) { String pargeSnapshotId = snapshot.getSnapshotId(); pargeSnapshot(client, pargeSnapshotId); logger.log("Parge EBS snapshot. snapshotId[" + pargeSnapshotId + "]"); } } } }
/** * Return true if snapshotId exists. */ public static boolean snapshotIdExists(AmazonEC2AsyncClient client, String snapshotId) { List<Snapshot> snapshots = client.describeSnapshots() .getSnapshots() .stream() .filter(snapshot -> snapshot.getSnapshotId().equals(snapshotId)) .collect(Collectors.toList()); return snapshots != null && !snapshots.isEmpty(); }
public List<AbstractResource<?>> toEc2Snapshots(List<Snapshot> snapshots, List<List<CreateVolumePermission>> createVolumePermissions, List<List<ProductCode>> productCodes, String accountId, Region region, DateTime dt) { List<AbstractResource<?>> resources = new ArrayList<>(); for (int i = 0; i < snapshots.size(); i++) { Ec2Snapshot ec2Snapshot = new Ec2Snapshot(); conf(ec2Snapshot, accountId, region, dt); ec2Snapshot.setResource(snapshots.get(i)); ec2Snapshot.setCreateVolumePermissions(createVolumePermissions.get(i)); ec2Snapshot.setProductCodes(productCodes.get(i)); resources.add(ec2Snapshot); } log.debug("{} snapshots found via api and converted to Ec2Snapshot", resources.size()); return resources; }
@Override public Event add(Ec2Snapshot newResource) { Snapshot newSnapshot = newResource.getResource(); String state = newSnapshot.getState(); switch (state) { case "pending": return createEvent(null, newResource, EventType.Ec2_Snapshot_Pending); case "completed": return createEvent(null, newResource, EventType.Ec2_Snapshot_Created); default: log.error("not handled instance state:{}", state); return createEvent(null, newResource, EventType.Unknown); } }
@Override protected void update(List<Event> result, Ec2Snapshot oldResource, Ec2Snapshot newResource) { Snapshot oldSnapshot = oldResource.getResource(); Snapshot newSnapshot = newResource.getResource(); compareSnapshotState(result, oldResource, newResource); compareCreateVolumePermission(result, oldResource.getCreateVolumePermissions(), newResource.getCreateVolumePermissions(), oldResource, newResource); compareTags(result, oldSnapshot.getTags(), newSnapshot.getTags(), oldResource, newResource); }
@Override public int compare(Snapshot o1, Snapshot o2) { Date startDateO1 = o1.getStartTime(); Date startDateO2 = o2.getStartTime(); return startDateO1.compareTo(startDateO2); }
/** * Find the snapshot the volumeId the key, to get a copy of the latest * Snapshot * * @param volumeIdRequest * VolumeIdRequest * @param context * Context */ public void copySnapshotFromVolumeId(VolumeIdRequest volumeIdRequest, Context context) { LambdaLogger logger = context.getLogger(); String regionName = System.getenv("AWS_DEFAULT_REGION"); AmazonEC2Async client = RegionUtils.getRegion(regionName).createClient(AmazonEC2AsyncClient.class, new DefaultAWSCredentialsProviderChain(), cc); try { String volumeId = volumeIdRequest.getVolumeId(); boolean isLockAcquisition = new LambdaLock().lock(volumeId, context); if (!isLockAcquisition) { logger.log("[ERROR][EBSCopySnapshot][" + volumeId + "]You can not acquire a lock."); return; } List<Filter> filters = new ArrayList<>(); filters.add(new Filter().withName("volume-id").withValues(volumeId)); filters.add(new Filter().withName("tag:VolumeId").withValues(volumeId)); filters.add(new Filter().withName("tag:BackupType").withValues("snapshot")); DescribeSnapshotsRequest snapshotRequest = new DescribeSnapshotsRequest().withFilters(filters); DescribeSnapshotsResult snapshotResult = client.describeSnapshots(snapshotRequest); // sort and get latest snapshot List<Snapshot> snapshots = snapshotResult.getSnapshots(); Collections.sort(snapshots, new SnapshotComparator()); int snapshotsSize = snapshots.size(); if (snapshotsSize == 0) { throw new RuntimeException("The copy source snapshot can not be found"); } Snapshot snapshot = snapshots.get(snapshots.size() - 1); String sourceSnapshotId = snapshot.getSnapshotId(); copySnapshot(sourceSnapshotId, volumeIdRequest.getDestinationRegion(), volumeIdRequest.getGenerationCount(), context); } catch (Exception e) { logger.log("[ERROR][EBSCopySnapshot][" + volumeIdRequest.getVolumeId() + "] message[" + e.getMessage() + "] stackTrace[" + getStackTrace(e) + "]"); } finally { client.shutdown(); } }
/** * Find the snap & volumes associated with the AMI we used and delete it. * AWS doesn't help us out much and the only relationship (as of 2/14/2015) * we can leverage is the description field. * * @param ami * to find associated snaps for * @throws ServiceBrokerExceptions */ public void deleteStorageArtifacts(String ami) throws ServiceBrokerException { DescribeSnapshotsResult desc = ec2Client.describeSnapshots(); if (null == desc.getSnapshots()) { return; } List<Snapshot> snapshots = desc.getSnapshots(); // The only way I can find to track the snaps that get created (but not // cleaned up) as part of the ami creation is by the description. This // code is brittle and will probably fail in unexpected and glamorous // ways. String amiDesc = "Created by CreateImage(" + sourceInstanceId + ") for " + ami + " from vol"; // Would be nice if the aws client return optionals... List<Snapshot> matching = snapshots.stream() .filter(s -> safeContains(s::getDescription, amiDesc)) .collect(Collectors.toList()); switch (matching.size()) { case 0: // Should this throw? Might have been manually cleaned up...but it // may orphan the volume. It's done this way to allow people to // create their own instances in AWS and not jack them up by // deleting the volume log.error("No snapshots found for AMI " + ami); break; case 1: String snap = matching.get(0).getSnapshotId(); log.info("Deleting snapshot " + snap); ec2Client.deleteSnapshot(new DeleteSnapshotRequest() .withSnapshotId(snap)); deleteVolumeForSnap(snap); break; default: throw new ServiceBrokerException( "Found too many snapshots for AMI " + ami); } }
@Test public void itShouldDeleteTheStorageArtifacts() throws ServiceInstanceBindingExistsException, ServiceBrokerException { when(ec2Client.describeSnapshots()) .thenReturn( new DescribeSnapshotsResult().withSnapshots(Arrays.asList( new Snapshot() .withDescription("Created by CreateImage(source_instance) for ami-9687d2fe from vol-e7526fac"), new Snapshot() .withDescription( "Created by CreateImage(source_instance) for test_image from vol-e7526fac") .withSnapshotId("test_snapshot"), new Snapshot() .withDescription("Created by CreateImage(i-bf72d345) for ami-xx from vol-e7526fac"), new Snapshot()))); Predicate<DescribeVolumesRequest> pred = new Predicate<DescribeVolumesRequest>() { @Override public boolean test(DescribeVolumesRequest r) { Filter filter = r.getFilters().stream().findFirst().get(); return filter.getName().equals("snapshot-id") && filter.getValues().stream().findFirst().get() .equals("test_snapshot"); } }; DescribeVolumesResult volumesResult = new DescribeVolumesResult() .withVolumes(Collections.singletonList(new Volume() .withVolumeId("test_volume"))); when(ec2Client.describeVolumes(awsRqst(pred))) .thenReturn(volumesResult); aws.deleteStorageArtifacts("test_image"); verify(ec2Client).deleteSnapshot( awsRqst(r -> r.getSnapshotId().equals("test_snapshot"))); verify(ec2Client).deleteVolume( awsRqst(r -> r.getVolumeId().equals("test_volume"))); }
private void waitingTempSnapshotStep(TaskEntry taskEntry) { setProgress(taskEntry, TaskProgress.WAITING_SNAPSHOT); Snapshot snapshot = awsCommunication.waitForCompleteState(awsCommunication.getSnapshot(taskEntry.getTempSnapshotId())); LOG.info("SnapshotEntry created: {}", snapshot.toString()); }
private Optional<String> checkThatSnapshotIsAvailable(AwsInstanceView awsInstanceView, AmazonEC2Client client) { Optional<Snapshot> describeSnapshotsResult = client.describeSnapshots(prepareDescribeSnapshotsRequest(awsInstanceView)).getSnapshots().stream().findFirst(); return describeSnapshotsResult.isPresent() ? Optional.ofNullable(describeSnapshotsResult.get().getSnapshotId()) : Optional.empty(); }