/** * Add the disk information to disk state so that the disk state reflects the volume * information */ private void addMandatoryProperties(DiskState diskState, BlockDeviceMapping deviceMapping, InstanceType instanceType) { if (diskState.customProperties == null) { diskState.customProperties = new HashMap<>(); } String deviceName = deviceMapping.getDeviceName(); diskState.customProperties.put(DEVICE_NAME, deviceName); EbsBlockDevice ebs = deviceMapping.getEbs(); if (ebs != null) { diskState.capacityMBytes = ebs.getVolumeSize() * 1024; diskState.customProperties.put(DEVICE_TYPE, AWSStorageType.EBS.getName()); } else { diskState.capacityMBytes = instanceType.dataDiskSizeInMB; diskState.customProperties.put(DEVICE_TYPE, AWSStorageType.INSTANCE_STORE.getName()); } }
private void customizeBootDiskProperties(DiskState bootDisk, String rootDeviceType, BlockDeviceMapping rootDeviceMapping, boolean hasHardConstraint, AWSInstanceContext aws) { if (rootDeviceType.equals(AWSStorageType.EBS.name().toLowerCase())) { String requestedType = bootDisk.customProperties.get(DEVICE_TYPE); EbsBlockDevice ebs = rootDeviceMapping.getEbs(); if (hasHardConstraint) { validateIfDeviceTypesAreMatching(rootDeviceType, requestedType); } bootDisk.capacityMBytes = ebs.getVolumeSize() * 1024; updateDeviceMapping(rootDeviceType, requestedType, rootDeviceMapping.getDeviceName(), ebs, bootDisk); bootDisk.customProperties.put(DEVICE_TYPE, AWSStorageType.EBS.getName()); bootDisk.customProperties.put(DEVICE_NAME, rootDeviceMapping.getDeviceName()); bootDisk.customProperties.put(VOLUME_TYPE, ebs.getVolumeType()); bootDisk.customProperties.put(DISK_IOPS, String.valueOf(ebs.getIops())); } else { if (aws.instanceTypeInfo.dataDiskSizeInMB != null) { this.logInfo( () -> "[AWSInstanceService] Instance-Store boot disk size is set to the " + "value supported by instance-type."); bootDisk.capacityMBytes = aws.instanceTypeInfo.dataDiskSizeInMB; bootDisk.customProperties.put(DEVICE_TYPE, AWSStorageType.INSTANCE_STORE.getName()); } } }
private List<BlockDeviceMapping> getEbsBlockDeviceMapping(int count, String volumeType, int volumeSizeGib, boolean enableEncryption) { List<String> deviceNames = ebsAllocator.getEbsDeviceNames(count); List<BlockDeviceMapping> mappings = Lists.newArrayList(); for (String deviceName : deviceNames) { EbsBlockDevice ebs = new EbsBlockDevice() .withVolumeType(volumeType) .withVolumeSize(volumeSizeGib) .withEncrypted(enableEncryption) .withDeleteOnTermination(true); BlockDeviceMapping mapping = new BlockDeviceMapping() .withDeviceName(deviceName) .withEbs(ebs); mappings.add(mapping); } return mappings; }
private static EbsBlockDevice parseEbs(String blockDevice) { String[] parts = blockDevice.split(":"); EbsBlockDevice ebs = new EbsBlockDevice(); if (StringUtils.isNotBlank(getOrEmpty(parts, 0))) { ebs.setSnapshotId(parts[0]); } if (StringUtils.isNotBlank(getOrEmpty(parts, 1))) { ebs.setVolumeSize(Integer.valueOf(parts[1])); } if (StringUtils.isNotBlank(getOrEmpty(parts, 2))) { ebs.setDeleteOnTermination(Boolean.valueOf(parts[2])); } if (StringUtils.isNotBlank(getOrEmpty(parts, 3))) { ebs.setVolumeType(parts[3]); } if (StringUtils.isNotBlank(getOrEmpty(parts, 4))) { ebs.setIops(Integer.valueOf(parts[4])); } if (StringUtils.isNotBlank(getOrEmpty(parts, 5))) { ebs.setEncrypted(parts[5].equals("encrypted")); } return ebs; }
public void testParserWithTermination() throws Exception { List<BlockDeviceMapping> expected = new ArrayList<BlockDeviceMapping>(); expected.add( new BlockDeviceMapping(). withDeviceName("/dev/sdc"). withEbs( new EbsBlockDevice(). withSnapshotId("snap-7eb96d16"). withVolumeSize(80). withDeleteOnTermination(false) ) ); String customDeviceMappings = "/dev/sdc=snap-7eb96d16:80:false"; List<BlockDeviceMapping> actual = DeviceMappingParser.parse(customDeviceMappings); assertEquals(expected, actual); }
public void testParserWithIo() throws Exception { List<BlockDeviceMapping> expected = new ArrayList<BlockDeviceMapping>(); expected.add( new BlockDeviceMapping(). withDeviceName("/dev/sdc"). withEbs( new EbsBlockDevice(). withSnapshotId("snap-7eb96d16"). withVolumeSize(80). withDeleteOnTermination(false). withVolumeType("io1"). withIops(100) ) ); String customDeviceMappings = "/dev/sdc=snap-7eb96d16:80:false:io1:100"; List<BlockDeviceMapping> actual = DeviceMappingParser.parse(customDeviceMappings); assertEquals(expected, actual); }
public void testParserWithEncrypted() throws Exception { List<BlockDeviceMapping> expected = new ArrayList<BlockDeviceMapping>(); expected.add( new BlockDeviceMapping(). withDeviceName("/dev/sdd"). withEbs( new EbsBlockDevice(). withVolumeSize(120). withEncrypted(true) ) ); String customDeviceMappings = "/dev/sdd=:120::::encrypted"; List<BlockDeviceMapping> actual = DeviceMappingParser.parse(customDeviceMappings); assertEquals(expected, actual); }
public void testParserWithMultiple() throws Exception { List<BlockDeviceMapping> expected = new ArrayList<BlockDeviceMapping>(); expected.add( new BlockDeviceMapping(). withDeviceName("/dev/sdd"). withEbs( new EbsBlockDevice(). withVolumeSize(120). withEncrypted(true) ) ); expected.add( new BlockDeviceMapping(). withDeviceName("/dev/sdc"). withEbs( new EbsBlockDevice(). withVolumeSize(120) ) ); String customDeviceMappings = "/dev/sdd=:120::::encrypted,/dev/sdc=:120"; List<BlockDeviceMapping> actual = DeviceMappingParser.parse(customDeviceMappings); assertEquals(expected, actual); }
public void deleteImage(Image image) { String imageId = image.getImageId(); logger.info("delete image, imageId={}", imageId); ec2.deregisterImage(new DeregisterImageRequest(imageId)); // our image always uses EBS as first and only drive List<BlockDeviceMapping> mappings = image.getBlockDeviceMappings(); if (!mappings.isEmpty()) { EbsBlockDevice ebs = mappings.get(0).getEbs(); if (ebs != null) { String snapshotId = ebs.getSnapshotId(); logger.info("delete snapshot, snapshotId={}", snapshotId); ec2.deleteSnapshot(new DeleteSnapshotRequest(snapshotId)); } } }
public void testParserWithAmi() throws Exception { List<BlockDeviceMapping> expected = new ArrayList<BlockDeviceMapping>(); expected.add( new BlockDeviceMapping(). withDeviceName("/dev/sdb"). withEbs( new EbsBlockDevice(). withSnapshotId("snap-7eb96d16") ) ); String customDeviceMappings = "/dev/sdb=snap-7eb96d16"; List<BlockDeviceMapping> actual = DeviceMappingParser.parse(customDeviceMappings); assertEquals(expected, actual); }
public void testParserWithSize() throws Exception { List<BlockDeviceMapping> expected = new ArrayList<BlockDeviceMapping>(); expected.add( new BlockDeviceMapping(). withDeviceName("/dev/sdd"). withEbs( new EbsBlockDevice(). withVolumeSize(120) ) ); String customDeviceMappings = "/dev/sdd=:120"; List<BlockDeviceMapping> actual = DeviceMappingParser.parse(customDeviceMappings); assertEquals(expected, actual); }
@Override @Nullable public List<VirtualMachineImage> apply(@Nullable List<Image> input) { List<VirtualMachineImage> images = newArrayList(); for (Image image: input) { VirtualMachineImage vmi = new VirtualMachineImage() .setArchitecture(OsArchitectureType.valueOfFromValue(image.getArchitecture())) .setDefaultUsername(DEFAULT_PLATFORM_USER_NAME) .setHypervisor(HypervisorType.valueOfFromValue(image.getHypervisor())) .setName(image.getImageId()) .setPlatform(image.getPlatform() == null ? Platform.LINUX : Platform.valueOfFromValue(image.getPlatform())) .setRegion(credentials_.getRegion()) .setVirtualizationType(VirtualizationType.valueOfFromValue(image.getVirtualizationType())); if ("ebs".equals(image.getRootDeviceType())) { EbsBlockDevice ebs = image.getBlockDeviceMappings().get(0).getEbs(); org.excalibur.core.cloud.api.Volume volume = new org.excalibur.core.cloud.api.Volume() .setName(image.getRootDeviceName()) .setType(new VolumeType().setName(ebs.getVolumeType())) .setSizeGb(ebs.getVolumeSize()); vmi.setRootVolume(volume); } images.add(vmi); } return images; }
private String getDeviceType(EbsBlockDevice ebs) { return ebs != null ? AWSStorageType.EBS.getName() : AWSStorageType.INSTANCE_STORE.getName(); }
@Override protected DeferredResult<LocalStateHolder> buildLocalResourceState( Image remoteImage, ImageState existingImageState) { LocalStateHolder holder = new LocalStateHolder(); holder.localState = new ImageState(); if (existingImageState == null) { // Create flow if (this.request.requestType == ImageEnumerateRequestType.PUBLIC) { holder.localState.endpointType = this.endpointState.endpointType; } } else { // Update flow: do nothing } // Both flows - populate from remote Image holder.localState.name = remoteImage.getName(); holder.localState.description = remoteImage.getDescription(); holder.localState.osFamily = remoteImage.getPlatform(); holder.localState.diskConfigs = new ArrayList<>(); if (DeviceType.Ebs == DeviceType.fromValue(remoteImage.getRootDeviceType())) { for (BlockDeviceMapping blockDeviceMapping : remoteImage.getBlockDeviceMappings()) { // blockDeviceMapping can be with noDevice EbsBlockDevice ebs = blockDeviceMapping.getEbs(); if (ebs != null) { DiskConfiguration diskConfig = new DiskConfiguration(); diskConfig.id = blockDeviceMapping.getDeviceName(); diskConfig.encrypted = ebs.getEncrypted(); diskConfig.persistent = true; if (ebs.getVolumeSize() != null) { diskConfig.capacityMBytes = ebs.getVolumeSize() * 1024; } diskConfig.properties = Collections.singletonMap( VOLUME_TYPE, ebs.getVolumeType()); holder.localState.diskConfigs.add(diskConfig); } } } for (Tag remoteImageTag : remoteImage.getTags()) { holder.remoteTags.put(remoteImageTag.getKey(), remoteImageTag.getValue()); } return DeferredResult.completed(holder); }
public void spotRequest() { if (instances <= instancesSet.size()) { logger.info("No more instances will be launched because there are already enough."); return; } com.amazonaws.services.ec2.AmazonEC2 client = AmazonEC2.connect(); RequestSpotInstancesRequest requestRequest = new RequestSpotInstancesRequest(); requestRequest.setSpotPrice(Double.valueOf(maxPrice).toString()); // requestRequest.setInstanceCount(instances); requestRequest.setInstanceCount(instances - instancesSet.size()); LaunchSpecification launchSpecification = new LaunchSpecification(); launchSpecification.setImageId(ami); launchSpecification.setInstanceType(size); if (userData != null) launchSpecification.setUserData(Base64.encodeAsString(userData .getBytes())); BlockDeviceMapping blockDeviceMapping = new BlockDeviceMapping(); blockDeviceMapping.setDeviceName("/dev/sda1"); EbsBlockDevice ebs = new EbsBlockDevice(); ebs.setDeleteOnTermination(Boolean.TRUE); ebs.setVolumeSize(diskSize); blockDeviceMapping.setEbs(ebs); ArrayList<BlockDeviceMapping> blockList = new ArrayList<BlockDeviceMapping>(); blockList.add(blockDeviceMapping); launchSpecification.setBlockDeviceMappings(blockList); ArrayList<String> securityGroups = new ArrayList<String>(); securityGroups.add(Configuration.SECURITY_GROUP_NAME); launchSpecification.setSecurityGroups(securityGroups); launchSpecification.setKeyName(keyName); requestRequest.setLaunchSpecification(launchSpecification); RequestSpotInstancesResult requestResult = client .requestSpotInstances(requestRequest); List<SpotInstanceRequest> reqs = requestResult .getSpotInstanceRequests(); for (SpotInstanceRequest req : reqs) instancesSet.add(new Instance(this, req.getInstanceId(), req .getSpotInstanceRequestId())); }
public void onDemandRequest() { if (instances <= instancesSet.size()) { logger.info("No more instances will be launched because there are already enough."); return; } com.amazonaws.services.ec2.AmazonEC2 client = AmazonEC2.connect(); RunInstancesRequest request = new RunInstancesRequest(); // request.setMinCount(instances); // request.setMaxCount(instances); request.setMinCount(instances - instancesSet.size()); request.setMaxCount(instances - instancesSet.size()); request.setImageId(ami); request.setInstanceType(size); if (userData != null) request.setUserData(Base64.encodeAsString(userData .getBytes())); BlockDeviceMapping blockDeviceMapping = new BlockDeviceMapping(); blockDeviceMapping.setDeviceName("/dev/sda1"); EbsBlockDevice ebs = new EbsBlockDevice(); ebs.setDeleteOnTermination(Boolean.TRUE); ebs.setVolumeSize(diskSize); blockDeviceMapping.setEbs(ebs); ArrayList<BlockDeviceMapping> blockList = new ArrayList<BlockDeviceMapping>(); blockList.add(blockDeviceMapping); request.setBlockDeviceMappings(blockList); ArrayList<String> securityGroups = new ArrayList<String>(); securityGroups.add(Configuration.SECURITY_GROUP_NAME); request.setSecurityGroups(securityGroups); request.setKeyName(keyName); RunInstancesResult requestResult = client.runInstances(request); Reservation reservation = requestResult.getReservation(); reservation.getInstances(); for (com.amazonaws.services.ec2.model.Instance i : reservation.getInstances()) instancesSet.add(new Instance(this, i.getInstanceId(), null)); }
private void createInstance(Context context, int count, String subnetId) throws Exception { String sgId = resource.securityGroup.remoteSecurityGroup.getGroupId(); RunInstancesRequest request = new RunInstancesRequest() .withKeyName(resource.keyPair.remoteKeyPair.getKeyName()) .withInstanceType(resource.instanceType) .withImageId(resource.ami.imageId()) .withSubnetId(subnetId) .withSecurityGroupIds(sgId) .withMinCount(count) .withMaxCount(count) .withUserData(Base64.encodeBase64String(Strings.bytes(userData(context.env)))); if (EBS.enableEBSOptimized(resource.instanceType)) { request.withEbsOptimized(true); } if (resource.instanceProfile != null) request.withIamInstanceProfile(new IamInstanceProfileSpecification() .withName(resource.instanceProfile.remoteInstanceProfile.getInstanceProfileName())); if (resource.ebs.rootVolumeSize != null) { request.getBlockDeviceMappings().add(new BlockDeviceMapping() .withDeviceName("/dev/sda1") .withEbs(new EbsBlockDevice().withVolumeSize(resource.ebs.rootVolumeSize).withVolumeType(resource.ebs.type))); } List<com.amazonaws.services.ec2.model.Instance> remoteInstances = AWS.ec2.runInstances(request, tags(context.env)); resource.remoteInstances.addAll(remoteInstances); for (com.amazonaws.services.ec2.model.Instance remoteInstance : remoteInstances) { String key = String.format("instance/%s/%s", resource.id, remoteInstance.getInstanceId()); StringBuilder builder = new StringBuilder(); builder.append("privateIP=").append(remoteInstance.getPrivateIpAddress()); if (resource.subnet == null || resource.subnet.type == SubnetType.PUBLIC) { builder.append(", publicDNS=").append(remoteInstance.getPublicDnsName()); } context.output(key, builder.toString()); } if (resource.elb != null) { List<String> instanceIds = remoteInstances.stream().map(com.amazonaws.services.ec2.model.Instance::getInstanceId).collect(Collectors.toList()); AWS.elb.attachInstances(resource.elb.remoteELB.getLoadBalancerName(), instanceIds, waitUntilInService); } }
public BlockDeviceMapping getBlockDeviceMapping(String deviceName, int sizeInGb) { return new BlockDeviceMapping().withDeviceName(deviceName).withEbs( new EbsBlockDevice().withVolumeSize(sizeInGb)); }
public List<Instance> launchEc2Instances(AmazonEC2Client ec2Client, Properties props) throws Exception { Integer totalExpectedWorkers = Integer.valueOf(props.getProperty("master.workers.total")); // disk size Collection<BlockDeviceMapping> blockDevices = new ArrayList<BlockDeviceMapping>(); blockDevices.add( new BlockDeviceMapping() .withDeviceName(props.getProperty("master.workers.ec2.disk.deviceName")) .withEbs(new EbsBlockDevice() .withVolumeType(VolumeType.valueOf(props.getProperty("master.workers.ec2.disk.volumeType"))) .withDeleteOnTermination(true) .withVolumeSize(Integer.valueOf(props.getProperty("master.workers.ec2.disk.size.gigabytes"))))); // create our run request for the total workers we expect RunInstancesRequest runInstancesRequest = new RunInstancesRequest(); runInstancesRequest.withImageId(props.getProperty("master.workers.ec2.ami.id")) .withInstanceType(props.getProperty("master.workers.ec2.instanceType")) .withMinCount(totalExpectedWorkers) .withMaxCount(totalExpectedWorkers) .withBlockDeviceMappings(blockDevices) .withKeyName(props.getProperty("master.workers.ec2.keyName")) .withSecurityGroupIds(props.getProperty("master.workers.ec2.securityGroupId")) .withInstanceInitiatedShutdownBehavior(ShutdownBehavior.valueOf(props.getProperty("master.workers.ec2.shutdownBehavior"))) .withSubnetId(props.getProperty("master.workers.ec2.subnetId")) .withUserData(Base64.encodeAsString(readFile(props.getProperty("master.workers.ec2.userDataFile")).getBytes())); // launch logger.debug("Launching " + totalExpectedWorkers + " EC2 instances, " + "it may take few minutes for workers to come up...: \n" + "\tamiId:" + runInstancesRequest.getImageId() +"\n"+ "\tsecGrpId:" + runInstancesRequest.getSecurityGroupIds().get(0) +"\n"+ "\tsubnetId:" + runInstancesRequest.getSubnetId() +"\n"+ "\tinstanceType:" + runInstancesRequest.getInstanceType() +"\n"+ "\tshutdownBehavior:" + runInstancesRequest.getInstanceInitiatedShutdownBehavior() +"\n"+ "\tkeyName:" + runInstancesRequest.getKeyName() ); // as the instances come up, assuming the "userData" above launches the worker we will be good // they will auto register w/ us the master RunInstancesResult result = ec2Client.runInstances(runInstancesRequest); Reservation reservation = result.getReservation(); return reservation.getInstances(); }
private AmazonWebServiceRequest createRequest(Pool pool, DelegateExecution execution, boolean spot) throws IOException { final String businessKey = execution.getProcessBusinessKey(); final String securityGroupName = SecurityGroups.formatNameFromBusinessKey(businessKey); final String keyPairName = KeyPairs.formatNameFromBusinessKey(businessKey); final String instanceType = pool.getHardware().getType(); final String imageId = getImageIdFromPoolConfigurationOrQueryImageTable( pool, pool.getProvider(), instanceType); final String userData = Resources.toString(Resources.getResource(RunInstances.class, "/org/apache/provisionr/amazon/userdata.sh"), Charsets.UTF_8); List<BlockDevice> blockDevices = pool.getHardware().getBlockDevices(); List<BlockDeviceMapping> blockDeviceMappings = Lists.newArrayList(); if (blockDevices != null && blockDevices.size() > 0) { for (BlockDevice device : blockDevices) { blockDeviceMappings.add(new BlockDeviceMapping() .withDeviceName(device.getName()) .withEbs(new EbsBlockDevice() .withVolumeSize(device.getSize()) .withDeleteOnTermination(true) )); } } if (spot) { Calendar validUntil = Calendar.getInstance(); validUntil.add(Calendar.MINUTE, 10); final String spotPrice = checkNotNull(pool.getProvider().getOption(ProviderOptions.SPOT_BID), "The bid for spot instances was not specified"); LaunchSpecification ls = new LaunchSpecification() .withInstanceType(instanceType) .withKeyName(keyPairName) .withImageId(imageId) .withBlockDeviceMappings(blockDeviceMappings) .withSecurityGroups(Lists.newArrayList(securityGroupName)) .withUserData(Base64.encodeBytes(userData.getBytes(Charsets.UTF_8))); return new RequestSpotInstancesRequest() .withSpotPrice(spotPrice) .withLaunchSpecification(ls) .withLaunchGroup(businessKey) .withInstanceCount(pool.getExpectedSize()) .withType(SpotInstanceType.OneTime) .withValidUntil(validUntil.getTime()); } else { return new RunInstancesRequest() .withClientToken(businessKey) .withSecurityGroups(securityGroupName) .withKeyName(keyPairName) .withInstanceType(instanceType) .withImageId(imageId) .withBlockDeviceMappings(blockDeviceMappings) .withMinCount(pool.getMinSize()) .withMaxCount(pool.getExpectedSize()) .withUserData(Base64.encodeBytes(userData.getBytes(Charsets.UTF_8))); } }
/** delete AMI image and related EBS snapshots */ public void imageDelete(final String imageId) throws Exception { final Image image = findImage(imageId); if (image == null) { logger.info("missing imageId = " + imageId); return; } else { logger.info("present imageId = " + imageId); } imageUnregister(imageId); for (final BlockDeviceMapping blockDevice : image .getBlockDeviceMappings()) { final EbsBlockDevice elasticDevice = blockDevice.getEbs(); if (elasticDevice == null) { continue; } final String snapshotId = elasticDevice.getSnapshotId(); if (snapshotId == null) { continue; } snapshotDelete(snapshotId); } logger.info("removed imageId = " + imageId); }
@Override public void execute() throws MojoFailureException { try { getLog().info("image create init [" + imageName() + "]"); final CarrotElasticCompute compute = newElasticCompute(); final Image image = compute.imageCreate( // imageInstanceId(), // imageName(), // imageDescription // ); final ImageState state = ImageState.fromValue(image.getState()); switch (state) { case AVAILABLE: break; default: throw new IllegalStateException("image create failed : \n" + image); } final String imageId = image.getImageId(); /** publish result */ project().getProperties().put(imageResultProperty, image); project().getProperties().put(imageIdResultProperty, imageId); /** tag image */ compute.tagCreate(imageId, amazonTagName(), imageName()); /** tag image devices */ for (final BlockDeviceMapping blockDevice : image .getBlockDeviceMappings()) { final EbsBlockDevice elasticDevice = blockDevice.getEbs(); if (elasticDevice == null) { continue; } final String snapshotId = elasticDevice.getSnapshotId(); if (snapshotId == null) { continue; } compute.tagCreate(snapshotId, amazonTagName(), imageName()); } getLog().info("image create image=\n" + image); getLog().info("image create done [" + imageName() + "]"); } catch (final Exception e) { throw new MojoFailureException("bada-boom", e); } }