com.amazonaws.services.elasticmapreduce.model.ScriptBootstrapActionConfig Java Examples

/**
 * Create the BootstrapActionConfig object from the bootstrap script.
 *
 * @param scriptDescription bootstrap script name to be displayed.
 * @param bootstrapScript location of the bootstrap script.
 *
 * @return bootstrap action configuration that contains all the bootstrap actions for the given configuration.
 */
private BootstrapActionConfig getBootstrapActionConfig(String scriptDescription, String bootstrapScript)
{
    // Create the BootstrapActionConfig object
    BootstrapActionConfig bootstrapConfig = new BootstrapActionConfig();
    ScriptBootstrapActionConfig bootstrapConfigScript = new ScriptBootstrapActionConfig();

    // Set the bootstrapScript
    bootstrapConfig.setName(scriptDescription);
    bootstrapConfigScript.setPath(bootstrapScript);
    bootstrapConfig.setScriptBootstrapAction(bootstrapConfigScript);

    // Return the object
    return bootstrapConfig;
}
 

private BootstrapActionConfig bootstrapAction(int index, JsonNode action, String tag, Filer filer, RemoteFile runner, ParameterCompiler parameterCompiler)
        throws IOException
{
    String script;
    String name;
    FileReference reference;

    Config config;
    if (action.isTextual()) {
        script = action.asText();
        reference = fileReference("bootstrap", script);
        name = reference.filename();
        config = request.getConfig().getFactory().create();
    }
    else if (action.isObject()) {
        config = request.getConfig().getFactory().create(action);
        script = config.get("path", String.class);
        reference = fileReference("bootstrap", script);
        name = config.get("name", String.class, reference.filename());
    }
    else {
        throw new ConfigException("Invalid bootstrap action: " + action);
    }

    RemoteFile file = filer.prepareRemoteFile(tag, "bootstrap", Integer.toString(index), reference, false);

    CommandRunnerConfiguration configuration = CommandRunnerConfiguration.builder()
            .workingDirectory(bootstrapWorkingDirectory(index))
            .env(parameterCompiler.parameters(config.getNestedOrGetEmpty("env"), (key, value) -> value))
            .addDownload(DownloadConfig.of(file, 0777))
            .addAllDownload(config.getListOrEmpty("files", String.class).stream()
                    .map(r -> fileReference("file", r))
                    .map(r -> filer.prepareRemoteFile(tag, "bootstrap", Integer.toString(index), r, false, bootstrapWorkingDirectory(index)))
                    .collect(toList()))
            .addCommand(file.localPath())
            .addAllCommand(parameterCompiler.parameters(config, "args"))
            .build();

    FileReference configurationFileReference = ImmutableFileReference.builder()
            .type(FileReference.Type.DIRECT)
            .contents(objectMapper.writeValueAsBytes(configuration))
            .filename("config.json")
            .build();
    RemoteFile remoteConfigurationFile = filer.prepareRemoteFile(tag, "bootstrap", Integer.toString(index), configurationFileReference, false);

    return new BootstrapActionConfig()
            .withName(name)
            .withScriptBootstrapAction(new ScriptBootstrapActionConfig()
                    .withPath(runner.s3Uri().toString())
                    .withArgs(remoteConfigurationFile.s3Uri().toString()));
}
 

/**
 * This method uses method the AWS Java to launch an Apache HBase cluster on Amazon EMR. 
 * 
 * @param client - AmazonElasticMapReduce client that interfaces directly with the Amazon EMR Web Service
 * @param clusterIdentifier - identifier of an existing cluster
 * @param amiVersion - AMI to use for launching this cluster
 * @param keypair - A keypair for SSHing into the Amazon EMR master node
 * @param masterInstanceType - Master node Amazon EC2 instance type 
 * @param coreInstanceType - core nodes Amazon EC2 instance type 
 * @param logUri - An Amazon S3 bucket for your 
 * @param numberOfNodes - total number of nodes in this cluster including master node
 * @return
 */
public static String createCluster(AmazonElasticMapReduce client,
		String clusterIdentifier,
		String amiVersion,
		String keypair,
		String masterInstanceType,
		String coreInstanceType,
		String logUri,
		int numberOfNodes) {

	if (clusterExists(client, clusterIdentifier)) {
		LOG.info("Cluster " + clusterIdentifier + " is available");
		return clusterIdentifier;
	}
	
	//Error checking
	if (amiVersion == null || amiVersion.isEmpty()) throw new RuntimeException("ERROR: Please specify an AMI Version");
	if (keypair == null || keypair.isEmpty()) throw new RuntimeException("ERROR: Please specify a valid Amazon Key Pair");
	if (masterInstanceType == null || masterInstanceType.isEmpty()) throw new RuntimeException("ERROR: Please specify a Master Instance Type");
	if (logUri == null || logUri.isEmpty()) throw new RuntimeException("ERROR: Please specify a valid Amazon S3 bucket for your logs.");
	if (numberOfNodes < 0) throw new RuntimeException("ERROR: Please specify at least 1 node");
	  		
	  RunJobFlowRequest request = new RunJobFlowRequest()
	    .withAmiVersion(amiVersion)
		.withBootstrapActions(new BootstrapActionConfig()
		             .withName("Install HBase")
		             .withScriptBootstrapAction(new ScriptBootstrapActionConfig()
		             .withPath("s3://elasticmapreduce/bootstrap-actions/setup-hbase")))
		.withName("Job Flow With HBAse Actions")	 
		.withSteps(new StepConfig() //enable debugging step
					.withName("Enable debugging")
					.withActionOnFailure("TERMINATE_CLUSTER")
					.withHadoopJarStep(new StepFactory().newEnableDebuggingStep()), 
					//Start HBase step - after installing it with a bootstrap action
					createStepConfig("Start HBase","TERMINATE_CLUSTER", "/home/hadoop/lib/hbase.jar", getHBaseArgs()), 
					//add HBase backup step
					createStepConfig("Modify backup schedule","TERMINATE_JOB_FLOW", "/home/hadoop/lib/hbase.jar", getHBaseBackupArgs()))
		.withLogUri(logUri)
		.withInstances(new JobFlowInstancesConfig()
		.withEc2KeyName(keypair)
		.withInstanceCount(numberOfNodes)
		.withKeepJobFlowAliveWhenNoSteps(true)
		.withMasterInstanceType(masterInstanceType)
		.withSlaveInstanceType(coreInstanceType));

	RunJobFlowResult result = client.runJobFlow(request);
	
	String state = null;
	while (!(state = clusterState(client, result.getJobFlowId())).equalsIgnoreCase("waiting")) {
		try {
			Thread.sleep(10 * 1000);
			LOG.info(result.getJobFlowId() + " is " + state + ". Waiting for cluster to become available.");
		} catch (InterruptedException e) {

		}
		
		if (state.equalsIgnoreCase("TERMINATED_WITH_ERRORS")){
			LOG.error("Could not create EMR Cluster");
			System.exit(-1);	
		}
	}
	LOG.info("Created cluster " + result.getJobFlowId());
	LOG.info("Cluster " + clusterIdentifier + " is available");	
	return result.getJobFlowId();
}