com.amazonaws.services.elasticmapreduce.model.DescribeClusterResult Java Examples

The following examples show how to use com.amazonaws.services.elasticmapreduce.model.DescribeClusterResult. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: EmrClusterTableProvider.java    From aws-athena-query-federation with Apache License 2.0 6 votes vote down vote up
/**
 * Calls ListClusters and DescribeCluster on the AWS EMR Client returning all clusters that match the supplied
 * predicate and attempting to push down certain predicates (namely queries for specific cluster) to EC2.
 *
 * @See TableProvider
 */
@Override
public void readWithConstraint(BlockSpiller spiller, ReadRecordsRequest recordsRequest, QueryStatusChecker queryStatusChecker)
{
    boolean done = false;
    ListClustersRequest request = new ListClustersRequest();

    while (!done) {
        ListClustersResult response = emr.listClusters(request);

        for (ClusterSummary next : response.getClusters()) {
            Cluster cluster = null;
            if (!next.getStatus().getState().toLowerCase().contains("terminated")) {
                DescribeClusterResult clusterResponse = emr.describeCluster(new DescribeClusterRequest().withClusterId(next.getId()));
                cluster = clusterResponse.getCluster();
            }
            clusterToRow(next, cluster, spiller);
        }

        request.setMarker(response.getMarker());

        if (response.getMarker() == null || !queryStatusChecker.isQueryRunning()) {
            done = true;
        }
    }
}
 
Example #2
Source File: ClusterManager.java    From herd-mdl with Apache License 2.0 6 votes vote down vote up
@Override
public void afterPropertiesSet() throws Exception {
    createEmrClient();

    String emrInfo = new String(Files.readAllBytes(Paths.get(jobFlowJson)));
    JsonReader reader = Json.createReader(new StringReader(emrInfo));
    JsonObject object = reader.readObject();
    if(object.containsKey("jobFlowId"))
    {
        setClusterID(object.getString("jobFlowId"));
        logger.info("Cluster ID = "+ clusterID);
        DescribeClusterRequest req = new DescribeClusterRequest().withClusterId(clusterID);
        DescribeClusterResult result = emrClient.describeCluster(req);
        String name = result.getCluster().getName();
        logger.info("Cluster Name = "+ name);
        setClusterName(name.substring(name.lastIndexOf(".")+1));
    }

}
 
Example #3
Source File: EmrDaoImpl.java    From herd with Apache License 2.0 6 votes vote down vote up
@Override
public Cluster getEmrClusterById(String clusterId, AwsParamsDto awsParams)
{
    Cluster cluster = null;
    if (StringUtils.isNotBlank(clusterId))
    {
        DescribeClusterResult describeClusterResult =
            emrOperations.describeClusterRequest(getEmrClient(awsParams), new DescribeClusterRequest().withClusterId(clusterId));
        if (describeClusterResult != null && describeClusterResult.getCluster() != null)
        {
            cluster = describeClusterResult.getCluster();
        }
    }

    return cluster;
}
 
Example #4
Source File: TestEmrClusterJob.java    From datacollector with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetClusterStatus() {
  Properties properties = new Properties();
  EmrClusterJob emrClusterJob = new EmrClusterJob();
  EmrClusterJob.Client client = Mockito.spy(emrClusterJob.getClient(properties));
  AmazonElasticMapReduce emr = Mockito.mock(AmazonElasticMapReduce.class);
  Mockito.doReturn(emr).when(client).getEmrClient(Mockito.any(EmrClusterConfig.class));
  DescribeClusterResult result = Mockito.mock(DescribeClusterResult.class);
  Mockito.doReturn(result).when(emr).describeCluster(Mockito.any(DescribeClusterRequest
      .class));
  Cluster cluster = Mockito.mock(Cluster.class);
  Mockito.doReturn(cluster).when(result).getCluster();
  Mockito.doReturn(Mockito.mock(ClusterStatus.class)).when(cluster).getStatus();
  client.getClusterStatus("foo");
  Mockito.verify(emr, Mockito.times(1)).describeCluster(Mockito.any(DescribeClusterRequest
      .class));
  Mockito.verify(client, Mockito.times(1)).getEmrClient(Mockito.any(EmrClusterConfig.class));
}
 
Example #5
Source File: InventoryUtil.java    From pacbot with Apache License 2.0 5 votes vote down vote up
/**
 * Fetch EMR info.
 *
 * @param temporaryCredentials the temporary credentials
 * @param skipRegions the skip regions
 * @param accountId the accountId
 * @param accountName the account name
 * @return the map
 */
public static Map<String,List<Cluster>> fetchEMRInfo(BasicSessionCredentials temporaryCredentials, String skipRegions,String accountId,String accountName){

	Map<String,List<Cluster>> clusterList = new LinkedHashMap<>();
	String expPrefix = InventoryConstants.ERROR_PREFIX_CODE+accountId + "\",\"Message\": \"Exception in fetching info for resource in specific region\" ,\"type\": \"EMR\" , \"region\":\"" ;
	for(Region region : RegionUtils.getRegions()){
		try{
			if(!skipRegions.contains(region.getName())){
				AmazonElasticMapReduce emrClient = AmazonElasticMapReduceClientBuilder.standard().
				 	withCredentials(new AWSStaticCredentialsProvider(temporaryCredentials)).withRegion(region.getName()).build();
				List<ClusterSummary> clusters = new ArrayList<>();
				String marker = null;
				ListClustersResult clusterResult ;
				do{
					clusterResult = emrClient.listClusters(new ListClustersRequest().withMarker(marker));
					clusters.addAll(clusterResult.getClusters());
					marker = clusterResult.getMarker();
				}while(marker!=null);

				List<Cluster> clustersList = new ArrayList<>();
				clusters.forEach(cluster ->
					{
						DescribeClusterResult descClstrRslt = emrClient.describeCluster(new DescribeClusterRequest().withClusterId(cluster.getId()));
						clustersList.add(descClstrRslt.getCluster());
					});

				if( !clustersList.isEmpty() ){
					log.debug(InventoryConstants.ACCOUNT + accountId +" Type : EMR "+region.getName() + " >> "+clustersList.size());
					clusterList.put(accountId+delimiter+accountName+delimiter+region.getName(),clustersList);
				}
			}
		}catch(Exception e){
			if(region.isServiceSupported(AmazonElasticMapReduce.ENDPOINT_PREFIX)){
				log.warn(expPrefix+ region.getName()+InventoryConstants.ERROR_CAUSE +e.getMessage()+"\"}");
				ErrorManageUtil.uploadError(accountId,region.getName(),"emr",e.getMessage());
			}
		}
	}
	return clusterList;
}
 
Example #6
Source File: InventoryUtilTest.java    From pacbot with Apache License 2.0 5 votes vote down vote up
/**
 * Fetch EMR info test.
 *
 * @throws Exception the exception
 */
@SuppressWarnings("static-access")
@Test
public void fetchEMRInfoTest() throws Exception {
    
    mockStatic(AmazonElasticMapReduceClientBuilder.class);
    AmazonElasticMapReduce emrClient = PowerMockito.mock(AmazonElasticMapReduce.class);
    AmazonElasticMapReduceClientBuilder amazonElasticFileSystemClientBuilder = PowerMockito.mock(AmazonElasticMapReduceClientBuilder.class);
    AWSStaticCredentialsProvider awsStaticCredentialsProvider = PowerMockito.mock(AWSStaticCredentialsProvider.class);
    PowerMockito.whenNew(AWSStaticCredentialsProvider.class).withAnyArguments().thenReturn(awsStaticCredentialsProvider);
    when(amazonElasticFileSystemClientBuilder.standard()).thenReturn(amazonElasticFileSystemClientBuilder);
    when(amazonElasticFileSystemClientBuilder.withCredentials(anyObject())).thenReturn(amazonElasticFileSystemClientBuilder);
    when(amazonElasticFileSystemClientBuilder.withRegion(anyString())).thenReturn(amazonElasticFileSystemClientBuilder);
    when(amazonElasticFileSystemClientBuilder.build()).thenReturn(emrClient);
    
    ListClustersResult listClustersResult = new ListClustersResult();
    List<ClusterSummary> clusters = new ArrayList<>();
    ClusterSummary clusterSummary = new ClusterSummary();
    clusterSummary.setId("id");
    clusters.add(clusterSummary);
    listClustersResult.setClusters(clusters);
    when(emrClient.listClusters(anyObject())).thenReturn(listClustersResult);
    
    DescribeClusterResult describeClusterResult = new DescribeClusterResult();
    describeClusterResult.setCluster(new Cluster());
    when(emrClient.describeCluster(anyObject())).thenReturn(describeClusterResult);
    assertThat(inventoryUtil.fetchEMRInfo(new BasicSessionCredentials("awsAccessKey", "awsSecretKey", "sessionToken"), 
            "skipRegions", "account","accountName").size(), is(1));
    
}
 
Example #7
Source File: EmrDaoTest.java    From herd with Apache License 2.0 5 votes vote down vote up
@Test
public void getEmrClusterByIdAssertReturnNullWhenDescribeClusterResponseClusterIsNull()
{
    String clusterId = "clusterId";

    when(mockEmrOperations.describeClusterRequest(any(), any())).thenReturn(new DescribeClusterResult());
    assertNull(emrDao.getEmrClusterById(clusterId, getAwsParamsDto()));
}
 
Example #8
Source File: EmrClusterJob.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
YarnClient getYarnClient(String clusterId, EmrClusterConfig emrClusterConfig) {
  DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest().withClusterId(clusterId);
  DescribeClusterResult result= getEmrClient(emrClusterConfig).describeCluster(describeClusterRequest);
  String rmHostname =  result.getCluster().getMasterPublicDnsName();
  if (yarnClient == null) {
    yarnClient = YarnClient.createYarnClient();
    org.apache.hadoop.conf.Configuration configuration = new org.apache.hadoop.conf.Configuration();
    configuration.set("yarn.resourcemanager.hostname", rmHostname);
    yarnClient.init(configuration);
    yarnClient.start();
  }
  return yarnClient;
}
 
Example #9
Source File: EmrClusterJob.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Override
public Properties getClusterStatus(String clusterId) {
  DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest()
      .withClusterId(clusterId);
  DescribeClusterResult result= getEmrClient(emrClusterConfig).describeCluster(describeClusterRequest);
  EmrState emrState = new EmrState();
  ClusterStatus clusterStatus = result.getCluster().getStatus();
  emrState.setState(clusterStatus.getState());
  if (clusterStatus.getStateChangeReason() != null) {
    emrState.setMessage(clusterStatus.getStateChangeReason().getCode() + ":" + clusterStatus.getStateChangeReason
        ().getMessage());
  }
  LOG.debug("State of cluster: {} is {}", clusterId, clusterStatus.getState());
  return emrState.toProperties();
}
 
Example #10
Source File: EMRUtils.java    From aws-big-data-blog with Apache License 2.0 5 votes vote down vote up
/**
 * Helper method to determine if an Amazon EMR cluster exists
 * 
 * @param client
 *        The {@link AmazonElasticMapReduceClient} with read permissions
 * @param clusterIdentifier
 *        The Amazon EMR cluster to check
 * @return true if the Amazon EMR cluster exists, otherwise false
 */
public static boolean clusterExists(AmazonElasticMapReduce client, String clusterIdentifier) {
	if (clusterIdentifier != null && !clusterIdentifier.isEmpty()) {
		ListClustersResult clustersList = client.listClusters();
		ListIterator<ClusterSummary> iterator = clustersList.getClusters().listIterator();
		ClusterSummary summary;
		for (summary = iterator.next() ; iterator.hasNext();summary = iterator.next()) {
			if (summary.getId().equals(clusterIdentifier)) {
				DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest().withClusterId(clusterIdentifier);	
				DescribeClusterResult result = client.describeCluster(describeClusterRequest);	
				if (result != null) {
					Cluster cluster = result.getCluster();
					//check if HBase is installed on this cluster
					if (isHBaseInstalled(client, cluster.getId())) return false;
					String state = cluster.getStatus().getState();
					LOG.info(clusterIdentifier + " is " + state + ". ");
					if (state.equalsIgnoreCase("RUNNING") ||state.equalsIgnoreCase("WAITING"))	{
						LOG.info("The cluster with id " + clusterIdentifier + " exists and is " + state);   
						return true;
					}
				}
			}		
		}					
	}
	LOG.info("The cluster with id " + clusterIdentifier + " does not exist");
	return false;  
}
 
Example #11
Source File: ClusterManager.java    From herd-mdl with Apache License 2.0 4 votes vote down vote up
public boolean registerCluster() {
    String sql = "INSERT IGNORE INTO `EMR_CLUSTER` (`CLUSTER_NAME`,`CLUSTER_ID`, CREATE_TIME, STATUS) VALUES (?, ?, now(), 'R')";

    template.update(sql, clusterName, clusterID);

    String runningClusterID = template.queryForObject(SELECT_CLUSTER_ID_SQL, new Object[]{clusterName}, String.class);
    boolean continueProcess = false;

    if (!runningClusterID.equals(clusterID)) {
        //Verify the running cluster is still up

        DescribeClusterRequest req = new DescribeClusterRequest().withClusterId(runningClusterID);
        DescribeClusterResult result = emrClient.describeCluster(req);

        String clusterState = result.getCluster().getStatus().getState();

        if (clusterState.equals(ClusterState.TERMINATED.toString()) || clusterState.equals(ClusterState.TERMINATED_WITH_ERRORS.toString())) {
            continueProcess = prepareProcessing();
        } else if (clusterState.equals(ClusterState.TERMINATING.toString())) {
            //Wait till cluster is terminated
            logger.info(String.format("Running Cluster %s is in TERMINCATING state, waiting for it to be terminated", runningClusterID));

            do {
                try {
                    Thread.sleep(30000);
                } catch (InterruptedException ex) {

                }
                result = emrClient.describeCluster(req);
                clusterState = result.getCluster().getStatus().getState();
            } while (clusterState.equals(ClusterState.TERMINATING.toString()));

            if (clusterState.equals(ClusterState.TERMINATED.toString()) || clusterState.equals(ClusterState.TERMINATED_WITH_ERRORS)) {
                continueProcess = prepareProcessing();
            }

        } else {
            logger.info("Running cluster is " + runningClusterID + ", exiting");
        }

    } else {
        continueProcess = true;
    }

    return continueProcess;
}
 
Example #12
Source File: ClusterManager.java    From herd-mdl with Apache License 2.0 4 votes vote down vote up
public void clusterAutoScale()
  {
      logger.info("Auto Scale Job started");
      Map<String, Object> lastCheck = template.queryForMap(AUTO_SCALE_QUERY);
      long minutes = (Long) lastCheck.get("age");
      if(minutes >= autoScaleIntervalInMin) {
          long id = (Long)lastCheck.get("ID") + 1;
          int updated = template.update("INSERT IGNORE INTO METASTOR_EMR_AUTOSCALE (`ID`," +
                  "`CLUSTER_ID`) VALUES (?, ?)", id, clusterID);

          if (updated > 0) {

              int clusterNumber = calculateNumberOfClustersNeeded();
              template.update("UPDATE METASTOR_EMR_AUTOSCALE SET TOTAL_CLUSTER= ? WHERE ID=?", clusterNumber, id);
              if (clusterNumber <= 1) return;

              List<Map<String, Object>> clusters = template.queryForList("select * from EMR_CLUSTER");
              List<String> existingCluster = new ArrayList<>();

              createEmrClient();
              for (Map<String, Object> record : clusters) {
                  String cid = (String) record.get("CLUSTER_ID");
                  String cname = (String) record.get("CLUSTER_NAME");

                  boolean clusterIsAlive = true;

                  try {

                      DescribeClusterRequest req = new DescribeClusterRequest().withClusterId(cid);
                      DescribeClusterResult result = emrClient.describeCluster(req);

                      String clusterState = result.getCluster().getStatus().getState();
                      if (clusterState.equals(ClusterState.TERMINATED.toString()) || clusterState.equals(ClusterState.TERMINATED_WITH_ERRORS.toString())) {
                          deleteCluster(cid);
                          clusterIsAlive = false;
                      }
                  } catch (Exception ex) {
                      logger.warning("Error getting info for cluster " + cid + ex.getMessage());
                      clusterIsAlive = false;
                  }


                  if (clusterIsAlive) {
                      existingCluster.add(cname);
                  }

              }

              // Start Additional if required
		startAdditionalClusters( clusterNumber, existingCluster );
	}
      }
logger.info("Auto Scale Job completed");

  }
 
Example #13
Source File: EmrDaoImplTest.java    From herd with Apache License 2.0 4 votes vote down vote up
private void getActiveEmrClusterByNameAndAccountIdClusterNameIsInCache(Cluster cluster, String accountId)
{
    // Create an AWS parameters DTO.
    AwsParamsDto awsParamsDto =
        new AwsParamsDto(AWS_ASSUMED_ROLE_ACCESS_KEY, AWS_ASSUMED_ROLE_SECRET_KEY, AWS_ASSUMED_ROLE_SESSION_TOKEN, HTTP_PROXY_HOST, HTTP_PROXY_PORT,
            AWS_REGION_NAME_US_EAST_1);

    // Create a mock AmazonElasticMapReduceClient.
    AmazonElasticMapReduceClient amazonElasticMapReduceClient = mock(AmazonElasticMapReduceClient.class);

    // Create a cluster summary.
    ClusterSummary clusterSummary =
        new ClusterSummary().withName(EMR_CLUSTER_NAME).withId(EMR_CLUSTER_ID).withStatus(cluster == null ? null : cluster.getStatus());

    // Create a list cluster result with the matching cluster.
    ListClustersResult listClusterResult = new ListClustersResult().withClusters(clusterSummary);

    // Create a describe cluster result.
    DescribeClusterResult describeClusterResult = new DescribeClusterResult().withCluster(cluster);

    // Create a describe cluster request.
    DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest().withClusterId(EMR_CLUSTER_ID);

    // Build the EMR cluster cache key
    EmrClusterCacheKey emrClusterCacheKey = new EmrClusterCacheKey(EMR_CLUSTER_NAME.toUpperCase(), accountId);

    // Build the EMR cluster cache
    Map<EmrClusterCacheKey, String> emrClusterCache = new ConcurrentHashMap<>();
    emrClusterCache.put(emrClusterCacheKey, EMR_CLUSTER_ID);

    // Mock the external calls.
    if (accountId == null)
    {
        when(emrClusterCacheMap.get(EMR_CLUSTER_CACHE_MAP_DEFAULT_AWS_ACCOUNT_ID_KEY)).thenReturn(emrClusterCache);
    }
    else
    {
        when(emrClusterCacheMap.get(accountId)).thenReturn(emrClusterCache);
    }

    when(emrOperations.describeClusterRequest(eq(amazonElasticMapReduceClient), any(DescribeClusterRequest.class))).thenReturn(describeClusterResult);
    when(configurationHelper.getProperty(ConfigurationValue.EMR_VALID_STATES)).thenReturn(ConfigurationValue.EMR_VALID_STATES.getDefaultValue().toString());
    when(configurationHelper.getProperty(ConfigurationValue.FIELD_DATA_DELIMITER))
        .thenReturn((String) ConfigurationValue.FIELD_DATA_DELIMITER.getDefaultValue());
    when(awsClientFactory.getEmrClient(awsParamsDto)).thenReturn(amazonElasticMapReduceClient);
    when(emrOperations.listEmrClusters(any(AmazonElasticMapReduceClient.class), any(ListClustersRequest.class))).thenReturn(listClusterResult);

    // Call the method under test.
    ClusterSummary result = emrDaoImpl.getActiveEmrClusterByNameAndAccountId(EMR_CLUSTER_NAME, accountId, awsParamsDto);

    // Verify the external calls.
    verify(emrOperations).describeClusterRequest(eq(amazonElasticMapReduceClient), eq(describeClusterRequest));

    if (cluster == null)
    {
        verify(configurationHelper).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
        verify(configurationHelper).getProperty(ConfigurationValue.EMR_VALID_STATES);
        verify(awsClientFactory, times(2)).getEmrClient(awsParamsDto);
        verify(emrOperations).listEmrClusters(eq(amazonElasticMapReduceClient), any(ListClustersRequest.class));
    }
    else if (cluster.getStatus().getState().equals(EMR_INVALID_STATE))
    {
        verify(configurationHelper, times(2)).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
        verify(configurationHelper, times(2)).getProperty(ConfigurationValue.EMR_VALID_STATES);
        verify(awsClientFactory, times(2)).getEmrClient(awsParamsDto);
        verify(emrOperations).listEmrClusters(eq(amazonElasticMapReduceClient), any(ListClustersRequest.class));
    }
    else
    {
        verify(configurationHelper).getProperty(ConfigurationValue.FIELD_DATA_DELIMITER);
        verify(configurationHelper).getProperty(ConfigurationValue.EMR_VALID_STATES);
        verify(awsClientFactory).getEmrClient(awsParamsDto);
    }

    verifyNoMoreInteractionsHelper();

    // Validate the results.
    assertEquals(clusterSummary, result);
}
 
Example #14
Source File: EmrOperatorFactory.java    From digdag with Apache License 2.0 4 votes vote down vote up
private Optional<String> checkClusterBootStatus(AmazonElasticMapReduce emr, NewCluster cluster, TaskState state)
{
    // Only creating a cluster, with no steps?
    boolean createOnly = cluster.steps() == 0;

    DescribeClusterResult describeClusterResult = pollingRetryExecutor(state, "describe-cluster")
            .withRetryInterval(DurationInterval.of(Duration.ofSeconds(30), Duration.ofMinutes(5)))
            .retryUnless(AmazonServiceException.class, Aws::isDeterministicException)
            .run(ds -> emr.describeCluster(new DescribeClusterRequest().withClusterId(cluster.id())));

    ClusterStatus clusterStatus = describeClusterResult.getCluster().getStatus();
    String clusterState = clusterStatus.getState();

    switch (clusterState) {
        case "STARTING":
            logger.info("EMR cluster starting: {}", cluster.id());
            return Optional.absent();
        case "BOOTSTRAPPING":
            logger.info("EMR cluster bootstrapping: {}", cluster.id());
            return Optional.absent();

        case "RUNNING":
        case "WAITING":
            logger.info("EMR cluster up: {}", cluster.id());
            return Optional.of(clusterState);

        case "TERMINATED_WITH_ERRORS":
            if (createOnly) {
                // TODO: log more information about the errors
                // TODO: inspect state change reason to figure out whether it was the boot that failed or e.g. steps submitted by another agent
                throw new TaskExecutionException("EMR boot failed: " + cluster.id());
            }
            return Optional.of(clusterState);

        case "TERMINATING":
            if (createOnly) {
                // Keep waiting for the final state
                // TODO: inspect state change reason and bail early here
                return Optional.absent();
            }
            return Optional.of(clusterState);

        case "TERMINATED":
            return Optional.of(clusterState);

        default:
            throw new RuntimeException("Unknown EMR cluster state: " + clusterState);
    }
}
 
Example #15
Source File: EMRUtils.java    From aws-big-data-blog with Apache License 2.0 3 votes vote down vote up
/**
 * Helper method to determine the Amazon EMR cluster state
 * 
 * @param client
 *        The {@link AmazonElasticMapReduceClient} with read permissions
 * @param clusterIdentifier
 *        The Amazon EMR cluster to get the state of - e.g. j-2A98VJHDSU48M
 * @return The String representation of the Amazon EMR cluster state
 */
public static String clusterState(AmazonElasticMapReduce client, String clusterIdentifier) {
	DescribeClusterRequest describeClusterRequest = new DescribeClusterRequest().withClusterId(clusterIdentifier);
	DescribeClusterResult result= client.describeCluster(describeClusterRequest);
	if (result != null) {
		return result.getCluster().getStatus().getState();
	}
	return null;
}