com.amazonaws.services.elasticmapreduce.model.ClusterState Java Examples

The following examples show how to use com.amazonaws.services.elasticmapreduce.model.ClusterState. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: EmrHelperTest.java From herd with Apache License 2.0

6 votes

@Test
public void testGetActiveEmrClusterIdAssertReturnActualClusterIdWhenClusterIdSpecifiedAndClusterStateActiveAndNameMatch()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(emrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        assertEquals(expectedEmrClusterId, emrHelper.getActiveEmrClusterId(emrClusterId, emrClusterName, null));

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId.trim()), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}

Example #2

Source File: LambdaContainer.java From aws-big-data-blog with Apache License 2.0

6 votes

protected List<String> getActiveTaggedClusters() throws Exception{
	AmazonElasticMapReduceClient emrClient = new AmazonElasticMapReduceClient();
	List<String> waitingClusters = new ArrayList<String>();
	ListClustersResult clusterResult = emrClient.listClusters(new ListClustersRequest().withClusterStates(ClusterState.WAITING));
	
	DescribeClusterRequest specifcTagDescribe = new DescribeClusterRequest();
	specifcTagDescribe.putCustomQueryParameter("Cluster.Tags",null);
	 for( ClusterSummary cluster : clusterResult.getClusters()){
		 	System.out.println("list cluster id "+cluster.getId());
		 	List<Tag> tagList = emrClient.describeCluster(specifcTagDescribe.withClusterId(cluster.getId())).getCluster().getTags();
		 	for(Tag tag:tagList){
		 		if(tag.getKey().equals(props.getProperty("edba.cluster.tag.key"))){
		 			waitingClusters.add(cluster.getId());
		 		}
		 	}
		 	
	}
	return waitingClusters;
	
}

Example #3

Source File: EmrHelperTest.java From herd with Apache License 2.0

5 votes

@Test
public void testGetActiveEmrClusterIdAssertErrorWhenClusterIdSpecifiedAndNameMismatch()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";
        String actualEmrClusterName = "actualEmrClusterName";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(actualEmrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        try
        {
            emrHelper.getActiveEmrClusterId(emrClusterId, emrClusterName, null);
            fail();
        }
        catch (IllegalArgumentException e)
        {
            assertEquals(String
                .format("The cluster with ID \"%s\" does not match the expected name \"%s\". The actual name is \"%s\".", expectedEmrClusterId,
                    emrClusterName, actualEmrClusterName), e.getMessage());
        }

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId.trim()), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}

Example #4

Source File: EmrHelperTest.java From herd with Apache License 2.0

5 votes

@Test
public void testGetActiveEmrClusterIdAssertReturnActualClusterIdWhenClusterStateActiveAndNameNotSpecified()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = null;
        String expectedEmrClusterId = "expectedEmrClusterId";
        String actualEmrClusterName = "actualEmrClusterName";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(actualEmrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        assertEquals(expectedEmrClusterId, emrHelper.getActiveEmrClusterId(emrClusterId, emrClusterName, null));

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}

Example #5

Source File: EmrHelperTest.java From herd with Apache License 2.0

5 votes

@Test
public void testGetActiveEmrClusterIdAssertErrorWhenClusterIdSpecifiedAndClusterStateNotActive()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";

        ClusterState actualClusterState = ClusterState.TERMINATED;
        when(mockEmrDao.getEmrClusterById(any(), any()))
            .thenReturn(new Cluster().withId(expectedEmrClusterId).withName(emrClusterName).withStatus(new ClusterStatus().withState(actualClusterState)));

        try
        {
            emrHelper.getActiveEmrClusterId(emrClusterId, emrClusterName, null);
            fail();
        }
        catch (IllegalArgumentException e)
        {
            assertEquals(String.format("The cluster with ID \"%s\" is not active. The cluster state must be in one of [STARTING, BOOTSTRAPPING, RUNNING, " +
                "WAITING]. Current state is \"%s\"", emrClusterId, actualClusterState), e.getMessage());
        }

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}

Example #6

Source File: EmrHelperTest.java From herd with Apache License 2.0

5 votes

@Test
public void testGetActiveEmrClusterIdAssertParametersTrimmed()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(emrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        assertEquals(expectedEmrClusterId,
            emrHelper.getActiveEmrClusterId(StringUtils.wrap(emrClusterId, BLANK_TEXT), StringUtils.wrap(emrClusterName, BLANK_TEXT), null));

        verify(mockEmrDao).getEmrClusterById(eq(emrClusterId.trim()), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}

Example #7

Source File: EmrHelperTest.java From herd with Apache License 2.0

5 votes

@Test
public void testGetActiveEmrClusterIdAssertParametersCaseIgnored()
{
    EmrDao originalEmrDao = emrHelper.getEmrDao();
    EmrDao mockEmrDao = mock(EmrDao.class);
    emrHelper.setEmrDao(mockEmrDao);

    try
    {
        String emrClusterId = "emrClusterId";
        String emrClusterName = "emrClusterName";
        String expectedEmrClusterId = "expectedEmrClusterId";

        when(mockEmrDao.getEmrClusterById(any(), any())).thenReturn(
            new Cluster().withId(expectedEmrClusterId).withName(emrClusterName).withStatus(new ClusterStatus().withState(ClusterState.RUNNING)));

        assertEquals(expectedEmrClusterId,
            emrHelper.getActiveEmrClusterId(StringUtils.upperCase(emrClusterId), StringUtils.upperCase(emrClusterName), null));

        verify(mockEmrDao).getEmrClusterById(eq(StringUtils.upperCase(emrClusterId)), any());
        verifyNoMoreInteractions(mockEmrDao);
    }
    finally
    {
        emrHelper.setEmrDao(originalEmrDao);
    }
}

Example #8

Source File: MockEmrOperationsImpl.java From herd with Apache License 2.0

5 votes

@Override
public void terminateEmrCluster(AmazonElasticMapReduceClient emrClient, String clusterId, boolean overrideTerminationProtection)
{
    MockEmrJobFlow cluster = getClusterById(clusterId);
    if (cluster.getJobFlowName().endsWith(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION))
    {
        throw new AmazonServiceException(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION);
    }
    cluster.setStatus(ClusterState.TERMINATED.toString());
}

Example #9

Source File: ClusterManager.java From herd-mdl with Apache License 2.0

4 votes

public boolean registerCluster() {
    String sql = "INSERT IGNORE INTO `EMR_CLUSTER` (`CLUSTER_NAME`,`CLUSTER_ID`, CREATE_TIME, STATUS) VALUES (?, ?, now(), 'R')";

    template.update(sql, clusterName, clusterID);

    String runningClusterID = template.queryForObject(SELECT_CLUSTER_ID_SQL, new Object[]{clusterName}, String.class);
    boolean continueProcess = false;

    if (!runningClusterID.equals(clusterID)) {
        //Verify the running cluster is still up

        DescribeClusterRequest req = new DescribeClusterRequest().withClusterId(runningClusterID);
        DescribeClusterResult result = emrClient.describeCluster(req);

        String clusterState = result.getCluster().getStatus().getState();

        if (clusterState.equals(ClusterState.TERMINATED.toString()) || clusterState.equals(ClusterState.TERMINATED_WITH_ERRORS.toString())) {
            continueProcess = prepareProcessing();
        } else if (clusterState.equals(ClusterState.TERMINATING.toString())) {
            //Wait till cluster is terminated
            logger.info(String.format("Running Cluster %s is in TERMINCATING state, waiting for it to be terminated", runningClusterID));

            do {
                try {
                    Thread.sleep(30000);
                } catch (InterruptedException ex) {

                }
                result = emrClient.describeCluster(req);
                clusterState = result.getCluster().getStatus().getState();
            } while (clusterState.equals(ClusterState.TERMINATING.toString()));

            if (clusterState.equals(ClusterState.TERMINATED.toString()) || clusterState.equals(ClusterState.TERMINATED_WITH_ERRORS)) {
                continueProcess = prepareProcessing();
            }

        } else {
            logger.info("Running cluster is " + runningClusterID + ", exiting");
        }

    } else {
        continueProcess = true;
    }

    return continueProcess;
}

Example #10

Source File: ClusterManager.java From herd-mdl with Apache License 2.0

4 votes

public void clusterAutoScale()
  {
      logger.info("Auto Scale Job started");
      Map<String, Object> lastCheck = template.queryForMap(AUTO_SCALE_QUERY);
      long minutes = (Long) lastCheck.get("age");
      if(minutes >= autoScaleIntervalInMin) {
          long id = (Long)lastCheck.get("ID") + 1;
          int updated = template.update("INSERT IGNORE INTO METASTOR_EMR_AUTOSCALE (`ID`," +
                  "`CLUSTER_ID`) VALUES (?, ?)", id, clusterID);

          if (updated > 0) {

              int clusterNumber = calculateNumberOfClustersNeeded();
              template.update("UPDATE METASTOR_EMR_AUTOSCALE SET TOTAL_CLUSTER= ? WHERE ID=?", clusterNumber, id);
              if (clusterNumber <= 1) return;

              List<Map<String, Object>> clusters = template.queryForList("select * from EMR_CLUSTER");
              List<String> existingCluster = new ArrayList<>();

              createEmrClient();
              for (Map<String, Object> record : clusters) {
                  String cid = (String) record.get("CLUSTER_ID");
                  String cname = (String) record.get("CLUSTER_NAME");

                  boolean clusterIsAlive = true;

                  try {

                      DescribeClusterRequest req = new DescribeClusterRequest().withClusterId(cid);
                      DescribeClusterResult result = emrClient.describeCluster(req);

                      String clusterState = result.getCluster().getStatus().getState();
                      if (clusterState.equals(ClusterState.TERMINATED.toString()) || clusterState.equals(ClusterState.TERMINATED_WITH_ERRORS.toString())) {
                          deleteCluster(cid);
                          clusterIsAlive = false;
                      }
                  } catch (Exception ex) {
                      logger.warning("Error getting info for cluster " + cid + ex.getMessage());
                      clusterIsAlive = false;
                  }


                  if (clusterIsAlive) {
                      existingCluster.add(cname);
                  }

              }

              // Start Additional if required
		startAdditionalClusters( clusterNumber, existingCluster );
	}
      }
logger.info("Auto Scale Job completed");

  }

Example #11

Source File: MockEmrOperationsImpl.java From herd with Apache License 2.0

4 votes

@Override
public String runEmrJobFlow(AmazonElasticMapReduceClient emrClient, RunJobFlowRequest jobFlowRequest)
{
    String clusterStatus = ClusterState.BOOTSTRAPPING.toString();

    StatusChangeReason reason = new StatusChangeReason(ClusterStateChangeReasonCode.USER_REQUEST.toString(), "Started " + clusterStatus);
    StatusTimeline timeline = new StatusTimeline();
    timeline.setCreationTime(HerdDateUtils.getXMLGregorianCalendarValue(new Date()));

    if (StringUtils.isNotBlank(jobFlowRequest.getAmiVersion()))
    {
        if (jobFlowRequest.getAmiVersion().equals(MockAwsOperationsHelper.AMAZON_THROTTLING_EXCEPTION))
        {
            AmazonServiceException throttlingException = new AmazonServiceException("test throttling exception");
            throttlingException.setErrorCode("ThrottlingException");

            throw throttlingException;
        }
        else if (jobFlowRequest.getAmiVersion().equals(MockAwsOperationsHelper.AMAZON_BAD_REQUEST))
        {
            AmazonServiceException badRequestException = new AmazonServiceException(MockAwsOperationsHelper.AMAZON_BAD_REQUEST);
            badRequestException.setStatusCode(HttpStatus.SC_BAD_REQUEST);
            throw badRequestException;
        }
        else if (jobFlowRequest.getAmiVersion().equals(MockAwsOperationsHelper.AMAZON_NOT_FOUND))
        {
            AmazonServiceException notFoundException = new AmazonServiceException(MockAwsOperationsHelper.AMAZON_NOT_FOUND);
            notFoundException.setStatusCode(HttpStatus.SC_NOT_FOUND);
            throw notFoundException;
        }
        else if (jobFlowRequest.getAmiVersion().equals(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION))
        {
            throw new AmazonServiceException(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION);
        }
        else if (jobFlowRequest.getAmiVersion().equals(MockAwsOperationsHelper.AMAZON_CLUSTER_STATUS_WAITING))
        {
            clusterStatus = ClusterState.WAITING.toString();
        }
        else if (jobFlowRequest.getAmiVersion().equals(MockAwsOperationsHelper.AMAZON_CLUSTER_STATUS_RUNNING))
        {
            clusterStatus = ClusterState.RUNNING.toString();
        }
    }


    return createNewCluster(jobFlowRequest, clusterStatus, reason, timeline).getJobFlowId();
}