Java Code Examples for org.apache.hadoop.registry.client.types.ServiceRecord#set()

The following examples show how to use org.apache.hadoop.registry.client.types.ServiceRecord#set() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Executor.java    From jstorm with Apache License 2.0 6 votes vote down vote up 
public boolean needUpgrade() {
    String containerPath = RegistryUtils.componentPath(
            JOYConstants.APP_TYPE, this.executorMeta.getInstanceName(),
            this.executorMeta.getApplicationId(), this.executorMeta.getRunningContainer());

    try {
        if (registryOperations.exists(containerPath)) {
            ServiceRecord sr = registryOperations.resolve(containerPath);
            if (sr.get(JOYConstants.NEED_UPGRADE) != null && sr.get(JOYConstants.NEED_UPGRADE).equals(JOYConstants.TRUE)) {
                sr.set(JOYConstants.NEED_UPGRADE, JOYConstants.FALSE);
                registryOperations.bind(containerPath, sr, BindFlags.OVERWRITE);
                LOG.info(JOYConstants.NEED_UPGRADE);
                return true;
            }
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    return false;
}
Example 2
Source File: JStormOnYarnBusiness.java    From PoseidonX with Apache License 2.0 6 votes vote down vote up 
/**
 * 删除应用
 * @param zkHosts
 * @param zkPort
 * @param appId
 */
public static void killApplication(List<String> zkHosts, Integer zkPort, String appId, String taskName) throws Exception{
    if(StringUtils.isNotBlank(appId) && StringUtils.isNotBlank(taskName)){

        String applicationPath = YarnZkRegistryBusiness.PathBuilder.applicationPath(taskName,appId);
        ServiceRecord serviceRecord = YarnZkRegistryBusiness.resolve(applicationPath);
        if(serviceRecord !=null){
            serviceRecord.set("killed","true");
            YarnZkRegistryBusiness.bind(applicationPath,serviceRecord);
        }
        ApplicationReport applicationReport = YarnClientProxy.getApplicationReportByAppId(appId);
        if(applicationReport !=null){
            try{
                YarnClientProxy.killApplicationByAppId(appId);
            }catch (Exception e){
                if(serviceRecord != null){
                    serviceRecord.set("killed","false");
                    YarnZkRegistryBusiness.bind(applicationPath,serviceRecord);
                }
                throw e;
            }
        }
    }
    JStormOnYarnBusiness.clearZkInfo(zkHosts,zkPort,taskName);
}
Example 3
Source File: RegistryTestHelper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static ServiceRecord createRecord(String id, String persistence,
    String description) {
  ServiceRecord serviceRecord = new ServiceRecord();
  serviceRecord.set(YarnRegistryAttributes.YARN_ID, id);
  serviceRecord.description = description;
  serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE, persistence);
  return serviceRecord;
}
Example 4
Source File: RegistryTestHelper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static ServiceRecord createRecord(String id, String persistence,
    String description) {
  ServiceRecord serviceRecord = new ServiceRecord();
  serviceRecord.set(YarnRegistryAttributes.YARN_ID, id);
  serviceRecord.description = description;
  serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE, persistence);
  return serviceRecord;
}
Example 5
Source File: RegistryTestHelper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Create a service entry with the sample endpoints
 * @param persistence persistence policy
 * @return the record
 * @throws IOException on a failure
 */
public static ServiceRecord buildExampleServiceEntry(String persistence) throws
    IOException,
    URISyntaxException {
  ServiceRecord record = new ServiceRecord();
  record.set(YarnRegistryAttributes.YARN_ID, "example-0001");
  record.set(YarnRegistryAttributes.YARN_PERSISTENCE, persistence);
  addSampleEndpoints(record, "namenode");
  return record;
}
Example 6
Source File: JstormMaster.java    From jstorm with Apache License 2.0 5 votes vote down vote up 
private ServiceRecord setupServiceRecord() {
    ServiceRecord application = new ServiceRecord();
    application.set(YarnRegistryAttributes.YARN_ID, jstormMasterContext.appAttemptID.getApplicationId().toString());
    application.description = JOYConstants.AM;
    application.set(YarnRegistryAttributes.YARN_PERSISTENCE,
            PersistencePolicies.PERMANENT);
    Map<String, String> addresses = new HashMap<String, String>();
    addresses.put(JOYConstants.HOST, jstormMasterContext.appMasterHostname);
    addresses.put(JOYConstants.PORT, String.valueOf(jstormMasterContext.appMasterThriftPort));
    Endpoint endpoint = new Endpoint(JOYConstants.HTTP, JOYConstants.HOST_PORT, JOYConstants.RPC, addresses);
    application.addExternalEndpoint(endpoint);
    return application;
}
Example 7
Source File: TestMarshalling.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testUnknownFieldsRoundTrip() throws Throwable {
  ServiceRecord record =
      createRecord(PersistencePolicies.APPLICATION_ATTEMPT);
  record.set("key", "value");
  record.set("intval", "2");
  assertEquals("value", record.get("key"));
  assertEquals("2", record.get("intval"));
  assertNull(record.get("null"));
  assertEquals("defval", record.get("null", "defval"));
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord r2 = marshal.fromBytes("", bytes);
  assertEquals("value", r2.get("key"));
  assertEquals("2", r2.get("intval"));
}
Example 8
Source File: TestMarshalling.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRoundTrip() throws Throwable {
  String persistence = PersistencePolicies.PERMANENT;
  ServiceRecord record = createRecord(persistence);
  record.set("customkey", "customvalue");
  record.set("customkey2", "customvalue2");
  RegistryTypeUtils.validateServiceRecord("", record);
  LOG.info(marshal.toJson(record));
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord r2 = marshal.fromBytes("", bytes);
  assertMatches(record, r2);
  RegistryTypeUtils.validateServiceRecord("", r2);
}
Example 9
Source File: TestRegistryOperations.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test(expected = PathNotFoundException.class)
public void testPutNoParent2() throws Throwable {
  ServiceRecord record = new ServiceRecord();
  record.set(YarnRegistryAttributes.YARN_ID, "testPutNoParent");
  String path = "/path/without/parent";
  operations.bind(path, record, 0);
}
Example 10
Source File: TestRegistryOperations.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testPutNoParent() throws Throwable {
  ServiceRecord record = new ServiceRecord();
  record.set(YarnRegistryAttributes.YARN_ID, "testPutNoParent");
  String path = "/path/without/parent";
  try {
    operations.bind(path, record, 0);
    // didn't get a failure
    // trouble
    RegistryPathStatus stat = operations.stat(path);
    fail("Got a status " + stat);
  } catch (PathNotFoundException expected) {
    // expected
  }
}
Example 11
Source File: RegistryTestHelper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Create a service entry with the sample endpoints
 * @param persistence persistence policy
 * @return the record
 * @throws IOException on a failure
 */
public static ServiceRecord buildExampleServiceEntry(String persistence) throws
    IOException,
    URISyntaxException {
  ServiceRecord record = new ServiceRecord();
  record.set(YarnRegistryAttributes.YARN_ID, "example-0001");
  record.set(YarnRegistryAttributes.YARN_PERSISTENCE, persistence);
  addSampleEndpoints(record, "namenode");
  return record;
}
Example 12
Source File: SlotPortsView.java    From jstorm with Apache License 2.0 5 votes vote down vote up 
/**
 * see if anyone is updating host's port list, if not start , update this host itself
 * timeout is 45 seconds
 *
 * @param hostPath
 * @throws InterruptedException
 * @throws IOException
 */
private void tryHostLock(String hostPath) throws Exception {

    //if path has created 60 seconds ago, then delete
    if (registryOperations.exists(hostPath)) {
        try {
            ServiceRecord host = registryOperations.resolve(hostPath);
            Long cTime = Long.parseLong(host.get(JOYConstants.CTIME, JOYConstants.DEFAULT_CTIME));
            Date now = new Date();
            if (now.getTime() - cTime > JOYConstants.HOST_LOCK_TIMEOUT || cTime > now.getTime())
                registryOperations.delete(hostPath, true);
        } catch (Exception ex) {
            LOG.error(ex);
        }
    }

    int failedCount = JOYConstants.RETRY_TIMES;
    while (!registryOperations.mknode(hostPath, true)) {
        Thread.sleep(JOYConstants.SLEEP_INTERVAL);
        failedCount--;
        if (failedCount <= 0)
            break;
    }

    if (failedCount > 0) {
        ServiceRecord sr = new ServiceRecord();
        Date date = new Date();
        date.getTime();
        sr.set(JOYConstants.CTIME, String.valueOf(date.getTime()));
        registryOperations.bind(hostPath, sr, BindFlags.OVERWRITE);
        return;
    }
    throw new Exception("can't get host lock");
}
Example 13
Source File: TestMarshalling.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testUnknownFieldsRoundTrip() throws Throwable {
  ServiceRecord record =
      createRecord(PersistencePolicies.APPLICATION_ATTEMPT);
  record.set("key", "value");
  record.set("intval", "2");
  assertEquals("value", record.get("key"));
  assertEquals("2", record.get("intval"));
  assertNull(record.get("null"));
  assertEquals("defval", record.get("null", "defval"));
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord r2 = marshal.fromBytes("", bytes);
  assertEquals("value", r2.get("key"));
  assertEquals("2", r2.get("intval"));
}
Example 14
Source File: TestMarshalling.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRoundTrip() throws Throwable {
  String persistence = PersistencePolicies.PERMANENT;
  ServiceRecord record = createRecord(persistence);
  record.set("customkey", "customvalue");
  record.set("customkey2", "customvalue2");
  RegistryTypeUtils.validateServiceRecord("", record);
  LOG.info(marshal.toJson(record));
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord r2 = marshal.fromBytes("", bytes);
  assertMatches(record, r2);
  RegistryTypeUtils.validateServiceRecord("", r2);
}
Example 15
Source File: TestRegistryOperations.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testPutNoParent() throws Throwable {
  ServiceRecord record = new ServiceRecord();
  record.set(YarnRegistryAttributes.YARN_ID, "testPutNoParent");
  String path = "/path/without/parent";
  try {
    operations.bind(path, record, 0);
    // didn't get a failure
    // trouble
    RegistryPathStatus stat = operations.stat(path);
    fail("Got a status " + stat);
  } catch (PathNotFoundException expected) {
    // expected
  }
}
Example 16
Source File: TestRegistryRMOperations.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testPurgeEntryCuratorCallback() throws Throwable {

  String path = "/users/example/hbase/hbase1/";
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.APPLICATION_ATTEMPT);
  written.set(YarnRegistryAttributes.YARN_ID,
      "testAsyncPurgeEntry_attempt_001");

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, 0);

  ZKPathDumper dump = registry.dumpPath(false);
  CuratorEventCatcher events = new CuratorEventCatcher();

  LOG.info("Initial state {}", dump);

  // container query
  String id = written.get(YarnRegistryAttributes.YARN_ID, "");
  int opcount = purge("/",
      id,
      PersistencePolicies.CONTAINER,
      RegistryAdminService.PurgePolicy.PurgeAll,
      events);
  assertPathExists(path);
  assertEquals(0, opcount);
  assertEquals("Event counter", 0, events.getCount());

  // now the application attempt
  opcount = purge("/",
      id,
      PersistencePolicies.APPLICATION_ATTEMPT,
      RegistryAdminService.PurgePolicy.PurgeAll,
      events);

  LOG.info("Final state {}", dump);

  assertPathNotFound(path);
  assertEquals("wrong no of delete operations in " + dump, 1, opcount);
  // and validate the callback event
  assertEquals("Event counter", 1, events.getCount());
}
Example 17
Source File: TestRegistryRMOperations.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testAsyncPurgeEntry() throws Throwable {

  String path = "/users/example/hbase/hbase1/";
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.APPLICATION_ATTEMPT);
  written.set(YarnRegistryAttributes.YARN_ID,
      "testAsyncPurgeEntry_attempt_001");

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, 0);

  ZKPathDumper dump = registry.dumpPath(false);

  LOG.info("Initial state {}", dump);

  DeleteCompletionCallback deletions = new DeleteCompletionCallback();
  int opcount = purge("/",
      written.get(YarnRegistryAttributes.YARN_ID, ""),
      PersistencePolicies.CONTAINER,
      RegistryAdminService.PurgePolicy.PurgeAll,
      deletions);
  assertPathExists(path);

  dump = registry.dumpPath(false);

  assertEquals("wrong no of delete operations in " + dump, 0,
      deletions.getEventCount());
  assertEquals("wrong no of delete operations in " + dump, 0, opcount);


  // now app attempt
  deletions = new DeleteCompletionCallback();
  opcount = purge("/",
      written.get(YarnRegistryAttributes.YARN_ID, ""),
      PersistencePolicies.APPLICATION_ATTEMPT,
      RegistryAdminService.PurgePolicy.PurgeAll,
      deletions);

  dump = registry.dumpPath(false);
  LOG.info("Final state {}", dump);

  assertPathNotFound(path);
  assertEquals("wrong no of delete operations in " + dump, 1,
      deletions.getEventCount());
  assertEquals("wrong no of delete operations in " + dump, 1, opcount);
  // and validate the callback event

}
Example 18
Source File: TestRegistryRMOperations.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testAsyncPurgeEntry() throws Throwable {

  String path = "/users/example/hbase/hbase1/";
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.APPLICATION_ATTEMPT);
  written.set(YarnRegistryAttributes.YARN_ID,
      "testAsyncPurgeEntry_attempt_001");

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, 0);

  ZKPathDumper dump = registry.dumpPath(false);

  LOG.info("Initial state {}", dump);

  DeleteCompletionCallback deletions = new DeleteCompletionCallback();
  int opcount = purge("/",
      written.get(YarnRegistryAttributes.YARN_ID, ""),
      PersistencePolicies.CONTAINER,
      RegistryAdminService.PurgePolicy.PurgeAll,
      deletions);
  assertPathExists(path);

  dump = registry.dumpPath(false);

  assertEquals("wrong no of delete operations in " + dump, 0,
      deletions.getEventCount());
  assertEquals("wrong no of delete operations in " + dump, 0, opcount);


  // now app attempt
  deletions = new DeleteCompletionCallback();
  opcount = purge("/",
      written.get(YarnRegistryAttributes.YARN_ID, ""),
      PersistencePolicies.APPLICATION_ATTEMPT,
      RegistryAdminService.PurgePolicy.PurgeAll,
      deletions);

  dump = registry.dumpPath(false);
  LOG.info("Final state {}", dump);

  assertPathNotFound(path);
  assertEquals("wrong no of delete operations in " + dump, 1,
      deletions.getEventCount());
  assertEquals("wrong no of delete operations in " + dump, 1, opcount);
  // and validate the callback event

}
Example 19
Source File: TestRegistryRMOperations.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testPurgeEntryCuratorCallback() throws Throwable {

  String path = "/users/example/hbase/hbase1/";
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.APPLICATION_ATTEMPT);
  written.set(YarnRegistryAttributes.YARN_ID,
      "testAsyncPurgeEntry_attempt_001");

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, 0);

  ZKPathDumper dump = registry.dumpPath(false);
  CuratorEventCatcher events = new CuratorEventCatcher();

  LOG.info("Initial state {}", dump);

  // container query
  String id = written.get(YarnRegistryAttributes.YARN_ID, "");
  int opcount = purge("/",
      id,
      PersistencePolicies.CONTAINER,
      RegistryAdminService.PurgePolicy.PurgeAll,
      events);
  assertPathExists(path);
  assertEquals(0, opcount);
  assertEquals("Event counter", 0, events.getCount());

  // now the application attempt
  opcount = purge("/",
      id,
      PersistencePolicies.APPLICATION_ATTEMPT,
      RegistryAdminService.PurgePolicy.PurgeAll,
      events);

  LOG.info("Final state {}", dump);

  assertPathNotFound(path);
  assertEquals("wrong no of delete operations in " + dump, 1, opcount);
  // and validate the callback event
  assertEquals("Event counter", 1, events.getCount());
}
Example 20
Source File: JstormEngineServiceImpl.java    From PoseidonX with Apache License 2.0 4 votes vote down vote up 
public static void stopTask(Integer taskId, String taskName, Integer processId) throws Exception {
    if(taskId == null){
        throw new Exception("任务停止失败，参数异常，taskId为空！");
    }
    if(StringUtils.isBlank(taskName)){
        throw new Exception("任务提交失败，参数异常，任务名为空！");
    }
    if(processId == null){
        throw new Exception("任务停止失败，参数异常，processId为空！");
    }
    JstormProcessPO jstormProcessPO = jstormEngineService.jstormProcessDao.getById(processId);
    if(jstormProcessPO == null){
        throw new Exception("任务停止失败，任务的执行信息不存在或者已经被删除！");
    }
    JstormTaskConfigDTO jstormTaskConfigDTO = JSONObject.parseObject(jstormProcessPO.getTaskConfig(), JstormTaskConfigDTO.class);

    String topId = jstormProcessPO.getTopId();
    String appId = jstormProcessPO.getYarnAppId();
    String applicationPath = YarnZkRegistryBusiness.PathBuilder.applicationPath(jstormTaskConfigDTO.getTaskName(),jstormProcessPO.getYarnAppId());
    ServiceRecord serviceRecord  = YarnZkRegistryBusiness.resolve(applicationPath);
    if(serviceRecord != null){
        // 设置为true
        serviceRecord.set("killed","true");
        YarnZkRegistryBusiness.bind(applicationPath,serviceRecord);
    }

    ApplicationReport report = YarnClientProxy.getApplicationReportByAppId(appId);
    if(report != null){
        YarnApplicationState state = report.getYarnApplicationState();

        LOGGER.error("JstormProcessServiceImpl stopTask report YarnApplicationState = " + state.toString() + " appId=" + appId);

        if (!(YarnApplicationState.FINISHED == state || YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state)) {
            try{
                YarnClientProxy.killApplicationByAppId(appId);
                LOGGER.error("JstormProcessServiceImpl stopTask killApplicationByAppId is ok appId=" + appId);
            }catch (Exception e){
                // kill失败时进行还原处理
                if(serviceRecord != null){
                    serviceRecord.set("killed","false");
                    YarnZkRegistryBusiness.bind(applicationPath,serviceRecord);
                }
                LOGGER.error("JstormProcessServiceImpl stopTask killApplicationByAppId is error appId=" + appId,e);
                throw e;
            }
        }
    }else{
        LOGGER.error("JstormProcessServiceImpl stopTask report is null appId=" + appId);
    }

    TaskPO taskPO = new TaskPO();
    taskPO.setTaskStatus(TaskStatusEnum.STOP.getValue());
    taskPO.setTaskStopTime(new Date());
    taskPO.setId(jstormProcessPO.getTaskId());
    jstormEngineService.taskDao.update4Stop(taskPO);

    //清理资源
    JStormOnYarnBusiness.clearZkInfo(jstormTaskConfigDTO.getJstormZkHost(), jstormTaskConfigDTO.getJstormZkPort(), jstormTaskConfigDTO.getTaskName());
    JstormMetricCollectImpl.workerErrorHistoryCache.remove(topId);
    MetricReportContainer.removeReports(topId);
    JstormEngineCheckPointImpl.recoveryFail.remove(jstormProcessPO.getTaskId());
}