org.apache.hadoop.registry.client.types.yarn.PersistencePolicies Java Examples

The following examples show how to use org.apache.hadoop.registry.client.types.yarn.PersistencePolicies. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestRegistryRMOperations.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testChildDeletion() throws Throwable {
  ServiceRecord app = createRecord("app1",
      PersistencePolicies.APPLICATION, "app",
      null);
  ServiceRecord container = createRecord("container1",
      PersistencePolicies.CONTAINER, "container",
      null);

  operations.bind("/app", app, BindFlags.OVERWRITE);
  operations.bind("/app/container", container, BindFlags.OVERWRITE);

  try {
    int p = purge("/",
        "app1",
        PersistencePolicies.APPLICATION,
        RegistryAdminService.PurgePolicy.FailOnChildren);
    fail("expected a failure, got a purge count of " + p);
  } catch (PathIsNotEmptyDirectoryException expected) {
    // expected
  }

}
 
Example #2
Source File: TestRegistryRMOperations.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testChildDeletion() throws Throwable {
  ServiceRecord app = createRecord("app1",
      PersistencePolicies.APPLICATION, "app",
      null);
  ServiceRecord container = createRecord("container1",
      PersistencePolicies.CONTAINER, "container",
      null);

  operations.bind("/app", app, BindFlags.OVERWRITE);
  operations.bind("/app/container", container, BindFlags.OVERWRITE);

  try {
    int p = purge("/",
        "app1",
        PersistencePolicies.APPLICATION,
        RegistryAdminService.PurgePolicy.FailOnChildren);
    fail("expected a failure, got a purge count of " + p);
  } catch (PathIsNotEmptyDirectoryException expected) {
    // expected
  }

}
 
Example #3
Source File: RMRegistryOperationsService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Actions to take when an application is completed
 * @param id  application  ID
 * @throws IOException problems
 */
public void onApplicationCompleted(ApplicationId id)
    throws IOException {
  LOG.info("Application {} completed, purging application-level records",
      id);
  purgeRecordsAsync("/",
      id.toString(),
      PersistencePolicies.APPLICATION);
}
 
Example #4
Source File: JstormMaster.java    From jstorm with Apache License 2.0 5 votes vote down vote up
private ServiceRecord setupServiceRecord() {
    ServiceRecord application = new ServiceRecord();
    application.set(YarnRegistryAttributes.YARN_ID, jstormMasterContext.appAttemptID.getApplicationId().toString());
    application.description = JOYConstants.AM;
    application.set(YarnRegistryAttributes.YARN_PERSISTENCE,
            PersistencePolicies.PERMANENT);
    Map<String, String> addresses = new HashMap<String, String>();
    addresses.put(JOYConstants.HOST, jstormMasterContext.appMasterHostname);
    addresses.put(JOYConstants.PORT, String.valueOf(jstormMasterContext.appMasterThriftPort));
    Endpoint endpoint = new Endpoint(JOYConstants.HTTP, JOYConstants.HOST_PORT, JOYConstants.RPC, addresses);
    application.addExternalEndpoint(endpoint);
    return application;
}
 
Example #5
Source File: TestMarshalling.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testFieldPropagationInCopy() throws Throwable {
  ServiceRecord record =
      createRecord(PersistencePolicies.APPLICATION_ATTEMPT);
  record.set("key", "value");
  record.set("intval", "2");
  ServiceRecord that = new ServiceRecord(record);
  assertMatches(record, that);
}
 
Example #6
Source File: TestMarshalling.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testUnknownFieldsRoundTrip() throws Throwable {
  ServiceRecord record =
      createRecord(PersistencePolicies.APPLICATION_ATTEMPT);
  record.set("key", "value");
  record.set("intval", "2");
  assertEquals("value", record.get("key"));
  assertEquals("2", record.get("intval"));
  assertNull(record.get("null"));
  assertEquals("defval", record.get("null", "defval"));
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord r2 = marshal.fromBytes("", bytes);
  assertEquals("value", r2.get("key"));
  assertEquals("2", r2.get("intval"));
}
 
Example #7
Source File: TestMarshalling.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testRoundTrip() throws Throwable {
  String persistence = PersistencePolicies.PERMANENT;
  ServiceRecord record = createRecord(persistence);
  record.set("customkey", "customvalue");
  record.set("customkey2", "customvalue2");
  RegistryTypeUtils.validateServiceRecord("", record);
  LOG.info(marshal.toJson(record));
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord r2 = marshal.fromBytes("", bytes);
  assertMatches(record, r2);
  RegistryTypeUtils.validateServiceRecord("", r2);
}
 
Example #8
Source File: TestRegistryOperations.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testListListFully() throws Throwable {
  ServiceRecord r1 = new ServiceRecord();
  ServiceRecord r2 = createRecord("i",
      PersistencePolicies.PERMANENT, "r2");

  String path = USERPATH + SC_HADOOP + "/listing" ;
  operations.mknode(path, true);
  String r1path = path + "/r1";
  operations.bind(r1path, r1, 0);
  String r2path = path + "/r2";
  operations.bind(r2path, r2, 0);

  RegistryPathStatus r1stat = operations.stat(r1path);
  assertEquals("r1", r1stat.path);
  RegistryPathStatus r2stat = operations.stat(r2path);
  assertEquals("r2", r2stat.path);
  assertNotEquals(r1stat, r2stat);

  // listings now
  List<String> list = operations.list(path);
  assertEquals("Wrong no. of children", 2, list.size());
  // there's no order here, so create one
  Map<String, String> names = new HashMap<String, String>();
  String entries = "";
  for (String child : list) {
    names.put(child, child);
    entries += child + " ";
  }
  assertTrue("No 'r1' in " + entries,
      names.containsKey("r1"));
  assertTrue("No 'r2' in " + entries,
      names.containsKey("r2"));

  Map<String, RegistryPathStatus> stats =
      RegistryUtils.statChildren(operations, path);
  assertEquals("Wrong no. of children", 2, stats.size());
  assertEquals(r1stat, stats.get("r1"));
  assertEquals(r2stat, stats.get("r2"));
}
 
Example #9
Source File: TestRegistryOperations.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testPutGetContainerPersistenceServiceEntry() throws Throwable {

  String path = ENTRY_PATH;
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.CONTAINER);

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, BindFlags.CREATE);
  ServiceRecord resolved = operations.resolve(path);
  validateEntry(resolved);
  assertMatches(written, resolved);
}
 
Example #10
Source File: TestRegistryOperations.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testPutGetServiceEntry() throws Throwable {
  ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0,
      PersistencePolicies.APPLICATION);
  ServiceRecord resolved = operations.resolve(ENTRY_PATH);
  validateEntry(resolved);
  assertMatches(written, resolved);
}
 
Example #11
Source File: TestRegistryRMOperations.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testPutGetContainerPersistenceServiceEntry() throws Throwable {

  String path = ENTRY_PATH;
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.CONTAINER);

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, BindFlags.CREATE);
  ServiceRecord resolved = operations.resolve(path);
  validateEntry(resolved);
  assertMatches(written, resolved);
}
 
Example #12
Source File: RMRegistryOperationsService.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Actions to take when the AM container is completed
 * @param id  container ID
 * @throws IOException problems
 */
public void onContainerFinished(ContainerId id) throws IOException {
  LOG.info("Container {} finished, purging container-level records",
      id);
  purgeRecordsAsync("/",
      id.toString(),
      PersistencePolicies.CONTAINER);
}
 
Example #13
Source File: RMRegistryOperationsService.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Actions to take when an application is completed
 * @param id  application  ID
 * @throws IOException problems
 */
public void onApplicationCompleted(ApplicationId id)
    throws IOException {
  LOG.info("Application {} completed, purging application-level records",
      id);
  purgeRecordsAsync("/",
      id.toString(),
      PersistencePolicies.APPLICATION);
}
 
Example #14
Source File: TestMarshalling.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testFieldPropagationInCopy() throws Throwable {
  ServiceRecord record =
      createRecord(PersistencePolicies.APPLICATION_ATTEMPT);
  record.set("key", "value");
  record.set("intval", "2");
  ServiceRecord that = new ServiceRecord(record);
  assertMatches(record, that);
}
 
Example #15
Source File: TestMarshalling.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testRoundTrip() throws Throwable {
  String persistence = PersistencePolicies.PERMANENT;
  ServiceRecord record = createRecord(persistence);
  record.set("customkey", "customvalue");
  record.set("customkey2", "customvalue2");
  RegistryTypeUtils.validateServiceRecord("", record);
  LOG.info(marshal.toJson(record));
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord r2 = marshal.fromBytes("", bytes);
  assertMatches(record, r2);
  RegistryTypeUtils.validateServiceRecord("", r2);
}
 
Example #16
Source File: TestMarshalling.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testUnknownFieldsRoundTrip() throws Throwable {
  ServiceRecord record =
      createRecord(PersistencePolicies.APPLICATION_ATTEMPT);
  record.set("key", "value");
  record.set("intval", "2");
  assertEquals("value", record.get("key"));
  assertEquals("2", record.get("intval"));
  assertNull(record.get("null"));
  assertEquals("defval", record.get("null", "defval"));
  byte[] bytes = marshal.toBytes(record);
  ServiceRecord r2 = marshal.fromBytes("", bytes);
  assertEquals("value", r2.get("key"));
  assertEquals("2", r2.get("intval"));
}
 
Example #17
Source File: RMRegistryOperationsService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Actions to take when the AM container is completed
 * @param id  container ID
 * @throws IOException problems
 */
public void onContainerFinished(ContainerId id) throws IOException {
  LOG.info("Container {} finished, purging container-level records",
      id);
  purgeRecordsAsync("/",
      id.toString(),
      PersistencePolicies.CONTAINER);
}
 
Example #18
Source File: TestRegistryOperations.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testListListFully() throws Throwable {
  ServiceRecord r1 = new ServiceRecord();
  ServiceRecord r2 = createRecord("i",
      PersistencePolicies.PERMANENT, "r2");

  String path = USERPATH + SC_HADOOP + "/listing" ;
  operations.mknode(path, true);
  String r1path = path + "/r1";
  operations.bind(r1path, r1, 0);
  String r2path = path + "/r2";
  operations.bind(r2path, r2, 0);

  RegistryPathStatus r1stat = operations.stat(r1path);
  assertEquals("r1", r1stat.path);
  RegistryPathStatus r2stat = operations.stat(r2path);
  assertEquals("r2", r2stat.path);
  assertNotEquals(r1stat, r2stat);

  // listings now
  List<String> list = operations.list(path);
  assertEquals("Wrong no. of children", 2, list.size());
  // there's no order here, so create one
  Map<String, String> names = new HashMap<String, String>();
  String entries = "";
  for (String child : list) {
    names.put(child, child);
    entries += child + " ";
  }
  assertTrue("No 'r1' in " + entries,
      names.containsKey("r1"));
  assertTrue("No 'r2' in " + entries,
      names.containsKey("r2"));

  Map<String, RegistryPathStatus> stats =
      RegistryUtils.statChildren(operations, path);
  assertEquals("Wrong no. of children", 2, stats.size());
  assertEquals(r1stat, stats.get("r1"));
  assertEquals(r2stat, stats.get("r2"));
}
 
Example #19
Source File: TestRegistryOperations.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testPutGetContainerPersistenceServiceEntry() throws Throwable {

  String path = ENTRY_PATH;
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.CONTAINER);

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, BindFlags.CREATE);
  ServiceRecord resolved = operations.resolve(path);
  validateEntry(resolved);
  assertMatches(written, resolved);
}
 
Example #20
Source File: TestRegistryOperations.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testPutGetServiceEntry() throws Throwable {
  ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0,
      PersistencePolicies.APPLICATION);
  ServiceRecord resolved = operations.resolve(ENTRY_PATH);
  validateEntry(resolved);
  assertMatches(written, resolved);
}
 
Example #21
Source File: TestRegistryRMOperations.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testPutGetContainerPersistenceServiceEntry() throws Throwable {

  String path = ENTRY_PATH;
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.CONTAINER);

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, BindFlags.CREATE);
  ServiceRecord resolved = operations.resolve(path);
  validateEntry(resolved);
  assertMatches(written, resolved);
}
 
Example #22
Source File: RMRegistryOperationsService.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * remove all application attempt entries
 * @param attemptId attempt ID
 */
protected void purgeAppAttemptRecords(ApplicationAttemptId attemptId) {
  purgeRecordsAsync("/",
      attemptId.toString(),
      PersistencePolicies.APPLICATION_ATTEMPT);
}
 
Example #23
Source File: TestYarnPolicySelector.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testByContainer() throws Throwable {
  assertSelected(false,
      new SelectByYarnPersistence("1",
          PersistencePolicies.CONTAINER));
}
 
Example #24
Source File: TestYarnPolicySelector.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testByApp() throws Throwable {
  assertSelected(true,
      new SelectByYarnPersistence("1",
          PersistencePolicies.APPLICATION));
}
 
Example #25
Source File: TestYarnPolicySelector.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testByAppName() throws Throwable {
  assertSelected(false,
      new SelectByYarnPersistence("2",
          PersistencePolicies.APPLICATION));
}
 
Example #26
Source File: TestRegistryRMOperations.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testPurgeEntryCuratorCallback() throws Throwable {

  String path = "/users/example/hbase/hbase1/";
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.APPLICATION_ATTEMPT);
  written.set(YarnRegistryAttributes.YARN_ID,
      "testAsyncPurgeEntry_attempt_001");

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, 0);

  ZKPathDumper dump = registry.dumpPath(false);
  CuratorEventCatcher events = new CuratorEventCatcher();

  LOG.info("Initial state {}", dump);

  // container query
  String id = written.get(YarnRegistryAttributes.YARN_ID, "");
  int opcount = purge("/",
      id,
      PersistencePolicies.CONTAINER,
      RegistryAdminService.PurgePolicy.PurgeAll,
      events);
  assertPathExists(path);
  assertEquals(0, opcount);
  assertEquals("Event counter", 0, events.getCount());

  // now the application attempt
  opcount = purge("/",
      id,
      PersistencePolicies.APPLICATION_ATTEMPT,
      RegistryAdminService.PurgePolicy.PurgeAll,
      events);

  LOG.info("Final state {}", dump);

  assertPathNotFound(path);
  assertEquals("wrong no of delete operations in " + dump, 1, opcount);
  // and validate the callback event
  assertEquals("Event counter", 1, events.getCount());
}
 
Example #27
Source File: TestRegistryRMOperations.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testAsyncPurgeEntry() throws Throwable {

  String path = "/users/example/hbase/hbase1/";
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.APPLICATION_ATTEMPT);
  written.set(YarnRegistryAttributes.YARN_ID,
      "testAsyncPurgeEntry_attempt_001");

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, 0);

  ZKPathDumper dump = registry.dumpPath(false);

  LOG.info("Initial state {}", dump);

  DeleteCompletionCallback deletions = new DeleteCompletionCallback();
  int opcount = purge("/",
      written.get(YarnRegistryAttributes.YARN_ID, ""),
      PersistencePolicies.CONTAINER,
      RegistryAdminService.PurgePolicy.PurgeAll,
      deletions);
  assertPathExists(path);

  dump = registry.dumpPath(false);

  assertEquals("wrong no of delete operations in " + dump, 0,
      deletions.getEventCount());
  assertEquals("wrong no of delete operations in " + dump, 0, opcount);


  // now app attempt
  deletions = new DeleteCompletionCallback();
  opcount = purge("/",
      written.get(YarnRegistryAttributes.YARN_ID, ""),
      PersistencePolicies.APPLICATION_ATTEMPT,
      RegistryAdminService.PurgePolicy.PurgeAll,
      deletions);

  dump = registry.dumpPath(false);
  LOG.info("Final state {}", dump);

  assertPathNotFound(path);
  assertEquals("wrong no of delete operations in " + dump, 1,
      deletions.getEventCount());
  assertEquals("wrong no of delete operations in " + dump, 1, opcount);
  // and validate the callback event

}
 
Example #28
Source File: TestRegistryRMOperations.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testAsyncPurgeEntry() throws Throwable {

  String path = "/users/example/hbase/hbase1/";
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.APPLICATION_ATTEMPT);
  written.set(YarnRegistryAttributes.YARN_ID,
      "testAsyncPurgeEntry_attempt_001");

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, 0);

  ZKPathDumper dump = registry.dumpPath(false);

  LOG.info("Initial state {}", dump);

  DeleteCompletionCallback deletions = new DeleteCompletionCallback();
  int opcount = purge("/",
      written.get(YarnRegistryAttributes.YARN_ID, ""),
      PersistencePolicies.CONTAINER,
      RegistryAdminService.PurgePolicy.PurgeAll,
      deletions);
  assertPathExists(path);

  dump = registry.dumpPath(false);

  assertEquals("wrong no of delete operations in " + dump, 0,
      deletions.getEventCount());
  assertEquals("wrong no of delete operations in " + dump, 0, opcount);


  // now app attempt
  deletions = new DeleteCompletionCallback();
  opcount = purge("/",
      written.get(YarnRegistryAttributes.YARN_ID, ""),
      PersistencePolicies.APPLICATION_ATTEMPT,
      RegistryAdminService.PurgePolicy.PurgeAll,
      deletions);

  dump = registry.dumpPath(false);
  LOG.info("Final state {}", dump);

  assertPathNotFound(path);
  assertEquals("wrong no of delete operations in " + dump, 1,
      deletions.getEventCount());
  assertEquals("wrong no of delete operations in " + dump, 1, opcount);
  // and validate the callback event

}
 
Example #29
Source File: TestRegistryRMOperations.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testPurgeEntryCuratorCallback() throws Throwable {

  String path = "/users/example/hbase/hbase1/";
  ServiceRecord written = buildExampleServiceEntry(
      PersistencePolicies.APPLICATION_ATTEMPT);
  written.set(YarnRegistryAttributes.YARN_ID,
      "testAsyncPurgeEntry_attempt_001");

  operations.mknode(RegistryPathUtils.parentOf(path), true);
  operations.bind(path, written, 0);

  ZKPathDumper dump = registry.dumpPath(false);
  CuratorEventCatcher events = new CuratorEventCatcher();

  LOG.info("Initial state {}", dump);

  // container query
  String id = written.get(YarnRegistryAttributes.YARN_ID, "");
  int opcount = purge("/",
      id,
      PersistencePolicies.CONTAINER,
      RegistryAdminService.PurgePolicy.PurgeAll,
      events);
  assertPathExists(path);
  assertEquals(0, opcount);
  assertEquals("Event counter", 0, events.getCount());

  // now the application attempt
  opcount = purge("/",
      id,
      PersistencePolicies.APPLICATION_ATTEMPT,
      RegistryAdminService.PurgePolicy.PurgeAll,
      events);

  LOG.info("Final state {}", dump);

  assertPathNotFound(path);
  assertEquals("wrong no of delete operations in " + dump, 1, opcount);
  // and validate the callback event
  assertEquals("Event counter", 1, events.getCount());
}
 
Example #30
Source File: TestYarnPolicySelector.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testByAppName() throws Throwable {
  assertSelected(false,
      new SelectByYarnPersistence("2",
          PersistencePolicies.APPLICATION));
}