org.apache.hadoop.registry.client.api.RegistryOperations Java Examples

The following examples show how to use org.apache.hadoop.registry.client.api.RegistryOperations. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RegistryUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * List children of a directory and retrieve their
 * {@link RegistryPathStatus} values.
 * <p>
 * This is not an atomic operation; A child may be deleted
 * during the iteration through the child entries. If this happens,
 * the <code>PathNotFoundException</code> is caught and that child
 * entry ommitted.
 *
 * @param path path
 * @return a possibly empty map of child entries listed by
 * their short name.
 * @throws PathNotFoundException path is not in the registry.
 * @throws InvalidPathnameException the path is invalid.
 * @throws IOException Any other IO Exception
 */
public static Map<String, RegistryPathStatus> statChildren(
    RegistryOperations registryOperations,
    String path)
    throws PathNotFoundException,
    InvalidPathnameException,
    IOException {
  List<String> childNames = registryOperations.list(path);
  Map<String, RegistryPathStatus> results =
      new HashMap<String, RegistryPathStatus>();
  for (String childName : childNames) {
    String child = join(path, childName);
    try {
      RegistryPathStatus stat = registryOperations.stat(child);
      results.put(childName, stat);
    } catch (PathNotFoundException pnfe) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("stat failed on {}: moved? {}", child, pnfe, pnfe);
      }
      // and continue
    }
  }
  return results;
}
 
Example #2
Source File: RegistryUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * List children of a directory and retrieve their
 * {@link RegistryPathStatus} values.
 * <p>
 * This is not an atomic operation; A child may be deleted
 * during the iteration through the child entries. If this happens,
 * the <code>PathNotFoundException</code> is caught and that child
 * entry ommitted.
 *
 * @param path path
 * @return a possibly empty map of child entries listed by
 * their short name.
 * @throws PathNotFoundException path is not in the registry.
 * @throws InvalidPathnameException the path is invalid.
 * @throws IOException Any other IO Exception
 */
public static Map<String, RegistryPathStatus> statChildren(
    RegistryOperations registryOperations,
    String path)
    throws PathNotFoundException,
    InvalidPathnameException,
    IOException {
  List<String> childNames = registryOperations.list(path);
  Map<String, RegistryPathStatus> results =
      new HashMap<String, RegistryPathStatus>();
  for (String childName : childNames) {
    String child = join(path, childName);
    try {
      RegistryPathStatus stat = registryOperations.stat(child);
      results.put(childName, stat);
    } catch (PathNotFoundException pnfe) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("stat failed on {}: moved? {}", child, pnfe, pnfe);
      }
      // and continue
    }
  }
  return results;
}
 
Example #3
Source File: YarnRegistryViewForProviders.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public YarnRegistryViewForProviders(RegistryOperations registryOperations,
                                    String user,
                                    String jstormServiceClass,
                                    String instanceName,
                                    ApplicationAttemptId applicationAttemptId) {
  Preconditions.checkArgument(registryOperations != null,
      "null registry operations");
  Preconditions.checkArgument(user != null, "null user");
  Preconditions.checkArgument(JstormYarnUtils.isSet(jstormServiceClass),
      "unset service class");
  Preconditions.checkArgument(JstormYarnUtils.isSet(instanceName),
      "instanceName");
  Preconditions.checkArgument(applicationAttemptId != null,
      "null applicationAttemptId");
  this.registryOperations = registryOperations;
  this.user = user;
  this.jstormServiceClass = jstormServiceClass;
  this.instanceName = instanceName;
  this.applicationAttemptId = applicationAttemptId;
}
 
Example #4
Source File: RegistryCli.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public RegistryCli(RegistryOperations reg,
    Configuration conf,
    PrintStream sysout,
    PrintStream syserr) {
  super(conf);
  Preconditions.checkArgument(reg != null, "Null registry");
  registry = reg;
  this.sysout = sysout;
  this.syserr = syserr;
}
 
Example #5
Source File: TestSecureRMRegistryOperations.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testAlicePathRestrictedAnonAccess() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  String aliceHome = rmRegistryOperations.initUserRegistry(ALICE);
  describe(LOG, "Creating anonymous accessor");
  RegistryOperations anonOperations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(anonOperations);
  anonOperations.start();
  anonOperations.list(aliceHome);
  expectMkNodeFailure(anonOperations, aliceHome + "/anon");
  expectDeleteFailure(anonOperations, aliceHome, true);
}
 
Example #6
Source File: TestSecureRMRegistryOperations.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testAnonNoWriteAccessOffRoot() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  describe(LOG, "testAnonNoWriteAccessOffRoot");
  RegistryOperations operations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(operations);
  operations.start();
  assertFalse("mknode(/)", operations.mknode("/", false));
  expectMkNodeFailure(operations, "/sub");
  expectDeleteFailure(operations, PATH_SYSTEM_SERVICES, true);
}
 
Example #7
Source File: TestSecureRMRegistryOperations.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testAnonNoWriteAccess() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  describe(LOG, "testAnonNoWriteAccess");
  RegistryOperations operations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(operations);
  operations.start();

  String servicePath = PATH_SYSTEM_SERVICES + "hdfs";
  expectMkNodeFailure(operations, servicePath);
}
 
Example #8
Source File: TestSecureRMRegistryOperations.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testAnonReadAccess() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  describe(LOG, "testAnonReadAccess");
  RegistryOperations operations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(operations);
  operations.start();

  assertFalse("RegistrySecurity.isClientSASLEnabled()==true",
      RegistrySecurity.isClientSASLEnabled());
  operations.list(PATH_SYSTEM_SERVICES);
}
 
Example #9
Source File: TestSecureRMRegistryOperations.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * test that ZK can write as itself
 * @throws Throwable
 */
@Test
public void testZookeeperCanWriteUnderSystem() throws Throwable {

  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  RegistryOperations operations = rmRegistryOperations;
  operations.mknode(PATH_SYSTEM_SERVICES + "hdfs",
      false);
  ZKPathDumper pathDumper = rmRegistryOperations.dumpPath(true);
  LOG.info(pathDumper.toString());
}
 
Example #10
Source File: RegistryCli.java    From big-c with Apache License 2.0 5 votes vote down vote up
public RegistryCli(RegistryOperations reg,
    Configuration conf,
    PrintStream sysout,
    PrintStream syserr) {
  super(conf);
  Preconditions.checkArgument(reg != null, "Null registry");
  registry = reg;
  this.sysout = sysout;
  this.syserr = syserr;
}
 
Example #11
Source File: JstormYarnUtils.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static List<String> getSupervisorPorts(int memory, int vcores, String instanceName, String supervisorHost, RegistryOperations registryOperations) {
    List<String> relist = new ArrayList<String>();
    int slotCount = getSlotCount(memory, vcores);
    for (int i = 9000; i < 15000; i++) {
        if (isPortAvailable(supervisorHost, i)) {
            if (!getSetPortUsedBySupervisor(instanceName, supervisorHost, i, registryOperations))
                relist.add(String.valueOf(i));
        }
        if (relist.size() >= slotCount) {
            break;
        }
    }
    return relist;
}
 
Example #12
Source File: RegistryUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * List service records directly under a path
 * @param registryOperations registry operations instance
 * @param path path to list
 * @return a mapping of the service records that were resolved, indexed
 * by their full path
 * @throws IOException
 */
public static Map<String, ServiceRecord> listServiceRecords(
    RegistryOperations registryOperations,
    String path) throws IOException {
  Map<String, RegistryPathStatus> children =
      statChildren(registryOperations, path);
  return extractServiceRecords(registryOperations,
      path,
      children.values());
}
 
Example #13
Source File: RegistryUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * List service records directly under a path
 * @param registryOperations registry operations instance
 * @param path path to list
 * @return a mapping of the service records that were resolved, indexed
 * by their full path
 * @throws IOException
 */
public static Map<String, ServiceRecord> listServiceRecords(
    RegistryOperations registryOperations,
    String path) throws IOException {
  Map<String, RegistryPathStatus> children =
      statChildren(registryOperations, path);
  return extractServiceRecords(registryOperations,
      path,
      children.values());
}
 
Example #14
Source File: TestSecureRMRegistryOperations.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testAlicePathRestrictedAnonAccess() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  String aliceHome = rmRegistryOperations.initUserRegistry(ALICE);
  describe(LOG, "Creating anonymous accessor");
  RegistryOperations anonOperations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(anonOperations);
  anonOperations.start();
  anonOperations.list(aliceHome);
  expectMkNodeFailure(anonOperations, aliceHome + "/anon");
  expectDeleteFailure(anonOperations, aliceHome, true);
}
 
Example #15
Source File: TestSecureRMRegistryOperations.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testAnonNoWriteAccessOffRoot() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  describe(LOG, "testAnonNoWriteAccessOffRoot");
  RegistryOperations operations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(operations);
  operations.start();
  assertFalse("mknode(/)", operations.mknode("/", false));
  expectMkNodeFailure(operations, "/sub");
  expectDeleteFailure(operations, PATH_SYSTEM_SERVICES, true);
}
 
Example #16
Source File: TestSecureRMRegistryOperations.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testAnonNoWriteAccess() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  describe(LOG, "testAnonNoWriteAccess");
  RegistryOperations operations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(operations);
  operations.start();

  String servicePath = PATH_SYSTEM_SERVICES + "hdfs";
  expectMkNodeFailure(operations, servicePath);
}
 
Example #17
Source File: TestSecureRMRegistryOperations.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testAnonReadAccess() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  describe(LOG, "testAnonReadAccess");
  RegistryOperations operations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(operations);
  operations.start();

  assertFalse("RegistrySecurity.isClientSASLEnabled()==true",
      RegistrySecurity.isClientSASLEnabled());
  operations.list(PATH_SYSTEM_SERVICES);
}
 
Example #18
Source File: TestSecureRMRegistryOperations.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * test that ZK can write as itself
 * @throws Throwable
 */
@Test
public void testZookeeperCanWriteUnderSystem() throws Throwable {

  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  RegistryOperations operations = rmRegistryOperations;
  operations.mknode(PATH_SYSTEM_SERVICES + "hdfs",
      false);
  ZKPathDumper pathDumper = rmRegistryOperations.dumpPath(true);
  LOG.info(pathDumper.toString());
}
 
Example #19
Source File: ContainersView.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public ContainersView(String instanceName, String host, ContainerId containerId, RegistryOperations registryOperations) {
    this.instanceName = instanceName;
    this.containerId = containerId;
    this.registryOperations = registryOperations;
    this.host = host;
}
 
Example #20
Source File: SlotPortsView.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public SlotPortsView(String instanceName, ContainerId containerId, RegistryOperations registryOperations) {
    this.instanceName = instanceName;
    this.containerId = containerId;
    this.registryOperations = registryOperations;
}
 
Example #21
Source File: YarnRegistryViewForProviders.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public RegistryOperations getRegistryOperations() {
  return registryOperations;
}
 
Example #22
Source File: TestSecureRMRegistryOperations.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testDigestAccess() throws Throwable {
  RMRegistryOperationsService registryAdmin =
      startRMRegistryOperations();
  String id = "username";
  String pass = "password";
  registryAdmin.addWriteAccessor(id, pass);
  List<ACL> clientAcls = registryAdmin.getClientAcls();
  LOG.info("Client ACLS=\n{}", RegistrySecurity.aclsToString(clientAcls));

  String base = "/digested";
  registryAdmin.mknode(base, false);
  List<ACL> baseACLs = registryAdmin.zkGetACLS(base);
  String aclset = RegistrySecurity.aclsToString(baseACLs);
  LOG.info("Base ACLs=\n{}", aclset);
  ACL found = null;
  for (ACL acl : baseACLs) {
    if (ZookeeperConfigOptions.SCHEME_DIGEST.equals(acl.getId().getScheme())) {
      found = acl;
      break;
    }
  }
  assertNotNull("Did not find digest entry in ACLs " + aclset, found);
  zkClientConf.set(KEY_REGISTRY_USER_ACCOUNTS,
      "sasl:[email protected], sasl:other");
  RegistryOperations operations =
      RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf,
          id,
          pass);
  addToTeardown(operations);
  operations.start();
  RegistryOperationsClient operationsClient =
      (RegistryOperationsClient) operations;
  List<ACL> digestClientACLs = operationsClient.getClientAcls();
  LOG.info("digest client ACLs=\n{}",
      RegistrySecurity.aclsToString(digestClientACLs));
  operations.stat(base);
  operations.mknode(base + "/subdir", false);
  ZKPathDumper pathDumper = registryAdmin.dumpPath(true);
  LOG.info(pathDumper.toString());
}
 
Example #23
Source File: TestSecureRMRegistryOperations.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testDigestAccess() throws Throwable {
  RMRegistryOperationsService registryAdmin =
      startRMRegistryOperations();
  String id = "username";
  String pass = "password";
  registryAdmin.addWriteAccessor(id, pass);
  List<ACL> clientAcls = registryAdmin.getClientAcls();
  LOG.info("Client ACLS=\n{}", RegistrySecurity.aclsToString(clientAcls));

  String base = "/digested";
  registryAdmin.mknode(base, false);
  List<ACL> baseACLs = registryAdmin.zkGetACLS(base);
  String aclset = RegistrySecurity.aclsToString(baseACLs);
  LOG.info("Base ACLs=\n{}", aclset);
  ACL found = null;
  for (ACL acl : baseACLs) {
    if (ZookeeperConfigOptions.SCHEME_DIGEST.equals(acl.getId().getScheme())) {
      found = acl;
      break;
    }
  }
  assertNotNull("Did not find digest entry in ACLs " + aclset, found);
  zkClientConf.set(KEY_REGISTRY_USER_ACCOUNTS,
      "sasl:[email protected], sasl:other");
  RegistryOperations operations =
      RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf,
          id,
          pass);
  addToTeardown(operations);
  operations.start();
  RegistryOperationsClient operationsClient =
      (RegistryOperationsClient) operations;
  List<ACL> digestClientACLs = operationsClient.getClientAcls();
  LOG.info("digest client ACLs=\n{}",
      RegistrySecurity.aclsToString(digestClientACLs));
  operations.stat(base);
  operations.mknode(base + "/subdir", false);
  ZKPathDumper pathDumper = registryAdmin.dumpPath(true);
  LOG.info(pathDumper.toString());
}
 
Example #24
Source File: RegistryUtils.java    From big-c with Apache License 2.0 3 votes vote down vote up
/**
 * Extract all service records under a list of stat operations...this
 * non-atomic action skips entries that are too short or simply not matching.
 * <p>
 * @param operations operation support for fetches
 * @param parentpath path of the parent of all the entries
 * @return a possibly empty map of fullpath:record.
 * @throws IOException for any IO Operation that wasn't ignored.
 */
public static Map<String, ServiceRecord> extractServiceRecords(
    RegistryOperations operations,
    String parentpath) throws IOException {
  return
  extractServiceRecords(operations,
      parentpath,
      statChildren(operations, parentpath).values());
}
 
Example #25
Source File: RegistryUtils.java    From big-c with Apache License 2.0 3 votes vote down vote up
/**
 * Extract all service records under a list of stat operations...this
 * non-atomic action skips entries that are too short or simply not matching.
 * <p>
 * @param operations operation support for fetches
 * @param parentpath path of the parent of all the entries
 * @return a possibly empty map of fullpath:record.
 * @throws IOException for any IO Operation that wasn't ignored.
 */
public static Map<String, ServiceRecord> extractServiceRecords(
    RegistryOperations operations,
    String parentpath,
    Map<String , RegistryPathStatus> stats) throws IOException {
  return extractServiceRecords(operations, parentpath, stats.values());
}
 
Example #26
Source File: RegistryUtils.java    From hadoop with Apache License 2.0 3 votes vote down vote up
/**
 * Extract all service records under a list of stat operations...this
 * non-atomic action skips entries that are too short or simply not matching.
 * <p>
 * @param operations operation support for fetches
 * @param parentpath path of the parent of all the entries
 * @return a possibly empty map of fullpath:record.
 * @throws IOException for any IO Operation that wasn't ignored.
 */
public static Map<String, ServiceRecord> extractServiceRecords(
    RegistryOperations operations,
    String parentpath) throws IOException {
  return
  extractServiceRecords(operations,
      parentpath,
      statChildren(operations, parentpath).values());
}
 
Example #27
Source File: RegistryUtils.java    From hadoop with Apache License 2.0 3 votes vote down vote up
/**
 * Extract all service records under a list of stat operations...this
 * non-atomic action skips entries that are too short or simply not matching.
 * <p>
 * @param operations operation support for fetches
 * @param parentpath path of the parent of all the entries
 * @return a possibly empty map of fullpath:record.
 * @throws IOException for any IO Operation that wasn't ignored.
 */
public static Map<String, ServiceRecord> extractServiceRecords(
    RegistryOperations operations,
    String parentpath,
    Map<String , RegistryPathStatus> stats) throws IOException {
  return extractServiceRecords(operations, parentpath, stats.values());
}
 
Example #28
Source File: JstormYarnUtils.java    From jstorm with Apache License 2.0 2 votes vote down vote up
/**
 * this is for jstorm configuration's format
 *
 * @param memory
 * @param vcores
 * @param supervisorHost
 * @return
 */
public static String getSupervisorSlotPorts(int memory, int vcores, String instanceName, String supervisorHost, RegistryOperations registryOperations) {
    return join(getSupervisorPorts(memory, vcores, instanceName, supervisorHost, registryOperations), JOYConstants.COMMA, false);
}