org.apache.hadoop.registry.client.api.RegistryOperations Java Examples

/**
 * List children of a directory and retrieve their
 * {@link RegistryPathStatus} values.
 * <p>
 * This is not an atomic operation; A child may be deleted
 * during the iteration through the child entries. If this happens,
 * the <code>PathNotFoundException</code> is caught and that child
 * entry ommitted.
 *
 * @param path path
 * @return a possibly empty map of child entries listed by
 * their short name.
 * @throws PathNotFoundException path is not in the registry.
 * @throws InvalidPathnameException the path is invalid.
 * @throws IOException Any other IO Exception
 */
public static Map<String, RegistryPathStatus> statChildren(
    RegistryOperations registryOperations,
    String path)
    throws PathNotFoundException,
    InvalidPathnameException,
    IOException {
  List<String> childNames = registryOperations.list(path);
  Map<String, RegistryPathStatus> results =
      new HashMap<String, RegistryPathStatus>();
  for (String childName : childNames) {
    String child = join(path, childName);
    try {
      RegistryPathStatus stat = registryOperations.stat(child);
      results.put(childName, stat);
    } catch (PathNotFoundException pnfe) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("stat failed on {}: moved? {}", child, pnfe, pnfe);
      }
      // and continue
    }
  }
  return results;
}
 

/**
 * List children of a directory and retrieve their
 * {@link RegistryPathStatus} values.
 * <p>
 * This is not an atomic operation; A child may be deleted
 * during the iteration through the child entries. If this happens,
 * the <code>PathNotFoundException</code> is caught and that child
 * entry ommitted.
 *
 * @param path path
 * @return a possibly empty map of child entries listed by
 * their short name.
 * @throws PathNotFoundException path is not in the registry.
 * @throws InvalidPathnameException the path is invalid.
 * @throws IOException Any other IO Exception
 */
public static Map<String, RegistryPathStatus> statChildren(
    RegistryOperations registryOperations,
    String path)
    throws PathNotFoundException,
    InvalidPathnameException,
    IOException {
  List<String> childNames = registryOperations.list(path);
  Map<String, RegistryPathStatus> results =
      new HashMap<String, RegistryPathStatus>();
  for (String childName : childNames) {
    String child = join(path, childName);
    try {
      RegistryPathStatus stat = registryOperations.stat(child);
      results.put(childName, stat);
    } catch (PathNotFoundException pnfe) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("stat failed on {}: moved? {}", child, pnfe, pnfe);
      }
      // and continue
    }
  }
  return results;
}
 

public YarnRegistryViewForProviders(RegistryOperations registryOperations,
                                    String user,
                                    String jstormServiceClass,
                                    String instanceName,
                                    ApplicationAttemptId applicationAttemptId) {
  Preconditions.checkArgument(registryOperations != null,
      "null registry operations");
  Preconditions.checkArgument(user != null, "null user");
  Preconditions.checkArgument(JstormYarnUtils.isSet(jstormServiceClass),
      "unset service class");
  Preconditions.checkArgument(JstormYarnUtils.isSet(instanceName),
      "instanceName");
  Preconditions.checkArgument(applicationAttemptId != null,
      "null applicationAttemptId");
  this.registryOperations = registryOperations;
  this.user = user;
  this.jstormServiceClass = jstormServiceClass;
  this.instanceName = instanceName;
  this.applicationAttemptId = applicationAttemptId;
}
 

public RegistryCli(RegistryOperations reg,
    Configuration conf,
    PrintStream sysout,
    PrintStream syserr) {
  super(conf);
  Preconditions.checkArgument(reg != null, "Null registry");
  registry = reg;
  this.sysout = sysout;
  this.syserr = syserr;
}
 

@Test
public void testAlicePathRestrictedAnonAccess() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  String aliceHome = rmRegistryOperations.initUserRegistry(ALICE);
  describe(LOG, "Creating anonymous accessor");
  RegistryOperations anonOperations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(anonOperations);
  anonOperations.start();
  anonOperations.list(aliceHome);
  expectMkNodeFailure(anonOperations, aliceHome + "/anon");
  expectDeleteFailure(anonOperations, aliceHome, true);
}
 

@Test
public void testAnonNoWriteAccessOffRoot() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  describe(LOG, "testAnonNoWriteAccessOffRoot");
  RegistryOperations operations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(operations);
  operations.start();
  assertFalse("mknode(/)", operations.mknode("/", false));
  expectMkNodeFailure(operations, "/sub");
  expectDeleteFailure(operations, PATH_SYSTEM_SERVICES, true);
}
 

@Test
public void testAnonNoWriteAccess() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  describe(LOG, "testAnonNoWriteAccess");
  RegistryOperations operations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(operations);
  operations.start();

  String servicePath = PATH_SYSTEM_SERVICES + "hdfs";
  expectMkNodeFailure(operations, servicePath);
}
 

@Test
public void testAnonReadAccess() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  describe(LOG, "testAnonReadAccess");
  RegistryOperations operations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(operations);
  operations.start();

  assertFalse("RegistrySecurity.isClientSASLEnabled()==true",
      RegistrySecurity.isClientSASLEnabled());
  operations.list(PATH_SYSTEM_SERVICES);
}
 

/**
 * test that ZK can write as itself
 * @throws Throwable
 */
@Test
public void testZookeeperCanWriteUnderSystem() throws Throwable {

  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  RegistryOperations operations = rmRegistryOperations;
  operations.mknode(PATH_SYSTEM_SERVICES + "hdfs",
      false);
  ZKPathDumper pathDumper = rmRegistryOperations.dumpPath(true);
  LOG.info(pathDumper.toString());
}
 

public RegistryCli(RegistryOperations reg,
    Configuration conf,
    PrintStream sysout,
    PrintStream syserr) {
  super(conf);
  Preconditions.checkArgument(reg != null, "Null registry");
  registry = reg;
  this.sysout = sysout;
  this.syserr = syserr;
}
 

public static List<String> getSupervisorPorts(int memory, int vcores, String instanceName, String supervisorHost, RegistryOperations registryOperations) {
    List<String> relist = new ArrayList<String>();
    int slotCount = getSlotCount(memory, vcores);
    for (int i = 9000; i < 15000; i++) {
        if (isPortAvailable(supervisorHost, i)) {
            if (!getSetPortUsedBySupervisor(instanceName, supervisorHost, i, registryOperations))
                relist.add(String.valueOf(i));
        }
        if (relist.size() >= slotCount) {
            break;
        }
    }
    return relist;
}
 

/**
 * List service records directly under a path
 * @param registryOperations registry operations instance
 * @param path path to list
 * @return a mapping of the service records that were resolved, indexed
 * by their full path
 * @throws IOException
 */
public static Map<String, ServiceRecord> listServiceRecords(
    RegistryOperations registryOperations,
    String path) throws IOException {
  Map<String, RegistryPathStatus> children =
      statChildren(registryOperations, path);
  return extractServiceRecords(registryOperations,
      path,
      children.values());
}
 

/**
 * List service records directly under a path
 * @param registryOperations registry operations instance
 * @param path path to list
 * @return a mapping of the service records that were resolved, indexed
 * by their full path
 * @throws IOException
 */
public static Map<String, ServiceRecord> listServiceRecords(
    RegistryOperations registryOperations,
    String path) throws IOException {
  Map<String, RegistryPathStatus> children =
      statChildren(registryOperations, path);
  return extractServiceRecords(registryOperations,
      path,
      children.values());
}
 

@Test
public void testAlicePathRestrictedAnonAccess() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  String aliceHome = rmRegistryOperations.initUserRegistry(ALICE);
  describe(LOG, "Creating anonymous accessor");
  RegistryOperations anonOperations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(anonOperations);
  anonOperations.start();
  anonOperations.list(aliceHome);
  expectMkNodeFailure(anonOperations, aliceHome + "/anon");
  expectDeleteFailure(anonOperations, aliceHome, true);
}
 

@Test
public void testAnonNoWriteAccessOffRoot() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  describe(LOG, "testAnonNoWriteAccessOffRoot");
  RegistryOperations operations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(operations);
  operations.start();
  assertFalse("mknode(/)", operations.mknode("/", false));
  expectMkNodeFailure(operations, "/sub");
  expectDeleteFailure(operations, PATH_SYSTEM_SERVICES, true);
}
 

@Test
public void testAnonNoWriteAccess() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  describe(LOG, "testAnonNoWriteAccess");
  RegistryOperations operations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(operations);
  operations.start();

  String servicePath = PATH_SYSTEM_SERVICES + "hdfs";
  expectMkNodeFailure(operations, servicePath);
}
 

@Test
public void testAnonReadAccess() throws Throwable {
  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  describe(LOG, "testAnonReadAccess");
  RegistryOperations operations =
      RegistryOperationsFactory.createAnonymousInstance(zkClientConf);
  addToTeardown(operations);
  operations.start();

  assertFalse("RegistrySecurity.isClientSASLEnabled()==true",
      RegistrySecurity.isClientSASLEnabled());
  operations.list(PATH_SYSTEM_SERVICES);
}
 

/**
 * test that ZK can write as itself
 * @throws Throwable
 */
@Test
public void testZookeeperCanWriteUnderSystem() throws Throwable {

  RMRegistryOperationsService rmRegistryOperations =
      startRMRegistryOperations();
  RegistryOperations operations = rmRegistryOperations;
  operations.mknode(PATH_SYSTEM_SERVICES + "hdfs",
      false);
  ZKPathDumper pathDumper = rmRegistryOperations.dumpPath(true);
  LOG.info(pathDumper.toString());
}
 

public ContainersView(String instanceName, String host, ContainerId containerId, RegistryOperations registryOperations) {
    this.instanceName = instanceName;
    this.containerId = containerId;
    this.registryOperations = registryOperations;
    this.host = host;
}
 

public SlotPortsView(String instanceName, ContainerId containerId, RegistryOperations registryOperations) {
    this.instanceName = instanceName;
    this.containerId = containerId;
    this.registryOperations = registryOperations;
}
 

public RegistryOperations getRegistryOperations() {
  return registryOperations;
}
 

@Test
public void testDigestAccess() throws Throwable {
  RMRegistryOperationsService registryAdmin =
      startRMRegistryOperations();
  String id = "username";
  String pass = "password";
  registryAdmin.addWriteAccessor(id, pass);
  List<ACL> clientAcls = registryAdmin.getClientAcls();
  LOG.info("Client ACLS=\n{}", RegistrySecurity.aclsToString(clientAcls));

  String base = "/digested";
  registryAdmin.mknode(base, false);
  List<ACL> baseACLs = registryAdmin.zkGetACLS(base);
  String aclset = RegistrySecurity.aclsToString(baseACLs);
  LOG.info("Base ACLs=\n{}", aclset);
  ACL found = null;
  for (ACL acl : baseACLs) {
    if (ZookeeperConfigOptions.SCHEME_DIGEST.equals(acl.getId().getScheme())) {
      found = acl;
      break;
    }
  }
  assertNotNull("Did not find digest entry in ACLs " + aclset, found);
  zkClientConf.set(KEY_REGISTRY_USER_ACCOUNTS,
      "sasl:[email protected], sasl:other");
  RegistryOperations operations =
      RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf,
          id,
          pass);
  addToTeardown(operations);
  operations.start();
  RegistryOperationsClient operationsClient =
      (RegistryOperationsClient) operations;
  List<ACL> digestClientACLs = operationsClient.getClientAcls();
  LOG.info("digest client ACLs=\n{}",
      RegistrySecurity.aclsToString(digestClientACLs));
  operations.stat(base);
  operations.mknode(base + "/subdir", false);
  ZKPathDumper pathDumper = registryAdmin.dumpPath(true);
  LOG.info(pathDumper.toString());
}
 

@Test
public void testDigestAccess() throws Throwable {
  RMRegistryOperationsService registryAdmin =
      startRMRegistryOperations();
  String id = "username";
  String pass = "password";
  registryAdmin.addWriteAccessor(id, pass);
  List<ACL> clientAcls = registryAdmin.getClientAcls();
  LOG.info("Client ACLS=\n{}", RegistrySecurity.aclsToString(clientAcls));

  String base = "/digested";
  registryAdmin.mknode(base, false);
  List<ACL> baseACLs = registryAdmin.zkGetACLS(base);
  String aclset = RegistrySecurity.aclsToString(baseACLs);
  LOG.info("Base ACLs=\n{}", aclset);
  ACL found = null;
  for (ACL acl : baseACLs) {
    if (ZookeeperConfigOptions.SCHEME_DIGEST.equals(acl.getId().getScheme())) {
      found = acl;
      break;
    }
  }
  assertNotNull("Did not find digest entry in ACLs " + aclset, found);
  zkClientConf.set(KEY_REGISTRY_USER_ACCOUNTS,
      "sasl:[email protected], sasl:other");
  RegistryOperations operations =
      RegistryOperationsFactory.createAuthenticatedInstance(zkClientConf,
          id,
          pass);
  addToTeardown(operations);
  operations.start();
  RegistryOperationsClient operationsClient =
      (RegistryOperationsClient) operations;
  List<ACL> digestClientACLs = operationsClient.getClientAcls();
  LOG.info("digest client ACLs=\n{}",
      RegistrySecurity.aclsToString(digestClientACLs));
  operations.stat(base);
  operations.mknode(base + "/subdir", false);
  ZKPathDumper pathDumper = registryAdmin.dumpPath(true);
  LOG.info(pathDumper.toString());
}
 

/**
 * Extract all service records under a list of stat operations...this
 * non-atomic action skips entries that are too short or simply not matching.
 * <p>
 * @param operations operation support for fetches
 * @param parentpath path of the parent of all the entries
 * @return a possibly empty map of fullpath:record.
 * @throws IOException for any IO Operation that wasn't ignored.
 */
public static Map<String, ServiceRecord> extractServiceRecords(
    RegistryOperations operations,
    String parentpath) throws IOException {
  return
  extractServiceRecords(operations,
      parentpath,
      statChildren(operations, parentpath).values());
}
 

/**
 * Extract all service records under a list of stat operations...this
 * non-atomic action skips entries that are too short or simply not matching.
 * <p>
 * @param operations operation support for fetches
 * @param parentpath path of the parent of all the entries
 * @return a possibly empty map of fullpath:record.
 * @throws IOException for any IO Operation that wasn't ignored.
 */
public static Map<String, ServiceRecord> extractServiceRecords(
    RegistryOperations operations,
    String parentpath,
    Map<String , RegistryPathStatus> stats) throws IOException {
  return extractServiceRecords(operations, parentpath, stats.values());
}
 

/**
 * Extract all service records under a list of stat operations...this
 * non-atomic action skips entries that are too short or simply not matching.
 * <p>
 * @param operations operation support for fetches
 * @param parentpath path of the parent of all the entries
 * @return a possibly empty map of fullpath:record.
 * @throws IOException for any IO Operation that wasn't ignored.
 */
public static Map<String, ServiceRecord> extractServiceRecords(
    RegistryOperations operations,
    String parentpath) throws IOException {
  return
  extractServiceRecords(operations,
      parentpath,
      statChildren(operations, parentpath).values());
}
 

/**
 * Extract all service records under a list of stat operations...this
 * non-atomic action skips entries that are too short or simply not matching.
 * <p>
 * @param operations operation support for fetches
 * @param parentpath path of the parent of all the entries
 * @return a possibly empty map of fullpath:record.
 * @throws IOException for any IO Operation that wasn't ignored.
 */
public static Map<String, ServiceRecord> extractServiceRecords(
    RegistryOperations operations,
    String parentpath,
    Map<String , RegistryPathStatus> stats) throws IOException {
  return extractServiceRecords(operations, parentpath, stats.values());
}
 

/**
 * this is for jstorm configuration's format
 *
 * @param memory
 * @param vcores
 * @param supervisorHost
 * @return
 */
public static String getSupervisorSlotPorts(int memory, int vcores, String instanceName, String supervisorHost, RegistryOperations registryOperations) {
    return join(getSupervisorPorts(memory, vcores, instanceName, supervisorHost, registryOperations), JOYConstants.COMMA, false);
}