Java Code Examples for org.apache.hadoop.conf.Configuration#addDefaultResource()

The following examples show how to use org.apache.hadoop.conf.Configuration#addDefaultResource() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MiniDFS.java    From incubator-sentry with Apache License 2.0 6 votes vote down vote up 
MiniDFS(File baseDir, String serverType) throws Exception {
  Configuration conf = new Configuration();
  if (HiveServer2Type.InternalMetastore.name().equalsIgnoreCase(serverType)) {
    // set the test group mapping that maps user to a group of same name
    conf.set("hadoop.security.group.mapping",
        "org.apache.sentry.tests.e2e.hive.fs.MiniDFS$PseudoGroupMappingService");
    // set umask for metastore test client can create tables in the warehouse dir
    conf.set("fs.permissions.umask-mode", "000");
    Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf);
  }
  File dfsDir = assertCreateDir(new File(baseDir, "dfs"));
  conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dfsDir.getPath());
  conf.set("hadoop.security.group.mapping",
      MiniDFS.PseudoGroupMappingService.class.getName());
  Configuration.addDefaultResource("test.xml");
  dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  fileSystem = dfsCluster.getFileSystem();
  String policyDir = System.getProperty("sentry.e2etest.hive.policy.location", "/user/hive/sentry");
  sentryDir = super.assertCreateDfsDir(new Path(fileSystem.getUri() + policyDir));
  dfsBaseDir = assertCreateDfsDir(new Path(new Path(fileSystem.getUri()), "/base"));
}
Example 2
Source File: TestRefreshUserMappings.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private void addNewConfigResource(String rsrcName, String keyGroup,
    String groups, String keyHosts, String hosts)
        throws FileNotFoundException, UnsupportedEncodingException {
  // location for temp resource should be in CLASSPATH
  Configuration conf = new Configuration();
  URL url = conf.getResource("hdfs-site.xml");

  String urlPath = URLDecoder.decode(url.getPath().toString(), "UTF-8");
  Path p = new Path(urlPath);
  Path dir = p.getParent();
  tempResource = dir.toString() + "/" + rsrcName;

  String newResource =
  "<configuration>"+
  "<property><name>" + keyGroup + "</name><value>"+groups+"</value></property>" +
  "<property><name>" + keyHosts + "</name><value>"+hosts+"</value></property>" +
  "</configuration>";
  PrintWriter writer = new PrintWriter(new FileOutputStream(tempResource));
  writer.println(newResource);
  writer.close();

  Configuration.addDefaultResource(rsrcName);
}
Example 3
Source File: TestRefreshUserMappings.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private void addNewConfigResource(String rsrcName, String keyGroup,
    String groups, String keyHosts, String hosts)
        throws FileNotFoundException, UnsupportedEncodingException {
  // location for temp resource should be in CLASSPATH
  Configuration conf = new Configuration();
  URL url = conf.getResource("hdfs-site.xml");

  String urlPath = URLDecoder.decode(url.getPath().toString(), "UTF-8");
  Path p = new Path(urlPath);
  Path dir = p.getParent();
  tempResource = dir.toString() + "/" + rsrcName;

  String newResource =
  "<configuration>"+
  "<property><name>" + keyGroup + "</name><value>"+groups+"</value></property>" +
  "<property><name>" + keyHosts + "</name><value>"+hosts+"</value></property>" +
  "</configuration>";
  PrintWriter writer = new PrintWriter(new FileOutputStream(tempResource));
  writer.println(newResource);
  writer.close();

  Configuration.addDefaultResource(rsrcName);
}
Example 4
Source File: TestResourceManagerAdministrationProtocolPBClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Start resource manager server
 */

@BeforeClass
public static void setUpResourceManager() throws IOException,
        InterruptedException {
  Configuration.addDefaultResource("config-with-security.xml");
  Configuration configuration = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    }
  };
  resourceManager.init(configuration);
  new Thread() {
    public void run() {
      resourceManager.start();
    }
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
          && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException("ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: "
          + configuration.get(YarnConfiguration.RM_ADMIN_ADDRESS));

  client = new ResourceManagerAdministrationProtocolPBClientImpl(1L,
          getProtocolAddress(configuration), configuration);

}
Example 5
Source File: ClusterBalancerTool.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
private ClusterBalancerAdminProtocol getClusterBalancerAdmin()
        throws IOException {
  Configuration.addDefaultResource(
          DynamicCloudsDaemon.CLUSTER_BALANCER_CONF_FILE);
  Configuration conf = this.getConf();
  String addr = conf.get(DynamicCloudsDaemon.CLUSTER_BALANCER_ADDR);

  InetSocketAddress isAddr = NetUtils.createSocketAddr(addr);

  return (ClusterBalancerAdminProtocol) RPC.getProxy(
          ClusterBalancerAdminProtocol.class,
          ClusterBalancerAdminProtocol.versionID, isAddr, conf);
}
Example 6
Source File: ConfigUtil.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Adds all the deprecated keys. Loads mapred-default.xml and mapred-site.xml
 */
public static void loadResources() {
  addDeprecatedKeys();
  Configuration.addDefaultResource("mapred-default.xml");
  Configuration.addDefaultResource("mapred-site.xml");
  Configuration.addDefaultResource("yarn-default.xml");
  Configuration.addDefaultResource("yarn-site.xml");
}
Example 7
Source File: TestResourceManagerAdministrationProtocolPBClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Start resource manager server
 */

@BeforeClass
public static void setUpResourceManager() throws IOException,
        InterruptedException {
  Configuration.addDefaultResource("config-with-security.xml");
  Configuration configuration = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    }
  };
  resourceManager.init(configuration);
  new Thread() {
    public void run() {
      resourceManager.start();
    }
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
          && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException("ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: "
          + configuration.get(YarnConfiguration.RM_ADMIN_ADDRESS));

  client = new ResourceManagerAdministrationProtocolPBClientImpl(1L,
          getProtocolAddress(configuration), configuration);

}
Example 8
Source File: ConfigUtil.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Adds all the deprecated keys. Loads mapred-default.xml and mapred-site.xml
 */
public static void loadResources() {
  addDeprecatedKeys();
  Configuration.addDefaultResource("mapred-default.xml");
  Configuration.addDefaultResource("mapred-site.xml");
  Configuration.addDefaultResource("yarn-default.xml");
  Configuration.addDefaultResource("yarn-site.xml");
}
Example 9
Source File: TestMalformedURLs.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() throws Exception {
  Configuration.addDefaultResource("hdfs-site.malformed.xml");
  config = new Configuration();
}
Example 10
Source File: TestMalformedURLs.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() throws Exception {
  Configuration.addDefaultResource("hdfs-site.malformed.xml");
  config = new Configuration();
}
Example 11
Source File: ImmutableConfiguration.java    From titan1withtp3.1 with Apache License 2.0 4 votes vote down vote up 
public static void addDefaultResource(String name) {
    Configuration.addDefaultResource(name);
}
Example 12
Source File: CoprocessorTool.java    From eagle with Apache License 2.0 4 votes vote down vote up 
@Override
public int run(String[] args) throws Exception {
    Options cmdOptions = new Options();
    cmdOptions.addOption(new Option("register", false, "Register coprocessor"));
    cmdOptions.addOption(new Option("unregister", false, "Unregister coprocessor"));

    cmdOptions.addOption("table", true, "HBase table name, separated with comma, for example, table1,table2,..");
    cmdOptions.addOption("jar", true, "Coprocessor target jar path");
    cmdOptions.addOption("localJar", true, "Coprocessor local source jar path");
    cmdOptions.addOption("config", true, "Configuration file");

    cmdOptions.getOption("table").setType(String.class);
    cmdOptions.getOption("table").setRequired(true);
    cmdOptions.getOption("jar").setType(String.class);
    cmdOptions.getOption("jar").setRequired(false);
    cmdOptions.getOption("localJar").setType(String.class);
    cmdOptions.getOption("localJar").setRequired(false);
    cmdOptions.getOption("config").setType(String.class);
    cmdOptions.getOption("config").setRequired(false);

    GnuParser parser = new GnuParser();
    CommandLine cmdCli = parser.parse(cmdOptions, args);
    String tableName = cmdCli.getOptionValue("table");
    String configFile = cmdCli.getOptionValue("config");

    if (configFile != null) {
        Configuration.addDefaultResource(configFile);
    }

    if (cmdCli.hasOption("register")) {
        if (args.length < 3) {
            System.err.println("Error: coprocessor jar path is missing");
            System.err.println("Usage: java " + CoprocessorTool.class.getName() + " enable " + tableName + " [jarOnHdfs] [jarOnLocal]");
            return 1;
        }
        String jarPath = cmdCli.getOptionValue("jar");
        LOGGER.info("Table name: {}", tableName);
        LOGGER.info("Coprocessor jar on hdfs: {}", jarPath);
        String localJarPath = cmdCli.getOptionValue("localJar");
        LOGGER.info("Coprocessor jar on local: {}", localJarPath);

        String[] tableNames = tableName.split(",\\s*");
        for (String table : tableNames) {
            LOGGER.info("Registering coprocessor for table {}", table);
            registerCoprocessor(jarPath, table, localJarPath);
        }
    } else if (cmdCli.hasOption("unregister")) {
        unregisterCoprocessor(tableName);
    } else {
        System.err.println("command is required, --register/--unregister");
        printHelpMessage(cmdOptions);
    }
    return 0;
}
Example 13
Source File: TestReadOnlyConfiguration.java    From hbase with Apache License 2.0 4 votes vote down vote up 
@Test
public void testAddDefaultResource() {
  Configuration configuration = new Configuration();
  Configuration readOnlyConf = new ReadOnlyConfiguration(configuration);
  configuration.addDefaultResource("abc.xml");
}
Example 14
Source File: HadoopUtils.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up 
/**
 * Add "gobblin-site.xml" as a {@link Configuration} resource.
 */
public static void addGobblinSite() {
  Configuration.addDefaultResource("gobblin-site.xml");
}
Example 15
Source File: ClusterBalancerTool.java    From RDFS with Apache License 2.0 4 votes vote down vote up 
@Override
public void setConf(Configuration conf) {
  super.setConf(conf);
  Configuration.addDefaultResource(
          DynamicCloudsDaemon.CLUSTER_BALANCER_CONF_FILE);
}
Example 16
Source File: DynamicCloudsDaemon.java    From RDFS with Apache License 2.0 4 votes vote down vote up 
public void setConf(Configuration conf) {
  this.conf = conf;
  Configuration.addDefaultResource(CLUSTER_BALANCER_CONF_FILE);
}
Example 17
Source File: OzoneConfiguration.java    From hadoop-ozone with Apache License 2.0 4 votes vote down vote up 
public static void activate() {
  // adds the default resources
  Configuration.addDefaultResource("hdfs-default.xml");
  Configuration.addDefaultResource("hdfs-site.xml");
  Configuration.addDefaultResource("ozone-default.xml");
}