Java Code Examples for org.apache.hadoop.conf.Configuration#addDefaultResource()

The following examples show how to use org.apache.hadoop.conf.Configuration#addDefaultResource() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: hadoop   File: TestRefreshUserMappings.java    License: Apache License 2.0 6 votes vote down vote up
private void addNewConfigResource(String rsrcName, String keyGroup,
    String groups, String keyHosts, String hosts)
        throws FileNotFoundException, UnsupportedEncodingException {
  // location for temp resource should be in CLASSPATH
  Configuration conf = new Configuration();
  URL url = conf.getResource("hdfs-site.xml");

  String urlPath = URLDecoder.decode(url.getPath().toString(), "UTF-8");
  Path p = new Path(urlPath);
  Path dir = p.getParent();
  tempResource = dir.toString() + "/" + rsrcName;

  String newResource =
  "<configuration>"+
  "<property><name>" + keyGroup + "</name><value>"+groups+"</value></property>" +
  "<property><name>" + keyHosts + "</name><value>"+hosts+"</value></property>" +
  "</configuration>";
  PrintWriter writer = new PrintWriter(new FileOutputStream(tempResource));
  writer.println(newResource);
  writer.close();

  Configuration.addDefaultResource(rsrcName);
}
 
Example 2
Source Project: big-c   File: TestRefreshUserMappings.java    License: Apache License 2.0 6 votes vote down vote up
private void addNewConfigResource(String rsrcName, String keyGroup,
    String groups, String keyHosts, String hosts)
        throws FileNotFoundException, UnsupportedEncodingException {
  // location for temp resource should be in CLASSPATH
  Configuration conf = new Configuration();
  URL url = conf.getResource("hdfs-site.xml");

  String urlPath = URLDecoder.decode(url.getPath().toString(), "UTF-8");
  Path p = new Path(urlPath);
  Path dir = p.getParent();
  tempResource = dir.toString() + "/" + rsrcName;

  String newResource =
  "<configuration>"+
  "<property><name>" + keyGroup + "</name><value>"+groups+"</value></property>" +
  "<property><name>" + keyHosts + "</name><value>"+hosts+"</value></property>" +
  "</configuration>";
  PrintWriter writer = new PrintWriter(new FileOutputStream(tempResource));
  writer.println(newResource);
  writer.close();

  Configuration.addDefaultResource(rsrcName);
}
 
Example 3
Source Project: incubator-sentry   File: MiniDFS.java    License: Apache License 2.0 6 votes vote down vote up
MiniDFS(File baseDir, String serverType) throws Exception {
  Configuration conf = new Configuration();
  if (HiveServer2Type.InternalMetastore.name().equalsIgnoreCase(serverType)) {
    // set the test group mapping that maps user to a group of same name
    conf.set("hadoop.security.group.mapping",
        "org.apache.sentry.tests.e2e.hive.fs.MiniDFS$PseudoGroupMappingService");
    // set umask for metastore test client can create tables in the warehouse dir
    conf.set("fs.permissions.umask-mode", "000");
    Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(conf);
  }
  File dfsDir = assertCreateDir(new File(baseDir, "dfs"));
  conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dfsDir.getPath());
  conf.set("hadoop.security.group.mapping",
      MiniDFS.PseudoGroupMappingService.class.getName());
  Configuration.addDefaultResource("test.xml");
  dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
  fileSystem = dfsCluster.getFileSystem();
  String policyDir = System.getProperty("sentry.e2etest.hive.policy.location", "/user/hive/sentry");
  sentryDir = super.assertCreateDfsDir(new Path(fileSystem.getUri() + policyDir));
  dfsBaseDir = assertCreateDfsDir(new Path(new Path(fileSystem.getUri()), "/base"));
}
 
Example 4
/**
 * Start resource manager server
 */

@BeforeClass
public static void setUpResourceManager() throws IOException,
        InterruptedException {
  Configuration.addDefaultResource("config-with-security.xml");
  Configuration configuration = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    }
  };
  resourceManager.init(configuration);
  new Thread() {
    public void run() {
      resourceManager.start();
    }
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
          && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException("ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: "
          + configuration.get(YarnConfiguration.RM_ADMIN_ADDRESS));

  client = new ResourceManagerAdministrationProtocolPBClientImpl(1L,
          getProtocolAddress(configuration), configuration);

}
 
Example 5
Source Project: hadoop   File: ConfigUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Adds all the deprecated keys. Loads mapred-default.xml and mapred-site.xml
 */
public static void loadResources() {
  addDeprecatedKeys();
  Configuration.addDefaultResource("mapred-default.xml");
  Configuration.addDefaultResource("mapred-site.xml");
  Configuration.addDefaultResource("yarn-default.xml");
  Configuration.addDefaultResource("yarn-site.xml");
}
 
Example 6
/**
 * Start resource manager server
 */

@BeforeClass
public static void setUpResourceManager() throws IOException,
        InterruptedException {
  Configuration.addDefaultResource("config-with-security.xml");
  Configuration configuration = new YarnConfiguration();
  resourceManager = new ResourceManager() {
    @Override
    protected void doSecureLogin() throws IOException {
    }
  };
  resourceManager.init(configuration);
  new Thread() {
    public void run() {
      resourceManager.start();
    }
  }.start();
  int waitCount = 0;
  while (resourceManager.getServiceState() == STATE.INITED
          && waitCount++ < 10) {
    LOG.info("Waiting for RM to start...");
    Thread.sleep(1000);
  }
  if (resourceManager.getServiceState() != STATE.STARTED) {
    throw new IOException("ResourceManager failed to start. Final state is "
            + resourceManager.getServiceState());
  }
  LOG.info("ResourceManager RMAdmin address: "
          + configuration.get(YarnConfiguration.RM_ADMIN_ADDRESS));

  client = new ResourceManagerAdministrationProtocolPBClientImpl(1L,
          getProtocolAddress(configuration), configuration);

}
 
Example 7
Source Project: big-c   File: ConfigUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Adds all the deprecated keys. Loads mapred-default.xml and mapred-site.xml
 */
public static void loadResources() {
  addDeprecatedKeys();
  Configuration.addDefaultResource("mapred-default.xml");
  Configuration.addDefaultResource("mapred-site.xml");
  Configuration.addDefaultResource("yarn-default.xml");
  Configuration.addDefaultResource("yarn-site.xml");
}
 
Example 8
Source Project: RDFS   File: ClusterBalancerTool.java    License: Apache License 2.0 5 votes vote down vote up
private ClusterBalancerAdminProtocol getClusterBalancerAdmin()
        throws IOException {
  Configuration.addDefaultResource(
          DynamicCloudsDaemon.CLUSTER_BALANCER_CONF_FILE);
  Configuration conf = this.getConf();
  String addr = conf.get(DynamicCloudsDaemon.CLUSTER_BALANCER_ADDR);

  InetSocketAddress isAddr = NetUtils.createSocketAddr(addr);

  return (ClusterBalancerAdminProtocol) RPC.getProxy(
          ClusterBalancerAdminProtocol.class,
          ClusterBalancerAdminProtocol.versionID, isAddr, conf);
}
 
Example 9
Source Project: hadoop-ozone   File: OzoneConfiguration.java    License: Apache License 2.0 4 votes vote down vote up
public static void activate() {
  // adds the default resources
  Configuration.addDefaultResource("hdfs-default.xml");
  Configuration.addDefaultResource("hdfs-site.xml");
  Configuration.addDefaultResource("ozone-default.xml");
}
 
Example 10
Source Project: hadoop   File: TestMalformedURLs.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {
  Configuration.addDefaultResource("hdfs-site.malformed.xml");
  config = new Configuration();
}
 
Example 11
Source Project: big-c   File: TestMalformedURLs.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {
  Configuration.addDefaultResource("hdfs-site.malformed.xml");
  config = new Configuration();
}
 
Example 12
public static void addDefaultResource(String name) {
    Configuration.addDefaultResource(name);
}
 
Example 13
Source Project: eagle   File: CoprocessorTool.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public int run(String[] args) throws Exception {
    Options cmdOptions = new Options();
    cmdOptions.addOption(new Option("register", false, "Register coprocessor"));
    cmdOptions.addOption(new Option("unregister", false, "Unregister coprocessor"));

    cmdOptions.addOption("table", true, "HBase table name, separated with comma, for example, table1,table2,..");
    cmdOptions.addOption("jar", true, "Coprocessor target jar path");
    cmdOptions.addOption("localJar", true, "Coprocessor local source jar path");
    cmdOptions.addOption("config", true, "Configuration file");

    cmdOptions.getOption("table").setType(String.class);
    cmdOptions.getOption("table").setRequired(true);
    cmdOptions.getOption("jar").setType(String.class);
    cmdOptions.getOption("jar").setRequired(false);
    cmdOptions.getOption("localJar").setType(String.class);
    cmdOptions.getOption("localJar").setRequired(false);
    cmdOptions.getOption("config").setType(String.class);
    cmdOptions.getOption("config").setRequired(false);

    GnuParser parser = new GnuParser();
    CommandLine cmdCli = parser.parse(cmdOptions, args);
    String tableName = cmdCli.getOptionValue("table");
    String configFile = cmdCli.getOptionValue("config");

    if (configFile != null) {
        Configuration.addDefaultResource(configFile);
    }

    if (cmdCli.hasOption("register")) {
        if (args.length < 3) {
            System.err.println("Error: coprocessor jar path is missing");
            System.err.println("Usage: java " + CoprocessorTool.class.getName() + " enable " + tableName + " [jarOnHdfs] [jarOnLocal]");
            return 1;
        }
        String jarPath = cmdCli.getOptionValue("jar");
        LOGGER.info("Table name: {}", tableName);
        LOGGER.info("Coprocessor jar on hdfs: {}", jarPath);
        String localJarPath = cmdCli.getOptionValue("localJar");
        LOGGER.info("Coprocessor jar on local: {}", localJarPath);

        String[] tableNames = tableName.split(",\\s*");
        for (String table : tableNames) {
            LOGGER.info("Registering coprocessor for table {}", table);
            registerCoprocessor(jarPath, table, localJarPath);
        }
    } else if (cmdCli.hasOption("unregister")) {
        unregisterCoprocessor(tableName);
    } else {
        System.err.println("command is required, --register/--unregister");
        printHelpMessage(cmdOptions);
    }
    return 0;
}
 
Example 14
Source Project: hbase   File: TestReadOnlyConfiguration.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testAddDefaultResource() {
  Configuration configuration = new Configuration();
  Configuration readOnlyConf = new ReadOnlyConfiguration(configuration);
  configuration.addDefaultResource("abc.xml");
}
 
Example 15
Source Project: incubator-gobblin   File: HadoopUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Add "gobblin-site.xml" as a {@link Configuration} resource.
 */
public static void addGobblinSite() {
  Configuration.addDefaultResource("gobblin-site.xml");
}
 
Example 16
Source Project: RDFS   File: ClusterBalancerTool.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void setConf(Configuration conf) {
  super.setConf(conf);
  Configuration.addDefaultResource(
          DynamicCloudsDaemon.CLUSTER_BALANCER_CONF_FILE);
}
 
Example 17
Source Project: RDFS   File: DynamicCloudsDaemon.java    License: Apache License 2.0 4 votes vote down vote up
public void setConf(Configuration conf) {
  this.conf = conf;
  Configuration.addDefaultResource(CLUSTER_BALANCER_CONF_FILE);
}