Java Code Examples for org.apache.hadoop.conf.Configuration#setEnum()

The following examples show how to use org.apache.hadoop.conf.Configuration#setEnum() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: examples   File: ApplicationTest.java    License: Apache License 2.0 6 votes vote down vote up
private Configuration getConfig() {
    Configuration conf = new Configuration(false);
    String pre = "dt.operator.kafkaIn.prop.";
    conf.setEnum(pre + "initialOffset",
                 AbstractKafkaInputOperator.InitialOffset.EARLIEST);
    conf.setInt(pre + "initialPartitionCount", 1);
    conf.set(   pre + "topics",                TOPIC);
    conf.set(   pre + "clusters",              BROKER);

    pre = "dt.operator.fileOut.prop.";
    conf.set(   pre + "filePath",        FILE_DIR);
    conf.set(   pre + "baseName",        FILE_NAME);
    conf.setInt(pre + "maxLength",       40);
    conf.setInt(pre + "rotationWindows", 3);

    return conf;
}
 
Example 2
private Configuration getConfig()
{
  Configuration conf = new Configuration(false);
  String pre = "dt.operator.kafkaIn.prop.";
  conf.setEnum(pre + "initialOffset", AbstractKafkaInputOperator.InitialOffset.EARLIEST);
  conf.setInt(pre + "initialPartitionCount", 1);
  conf.set(pre + "topics", TOPIC);
  conf.set(pre + "clusters", BROKER);

  pre = "dt.operator.fileOut.prop.";
  conf.set(pre + "filePath", FILE_DIR);
  conf.set(pre + "baseName", FILE_NAME);
  conf.setInt(pre + "maxLength", 40);
  conf.setInt(pre + "rotationWindows", 3);

  return conf;
}
 
Example 3
Source Project: tez   File: TestHistoryEventHandler.java    License: Apache License 2.0 6 votes vote down vote up
private HistoryEventHandler createHandler(HistoryLogLevel logLevel) {
  Configuration conf = new Configuration(baseConfig);
  conf.setBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, false);
  conf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
      InMemoryHistoryLoggingService.class.getName());
  if (logLevel != null) {
    conf.setEnum(TezConfiguration.TEZ_HISTORY_LOGGING_LOGLEVEL, logLevel);
  }

  DAG dag = mock(DAG.class);
  when(dag.getConf()).thenReturn(conf);

  AppContext appContext = mock(AppContext.class);
  when(appContext.getApplicationID()).thenReturn(appId);
  when(appContext.getHadoopShim()).thenReturn(new HadoopShim() {});
  when(appContext.getAMConf()).thenReturn(conf);
  when(appContext.getCurrentDAG()).thenReturn(dag);

  HistoryEventHandler handler =  new HistoryEventHandler(appContext);
  handler.init(conf);

  return handler;
}
 
Example 4
Source Project: iceberg   File: IcebergInputFormat.java    License: Apache License 2.0 5 votes vote down vote up
public ConfigBuilder(Configuration conf) {
  this.conf = conf;
  // defaults
  conf.setEnum(IN_MEMORY_DATA_MODEL, InMemoryDataModel.GENERIC);
  conf.setBoolean(SKIP_RESIDUAL_FILTERING, false);
  conf.setBoolean(CASE_SENSITIVE, true);
  conf.setBoolean(REUSE_CONTAINERS, false);
  conf.setBoolean(LOCALITY, false);
}
 
Example 5
@Test
public void hflush_syncsEverything() throws Exception {
  URI path = gcsFsIHelper.getUniqueObjectUri("hflush_syncsEverything");
  Path hadoopPath = new Path(path);

  Configuration config = getTestConfig();
  config.setEnum(GCS_OUTPUT_STREAM_TYPE.getKey(), OutputStreamType.FLUSHABLE_COMPOSITE);
  FileSystem ghfs = GoogleHadoopFileSystemIntegrationHelper.createGhfs(path, config);

  byte[] testData = new byte[5];
  new Random().nextBytes(testData);

  try (FSDataOutputStream out = ghfs.create(hadoopPath)) {
    for (int i = 0; i < testData.length; i++) {
      out.write(testData[i]);
      out.hflush();

      // Validate partly composed data always just contain the first byte because only the
      // first hflush() succeeds and all subsequent hflush() calls should be rate limited.
      int composedLength = i + 1;
      assertThat(ghfs.getFileStatus(hadoopPath).getLen()).isEqualTo(composedLength);
      assertThat(gcsFsIHelper.readFile(path)).isEqualTo(Arrays.copyOf(testData, composedLength));
    }
  }

  // Assert that data was fully written after close
  assertThat(ghfs.getFileStatus(hadoopPath).getLen()).isEqualTo(testData.length);
  assertThat(gcsFsIHelper.readFile(path)).isEqualTo(testData);
}
 
Example 6
@Test
public void hflush_rateLimited_writesEverything() throws Exception {
  URI path = gcsFsIHelper.getUniqueObjectUri("hflush_syncsEverything");
  Path hadoopPath = new Path(path);

  Configuration config = getTestConfig();
  config.setEnum(GCS_OUTPUT_STREAM_TYPE.getKey(), OutputStreamType.FLUSHABLE_COMPOSITE);
  config.setLong(GCS_OUTPUT_STREAM_SYNC_MIN_INTERVAL_MS.getKey(), Duration.ofDays(1).toMillis());
  FileSystem ghfs = GoogleHadoopFileSystemIntegrationHelper.createGhfs(path, config);

  byte[] testData = new byte[10];
  new Random().nextBytes(testData);

  try (FSDataOutputStream out = ghfs.create(hadoopPath)) {
    for (byte testDataByte : testData) {
      out.write(testDataByte);
      out.hflush();

      // Validate partly composed data always just contain the first byte because only the
      // first hflush() succeeds and all subsequent hflush() calls should be rate limited.
      assertThat(ghfs.getFileStatus(hadoopPath).getLen()).isEqualTo(1);
      assertThat(gcsFsIHelper.readFile(path)).isEqualTo(new byte[] {testData[0]});
    }
  }

  // Assert that data was fully written after close
  assertThat(ghfs.getFileStatus(hadoopPath).getLen()).isEqualTo(testData.length);
  assertThat(gcsFsIHelper.readFile(path)).isEqualTo(testData);
}
 
Example 7
Source Project: Bats   File: StramAppLauncher.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Submit application to the cluster and return the app id.
 * Sets the context class loader for application dependencies.
 *
 * @param appConfig
 * @return ApplicationId
 * @throws Exception
 */
public ApplicationId launchApp(AppFactory appConfig) throws Exception
{
  loadDependencies();
  Configuration conf = propertiesBuilder.conf;
  conf.setEnum(StreamingApplication.ENVIRONMENT, StreamingApplication.Environment.CLUSTER);
  LogicalPlan dag = appConfig.createApp(propertiesBuilder);
  if (UserGroupInformation.isSecurityEnabled()) {
    long hdfsTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_MAX_LIFE_TIME, conf.getLong(StramClientUtils.HDFS_TOKEN_MAX_LIFE_TIME, StramClientUtils.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
    dag.setAttribute(LogicalPlan.HDFS_TOKEN_LIFE_TIME, hdfsTokenMaxLifeTime);
    long rmTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_RM_TOKEN_MAX_LIFE_TIME, conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY, YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
    dag.setAttribute(LogicalPlan.RM_TOKEN_LIFE_TIME, rmTokenMaxLifeTime);
    setTokenRefreshCredentials(dag, conf);
  }
  String tokenRefreshFactor = conf.get(StramClientUtils.TOKEN_ANTICIPATORY_REFRESH_FACTOR);
  if (tokenRefreshFactor != null && tokenRefreshFactor.trim().length() > 0) {
    dag.setAttribute(LogicalPlan.TOKEN_REFRESH_ANTICIPATORY_FACTOR, Double.parseDouble(tokenRefreshFactor));
  }
  StramClient client = new StramClient(conf, dag);
  try {
    client.start();
    LinkedHashSet<String> libjars = Sets.newLinkedHashSet();
    String libjarsCsv = conf.get(LIBJARS_CONF_KEY_NAME);
    if (libjarsCsv != null) {
      String[] jars = StringUtils.splitByWholeSeparator(libjarsCsv, StramClient.LIB_JARS_SEP);
      libjars.addAll(Arrays.asList(jars));
    }
    if (deployJars != null) {
      for (File deployJar : deployJars) {
        libjars.add(deployJar.getAbsolutePath());
      }
    }

    client.setResources(libjars);
    client.setFiles(conf.get(FILES_CONF_KEY_NAME));
    client.setArchives(conf.get(ARCHIVES_CONF_KEY_NAME));
    client.setOriginalAppId(conf.get(ORIGINAL_APP_ID));
    client.setQueueName(conf.get(QUEUE_NAME));
    String tags = conf.get(TAGS);
    if (tags != null) {
      for (String tag : tags.split(",")) {
        client.addTag(tag.trim());
      }
    }
    client.startApplication();
    return client.getApplicationReport().getApplicationId();
  } finally {
    client.stop();
  }
}
 
Example 8
Source Project: hadoop   File: CommonJobTest.java    License: Apache License 2.0 4 votes vote down vote up
protected void doSubmission(String jobCreatorName, boolean defaultOutputPath)
        throws Exception {
  final Path in = new Path("foo").makeQualified(
          GridmixTestUtils.dfs.getUri(),
          GridmixTestUtils.dfs.getWorkingDirectory());
  final Path out = GridmixTestUtils.DEST.makeQualified(
          GridmixTestUtils.dfs.getUri(),
          GridmixTestUtils.dfs.getWorkingDirectory());
  final Path root = new Path(workspace.getName()).makeQualified(
      GridmixTestUtils.dfs.getUri(), GridmixTestUtils.dfs.getWorkingDirectory());
  if (!workspace.exists()) {
    assertTrue(workspace.mkdirs());
  }
  Configuration conf = null;

  try {
    ArrayList<String> argsList = new ArrayList<String>();

    argsList.add("-D" + FilePool.GRIDMIX_MIN_FILE + "=0");
    argsList.add("-D" + Gridmix.GRIDMIX_USR_RSV + "="
            + EchoUserResolver.class.getName());
    if (jobCreatorName != null) {
      argsList.add("-D" + JobCreator.GRIDMIX_JOB_TYPE + "=" + jobCreatorName);
    }

    // Set the config property gridmix.output.directory only if
    // defaultOutputPath is false. If defaultOutputPath is true, then
    // let us allow gridmix to use the path foo/gridmix/ as output dir.
    if (!defaultOutputPath) {
      argsList.add("-D" + Gridmix.GRIDMIX_OUT_DIR + "=" + out);
    }
    argsList.add("-generate");
    argsList.add(String.valueOf(GENDATA) + "m");
    argsList.add(in.toString());
    argsList.add("-"); // ignored by DebugGridmix

    String[] argv = argsList.toArray(new String[argsList.size()]);

    DebugGridmix client = new DebugGridmix();
    conf = GridmixTestUtils.mrvl.getConfig();

    CompressionEmulationUtil.setCompressionEmulationEnabled(conf, true);
    conf.setEnum(GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, policy);

    conf.setBoolean(GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE, true);
    UserGroupInformation ugi = UserGroupInformation.getLoginUser();
    conf.set(MRJobConfig.USER_NAME, ugi.getUserName());

    // allow synthetic users to create home directories
    GridmixTestUtils.dfs.mkdirs(root, new FsPermission((short) 777));
    GridmixTestUtils.dfs.setPermission(root, new FsPermission((short) 777));

    int res = ToolRunner.run(conf, client, argv);
    assertEquals("Client exited with nonzero status", 0, res);
    client.checkMonitor();
  } catch (Exception e) {
    e.printStackTrace();
  } finally {
    in.getFileSystem(conf).delete(in, true);
    out.getFileSystem(conf).delete(out, true);
    root.getFileSystem(conf).delete(root, true);
  }
}
 
Example 9
Source Project: hadoop   File: ZlibFactory.java    License: Apache License 2.0 4 votes vote down vote up
public static void setCompressionStrategy(Configuration conf,
    CompressionStrategy strategy) {
  conf.setEnum("zlib.compress.strategy", strategy);
}
 
Example 10
Source Project: hadoop   File: ZlibFactory.java    License: Apache License 2.0 4 votes vote down vote up
public static void setCompressionLevel(Configuration conf,
    CompressionLevel level) {
  conf.setEnum("zlib.compress.level", level);
}
 
Example 11
Source Project: big-c   File: CommonJobTest.java    License: Apache License 2.0 4 votes vote down vote up
protected void doSubmission(String jobCreatorName, boolean defaultOutputPath)
        throws Exception {
  final Path in = new Path("foo").makeQualified(
          GridmixTestUtils.dfs.getUri(),
          GridmixTestUtils.dfs.getWorkingDirectory());
  final Path out = GridmixTestUtils.DEST.makeQualified(
          GridmixTestUtils.dfs.getUri(),
          GridmixTestUtils.dfs.getWorkingDirectory());
  final Path root = new Path(workspace.getName()).makeQualified(
      GridmixTestUtils.dfs.getUri(), GridmixTestUtils.dfs.getWorkingDirectory());
  if (!workspace.exists()) {
    assertTrue(workspace.mkdirs());
  }
  Configuration conf = null;

  try {
    ArrayList<String> argsList = new ArrayList<String>();

    argsList.add("-D" + FilePool.GRIDMIX_MIN_FILE + "=0");
    argsList.add("-D" + Gridmix.GRIDMIX_USR_RSV + "="
            + EchoUserResolver.class.getName());
    if (jobCreatorName != null) {
      argsList.add("-D" + JobCreator.GRIDMIX_JOB_TYPE + "=" + jobCreatorName);
    }

    // Set the config property gridmix.output.directory only if
    // defaultOutputPath is false. If defaultOutputPath is true, then
    // let us allow gridmix to use the path foo/gridmix/ as output dir.
    if (!defaultOutputPath) {
      argsList.add("-D" + Gridmix.GRIDMIX_OUT_DIR + "=" + out);
    }
    argsList.add("-generate");
    argsList.add(String.valueOf(GENDATA) + "m");
    argsList.add(in.toString());
    argsList.add("-"); // ignored by DebugGridmix

    String[] argv = argsList.toArray(new String[argsList.size()]);

    DebugGridmix client = new DebugGridmix();
    conf = GridmixTestUtils.mrvl.getConfig();

    CompressionEmulationUtil.setCompressionEmulationEnabled(conf, true);
    conf.setEnum(GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, policy);

    conf.setBoolean(GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE, true);
    UserGroupInformation ugi = UserGroupInformation.getLoginUser();
    conf.set(MRJobConfig.USER_NAME, ugi.getUserName());

    // allow synthetic users to create home directories
    GridmixTestUtils.dfs.mkdirs(root, new FsPermission((short) 777));
    GridmixTestUtils.dfs.setPermission(root, new FsPermission((short) 777));

    int res = ToolRunner.run(conf, client, argv);
    assertEquals("Client exited with nonzero status", 0, res);
    client.checkMonitor();
  } catch (Exception e) {
    e.printStackTrace();
  } finally {
    in.getFileSystem(conf).delete(in, true);
    out.getFileSystem(conf).delete(out, true);
    root.getFileSystem(conf).delete(root, true);
  }
}
 
Example 12
Source Project: big-c   File: ZlibFactory.java    License: Apache License 2.0 4 votes vote down vote up
public static void setCompressionStrategy(Configuration conf,
    CompressionStrategy strategy) {
  conf.setEnum("zlib.compress.strategy", strategy);
}
 
Example 13
Source Project: big-c   File: ZlibFactory.java    License: Apache License 2.0 4 votes vote down vote up
public static void setCompressionLevel(Configuration conf,
    CompressionLevel level) {
  conf.setEnum("zlib.compress.level", level);
}
 
Example 14
/**
 * Submit application to the cluster and return the app id.
 * Sets the context class loader for application dependencies.
 *
 * @param appConfig
 * @return ApplicationId
 * @throws Exception
 */
public ApplicationId launchApp(AppFactory appConfig) throws Exception
{
  loadDependencies();
  Configuration conf = propertiesBuilder.conf;
  conf.setEnum(StreamingApplication.ENVIRONMENT, StreamingApplication.Environment.CLUSTER);
  LogicalPlan dag = appConfig.createApp(propertiesBuilder);
  if (UserGroupInformation.isSecurityEnabled()) {
    long hdfsTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_MAX_LIFE_TIME, conf.getLong(StramClientUtils.HDFS_TOKEN_MAX_LIFE_TIME, StramClientUtils.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
    dag.setAttribute(LogicalPlan.HDFS_TOKEN_LIFE_TIME, hdfsTokenMaxLifeTime);
    LOG.debug("HDFS token life time {}", hdfsTokenMaxLifeTime);
    long hdfsTokenRenewInterval = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_RENEW_INTERVAL, conf.getLong(StramClientUtils.HDFS_TOKEN_RENEW_INTERVAL, StramClientUtils.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT));
    dag.setAttribute(LogicalPlan.HDFS_TOKEN_RENEWAL_INTERVAL, hdfsTokenRenewInterval);
    LOG.debug("HDFS token renew interval {}", hdfsTokenRenewInterval);
    long rmTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_RM_TOKEN_MAX_LIFE_TIME, conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY, YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
    dag.setAttribute(LogicalPlan.RM_TOKEN_LIFE_TIME, rmTokenMaxLifeTime);
    LOG.debug("RM token life time {}", rmTokenMaxLifeTime);
    long rmTokenRenewInterval = conf.getLong(StramClientUtils.DT_RM_TOKEN_RENEW_INTERVAL, conf.getLong(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_KEY, YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT));
    dag.setAttribute(LogicalPlan.RM_TOKEN_RENEWAL_INTERVAL, rmTokenRenewInterval);
    LOG.debug("RM token renew interval {}", rmTokenRenewInterval);
    setTokenRefreshCredentials(dag, conf);
  }
  String tokenRefreshFactor = conf.get(StramClientUtils.TOKEN_ANTICIPATORY_REFRESH_FACTOR);
  if (tokenRefreshFactor != null && tokenRefreshFactor.trim().length() > 0) {
    double refreshFactor = Double.parseDouble(tokenRefreshFactor);
    dag.setAttribute(LogicalPlan.TOKEN_REFRESH_ANTICIPATORY_FACTOR, refreshFactor);
    LOG.debug("Token refresh anticipatory factor {}", refreshFactor);
  }
  StramClient client = new StramClient(conf, dag);
  try {
    client.start();
    LinkedHashSet<String> libjars = Sets.newLinkedHashSet();
    String libjarsCsv = conf.get(LIBJARS_CONF_KEY_NAME);
    if (libjarsCsv != null) {
      String[] jars = StringUtils.splitByWholeSeparator(libjarsCsv, StramClient.LIB_JARS_SEP);
      libjars.addAll(Arrays.asList(jars));
    }
    if (deployJars != null) {
      for (File deployJar : deployJars) {
        libjars.add(deployJar.getAbsolutePath());
      }
    }

    client.setResources(libjars);
    client.setFiles(conf.get(FILES_CONF_KEY_NAME));
    client.setArchives(conf.get(ARCHIVES_CONF_KEY_NAME));
    client.setOriginalAppId(conf.get(ORIGINAL_APP_ID));
    client.setQueueName(conf.get(QUEUE_NAME));
    String tags = conf.get(TAGS);
    if (tags != null) {
      for (String tag : tags.split(",")) {
        client.addTag(tag.trim());
      }
    }
    client.startApplication();
    return client.getApplicationReport().getApplicationId();
  } finally {
    client.stop();
  }
}
 
Example 15
private Configuration getTestConfig() {
  Configuration conf = GoogleHadoopFileSystemIntegrationHelper.getTestConfig();
  conf.setEnum(GCS_OUTPUT_STREAM_TYPE.getKey(), outputStreamType);
  conf.setEnum(GCS_OUTPUT_STREAM_PIPE_TYPE.getKey(), pipeType);
  return conf;
}
 
Example 16
private static Configuration getTestConfig() {
  Configuration conf = GoogleHadoopFileSystemIntegrationHelper.getTestConfig();
  conf.setEnum(GCS_OUTPUT_STREAM_TYPE.getKey(), OutputStreamType.SYNCABLE_COMPOSITE);
  return conf;
}