Java Code Examples for org.apache.hadoop.util.StringUtils#join()

The following examples show how to use org.apache.hadoop.util.StringUtils#join() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestFileSystemAccessService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
@TestException(exception = ServiceException.class, msgRegExp = "H01.*")
@TestDir
public void noKerberosKeytabProperty() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
  Arrays.asList(InstrumentationService.class.getName(),
                SchedulerService.class.getName(),
                FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);
  conf.set("server.hadoop.authentication.type", "kerberos");
  conf.set("server.hadoop.authentication.kerberos.keytab", " ");
  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
}
 
Example 2
Source File: TestFileSystemAccessService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
@TestException(exception = FileSystemAccessException.class, msgRegExp = "H05.*")
@TestDir
public void NameNodeNotinWhitelists() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
    Arrays.asList(InstrumentationService.class.getName(),
                  SchedulerService.class.getName(),
                  FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);
  conf.set("server.hadoop.name.node.whitelist", "NN");
  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
  FileSystemAccessService fsAccess = (FileSystemAccessService) server.get(FileSystemAccess.class);
  fsAccess.validateNamenode("NNx");
}
 
Example 3
Source File: TestFileSystemAccessService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
@TestDir
public void serviceHadoopConf() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
    Arrays.asList(InstrumentationService.class.getName(),
                  SchedulerService.class.getName(),
                  FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);

  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
  FileSystemAccessService fsAccess = (FileSystemAccessService) server.get(FileSystemAccess.class);
  Assert.assertEquals(fsAccess.serviceHadoopConf.get("foo"), "FOO");
  server.destroy();
}
 
Example 4
Source File: HttpFSFileSystem.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Concat existing files together.
 * @param f the path to the target destination.
 * @param psrcs the paths to the sources to use for the concatenation.
 *
 * @throws IOException
 */
@Override
public void concat(Path f, Path[] psrcs) throws IOException {
  List<String> strPaths = new ArrayList<String>(psrcs.length);
  for(Path psrc : psrcs) {
    strPaths.add(psrc.toUri().getPath());
  }
  String srcs = StringUtils.join(",", strPaths);

  Map<String, String> params = new HashMap<String, String>();
  params.put(OP_PARAM, Operation.CONCAT.toString());
  params.put(SOURCES_PARAM, srcs);
  HttpURLConnection conn = getConnection(Operation.CONCAT.getMethod(),
      params, f, true);
  HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
}
 
Example 5
Source File: TestShuffleHandler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static void createShuffleHandlerFiles(File logDir, String user,
    String appId, String appAttemptId, Configuration conf,
    List<File> fileMap) throws IOException {
  String attemptDir =
      StringUtils.join(Path.SEPARATOR,
          Arrays.asList(new String[] { logDir.getAbsolutePath(),
              ContainerLocalizer.USERCACHE, user,
              ContainerLocalizer.APPCACHE, appId, "output", appAttemptId }));
  File appAttemptDir = new File(attemptDir);
  appAttemptDir.mkdirs();
  System.out.println(appAttemptDir.getAbsolutePath());
  File indexFile = new File(appAttemptDir, "file.out.index");
  fileMap.add(indexFile);
  createIndexFile(indexFile, conf);
  File mapOutputFile = new File(appAttemptDir, "file.out");
  fileMap.add(mapOutputFile);
  createMapOutputFile(mapOutputFile, conf);
}
 
Example 6
Source File: TestMRApps.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 120000)
public void testSetClasspathWithJobClassloader() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
  Map<String, String> env = new HashMap<String, String>();
  MRApps.setClasspath(env, conf);
  String cp = env.get("CLASSPATH");
  String appCp = env.get("APP_CLASSPATH");
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
    + " classpath!", cp.contains("jar" + ApplicationConstants.CLASS_PATH_SEPARATOR + "job"));
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
    cp.contains("PWD"));
  String expectedAppClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
      "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
    + " classpath!", expectedAppClasspath, appCp);
}
 
Example 7
Source File: HttpFSFileSystem.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Concat existing files together.
 * @param f the path to the target destination.
 * @param psrcs the paths to the sources to use for the concatenation.
 *
 * @throws IOException
 */
@Override
public void concat(Path f, Path[] psrcs) throws IOException {
  List<String> strPaths = new ArrayList<String>(psrcs.length);
  for(Path psrc : psrcs) {
    strPaths.add(psrc.toUri().getPath());
  }
  String srcs = StringUtils.join(",", strPaths);

  Map<String, String> params = new HashMap<String, String>();
  params.put(OP_PARAM, Operation.CONCAT.toString());
  params.put(SOURCES_PARAM, srcs);
  HttpURLConnection conn = getConnection(Operation.CONCAT.getMethod(),
      params, f, true);
  HttpExceptionUtils.validateResponse(conn, HttpURLConnection.HTTP_OK);
}
 
Example 8
Source File: TestFileSystemAccessService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
@TestException(exception = ServiceException.class, msgRegExp = "H02.*")
@TestDir
public void kerberosInitializationFailure() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
    Arrays.asList(InstrumentationService.class.getName(),
                  SchedulerService.class.getName(),
                  FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);
  conf.set("server.hadoop.authentication.type", "kerberos");
  conf.set("server.hadoop.authentication.kerberos.keytab", "/tmp/foo");
  conf.set("server.hadoop.authentication.kerberos.principal", "foo@FOO");
  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
}
 
Example 9
Source File: TestFileSystemAccessService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
@TestDir
public void serviceHadoopConf() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
    Arrays.asList(InstrumentationService.class.getName(),
                  SchedulerService.class.getName(),
                  FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);

  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
  FileSystemAccessService fsAccess = (FileSystemAccessService) server.get(FileSystemAccess.class);
  Assert.assertEquals(fsAccess.serviceHadoopConf.get("foo"), "FOO");
  server.destroy();
}
 
Example 10
Source File: TestInstrumentationService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
@TestDir
@SuppressWarnings("unchecked")
public void sampling() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",", Arrays.asList(InstrumentationService.class.getName(),
                                                        SchedulerService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);
  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
  Instrumentation instrumentation = server.get(Instrumentation.class);

  final AtomicInteger count = new AtomicInteger();

  Instrumentation.Variable<Long> varToSample = new Instrumentation.Variable<Long>() {
    @Override
    public Long getValue() {
      return (long) count.incrementAndGet();
    }
  };
  instrumentation.addSampler("g", "s", 10, varToSample);

  sleep(2000);
  int i = count.get();
  assertTrue(i > 0);

  Map<String, Map<String, ?>> snapshot = instrumentation.getSnapshot();
  Map<String, Map<String, Object>> samplers = (Map<String, Map<String, Object>>) snapshot.get("samplers");
  InstrumentationService.Sampler sampler = (InstrumentationService.Sampler) samplers.get("g").get("s");
  assertTrue(sampler.getRate() > 0);

  server.destroy();
}
 
Example 11
Source File: TestFileSystemAccessService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
@TestException(exception = ServiceException.class, msgRegExp = "H09.*")
@TestDir
public void invalidSecurity() throws Exception {
  String dir = TestDirHelper.getTestDir().getAbsolutePath();
  String services = StringUtils.join(",",
    Arrays.asList(InstrumentationService.class.getName(),
                  SchedulerService.class.getName(),
                  FileSystemAccessService.class.getName()));
  Configuration conf = new Configuration(false);
  conf.set("server.services", services);
  conf.set("server.hadoop.authentication.type", "foo");
  Server server = new Server("server", dir, dir, dir, dir, conf);
  server.init();
}
 
Example 12
Source File: VertexImpl.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
void logJobHistoryVertexFailedEvent(VertexState state) throws IOException {
  VertexFinishedEvent finishEvt = new VertexFinishedEvent(vertexId,
      vertexName, initTimeRequested, initedTime, startTimeRequested,
      startedTime, clock.getTime(), state, StringUtils.join(LINE_SEPARATOR,
          getDiagnostics()), getAllCounters(), getVertexStats());
  this.appContext.getHistoryHandler().handleCriticalEvent(
      new DAGHistoryEvent(getDAGId(), finishEvt));
}
 
Example 13
Source File: TestMRApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 3000000)
public void testSetClasspathWithFramework() throws IOException {
  final String FRAMEWORK_NAME = "some-framework-name";
  final String FRAMEWORK_PATH = "some-framework-path#" + FRAMEWORK_NAME;
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.set(MRJobConfig.MAPREDUCE_APPLICATION_FRAMEWORK_PATH, FRAMEWORK_PATH);
  Map<String, String> env = new HashMap<String, String>();
  try {
    MRApps.setClasspath(env, conf);
    fail("Failed to catch framework path set without classpath change");
  } catch (IllegalArgumentException e) {
    assertTrue("Unexpected IllegalArgumentException",
        e.getMessage().contains("Could not locate MapReduce framework name '"
            + FRAMEWORK_NAME + "'"));
  }

  env.clear();
  final String FRAMEWORK_CLASSPATH = FRAMEWORK_NAME + "/*.jar";
  conf.set(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, FRAMEWORK_CLASSPATH);
  MRApps.setClasspath(env, conf);
  final String stdClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
          ApplicationConstants.Environment.PWD.$$() + "/*"));
  String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList(ApplicationConstants.Environment.PWD.$$(),
          FRAMEWORK_CLASSPATH, stdClasspath));
  assertEquals("Incorrect classpath with framework and no user precedence",
      expectedClasspath, env.get("CLASSPATH"));

  env.clear();
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
  MRApps.setClasspath(env, conf);
  expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      Arrays.asList(ApplicationConstants.Environment.PWD.$$(),
          stdClasspath, FRAMEWORK_CLASSPATH));
  assertEquals("Incorrect classpath with framework and user precedence",
      expectedClasspath, env.get("CLASSPATH"));
}
 
Example 14
Source File: TestParam.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testConcatSourcesParam() {
  final String[] strings = {"/", "/foo", "/bar"};
  for(int n = 0; n < strings.length; n++) {
    final String[] sub = new String[n]; 
    final Path[] paths = new Path[n];
    for(int i = 0; i < paths.length; i++) {
      paths[i] = new Path(sub[i] = strings[i]);
    }

    final String expected = StringUtils.join(",", Arrays.asList(sub));
    final ConcatSourcesParam computed = new ConcatSourcesParam(paths);
    Assert.assertEquals(expected, computed.getValue());
  }
}
 
Example 15
Source File: CapacitySchedulerConfiguration.java    From big-c with Apache License 2.0 5 votes vote down vote up
public void setAccessibleNodeLabels(String queue, Set<String> labels) {
  if (labels == null) {
    return;
  }
  String str = StringUtils.join(",", labels);
  set(getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS, str);
}
 
Example 16
Source File: SqoopHCatUtilities.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 5 votes vote down vote up
public void launchHCatCli(String cmdLine)
  throws IOException {
  String tmpFileName = null;


  String tmpDir = System.getProperty("java.io.tmpdir");
  if (options != null) {
    tmpDir = options.getTempDir();
  }
  tmpFileName =
    new File(tmpDir, "hcat-script-"
      + System.currentTimeMillis()).getAbsolutePath();

  writeHCatScriptFile(tmpFileName, cmdLine);
  // Create the argv for the HCatalog Cli Driver.
  String[] argArray = new String[2];
  argArray[0] = "-f";
  argArray[1] = tmpFileName;
  String argLine = StringUtils.join(",", Arrays.asList(argArray));

  if (testMode) {
    LOG.debug("Executing HCatalog CLI in-process with " + argLine);
    executeHCatProgramInProcess(argArray);
  } else {
    LOG.info("Executing external HCatalog CLI process with args :" + argLine);
    executeExternalHCatProgram(Executor.getCurEnvpStrings(), argArray);
  }
}
 
Example 17
Source File: MonitoredTaskImpl.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public String prettyPrintJournal() {
  return StringUtils.join("\n\t", getStatusJournal());
}
 
Example 18
Source File: ResourceLocalizationService.java    From big-c with Apache License 2.0 4 votes vote down vote up
private String getUserFileCachePath(String user) {
  return StringUtils.join(Path.SEPARATOR, Arrays.asList(".",
    ContainerLocalizer.USERCACHE, user, ContainerLocalizer.FILECACHE));

}
 
Example 19
Source File: IntegrationTestIngest.java    From hbase with Apache License 2.0 4 votes vote down vote up
private String getColumnFamiliesAsString() {
  return StringUtils.join(",", getColumnFamilies());
}
 
Example 20
Source File: ResourceLocalizationService.java    From big-c with Apache License 2.0 4 votes vote down vote up
private String getAppFileCachePath(String user, String appId) {
  return StringUtils.join(Path.SEPARATOR, Arrays.asList(".",
      ContainerLocalizer.USERCACHE, user, ContainerLocalizer.APPCACHE, appId,
      ContainerLocalizer.FILECACHE));
}