Java Code Examples for org.apache.hadoop.util.Shell#WINDOWS

private ServerConnector createHttpChannelConnector(
    Server server, HttpConfiguration httpConfig) {
  ServerConnector conn = new ServerConnector(server,
      conf.getInt(HTTP_ACCEPTOR_COUNT_KEY, HTTP_ACCEPTOR_COUNT_DEFAULT),
      conf.getInt(HTTP_SELECTOR_COUNT_KEY, HTTP_SELECTOR_COUNT_DEFAULT));
  ConnectionFactory connFactory = new HttpConnectionFactory(httpConfig);
  conn.addConnectionFactory(connFactory);
  if (Shell.WINDOWS) {
    // result of setting the SO_REUSEADDR flag is different on Windows
    // http://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx
    // without this 2 NN's can start on the same machine and listen on
    // the same port with indeterminate routing of incoming requests to them
    conn.setReuseAddress(false);
  }
  return conn;
}
 

@Override
protected synchronized void serviceStop() throws Exception {
    if (resourceManagers[index] != null) {
        waitForAppMastersToFinish(5000);
        resourceManagers[index].stop();
    }

    if (Shell.WINDOWS) {
        // On Windows, clean up the short temporary symlink that was created to
        // work around path length limitation.
        String testWorkDirPath = testWorkDir.getAbsolutePath();
        try {
            FileContext.getLocalFSFileContext().delete(new Path(testWorkDirPath),
                                                       true);
        } catch (IOException e) {
            LOG.warn("could not cleanup symlink: " +
                         testWorkDir.getAbsolutePath());
        }
    }
    super.serviceStop();
}
 

private LocalFSFileOutputStream(Path f, boolean append,
    FsPermission permission) throws IOException {
  File file = pathToFile(f);
  if (permission == null) {
    this.fos = new FileOutputStream(file, append);
  } else {
    if (Shell.WINDOWS && NativeIO.isAvailable()) {
      this.fos = NativeIO.Windows.createFileOutputStreamWithMode(file,
          append, permission.toShort());
    } else {
      this.fos = new FileOutputStream(file, append);
      boolean success = false;
      try {
        setPermission(f, permission);
        success = true;
      } finally {
        if (!success) {
          IOUtils.cleanup(LOG, this.fos);
        }
      }
    }
  }
}
 

protected boolean mkOneDirWithMode(Path p, File p2f, FsPermission permission)
    throws IOException {
  if (permission == null) {
    permission = FsPermission.getDirDefault();
  }
  permission = permission.applyUMask(FsPermission.getUMask(getConf()));
  if (Shell.WINDOWS && NativeIO.isAvailable()) {
    try {
      NativeIO.Windows.createDirectoryWithMode(p2f, permission.toShort());
      return true;
    } catch (IOException e) {
      if (LOG.isDebugEnabled()) {
        LOG.debug(String.format(
            "NativeIO.createDirectoryWithMode error, path = %s, mode = %o",
            p2f, permission.toShort()), e);
      }
      return false;
    }
  } else {
    boolean b = p2f.mkdir();
    if (b) {
      setPermission(p, permission);
    }
    return b;
  }
}
 

/**
 * Given a Tar File as input it will untar the file in a the untar directory
 * passed as the second parameter
 *
 * This utility will untar ".tar" files and ".tar.gz","tgz" files.
 *
 * @param inFile The tar file as input.
 * @param untarDir The untar directory where to untar the tar file.
 * @throws IOException exception untarring
 */
public static void unTar(File inFile, File untarDir) throws IOException {
  if (!untarDir.mkdirs()) {
    if (!untarDir.isDirectory()) {
      throw new IOException("Mkdirs failed to create " + untarDir);
    }
  }

  boolean gzipped = inFile.toString().endsWith("gz");
  if(Shell.WINDOWS) {
    // Tar is not native to Windows. Use simple Java based implementation for
    // tests and simple tar archives
    unTarUsingJava(inFile, untarDir, gzipped);
  }
  else {
    // spawn tar utility to untar archive for full fledged unix behavior such
    // as resolving symlinks in tar archives
    unTarUsingTar(inFile, untarDir, gzipped);
  }
}
 

/**
 * Returns the file stat for a file descriptor.
 *
 * @param fd file descriptor.
 * @return the file descriptor file stat.
 * @throws IOException thrown if there was an IO error while obtaining the file stat.
 */
public static Stat getFstat(FileDescriptor fd) throws IOException {
  Stat stat = null;
  if (!Shell.WINDOWS) {
    stat = fstat(fd); 
    stat.owner = getName(IdCache.USER, stat.ownerId);
    stat.group = getName(IdCache.GROUP, stat.groupId);
  } else {
    try {
      stat = fstat(fd);
    } catch (NativeIOException nioe) {
      if (nioe.getErrorCode() == 6) {
        throw new NativeIOException("The handle is invalid.",
            Errno.EBADF);
      } else {
        LOG.warn(String.format("NativeIO.getFstat error (%d): %s",
            nioe.getErrorCode(), nioe.getMessage()));
        throw new NativeIOException("Unknown error", Errno.UNKNOWN);
      }
    }
  }
  return stat;
}
 

/**
 * Platform independent implementation for {@link File#canWrite()}
 * @param f input file
 * @return On Unix, same as {@link File#canWrite()}
 *         On Windows, true if process has write access on the path
 */
public static boolean canWrite(File f) {
  if (Shell.WINDOWS) {
    try {
      return NativeIO.Windows.access(f.getCanonicalPath(),
          NativeIO.Windows.AccessRight.ACCESS_WRITE);
    } catch (IOException e) {
      return false;
    }
  } else {
    return f.canWrite();
  }
}
 

/**
 * Verify that information about the fencing target gets passed as
 * environment variables to the fencer.
 */
@Test
public void testTargetAsEnvironment() {
  if (!Shell.WINDOWS) {
    fencer.tryFence(TEST_TARGET, "echo $target_host $target_port");
    Mockito.verify(ShellCommandFencer.LOG).info(
        Mockito.endsWith("echo $ta...rget_port: dummyhost 1234"));
  } else {
    fencer.tryFence(TEST_TARGET, "echo %target_host% %target_port%");
    Mockito.verify(ShellCommandFencer.LOG).info(
        Mockito.endsWith("echo %ta...get_port%: dummyhost 1234"));
  }
}
 

public void configure(JobConf job) {
  String executionEnvPathVariable = System.getenv(Shell.WINDOWS ? "PATH"
      : "LD_LIBRARY_PATH");
  String hadoopHome = System.getenv("HADOOP_COMMON_HOME");
  if (hadoopHome == null) {
    hadoopHome = "";
  }
  String hadoopLibLocation = hadoopHome 
      + (Shell.WINDOWS ? "\\bin" : "/lib/native");
  assertTrue(executionEnvPathVariable.contains(hadoopLibLocation));
}
 

/**
 * Platform independent implementation for {@link File#canExecute()}
 * @param f input file
 * @return On Unix, same as {@link File#canExecute()}
 *         On Windows, true if process has execute access on the path
 */
public static boolean canExecute(File f) {
  if (Shell.WINDOWS) {
    try {
      return NativeIO.Windows.access(f.getCanonicalPath(),
          NativeIO.Windows.AccessRight.ACCESS_EXECUTE);
    } catch (IOException e) {
      return false;
    }
  } else {
    return f.canExecute();
  }
}
 

/**
 * Platform independent implementation for {@link File#setExecutable(boolean)}
 * File#setExecutable does not work as expected on Windows.
 * Note: revoking execute permission on folders does not have the same
 * behavior on Windows as on Unix platforms. Creating, deleting or renaming
 * a file within that folder will still succeed on Windows.
 * @param f input file
 * @param executable
 * @return true on success, false otherwise
 */
public static boolean setExecutable(File f, boolean executable) {
  if (Shell.WINDOWS) {
    try {
      String permission = executable ? "u+x" : "u-x";
      FileUtil.chmod(f.getCanonicalPath(), permission, false);
      return true;
    } catch (IOException ex) {
      return false;
    }
  } else {
    return f.setExecutable(executable);
  }
}
 

/**
 * Allows dynamic update to the environment variables. After calling put,
 * System.getenv(key) will then return value.
 *
 * @param key System environment variable
 * @param value Value to assign to system environment variable
 */
public synchronized static void put(String key, String value){
  Map<String, String> environment = new HashMap<String, String>(System.getenv());
  environment.put(key, value);
  if (!Shell.WINDOWS) {
    updateEnvironment(environment);
  } else {
    updateEnvironmentOnWindows(environment);
  }
}
 

/**
 * Verify that the Configuration gets passed as
 * environment variables to the fencer.
 */
@Test
public void testConfAsEnvironment() {
  if (!Shell.WINDOWS) {
    fencer.tryFence(TEST_TARGET, "echo $in_fencing_tests");
    Mockito.verify(ShellCommandFencer.LOG).info(
        Mockito.endsWith("echo $in...ing_tests: yessir"));
  } else {
    fencer.tryFence(TEST_TARGET, "echo %in_fencing_tests%");
    Mockito.verify(ShellCommandFencer.LOG).info(
        Mockito.endsWith("echo %in...ng_tests%: yessir"));
  }
}
 

private static String getFencerTrueCommand() {
  return Shell.WINDOWS ?
      FENCER_TRUE_COMMAND_WINDOWS : FENCER_TRUE_COMMAND_UNIX;
}
 

private void testContainerLaunchAndExit(int exitCode) throws IOException,
    InterruptedException, YarnException {

 File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
 PrintWriter fileWriter = new PrintWriter(scriptFile);
 File processStartFile =
	  new File(tmpDir, "start_file.txt").getAbsoluteFile();

 // ////// Construct the Container-id
 ContainerId cId = createContainerId(0);

 if (Shell.WINDOWS) {
   fileWriter.println("@echo Hello World!> " + processStartFile);
   fileWriter.println("@echo " + cId + ">> " + processStartFile);
   if (exitCode != 0) {
     fileWriter.println("@exit " + exitCode);
   }
 } else {
   fileWriter.write("\numask 0"); // So that start file is readable by the test
   fileWriter.write("\necho Hello World! > " + processStartFile);
   fileWriter.write("\necho $$ >> " + processStartFile); 
   // Have script throw an exit code at the end
   if (exitCode != 0) {
     fileWriter.write("\nexit "+exitCode);
   }
 }
 
 fileWriter.close();

 ContainerLaunchContext containerLaunchContext = 
	  recordFactory.newRecordInstance(ContainerLaunchContext.class);

 URL resource_alpha =
	  ConverterUtils.getYarnUrlFromPath(localFS
			  .makeQualified(new Path(scriptFile.getAbsolutePath())));
 LocalResource rsrc_alpha =
	  recordFactory.newRecordInstance(LocalResource.class);
 rsrc_alpha.setResource(resource_alpha);
 rsrc_alpha.setSize(-1);
 rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
 rsrc_alpha.setType(LocalResourceType.FILE);
 rsrc_alpha.setTimestamp(scriptFile.lastModified());
 String destinationFile = "dest_file";
 Map<String, LocalResource> localResources = 
	  new HashMap<String, LocalResource>();
 localResources.put(destinationFile, rsrc_alpha);
 containerLaunchContext.setLocalResources(localResources);
 List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
 containerLaunchContext.setCommands(commands);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

 BaseContainerManagerTest.waitForContainerState(containerManager, cId,
	  ContainerState.COMPLETE);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(cId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
 ContainerStatus containerStatus = 
	  containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);

 // Verify exit status matches exit state of script
 Assert.assertEquals(exitCode,
	  containerStatus.getExitStatus());	    
}
 

@Test
public void testNodeHealthScript() throws Exception {
  RecordFactory factory = RecordFactoryProvider.getRecordFactory(null);
  NodeHealthStatus healthStatus =
      factory.newRecordInstance(NodeHealthStatus.class);
  String errorScript = "echo ERROR\n echo \"Tracker not healthy\"";
  String normalScript = "echo \"I am all fine\"";
  String timeOutScript = Shell.WINDOWS ? "@echo off\nping -n 4 127.0.0.1 >nul\necho \"I am fine\""
      : "sleep 4\necho \"I am fine\"";
  Configuration conf = getConfForNodeHealthScript();
  conf.writeXml(new FileOutputStream(nodeHealthConfigFile));
  conf.addResource(nodeHealthConfigFile.getName());

  writeNodeHealthScriptFile(normalScript, true);
  NodeHealthCheckerService nodeHealthChecker = new NodeHealthCheckerService();
  nodeHealthChecker.init(conf);
  NodeHealthScriptRunner nodeHealthScriptRunner =
      nodeHealthChecker.getNodeHealthScriptRunner();
  TimerTask timerTask = nodeHealthScriptRunner.getTimerTask();

  timerTask.run();

  setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(),
      nodeHealthChecker.getHealthReport(),
      nodeHealthChecker.getLastHealthReportTime());
  LOG.info("Checking initial healthy condition");
  // Check proper report conditions.
  Assert.assertTrue("Node health status reported unhealthy", healthStatus
      .getIsNodeHealthy());
  Assert.assertTrue("Node health status reported unhealthy", healthStatus
      .getHealthReport().equals(nodeHealthChecker.getHealthReport()));

  // write out error file.
  // Healthy to unhealthy transition
  writeNodeHealthScriptFile(errorScript, true);
  // Run timer
  timerTask.run();
  // update health status
  setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(),
      nodeHealthChecker.getHealthReport(),
      nodeHealthChecker.getLastHealthReportTime());
  LOG.info("Checking Healthy--->Unhealthy");
  Assert.assertFalse("Node health status reported healthy", healthStatus
      .getIsNodeHealthy());
  Assert.assertTrue("Node health status reported healthy", healthStatus
      .getHealthReport().equals(nodeHealthChecker.getHealthReport()));
  
  // Check unhealthy to healthy transitions.
  writeNodeHealthScriptFile(normalScript, true);
  timerTask.run();
  setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(),
      nodeHealthChecker.getHealthReport(),
      nodeHealthChecker.getLastHealthReportTime());
  LOG.info("Checking UnHealthy--->healthy");
  // Check proper report conditions.
  Assert.assertTrue("Node health status reported unhealthy", healthStatus
      .getIsNodeHealthy());
  Assert.assertTrue("Node health status reported unhealthy", healthStatus
      .getHealthReport().equals(nodeHealthChecker.getHealthReport()));

  // Healthy to timeout transition.
  writeNodeHealthScriptFile(timeOutScript, true);
  timerTask.run();
  setHealthStatus(healthStatus, nodeHealthChecker.isHealthy(),
      nodeHealthChecker.getHealthReport(),
      nodeHealthChecker.getLastHealthReportTime());
  LOG.info("Checking Healthy--->timeout");
  Assert.assertFalse("Node health status reported healthy even after timeout",
      healthStatus.getIsNodeHealthy());
  Assert.assertTrue("Node script time out message not propogated",
      healthStatus.getHealthReport().equals(
          NodeHealthScriptRunner.NODE_HEALTH_SCRIPT_TIMED_OUT_MSG
          + NodeHealthCheckerService.SEPARATOR
          + nodeHealthChecker.getDiskHandler().getDisksHealthReport(false)));
}
 

@Test (timeout = 30000)
public void testGlobEscapeStatus() throws Exception {
  // This test is not meaningful on Windows where * is disallowed in file name.
  if (Shell.WINDOWS) return;
  FileSystem lfs = FileSystem.getLocal(new Configuration());
  Path testRoot = lfs.makeQualified(new Path(
      System.getProperty("test.build.data","test/build/data"),
      "testPathGlob"));
  lfs.delete(testRoot, true);
  lfs.mkdirs(testRoot);
  assertTrue(lfs.isDirectory(testRoot));
  lfs.setWorkingDirectory(testRoot);
  
  // create a couple dirs with file in them
  Path paths[] = new Path[]{
      new Path(testRoot, "*/f"),
      new Path(testRoot, "d1/f"),
      new Path(testRoot, "d2/f")
  };
  Arrays.sort(paths);
  for (Path p : paths) {
    lfs.create(p).close();
    assertTrue(lfs.exists(p));
  }

  // try the non-globbed listStatus
  FileStatus stats[] = lfs.listStatus(new Path(testRoot, "*"));
  assertEquals(1, stats.length);
  assertEquals(new Path(testRoot, "*/f"), stats[0].getPath());

  // ensure globStatus with "*" finds all dir contents
  stats = lfs.globStatus(new Path(testRoot, "*"));
  Arrays.sort(stats);
  Path parentPaths[] = new Path[paths.length];
  for (int i = 0; i < paths.length; i++) {
    parentPaths[i] = paths[i].getParent();
  }
  assertEquals(mergeStatuses(parentPaths), mergeStatuses(stats));

  // ensure that globStatus with an escaped "\*" only finds "*"
  stats = lfs.globStatus(new Path(testRoot, "\\*"));
  assertEquals(1, stats.length);
  assertEquals(new Path(testRoot, "*"), stats[0].getPath());

  // try to glob the inner file for all dirs
  stats = lfs.globStatus(new Path(testRoot, "*/f"));
  assertEquals(paths.length, stats.length);
  assertEquals(mergeStatuses(paths), mergeStatuses(stats));

  // try to get the inner file for only the "*" dir
  stats = lfs.globStatus(new Path(testRoot, "\\*/f"));
  assertEquals(1, stats.length);
  assertEquals(new Path(testRoot, "*/f"), stats[0].getPath());

  // try to glob all the contents of the "*" dir
  stats = lfs.globStatus(new Path(testRoot, "\\*/*"));
  assertEquals(1, stats.length);
  assertEquals(new Path(testRoot, "*/f"), stats[0].getPath());
}
 

public static String getFencerTrueCommand() {
  return Shell.WINDOWS ?
      FENCER_TRUE_COMMAND_WINDOWS : FENCER_TRUE_COMMAND_UNIX;
}
 

@Test
public void testSocketIOWithTimeout() throws Exception {
  
  // first open pipe:
  Pipe pipe = Pipe.open();
  Pipe.SourceChannel source = pipe.source();
  Pipe.SinkChannel sink = pipe.sink();
  
  try {
    final InputStream in = new SocketInputStream(source, TIMEOUT);
    OutputStream out = new SocketOutputStream(sink, TIMEOUT);
    
    byte[] writeBytes = TEST_STRING.getBytes();
    byte[] readBytes = new byte[writeBytes.length];
    byte byteWithHighBit = (byte)0x80;
    
    out.write(writeBytes);
    out.write(byteWithHighBit);
    doIO(null, out, TIMEOUT);
    
    in.read(readBytes);
    assertTrue(Arrays.equals(writeBytes, readBytes));
    assertEquals(byteWithHighBit & 0xff, in.read());
    doIO(in, null, TIMEOUT);
    
    // Change timeout on the read side.
    ((SocketInputStream)in).setTimeout(TIMEOUT * 2);
    doIO(in, null, TIMEOUT * 2);
    
    
    /*
     * Verify that it handles interrupted threads properly.
     * Use a large timeout and expect the thread to return quickly
     * upon interruption.
     */
    ((SocketInputStream)in).setTimeout(0);
    TestingThread thread = new TestingThread(ctx) {
      @Override
      public void doWork() throws Exception {
        try {
          in.read();
          fail("Did not fail with interrupt");
        } catch (InterruptedIOException ste) {
          LOG.info("Got expection while reading as expected : " + 
              ste.getMessage());
        }
      }
    };
    ctx.addThread(thread);
    ctx.startThreads();
    // If the thread is interrupted before it calls read()
    // then it throws ClosedByInterruptException due to
    // some Java quirk. Waiting for it to call read()
    // gets it into select(), so we get the expected
    // InterruptedIOException.
    Thread.sleep(1000);
    thread.interrupt();
    ctx.stop();

    //make sure the channels are still open
    assertTrue(source.isOpen());
    assertTrue(sink.isOpen());
    
    // Nevertheless, the output stream is closed, because
    // a partial write may have succeeded (see comment in
    // SocketOutputStream#write(byte[]), int, int)
    // This portion of the test cannot pass on Windows due to differences in
    // behavior of partial writes.  Windows appears to buffer large amounts of
    // written data and send it all atomically, thus making it impossible to
    // simulate a partial write scenario.  Attempts were made to switch the
    // test from using a pipe to a network socket and also to use larger and
    // larger buffers in doIO.  Nothing helped the situation though.
    if (!Shell.WINDOWS) {
      try {
        out.write(1);
        fail("Did not throw");
      } catch (IOException ioe) {
        GenericTestUtils.assertExceptionContains(
            "stream is closed", ioe);
      }
    }
    
    out.close();
    assertFalse(sink.isOpen());
    
    // close sink and expect -1 from source.read()
    assertEquals(-1, in.read());
    
    // make sure close() closes the underlying channel.
    in.close();
    assertFalse(source.isOpen());
    
  } finally {
    if (source != null) {
      source.close();
    }
    if (sink != null) {
      sink.close();
    }
  }
}
 

public static ShellScriptBuilder create() {
  return Shell.WINDOWS ? new WindowsShellScriptBuilder() :
    new UnixShellScriptBuilder();
}