Java Code Examples for org.apache.hadoop.util.Shell#execCommand()

/**
 * Used on Windows to determine if the specified file is a symlink that
 * targets a directory.  On most platforms, these checks can be done using
 * commons-io.  On Windows, the commons-io implementation is unreliable and
 * always returns false.  Instead, this method checks the output of the dir
 * command.  After migrating to Java 7, this method can be removed in favor
 * of the new method java.nio.file.Files.isSymbolicLink, which is expected to
 * work cross-platform.
 * 
 * @param file File to check
 * @return boolean true if the file is a symlink that targets a directory
 * @throws IOException thrown for any I/O error
 */
private static boolean isWindowsSymlinkedDirectory(File file)
    throws IOException {
  String dirOut = Shell.execCommand("cmd", "/c", "dir",
    file.getAbsoluteFile().getParent());
  StringReader sr = new StringReader(dirOut);
  BufferedReader br = new BufferedReader(sr);
  try {
    String line = br.readLine();
    while (line != null) {
      line = br.readLine();
      if (line.contains(file.getName()) && line.contains("<SYMLINKD>")) {
        return true;
      }
    }
    return false;
  } finally {
    IOUtils.closeStream(br);
    IOUtils.closeStream(sr);
  }
}
 

private void _checkDirs(boolean isDir, FsPermission perm, boolean success)
    throws Throwable {
  File localDir = File.createTempFile("test", "tmp");
  if (isDir) {
    localDir.delete();
    localDir.mkdir();
  }
  Shell.execCommand(Shell.getSetPermissionCommand(String.format("%04o",
    perm.toShort()), false, localDir.getAbsolutePath()));
  try {
    DiskChecker.checkDir(FileSystem.getLocal(new Configuration()),
      new Path(localDir.getAbsolutePath()), perm);
    assertTrue("checkDir success", success);
  } catch (DiskErrorException e) {
    assertFalse("checkDir success", success);
  }
  localDir.delete();
}
 

/**
 * Used on Windows to determine if the specified file is a symlink that
 * targets a directory.  On most platforms, these checks can be done using
 * commons-io.  On Windows, the commons-io implementation is unreliable and
 * always returns false.  Instead, this method checks the output of the dir
 * command.  After migrating to Java 7, this method can be removed in favor
 * of the new method java.nio.file.Files.isSymbolicLink, which is expected to
 * work cross-platform.
 * 
 * @param file File to check
 * @return boolean true if the file is a symlink that targets a directory
 * @throws IOException thrown for any I/O error
 */
private static boolean isWindowsSymlinkedDirectory(File file)
    throws IOException {
  String dirOut = Shell.execCommand("cmd", "/c", "dir",
    file.getAbsoluteFile().getParent());
  StringReader sr = new StringReader(dirOut);
  BufferedReader br = new BufferedReader(sr);
  try {
    String line = br.readLine();
    while (line != null) {
      line = br.readLine();
      if (line.contains(file.getName()) && line.contains("<SYMLINKD>")) {
        return true;
      }
    }
    return false;
  } finally {
    IOUtils.closeStream(br);
    IOUtils.closeStream(sr);
  }
}
 

private void _checkDirs(boolean isDir, String perm, boolean success)
    throws Throwable {
  File localDir = File.createTempFile("test", "tmp");
  if (isDir) {
    localDir.delete();
    localDir.mkdir();
  }
  Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
                                                  localDir.getAbsolutePath()));
  try {
    DiskChecker.checkDir(localDir);
    assertTrue("checkDir success", success);
  } catch (DiskErrorException e) {
    e.printStackTrace();
    assertFalse("checkDir success", success);
  }
  localDir.delete();
  System.out.println("checkDir success: " + success);

}
 

/**
 * Test getLocalPathToRead() returns correct filename and "file" schema.
 *
 * @throws IOException
 */
@Test (timeout = 30000)
public void testGetLocalPathToRead() throws IOException {
  assumeTrue(!isWindows);
  String dir = buildBufferDir(ROOT, 0);
  try {
    conf.set(CONTEXT, dir);
    assertTrue(localFs.mkdirs(new Path(dir)));
    File f1 = dirAllocator.createTmpFileForWrite(FILENAME, SMALL_FILE_SIZE,
        conf);
    Path p1 = dirAllocator.getLocalPathToRead(f1.getName(), conf);
    assertEquals(f1.getName(), p1.getName());
    assertEquals("file", p1.getFileSystem(conf).getUri().getScheme());
  } finally {
    Shell.execCommand(Shell.getSetPermissionCommand("u+w", false,
                                                    BUFFER_DIR_ROOT));
    rmBufferDirs();
  }
}
 

private void _checkDirs(boolean isDir, String perm, boolean success)
    throws Throwable {
  File localDir = File.createTempFile("test", "tmp");
  if (isDir) {
    localDir.delete();
    localDir.mkdir();
  }
  Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
                                                  localDir.getAbsolutePath()));
  try {
    DiskChecker.checkDir(localDir);
    assertTrue("checkDir success", success);
  } catch (DiskErrorException e) {
    e.printStackTrace();
    assertFalse("checkDir success", success);
  }
  localDir.delete();
  System.out.println("checkDir success: " + success);

}
 

/** Two buffer dirs. The first dir does not exist & is on a read-only disk;
 * The second dir exists & is RW
 * @throws Exception
 */
@Test (timeout = 30000)
public void test0() throws Exception {
  if (isWindows) return;
  String dir0 = buildBufferDir(ROOT, 0);
  String dir1 = buildBufferDir(ROOT, 1);
  try {
    conf.set(CONTEXT, dir0 + "," + dir1);
    assertTrue(localFs.mkdirs(new Path(dir1)));
    BUFFER_ROOT.setReadOnly();
    validateTempDirCreation(dir1);
    validateTempDirCreation(dir1);
  } finally {
    Shell.execCommand(Shell.getSetPermissionCommand("u+w", false,
                                                    BUFFER_DIR_ROOT));
    rmBufferDirs();
  }
}
 

@Override
public String getStorageAccountKey(String accountName, Configuration conf)
    throws KeyProviderException {
  String envelope = super.getStorageAccountKey(accountName, conf);

  final String command = conf.get(KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT);
  if (command == null) {
    throw new KeyProviderException(
        "Script path is not specified via fs.azure.shellkeyprovider.script");
  }

  String[] cmd = command.split(" ");
  String[] cmdWithEnvelope = Arrays.copyOf(cmd, cmd.length + 1);
  cmdWithEnvelope[cmdWithEnvelope.length - 1] = envelope;

  String decryptedKey = null;
  try {
    decryptedKey = Shell.execCommand(cmdWithEnvelope);
  } catch (IOException ex) {
    throw new KeyProviderException(ex);
  }

  // trim any whitespace
  return decryptedKey.trim();
}
 

/**
 * Test getLocalPathToRead() returns correct filename and "file" schema.
 *
 * @throws IOException
 */
@Test (timeout = 30000)
public void testGetLocalPathToRead() throws IOException {
  assumeTrue(!isWindows);
  String dir = buildBufferDir(ROOT, 0);
  try {
    conf.set(CONTEXT, dir);
    assertTrue(localFs.mkdirs(new Path(dir)));
    File f1 = dirAllocator.createTmpFileForWrite(FILENAME, SMALL_FILE_SIZE,
        conf);
    Path p1 = dirAllocator.getLocalPathToRead(f1.getName(), conf);
    assertEquals(f1.getName(), p1.getName());
    assertEquals("file", p1.getFileSystem(conf).getUri().getScheme());
  } finally {
    Shell.execCommand(Shell.getSetPermissionCommand("u+w", false,
                                                    BUFFER_DIR_ROOT));
    rmBufferDirs();
  }
}
 

/**
 * Use the command chmod to set permission.
 */
@Override
public void setPermission(Path p, FsPermission permission)
  throws IOException {
  if (NativeIO.isAvailable()) {
    NativeIO.POSIX.chmod(pathToFile(p).getCanonicalPath(),
                   permission.toShort());
  } else {
    String perm = String.format("%04o", permission.toShort());
    Shell.execCommand(Shell.getSetPermissionCommand(perm, false,
      FileUtil.makeShellPath(pathToFile(p), true)));
  }
}
 

static String execCommand(File f, String... cmd) throws IOException {
  String[] args = new String[cmd.length + 1];
  System.arraycopy(cmd, 0, args, 0, cmd.length);
  args[cmd.length] = f.getCanonicalPath();
  String output = Shell.execCommand(args);
  return output;
}
 

static String execCommand(File f, String... cmd) throws IOException {
  String[] args = new String[cmd.length + 1];
  System.arraycopy(cmd, 0, args, 0, cmd.length);
  args[cmd.length] = f.getCanonicalPath();
  String output = Shell.execCommand(args);
  return output;
}
 

/** Two buffer dirs. The first dir does not exist & is on a read-only disk;
 * The second dir exists & is RW
 * getLocalPathForWrite with checkAccess set to false should create a parent
 * directory. With checkAccess true, the directory should not be created.
 * @throws Exception
 */
@Test (timeout = 30000)
public void testLocalPathForWriteDirCreation() throws IOException {
  String dir0 = buildBufferDir(ROOT, 0);
  String dir1 = buildBufferDir(ROOT, 1);
  try {
    conf.set(CONTEXT, dir0 + "," + dir1);
    assertTrue(localFs.mkdirs(new Path(dir1)));
    BUFFER_ROOT.setReadOnly();
    Path p1 =
      dirAllocator.getLocalPathForWrite("p1/x", SMALL_FILE_SIZE, conf);
    assertTrue(localFs.getFileStatus(p1.getParent()).isDirectory());

    Path p2 =
      dirAllocator.getLocalPathForWrite("p2/x", SMALL_FILE_SIZE, conf,
          false);
    try {
      localFs.getFileStatus(p2.getParent());
    } catch (Exception e) {
      assertEquals(e.getClass(), FileNotFoundException.class);
    }
  } finally {
    Shell.execCommand(Shell.getSetPermissionCommand("u+w", false,
                                                    BUFFER_DIR_ROOT));
    rmBufferDirs();
  }
}
 

private static String[] executeShellCommand(String[] command)
throws IOException {
  String groups = Shell.execCommand(command);
  StringTokenizer tokenizer = new StringTokenizer(groups);
  int numOfTokens = tokenizer.countTokens();
  String[] tokens = new String[numOfTokens];
  for (int i=0; tokenizer.hasMoreTokens(); i++) {
    tokens[i] = tokenizer.nextToken();
  }

  return tokens;
}
 

/**
 * Calls shell to get users for a netgroup by calling getent
 * netgroup, this is a low level function that just returns string
 * that 
 *
 * @param netgroup get users for this netgroup
 * @return string of users for a given netgroup in getent netgroups format
 */
protected String execShellGetUserForNetgroup(final String netgroup)
    throws IOException {
  String result = "";
  try {
    // shell command does not expect '@' at the begining of the group name
    result = Shell.execCommand(
      Shell.getUsersForNetgroupCommand(netgroup.substring(1)));
  } catch (ExitCodeException e) {
    // if we didn't get the group - just return empty list;
    LOG.warn("error getting users for netgroup " + netgroup, e);
  }
  return result;
}
 

/** Two buffer dirs. The first dir exists & is on a read-only disk; 
 * The second dir exists & is RW
 * @throws Exception
 */
public void test1() throws Exception {
  if (isWindows) return;
  try {
    conf.set(CONTEXT, BUFFER_DIR[1]+","+BUFFER_DIR[2]);
    assertTrue(localFs.mkdirs(BUFFER_PATH[2]));
    BUFFER_ROOT.setReadOnly();
    validateTempDirCreation(2);
    validateTempDirCreation(2);
  } finally {
    Shell.execCommand(new String[]{"chmod", "u+w", BUFFER_DIR_ROOT});
    rmBufferDirs();
  }
}
 

/**
 * Dump out contents of $CWD and the environment to stdout for debugging
 */
private static void dumpOutDebugInfo() {

    LOG.info("Dump debug output");
    Map<String, String> envs = System.getenv();
    for (Map.Entry<String, String> env : envs.entrySet()) {
        LOG.info("System env: key=" + env.getKey() + ", val=" + env.getValue());
        System.out.println("System env: key=" + env.getKey() + ", val="
                + env.getValue());
    }

    BufferedReader buf = null;
    try {
        String lines = Shell.WINDOWS ? Shell.execCommand("cmd", "/c", "dir") :
                Shell.execCommand("ls", "-al");
        buf = new BufferedReader(new StringReader(lines));
        String line = "";
        while ((line = buf.readLine()) != null) {
            LOG.info("System CWD content: " + line);
            System.out.println("System CWD content: " + line);
        }
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        org.apache.hadoop.io.IOUtils.cleanup(LOG, buf);
    }
}
 

static List<String> getGroups() throws IOException {
  List<String> a = new ArrayList<String>();
  String s = Shell.execCommand(Shell.getGROUPS_COMMAND());
  for(StringTokenizer t = new StringTokenizer(s); t.hasMoreTokens(); ) {
    a.add(t.nextToken());
  }
  return a;
}
 

/** 
 * Get the current user's group list from Unix by running the command 'groups'
 * NOTE. For non-existing user it will return EMPTY list
 * @param user user name
 * @return the groups list that the <code>user</code> belongs to. The primary
 *         group is returned first.
 * @throws IOException if encounter any error when running the command
 */
private static List<String> getUnixGroups(final String user) throws IOException {
  String result = "";
  try {
    result = Shell.execCommand(Shell.getGroupsForUserCommand(user));
  } catch (ExitCodeException e) {
    // if we didn't get the group - just return empty list;
    LOG.warn("got exception trying to get groups for user " + user + ": "
        + e.getMessage());
    return new LinkedList<String>();
  }
  
  StringTokenizer tokenizer =
      new StringTokenizer(result, Shell.TOKEN_SEPARATOR_REGEX);
  List<String> groups = new LinkedList<String>();
  while (tokenizer.hasMoreTokens()) {
    groups.add(tokenizer.nextToken());
  }

  // remove duplicated primary group
  if (!Shell.WINDOWS) {
    for (int i = 1; i < groups.size(); i++) {
      if (groups.get(i).equals(groups.get(0))) {
        groups.remove(i);
        break;
      }
    }
  }

  return groups;
}
 

static String execCommand(File f, String... cmd) throws IOException {
  String[] args = new String[cmd.length + 1];
  System.arraycopy(cmd, 0, args, 0, cmd.length);
  args[cmd.length] = f.getCanonicalPath();
  return Shell.execCommand(args);
}