Java Code Examples for org.apache.hadoop.fs.shell.CommandFormat

The following examples show how to use org.apache.hadoop.fs.shell.CommandFormat. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: DFSAdmin.java    License: Apache License 2.0 6 votes vote down vote up
/** Constructor */
SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String str = parameters.remove(0).trim();
  try {
    quota = StringUtils.TraditionalBinaryPrefix.string2long(str);
  } catch (NumberFormatException nfe) {
    throw new IllegalArgumentException("\"" + str + "\" is not a valid value for a quota.");
  }
  String storageTypeString =
      StringUtils.popOptionWithArgument("-storageType", parameters);
  if (storageTypeString != null) {
    this.type = StorageType.parseStorageType(storageTypeString);
  }
  
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 2
Source Project: hadoop   Source File: FsShellPermissions.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "R", null);
  cf.parse(args);
  setRecursive(cf.getOpt("R"));

  String modeStr = args.removeFirst();
  try {
    pp = new ChmodParser(modeStr);
  } catch (IllegalArgumentException iea) {
    // TODO: remove "chmod : " so it's not doubled up in output, but it's
    // here for backwards compatibility...
    throw new IllegalArgumentException(
        "chmod : mode '" + modeStr + "' does not match the expected pattern.");      
  }
}
 
Example 3
Source Project: hadoop   Source File: TestCommandFormat.java    License: Apache License 2.0 6 votes vote down vote up
private static <T> CommandFormat checkArgLimits(
    Class<? extends IllegalArgumentException> expectedErr,
    int min, int max, String ... opts)
{
  CommandFormat cf = new CommandFormat(min, max, opts);
  List<String> parsedArgs = new ArrayList<String>(args);
  
  Class<?> cfError = null;
  try {
    cf.parse(parsedArgs);
  } catch (IllegalArgumentException e) {
    System.out.println(e.getMessage());
    cfError = e.getClass();
  }

  assertEquals(expectedErr, cfError);
  if (expectedErr == null) {
    assertEquals(expectedArgs, parsedArgs);
    assertEquals(expectedOpts, cf.getOpts());
  }
  return cf;
}
 
Example 4
Source Project: big-c   Source File: DFSAdmin.java    License: Apache License 2.0 6 votes vote down vote up
/** Constructor */
SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String str = parameters.remove(0).trim();
  try {
    quota = StringUtils.TraditionalBinaryPrefix.string2long(str);
  } catch (NumberFormatException nfe) {
    throw new IllegalArgumentException("\"" + str + "\" is not a valid value for a quota.");
  }
  String storageTypeString =
      StringUtils.popOptionWithArgument("-storageType", parameters);
  if (storageTypeString != null) {
    this.type = StorageType.parseStorageType(storageTypeString);
  }
  
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 5
Source Project: big-c   Source File: FsShellPermissions.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "R", null);
  cf.parse(args);
  setRecursive(cf.getOpt("R"));

  String modeStr = args.removeFirst();
  try {
    pp = new ChmodParser(modeStr);
  } catch (IllegalArgumentException iea) {
    // TODO: remove "chmod : " so it's not doubled up in output, but it's
    // here for backwards compatibility...
    throw new IllegalArgumentException(
        "chmod : mode '" + modeStr + "' does not match the expected pattern.");      
  }
}
 
Example 6
Source Project: big-c   Source File: TestCommandFormat.java    License: Apache License 2.0 6 votes vote down vote up
private static <T> CommandFormat checkArgLimits(
    Class<? extends IllegalArgumentException> expectedErr,
    int min, int max, String ... opts)
{
  CommandFormat cf = new CommandFormat(min, max, opts);
  List<String> parsedArgs = new ArrayList<String>(args);
  
  Class<?> cfError = null;
  try {
    cf.parse(parsedArgs);
  } catch (IllegalArgumentException e) {
    System.out.println(e.getMessage());
    cfError = e.getClass();
  }

  assertEquals(expectedErr, cfError);
  if (expectedErr == null) {
    assertEquals(expectedArgs, parsedArgs);
    assertEquals(expectedOpts, cf.getOpts());
  }
  return cf;
}
 
Example 7
Source Project: hadoop   Source File: DFSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 8
Source Project: hadoop   Source File: DFSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/** Constructor */
SetQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.quota = Long.parseLong(parameters.remove(0));
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 9
Source Project: hadoop   Source File: DFSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String storageTypeString =
      StringUtils.popOptionWithArgument("-storageType", parameters);
  if (storageTypeString != null) {
    this.type = StorageType.parseStorageType(storageTypeString);
  }
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 10
Source Project: hadoop   Source File: FsShellPermissions.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "R");
  cf.parse(args);
  setRecursive(cf.getOpt("R"));
  parseOwnerGroup(args.removeFirst());
}
 
Example 11
Source Project: big-c   Source File: DFSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 12
Source Project: big-c   Source File: DFSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/** Constructor */
SetQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.quota = Long.parseLong(parameters.remove(0));
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 13
Source Project: big-c   Source File: DFSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String storageTypeString =
      StringUtils.popOptionWithArgument("-storageType", parameters);
  if (storageTypeString != null) {
    this.type = StorageType.parseStorageType(storageTypeString);
  }
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 14
Source Project: big-c   Source File: FsShellPermissions.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "R");
  cf.parse(args);
  setRecursive(cf.getOpt("R"));
  parseOwnerGroup(args.removeFirst());
}
 
Example 15
Source Project: examples   Source File: Find.java    License: Apache License 2.0 5 votes vote down vote up
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, OPTION_FOLLOW_LINK, OPTION_FOLLOW_ARG_LINK);
  cf.parse(args);
  
  if(cf.getOpt(OPTION_FOLLOW_LINK)) {
    options.setFollowLink(true);
  }
  else if(cf.getOpt(OPTION_FOLLOW_ARG_LINK)) {
    options.setFollowArgLink(true);
  }
}
 
Example 16
Source Project: RDFS   Source File: FsShell.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException
 */
private void setReplication(String[] cmd, int pos) throws IOException {
  final int minArgs = 2;  // We need the replication and at least one path.
  CommandFormat c =
    new CommandFormat("setrep", minArgs, SETREP_MAX_PATHS, "R", "w");
  short rep = 0;
  List<String> dsts = null;
  try {
    List<String> parameters = c.parse(cmd, pos);
    rep = Short.parseShort(parameters.get(0));
    dsts = parameters.subList(1, parameters.size());
  } catch (NumberFormatException nfe) {
    System.err.println("Illegal replication, a positive integer expected");
    throw nfe;
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + SETREP_SHORT_USAGE);
    throw iae;
  }

  if (rep < 1) {
    System.err.println("Cannot set replication to: " + rep);
    throw new IllegalArgumentException("replication must be >= 1");
  }

  List<Path> waitList = c.getOpt("w")? new ArrayList<Path>(): null;
  for (String dst: dsts) {
    setReplication(rep, dst, c.getOpt("R"), waitList);
  }

  if (waitList != null) {
    waitForReplication(waitList, rep);
  }
}
 
Example 17
Source Project: RDFS   Source File: FsShell.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException
 */
private void head(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("head", 1, 1);
  String src = null;
  Path path = null;

  try {
    List<String> parameters = c.parse(cmd, pos);
    src = parameters.get(0);
  } catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + HEAD_USAGE);
    throw iae;
  }

  path = new Path(src);
  FileSystem srcFs = path.getFileSystem(getConf());
  if (srcFs.isDirectory(path)) {
    throw new IOException("Source must be a file.");
  }

  long fileSize = srcFs.getFileStatus(path).getLen();
  int len = (fileSize > 1024) ? 1024 : (int) fileSize;

  FSDataInputStream in = srcFs.open(path);
  byte buf[] = new byte[len];
  IOUtils.readFully(in, buf, 0, len);
  System.out.write(buf);
  System.out.write('\n');
  in.close();
}
 
Example 18
Source Project: RDFS   Source File: DFSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 19
Source Project: RDFS   Source File: DFSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/** Constructor */
SetQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.quota = Long.parseLong(parameters.remove(0));
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 20
Source Project: RDFS   Source File: DFSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 21
Source Project: RDFS   Source File: DFSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/** Constructor */
SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String str = parameters.remove(0).trim();
  quota = StringUtils.TraditionalBinaryPrefix.string2long(str);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 22
Source Project: hadoop-gpu   Source File: FsShell.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException 
 */
private void setReplication(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("setrep", 2, 2, "R", "w");
  String dst = null;
  short rep = 0;

  try {
    List<String> parameters = c.parse(cmd, pos);
    rep = Short.parseShort(parameters.get(0));
    dst = parameters.get(1);
  } catch (NumberFormatException nfe) {
    System.err.println("Illegal replication, a positive integer expected");
    throw nfe;
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + SETREP_SHORT_USAGE);
    throw iae;
  }

  if (rep < 1) {
    System.err.println("Cannot set replication to: " + rep);
    throw new IllegalArgumentException("replication must be >= 1");
  }

  List<Path> waitList = c.getOpt("w")? new ArrayList<Path>(): null;
  setReplication(rep, dst, c.getOpt("R"), waitList);

  if (waitList != null) {
    waitForReplication(waitList, rep);
  }
}
 
Example 23
Source Project: hadoop-gpu   Source File: DFSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 24
Source Project: hadoop-gpu   Source File: DFSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/** Constructor */
SetQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.quota = Long.parseLong(parameters.remove(0));
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 25
Source Project: hadoop-gpu   Source File: DFSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 26
Source Project: hadoop-gpu   Source File: DFSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
/** Constructor */
SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String str = parameters.remove(0).trim();
  quota = StringUtils.TraditionalBinaryPrefix.string2long(str);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example 27
Source Project: hadoop   Source File: Find.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf =
      new CommandFormat(1, Integer.MAX_VALUE, OPTION_FOLLOW_LINK,
          OPTION_FOLLOW_ARG_LINK, null);
  cf.parse(args);

  if (cf.getOpt(OPTION_FOLLOW_LINK)) {
    getOptions().setFollowLink(true);
  } else if (cf.getOpt(OPTION_FOLLOW_ARG_LINK)) {
    getOptions().setFollowArgLink(true);
  }

  // search for first non-path argument (ie starts with a "-") and capture and
  // remove the remaining arguments as expressions
  LinkedList<String> expressionArgs = new LinkedList<String>();
  Iterator<String> it = args.iterator();
  boolean isPath = true;
  while (it.hasNext()) {
    String arg = it.next();
    if (isPath) {
      if (arg.startsWith("-")) {
        isPath = false;
      }
    }
    if (!isPath) {
      expressionArgs.add(arg);
      it.remove();
    }
  }

  if (args.isEmpty()) {
    args.add(Path.CUR_DIR);
  }

  Expression expression = parseExpression(expressionArgs);
  if (!expression.isAction()) {
    Expression and = getExpression(And.class);
    Deque<Expression> children = new LinkedList<Expression>();
    children.add(getExpression(Print.class));
    children.add(expression);
    and.addChildren(children);
    expression = and;
  }

  setRootExpression(expression);
}
 
Example 28
Source Project: big-c   Source File: Find.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf =
      new CommandFormat(1, Integer.MAX_VALUE, OPTION_FOLLOW_LINK,
          OPTION_FOLLOW_ARG_LINK, null);
  cf.parse(args);

  if (cf.getOpt(OPTION_FOLLOW_LINK)) {
    getOptions().setFollowLink(true);
  } else if (cf.getOpt(OPTION_FOLLOW_ARG_LINK)) {
    getOptions().setFollowArgLink(true);
  }

  // search for first non-path argument (ie starts with a "-") and capture and
  // remove the remaining arguments as expressions
  LinkedList<String> expressionArgs = new LinkedList<String>();
  Iterator<String> it = args.iterator();
  boolean isPath = true;
  while (it.hasNext()) {
    String arg = it.next();
    if (isPath) {
      if (arg.startsWith("-")) {
        isPath = false;
      }
    }
    if (!isPath) {
      expressionArgs.add(arg);
      it.remove();
    }
  }

  if (args.isEmpty()) {
    args.add(Path.CUR_DIR);
  }

  Expression expression = parseExpression(expressionArgs);
  if (!expression.isAction()) {
    Expression and = getExpression(And.class);
    Deque<Expression> children = new LinkedList<Expression>();
    children.add(getExpression(Print.class));
    children.add(expression);
    and.addChildren(children);
    expression = and;
  }

  setRootExpression(expression);
}
 
Example 29
Source Project: RDFS   Source File: FsShell.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Obtain the indicated files that match the file pattern <i>srcf</i>
 * and copy them to the local name. srcf is kept.
 * When copying multiple files, the destination must be a directory.
 * Otherwise, IOException is thrown.
 * @param argv: arguments
 * @param pos: Ignore everything before argv[pos]
 * @exception: IOException
 * @see org.apache.hadoop.fs.FileSystem.globStatus
 */
void copyToLocal(String[]argv, int pos) throws IOException {
  CommandFormat cf = new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc", "gencrc");

  String srcstr = null;
  String dststr = null;
  try {
    List<String> parameters = cf.parse(argv, pos);
    srcstr = parameters.get(0);
    dststr = parameters.get(1);
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + GET_SHORT_USAGE);
    throw iae;
  }
  boolean copyCrc = cf.getOpt("crc");
  final boolean genCrc = cf.getOpt("gencrc");
  final boolean verifyChecksum = !cf.getOpt("ignoreCrc");

  if (dststr.equals("-")) {
    if (copyCrc) {
      System.err.println("-crc option is not valid when destination is stdout.");
    }
    cat(srcstr, verifyChecksum, genCrc);
  } else {
    File dst = new File(dststr);
    Path srcpath = new Path(srcstr);
    FileSystem srcFS = getSrcFileSystem(srcpath, verifyChecksum);
    if (copyCrc && !(srcFS instanceof ChecksumFileSystem)) {
      System.err.println("-crc option is not valid when source file system " +
          "does not have crc files. Automatically turn the option off.");
      copyCrc = false;
    }
    FileStatus[] srcs = srcFS.globStatus(srcpath);
    boolean dstIsDir = dst.isDirectory();
    if (srcs.length > 1 && !dstIsDir) {
      throw new IOException("When copying multiple files, "
                            + "destination should be a directory.");
    }
    for (FileStatus status : srcs) {
      Path p = status.getPath();
      File f = dstIsDir? new File(dst, p.getName()): dst;
      copyToLocal(srcFS, p, f, copyCrc, genCrc);
    }
  }
}
 
Example 30
Source Project: RDFS   Source File: FsShell.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException
 */
private void tail(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("tail", 1, 1, "f");
  String src = null;
  Path path = null;

  try {
    List<String> parameters = c.parse(cmd, pos);
    src = parameters.get(0);
  } catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + TAIL_USAGE);
    throw iae;
  }
  boolean foption = c.getOpt("f") ? true: false;
  path = new Path(src);
  FileSystem srcFs = path.getFileSystem(getConf());
  if (srcFs.isDirectory(path)) {
    throw new IOException("Source must be a file.");
  }

  long fileSize = srcFs.getFileStatus(path).getLen();
  long offset = (fileSize > 1024) ? fileSize - 1024: 0;

  while (true) {
    FSDataInputStream in = srcFs.open(path);
    in.seek(offset);
    IOUtils.copyBytes(in, System.out, 1024, false);
    offset = in.getPos();
    in.close();
    if (!foption) {
      break;
    }
    fileSize = srcFs.getFileStatus(path).getLen();
    offset = (fileSize > offset) ? offset: fileSize;
    try {
      Thread.sleep(5000);
    } catch (InterruptedException e) {
      break;
    }
  }
}