org.apache.hadoop.fs.shell.CommandFormat Java Examples

The following examples show how to use org.apache.hadoop.fs.shell.CommandFormat. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DFSAdmin.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/** Constructor */
SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String str = parameters.remove(0).trim();
  try {
    quota = StringUtils.TraditionalBinaryPrefix.string2long(str);
  } catch (NumberFormatException nfe) {
    throw new IllegalArgumentException("\"" + str + "\" is not a valid value for a quota.");
  }
  String storageTypeString =
      StringUtils.popOptionWithArgument("-storageType", parameters);
  if (storageTypeString != null) {
    this.type = StorageType.parseStorageType(storageTypeString);
  }
  
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #2
Source File: TestCommandFormat.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static <T> CommandFormat checkArgLimits(
    Class<? extends IllegalArgumentException> expectedErr,
    int min, int max, String ... opts)
{
  CommandFormat cf = new CommandFormat(min, max, opts);
  List<String> parsedArgs = new ArrayList<String>(args);
  
  Class<?> cfError = null;
  try {
    cf.parse(parsedArgs);
  } catch (IllegalArgumentException e) {
    System.out.println(e.getMessage());
    cfError = e.getClass();
  }

  assertEquals(expectedErr, cfError);
  if (expectedErr == null) {
    assertEquals(expectedArgs, parsedArgs);
    assertEquals(expectedOpts, cf.getOpts());
  }
  return cf;
}
 
Example #3
Source File: FsShellPermissions.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "R", null);
  cf.parse(args);
  setRecursive(cf.getOpt("R"));

  String modeStr = args.removeFirst();
  try {
    pp = new ChmodParser(modeStr);
  } catch (IllegalArgumentException iea) {
    // TODO: remove "chmod : " so it's not doubled up in output, but it's
    // here for backwards compatibility...
    throw new IllegalArgumentException(
        "chmod : mode '" + modeStr + "' does not match the expected pattern.");      
  }
}
 
Example #4
Source File: TestCommandFormat.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static <T> CommandFormat checkArgLimits(
    Class<? extends IllegalArgumentException> expectedErr,
    int min, int max, String ... opts)
{
  CommandFormat cf = new CommandFormat(min, max, opts);
  List<String> parsedArgs = new ArrayList<String>(args);
  
  Class<?> cfError = null;
  try {
    cf.parse(parsedArgs);
  } catch (IllegalArgumentException e) {
    System.out.println(e.getMessage());
    cfError = e.getClass();
  }

  assertEquals(expectedErr, cfError);
  if (expectedErr == null) {
    assertEquals(expectedArgs, parsedArgs);
    assertEquals(expectedOpts, cf.getOpts());
  }
  return cf;
}
 
Example #5
Source File: FsShellPermissions.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "R", null);
  cf.parse(args);
  setRecursive(cf.getOpt("R"));

  String modeStr = args.removeFirst();
  try {
    pp = new ChmodParser(modeStr);
  } catch (IllegalArgumentException iea) {
    // TODO: remove "chmod : " so it's not doubled up in output, but it's
    // here for backwards compatibility...
    throw new IllegalArgumentException(
        "chmod : mode '" + modeStr + "' does not match the expected pattern.");      
  }
}
 
Example #6
Source File: DFSAdmin.java    From big-c with Apache License 2.0 6 votes vote down vote up
/** Constructor */
SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String str = parameters.remove(0).trim();
  try {
    quota = StringUtils.TraditionalBinaryPrefix.string2long(str);
  } catch (NumberFormatException nfe) {
    throw new IllegalArgumentException("\"" + str + "\" is not a valid value for a quota.");
  }
  String storageTypeString =
      StringUtils.popOptionWithArgument("-storageType", parameters);
  if (storageTypeString != null) {
    this.type = StorageType.parseStorageType(storageTypeString);
  }
  
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #7
Source File: DFSAdmin.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/** Constructor */
SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String str = parameters.remove(0).trim();
  quota = StringUtils.TraditionalBinaryPrefix.string2long(str);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #8
Source File: FsShell.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException
 */
private void setReplication(String[] cmd, int pos) throws IOException {
  final int minArgs = 2;  // We need the replication and at least one path.
  CommandFormat c =
    new CommandFormat("setrep", minArgs, SETREP_MAX_PATHS, "R", "w");
  short rep = 0;
  List<String> dsts = null;
  try {
    List<String> parameters = c.parse(cmd, pos);
    rep = Short.parseShort(parameters.get(0));
    dsts = parameters.subList(1, parameters.size());
  } catch (NumberFormatException nfe) {
    System.err.println("Illegal replication, a positive integer expected");
    throw nfe;
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + SETREP_SHORT_USAGE);
    throw iae;
  }

  if (rep < 1) {
    System.err.println("Cannot set replication to: " + rep);
    throw new IllegalArgumentException("replication must be >= 1");
  }

  List<Path> waitList = c.getOpt("w")? new ArrayList<Path>(): null;
  for (String dst: dsts) {
    setReplication(rep, dst, c.getOpt("R"), waitList);
  }

  if (waitList != null) {
    waitForReplication(waitList, rep);
  }
}
 
Example #9
Source File: FsShell.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException
 */
private void head(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("head", 1, 1);
  String src = null;
  Path path = null;

  try {
    List<String> parameters = c.parse(cmd, pos);
    src = parameters.get(0);
  } catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + HEAD_USAGE);
    throw iae;
  }

  path = new Path(src);
  FileSystem srcFs = path.getFileSystem(getConf());
  if (srcFs.isDirectory(path)) {
    throw new IOException("Source must be a file.");
  }

  long fileSize = srcFs.getFileStatus(path).getLen();
  int len = (fileSize > 1024) ? 1024 : (int) fileSize;

  FSDataInputStream in = srcFs.open(path);
  byte buf[] = new byte[len];
  IOUtils.readFully(in, buf, 0, len);
  System.out.write(buf);
  System.out.write('\n');
  in.close();
}
 
Example #10
Source File: DFSAdmin.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #11
Source File: DFSAdmin.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/** Constructor */
SetQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.quota = Long.parseLong(parameters.remove(0));
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #12
Source File: DFSAdmin.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #13
Source File: DFSAdmin.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/** Constructor */
SetQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.quota = Long.parseLong(parameters.remove(0));
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #14
Source File: FsShell.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException 
 */
private void setReplication(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("setrep", 2, 2, "R", "w");
  String dst = null;
  short rep = 0;

  try {
    List<String> parameters = c.parse(cmd, pos);
    rep = Short.parseShort(parameters.get(0));
    dst = parameters.get(1);
  } catch (NumberFormatException nfe) {
    System.err.println("Illegal replication, a positive integer expected");
    throw nfe;
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + SETREP_SHORT_USAGE);
    throw iae;
  }

  if (rep < 1) {
    System.err.println("Cannot set replication to: " + rep);
    throw new IllegalArgumentException("replication must be >= 1");
  }

  List<Path> waitList = c.getOpt("w")? new ArrayList<Path>(): null;
  setReplication(rep, dst, c.getOpt("R"), waitList);

  if (waitList != null) {
    waitForReplication(waitList, rep);
  }
}
 
Example #15
Source File: DFSAdmin.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #16
Source File: DFSAdmin.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/** Constructor */
SetQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.quota = Long.parseLong(parameters.remove(0));
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #17
Source File: DFSAdmin.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #18
Source File: DFSAdmin.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/** Constructor */
SetSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(NAME, 2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String str = parameters.remove(0).trim();
  quota = StringUtils.TraditionalBinaryPrefix.string2long(str);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #19
Source File: Find.java    From examples with Apache License 2.0 5 votes vote down vote up
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, OPTION_FOLLOW_LINK, OPTION_FOLLOW_ARG_LINK);
  cf.parse(args);
  
  if(cf.getOpt(OPTION_FOLLOW_LINK)) {
    options.setFollowLink(true);
  }
  else if(cf.getOpt(OPTION_FOLLOW_ARG_LINK)) {
    options.setFollowArgLink(true);
  }
}
 
Example #20
Source File: FsShellPermissions.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "R");
  cf.parse(args);
  setRecursive(cf.getOpt("R"));
  parseOwnerGroup(args.removeFirst());
}
 
Example #21
Source File: DFSAdmin.java    From big-c with Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String storageTypeString =
      StringUtils.popOptionWithArgument("-storageType", parameters);
  if (storageTypeString != null) {
    this.type = StorageType.parseStorageType(storageTypeString);
  }
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #22
Source File: DFSAdmin.java    From big-c with Apache License 2.0 5 votes vote down vote up
/** Constructor */
SetQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(2, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.quota = Long.parseLong(parameters.remove(0));
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #23
Source File: DFSAdmin.java    From big-c with Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #24
Source File: FsShellPermissions.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected void processOptions(LinkedList<String> args) throws IOException {
  CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "R");
  cf.parse(args);
  setRecursive(cf.getOpt("R"));
  parseOwnerGroup(args.removeFirst());
}
 
Example #25
Source File: DFSAdmin.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearSpaceQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  String storageTypeString =
      StringUtils.popOptionWithArgument("-storageType", parameters);
  if (storageTypeString != null) {
    this.type = StorageType.parseStorageType(storageTypeString);
  }
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #26
Source File: DFSAdmin.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/** Constructor */
ClearQuotaCommand(String[] args, int pos, FileSystem fs) {
  super(fs);
  CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
  List<String> parameters = c.parse(args, pos);
  this.args = parameters.toArray(new String[parameters.size()]);
}
 
Example #27
Source File: FsShell.java    From RDFS with Apache License 2.0 4 votes vote down vote up
/**
 * Obtain the indicated files that match the file pattern <i>srcf</i>
 * and copy them to the local name. srcf is kept.
 * When copying multiple files, the destination must be a directory.
 * Otherwise, IOException is thrown.
 * @param argv: arguments
 * @param pos: Ignore everything before argv[pos]
 * @exception: IOException
 * @see org.apache.hadoop.fs.FileSystem.globStatus
 */
void copyToLocal(String[]argv, int pos) throws IOException {
  CommandFormat cf = new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc", "gencrc");

  String srcstr = null;
  String dststr = null;
  try {
    List<String> parameters = cf.parse(argv, pos);
    srcstr = parameters.get(0);
    dststr = parameters.get(1);
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + GET_SHORT_USAGE);
    throw iae;
  }
  boolean copyCrc = cf.getOpt("crc");
  final boolean genCrc = cf.getOpt("gencrc");
  final boolean verifyChecksum = !cf.getOpt("ignoreCrc");

  if (dststr.equals("-")) {
    if (copyCrc) {
      System.err.println("-crc option is not valid when destination is stdout.");
    }
    cat(srcstr, verifyChecksum, genCrc);
  } else {
    File dst = new File(dststr);
    Path srcpath = new Path(srcstr);
    FileSystem srcFS = getSrcFileSystem(srcpath, verifyChecksum);
    if (copyCrc && !(srcFS instanceof ChecksumFileSystem)) {
      System.err.println("-crc option is not valid when source file system " +
          "does not have crc files. Automatically turn the option off.");
      copyCrc = false;
    }
    FileStatus[] srcs = srcFS.globStatus(srcpath);
    boolean dstIsDir = dst.isDirectory();
    if (srcs.length > 1 && !dstIsDir) {
      throw new IOException("When copying multiple files, "
                            + "destination should be a directory.");
    }
    for (FileStatus status : srcs) {
      Path p = status.getPath();
      File f = dstIsDir? new File(dst, p.getName()): dst;
      copyToLocal(srcFS, p, f, copyCrc, genCrc);
    }
  }
}
 
Example #28
Source File: FsShell.java    From RDFS with Apache License 2.0 4 votes vote down vote up
/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException
 */
private void tail(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("tail", 1, 1, "f");
  String src = null;
  Path path = null;

  try {
    List<String> parameters = c.parse(cmd, pos);
    src = parameters.get(0);
  } catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FsShell " + TAIL_USAGE);
    throw iae;
  }
  boolean foption = c.getOpt("f") ? true: false;
  path = new Path(src);
  FileSystem srcFs = path.getFileSystem(getConf());
  if (srcFs.isDirectory(path)) {
    throw new IOException("Source must be a file.");
  }

  long fileSize = srcFs.getFileStatus(path).getLen();
  long offset = (fileSize > 1024) ? fileSize - 1024: 0;

  while (true) {
    FSDataInputStream in = srcFs.open(path);
    in.seek(offset);
    IOUtils.copyBytes(in, System.out, 1024, false);
    offset = in.getPos();
    in.close();
    if (!foption) {
      break;
    }
    fileSize = srcFs.getFileStatus(path).getLen();
    offset = (fileSize > offset) ? offset: fileSize;
    try {
      Thread.sleep(5000);
    } catch (InterruptedException e) {
      break;
    }
  }
}
 
Example #29
Source File: FreightStreamer.java    From RDFS with Apache License 2.0 4 votes vote down vote up
/**
 * Obtain the indicated files that match the file pattern <i>srcf</i>
 * and copy them to the local name. srcf is kept.
 * When copying multiple files, the destination must be a directory. 
 * Otherwise, IOException is thrown.
 * @param argv: arguments
 * @param pos: Ignore everything before argv[pos]  
 * @exception: IOException  
 * @see org.apache.hadoop.fs.FileSystem.globStatus 
 */
void copyToLocal(String[]argv, int pos) throws IOException {
  CommandFormat cf = new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc");
  
  String srcstr = null;
  String dststr = null;
  try {
    List<String> parameters = cf.parse(argv, pos);
    srcstr = parameters.get(0);
    dststr = parameters.get(1);
  }
  catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FreightStreamer " + GET_SHORT_USAGE);
    throw iae;
  }
  boolean copyCrc = cf.getOpt("crc");
  final boolean verifyChecksum = !cf.getOpt("ignoreCrc");

  if (dststr.equals("-")) {
    if (copyCrc) {
      System.err.println("-crc option is not valid when destination is stdout.");
    }
    cat(srcstr, verifyChecksum);
  } else {
    File dst = new File(dststr);      
    Path srcpath = new Path(srcstr);
    FileSystem srcFS = getSrcFileSystem(srcpath, verifyChecksum);
    if (copyCrc && !(srcFS instanceof ChecksumFileSystem)) {
      System.err.println("-crc option is not valid when source file system " +
          "does not have crc files. Automatically turn the option off.");
      copyCrc = false;
    }
    FileStatus[] srcs = srcFS.globStatus(srcpath);
    boolean dstIsDir = dst.isDirectory(); 
    if (srcs.length > 1 && !dstIsDir) {
      throw new IOException("When copying multiple files, "
                            + "destination should be a directory.");
    }
    for (FileStatus status : srcs) {
      Path p = status.getPath();
      File f = dstIsDir? new File(dst, p.getName()): dst;
      copyToLocal(srcFS, p, f, copyCrc);
    }
  }
}
 
Example #30
Source File: FreightStreamer.java    From RDFS with Apache License 2.0 4 votes vote down vote up
/**
 * Parse the incoming command string
 * @param cmd
 * @param pos ignore anything before this pos in cmd
 * @throws IOException 
 */
private void tail(String[] cmd, int pos) throws IOException {
  CommandFormat c = new CommandFormat("tail", 1, 1, "f");
  String src = null;
  Path path = null;

  try {
    List<String> parameters = c.parse(cmd, pos);
    src = parameters.get(0);
  } catch(IllegalArgumentException iae) {
    System.err.println("Usage: java FreightStreamer " + TAIL_USAGE);
    throw iae;
  }
  boolean foption = c.getOpt("f") ? true: false;
  path = new Path(src);
  FileSystem srcFs = path.getFileSystem(getConf());
  if (srcFs.isDirectory(path)) {
    throw new IOException("Source must be a file.");
  }

  long fileSize = srcFs.getFileStatus(path).getLen();
  long offset = (fileSize > 1024) ? fileSize - 1024: 0;

  while (true) {
    FSDataInputStream in = srcFs.open(path);
    in.seek(offset);
    IOUtils.copyBytes(in, System.out, 1024, false);
    offset = in.getPos();
    in.close();
    if (!foption) {
      break;
    }
    fileSize = srcFs.getFileStatus(path).getLen();
    offset = (fileSize > offset) ? offset: fileSize;
    try {
      Thread.sleep(5000);
    } catch (InterruptedException e) {
      break;
    }
  }
}