org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner Java Examples

The following examples show how to use org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestStreamDataProtocol.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
protected String[] genArgs() {
  return new String[] {
    "-input", INPUT_FILE.getAbsolutePath(),
    "-output", OUTPUT_DIR.getAbsolutePath(),
    "-mapper", map,
    "-reducer", reduce,
    "-partitioner", KeyFieldBasedPartitioner.class.getCanonicalName(),
    "-jobconf", "stream.map.output.field.separator=.",
    "-jobconf", "stream.num.map.output.key.fields=2",
    "-jobconf", "mapreduce.map.output.key.field.separator=.",
    "-jobconf", "num.key.fields.for.partition=1",
    "-jobconf", "mapreduce.job.reduces=2",
    "-jobconf", "mapreduce.task.files.preserve.failedtasks=true",
    "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp")
  };
}
Example #2
Source File: TestStreamDataProtocol.java    From big-c with Apache License 2.0 6 votes vote down vote up 
protected String[] genArgs() {
  return new String[] {
    "-input", INPUT_FILE.getAbsolutePath(),
    "-output", OUTPUT_DIR.getAbsolutePath(),
    "-mapper", map,
    "-reducer", reduce,
    "-partitioner", KeyFieldBasedPartitioner.class.getCanonicalName(),
    "-jobconf", "stream.map.output.field.separator=.",
    "-jobconf", "stream.num.map.output.key.fields=2",
    "-jobconf", "mapreduce.map.output.key.field.separator=.",
    "-jobconf", "num.key.fields.for.partition=1",
    "-jobconf", "mapreduce.job.reduces=2",
    "-jobconf", "mapreduce.task.files.preserve.failedtasks=true",
    "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp")
  };
}
Example #3
Source File: TestStreamDataProtocol.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
protected String[] genArgs() {
  return new String[] {
    "-input", INPUT_FILE.getAbsolutePath(),
    "-output", OUTPUT_DIR.getAbsolutePath(),
    "-mapper", map,
    "-reducer", reduce,
    "-partitioner", KeyFieldBasedPartitioner.class.getCanonicalName(),
    //"-verbose",
    "-jobconf", "stream.map.output.field.separator=.",
    "-jobconf", "stream.num.map.output.key.fields=2",
    "-jobconf", "map.output.key.field.separator=.",
    "-jobconf", "num.key.fields.for.partition=1",
    "-jobconf", "mapred.reduce.tasks=2",
    "-jobconf", "keep.failed.task.files=true",
    "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp")
  };
}
Example #4
Source File: TestStreamDataProtocol.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
protected String[] genArgs() {
  return new String[] {
    "-input", INPUT_FILE.getAbsolutePath(),
    "-output", OUTPUT_DIR.getAbsolutePath(),
    "-mapper", map,
    "-reducer", reduce,
    "-partitioner", KeyFieldBasedPartitioner.class.getCanonicalName(),
    //"-verbose",
    "-jobconf", "stream.map.output.field.separator=.",
    "-jobconf", "stream.num.map.output.key.fields=2",
    "-jobconf", "map.output.key.field.separator=.",
    "-jobconf", "num.key.fields.for.partition=1",
    "-jobconf", "mapred.reduce.tasks=2",
    "-jobconf", "keep.failed.task.files=true",
    "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp")
  };
}
Example #5
Source File: TestStreamDataProtocol.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
protected String[] genArgs() {
  return new String[] {
    "-input", INPUT_FILE.getAbsolutePath(),
    "-output", OUTPUT_DIR.getAbsolutePath(),
    "-mapper", map,
    "-reducer", reduce,
    "-partitioner", KeyFieldBasedPartitioner.class.getCanonicalName(),
    //"-verbose",
    "-jobconf", "stream.map.output.field.separator=.",
    "-jobconf", "stream.num.map.output.key.fields=2",
    "-jobconf", "map.output.key.field.separator=.",
    "-jobconf", "num.key.fields.for.partition=1",
    "-jobconf", "mapred.reduce.tasks=2",
    "-jobconf", "keep.failed.task.files=true",
    "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp")
  };
}
Example #6
Source File: JobConf.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Get the {@link KeyFieldBasedPartitioner} options
 */
public String getKeyFieldPartitionerOption() {
  return get(KeyFieldBasedPartitioner.PARTITIONER_OPTIONS);
}
Example #7
Source File: JobConf.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Get the {@link KeyFieldBasedPartitioner} options
 */
public String getKeyFieldPartitionerOption() {
  return get(KeyFieldBasedPartitioner.PARTITIONER_OPTIONS);
}
Example #8
Source File: JobConf.java    From hadoop with Apache License 2.0 2 votes vote down vote up 
/**
 * Set the {@link KeyFieldBasedPartitioner} options used for 
 * {@link Partitioner}
 * 
 * @param keySpec the key specification of the form -k pos1[,pos2], where,
 *  pos is of the form f[.c][opts], where f is the number
 *  of the key field to use, and c is the number of the first character from
 *  the beginning of the field. Fields and character posns are numbered 
 *  starting with 1; a character position of zero in pos2 indicates the
 *  field's last character. If '.c' is omitted from pos1, it defaults to 1
 *  (the beginning of the field); if omitted from pos2, it defaults to 0 
 *  (the end of the field).
 */
public void setKeyFieldPartitionerOptions(String keySpec) {
  setPartitionerClass(KeyFieldBasedPartitioner.class);
  set(KeyFieldBasedPartitioner.PARTITIONER_OPTIONS, keySpec);
}
Example #9
Source File: JobConf.java    From big-c with Apache License 2.0 2 votes vote down vote up 
/**
 * Set the {@link KeyFieldBasedPartitioner} options used for 
 * {@link Partitioner}
 * 
 * @param keySpec the key specification of the form -k pos1[,pos2], where,
 *  pos is of the form f[.c][opts], where f is the number
 *  of the key field to use, and c is the number of the first character from
 *  the beginning of the field. Fields and character posns are numbered 
 *  starting with 1; a character position of zero in pos2 indicates the
 *  field's last character. If '.c' is omitted from pos1, it defaults to 1
 *  (the beginning of the field); if omitted from pos2, it defaults to 0 
 *  (the end of the field).
 */
public void setKeyFieldPartitionerOptions(String keySpec) {
  setPartitionerClass(KeyFieldBasedPartitioner.class);
  set(KeyFieldBasedPartitioner.PARTITIONER_OPTIONS, keySpec);
}
Example #10
Source File: JobConf.java    From RDFS with Apache License 2.0 2 votes vote down vote up 
/**
 * Set the {@link KeyFieldBasedPartitioner} options used for
 * {@link Partitioner}
 *
 * @param keySpec the key specification of the form -k pos1[,pos2], where,
 *  pos is of the form f[.c][opts], where f is the number
 *  of the key field to use, and c is the number of the first character from
 *  the beginning of the field. Fields and character posns are numbered
 *  starting with 1; a character position of zero in pos2 indicates the
 *  field's last character. If '.c' is omitted from pos1, it defaults to 1
 *  (the beginning of the field); if omitted from pos2, it defaults to 0
 *  (the end of the field).
 */
public void setKeyFieldPartitionerOptions(String keySpec) {
  setPartitionerClass(KeyFieldBasedPartitioner.class);
  set("mapred.text.key.partitioner.options", keySpec);
}
Example #11
Source File: JobConf.java    From hadoop-gpu with Apache License 2.0 2 votes vote down vote up 
/**
 * Set the {@link KeyFieldBasedPartitioner} options used for 
 * {@link Partitioner}
 * 
 * @param keySpec the key specification of the form -k pos1[,pos2], where,
 *  pos is of the form f[.c][opts], where f is the number
 *  of the key field to use, and c is the number of the first character from
 *  the beginning of the field. Fields and character posns are numbered 
 *  starting with 1; a character position of zero in pos2 indicates the
 *  field's last character. If '.c' is omitted from pos1, it defaults to 1
 *  (the beginning of the field); if omitted from pos2, it defaults to 0 
 *  (the end of the field).
 */
public void setKeyFieldPartitionerOptions(String keySpec) {
  setPartitionerClass(KeyFieldBasedPartitioner.class);
  set("mapred.text.key.partitioner.options", keySpec);
}