Java Code Examples for org.apache.hadoop.io.LongWritable

The following are top voted examples for showing how to use org.apache.hadoop.io.LongWritable. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: hadoop   File: WordMean.java   Source Code and License 6 votes vote down vote up
@Override
public int run(String[] args) throws Exception {
  if (args.length != 2) {
    System.err.println("Usage: wordmean <in> <out>");
    return 0;
  }

  Configuration conf = getConf();

  Job job = Job.getInstance(conf, "word mean");
  job.setJarByClass(WordMean.class);
  job.setMapperClass(WordMeanMapper.class);
  job.setCombinerClass(WordMeanReducer.class);
  job.setReducerClass(WordMeanReducer.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(LongWritable.class);
  FileInputFormat.addInputPath(job, new Path(args[0]));
  Path outputpath = new Path(args[1]);
  FileOutputFormat.setOutputPath(job, outputpath);
  boolean result = job.waitForCompletion(true);
  mean = readAndCalcMean(outputpath, conf);

  return (result ? 0 : 1);
}
 
Example 2
Project: hadoop   File: TestBadRecords.java   Source Code and License 6 votes vote down vote up
public void map(LongWritable key, Text val,
    OutputCollector<LongWritable, Text> output, Reporter reporter)
    throws IOException {
  String str = val.toString();
  LOG.debug("MAP key:" +key +"  value:" + str);
  if(MAPPER_BAD_RECORDS.get(0).equals(str)) {
    LOG.warn("MAP Encountered BAD record");
    System.exit(-1);
  }
  else if(MAPPER_BAD_RECORDS.get(1).equals(str)) {
    LOG.warn("MAP Encountered BAD record");
    throw new RuntimeException("Bad record "+str);
  }
  else if(MAPPER_BAD_RECORDS.get(2).equals(str)) {
    try {
      LOG.warn("MAP Encountered BAD record");
      Thread.sleep(15*60*1000);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }
  }
  output.collect(key, val);
}
 
Example 3
Project: hadoop   File: TestIPC.java   Source Code and License 6 votes vote down vote up
@Test(timeout=60000)
public void testStandAloneClient() throws IOException {
  Client client = new Client(LongWritable.class, conf);
  InetSocketAddress address = new InetSocketAddress("127.0.0.1", 10);
  try {
    client.call(new LongWritable(RANDOM.nextLong()),
            address, null, null, 0, conf);
    fail("Expected an exception to have been thrown");
  } catch (IOException e) {
    String message = e.getMessage();
    String addressText = address.getHostName() + ":" + address.getPort();
    assertTrue("Did not find "+addressText+" in "+message,
            message.contains(addressText));
    Throwable cause=e.getCause();
    assertNotNull("No nested exception in "+e,cause);
    String causeText=cause.getMessage();
    assertTrue("Did not find " + causeText + " in " + message,
            message.contains(causeText));
  }
}
 
Example 4
Project: MRNMF   File: MM2.java   Source Code and License 6 votes vote down vote up
@Override
protected void reduce(LongWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {

    int k = context.getConfiguration().getInt("k", -1);

    double[] result = new double[k];

    for (Text value : values) {
        String[] ai = value.toString().split(",");
        for (int j = 0; j < k; j++) {
            result[j] += Double.parseDouble(ai[j]);
        }
    }

    StringBuilder res = new StringBuilder(prefix);

    for (int i = 0; i < k; i++) {
        res.append(result[i]);
        if (i < k - 1) {
            res.append(",");
        }
    }
    context.write(key, new Text(res.toString()));
}
 
Example 5
Project: hadoop   File: TestLocalRunner.java   Source Code and License 6 votes vote down vote up
public void map(LongWritable key, Text val, Context c)
    throws IOException, InterruptedException {

  // Create a whole bunch of objects.
  List<Integer> lst = new ArrayList<Integer>();
  for (int i = 0; i < 20000; i++) {
    lst.add(new Integer(i));
  }

  // Actually use this list, to ensure that it isn't just optimized away.
  int sum = 0;
  for (int x : lst) {
    sum += x;
  }

  // throw away the list and run a GC.
  lst = null;
  System.gc();

  c.write(new LongWritable(sum), val);
}
 
Example 6
Project: hadoop   File: TestDFSIO.java   Source Code and License 6 votes vote down vote up
private void runIOTest(
        Class<? extends Mapper<Text, LongWritable, Text, Text>> mapperClass, 
        Path outputDir) throws IOException {
  JobConf job = new JobConf(config, TestDFSIO.class);

  FileInputFormat.setInputPaths(job, getControlDir(config));
  job.setInputFormat(SequenceFileInputFormat.class);

  job.setMapperClass(mapperClass);
  job.setReducerClass(AccumulatingReducer.class);

  FileOutputFormat.setOutputPath(job, outputDir);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);
  job.setNumReduceTasks(1);
  JobClient.runJob(job);
}
 
Example 7
Project: hadoop-oss   File: TestIPC.java   Source Code and License 6 votes vote down vote up
@Override
public void run() {
  for (int i = 0; i < count; i++) {
    try {
      final long param = RANDOM.nextLong();
      LongWritable value = call(client, param, server, conf);
      if (value.get() != param) {
        LOG.fatal("Call failed!");
        failed = true;
        break;
      }
    } catch (Exception e) {
      LOG.fatal("Caught: " + StringUtils.stringifyException(e));
      failed = true;
    }
  }
}
 
Example 8
Project: hadoop-oss   File: TestIPC.java   Source Code and License 6 votes vote down vote up
@Test(timeout=60000)
public void testStandAloneClient() throws IOException {
  Client client = new Client(LongWritable.class, conf);
  InetSocketAddress address = new InetSocketAddress("127.0.0.1", 10);
  try {
    call(client, RANDOM.nextLong(), address, conf);
    fail("Expected an exception to have been thrown");
  } catch (IOException e) {
    String message = e.getMessage();
    String addressText = address.getHostName() + ":" + address.getPort();
    assertTrue("Did not find "+addressText+" in "+message,
            message.contains(addressText));
    Throwable cause=e.getCause();
    assertNotNull("No nested exception in "+e,cause);
    String causeText=cause.getMessage();
    assertTrue("Did not find " + causeText + " in " + message,
            message.contains(causeText));
  } finally {
    client.stop();
  }
}
 
Example 9
Project: hadoop-oss   File: TestIPC.java   Source Code and License 6 votes vote down vote up
@Test(timeout=60000)
public void testIpcConnectTimeout() throws IOException {
  // start server
  Server server = new TestServer(1, true);
  InetSocketAddress addr = NetUtils.getConnectAddress(server);
  //Intentionally do not start server to get a connection timeout

  // start client
  Client.setConnectTimeout(conf, 100);
  Client client = new Client(LongWritable.class, conf);
  // set the rpc timeout to twice the MIN_SLEEP_TIME
  try {
    call(client, new LongWritable(RANDOM.nextLong()), addr,
        MIN_SLEEP_TIME * 2, conf);
    fail("Expected an exception to have been thrown");
  } catch (SocketTimeoutException e) {
    LOG.info("Get a SocketTimeoutException ", e);
  }
  client.stop();
}
 
Example 10
Project: hadoop   File: GenericMRLoadGenerator.java   Source Code and License 6 votes vote down vote up
public List<InputSplit> getSplits(JobContext job)
    throws IOException {

  Configuration conf = job.getConfiguration();
  Path src = new Path(conf.get(INDIRECT_INPUT_FILE, null));
  FileSystem fs = src.getFileSystem(conf);

  List<InputSplit> splits = new ArrayList<InputSplit>();
  LongWritable key = new LongWritable();
  Text value = new Text();
  for (SequenceFile.Reader sl = new SequenceFile.Reader(fs, src, conf);
       sl.next(key, value);) {
    splits.add(new IndirectSplit(new Path(value.toString()), key.get()));
  }

  return splits;
}
 
Example 11
Project: hadoop   File: TestChainErrors.java   Source Code and License 6 votes vote down vote up
/**
 * Tests reducer consuming output.
 * 
 * @throws Exception
 */
public void testChainReduceNoOuptut() throws Exception {
  Configuration conf = createJobConf();
  String expectedOutput = "";

  Job job = MapReduceTestUtil.createJob(conf, inDir, outDir, 1, 1, input);
  job.setJobName("chain");

  ChainMapper.addMapper(job, Mapper.class, IntWritable.class, Text.class,
      LongWritable.class, Text.class, null);

  ChainReducer.setReducer(job, ConsumeReduce.class, LongWritable.class,
      Text.class, LongWritable.class, Text.class, null);

  ChainReducer.addMapper(job, Mapper.class, LongWritable.class, Text.class,
      LongWritable.class, Text.class, null);

  job.waitForCompletion(true);
  assertTrue("Job failed", job.isSuccessful());
  assertEquals("Outputs doesn't match", expectedOutput, MapReduceTestUtil
      .readOutput(outDir, conf));
}
 
Example 12
Project: hadoop   File: TestChainErrors.java   Source Code and License 6 votes vote down vote up
/**
 * Tests one of the mappers throwing exception.
 * 
 * @throws Exception
 */
public void testChainFail() throws Exception {

  Configuration conf = createJobConf();

  Job job = MapReduceTestUtil.createJob(conf, inDir, outDir, 1, 0, input);
  job.setJobName("chain");

  ChainMapper.addMapper(job, Mapper.class, LongWritable.class, Text.class,
      LongWritable.class, Text.class, null);

  ChainMapper.addMapper(job, FailMap.class, LongWritable.class, Text.class,
      IntWritable.class, Text.class, null);

  ChainMapper.addMapper(job, Mapper.class, IntWritable.class, Text.class,
      LongWritable.class, Text.class, null);

  job.waitForCompletion(true);
  assertTrue("Job Not failed", !job.isSuccessful());
}
 
Example 13
Project: hadoop   File: TestFileSystem.java   Source Code and License 6 votes vote down vote up
public static void seekTest(FileSystem fs, boolean fastCheck)
  throws Exception {

  fs.delete(READ_DIR, true);

  JobConf job = new JobConf(conf, TestFileSystem.class);
  job.setBoolean("fs.test.fastCheck", fastCheck);

  FileInputFormat.setInputPaths(job,CONTROL_DIR);
  job.setInputFormat(SequenceFileInputFormat.class);

  job.setMapperClass(SeekMapper.class);
  job.setReducerClass(LongSumReducer.class);

  FileOutputFormat.setOutputPath(job, READ_DIR);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(LongWritable.class);
  job.setNumReduceTasks(1);
  JobClient.runJob(job);
}
 
Example 14
Project: hadoop   File: TestJoinTupleWritable.java   Source Code and License 6 votes vote down vote up
public void testNestedIterable() throws Exception {
  Random r = new Random();
  Writable[] writs = {
    new BooleanWritable(r.nextBoolean()),
    new FloatWritable(r.nextFloat()),
    new FloatWritable(r.nextFloat()),
    new IntWritable(r.nextInt()),
    new LongWritable(r.nextLong()),
    new BytesWritable("dingo".getBytes()),
    new LongWritable(r.nextLong()),
    new IntWritable(r.nextInt()),
    new BytesWritable("yak".getBytes()),
    new IntWritable(r.nextInt())
  };
  TupleWritable sTuple = makeTuple(writs);
  assertTrue("Bad count", writs.length == verifIter(writs, sTuple, 0));
}
 
Example 15
Project: hadoop   File: CompressionEmulationUtil.java   Source Code and License 6 votes vote down vote up
/**
 * Emits random words sequence of desired size. Note that the desired output
 * size is passed as the value parameter to this map.
 */
@Override
public void map(NullWritable key, LongWritable value, Context context)
throws IOException, InterruptedException {
  //TODO Control the extra data written ..
  //TODO Should the key\tvalue\n be considered for measuring size?
  //     Can counters like BYTES_WRITTEN be used? What will be the value of
  //     such counters in LocalJobRunner?
  for (long bytes = value.get(); bytes > 0;) {
    String randomKey = rtg.getRandomWord();
    String randomValue = rtg.getRandomWord();
    context.write(new Text(randomKey), new Text(randomValue));
    bytes -= (randomValue.getBytes(charsetUTF8).length +
        randomKey.getBytes(charsetUTF8).length);
  }
}
 
Example 16
Project: Wikipedia-Index   File: TF.java   Source Code and License 6 votes vote down vote up
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
	String doc = value.toString();
				
	String text = slice(doc, "<text", "</text>", true);
	if (text.length() < 1) return;
	
	char txt[] = text.toLowerCase().toCharArray();
	for (int i = 0; i < txt.length; ++i) {
		if (!((txt[i] >= 'a' && txt[i] <= 'z') || (txt[i] >= 'A' && txt[i] <= 'Z')))
			txt[i] = ' ';
	}
	
	String id = slice(doc, "<id>", "</id>", false);
	if (id.length() < 1) return;
	StringTokenizer itr = new StringTokenizer(String.valueOf(txt));
	int sum = itr.countTokens();
	while (itr.hasMoreTokens()) {
		String s = itr.nextToken();
		word.set(id + '-' + s);
		IntWritable tmp[] = {new IntWritable(sum), new IntWritable(1)};
		IntArrayWritable temp = new IntArrayWritable(tmp);
		context.write(word, temp);
	}
}
 
Example 17
Project: aliyun-maxcompute-data-collectors   File: CombineShimRecordReader.java   Source Code and License 6 votes vote down vote up
/**
 * Actually instantiate the user's chosen RecordReader implementation.
 */
@SuppressWarnings("unchecked")
private void createChildReader() throws IOException, InterruptedException {
  LOG.debug("ChildSplit operates on: " + split.getPath(index));

  Configuration conf = context.getConfiguration();

  // Determine the file format we're reading.
  Class rrClass;
  if (ExportJobBase.isSequenceFiles(conf, split.getPath(index))) {
    rrClass = SequenceFileRecordReader.class;
  } else {
    rrClass = LineRecordReader.class;
  }

  // Create the appropriate record reader.
  this.rr = (RecordReader<LongWritable, Object>)
      ReflectionUtils.newInstance(rrClass, conf);
}
 
Example 18
Project: hadoop   File: TestDBInputFormat.java   Source Code and License 6 votes vote down vote up
/**
 * test DBInputFormat class. Class should split result for chunks
 * @throws Exception
 */
@Test(timeout = 10000)
public void testDBInputFormat() throws Exception {
  JobConf configuration = new JobConf();
  setupDriver(configuration);
  
  DBInputFormat<NullDBWritable> format = new DBInputFormat<NullDBWritable>();
  format.setConf(configuration);
  format.setConf(configuration);
  DBInputFormat.DBInputSplit splitter = new DBInputFormat.DBInputSplit(1, 10);
  Reporter reporter = mock(Reporter.class);
  RecordReader<LongWritable, NullDBWritable> reader = format.getRecordReader(
      splitter, configuration, reporter);

  configuration.setInt(MRJobConfig.NUM_MAPS, 3);
  InputSplit[] lSplits = format.getSplits(configuration, 3);
  assertEquals(5, lSplits[0].getLength());
  assertEquals(3, lSplits.length);

  // test reader .Some simple tests
  assertEquals(LongWritable.class, reader.createKey().getClass());
  assertEquals(0, reader.getPos());
  assertEquals(0, reader.getProgress(), 0.001);
  reader.close();
}
 
Example 19
Project: MRNMF   File: MatrixUpdater.java   Source Code and License 5 votes vote down vote up
public void map(LongWritable key, Text value, Context context)
        throws IOException, NumberFormatException, InterruptedException {

    String[] vals = value.toString().split("\t");
    if (!vals[1].contains(":")) {
        vals[1] = "m:" + vals[1];
    }
    context.write(new LongWritable(Long.parseLong(vals[0])), new Text(vals[1]));

}
 
Example 20
Project: MRNMF   File: MatrixUpdater.java   Source Code and License 5 votes vote down vote up
public void reduce(LongWritable key, Iterable<Text> values,
                   Context context) throws IOException, InterruptedException {

    boolean sqrt = context.getConfiguration().getBoolean("sqrt", false);

    StringBuilder result = new StringBuilder();

    String[] arrayNames = new String[] {"m", "a", "b"};
    Map<String, double[]> arrays = new HashMap<>();
    for (String arrayName : arrayNames) {
        arrays.put(arrayName, new double[k]);
    }

    for (Text value : values) {
        String[] keyVal = value.toString().split(":");
        String[] xi = keyVal[1].split(",");
        for (int j = 0; j < k; j++) {
            arrays.get(keyVal[0])[j] = Double.parseDouble(xi[j]);
        }
    }

    for (int j = 0; j < k; j++) {
        double frac = arrays.get("a")[j] / arrays.get("b")[j];
        if (sqrt) {
            frac = Math.sqrt(frac);
        }
        result.append(arrays.get("m")[j] * frac);
        if (j != k - 1)
            result.append(",");
    }

    context.write(key, new Text(result.toString()));
}
 
Example 21
Project: Hydrograph   File: DelimitedAndFixedWidthInputFormat.java   Source Code and License 5 votes vote down vote up
@Override
public RecordReader<LongWritable, Text> getRecordReader(
		InputSplit genericSplit, JobConf job, Reporter reporter)
		throws IOException {

	reporter.setStatus(genericSplit.toString());
	return new DelimitedAndFixedWidthRecordReader(job,
			(FileSplit) genericSplit);
}
 
Example 22
Project: hadoop   File: TestValueIterReset.java   Source Code and License 5 votes vote down vote up
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {

  IntWritable outKey = new IntWritable();
  IntWritable outValue = new IntWritable();

  for (int j = 0; j < NUM_TESTS; j++) {
    for (int i = 0; i < NUM_VALUES; i++) {
      outKey.set(j);
      outValue.set(i);
      context.write(outKey, outValue);
    }
  }
}
 
Example 23
Project: hadoop   File: FailJob.java   Source Code and License 5 votes vote down vote up
public void map(LongWritable key, Text value, Context context
           ) throws IOException, InterruptedException {
  if (context.getConfiguration().getBoolean(FAIL_MAP, true)) {
    throw new RuntimeException("Intentional map failure");
  }
  context.write(key, NullWritable.get());
}
 
Example 24
Project: hadoop   File: BaileyBorweinPlouffe.java   Source Code and License 5 votes vote down vote up
/** Create and setup a job */
private static Job createJob(String name, Configuration conf
    ) throws IOException {
  final Job job = Job.getInstance(conf, NAME + "_" + name);
  final Configuration jobconf = job.getConfiguration();
  job.setJarByClass(BaileyBorweinPlouffe.class);

  // setup mapper
  job.setMapperClass(BbpMapper.class);
  job.setMapOutputKeyClass(LongWritable.class);
  job.setMapOutputValueClass(BytesWritable.class);

  // setup reducer
  job.setReducerClass(BbpReducer.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(BytesWritable.class);
  job.setNumReduceTasks(1);

  // setup input
  job.setInputFormatClass(BbpInputFormat.class);

  // disable task timeout
  jobconf.setLong(MRJobConfig.TASK_TIMEOUT, 0);

  // do not use speculative execution
  jobconf.setBoolean(MRJobConfig.MAP_SPECULATIVE, false);
  jobconf.setBoolean(MRJobConfig.REDUCE_SPECULATIVE, false);
  return job;
}
 
Example 25
Project: aliyun-maxcompute-data-collectors   File: TextImportMapper.java   Source Code and License 5 votes vote down vote up
@Override
public void map(LongWritable key, SqoopRecord val, Context context)
    throws IOException, InterruptedException {

  try {
    // Loading of LOBs was delayed until we have a Context.
    val.loadLargeObjects(lobLoader);
  } catch (SQLException sqlE) {
    throw new IOException(sqlE);
  }

  outkey.set(val.toString());
  context.write(outkey, NullWritable.get());
}
 
Example 26
Project: hadoop   File: TestLineRecordReader.java   Source Code and License 5 votes vote down vote up
public ArrayList<String> readRecords(URL testFileUrl, int splitSize)
    throws IOException {

  // Set up context
  File testFile = new File(testFileUrl.getFile());
  long testFileSize = testFile.length();
  Path testFilePath = new Path(testFile.getAbsolutePath());
  Configuration conf = new Configuration();
  conf.setInt("io.file.buffer.size", 1);

  // Gather the records returned by the record reader
  ArrayList<String> records = new ArrayList<String>();

  long offset = 0;
  LongWritable key = new LongWritable();
  Text value = new Text();
  while (offset < testFileSize) {
    FileSplit split =
        new FileSplit(testFilePath, offset, splitSize, (String[]) null);
    LineRecordReader reader = new LineRecordReader(conf, split);

    while (reader.next(key, value)) {
      records.add(value.toString());
    }
    offset += splitSize;
  }
  return records;
}
 
Example 27
Project: hadoop   File: TestDFSIO.java   Source Code and License 5 votes vote down vote up
@SuppressWarnings("deprecation")
private void createControlFile(FileSystem fs,
                                long nrBytes, // in bytes
                                int nrFiles
                              ) throws IOException {
  LOG.info("creating control file: "+nrBytes+" bytes, "+nrFiles+" files");

  Path controlDir = getControlDir(config);
  fs.delete(controlDir, true);

  for(int i=0; i < nrFiles; i++) {
    String name = getFileName(i);
    Path controlFile = new Path(controlDir, "in_file_" + name);
    SequenceFile.Writer writer = null;
    try {
      writer = SequenceFile.createWriter(fs, config, controlFile,
                                         Text.class, LongWritable.class,
                                         CompressionType.NONE);
      writer.append(new Text(name), new LongWritable(nrBytes));
    } catch(Exception e) {
      throw new IOException(e.getLocalizedMessage());
    } finally {
      if (writer != null)
        writer.close();
      writer = null;
    }
  }
  LOG.info("created control files for: "+nrFiles+" files");
}
 
Example 28
Project: hadoop   File: QuasiMonteCarlo.java   Source Code and License 5 votes vote down vote up
/**
 * Reduce task done, write output to a file.
 */
@Override
public void cleanup(Context context) throws IOException {
  //write output to a file
  Configuration conf = context.getConfiguration();
  Path outDir = new Path(conf.get(FileOutputFormat.OUTDIR));
  Path outFile = new Path(outDir, "reduce-out");
  FileSystem fileSys = FileSystem.get(conf);
  SequenceFile.Writer writer = SequenceFile.createWriter(fileSys, conf,
      outFile, LongWritable.class, LongWritable.class, 
      CompressionType.NONE);
  writer.append(new LongWritable(numInside), new LongWritable(numOutside));
  writer.close();
}
 
Example 29
Project: hadoop   File: TestJoinProperties.java   Source Code and License 5 votes vote down vote up
private static SequenceFile.Writer[] createWriters(Path testdir,
    Configuration conf, int srcs, Path[] src) throws IOException {
  for (int i = 0; i < srcs; ++i) {
    src[i] = new Path(testdir, Integer.toString(i + 10, 36));
  }
  SequenceFile.Writer out[] = new SequenceFile.Writer[srcs];
  for (int i = 0; i < srcs - 1; ++i) {
    out[i] = new SequenceFile.Writer(testdir.getFileSystem(conf), conf,
        src[i], IntWritable.class, IntWritable.class);
  }
  out[srcs - 1] = new SequenceFile.Writer(testdir.getFileSystem(conf), conf,
          src[srcs - 1], IntWritable.class, LongWritable.class);
  return out;
}
 
Example 30
Project: hadoop   File: FixedLengthInputFormat.java   Source Code and License 5 votes vote down vote up
@Override
public RecordReader<LongWritable, BytesWritable>
    createRecordReader(InputSplit split, TaskAttemptContext context)
    throws IOException, InterruptedException {
  int recordLength = getRecordLength(context.getConfiguration());
  if (recordLength <= 0) {
    throw new IOException("Fixed record length " + recordLength
        + " is invalid.  It should be set to a value greater than zero");
  }
  return new FixedLengthRecordReader(recordLength);
}
 
Example 31
Project: hadoop-oss   File: TestIPC.java   Source Code and License 5 votes vote down vote up
/**
 * Generic test case for exceptions thrown at some point in the IPC
 * process.
 * 
 * @param clientParamClass - client writes this writable for parameter
 * @param serverParamClass - server reads this writable for parameter
 * @param serverResponseClass - server writes this writable for response
 * @param clientResponseClass - client reads this writable for response
 */
private void doErrorTest(
    Class<? extends LongWritable> clientParamClass,
    Class<? extends LongWritable> serverParamClass,
    Class<? extends LongWritable> serverResponseClass,
    Class<? extends LongWritable> clientResponseClass) 
    throws IOException, InstantiationException, IllegalAccessException {
  
  // start server
  Server server = new TestServer(1, false,
      serverParamClass, serverResponseClass);
  InetSocketAddress addr = NetUtils.getConnectAddress(server);
  server.start();

  // start client
  WRITABLE_FAULTS_ENABLED = true;
  Client client = new Client(clientResponseClass, conf);
  try {
    LongWritable param = clientParamClass.newInstance();

    try {
      call(client, param, addr, 0, conf);
      fail("Expected an exception to have been thrown");
    } catch (Throwable t) {
      assertExceptionContains(t, "Injected fault");
    }
    
    // Doing a second call with faults disabled should return fine --
    // ie the internal state of the client or server should not be broken
    // by the failed call
    WRITABLE_FAULTS_ENABLED = false;
    call(client, param, addr, 0, conf);
    
  } finally {
    client.stop();
    server.stop();
  }
}
 
Example 32
Project: hadoop   File: TestIPC.java   Source Code and License 5 votes vote down vote up
@Test(timeout=60000)
public void testIOEOnClientReadResponse() throws Exception {
  doErrorTest(LongWritable.class,
      LongWritable.class,
      LongWritable.class,
      IOEOnReadWritable.class);
}
 
Example 33
Project: hadoop-oss   File: TestIPC.java   Source Code and License 5 votes vote down vote up
@Test(timeout=60000)
public void testRTEOnClientWriteParam() throws Exception {
  doErrorTest(RTEOnWriteWritable.class,
      LongWritable.class,
      LongWritable.class,
      LongWritable.class);
}
 
Example 34
Project: spark-util   File: AtlasTextInputFileFormat.java   Source Code and License 5 votes vote down vote up
public RecordReader<LongWritable, Text> createRecordReader(InputSplit split, TaskAttemptContext context) {
    String delimiter = context.getConfiguration().get("textinputformat.record.delimiter");
    byte[] recordDelimiterBytes = null;
    if(null != delimiter) {
        recordDelimiterBytes = delimiter.getBytes(Charsets.UTF_8);
    }
    return new ErrorHandlingLineRecordReader(recordDelimiterBytes);

}
 
Example 35
Project: WIFIProbe   File: MapperWriter.java   Source Code and License 5 votes vote down vote up
private void writNewOldCustomer() throws IOException, InterruptedException {

        KeyWrapper newOldKey = new KeyWrapper();
        newOldKey.setType(new Text(MapKeyConfig.NEW_OLD_CUSTOMER));

        LongWritable longWritable = new LongWritable();
        newOldKey.setMillisTime(longWritable);

        for (NewOldCustomElement newOldCustomElement : statistic.getNewOldCustomElements()) {
            longWritable.set(newOldCustomElement.getHour());
            context.write(newOldKey, new ValueWrapper(newOldCustomElement));
        }
    }
 
Example 36
Project: hadoop-oss   File: TestIPC.java   Source Code and License 5 votes vote down vote up
@Test(timeout=60000)
public void testIOEOnServerWriteResponse() throws Exception {
  doErrorTest(LongWritable.class,
      LongWritable.class,
      IOEOnWriteWritable.class,
      LongWritable.class);
}
 
Example 37
Project: hadoop   File: TestLineRecordReader.java   Source Code and License 5 votes vote down vote up
@Test
public void testStripBOM() throws IOException {
  // the test data contains a BOM at the start of the file
  // confirm the BOM is skipped by LineRecordReader
  String UTF8_BOM = "\uFEFF";
  URL testFileUrl = getClass().getClassLoader().getResource("testBOM.txt");
  assertNotNull("Cannot find testBOM.txt", testFileUrl);
  File testFile = new File(testFileUrl.getFile());
  Path testFilePath = new Path(testFile.getAbsolutePath());
  long testFileSize = testFile.length();
  Configuration conf = new Configuration();
  conf.setInt(org.apache.hadoop.mapreduce.lib.input.
      LineRecordReader.MAX_LINE_LENGTH, Integer.MAX_VALUE);

  // read the data and check whether BOM is skipped
  FileSplit split = new FileSplit(testFilePath, 0, testFileSize,
      (String[])null);
  LineRecordReader reader = new LineRecordReader(conf, split);
  LongWritable key = new LongWritable();
  Text value = new Text();
  int numRecords = 0;
  boolean firstLine = true;
  boolean skipBOM = true;
  while (reader.next(key, value)) {
    if (firstLine) {
      firstLine = false;
      if (value.toString().startsWith(UTF8_BOM)) {
        skipBOM = false;
      }
    }
    ++numRecords;
  }
  reader.close();

  assertTrue("BOM is not skipped", skipBOM);
}
 
Example 38
Project: hadoop-oss   File: TestIPC.java   Source Code and License 5 votes vote down vote up
@Test(timeout=60000)
public void testRTEOnClientReadResponse() throws Exception {
  doErrorTest(LongWritable.class,
      LongWritable.class,
      LongWritable.class,
      RTEOnReadWritable.class);
}
 
Example 39
Project: ditb   File: TestMapReduceExamples.java   Source Code and License 5 votes vote down vote up
/**
 * Test SampleUploader from examples
 */

@SuppressWarnings("unchecked")
@Test
public void testSampleUploader() throws Exception {

  Configuration configuration = new Configuration();
  Uploader uploader = new Uploader();
  Mapper<LongWritable, Text, ImmutableBytesWritable, Put>.Context ctx = mock(Context.class);
  doAnswer(new Answer<Void>() {

    @Override
    public Void answer(InvocationOnMock invocation) throws Throwable {
      ImmutableBytesWritable writer = (ImmutableBytesWritable) invocation.getArguments()[0];
      Put put = (Put) invocation.getArguments()[1];
      assertEquals("row", Bytes.toString(writer.get()));
      assertEquals("row", Bytes.toString(put.getRow()));
      return null;
    }
  }).when(ctx).write(any(ImmutableBytesWritable.class), any(Put.class));

  uploader.map(null, new Text("row,family,qualifier,value"), ctx);

  Path dir = util.getDataTestDirOnTestFS("testSampleUploader");

  String[] args = { dir.toString(), "simpleTable" };
  Job job = SampleUploader.configureJob(configuration, args);
  assertEquals(SequenceFileInputFormat.class, job.getInputFormatClass());

}
 
Example 40
Project: hadoop   File: TestCombineTextInputFormat.java   Source Code and License 5 votes vote down vote up
private static List<Text> readSplit(InputFormat<LongWritable,Text> format,
                                    InputSplit split,
                                    JobConf job) throws IOException {
  List<Text> result = new ArrayList<Text>();
  RecordReader<LongWritable, Text> reader =
    format.getRecordReader(split, job, voidReporter);
  LongWritable key = reader.createKey();
  Text value = reader.createValue();
  while (reader.next(key, value)) {
    result.add(value);
    value = reader.createValue();
  }
  reader.close();
  return result;
}
 
Example 41
Project: hadoop   File: TestJobCleanup.java   Source Code and License 5 votes vote down vote up
private void configureJob(JobConf jc, String jobName, int maps, int reds,
    Path outDir) {
  jc.setJobName(jobName);
  jc.setInputFormat(TextInputFormat.class);
  jc.setOutputKeyClass(LongWritable.class);
  jc.setOutputValueClass(Text.class);
  FileInputFormat.setInputPaths(jc, inDir);
  FileOutputFormat.setOutputPath(jc, outDir);
  jc.setMapperClass(IdentityMapper.class);
  jc.setReducerClass(IdentityReducer.class);
  jc.setNumMapTasks(maps);
  jc.setNumReduceTasks(reds);
}
 
Example 42
Project: Mastering-Apache-Storm   File: DefaultSequenceFormat.java   Source Code and License 5 votes vote down vote up
public Writable key(Tuple tuple) {
    if(this.key == null){
        this.key  = new LongWritable();
    }
    this.key.set(tuple.getLongByField(this.keyField));
    return this.key;
}
 
Example 43
Project: hadoop   File: TestGridMixClasses.java   Source Code and License 5 votes vote down vote up
@SuppressWarnings({"unchecked", "rawtypes"})
@Test (timeout=30000)
public void testSleepMapper() throws Exception {
  SleepJob.SleepMapper test = new SleepJob.SleepMapper();

  Configuration conf = new Configuration();
  conf.setInt(JobContext.NUM_REDUCES, 2);

  CompressionEmulationUtil.setCompressionEmulationEnabled(conf, true);
  conf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, true);
  TaskAttemptID taskId = new TaskAttemptID();
  FakeRecordLLReader reader = new FakeRecordLLReader();
  LoadRecordGkNullWriter writer = new LoadRecordGkNullWriter();
  OutputCommitter committer = new CustomOutputCommitter();
  StatusReporter reporter = new TaskAttemptContextImpl.DummyReporter();
  SleepSplit split = getSleepSplit();
  MapContext<LongWritable, LongWritable, GridmixKey, NullWritable> mapcontext = new MapContextImpl<LongWritable, LongWritable, GridmixKey, NullWritable>(
          conf, taskId, reader, writer, committer, reporter, split);
  Context context = new WrappedMapper<LongWritable, LongWritable, GridmixKey, NullWritable>()
          .getMapContext(mapcontext);

  long start = System.currentTimeMillis();
  LOG.info("start:" + start);
  LongWritable key = new LongWritable(start + 2000);
  LongWritable value = new LongWritable(start + 2000);
  // should slip 2 sec
  test.map(key, value, context);
  LOG.info("finish:" + System.currentTimeMillis());
  assertTrue(System.currentTimeMillis() >= (start + 2000));

  test.cleanup(context);
  assertEquals(1, writer.getData().size());
}
 
Example 44
Project: hadoop   File: TestTaskContext.java   Source Code and License 5 votes vote down vote up
@Override
protected void map(LongWritable key, Text value, 
    org.apache.hadoop.mapreduce.Mapper.Context context) 
throws IOException ,InterruptedException {
  // get the map phase progress
  float mapPhaseProgress = ((float)++recordCount)/INPUT_LINES;
  // get the weighted map phase progress
  float weightedMapProgress = progressRange * mapPhaseProgress;
  // check the map progress
  assertEquals("Invalid progress in map", 
               weightedMapProgress, context.getProgress(), 0f);
  
  context.write(new Text(value.toString() + recordCount), value);
}
 
Example 45
Project: hadoop   File: TestFixedLengthInputFormat.java   Source Code and License 5 votes vote down vote up
/**
 * Test with no record length set.
 */
@Test (timeout=5000)
public void testNoRecordLength() throws Exception {
  localFs.delete(workDir, true);
  Path file = new Path(workDir, new String("testFormat.txt"));
  createFile(file, null, 10, 10);
  // Create the job and do not set fixed record length
  Job job = Job.getInstance(defaultConf);
  FileInputFormat.setInputPaths(job, workDir);
  FixedLengthInputFormat format = new FixedLengthInputFormat();
  List<InputSplit> splits = format.getSplits(job);
  boolean exceptionThrown = false;
  for (InputSplit split : splits) {
    try {
      TaskAttemptContext context = MapReduceTestUtil.
          createDummyMapTaskAttemptContext(job.getConfiguration());
      RecordReader<LongWritable, BytesWritable> reader =
          format.createRecordReader(split, context);
      MapContext<LongWritable, BytesWritable, LongWritable, BytesWritable>
          mcontext =
          new MapContextImpl<LongWritable, BytesWritable, LongWritable,
          BytesWritable>(job.getConfiguration(), context.getTaskAttemptID(),
          reader, null, null, MapReduceTestUtil.createDummyReporter(), split);
      reader.initialize(split, mcontext);
    } catch(IOException ioe) {
      exceptionThrown = true;
      LOG.info("Exception message:" + ioe.getMessage());
    }
  }
  assertTrue("Exception for not setting record length:", exceptionThrown);
}
 
Example 46
Project: Wikipedia-Index   File: XmlInputFormat.java   Source Code and License 5 votes vote down vote up
@Override
public RecordReader<LongWritable, Text> createRecordReader(
        InputSplit inputSplit, TaskAttemptContext context) {
    try {
        return new XMLRecordReader(inputSplit, context.getConfiguration());
    } catch (IOException e) {
        return null;
    }
}
 
Example 47
Project: ditb   File: TestTableMapReduceUtil.java   Source Code and License 5 votes vote down vote up
@Test
public void testInitTableMapperJob2() throws Exception {
  Configuration configuration = new Configuration();
  Job job = new Job(configuration, "tableName");
  TableMapReduceUtil.initTableMapperJob(Bytes.toBytes("Table"), new Scan(),
      Import.Importer.class, Text.class, Text.class, job, false, WALInputFormat.class);
  assertEquals(WALInputFormat.class, job.getInputFormatClass());
  assertEquals(Import.Importer.class, job.getMapperClass());
  assertEquals(LongWritable.class, job.getOutputKeyClass());
  assertEquals(Text.class, job.getOutputValueClass());
  assertNull(job.getCombinerClass());
  assertEquals("Table", job.getConfiguration().get(TableInputFormat.INPUT_TABLE));
}
 
Example 48
Project: aliyun-maxcompute-data-collectors   File: SQLServerDatatypeExportSequenceFileManualTest.java   Source Code and License 5 votes vote down vote up
@Override
public void createFile(DATATYPES dt, String[] data) throws Exception {
  try {
    codeGen(dt);
    // Instantiate the value record object via reflection.
    Class cls = Class.forName(getTableName(dt), true, Thread
      .currentThread().getContextClassLoader());
    SqoopRecord record = (SqoopRecord) ReflectionUtils.newInstance(cls,
      new Configuration());

    // Create the SequenceFile.
    Configuration conf = new Configuration();
    String hdfsroot;
    hdfsroot = System.getProperty("ms.datatype.test.hdfsprefix");
    if (hdfsroot == null){
      hdfsroot ="hdfs://localhost/";
    }
    conf.set("fs.default.name", hdfsroot);
    FileSystem fs = FileSystem.get(conf);
    Path tablePath = getTablePath(dt);
    Path filePath = new Path(tablePath, getTableName(dt));
    fs.mkdirs(tablePath);
    SequenceFile.Writer w = SequenceFile.createWriter(fs, conf,
      filePath, LongWritable.class, cls);

    int cnt = 0;
    for (String tmp : data) {
      record.parse(tmp + "\n");
      w.append(new LongWritable(cnt), record);
    }

    w.close();
  } catch (ClassNotFoundException cnfe) {
   throw new IOException(cnfe);
  } catch (RecordParser.ParseError pe) {
   throw new IOException(pe);
  }
}
 
Example 49
Project: Wikipedia-Index   File: XmlInputFormat.java   Source Code and License 5 votes vote down vote up
private boolean next(LongWritable key, Text value) throws IOException {
    /**
     *  通过readUntilMatch方法查找xml段开始的标签,直到找到了,才开始
     *  写xml片段到buffer中去,如readUntilMatch的第二个参数为false则不查找的过
     *  程中写入数据到buffer,如果为true的话就边查找边写入
     */
    if( fsin.getPos() < end && readUntilMatch(startTag, false)) {
        //进入代码段则说明找到了开始标签,现在fsin的指针指在找到的开始标签的
        //最后一位上,所以向buffer中写入开始标签
        buffer.write(startTag);
        try {
            /**
             * 在fsin中去查找结束标签边查找边记录直到找到结束标签为止
             */
            if(readUntilMatch(endTag, true)) {
                /**
                 * 找到标签后把start标签的指针位置的偏移量赋值给key
                 * 把buffer中记录的整个xml完整片断赋值给value
                 */
                key.set(fsin.getPos() - buffer.getLength());
                value.set(buffer.getData(), 0, buffer.getLength());
                return true;
            }
        } finally {
            buffer.reset();
        }
    }
    return false;
}
 
Example 50
Project: hadoop   File: TokenCountMapper.java   Source Code and License 5 votes vote down vote up
public void map(K key, Text value,
                OutputCollector<Text, LongWritable> output,
                Reporter reporter)
  throws IOException {
  // get input text
  String text = value.toString();       // value is line of text

  // tokenize the value
  StringTokenizer st = new StringTokenizer(text);
  while (st.hasMoreTokens()) {
    // output <token,1> pairs
    output.collect(new Text(st.nextToken()), new LongWritable(1));
  }  
}
 
Example 51
Project: WIFIProbe   File: MapperWriter.java   Source Code and License 5 votes vote down vote up
private void writCustomerFlow() throws IOException, InterruptedException{

        KeyWrapper customerFlowKey = new KeyWrapper();
        customerFlowKey.setType(new Text(MapKeyConfig.CUSTOMER_FLOW_KEY));

        LongWritable longWritable = new LongWritable();
        customerFlowKey.setMillisTime(longWritable);

        for (CustomerFlowElement customerFlowElement:statistic.getCustomerFlowElements()) {
            longWritable.set(customerFlowElement.getHour());
            context.write(customerFlowKey, new ValueWrapper(customerFlowElement));
        }
    }
 
Example 52
Project: flume-release-1.7.0   File: HDFSTextSerializer.java   Source Code and License 5 votes vote down vote up
private Object getKey(Event e) {
  // Write the data to HDFS
  String timestamp = e.getHeaders().get("timestamp");
  long eventStamp;

  if (timestamp == null) {
    eventStamp = System.currentTimeMillis();
  } else {
    eventStamp = Long.valueOf(timestamp);
  }
  return new LongWritable(eventStamp);
}
 
Example 53
Project: flume-release-1.7.0   File: HDFSWritableSerializer.java   Source Code and License 5 votes vote down vote up
private Object getKey(Event e) {
  String timestamp = e.getHeaders().get("timestamp");
  long eventStamp;

  if (timestamp == null) {
    eventStamp = System.currentTimeMillis();
  } else {
    eventStamp = Long.valueOf(timestamp);
  }
  return new LongWritable(eventStamp);
}
 
Example 54
Project: aliyun-maxcompute-data-collectors   File: MainframeDatasetImportMapper.java   Source Code and License 5 votes vote down vote up
public void map(LongWritable key,  SqoopRecord val, Context context)
    throws IOException, InterruptedException {
  String dataset = inputSplit.getCurrentDataset();
  outkey.set(val.toString());
  numberOfRecords++;
  mos.write(outkey, NullWritable.get(), dataset);
}
 
Example 55
Project: hadoop   File: TextInputFormat.java   Source Code and License 5 votes vote down vote up
@Override
public RecordReader<LongWritable, Text> 
  createRecordReader(InputSplit split,
                     TaskAttemptContext context) {
  String delimiter = context.getConfiguration().get(
      "textinputformat.record.delimiter");
  byte[] recordDelimiterBytes = null;
  if (null != delimiter)
    recordDelimiterBytes = delimiter.getBytes(Charsets.UTF_8);
  return new LineRecordReader(recordDelimiterBytes);
}
 
Example 56
Project: mumu-mapreduce   File: YearTrafficStatisticsMapReduce.java   Source Code and License 5 votes vote down vote up
@Override
protected void map(final LongWritable key, final Text value, final Context context) throws IOException, InterruptedException {
    Map<String, Object> nginxLogMap = NginxAccessLogParser.parseLine(value.toString());
    String remoteAddr = nginxLogMap.get("remoteAddr").toString();
    Date accessTimeDate = (Date) nginxLogMap.get("accessTime");
    String year = DateFormatUtils.format(accessTimeDate, "yyyy");
    context.write(new Text(year), new Text(remoteAddr));
}
 
Example 57
Project: hadoop   File: TestMultithreadedMapper.java   Source Code and License 5 votes vote down vote up
public void map(LongWritable key, Text value, Context context)
    throws IOException, InterruptedException {
  if (ioEx) {
    throw new IOException();
  }
  if (rtEx) {
    throw new RuntimeException();
  }
  super.map(key, value, context);
}
 
Example 58
Project: hadoop   File: TestJobName.java   Source Code and License 5 votes vote down vote up
public void testComplexNameWithRegex() throws Exception {
  OutputStream os = getFileSystem().create(new Path(getInputDir(),
      "text.txt"));
  Writer wr = new OutputStreamWriter(os);
  wr.write("b a\n");
  wr.close();

  JobConf conf = createJobConf();
  conf.setJobName("name \\Evalue]");

  conf.setInputFormat(TextInputFormat.class);

  conf.setOutputKeyClass(LongWritable.class);
  conf.setOutputValueClass(Text.class);

  conf.setMapperClass(IdentityMapper.class);

  FileInputFormat.setInputPaths(conf, getInputDir());

  FileOutputFormat.setOutputPath(conf, getOutputDir());

  JobClient.runJob(conf);

  Path[] outputFiles = FileUtil.stat2Paths(
                         getFileSystem().listStatus(getOutputDir(),
                         new Utils.OutputFileUtils.OutputFilesFilter()));
  assertEquals(1, outputFiles.length);
  InputStream is = getFileSystem().open(outputFiles[0]);
  BufferedReader reader = new BufferedReader(new InputStreamReader(is));
  assertEquals("0\tb a", reader.readLine());
  assertNull(reader.readLine());
  reader.close();
}
 
Example 59
Project: aliyun-maxcompute-data-collectors   File: NetezzaExternalTableHCatExportMapper.java   Source Code and License 5 votes vote down vote up
@Override
public void map(LongWritable key, HCatRecord hcr, Context context)
  throws IOException, InterruptedException {
  SqoopRecord sqr = helper.convertToSqoopRecord(hcr);
  writeSqoopRecord(sqr);
  context.progress();
}
 
Example 60
Project: hadoop   File: TestIPC.java   Source Code and License 5 votes vote down vote up
@Test(timeout=60000)
public void testIOEOnClientWriteParam() throws Exception {
  doErrorTest(IOEOnWriteWritable.class,
      LongWritable.class,
      LongWritable.class,
      LongWritable.class);
}