org.apache.hadoop.mapreduce.Mapper.Context Java Examples

The following examples show how to use org.apache.hadoop.mapreduce.Mapper.Context. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PGBulkloadExportMapper.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 7 votes vote down vote up
public void map(LongWritable key, Writable value, Context context)
  throws IOException, InterruptedException {
  try {
    String str = value.toString();
    if (value instanceof Text) {
      writer.write(str, 0, str.length());
      writer.newLine();
    } else if (value instanceof SqoopRecord) {
      writer.write(str, 0, str.length());
    }
  } catch (Exception e) {
    doExecuteUpdate("DROP TABLE " + tmpTableName);
    cleanup(context);
    throw new IOException(e);
  }
}
 
Example #2
Source File: HiveToBaseCuboidMapperPerformanceTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Ignore("convenient trial tool for dev")
@Test
public void test() throws IOException, InterruptedException {
    Configuration hconf = HadoopUtil.getCurrentConfiguration();
    HiveToBaseCuboidMapper mapper = new HiveToBaseCuboidMapper();
    Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);

    mapper.doSetup(context);

    Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
    Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
    Text value = new Text();

    while (reader.next(key, value)) {
        mapper.map(key, value, context);
    }

    reader.close();
}
 
Example #3
Source File: PopulateTable.java    From HBase-ToHDFS with Apache License 2.0 6 votes vote down vote up
@Override
public void map(NullWritable key, NullWritable value, Context context) throws IOException, InterruptedException {

  int counter = 0;
  
  System.out.println("starting mapper");
  System.out.println();
  for (int i = 0; i < numberOfRecords; i++) {
    String keyRoot = StringUtils.leftPad(Integer.toString(r.nextInt(Short.MAX_VALUE)), 5, '0');

    if (i % 1000 == 0) {
      System.out.print(".");
    }

    for (int j = 0; j < 10; j++) {
      hKey.set(Bytes.toBytes(keyRoot + "|" + runID + "|" + taskId));
      kv = new KeyValue(hKey.get(), columnFamily, Bytes.toBytes("C" + j), Bytes.toBytes("counter:" + counter++ ));
      context.write(hKey, kv);
    }
  }

  System.out.println("finished mapper");
}
 
Example #4
Source File: WordCount.java    From stratio-cassandra with Apache License 2.0 6 votes vote down vote up
public void map(Map<String, ByteBuffer> keys, Map<String, ByteBuffer> columns, Context context) throws IOException, InterruptedException
{
    for (Entry<String, ByteBuffer> column : columns.entrySet())
    {
        if (!"line".equalsIgnoreCase(column.getKey()))
            continue;

        String value = ByteBufferUtil.string(column.getValue());

        StringTokenizer itr = new StringTokenizer(value);
        while (itr.hasMoreTokens())
        {
            word.set(itr.nextToken());
            context.write(word, one);
        }
    }
}
 
Example #5
Source File: BWAMemInstance.java    From halvade with GNU General Public License v3.0 6 votes vote down vote up
@Override
protected void startAligner(Mapper.Context context) throws IOException, InterruptedException {
    // make command
    String customArgs = HalvadeConf.getCustomArgs(context.getConfiguration(), "bwa", "mem");
    String[] command = CommandGenerator.bwaMem(bin, ref, null, null, isPaired, true, threads, customArgs);
    pbw = new ProcessBuilderWrapper(command, bin);
    // run command
    // needs to be streamed to output otherwise the process blocks ...
    pbw.startProcess(null, System.err);
    // check if alive.
    if(!pbw.isAlive())
        throw new ProcessException("BWA mem", pbw.getExitState());
    pbw.getSTDINWriter();
    // make a SAMstream handler
    ssh = new SAMStreamHandler(instance, context, false);
    ssh.start();
}
 
Example #6
Source File: BaseCuboidMapperPerformanceTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up
@Ignore("convenient trial tool for dev")
@Test
public void test() throws IOException, InterruptedException {
    Configuration hconf = new Configuration();
    BaseCuboidMapper mapper = new BaseCuboidMapper();
    Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);

    mapper.setup(context);

    Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
    Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
    Text value = new Text();

    while (reader.next(key, value)) {
        mapper.map(key, value, context);
    }

    reader.close();
}
 
Example #7
Source File: PostgreSQLCopyExportMapper.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 6 votes vote down vote up
@Override
public void map(LongWritable key, Writable value, Context context)
  throws IOException, InterruptedException {
  line.setLength(0);
  line.append(value.toString());
  if (value instanceof Text) {
    line.append(System.getProperty("line.separator"));
  }
  try {
    byte[]data = line.toString().getBytes("UTF-8");
    copyin.writeToCopy(data, 0, data.length);
  } catch (SQLException ex) {
    LoggingUtils.logAll(LOG, "Unable to execute copy", ex);
    close();
    throw new IOException(ex);
  }
}
 
Example #8
Source File: HiveToBaseCuboidMapperPerformanceTest.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Ignore("convenient trial tool for dev")
@Test
public void test() throws IOException, InterruptedException {
    Configuration hconf = HadoopUtil.getCurrentConfiguration();
    HiveToBaseCuboidMapper mapper = new HiveToBaseCuboidMapper();
    Context context = MockupMapContext.create(hconf, metadataUrl, cubeName, null);

    mapper.doSetup(context);

    Reader reader = new Reader(hconf, SequenceFile.Reader.file(srcPath));
    Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), hconf);
    Text value = new Text();

    while (reader.next(key, value)) {
        mapper.map(key, value, context);
    }

    reader.close();
}
 
Example #9
Source File: SampleMachineConsumer.java    From DataGenerator with Apache License 2.0 6 votes vote down vote up
/**
 * Constructor for SampleMachineConsumer - needs the Mapper Context
 *
 * @param context A Hadoop MapReduce Mapper.Context to which this consumer
 *                should writer
 */
public SampleMachineConsumer(final Context context) {
    super();

    ContextWriter contextWrite = new ContextWriter(context, template);
    this.addDataWriter(contextWrite);
    this.addDataTransformer(new SampleMachineTransformer());

    exit = new AtomicBoolean(false);
    handler = new JenkinsReportingHandler(exit);

    currentRow = -1;
    finalRow = -2;

    setReportGap(1000);
}
 
Example #10
Source File: BoaAstIntrinsics.java    From compiler with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
public static void cleanup(final Context context) {
	closeMap();
	closeCommentMap();
	closeIssuesMap();
	closeCommitMap();
}
 
Example #11
Source File: LogUtil.java    From jumbune with GNU Lesser General Public License v3.0 5 votes vote down vote up
/**
 * Add logging in map cleanup method
 * 
 * @param context
 *            - map context
 * @param className
 *            - Class which is calling this method
 * @param methodName
 *            - Class Method which is calling this method
 */
@SuppressWarnings(RAW_TYPES)
public static void getMapContextInfoCleanup(Context context,
		String className, String methodName) {
	Counter counter = context.getCounter(MAPRED_COUNTER, MAP_INPUT_RECORDS);
	getLogMsg(className, methodName, counter.getDisplayName(), COUNTERS,
			counter.getValue());

	counter = context.getCounter(MAPRED_COUNTER, MAP_OUTPUT_RECORDS);
	getLogMsg(className, methodName, counter.getDisplayName(), COUNTERS,
			counter.getValue());
}
 
Example #12
Source File: TestImportExport.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Test map method of Importer
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testKeyValueImporter() throws Throwable {
  CellImporter importer = new CellImporter();
  Configuration configuration = new Configuration();
  Context ctx = mock(Context.class);
  when(ctx.getConfiguration()).thenReturn(configuration);

  doAnswer(new Answer<Void>() {

    @Override
    public Void answer(InvocationOnMock invocation) throws Throwable {
      ImmutableBytesWritable writer = (ImmutableBytesWritable) invocation.getArgument(0);
      MapReduceExtendedCell key = (MapReduceExtendedCell) invocation.getArgument(1);
      assertEquals("Key", Bytes.toString(writer.get()));
      assertEquals("row", Bytes.toString(CellUtil.cloneRow(key)));
      return null;
    }
  }).when(ctx).write(any(), any());

  importer.setup(ctx);
  Result value = mock(Result.class);
  KeyValue[] keys = {
      new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("family"), Bytes.toBytes("qualifier"),
          Bytes.toBytes("value")),
      new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("family"), Bytes.toBytes("qualifier"),
          Bytes.toBytes("value1")) };
  when(value.rawCells()).thenReturn(keys);
  importer.map(new ImmutableBytesWritable(Bytes.toBytes("Key")), value, ctx);

}
 
Example #13
Source File: TestMapReduceExamples.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Test IndexBuilder from examples
 */
@SuppressWarnings("unchecked")
@Test
public void testIndexBuilder() throws Exception {
  Configuration configuration = new Configuration();
  String[] args = { "tableName", "columnFamily", "column1", "column2" };
  IndexBuilder.configureJob(configuration, args);
  assertEquals("tableName", configuration.get("index.tablename"));
  assertEquals("tableName", configuration.get(TableInputFormat.INPUT_TABLE));
  assertEquals("column1,column2", configuration.get("index.fields"));

  Map map = new Map();
  ImmutableBytesWritable rowKey = new ImmutableBytesWritable(Bytes.toBytes("test"));
  Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Put>.Context ctx =
      mock(Context.class);
  when(ctx.getConfiguration()).thenReturn(configuration);
  doAnswer(new Answer<Void>() {

    @Override
    public Void answer(InvocationOnMock invocation) throws Throwable {
      ImmutableBytesWritable writer = (ImmutableBytesWritable) invocation.getArgument(0);
      Put put = (Put) invocation.getArgument(1);
      assertEquals("tableName-column1", Bytes.toString(writer.get()));
      assertEquals("test", Bytes.toString(put.getRow()));
      return null;
    }
  }).when(ctx).write(any(), any());
  Result result = mock(Result.class);
  when(result.getValue(Bytes.toBytes("columnFamily"), Bytes.toBytes("column1"))).thenReturn(
      Bytes.toBytes("test"));
  map.setup(ctx);
  map.map(rowKey, result, ctx);
}
 
Example #14
Source File: DateSortDesc.java    From MapReduce-Demo with MIT License 5 votes vote down vote up
public void reduce(IntWritable key, Iterable<Text> values, Context context)
		throws IOException, InterruptedException {
	for (Text value : values) {
		// 排序后再次颠倒k-v,将日期作为key
		System.out.println(value.toString() + ":" + key.get());
		context.write(value, key);
	}
}
 
Example #15
Source File: TestMapReduceExamples.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Test SampleUploader from examples
 */
@SuppressWarnings("unchecked")
@Test
public void testSampleUploader() throws Exception {
  Configuration configuration = new Configuration();
  Uploader uploader = new Uploader();
  Mapper<LongWritable, Text, ImmutableBytesWritable, Put>.Context ctx = mock(Context.class);
  doAnswer(new Answer<Void>() {

    @Override
    public Void answer(InvocationOnMock invocation) throws Throwable {
      ImmutableBytesWritable writer = (ImmutableBytesWritable) invocation.getArgument(0);
      Put put = (Put) invocation.getArgument(1);
      assertEquals("row", Bytes.toString(writer.get()));
      assertEquals("row", Bytes.toString(put.getRow()));
      return null;
    }
  }).when(ctx).write(any(), any());

  uploader.map(null, new Text("row,family,qualifier,value"), ctx);

  Path dir = util.getDataTestDirOnTestFS("testSampleUploader");

  String[] args = { dir.toString(), "simpleTable" };
  Job job = SampleUploader.configureJob(configuration, args);
  assertEquals(SequenceFileInputFormat.class, job.getInputFormatClass());
}
 
Example #16
Source File: WordCount.java    From stratio-cassandra with Apache License 2.0 5 votes vote down vote up
public void map(Long key, Row row, Context context) throws IOException, InterruptedException
{
    String value = row.getString("line");
    logger.debug("read {}:{}={} from {}", new Object[] {key, "line", value, context.getInputSplit()});
    StringTokenizer itr = new StringTokenizer(value);
    while (itr.hasMoreTokens())
    {
        word.set(itr.nextToken());
        context.write(word, one);
    }
}
 
Example #17
Source File: PostgreSQLCopyExportMapper.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 5 votes vote down vote up
@Override
protected void cleanup(Context context)
  throws IOException, InterruptedException {
  try {
    copyin.endCopy();
  } catch (SQLException ex) {
    LoggingUtils.logAll(LOG, "Unable to finalize copy", ex);
    throw new IOException(ex);
  }
  close();
}
 
Example #18
Source File: WordCount.java    From stratio-cassandra with Apache License 2.0 5 votes vote down vote up
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException
{
    int sum = 0;
    for (IntWritable val : values)
        sum += val.get();
    context.write(key, new IntWritable(sum));
}
 
Example #19
Source File: WordCount.java    From stratio-cassandra with Apache License 2.0 5 votes vote down vote up
public void reduce(Text word, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException
{
    int sum = 0;
    for (IntWritable val : values)
        sum += val.get();
    keys.put("word", ByteBufferUtil.bytes(word.toString()));
    context.write(keys, getBindVariables(word, sum));
}
 
Example #20
Source File: PopulateTable.java    From HBase-ToHDFS with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(Context context) {
  System.out.println("starting setup");

  columnFamily = Bytes.toBytes(context.getConfiguration().get(COLUMN_FAMILY));
  runID = context.getConfiguration().get(RUN_ID);
  taskId = context.getTaskAttemptID().getTaskID().getId();
  numberOfRecords = context.getConfiguration().getInt(NUMBER_OF_RECORDS, 1000) / context.getConfiguration().getInt("nmapinputformat.num.maps", 1);

  System.out.println("finished setup");
}
 
Example #21
Source File: SAMStreamHandler.java    From halvade with GNU General Public License v3.0 5 votes vote down vote up
public SAMStreamHandler(AlignerInstance instance, Context context, boolean useCompact) {
    this.is = instance.getSTDOUTStream();
    this.mFileHeader = instance.getFileHeader();
    this.instance = instance;
    this.useCompact = useCompact;
    mCurrentLine = null;
    mFile = null;
    validationStringency = ValidationStringency.LENIENT;
    mReader = new BufferedLineReader(this.is);
    samRecordFactory = new DefaultSAMRecordFactory();
    this.context = context;
    isPaired = HalvadeConf.getIsPaired(context.getConfiguration());
}
 
Example #22
Source File: BWAMemInstance.java    From halvade with GNU General Public License v3.0 5 votes vote down vote up
/**
 * 
 * This BWA instance runs BWA from stdin (custom provided BWA is needed)
 */
private BWAMemInstance(Context context, String bin) throws IOException, URISyntaxException {
    super(context, bin);
    String taskid = context.getTaskAttemptID().toString();
    taskid = taskid.substring(taskid.indexOf("m_"));
    ref = HalvadeFileUtils.downloadBWAIndex(context, taskid);
}
 
Example #23
Source File: BWAMemInstance.java    From halvade with GNU General Public License v3.0 5 votes vote down vote up
static public BWAMemInstance getBWAInstance(Mapper.Context context, String bin) throws IOException, InterruptedException, URISyntaxException {
    if(instance == null) {
        instance = new BWAMemInstance(context, bin);
        instance.startAligner(context);
    }
    BWAMemInstance.context = context;
    return instance;
}
 
Example #24
Source File: Crawler.java    From DistributedCrawler with Apache License 2.0 5 votes vote down vote up
public Crawler(String id, String seed, int topN, int deepth, CrawlDB db,
		Context context) {
	this.id = id;
	this.topN = topN;
	this.deepth = deepth;
	this.seed = seed;
	this.db = db;
	this.context = context;
}
 
Example #25
Source File: NegativeCellIdCounter.java    From geowave with Apache License 2.0 5 votes vote down vote up
public NegativeCellIdCounter(
    final Context context,
    final long level,
    final long minLevel,
    final long maxLevel) {
  super(context, level, minLevel, maxLevel);
}
 
Example #26
Source File: MapContextCellCounter.java    From geowave with Apache License 2.0 5 votes vote down vote up
public MapContextCellCounter(
    final Context context,
    final long level,
    final long minLevel,
    final long maxLevel) {
  this.context = context;
  this.level = level;
  this.minLevel = minLevel;
  this.maxLevel = maxLevel;
  numLevels = (maxLevel - minLevel) + 1;
}
 
Example #27
Source File: Decoder.java    From RDFS with Apache License 2.0 5 votes vote down vote up
public void recoverParityBlockToFile(FileSystem srcFs, Path srcPath,
		FileSystem parityFs, Path parityPath, long blockSize,
		long blockOffset, File localBlockFile, Context context)
		throws IOException, InterruptedException {
	OutputStream out = new FileOutputStream(localBlockFile);
	fixErasedBlock(srcFs, srcPath, parityFs, parityPath, false, blockSize,
			blockOffset, blockSize, false, out, context, false);
	out.close();
}
 
Example #28
Source File: Decoder.java    From RDFS with Apache License 2.0 5 votes vote down vote up
DecoderInputStream generateAlternateStream(FileSystem srcFs, Path srcFile,
		FileSystem parityFs, Path parityFile, long blockSize,
		long errorOffset, long limit, Context context) {
	configureBuffers(blockSize);
	Progressable reporter = context;
	if (reporter == null) {
		reporter = RaidUtils.NULL_PROGRESSABLE;
	}

	DecoderInputStream decoderInputStream = null;
	
	if(codec.id.equals("crs") || codec.id.equals("lrc")) {
		decoderInputStream = new CRSDecoderInputStream(
				reporter, limit, blockSize, errorOffset, srcFs, srcFile,
				parityFs, parityFile);
	} else {
		decoderInputStream = new DecoderInputStream(
				reporter, limit, blockSize, errorOffset, srcFs, srcFile,
				parityFs, parityFile);
	}
	
	/*
	decoderInputStream = new DecoderInputStream(
			reporter, limit, blockSize, errorOffset, srcFs, srcFile,
			parityFs, parityFile);
	*/
	return decoderInputStream;
}
 
Example #29
Source File: ValueMapperTest.java    From secure-data-service with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testMap() throws Exception {
    TenantAndIdEmittableKey key = new TenantAndIdEmittableKey();
    ValueMapper m = new MockValueMapper();
    BSONObject entry = new BasicBSONObject("found", "data");
    BSONWritable entity = new BSONWritable(entry);

    Context context = Mockito.mock(Context.class);
    PowerMockito.when(context, "write", Matchers.any(EmittableKey.class),
        Matchers.any(BSONObject.class)).thenAnswer(new Answer<BSONObject>() {

        @Override
        public BSONObject answer(InvocationOnMock invocation) throws Throwable {

            Object[] args = invocation.getArguments();

            assertNotNull(args);
            assertEquals(args.length, 2);

            assertTrue(args[0] instanceof TenantAndIdEmittableKey);
            assertTrue(args[1] instanceof ContentSummary);

            TenantAndIdEmittableKey id = (TenantAndIdEmittableKey) args[0];
            assertNotNull(id);

            ContentSummary e = (ContentSummary) args[1];
            assertEquals(e.getLength(), 1);
            assertEquals(e.getFileCount(), 2);
            assertEquals(e.getDirectoryCount(), 3);

            return null;
        }
    });

    m.map(key, entity, context);
}
 
Example #30
Source File: ValueMapperTest.java    From secure-data-service with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testMapValueNotFound() throws Exception {
    TenantAndIdEmittableKey key = new TenantAndIdEmittableKey();
    ValueMapper m = new MockValueMapper();
    BSONObject entry = new BasicBSONObject("not_found", "data");
    BSONWritable entity = new BSONWritable(entry);

    Context context = Mockito.mock(Context.class);
    PowerMockito.when(context, "write", Matchers.any(TenantAndIdEmittableKey.class),
        Matchers.any(BSONObject.class)).thenAnswer(new Answer<BSONObject>() {

        @Override
        public BSONObject answer(InvocationOnMock invocation) throws Throwable {

            Object[] args = invocation.getArguments();

            assertNotNull(args);
            assertEquals(args.length, 2);

            assertTrue(args[0] instanceof TenantAndIdEmittableKey);
            assertTrue(args[1] instanceof NullWritable);

            return null;
        }
    });

    m.map(key, entity, context);
}