Java Code Examples for org.apache.hadoop.io.LongWritable#set()

The following examples show how to use org.apache.hadoop.io.LongWritable#set() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto   File: S3SelectLineRecordReader.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized boolean next(LongWritable key, Text value)
        throws IOException
{
    while (true) {
        int bytes = readLine(value);
        if (bytes <= 0) {
            if (!selectClient.isRequestComplete()) {
                throw new IOException("S3 Select request was incomplete as End Event was not received");
            }
            return false;
        }
        recordsFromS3++;
        if (recordsFromS3 > processedRecords) {
            position += bytes;
            processedRecords++;
            key.set(processedRecords);
            return true;
        }
    }
}
 
Example 2
Source Project: WIFIProbe   File: MapperWriter.java    License: Apache License 2.0 6 votes vote down vote up
private void writeInStoreHour() throws IOException, InterruptedException{

        KeyWrapper cycleKey = new KeyWrapper();
        cycleKey.setType(new Text(MapKeyConfig.IN_STORE_HOUR));

        LongWritable longWritable = new LongWritable();
        cycleKey.setMillisTime(longWritable);

        IntWritable value = new IntWritable(1);


        List<Long> inStoreHours = statistic.getInStoreHours();
        for (Long inStoreTime : inStoreHours) {
            longWritable.set(IntervalCalculator.getInStoreInterval(inStoreTime));
            context.write(cycleKey, new ValueWrapper(value));
        }
    }
 
Example 3
Source Project: hadoop-gpu   File: TestChainMapReduce.java    License: Apache License 2.0 6 votes vote down vote up
public void reduce(LongWritable key, Iterator<Text> values,
                   OutputCollector<LongWritable, Text> output,
                   Reporter reporter) throws IOException {
  while (values.hasNext()) {
    Text value = values.next();
    writeFlag(conf, "reduce." + name + ".value." + value);
    key.set(10);
    output.collect(key, value);
    if (byValue) {
      assertEquals(10, key.get());
    } else {
      assertNotSame(10, key.get());
    }
    key.set(11);
  }
}
 
Example 4
Source Project: RDFS   File: DBInputFormat.java    License: Apache License 2.0 6 votes vote down vote up
/** {@inheritDoc} */
public boolean next(LongWritable key, T value) throws IOException {
  try {
    if (!results.next())
      return false;

    // Set the key field value as the output key value
    key.set(pos + split.getStart());

    value.readFields(results);

    pos ++;
  } catch (SQLException e) {
    throw new IOException(e.getMessage());
  }
  return true;
}
 
Example 5
Source Project: spork   File: Bzip2TextInputFormat.java    License: Apache License 2.0 6 votes vote down vote up
/** Read a line. */
public  boolean next(LongWritable key, Text value)
throws IOException {
    if (pos > end)
        return false;

    key.set(pos); // key is position
    buffer.reset();
    // long bytesRead = LineRecordReader.readLine(in, buffer); 
    long bytesRead = readLine(in, buffer);
    if (bytesRead == 0) {
        return false;
    }
    pos = in.getPos();
    // if we have read ahead because we encountered a carriage return
    // char followed by a non line feed char, decrement the pos
    if(CRFollowedByNonLF) {
        pos--;
    }

    bridge.target = value;
    buffer.writeTo(bridge);
    return true;
}
 
Example 6
Source Project: Arabesque   File: ExecutionEngine.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void postSuperstep() {
    super.postSuperstep();

    try {
        for (Map.Entry<String, AggregationStorage> aggregationStorageEntry : aggregationStorages.entrySet()) {
            String aggregationStorageName = aggregationStorageEntry.getKey();
            AggregationStorage aggregationStorage = aggregationStorageEntry.getValue();

            workerContext.addAggregationStorage(aggregationStorageName, aggregationStorage);
        }
    } catch (RuntimeException e) {
        LOG.error(e);
        throw e;
    }

    LongWritable longWritable = new LongWritable();

    LOG.info("Num embeddings processed: " + numEmbeddingsProcessed);
    longWritable.set(numEmbeddingsProcessed);
    aggregate(MasterExecutionEngine.AGG_EMBEDDINGS_PROCESSED, longWritable);
    LOG.info("Num embeddings generated: " + numEmbeddingsGenerated);
    longWritable.set(numEmbeddingsGenerated);
    aggregate(MasterExecutionEngine.AGG_EMBEDDINGS_GENERATED, longWritable);
    LOG.info("Num embeddings output: " + numberOfEmbeddingsOutput);
    longWritable.set(numberOfEmbeddingsOutput);
    aggregate(MasterExecutionEngine.AGG_EMBEDDINGS_OUTPUT, longWritable);
}
 
Example 7
@Override
public void finish() {
    flush();

    LongWritable longWritable = new LongWritable();
    longWritable.set(totalSizeODAGs);

    getExecutionEngine().aggregate(MasterExecutionEngine.AGG_PROCESSED_SIZE_ODAG, longWritable);
}
 
Example 8
/** Read a line. */
public synchronized boolean next(LongWritable key, Text value)
		throws IOException {
	while (pos < end) {
		key.set(pos);

		int newSize = lineReader.readLine(value, maxLineLength,
				Math.max((int) Math.min(Integer.MAX_VALUE, end - pos),
						maxLineLength));

		// start
		String strReplace = value.toString().toLowerCase()
				.replaceAll("\\|\\|\\|", "\001");
		Text txtReplace = new Text();
		txtReplace.set(strReplace);
		value.set(txtReplace.getBytes(), 0, txtReplace.getLength());
		// end

		if (newSize == 0) {
			return false;
		}
		pos += newSize;
		if (newSize < maxLineLength) {
			return true;
		}

		// line too long. try again
		LOG.info("Skipped line of size " + newSize + " at pos "
				+ (pos - newSize));
	}

	return false;
}
 
Example 9
Source Project: big-c   File: FixedLengthRecordReader.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized boolean next(LongWritable key, BytesWritable value)
    throws IOException {
  boolean dataRead = reader.nextKeyValue();
  if (dataRead) {
    LongWritable newKey = reader.getCurrentKey();
    BytesWritable newValue = reader.getCurrentValue();
    key.set(newKey.get());
    value.set(newValue);
  }
  return dataRead;
}
 
Example 10
@Override
public synchronized boolean next(LongWritable key, BytesWritable value)
		throws IOException {
	boolean dataRead = reader.nextKeyValue();
	if (dataRead) {
		LongWritable newKey = reader.getCurrentKey();
		BytesWritable newValue = reader.getCurrentValue();
		key.set(newKey.get());
		value.set(newValue);
	}
	return dataRead;
}
 
Example 11
Source Project: WIFIProbe   File: MapperWriter.java    License: Apache License 2.0 5 votes vote down vote up
private void writCustomerFlow() throws IOException, InterruptedException{

        KeyWrapper customerFlowKey = new KeyWrapper();
        customerFlowKey.setType(new Text(MapKeyConfig.CUSTOMER_FLOW_KEY));

        LongWritable longWritable = new LongWritable();
        customerFlowKey.setMillisTime(longWritable);

        for (CustomerFlowElement customerFlowElement:statistic.getCustomerFlowElements()) {
            longWritable.set(customerFlowElement.getHour());
            context.write(customerFlowKey, new ValueWrapper(customerFlowElement));
        }
    }
 
Example 12
Source Project: WIFIProbe   File: MapperWriter.java    License: Apache License 2.0 5 votes vote down vote up
private void writeCycle() throws IOException, InterruptedException{

        KeyWrapper cycleKey = new KeyWrapper();
        cycleKey.setType(new Text(MapKeyConfig.CYCLE));

        LongWritable longWritable = new LongWritable();
        cycleKey.setMillisTime(longWritable);

        IntWritable value = new IntWritable(1);

        for (Long cycle : statistic.getCycles()) {
            longWritable.set(IntervalCalculator.getCycleInterval(cycle));
            context.write(cycleKey, new ValueWrapper(value));
        }
    }
 
Example 13
Source Project: big-c   File: LineRecordReader.java    License: Apache License 2.0 5 votes vote down vote up
/** Read a line. */
public synchronized boolean next(LongWritable key, Text value)
  throws IOException {

  // We always read one extra line, which lies outside the upper
  // split limit i.e. (end - 1)
  while (getFilePosition() <= end || in.needAdditionalRecordAfterSplit()) {
    key.set(pos);

    int newSize = 0;
    if (pos == 0) {
      newSize = skipUtfByteOrderMark(value);
    } else {
      newSize = in.readLine(value, maxLineLength, maxBytesToConsume(pos));
      pos += newSize;
    }

    if (newSize == 0) {
      return false;
    }
    if (newSize < maxLineLength) {
      return true;
    }

    // line too long. try again
    LOG.info("Skipped line of size " + newSize + " at pos " + (pos - newSize));
  }

  return false;
}
 
Example 14
Source Project: hadoop   File: FixedLengthRecordReader.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized boolean next(LongWritable key, BytesWritable value)
    throws IOException {
  boolean dataRead = reader.nextKeyValue();
  if (dataRead) {
    LongWritable newKey = reader.getCurrentKey();
    BytesWritable newValue = reader.getCurrentValue();
    key.set(newKey.get());
    value.set(newValue);
  }
  return dataRead;
}
 
Example 15
Source Project: recsys-offline   File: Step1.java    License: Apache License 2.0 5 votes vote down vote up
public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
    VarLongWritable userID = new VarLongWritable();
    LongWritable itemID = new LongWritable();
    FloatWritable itemValue = new FloatWritable();System.err.println("key:"+key+"    value:"+value+"   ");
    String line = value.toString();
    String[] info = line.split(",");
    if (info.length != 3) {
        return;
    }
    userID.set(Long.parseLong(info[0]));
    itemID.set(Long.parseLong(info[1]));
    itemValue.set(Float.parseFloat(info[2]));
    context.write(userID, new LongAndFloat(itemID, itemValue));
}
 
Example 16
@Override
public LongWritable convert( ValueMetaInterface meta, Object obj ) throws TypeConversionException {
  try {
    LongWritable result = new LongWritable();
    result.set( meta.getInteger( obj ) );
    return result;
  } catch ( KettleValueException ex ) {
    throw new TypeConversionException( BaseMessages
      .getString( TypeConverterFactory.class, "ErrorConverting", LongWritable.class.getSimpleName(), obj ), ex );
  }
}
 
Example 17
Source Project: hadoop-gpu   File: TestChainMapReduce.java    License: Apache License 2.0 5 votes vote down vote up
public void map(LongWritable key, Text value,
                OutputCollector<LongWritable, Text> output,
                Reporter reporter) throws IOException {
  writeFlag(conf, "map." + name + ".value." + value);
  key.set(10);
  output.collect(key, value);
  if (byValue) {
    assertEquals(10, key.get());
  } else {
    assertNotSame(10, key.get());
  }
  key.set(11);
}
 
Example 18
Source Project: big-c   File: TestChainMapReduce.java    License: Apache License 2.0 5 votes vote down vote up
public void map(LongWritable key, Text value,
                OutputCollector<LongWritable, Text> output,
                Reporter reporter) throws IOException {
  writeFlag(conf, "map." + name + ".value." + value);
  key.set(10);
  output.collect(key, value);
  if (byValue) {
    assertEquals(10, key.get());
  } else {
    assertNotSame(10, key.get());
  }
  key.set(11);
}
 
Example 19
Source Project: systemds   File: FrameWriterBinaryBlock.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Internal primitive to write a block-aligned row range of a frame to a single sequence file, 
 * which is used for both single- and multi-threaded writers (for consistency). 
 * 
 * @param path file path
 * @param job job configuration
 * @param fs file system
 * @param src frame block
 * @param blen block length
 * @param rl lower row
 * @param ru upper row
 * @throws IOException if IOException occurs
 */
@SuppressWarnings("deprecation")
protected static void writeBinaryBlockFrameToSequenceFile( Path path, JobConf job, FileSystem fs, FrameBlock src, int blen, int rl, int ru ) 
	throws IOException
{
	//1) create sequence file writer 
	SequenceFile.Writer writer = null;
	writer = new SequenceFile.Writer(fs, job, path, LongWritable.class, FrameBlock.class);
	
	try
	{
		//2) reblock and write
		LongWritable index = new LongWritable();

		if( src.getNumRows() <= blen ) //opt for single block
		{
			//directly write single block
			index.set(1);
			writer.append(index, src);
		}
		else //general case
		{
			//initialize blocks for reuse (at most 4 different blocks required)
			FrameBlock[] blocks = createFrameBlocksForReuse(src.getSchema(), src.getColumnNames(), src.getNumRows());  
			
			//create and write subblocks of frame
			for(int bi = rl; bi < ru; bi += blen) {
				int len = Math.min(blen,  src.getNumRows()-bi);
				
				//get reuse frame block and copy subpart to block (incl meta on first)
				FrameBlock block = getFrameBlockForReuse(blocks);
				src.slice( bi, bi+len-1, 0, src.getNumColumns()-1, block );
				if( bi==0 ) //first block
					block.setColumnMetadata(src.getColumnMetadata());
				
				//append block to sequence file
				index.set(bi+1);
				writer.append(index, block);
			}
		}
	}
	finally {
		IOUtilFunctions.closeSilently(writer);
	}		
}
 
Example 20
Source Project: big-c   File: FSImageSerialization.java    License: Apache License 2.0 4 votes vote down vote up
/** write the long value */
static void writeLong(long value, DataOutputStream out) throws IOException {
  LongWritable uLong = TL_DATA.get().U_LONG;
  uLong.set(value);
  uLong.write(out);
}