Java Code Examples for org.apache.hadoop.io.BytesWritable

The following examples show how to use org.apache.hadoop.io.BytesWritable. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void parseBitcoinRawBlock() throws HadoopCryptoLedgerConfigurationException, IOException {
  ClassLoader classLoader = getClass().getClassLoader();
    String fileName="genesis.blk";
    String fileNameBlock=classLoader.getResource("testdata/"+fileName).getFile();	
    Path file = new Path(fileNameBlock); 
    FileInputSplit blockInputSplit = new FileInputSplit(0,file,0, -1, null);
    BitcoinRawBlockFlinkInputFormat inputFormat = new BitcoinRawBlockFlinkInputFormat(1024*1024,"F9BEB4D9",false);
    inputFormat.open(blockInputSplit);
    assertFalse(inputFormat.reachedEnd(),"End not reached");
    BytesWritable reuse = new BytesWritable();
    BytesWritable nextBlock = inputFormat.nextRecord(reuse);
    assertNotNull(nextBlock,"First Block returned");
	assertEquals( 293, nextBlock.getLength(),"First Block must have size of 293");
    nextBlock=inputFormat.nextRecord(reuse);
    assertNull(nextBlock,"No further block");
    assertTrue(inputFormat.reachedEnd(),"End reached");
}
 
Example 2
Source Project: RDFS   Source File: TestTupleWritable.java    License: Apache License 2.0 6 votes vote down vote up
public void testNestedIterable() throws Exception {
  Random r = new Random();
  Writable[] writs = {
    new BooleanWritable(r.nextBoolean()),
    new FloatWritable(r.nextFloat()),
    new FloatWritable(r.nextFloat()),
    new IntWritable(r.nextInt()),
    new LongWritable(r.nextLong()),
    new BytesWritable("dingo".getBytes()),
    new LongWritable(r.nextLong()),
    new IntWritable(r.nextInt()),
    new BytesWritable("yak".getBytes()),
    new IntWritable(r.nextInt())
  };
  TupleWritable sTuple = makeTuple(writs);
  assertTrue("Bad count", writs.length == verifIter(writs, sTuple, 0));
}
 
Example 3
Source Project: spatial-framework-for-hadoop   Source File: ST_MinY.java    License: Apache License 2.0 6 votes vote down vote up
public DoubleWritable evaluate(BytesWritable geomref) {
	if (geomref == null || geomref.getLength() == 0) {
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
	if (ogcGeometry == null) {
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	Envelope envBound = new Envelope();
	ogcGeometry.getEsriGeometry().queryEnvelope(envBound);
	resultDouble.set(envBound.getYMin());
	return resultDouble;
}
 
Example 4
Source Project: pentaho-hadoop-shims   Source File: CommonHadoopShim.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Class<? extends Writable> getHadoopWritableCompatibleClass( ValueMetaInterface kettleType ) {
  if ( kettleType == null ) {
    return NullWritable.class;
  }
  switch ( kettleType.getType() ) {
    case ValueMetaInterface.TYPE_STRING:
    case ValueMetaInterface.TYPE_BIGNUMBER:
    case ValueMetaInterface.TYPE_DATE:
      return Text.class;
    case ValueMetaInterface.TYPE_INTEGER:
      return LongWritable.class;
    case ValueMetaInterface.TYPE_NUMBER:
      return DoubleWritable.class;
    case ValueMetaInterface.TYPE_BOOLEAN:
      return BooleanWritable.class;
    case ValueMetaInterface.TYPE_BINARY:
      return BytesWritable.class;
    default:
      return Text.class;
  }
}
 
Example 5
public BytesWritable evaluate(Text wkwrap, int wkid) throws UDFArgumentException {

		String wkt = wkwrap.toString();
		try {
			SpatialReference spatialReference = null;
			if (wkid != GeometryUtils.WKID_UNKNOWN) {
				spatialReference = SpatialReference.create(wkid);
			}
			OGCGeometry ogcObj = OGCGeometry.fromText(wkt);
			ogcObj.setSpatialReference(spatialReference);
			return GeometryUtils.geometryToEsriShapeBytesWritable(ogcObj);
		} catch (Exception e) {  // IllegalArgumentException, GeometryException
			LogUtils.Log_InvalidText(LOG, wkt);
			return null;
		}
	}
 
Example 6
@Test
public void partial1ModeIntKeysDefaultParams() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { intInspector, doubleInspector };
  GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
  try (GenericUDAFEvaluator eval = new DataToDoubleSummarySketchUDAF().getEvaluator(info)) {
    ObjectInspector resultInspector = eval.init(Mode.PARTIAL1, inspectors);
    checkIntermediateResultInspector(resultInspector);

    @SuppressWarnings("unchecked")
    State<DoubleSummary> state = (State<DoubleSummary>) eval.getNewAggregationBuffer();
    eval.iterate(state, new Object[] {new IntWritable(1), new DoubleWritable(1)});
    eval.iterate(state, new Object[] {new IntWritable(2), new DoubleWritable(1)});

    Object result = eval.terminatePartial(state);
    Assert.assertNotNull(result);
    Assert.assertTrue(result instanceof List);
    List<?> r = (List<?>) result;
    Assert.assertEquals(r.size(), 2);
    Assert.assertEquals(((IntWritable) r.get(0)).get(), DEFAULT_NOMINAL_ENTRIES);
    Sketch<DoubleSummary> resultSketch = Sketches.heapifySketch(
        BytesWritableHelper.wrapAsMemory((BytesWritable) r.get(1)), new DoubleSummaryDeserializer());
    Assert.assertFalse(resultSketch.isEstimationMode());
    Assert.assertEquals(resultSketch.getEstimate(), 2.0);
  }
}
 
Example 7
Source Project: nutch-htmlunit   Source File: DeduplicationJob.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void map(Text key, CrawlDatum value,
        OutputCollector<BytesWritable, CrawlDatum> output,
        Reporter reporter) throws IOException {

    if (value.getStatus() == CrawlDatum.STATUS_DB_FETCHED
            || value.getStatus() == CrawlDatum.STATUS_DB_NOTMODIFIED) {
        // || value.getStatus() ==CrawlDatum.STATUS_DB_GONE){
        byte[] signature = value.getSignature();
        if (signature == null) return;
        BytesWritable sig = new BytesWritable(signature);
        // add the URL as a temporary MD
        value.getMetaData().put(urlKey, key);
        // reduce on the signature
        output.collect(sig, value);
    }
}
 
Example 8
Source Project: big-c   Source File: UtilsForTests.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Configure a waiting job
 */
static void configureWaitingJobConf(JobConf jobConf, Path inDir,
                                    Path outputPath, int numMaps, int numRed,
                                    String jobName, String mapSignalFilename,
                                    String redSignalFilename)
throws IOException {
  jobConf.setJobName(jobName);
  jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class);
  jobConf.setOutputFormat(SequenceFileOutputFormat.class);
  FileInputFormat.setInputPaths(jobConf, inDir);
  FileOutputFormat.setOutputPath(jobConf, outputPath);
  jobConf.setMapperClass(UtilsForTests.HalfWaitingMapper.class);
  jobConf.setReducerClass(IdentityReducer.class);
  jobConf.setOutputKeyClass(BytesWritable.class);
  jobConf.setOutputValueClass(BytesWritable.class);
  jobConf.setInputFormat(RandomInputFormat.class);
  jobConf.setNumMapTasks(numMaps);
  jobConf.setNumReduceTasks(numRed);
  jobConf.setJar("build/test/mapred/testjar/testjob.jar");
  jobConf.set(getTaskSignalParameter(true), mapSignalFilename);
  jobConf.set(getTaskSignalParameter(false), redSignalFilename);
}
 
Example 9
Source Project: hadoop-gpu   Source File: UtilsForTests.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Configure a waiting job
 */
static void configureWaitingJobConf(JobConf jobConf, Path inDir,
                                    Path outputPath, int numMaps, int numRed,
                                    String jobName, String mapSignalFilename,
                                    String redSignalFilename)
throws IOException {
  jobConf.setJobName(jobName);
  jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class);
  jobConf.setOutputFormat(SequenceFileOutputFormat.class);
  FileInputFormat.setInputPaths(jobConf, inDir);
  FileOutputFormat.setOutputPath(jobConf, outputPath);
  jobConf.setMapperClass(UtilsForTests.HalfWaitingMapper.class);
  jobConf.setReducerClass(IdentityReducer.class);
  jobConf.setOutputKeyClass(BytesWritable.class);
  jobConf.setOutputValueClass(BytesWritable.class);
  jobConf.setInputFormat(RandomInputFormat.class);
  jobConf.setNumMapTasks(numMaps);
  jobConf.setNumReduceTasks(numRed);
  jobConf.setJar("build/test/testjar/testjob.jar");
  jobConf.set(getTaskSignalParameter(true), mapSignalFilename);
  jobConf.set(getTaskSignalParameter(false), redSignalFilename);
}
 
Example 10
Source Project: spatial-framework-for-hadoop   Source File: ST_Point.java    License: Apache License 2.0 6 votes vote down vote up
public BytesWritable evaluate(Text wkwrap) throws UDFArgumentException {
	String wkt = wkwrap.toString();
	try {
		OGCGeometry ogcObj = OGCGeometry.fromText(wkt);
		ogcObj.setSpatialReference(null);
		if (ogcObj.geometryType().equals("Point")) {
			return GeometryUtils.geometryToEsriShapeBytesWritable(ogcObj);
		} else {
			LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_POINT, GeometryUtils.OGCType.UNKNOWN);
			return null;
		}

	} catch (Exception e) {  // IllegalArgumentException, GeometryException
		LogUtils.Log_InvalidText(LOG, wkt);
		return null;
	}
}
 
Example 11
Source Project: compiler   Source File: SeqCombiner.java    License: Apache License 2.0 6 votes vote down vote up
public static long readAndAppendCommit(Configuration conf, FileSystem fileSystem, MapFile.Writer writer, String fileName, long lastAstKey, long lastCommitKey) throws IOException {
	long newLastKey = lastCommitKey;
	SequenceFile.Reader r = new SequenceFile.Reader(fileSystem, new Path(fileName), conf);
	LongWritable longKey = new LongWritable();
	BytesWritable value = new BytesWritable();
	try {
		while (r.next(longKey, value)) {
			newLastKey = longKey.get() + lastCommitKey;
			Revision rev = Revision.parseFrom(CodedInputStream.newInstance(value.getBytes(), 0, value.getLength()));
			Revision.Builder rb = Revision.newBuilder(rev);
			for (ChangedFile.Builder cfb : rb.getFilesBuilderList()) {
				long key = cfb.getKey();
				if (key > 0)
					cfb.setKey(lastAstKey + key);
			}
			writer.append(new LongWritable(newLastKey), new BytesWritable(rb.build().toByteArray()));
		}
	} catch (Exception e) {
		System.err.println(fileName);
		e.printStackTrace();
	} finally {
		r.close();
	}
	return newLastKey;
}
 
Example 12
Source Project: hadoop   Source File: TestCombineSequenceFileInputFormat.java    License: Apache License 2.0 6 votes vote down vote up
private static void createFiles(int length, int numFiles, Random random,
  Job job) throws IOException {
  Range[] ranges = createRanges(length, numFiles, random);

  for (int i = 0; i < numFiles; i++) {
    Path file = new Path(workDir, "test_" + i + ".seq");
    // create a file with length entries
    @SuppressWarnings("deprecation")
    SequenceFile.Writer writer =
      SequenceFile.createWriter(localFs, job.getConfiguration(), file,
                                IntWritable.class, BytesWritable.class);
    Range range = ranges[i];
    try {
      for (int j = range.start; j < range.end; j++) {
        IntWritable key = new IntWritable(j);
        byte[] data = new byte[random.nextInt(10)];
        random.nextBytes(data);
        BytesWritable value = new BytesWritable(data);
        writer.append(key, value);
      }
    } finally {
      writer.close();
    }
  }
}
 
Example 13
Source Project: hadoopcryptoledger   Source File: NamecoinUDFTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void extractNamecoinFieldFirstUpdate() throws HiveException {
	String firstUpdateScript ="520A642F666C6173687570641460C7B068EDEA60281DAF424C38D8DAB87C96CF993D7B226970223A223134352E3234392E3130362E323238222C226D6170223A7B222A223A7B226970223A223134352E3234392E3130362E323238227D7D7D6D6D76A91451B4FC93AAB8CBDBD0AC9BC8EAF824643FC1E29B88AC";
	byte[] firstUpdateScriptBytes = BitcoinUtil.convertHexStringToByteArray(firstUpdateScript);
	NamecoinExtractFieldUDF nefu = new NamecoinExtractFieldUDF();
	ObjectInspector[] arguments = new ObjectInspector[1];
	arguments[0] =  PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;;
	nefu.initialize(arguments);	
	
	GenericUDF.DeferredObject[] doa = new GenericUDF.DeferredObject[1];
	
	doa[0]=new GenericUDF.DeferredJavaObject(new BytesWritable(firstUpdateScriptBytes));
	List<Text> resultList = (List<Text>) nefu.evaluate(doa);
	
	Text[] result=resultList.toArray(new Text[resultList.size()]);
	assertNotNull( result,"Valid result obtained");
	// test for domain name
	assertEquals("d/flashupd",result[0].toString(),"Domain name of first update detected correctly");
	// test for domain value
	assertEquals("{\"ip\":\"145.249.106.228\",\"map\":{\"*\":{\"ip\":\"145.249.106.228\"}}}",result[1].toString(),"Domain value of first update detected correctly");
	
}
 
Example 14
Source Project: RDFS   Source File: TestIPCServerResponder.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
  for (int i = 0; i < count; i++) {
    try {
      int byteSize = RANDOM.nextInt(BYTE_COUNT);
      byte[] bytes = new byte[byteSize];
      System.arraycopy(BYTES, 0, bytes, 0, byteSize);
      Writable param = new BytesWritable(bytes);
      Writable value = client.call(param, address);
      Thread.sleep(RANDOM.nextInt(20));
    } catch (Exception e) {
      LOG.fatal("Caught: " + e);
      failed = true;
    }
  }
}
 
Example 15
Source Project: hadoop   Source File: BaileyBorweinPlouffe.java    License: Apache License 2.0 6 votes vote down vote up
/** Compute the (offset+1)th to (offset+length)th digits. */
protected void map(LongWritable offset, IntWritable length,
    final Context context) throws IOException, InterruptedException {
  LOG.info("offset=" + offset + ", length=" + length);

  // compute digits
  final byte[] bytes = new byte[length.get() >> 1];
  long d = offset.get();
  for (int i = 0; i < bytes.length; d += 4) {
    final long digits = hexDigits(d);
    bytes[i++] = (byte) (digits >> 8);
    bytes[i++] = (byte) digits;
  }

  // output map results
  context.write(offset, new BytesWritable(bytes));
}
 
Example 16
Source Project: big-c   Source File: CommonStub.java    License: Apache License 2.0 6 votes vote down vote up
protected void readObject(Writable obj, DataInputStream inStream) throws IOException {
  int numBytes = WritableUtils.readVInt(inStream);
  byte[] buffer;
  // For BytesWritable and Text, use the specified length to set the length
  // this causes the "obvious" translations to work. So that if you emit
  // a string "abc" from C++, it shows up as "abc".
  if (obj instanceof BytesWritable) {
    buffer = new byte[numBytes];
    inStream.readFully(buffer);
    ((BytesWritable) obj).set(buffer, 0, numBytes);
  } else if (obj instanceof Text) {
    buffer = new byte[numBytes];
    inStream.readFully(buffer);
    ((Text) obj).set(buffer);
  } else {
    obj.readFields(inStream);
  }
}
 
Example 17
Source Project: Eagle   Source File: GroupbyKeyComparator.java    License: Apache License 2.0 6 votes vote down vote up
@Override 
   public int compare(GroupbyKey key1, GroupbyKey key2){
	List<BytesWritable> list1 = key1.getValue();
	List<BytesWritable> list2 = key2.getValue();
	
	if(list1 == null || list2 == null || list1.size() != list2.size())
		throw new IllegalArgumentException("2 list of groupby fields must be non-null and have the same size");
	ListIterator<BytesWritable> e1 = list1.listIterator();
	ListIterator<BytesWritable> e2 = list2.listIterator();
	while(e1.hasNext() && e2.hasNext()){
		int r = Bytes.compareTo(e1.next().copyBytes(), e2.next().copyBytes());
		if(r != 0)
			return r;
	}
	return 0;
}
 
Example 18
Source Project: eagle   Source File: GroupbyKeyComparator.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public int compare(GroupbyKey key1, GroupbyKey key2) {
    List<BytesWritable> list1 = key1.getValue();
    List<BytesWritable> list2 = key2.getValue();

    if (list1 == null || list2 == null || list1.size() != list2.size()) {
        throw new IllegalArgumentException("2 list of groupby fields must be non-null and have the same size");
    }
    ListIterator<BytesWritable> e1 = list1.listIterator();
    ListIterator<BytesWritable> e2 = list2.listIterator();
    while (e1.hasNext() && e2.hasNext()) {
        int r = Bytes.compareTo(e1.next().copyBytes(), e2.next().copyBytes());
        if (r != 0) {
            return r;
        }
    }
    return 0;
}
 
Example 19
Source Project: big-c   Source File: TestTupleWritable.java    License: Apache License 2.0 6 votes vote down vote up
public void testWritable() throws Exception {
  Random r = new Random();
  Writable[] writs = {
    new BooleanWritable(r.nextBoolean()),
    new FloatWritable(r.nextFloat()),
    new FloatWritable(r.nextFloat()),
    new IntWritable(r.nextInt()),
    new LongWritable(r.nextLong()),
    new BytesWritable("dingo".getBytes()),
    new LongWritable(r.nextLong()),
    new IntWritable(r.nextInt()),
    new BytesWritable("yak".getBytes()),
    new IntWritable(r.nextInt())
  };
  TupleWritable sTuple = makeTuple(writs);
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  sTuple.write(new DataOutputStream(out));
  ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray());
  TupleWritable dTuple = new TupleWritable();
  dTuple.readFields(new DataInputStream(in));
  assertTrue("Failed to write/read tuple", sTuple.equals(dTuple));
}
 
Example 20
Source Project: hadoop   Source File: TFile.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Copy the value into BytesWritable. The input BytesWritable will be
 * automatically resized to the actual value size. The implementation
 * directly uses the buffer inside BytesWritable for storing the value.
 * The call does not require the value length to be known.
 * 
 * @param value
 * @throws IOException
 */
public long getValue(BytesWritable value) throws IOException {
  DataInputStream dis = getValueStream();
  int size = 0;
  try {
    int remain;
    while ((remain = valueBufferInputStream.getRemain()) > 0) {
      value.setSize(size + remain);
      dis.readFully(value.getBytes(), size, remain);
      size += remain;
    }
    return value.getLength();
  } finally {
    dis.close();
  }
}
 
Example 21
@SuppressWarnings("rawtypes")
public static Class cloverType2Hadoop(DataFieldMetadata field) throws IOException{
	switch (field.getDataType()){
	case BOOLEAN:
		return BooleanWritable.class;
	case BYTE:
	case CBYTE:
		return BytesWritable.class;
	case DATE:
		return LongWritable.class;
	case INTEGER:
		return IntWritable.class;
	case LONG:
		return LongWritable.class;
	case NUMBER:
		return DoubleWritable.class;
	case STRING:
		return Text.class;
	default:
		throw new IOException(String.format("Unsupported CloverDX data type \"%s\" of field \"%s\" in conversion to Hadoop.",field.getDataType().getName(),field.getName()));
		
	}
}
 
Example 22
Source Project: spork   Source File: SequenceFileLoader.java    License: Apache License 2.0 5 votes vote down vote up
protected byte inferPigDataType(Type t) {
  if (t == BytesWritable.class) return DataType.BYTEARRAY;
  else if (t == Text.class) return DataType.CHARARRAY;
  else if (t == IntWritable.class) return DataType.INTEGER;
  else if (t == LongWritable.class) return DataType.LONG;
  else if (t == FloatWritable.class) return DataType.FLOAT;
  else if (t == DoubleWritable.class) return DataType.DOUBLE;
  else if (t == BooleanWritable.class) return DataType.BOOLEAN;
  else if (t == ByteWritable.class) return DataType.BYTE;
  else if (t == DateTimeWritable.class) return DataType.DATETIME;
  // not doing maps or other complex types for now
  else return DataType.ERROR;
}
 
Example 23
Source Project: eagle   Source File: TestGroupAggregateTimeSeriesClient.java    License: Apache License 2.0 5 votes vote down vote up
private void logGroupbyKeyValue(List<GroupbyKeyValue> keyValues) {
    for (GroupbyKeyValue keyValue : keyValues) {
        GroupbyKey key = keyValue.getKey();
        List<String> keys = new ArrayList<String>();
        for (BytesWritable bytes : key.getValue()) {
            keys.add(new String(bytes.copyBytes()));
        }
        List<Double> vals = new ArrayList<Double>();
        GroupbyValue val = keyValue.getValue();
        for (DoubleWritable dw : val.getValue()) {
            vals.add(dw.get());
        }
        if (LOG.isDebugEnabled()) LOG.debug("KEY: " + keys + ", VALUE: " + vals);
    }
}
 
Example 24
Source Project: emr-sample-apps   Source File: CloudBurst.java    License: Apache License 2.0 5 votes vote down vote up
public static void filter(String alignpath, 
	                  String outpath,
                            int nummappers,
                            int numreducers) throws IOException, Exception
  {
System.out.println("NUM_FMAP_TASKS: "     + nummappers);
System.out.println("NUM_FREDUCE_TASKS: "  + numreducers);

JobConf conf = new JobConf(FilterAlignments.class);
conf.setJobName("FilterAlignments");
conf.setNumMapTasks(nummappers);
conf.setNumReduceTasks(numreducers);

FileInputFormat.addInputPath(conf, new Path(alignpath));

conf.setMapperClass(FilterMapClass.class);

conf.setInputFormat(SequenceFileInputFormat.class);			
conf.setMapOutputKeyClass(IntWritable.class);
conf.setMapOutputValueClass(BytesWritable.class);

conf.setCombinerClass(FilterCombinerClass.class);

conf.setReducerClass(FilterReduceClass.class);		
conf.setOutputKeyClass(IntWritable.class);
conf.setOutputValueClass(BytesWritable.class);
conf.setOutputFormat(SequenceFileOutputFormat.class);

Path oPath = new Path(outpath);
FileOutputFormat.setOutputPath(conf, oPath);
System.err.println("  Removing old results");
FileSystem.get(conf).delete(oPath);

JobClient.runJob(conf);

System.err.println("FilterAlignments Finished");		
  }
 
Example 25
@Test
public void emptyListOfFractions() {
  UpdateDoublesSketch sketch = DoublesSketch.builder().build();
  sketch.update(1);
  sketch.update(2);
  sketch.update(3);
  List<Double> result = new GetQuantilesFromDoublesSketchUDF().evaluate(new BytesWritable(sketch.toByteArray()));
  Assert.assertNotNull(result);
  Assert.assertEquals(result.size(), 0);
}
 
Example 26
@Override
public void merge(final @SuppressWarnings("deprecation") AggregationBuffer buf, final Object data)
    throws HiveException {
  if (data == null) { return; }
  final ArrayOfDoublesUnionState state = (ArrayOfDoublesUnionState) buf;
  if (!state.isInitialized()) {
    initializeState(state, data);
  }
  final Memory serializedSketch = BytesWritableHelper.wrapAsMemory(
      (BytesWritable) intermediateInspector_.getStructFieldData(
          data, intermediateInspector_.getStructFieldRef(SKETCH_FIELD)));
  state.update(ArrayOfDoublesSketches.wrapSketch(serializedSketch));
}
 
Example 27
Source Project: big-c   Source File: UtilsForTests.java    License: Apache License 2.0 5 votes vote down vote up
static void writeFile(NameNode namenode, Configuration conf, Path name, 
                      short replication)
    throws IOException, TimeoutException, InterruptedException {
  FileSystem fileSys = FileSystem.get(conf);
  SequenceFile.Writer writer = 
    SequenceFile.createWriter(fileSys, conf, name, 
                              BytesWritable.class, BytesWritable.class,
                              CompressionType.NONE);
  writer.append(new BytesWritable(), new BytesWritable());
  writer.close();
  fileSys.setReplication(name, replication);
  DFSTestUtil.waitReplication(fileSys, name, replication);
}
 
Example 28
@Test
public void complete1ModeDefaultK() throws Exception {
  ObjectInspector[] inspectors = new ObjectInspector[] { binaryInspector };
  GenericUDAFParameterInfo info = new SimpleGenericUDAFParameterInfo(inspectors, false, false, false);
  try (GenericUDAFEvaluator eval = new UnionStringsSketchUDAF().getEvaluator(info)) {
    ObjectInspector resultInspector = eval.init(Mode.COMPLETE, inspectors);
    DataToDoublesSketchUDAFTest.checkResultInspector(resultInspector);

    @SuppressWarnings("unchecked")
    ItemsUnionState<String> state = (ItemsUnionState<String>) eval.getNewAggregationBuffer();

    ItemsSketch<String> sketch1 = ItemsSketch.getInstance(comparator);
    sketch1.update("a");
    eval.iterate(state, new Object[] { new BytesWritable(sketch1.toByteArray(serDe)) });

    ItemsSketch<String> sketch2 = ItemsSketch.getInstance(comparator);
    sketch2.update("b");
    eval.iterate(state, new Object[] { new BytesWritable(sketch2.toByteArray(serDe)) });

    BytesWritable bytes = (BytesWritable) eval.terminate(state);
    ItemsSketch<String> resultSketch = ItemsSketch.getInstance(BytesWritableHelper.wrapAsMemory(bytes), comparator, serDe);
    Assert.assertEquals(resultSketch.getK(), 128);
    Assert.assertEquals(resultSketch.getRetainedItems(), 2);
    Assert.assertEquals(resultSketch.getMinValue(), "a");
    Assert.assertEquals(resultSketch.getMaxValue(), "b");

    eval.reset(state);
    Assert.assertNull(eval.terminate(state));
  }
}
 
Example 29
Source Project: emr-sample-apps   Source File: CountKmers.java    License: Apache License 2.0 5 votes vote down vote up
public synchronized void reduce(BytesWritable mer, Iterator<IntWritable> values,
		OutputCollector<Text, Text> output, Reporter reporter)
		throws IOException 
{
	int cnt = 0;
	builder.setLength(0);
	
	while (values.hasNext()) 
	{
		cnt++;
		if (SHOW_POS)
		{
			builder.append('\t');
			builder.append(values.next().get());
		}
	}
	
	String val = DNAString.bytesToString(DNAString.bytesWritableDNAToArr(mer));
	mertext.set(val);
	
	if (SHOW_POS)
	{
		builder.insert(0, cnt);
		String locs = builder.toString();
		locations.set(locs);
	}
	else
	{
		locations.set(Integer.toString(cnt));
	}
	
	output.collect(mertext, locations);
}
 
Example 30
Source Project: hadoop-gpu   Source File: MapTask.java    License: Apache License 2.0 5 votes vote down vote up
public MapTask(String jobFile, TaskAttemptID taskId, 
               int partition, String splitClass, BytesWritable split
               ) {
  super(jobFile, taskId, partition);
  this.splitClass = splitClass;
  this.split = split;
}