Java Code Examples for org.apache.hadoop.io.BytesWritable#getBytes()

public BytesWritable evaluate(BytesWritable wkb, int wkid) throws UDFArgumentException {

		try {
			SpatialReference spatialReference = null;
			if (wkid != GeometryUtils.WKID_UNKNOWN) {
				spatialReference = SpatialReference.create(wkid);
			}
			byte [] byteArr = wkb.getBytes();
            ByteBuffer byteBuf = ByteBuffer.allocate(byteArr.length);
			byteBuf.put(byteArr);
			OGCGeometry ogcObj = OGCGeometry.fromBinary(byteBuf);
			ogcObj.setSpatialReference(spatialReference);
			if (ogcObj.geometryType().equals("Point")) {
				return GeometryUtils.geometryToEsriShapeBytesWritable(ogcObj);
			} else {
				LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_POINT, GeometryUtils.OGCType.UNKNOWN);
				return null;
			}
		} catch (Exception e) {  // IllegalArgumentException, GeometryException
			LOG.error(e.getMessage());
			return null;
		}
	}
 

@Override
public void write(BytesWritable key, BytesWritable value)
    throws IOException, InterruptedException {
  // Mapreduce reuses the same Text objects and hence sometimes they have some
  // additional left over garbage at the end from previous keys. So we need to
  // retrieve the String objects which do not have the additional garbage. getBytes()
  // call does not work well.
  byte[] row = new byte[key.getLength()];
  byte[] val = new byte[value.getLength()];
  for (int i = 0; i < row.length; i++) {
    row[i] = key.getBytes()[i];
  }
  for (int i = 0; i < val.length; i++) {
    val[i] = value.getBytes()[i];
  }
  writer.append(new KeyValue(row,
      Bytes.toBytes("cf"),
      Bytes.toBytes(""),
      val));
}
 

protected static HashMap<Integer, HashMap<Integer, Declaration>> collectDeclarations(final SequenceFile.Reader ar, List<ChangedFile> snapshot) throws IOException {
	HashMap<Integer, HashMap<Integer, Declaration>> fileNodeDeclaration = new HashMap<Integer, HashMap<Integer, Declaration>>();
	for (int fileIndex = 0; fileIndex < snapshot.size(); fileIndex++) {
		ChangedFile cf = snapshot.get(fileIndex);
		long astpos = cf.getKey();
		if (!cf.getAst())
			continue;
		if (astpos > -1) {
			ar.seek(astpos);
			Writable astkey = new LongWritable();
			BytesWritable val = new BytesWritable();
			ar.next(astkey, val);
			byte[] bytes = val.getBytes();
			ASTRoot root = ASTRoot.parseFrom(CodedInputStream.newInstance(bytes, 0, val.getLength()));
			HashMap<Integer, Declaration> nodeDeclaration = collectDeclarations(root);
			fileNodeDeclaration.put(fileIndex, nodeDeclaration);
		}
	}
	return fileNodeDeclaration;
}
 

@Override
public void map(LongWritable key, BytesWritable value, Context context)
    throws IOException, InterruptedException {

  String fileName = new String(value.getBytes(), 0,
      value.getLength(), charsetUTF8);
  Path path = new Path(fileName);

  FSDataOutputStream dos =
      FileSystem.create(fs, path, new FsPermission(GRIDMIX_DISTCACHE_FILE_PERM));

  int size = 0;
  for (long bytes = key.get(); bytes > 0; bytes -= size) {
    r.nextBytes(val.getBytes());
    size = (int)Math.min(val.getLength(), bytes);
    dos.write(val.getBytes(), 0, size);// Write to distCache file
  }
  dos.close();
}
 

public BytesWritable evaluate(BytesWritable wkb, int wkid) throws UDFArgumentException {

		try {
			SpatialReference spatialReference = null;
			if (wkid != GeometryUtils.WKID_UNKNOWN) {
				spatialReference = SpatialReference.create(wkid);
			}
			byte [] byteArr = wkb.getBytes();
            ByteBuffer byteBuf = ByteBuffer.allocate(byteArr.length);
			byteBuf.put(byteArr);
			OGCGeometry ogcObj = OGCGeometry.fromBinary(byteBuf);
			ogcObj.setSpatialReference(spatialReference);
			String gType = ogcObj.geometryType();
			if (gType.equals("MultiPoint") || gType.equals("Point")) {
				return GeometryUtils.geometryToEsriShapeBytesWritable(ogcObj);
			} else {
				LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_LINESTRING, GeometryUtils.OGCType.UNKNOWN);
				return null;
			}
		} catch (Exception e) {  // IllegalArgumentException, GeometryException
			LOG.error(e.getMessage());
			return null;
		}
	}
 

/**
 * @param compressionAlgo
 *          The compression algorithm to be used to for compression.
 * @throws IOException
 */
public WBlockState(Algorithm compressionAlgo, FSDataOutputStream fsOut,
    BytesWritable fsOutputBuffer, Configuration conf) throws IOException {
  this.compressAlgo = compressionAlgo;
  this.fsOut = fsOut;
  this.posStart = fsOut.getPos();

  fsOutputBuffer.setCapacity(TFile.getFSOutputBufferSize(conf));

  this.fsBufferedOutput =
      new SimpleBufferedOutputStream(this.fsOut, fsOutputBuffer.getBytes());
  this.compressor = compressAlgo.getCompressor();

  try {
    this.out =
        compressionAlgo.createCompressionStream(fsBufferedOutput,
            compressor, 0);
  } catch (IOException e) {
    compressAlgo.returnCompressor(compressor);
    throw e;
  }
}
 

public BytesWritable evaluate(BytesWritable wkb, int wkid) throws UDFArgumentException {

		try {
			SpatialReference spatialReference = null;
			if (wkid != GeometryUtils.WKID_UNKNOWN) {
				spatialReference = SpatialReference.create(wkid);
			}
			byte [] byteArr = wkb.getBytes();
            ByteBuffer byteBuf = ByteBuffer.allocate(byteArr.length);
			byteBuf.put(byteArr);
			OGCGeometry ogcObj = OGCGeometry.fromBinary(byteBuf);
			ogcObj.setSpatialReference(spatialReference);
			if (ogcObj.geometryType().equals("Polygon")) {
				return GeometryUtils.geometryToEsriShapeBytesWritable(ogcObj);
			} else {
				LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_POLYGON, GeometryUtils.OGCType.UNKNOWN);
				return null;
			}
		} catch (Exception e) {  // IllegalArgumentException, GeometryException
			LOG.error(e.getMessage());
			return null;
		}
	}
 

private static Rename[] readRenameOptions(DataInputStream in) throws IOException {
  BytesWritable writable = new BytesWritable();
  writable.readFields(in);

  byte[] bytes = writable.getBytes();
  Rename[] options = new Rename[bytes.length];

  for (int i = 0; i < bytes.length; i++) {
    options[i] = Rename.valueOf(bytes[i]);
  }
  return options;
}
 

public String getString(Object k) {
  BytesWritable bytes = get(k);
  if (null == bytes) {
    return null;
  } else {
    try {
      return new String(bytes.getBytes(), 0, bytes.getLength(), "UTF-8");
    } catch (UnsupportedEncodingException uee) {
      // Shouldn't happen; UTF-8 is always supported.
      throw new RuntimeException(uee);
    }
  }
}
 

public static BytesWritable bytes(int... items) {
  BytesWritable result = new BytesWritable();
  result.setSize(items.length);
  for(int i=0; i < items.length; ++i) {
    result.getBytes()[i] = (byte) items[i];
  }
  return result;
}
 

protected static Message getMessage(final SequenceFile.Reader ar, final ChangedFile cf, final int nodeId) {
	long astpos = cf.getKey();
	if (cf.getAst() && astpos > -1) {
		try {
			ar.seek(astpos);
			Writable astkey = new LongWritable();
			BytesWritable val = new BytesWritable();
			ar.next(astkey, val);
			byte[] bytes = val.getBytes();
			ASTRoot root = ASTRoot.parseFrom(CodedInputStream.newInstance(bytes, 0, val.getLength()));
			return getMessage(root, nodeId);
		} catch (IOException e) {}
	}
	return null;
}
 

private void printBytes(BytesWritable key) {
  byte[] bytes = key.getBytes();
  for (int i = 0; i < key.getLength(); i++) {
    System.out.printf("%02x", bytes[i]);
  }
  System.out.println();
}
 

/**
 * Validate setupGenerateDistCacheData by validating <li>permissions of the
 * distributed cache directory and <li>content of the generated sequence file.
 * This includes validation of dist cache file paths and their file sizes.
 */
private void doValidateSetupGenDC(
    RecordReader<LongWritable, BytesWritable> reader, FileSystem fs,
    long[] sortedFileSizes) throws IOException, InterruptedException {

  // Validate permissions of dist cache directory
  Path distCacheDir = dce.getDistributedCacheDir();
  assertEquals(
      "Wrong permissions for distributed cache dir " + distCacheDir,
      fs.getFileStatus(distCacheDir).getPermission().getOtherAction()
          .and(FsAction.EXECUTE), FsAction.EXECUTE);

  // Validate the content of the sequence file generated by
  // dce.setupGenerateDistCacheData().
  LongWritable key = new LongWritable();
  BytesWritable val = new BytesWritable();
  for (int i = 0; i < sortedFileSizes.length; i++) {
    assertTrue("Number of files written to the sequence file by "
        + "setupGenerateDistCacheData is less than the expected.",
        reader.nextKeyValue());
    key = reader.getCurrentKey();
    val = reader.getCurrentValue();
    long fileSize = key.get();
    String file = new String(val.getBytes(), 0, val.getLength());

    // Dist Cache files should be sorted based on file size.
    assertEquals("Dist cache file size is wrong.", sortedFileSizes[i],
        fileSize);

    // Validate dist cache file path.

    // parent dir of dist cache file
    Path parent = new Path(file).getParent().makeQualified(fs.getUri(),fs.getWorkingDirectory());
    // should exist in dist cache dir
    assertTrue("Public dist cache file path is wrong.",
        distCacheDir.equals(parent));
  }
}
 

private void runRandomTests(CompressionCodec codec) throws IOException {
  StringBuilder fileName = new StringBuilder("testFormat.txt");
  if (codec != null) {
    fileName.append(".gz");
  }
  localFs.delete(workDir, true);
  Path file = new Path(workDir, fileName.toString());
  int seed = new Random().nextInt();
  LOG.info("Seed = " + seed);
  Random random = new Random(seed);
  int MAX_TESTS = 20;
  LongWritable key = new LongWritable();
  BytesWritable value = new BytesWritable();

  for (int i = 0; i < MAX_TESTS; i++) {
    LOG.info("----------------------------------------------------------");
    // Maximum total records of 999
    int totalRecords = random.nextInt(999)+1;
    // Test an empty file
    if (i == 8) {
       totalRecords = 0;
    }
    // Maximum bytes in a record of 100K
    int recordLength = random.nextInt(1024*100)+1;
    // For the 11th test, force a record length of 1
    if (i == 10) {
      recordLength = 1;
    }
    // The total bytes in the test file
    int fileSize = (totalRecords * recordLength);
    LOG.info("totalRecords=" + totalRecords + " recordLength="
        + recordLength);
    // Create the job 
    JobConf job = new JobConf(defaultConf);
    if (codec != null) {
      ReflectionUtils.setConf(codec, job);
    }
    // Create the test file
    ArrayList<String> recordList
        = createFile(file, codec, recordLength, totalRecords);
    assertTrue(localFs.exists(file));
    //set the fixed length record length config property for the job
    FixedLengthInputFormat.setRecordLength(job, recordLength);

    int numSplits = 1;
    // Arbitrarily set number of splits.
    if (i > 0) {
      if (i == (MAX_TESTS-1)) {
        // Test a split size that is less than record len
        numSplits = (int)(fileSize/Math.floor(recordLength/2));
      } else {
        if (MAX_TESTS % i == 0) {
          // Let us create a split size that is forced to be 
          // smaller than the end file itself, (ensures 1+ splits)
          numSplits = fileSize/(fileSize - random.nextInt(fileSize));
        } else {
          // Just pick a random split size with no upper bound 
          numSplits = Math.max(1, fileSize/random.nextInt(Integer.MAX_VALUE));
        }
      }
      LOG.info("Number of splits set to: " + numSplits);
    }

    // Setup the input path
    FileInputFormat.setInputPaths(job, workDir);
    // Try splitting the file in a variety of sizes
    FixedLengthInputFormat format = new FixedLengthInputFormat();
    format.configure(job);
    InputSplit splits[] = format.getSplits(job, numSplits);
    LOG.info("Actual number of splits = " + splits.length);
    // Test combined split lengths = total file size
    long recordOffset = 0;
    int recordNumber = 0;
    for (InputSplit split : splits) {
      RecordReader<LongWritable, BytesWritable> reader = 
          format.getRecordReader(split, job, voidReporter);
      Class<?> clazz = reader.getClass();
      assertEquals("RecordReader class should be FixedLengthRecordReader:", 
          FixedLengthRecordReader.class, clazz);
      // Plow through the records in this split
      while (reader.next(key, value)) {
        assertEquals("Checking key", (long)(recordNumber*recordLength),
            key.get());
        String valueString =
            new String(value.getBytes(), 0, value.getLength());
        assertEquals("Checking record length:", recordLength,
            value.getLength());
        assertTrue("Checking for more records than expected:",
            recordNumber < totalRecords);
        String origRecord = recordList.get(recordNumber);
        assertEquals("Checking record content:", origRecord, valueString);
        recordNumber++;
      }
      reader.close();
    }
    assertEquals("Total original records should be total read records:",
        recordList.size(), recordNumber);
  }
}
 

protected InputStream getInputStreamFromKeyValue(NullWritable key, BytesWritable value) {
    return new ByteArrayInputStream(value.getBytes(), 0, value.getLength());
}
 

private void runRandomTests(CompressionCodec codec) throws IOException {
  StringBuilder fileName = new StringBuilder("testFormat.txt");
  if (codec != null) {
    fileName.append(".gz");
  }
  localFs.delete(workDir, true);
  Path file = new Path(workDir, fileName.toString());
  int seed = new Random().nextInt();
  LOG.info("Seed = " + seed);
  Random random = new Random(seed);
  int MAX_TESTS = 20;
  LongWritable key = new LongWritable();
  BytesWritable value = new BytesWritable();

  for (int i = 0; i < MAX_TESTS; i++) {
    LOG.info("----------------------------------------------------------");
    // Maximum total records of 999
    int totalRecords = random.nextInt(999)+1;
    // Test an empty file
    if (i == 8) {
       totalRecords = 0;
    }
    // Maximum bytes in a record of 100K
    int recordLength = random.nextInt(1024*100)+1;
    // For the 11th test, force a record length of 1
    if (i == 10) {
      recordLength = 1;
    }
    // The total bytes in the test file
    int fileSize = (totalRecords * recordLength);
    LOG.info("totalRecords=" + totalRecords + " recordLength="
        + recordLength);
    // Create the job 
    JobConf job = new JobConf(defaultConf);
    if (codec != null) {
      ReflectionUtils.setConf(codec, job);
    }
    // Create the test file
    ArrayList<String> recordList
        = createFile(file, codec, recordLength, totalRecords);
    assertTrue(localFs.exists(file));
    //set the fixed length record length config property for the job
    FixedLengthInputFormat.setRecordLength(job, recordLength);

    int numSplits = 1;
    // Arbitrarily set number of splits.
    if (i > 0) {
      if (i == (MAX_TESTS-1)) {
        // Test a split size that is less than record len
        numSplits = (int)(fileSize/Math.floor(recordLength/2));
      } else {
        if (MAX_TESTS % i == 0) {
          // Let us create a split size that is forced to be 
          // smaller than the end file itself, (ensures 1+ splits)
          numSplits = fileSize/(fileSize - random.nextInt(fileSize));
        } else {
          // Just pick a random split size with no upper bound 
          numSplits = Math.max(1, fileSize/random.nextInt(Integer.MAX_VALUE));
        }
      }
      LOG.info("Number of splits set to: " + numSplits);
    }

    // Setup the input path
    FileInputFormat.setInputPaths(job, workDir);
    // Try splitting the file in a variety of sizes
    FixedLengthInputFormat format = new FixedLengthInputFormat();
    format.configure(job);
    InputSplit splits[] = format.getSplits(job, numSplits);
    LOG.info("Actual number of splits = " + splits.length);
    // Test combined split lengths = total file size
    long recordOffset = 0;
    int recordNumber = 0;
    for (InputSplit split : splits) {
      RecordReader<LongWritable, BytesWritable> reader = 
          format.getRecordReader(split, job, voidReporter);
      Class<?> clazz = reader.getClass();
      assertEquals("RecordReader class should be FixedLengthRecordReader:", 
          FixedLengthRecordReader.class, clazz);
      // Plow through the records in this split
      while (reader.next(key, value)) {
        assertEquals("Checking key", (long)(recordNumber*recordLength),
            key.get());
        String valueString =
            new String(value.getBytes(), 0, value.getLength());
        assertEquals("Checking record length:", recordLength,
            value.getLength());
        assertTrue("Checking for more records than expected:",
            recordNumber < totalRecords);
        String origRecord = recordList.get(recordNumber);
        assertEquals("Checking record content:", origRecord, valueString);
        recordNumber++;
      }
      reader.close();
    }
    assertEquals("Total original records should be total read records:",
        recordList.size(), recordNumber);
  }
}
 

private Object toHCat(Object val, HCatFieldSchema hfs) {
  HCatFieldSchema.Type hfsType = hfs.getType();
  if (val == null) {
    return null;
  }

  Object retVal = null;

  if (val instanceof Number) {
    retVal = convertNumberTypes(val, hfs);
  } else if (val instanceof Boolean) {
    retVal = convertBooleanTypes(val, hfs);
  } else if (val instanceof String) {
    retVal = convertStringTypes(val, hfs);
  } else if (val instanceof java.util.Date) {
    retVal = converDateTypes(val, hfs);
  } else if (val instanceof BytesWritable) {
    if (hfsType == HCatFieldSchema.Type.BINARY) {
      BytesWritable bw = (BytesWritable) val;
      retVal = bw.getBytes();
    }
  } else if (val instanceof BlobRef) {
    if (hfsType == HCatFieldSchema.Type.BINARY) {
      BlobRef br = (BlobRef) val;
      byte[] bytes = br.isExternal() ? br.toString().getBytes() : br
        .getData();
      retVal = bytes;
    }
  } else if (val instanceof ClobRef) {
    retVal = convertClobType(val, hfs);
  } else {
    throw new UnsupportedOperationException("Objects of type "
      + val.getClass().getName() + " are not suported");
  }
  if (retVal == null) {
    LOG.error("Unable to convert [" + val
      + "]  of type " + val.getClass().getName()
      + " to HCatalog type " + hfs.getTypeString());
  }
  return retVal;
}
 

@Override
public void readFields(DataInput in) throws IOException {
    // 0. Timestamp
    Text tms_text = new Text(tms);
    tms_text.readFields(in);
    tms = tms_text.toString();

    // 1. integers
    IntWritable intw = new IntWritable();

    for (int i = 0; i < num.length; i++) {
        intw.readFields(in);
        num[i] = intw.get();
    }

    intw.readFields(in);
    int1 = intw.get();

    intw.readFields(in);
    int2 = intw.get();

    // 2. strings
    Text txt = new Text();

    for (int i = 0; i < strings.length; i++) {
        txt.readFields(in);
        strings[i] = txt.toString();
    }

    txt.readFields(in);
    st1 = txt.toString();

    // 3. doubles
    DoubleWritable dw = new DoubleWritable();
    for (int i = 0; i < dubs.length; i++) {
        dw.readFields(in);
        dubs[i] = dw.get();
    }

    dw.readFields(in);
    db = dw.get();

    // 4. floats
    FloatWritable fw = new FloatWritable();
    for (int i = 0; i < fts.length; i++) {
        fw.readFields(in);
        fts[i] = fw.get();
    }

    fw.readFields(in);
    ft = fw.get();

    // 5. longs
    LongWritable lw = new LongWritable();
    for (int i = 0; i < lngs.length; i++) {
        lw.readFields(in);
        lngs[i] = lw.get();
    }

    lw.readFields(in);
    lng = lw.get();

    // 6. booleans
    BooleanWritable bw = new BooleanWritable();
    for (int i = 0; i < bools.length; ++i) {
        bw.readFields(in);
        bools[i] = bw.get();
    }

    bw.readFields(in);
    bool = bw.get();

    // 7. shorts
    ShortWritable sw = new ShortWritable();
    for (int i = 0; i < shrts.length; ++i) {
        sw.readFields(in);
        shrts[i] = sw.get();
    }
    sw.readFields(in);
    shrt = sw.get();

    // 8. bytes
    BytesWritable btsw = new BytesWritable();
    btsw.readFields(in);
    byte[] buffer = btsw.getBytes();
    bts = new byte[btsw.getLength()];
    for (int i = 0; i < btsw.getLength(); i++) {
        bts[i] = buffer[i];
    }
}
 

private static ByteBuffer getShapeByteBuffer(BytesWritable geomref){
	byte [] geomBytes = geomref.getBytes();
	int offset = SIZE_WKID + SIZE_TYPE;
	
	return ByteBuffer.wrap(geomBytes, offset, geomBytes.length - offset).slice().order(ByteOrder.LITTLE_ENDIAN);
}
 

public void map(LongWritable key, BytesWritable value, Context context ) 
	throws IOException, InterruptedException {
String val = new String(value.getBytes(), 0, value.getLength()-1);	
String[] strs = val.split(" ");
context.write(new Text(strs[0]), new IntWritable(Integer.parseInt(strs[1])));
  }