Java Code Examples for org.apache.hadoop.io.BytesWritable#getLength()

public DoubleWritable evaluate(BytesWritable geomref) {
	if (geomref == null || geomref.getLength() == 0) {
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
	if (ogcGeometry == null) {
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}
	if (!ogcGeometry.is3D()) {
		LogUtils.Log_Not3D(LOG);
		return null;
	}

	resultDouble.set(ogcGeometry.MinZ());
	return resultDouble;
}
 

/**
 * Copy the value into BytesWritable. The input BytesWritable will be
 * automatically resized to the actual value size. The implementation
 * directly uses the buffer inside BytesWritable for storing the value.
 * The call does not require the value length to be known.
 * 
 * @param value
 * @throws IOException
 */
public long getValue(BytesWritable value) throws IOException {
  DataInputStream dis = getValueStream();
  int size = 0;
  try {
    int remain;
    while ((remain = valueBufferInputStream.getRemain()) > 0) {
      value.setSize(size + remain);
      dis.readFully(value.getBytes(), size, remain);
      size += remain;
    }
    return value.getLength();
  } finally {
    dis.close();
  }
}
 

public DoubleWritable evaluate(BytesWritable geomref) {
	if (geomref == null || geomref.getLength() == 0) {
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
	if (ogcGeometry == null){
		return null;
	}
	if (!ogcGeometry.is3D()) {
		LogUtils.Log_Not3D(LOG);
		return null;
	}

	switch(GeometryUtils.getType(geomref)) {
	case ST_POINT:
		OGCPoint pt = (OGCPoint)ogcGeometry;
		resultDouble.set(pt.Z());
		return resultDouble;
	default:
		LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_POINT, GeometryUtils.getType(geomref));
		return null;
	}
}
 

public void printFile(String path) throws Exception {
    FileSystem fileSystem = FileUtil.getFileSystem(path);
    Path fsPath = new Path(path);
    SequenceFile.Reader reader = new SequenceFile.Reader(fileSystem, fsPath,
            new Configuration());
    LongWritable key = (LongWritable) reader.getKeyClass().newInstance();
    BytesWritable value = (BytesWritable) reader.getValueClass().newInstance();
    System.out.println("reading file " + path);
    while (reader.next(key, value)) {
        if (mPrintOffsetsOnly) {
            System.out.println(Long.toString(key.get()));
        } else {
            byte[] nonPaddedBytes = new byte[value.getLength()];
            System.arraycopy(value.getBytes(), 0, nonPaddedBytes, 0, value.getLength());
            System.out.println(Long.toString(key.get()) + ": " + new String(nonPaddedBytes)); 
        }
    }
}
 

public BytesWritable evaluate(BytesWritable geomref) {
	if (geomref == null || geomref.getLength() == 0){
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
	if (ogcGeometry == null){
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	try {
		// Get Esri shape representation
		Geometry esriGeometry = ogcGeometry.getEsriGeometry();
		byte[] esriShape = GeometryEngine.geometryToEsriShape(esriGeometry);
		return new BytesWritable(esriShape);
	} catch (Exception e){
		LOG.error(e.getMessage());
		return null;
	}
}
 

public Text evaluate(BytesWritable geomref){
	if (geomref == null || geomref.getLength() == 0){
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
	if (ogcGeometry == null){
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	Geometry esriGeom = ogcGeometry.getEsriGeometry();
	int wkid = GeometryUtils.getWKID(geomref);
	return new Text(GeometryEngine.geometryToJson(wkid, esriGeom));
}
 

public BytesWritable evaluate(BytesWritable geometryref)
{
	if (geometryref == null || geometryref.getLength() == 0) {
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geometryref);
	if (ogcGeometry == null){
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	int wkid = GeometryUtils.getWKID(geometryref);
	SpatialReference spatialReference = null;
	if (wkid != GeometryUtils.WKID_UNKNOWN) {
		spatialReference = SpatialReference.create(wkid);
	}
	Envelope envBound = new Envelope();
	ogcGeometry.getEsriGeometry().queryEnvelope(envBound);
	return GeometryUtils.geometryToEsriShapeBytesWritable(OGCGeometry.createFromEsriGeometry(envBound,
															  spatialReference));
}
 

@Override
public LWDocument[] toDocuments(Writable key, Writable value, Reporter reporter,
    Configuration conf) throws IOException {
  LWDocument doc = createDocument(key.toString(), null);
  if (value instanceof Text) {
    doc.setContent(((Text) value).getBytes());
    reporter.getCounter(Counters.TEXT).increment(1);
  } else if (value instanceof BytesWritable) {
    // Copy the bytes for this one
    BytesWritable value_ = (BytesWritable) value;
    byte[] data = new byte[value_.getLength()];
    doc.setContent(data);
    System.arraycopy(value_.getBytes(), 0, data, 0, value_.getLength());
    reporter.getCounter(Counters.BYTES_WRITABLE).increment(1);
  } else {
    doc.setContent(WritableUtils.toByteArray(value));
    reporter.getCounter(Counters.RAW_WRITABLE).increment(1);
  }
  return new LWDocument[] {doc};
}
 

public DoubleWritable evaluate(BytesWritable geomref) {
	if (geomref == null || geomref.getLength() == 0) {
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
	if (ogcGeometry == null){
		return null;
	}
	if (!ogcGeometry.isMeasured()) {
		LogUtils.Log_NotMeasured(LOG);
		return null;
	}

	switch(GeometryUtils.getType(geomref)) {
	case ST_POINT:
		OGCPoint pt = (OGCPoint)ogcGeometry;
		resultDouble.set(pt.M());
		return resultDouble;
	default:
		LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_POINT, GeometryUtils.getType(geomref));
		return null;
	}
}
 

@Override
protected void map(BytesWritable key, NullWritable value, Context output) throws IOException {
  current[i] = new byte[key.getLength()];
  System.arraycopy(key.getBytes(), 0, current[i], 0, key.getLength());
  if (++i == current.length) {
    LOG.debug("Persisting current.length={}, count={}, id={}, current={}, i=",
      current.length, count, Bytes.toStringBinary(id), Bytes.toStringBinary(current[0]), i);
    persist(output, count, prev, current, id);
    i = 0;

    if (first == null) {
      first = current;
    }
    prev = current;
    current = new byte[this.width][];

    count += current.length;
    output.setStatus("Count " + count);

    if (count % wrap == 0) {
      // this block of code turns the 1 million linked list of length 25 into one giant
      //circular linked list of 25 million
      circularLeftShift(first);
      persist(output, -1, prev, first, null);
      // At this point the entire loop has been flushed so we can add one of its nodes to the
      // concurrent walker
      if (numWalkers > 0) {
        addFlushed(key.getBytes());
        if (walkers.isEmpty()) {
          startWalkers(numWalkers, conf, output);
        }
      }
      first = null;
      prev = null;
    }
  }
}
 

/**
 * Validate setupGenerateDistCacheData by validating <li>permissions of the
 * distributed cache directory and <li>content of the generated sequence file.
 * This includes validation of dist cache file paths and their file sizes.
 */
private void doValidateSetupGenDC(
    RecordReader<LongWritable, BytesWritable> reader, FileSystem fs,
    long[] sortedFileSizes) throws IOException, InterruptedException {

  // Validate permissions of dist cache directory
  Path distCacheDir = dce.getDistributedCacheDir();
  assertEquals(
      "Wrong permissions for distributed cache dir " + distCacheDir,
      fs.getFileStatus(distCacheDir).getPermission().getOtherAction()
          .and(FsAction.EXECUTE), FsAction.EXECUTE);

  // Validate the content of the sequence file generated by
  // dce.setupGenerateDistCacheData().
  LongWritable key = new LongWritable();
  BytesWritable val = new BytesWritable();
  for (int i = 0; i < sortedFileSizes.length; i++) {
    assertTrue("Number of files written to the sequence file by "
        + "setupGenerateDistCacheData is less than the expected.",
        reader.nextKeyValue());
    key = reader.getCurrentKey();
    val = reader.getCurrentValue();
    long fileSize = key.get();
    String file = new String(val.getBytes(), 0, val.getLength());

    // Dist Cache files should be sorted based on file size.
    assertEquals("Dist cache file size is wrong.", sortedFileSizes[i],
        fileSize);

    // Validate dist cache file path.

    // parent dir of dist cache file
    Path parent = new Path(file).getParent().makeQualified(fs.getUri(),fs.getWorkingDirectory());
    // should exist in dist cache dir
    assertTrue("Public dist cache file path is wrong.",
        distCacheDir.equals(parent));
  }
}
 

public BytesWritable evaluate(BytesWritable geometryref1, BytesWritable geometryref2)
{
	if (geometryref1 == null || geometryref2 == null ||
	    geometryref1.getLength() == 0 || geometryref2.getLength() == 0) {
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}
	
	if (!GeometryUtils.compareSpatialReferences(geometryref1, geometryref2)) {
		LogUtils.Log_SRIDMismatch(LOG, geometryref1, geometryref2);
		return null;
	}

	OGCGeometry ogcGeom1 = GeometryUtils.geometryFromEsriShape(geometryref1);
	OGCGeometry ogcGeom2 = GeometryUtils.geometryFromEsriShape(geometryref2);
	if (ogcGeom1 == null || ogcGeom2 == null){
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}
	
	try {
		OGCGeometry diffGeometry = ogcGeom1.symDifference(ogcGeom2);
		return GeometryUtils.geometryToEsriShapeBytesWritable(diffGeometry);
	} catch (Exception e) {
	    LogUtils.Log_InternalError(LOG, "ST_SymmetricDiff: " + e);
	    return null;
	}
}
 

/** Concatenate map outputs. */
@Override
protected void reduce(LongWritable offset, Iterable<BytesWritable> values,
    Context context) throws IOException, InterruptedException {
  // read map outputs
  for (BytesWritable bytes : values) {
    for (int i = 0; i < bytes.getLength(); i++)
      hex.add(bytes.getBytes()[i]);
  }

  LOG.info("hex.size() = " + hex.size());
}
 

/**
 * Return the first point of the ST_Linestring.
 * @param geomref hive geometry bytes
 * @return byte-reference of the first ST_Point
 */
public BytesWritable evaluate(BytesWritable geomref) {
	if (geomref == null || geomref.getLength() == 0){
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
	if (ogcGeometry == null){
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	if (GeometryUtils.getType(geomref) == GeometryUtils.OGCType.ST_LINESTRING) {
		MultiPath lines = (MultiPath)(ogcGeometry.getEsriGeometry());
		int wkid = GeometryUtils.getWKID(geomref);
		SpatialReference spatialReference = null;
		if (wkid != GeometryUtils.WKID_UNKNOWN) {
			spatialReference = SpatialReference.create(wkid);
		}
		return GeometryUtils.geometryToEsriShapeBytesWritable(OGCGeometry.createFromEsriGeometry(lines.getPoint(0),
																								 spatialReference));
	} else {
		LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_LINESTRING, GeometryUtils.getType(geomref));
		return null;
	}
}
 

public static void writeBytesWritable(BytesWritable val, int paramIdx,
    int sqlType, PreparedStatement s) throws SQLException {
  if (null == val) {
    s.setNull(paramIdx, sqlType);
  } else {
    // val.getBytes() is only valid in [0, len)
    byte [] rawBytes = val.getBytes();
    int len = val.getLength();
    byte [] outBytes = new byte[len];
    System.arraycopy(rawBytes, 0, outBytes, 0, len);
    s.setBytes(paramIdx, outBytes);
  }
}
 

private void runRandomTests(CompressionCodec codec) throws IOException {
  StringBuilder fileName = new StringBuilder("testFormat.txt");
  if (codec != null) {
    fileName.append(".gz");
  }
  localFs.delete(workDir, true);
  Path file = new Path(workDir, fileName.toString());
  int seed = new Random().nextInt();
  LOG.info("Seed = " + seed);
  Random random = new Random(seed);
  int MAX_TESTS = 20;
  LongWritable key = new LongWritable();
  BytesWritable value = new BytesWritable();

  for (int i = 0; i < MAX_TESTS; i++) {
    LOG.info("----------------------------------------------------------");
    // Maximum total records of 999
    int totalRecords = random.nextInt(999)+1;
    // Test an empty file
    if (i == 8) {
       totalRecords = 0;
    }
    // Maximum bytes in a record of 100K
    int recordLength = random.nextInt(1024*100)+1;
    // For the 11th test, force a record length of 1
    if (i == 10) {
      recordLength = 1;
    }
    // The total bytes in the test file
    int fileSize = (totalRecords * recordLength);
    LOG.info("totalRecords=" + totalRecords + " recordLength="
        + recordLength);
    // Create the job 
    JobConf job = new JobConf(defaultConf);
    if (codec != null) {
      ReflectionUtils.setConf(codec, job);
    }
    // Create the test file
    ArrayList<String> recordList
        = createFile(file, codec, recordLength, totalRecords);
    assertTrue(localFs.exists(file));
    //set the fixed length record length config property for the job
    FixedLengthInputFormat.setRecordLength(job, recordLength);

    int numSplits = 1;
    // Arbitrarily set number of splits.
    if (i > 0) {
      if (i == (MAX_TESTS-1)) {
        // Test a split size that is less than record len
        numSplits = (int)(fileSize/Math.floor(recordLength/2));
      } else {
        if (MAX_TESTS % i == 0) {
          // Let us create a split size that is forced to be 
          // smaller than the end file itself, (ensures 1+ splits)
          numSplits = fileSize/(fileSize - random.nextInt(fileSize));
        } else {
          // Just pick a random split size with no upper bound 
          numSplits = Math.max(1, fileSize/random.nextInt(Integer.MAX_VALUE));
        }
      }
      LOG.info("Number of splits set to: " + numSplits);
    }

    // Setup the input path
    FileInputFormat.setInputPaths(job, workDir);
    // Try splitting the file in a variety of sizes
    FixedLengthInputFormat format = new FixedLengthInputFormat();
    format.configure(job);
    InputSplit splits[] = format.getSplits(job, numSplits);
    LOG.info("Actual number of splits = " + splits.length);
    // Test combined split lengths = total file size
    long recordOffset = 0;
    int recordNumber = 0;
    for (InputSplit split : splits) {
      RecordReader<LongWritable, BytesWritable> reader = 
          format.getRecordReader(split, job, voidReporter);
      Class<?> clazz = reader.getClass();
      assertEquals("RecordReader class should be FixedLengthRecordReader:", 
          FixedLengthRecordReader.class, clazz);
      // Plow through the records in this split
      while (reader.next(key, value)) {
        assertEquals("Checking key", (long)(recordNumber*recordLength),
            key.get());
        String valueString =
            new String(value.getBytes(), 0, value.getLength());
        assertEquals("Checking record length:", recordLength,
            value.getLength());
        assertTrue("Checking for more records than expected:",
            recordNumber < totalRecords);
        String origRecord = recordList.get(recordNumber);
        assertEquals("Checking record content:", origRecord, valueString);
        recordNumber++;
      }
      reader.close();
    }
    assertEquals("Total original records should be total read records:",
        recordList.size(), recordNumber);
  }
}
 

static private byte[] pair(BytesWritable a, BytesWritable b) {
  byte[] pairData = new byte[a.getLength()+ b.getLength()];
  System.arraycopy(a.getBytes(), 0, pairData, 0, a.getLength());
  System.arraycopy(b.getBytes(), 0, pairData, a.getLength(), b.getLength());
  return pairData;
}
 

public BytesWritable evaluate(BytesWritable geomref, IntWritable index) {
	if (geomref == null || geomref.getLength() == 0 || index == null) {
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	OGCGeometry ogcGeometry = GeometryUtils.geometryFromEsriShape(geomref);
	if (ogcGeometry == null){
		LogUtils.Log_ArgumentsNull(LOG);
		return null;
	}

	int idx = index.get() - 1;  // 1-based UI, 0-based engine
	try {
		GeometryUtils.OGCType ogcType = GeometryUtils.getType(geomref);
		OGCGeometry ogcGeom = null;
		switch(ogcType) {
		case ST_POINT:
			LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_MULTIPOINT, ogcType);
			return null;
		case ST_LINESTRING:
			LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_MULTILINESTRING, ogcType);
			return null;
		case ST_POLYGON:
			LogUtils.Log_InvalidType(LOG, GeometryUtils.OGCType.ST_MULTIPOLYGON, ogcType);
			return null;
		case ST_MULTIPOINT:
			ogcGeom = ((OGCMultiPoint)ogcGeometry).geometryN(idx);
			break;
		case ST_MULTILINESTRING:
			ogcGeom = ((OGCMultiLineString)ogcGeometry).geometryN(idx);
			break;
		case ST_MULTIPOLYGON:
			ogcGeom = ((OGCMultiPolygon)ogcGeometry).geometryN(idx);
			break;
		}
		return GeometryUtils.geometryToEsriShapeBytesWritable(ogcGeom);
	} catch (Exception e) {
		LogUtils.Log_InternalError(LOG, "ST_GeometryN: " + e);
		return null;
	}
}
 

private void runRandomTests(CompressionCodec codec) throws IOException {
  StringBuilder fileName = new StringBuilder("testFormat.txt");
  if (codec != null) {
    fileName.append(".gz");
  }
  localFs.delete(workDir, true);
  Path file = new Path(workDir, fileName.toString());
  int seed = new Random().nextInt();
  LOG.info("Seed = " + seed);
  Random random = new Random(seed);
  int MAX_TESTS = 20;
  LongWritable key = new LongWritable();
  BytesWritable value = new BytesWritable();

  for (int i = 0; i < MAX_TESTS; i++) {
    LOG.info("----------------------------------------------------------");
    // Maximum total records of 999
    int totalRecords = random.nextInt(999)+1;
    // Test an empty file
    if (i == 8) {
       totalRecords = 0;
    }
    // Maximum bytes in a record of 100K
    int recordLength = random.nextInt(1024*100)+1;
    // For the 11th test, force a record length of 1
    if (i == 10) {
      recordLength = 1;
    }
    // The total bytes in the test file
    int fileSize = (totalRecords * recordLength);
    LOG.info("totalRecords=" + totalRecords + " recordLength="
        + recordLength);
    // Create the job 
    JobConf job = new JobConf(defaultConf);
    if (codec != null) {
      ReflectionUtils.setConf(codec, job);
    }
    // Create the test file
    ArrayList<String> recordList
        = createFile(file, codec, recordLength, totalRecords);
    assertTrue(localFs.exists(file));
    //set the fixed length record length config property for the job
    FixedLengthInputFormat.setRecordLength(job, recordLength);

    int numSplits = 1;
    // Arbitrarily set number of splits.
    if (i > 0) {
      if (i == (MAX_TESTS-1)) {
        // Test a split size that is less than record len
        numSplits = (int)(fileSize/Math.floor(recordLength/2));
      } else {
        if (MAX_TESTS % i == 0) {
          // Let us create a split size that is forced to be 
          // smaller than the end file itself, (ensures 1+ splits)
          numSplits = fileSize/(fileSize - random.nextInt(fileSize));
        } else {
          // Just pick a random split size with no upper bound 
          numSplits = Math.max(1, fileSize/random.nextInt(Integer.MAX_VALUE));
        }
      }
      LOG.info("Number of splits set to: " + numSplits);
    }

    // Setup the input path
    FileInputFormat.setInputPaths(job, workDir);
    // Try splitting the file in a variety of sizes
    FixedLengthInputFormat format = new FixedLengthInputFormat();
    format.configure(job);
    InputSplit splits[] = format.getSplits(job, numSplits);
    LOG.info("Actual number of splits = " + splits.length);
    // Test combined split lengths = total file size
    long recordOffset = 0;
    int recordNumber = 0;
    for (InputSplit split : splits) {
      RecordReader<LongWritable, BytesWritable> reader = 
          format.getRecordReader(split, job, voidReporter);
      Class<?> clazz = reader.getClass();
      assertEquals("RecordReader class should be FixedLengthRecordReader:", 
          FixedLengthRecordReader.class, clazz);
      // Plow through the records in this split
      while (reader.next(key, value)) {
        assertEquals("Checking key", (long)(recordNumber*recordLength),
            key.get());
        String valueString =
            new String(value.getBytes(), 0, value.getLength());
        assertEquals("Checking record length:", recordLength,
            value.getLength());
        assertTrue("Checking for more records than expected:",
            recordNumber < totalRecords);
        String origRecord = recordList.get(recordNumber);
        assertEquals("Checking record content:", origRecord, valueString);
        recordNumber++;
      }
      reader.close();
    }
    assertEquals("Total original records should be total read records:",
        recordList.size(), recordNumber);
  }
}
 

/**
 * Copy the key into BytesWritable. The input BytesWritable will be
 * automatically resized to the actual key size.
 * 
 * @param key
 *          BytesWritable to hold the key.
 * @throws IOException
 */
public int getKey(BytesWritable key) throws IOException {
  key.setSize(getKeyLength());
  getKey(key.getBytes());
  return key.getLength();
}