Java Code Examples for org.apache.hadoop.io.WritableComparator#hashBytes()

The following examples show how to use org.apache.hadoop.io.WritableComparator#hashBytes() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: BinaryPartitioner.java From hadoop with Apache License 2.0

5 votes

/** 
 * Use (the specified slice of the array returned by) 
 * {@link BinaryComparable#getBytes()} to partition. 
 */
@Override
public int getPartition(BinaryComparable key, V value, int numPartitions) {
  int length = key.getLength();
  int leftIndex = (leftOffset + length) % length;
  int rightIndex = (rightOffset + length) % length;
  int hash = WritableComparator.hashBytes(key.getBytes(), 
    leftIndex, rightIndex - leftIndex + 1);
  return (hash & Integer.MAX_VALUE) % numPartitions;
}

Example 2

Source File: BinaryPartitioner.java From big-c with Apache License 2.0

5 votes

/** 
 * Use (the specified slice of the array returned by) 
 * {@link BinaryComparable#getBytes()} to partition. 
 */
@Override
public int getPartition(BinaryComparable key, V value, int numPartitions) {
  int length = key.getLength();
  int leftIndex = (leftOffset + length) % length;
  int rightIndex = (rightOffset + length) % length;
  int hash = WritableComparator.hashBytes(key.getBytes(), 
    leftIndex, rightIndex - leftIndex + 1);
  return (hash & Integer.MAX_VALUE) % numPartitions;
}

Example 3

Source File: SortValidator.java From hadoop with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
public void map(WritableComparable key, Writable value,
                OutputCollector<IntWritable, RecordStatsWritable> output, 
                Reporter reporter) throws IOException {
  // Set up rawKey and rawValue on the first call to 'map'
  if (recordId == -1) {
   rawKey = createRaw(key.getClass());
   rawValue = createRaw(value.getClass());
  }
  ++recordId;
  
  if (this.key == sortOutput) {
    // Check if keys are 'sorted' if this  
    // record is from sort's output
    if (prevKey == null) {
      prevKey = key;
      keyClass = prevKey.getClass();
    } else {
      // Sanity check
      if (keyClass != key.getClass()) {
        throw new IOException("Type mismatch in key: expected " +
                              keyClass.getName() + ", received " +
                              key.getClass().getName());
      }
      
      // Check if they were sorted correctly
      if (prevKey.compareTo(key) > 0) {
        throw new IOException("The 'map-reduce' framework wrongly" +
                              " classifed (" + prevKey + ") > (" + 
                              key + ") "+ "for record# " + recordId); 
      }
      prevKey = key;
    }

    // Check if the sorted output is 'partitioned' right
    int keyPartition = 
      partitioner.getPartition(key, value, noSortReducers);
    if (partition != keyPartition) {
      throw new IOException("Partitions do not match for record# " + 
                            recordId + " ! - '" + partition + "' v/s '" + 
                            keyPartition + "'");
    }
  }

  // Construct the record-stats and output (this.key, record-stats)
  byte[] keyBytes = rawKey.getRawBytes(key);
  int keyBytesLen = rawKey.getRawBytesLength(key);
  byte[] valueBytes = rawValue.getRawBytes(value);
  int valueBytesLen = rawValue.getRawBytesLength(value);
  
  int keyValueChecksum = 
    (WritableComparator.hashBytes(keyBytes, keyBytesLen) ^
     WritableComparator.hashBytes(valueBytes, valueBytesLen));

  output.collect(this.key, 
                 new RecordStatsWritable((keyBytesLen+valueBytesLen),
                 1, keyValueChecksum)
                );
}

Example 4

Source File: TFile.java From hadoop with Apache License 2.0

4 votes

@Override
public int hashCode() {
  return WritableComparator.hashBytes(keyBuffer, 0, getKeyLength());
}

Example 5

Source File: SortValidator.java From big-c with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
public void map(WritableComparable key, Writable value,
                OutputCollector<IntWritable, RecordStatsWritable> output, 
                Reporter reporter) throws IOException {
  // Set up rawKey and rawValue on the first call to 'map'
  if (recordId == -1) {
   rawKey = createRaw(key.getClass());
   rawValue = createRaw(value.getClass());
  }
  ++recordId;
  
  if (this.key == sortOutput) {
    // Check if keys are 'sorted' if this  
    // record is from sort's output
    if (prevKey == null) {
      prevKey = key;
      keyClass = prevKey.getClass();
    } else {
      // Sanity check
      if (keyClass != key.getClass()) {
        throw new IOException("Type mismatch in key: expected " +
                              keyClass.getName() + ", received " +
                              key.getClass().getName());
      }
      
      // Check if they were sorted correctly
      if (prevKey.compareTo(key) > 0) {
        throw new IOException("The 'map-reduce' framework wrongly" +
                              " classifed (" + prevKey + ") > (" + 
                              key + ") "+ "for record# " + recordId); 
      }
      prevKey = key;
    }

    // Check if the sorted output is 'partitioned' right
    int keyPartition = 
      partitioner.getPartition(key, value, noSortReducers);
    if (partition != keyPartition) {
      throw new IOException("Partitions do not match for record# " + 
                            recordId + " ! - '" + partition + "' v/s '" + 
                            keyPartition + "'");
    }
  }

  // Construct the record-stats and output (this.key, record-stats)
  byte[] keyBytes = rawKey.getRawBytes(key);
  int keyBytesLen = rawKey.getRawBytesLength(key);
  byte[] valueBytes = rawValue.getRawBytes(value);
  int valueBytesLen = rawValue.getRawBytesLength(value);
  
  int keyValueChecksum = 
    (WritableComparator.hashBytes(keyBytes, keyBytesLen) ^
     WritableComparator.hashBytes(valueBytes, valueBytesLen));

  output.collect(this.key, 
                 new RecordStatsWritable((keyBytesLen+valueBytesLen),
                 1, keyValueChecksum)
                );
}

Example 6

Source File: TFile.java From big-c with Apache License 2.0

4 votes

@Override
public int hashCode() {
  return WritableComparator.hashBytes(keyBuffer, 0, getKeyLength());
}

Example 7

Source File: DTFile.java From attic-apex-malhar with Apache License 2.0

4 votes

@Override
public int hashCode() {
  return WritableComparator.hashBytes(blockBuffer, getKeyOffset(), getKeyLength());
}

Example 8

Source File: AccumuloMetadataReader.java From geowave with Apache License 2.0

4 votes

@Override
public int hashCode() {
  return WritableComparator.hashBytes(row, row.length)
      + WritableComparator.hashBytes(colFamily, colFamily.length)
      + WritableComparator.hashBytes(colQualifier, colQualifier.length);
}

Example 9

Source File: SortValidator.java From RDFS with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
public void map(WritableComparable key, Writable value,
                OutputCollector<IntWritable, RecordStatsWritable> output, 
                Reporter reporter) throws IOException {
  // Set up rawKey and rawValue on the first call to 'map'
  if (recordId == -1) {
   rawKey = createRaw(key.getClass());
   rawValue = createRaw(value.getClass());
  }
  ++recordId;
  
  if (this.key == sortOutput) {
    // Check if keys are 'sorted' if this  
    // record is from sort's output
    if (prevKey == null) {
      prevKey = key;
      keyClass = prevKey.getClass();
    } else {
      // Sanity check
      if (keyClass != key.getClass()) {
        throw new IOException("Type mismatch in key: expected " +
                              keyClass.getName() + ", recieved " +
                              key.getClass().getName());
      }
      
      // Check if they were sorted correctly
      if (prevKey.compareTo(key) > 0) {
        throw new IOException("The 'map-reduce' framework wrongly" +
                              " classifed (" + prevKey + ") > (" + 
                              key + ") "+ "for record# " + recordId); 
      }
      prevKey = key;
    }

    // Check if the sorted output is 'partitioned' right
    int keyPartition = 
      partitioner.getPartition(key, value, noSortReducers);
    if (partition != keyPartition) {
      throw new IOException("Partitions do not match for record# " + 
                            recordId + " ! - '" + partition + "' v/s '" + 
                            keyPartition + "'");
    }
  }

  // Construct the record-stats and output (this.key, record-stats)
  byte[] keyBytes = rawKey.getRawBytes(key);
  int keyBytesLen = rawKey.getRawBytesLength(key);
  byte[] valueBytes = rawValue.getRawBytes(value);
  int valueBytesLen = rawValue.getRawBytesLength(value);
  
  int keyValueChecksum = 
    (WritableComparator.hashBytes(keyBytes, keyBytesLen) ^
     WritableComparator.hashBytes(valueBytes, valueBytesLen));

  output.collect(this.key, 
                 new RecordStatsWritable((keyBytesLen+valueBytesLen),
                 1, keyValueChecksum)
                );
}

Example 10

Source File: TFile.java From RDFS with Apache License 2.0

4 votes

@Override
public int hashCode() {
  return WritableComparator.hashBytes(keyBuffer, getKeyLength());
}

Example 11

Source File: HiveBytesArrayWritable.java From elasticsearch-hadoop with Apache License 2.0

4 votes

@Override
public int hashCode() {
    return WritableComparator.hashBytes(ba.bytes(), ba.length());
}

Example 12

Source File: SortValidator.java From hadoop-gpu with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
public void map(WritableComparable key, Writable value,
                OutputCollector<IntWritable, RecordStatsWritable> output, 
                Reporter reporter) throws IOException {
  // Set up rawKey and rawValue on the first call to 'map'
  if (recordId == -1) {
   rawKey = createRaw(key.getClass());
   rawValue = createRaw(value.getClass());
  }
  ++recordId;
  
  if (this.key == sortOutput) {
    // Check if keys are 'sorted' if this  
    // record is from sort's output
    if (prevKey == null) {
      prevKey = key;
      keyClass = prevKey.getClass();
    } else {
      // Sanity check
      if (keyClass != key.getClass()) {
        throw new IOException("Type mismatch in key: expected " +
                              keyClass.getName() + ", recieved " +
                              key.getClass().getName());
      }
      
      // Check if they were sorted correctly
      if (prevKey.compareTo(key) > 0) {
        throw new IOException("The 'map-reduce' framework wrongly" +
                              " classifed (" + prevKey + ") > (" + 
                              key + ") "+ "for record# " + recordId); 
      }
      prevKey = key;
    }

    // Check if the sorted output is 'partitioned' right
    int keyPartition = 
      partitioner.getPartition(key, value, noSortReducers);
    if (partition != keyPartition) {
      throw new IOException("Partitions do not match for record# " + 
                            recordId + " ! - '" + partition + "' v/s '" + 
                            keyPartition + "'");
    }
  }

  // Construct the record-stats and output (this.key, record-stats)
  byte[] keyBytes = rawKey.getRawBytes(key);
  int keyBytesLen = rawKey.getRawBytesLength(key);
  byte[] valueBytes = rawValue.getRawBytes(value);
  int valueBytesLen = rawValue.getRawBytesLength(value);
  
  int keyValueChecksum = 
    (WritableComparator.hashBytes(keyBytes, keyBytesLen) ^
     WritableComparator.hashBytes(valueBytes, valueBytesLen));

  output.collect(this.key, 
                 new RecordStatsWritable((keyBytesLen+valueBytesLen),
                 1, keyValueChecksum)
                );
}

Example 13

Source File: TFile.java From hadoop-gpu with Apache License 2.0

4 votes

@Override
public int hashCode() {
  return WritableComparator.hashBytes(keyBuffer, getKeyLength());
}

Example 14

Source File: Bytes.java From tajo with Apache License 2.0

2 votes

/**
 * @param b value
 * @param length length of the value
 * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
 * passed in array.  This method is what {@link org.apache.hadoop.io.Text} and
 * {@link ImmutableBytesWritable} use calculating hash code.
 */
public static int hashCode(final byte [] b, final int length) {
  return WritableComparator.hashBytes(b, length);
}

Example 15

Source File: Bytes.java From hbase with Apache License 2.0

2 votes

/**
 * @param b value
 * @param length length of the value
 * @return Runs {@link WritableComparator#hashBytes(byte[], int)} on the
 * passed in array.  This method is what {@link org.apache.hadoop.io.Text}
 * use calculating hash code.
 */
public static int hashCode(final byte [] b, final int length) {
  return WritableComparator.hashBytes(b, length);
}

Example 16

Source File: Bytes.java From incubator-tajo with Apache License 2.0

2 votes

/**
 * @param b value
 * @param length length of the value
 * @return Runs {@link org.apache.hadoop.io.WritableComparator#hashBytes(byte[], int)} on the
 * passed in array.  This method is what {@link org.apache.hadoop.io.Text} and
 */
public static int hashCode(final byte [] b, final int length) {
  return WritableComparator.hashBytes(b, length);
}