org.apache.accumulo.core.data.KeyValue Java Examples

public List<KeyValue> constructKeyValuePairs(final T entry) {
  final List<KeyValue> keyValuePairs = new ArrayList<>();
  final GeoWaveRow[] rows =
      BaseDataStoreUtils.getGeoWaveRows(entry, adapter, index, visibilityWriter);
  if ((rows != null) && (rows.length > 0)) {
    for (final GeoWaveRow row : rows) {
      final Mutation m = AccumuloWriter.rowToMutation(row);
      for (final ColumnUpdate cu : m.getUpdates()) {
        keyValuePairs.add(
            new KeyValue(
                new Key(
                    m.getRow(),
                    cu.getColumnFamily(),
                    cu.getColumnQualifier(),
                    cu.getColumnVisibility(),
                    cu.getTimestamp()),
                cu.getValue()));
      }
    }
  }

  return keyValuePairs;
}
 

@Override
protected void map(final LongWritable key, final Text value, final Context context)
    throws IOException, InterruptedException {

  simpleFeature = parseGeonamesValue(value);
  ((InitializeWithIndicesDataAdapter) adapter).init(index);

  // build Geowave-formatted Accumulo [Key,Value] pairs
  keyValuePairs = generator.constructKeyValuePairs(simpleFeature);

  // output each [Key,Value] pair to shuffle-and-sort phase where we rely
  // on MapReduce to sort by Key
  for (final KeyValue accumuloKeyValuePair : keyValuePairs) {
    context.write(accumuloKeyValuePair.getKey(), accumuloKeyValuePair.getValue());
  }
}
 

public DataStoreCacheIterator(DataStoreCache store, Collection<VisibilityFilter> visibilityFilters,
        QueryRequest.SubQuery query, long startTs, long endTs) {

    this.store = store;
    this.visibilityFilters = visibilityFilters;
    this.query = query;
    this.startTs = startTs;
    this.endTs = endTs;
    Map<TaggedMetric, GorillaStore> storeMap = this.store.getGorillaStores(query.getMetric());
    this.storeItr = storeMap.entrySet().iterator();
    this.decompressors = getNextDecompressorIterable();

    long start = System.currentTimeMillis();
    Map<Key, Value> entries = getEntries();
    for (Map.Entry<Key, Value> entry : entries.entrySet()) {
        kvQueue.add(new KeyValue(entry.getKey(), entry.getValue()));
    }
    LOG.info(
            "Time to initialize cache iterator for {} with {} TaggedMetric/GorillaStore pairs and {} K/V entries - {}ms",
            query.toString(), storeMap.size(), entries.size(), System.currentTimeMillis() - start);
}
 

@Test
public void testTwoValuesEqual() throws Exception {
    KeyValue item1 = new KeyValue(new Key("row1", "cf1"), "value1".getBytes());
    KeyValue item2 = new KeyValue(new Key("row1", "cf2"), "value2".getBytes());
    List<KeyValue> list = Arrays.asList(item1, item2);
    
    KeyAggregatingTransformIterator it = new KeyAggregatingTransformIterator(list.iterator(), NOPTransformer.nopTransformer());
    assertTrue(it.hasNext());
    @SuppressWarnings("unchecked")
    List<Entry<Key,Value>> entries = (List<Entry<Key,Value>>) it.next();
    assertNotNull(entries);
    assertEquals(2, entries.size());
    assertEquals(item1, entries.get(0));
    assertEquals(item2, entries.get(1));
    assertFalse(it.hasNext());
}
 

public static KeyValue getDateIndexEntry(String shardDate, int[] shardIndicies, String dataType, String type, String dateField, String dateValue,
                ColumnVisibility visibility) throws ParseException {
    // The row is the date to index yyyyMMdd
    
    // the colf is the type (e.g. LOAD or ACTIVITY)
    
    // the colq is the event date yyyyMMdd \0 the datatype \0 the field name
    String colq = shardDate + '\0' + dataType + '\0' + dateField;
    
    // the value is a bitset denoting the shard
    BitSet bits = DateIndexUtil.getBits(shardIndicies[0]);
    for (int i = 1; i < shardIndicies.length; i++) {
        bits = DateIndexUtil.merge(bits, DateIndexUtil.getBits(shardIndicies[i]));
    }
    Value shardList = new Value(bits.toByteArray());
    
    // create the key
    Key key = new Key(dateValue, type, colq, visibility, DateIndexUtil.getBeginDate(dateValue).getTime());
    
    return new KeyValue(key, shardList);
}
 

/**
 * Map the uid in the supplied key and value. The formats expected are for the shard table only.
 * 
 * @param keyValue
 * @return the key with the uid mapped appropriately. The original UID if any is placed in the value.
 */
@Override
protected KeyValue mapUid(KeyValue keyValue, boolean startKey, boolean startKeyInclusive, boolean endKey, boolean endKeyInclusive) {
    if (keyValue == null || keyValue.getKey() == null) {
        return keyValue;
    }
    // pull the column family
    String cf = keyValue.getKey().getColumnFamily().toString();
    int index = cf.indexOf('\0');
    if (index > 0) {
        if (cf.startsWith("fi\0")) {
            keyValue = replaceEventUidInCQ(keyValue, 2, startKey, startKeyInclusive, endKey, endKeyInclusive);
        } else { // assume DataType\0UID column family
            keyValue = replaceEventUidInCF(keyValue, 1, startKey, startKeyInclusive, endKey, endKeyInclusive);
        }
    } else if (cf.equals("d")) {
        keyValue = replaceEventUidInCQ(keyValue, 1, startKey, startKeyInclusive, endKey, endKeyInclusive);
    } else if (cf.equals("tf")) {
        keyValue = replaceEventUidInCQ(keyValue, 1, startKey, startKeyInclusive, endKey, endKeyInclusive);
    }
    return keyValue;
}
 

private KeyValue replaceEventUidInCF(KeyValue keyValue, int partIndex, boolean startKey, boolean startKeyInclusive, boolean endKey, boolean endKeyInclusive) {
    Key key = keyValue.getKey();
    String[] replacement = replaceUid(key.getColumnFamily().toString(), partIndex, startKey, startKeyInclusive, endKey, endKeyInclusive);
    // if no change in the uid, then return the original key
    if (replacement == null) {
        return keyValue;
    }
    
    Key newKey = new Key(key.getRow(), new Text(replacement[CQ_INDEX]), key.getColumnQualifier(), key.getColumnVisibility(), key.getTimestamp());
    return new KeyValue(newKey, replacement[ORG_UID_INDEX].getBytes());
}
 

static void addData(List<Entry<Key,Value>> data, String row, String cf, int chunkSize,
    int chunkCount, String vis, String value) {
  Text chunkCQ = new Text(FileDataIngest.intToBytes(chunkSize));
  chunkCQ.append(FileDataIngest.intToBytes(chunkCount), 0, 4);
  data.add(new KeyValue(new Key(new Text(row), new Text(cf), chunkCQ, new Text(vis)),
      value.getBytes()));
}
 

private static void addData(List<Entry<Key,Value>> data, String row, String cf, int chunkSize,
    int chunkCount, String vis, String value) {
  Text chunkCQ = new Text(FileDataIngest.intToBytes(chunkSize));
  chunkCQ.append(FileDataIngest.intToBytes(chunkCount), 0, 4);
  data.add(new KeyValue(new Key(new Text(row), new Text(cf), chunkCQ, new Text(vis)),
      value.getBytes()));
}
 

@Test
@SuppressWarnings("unchecked")
public void testMultipleValuesPartial() throws Exception {
    KeyValue item1 = new KeyValue(new Key("row1", "cf1", "cq1"), "value1".getBytes());
    KeyValue item2 = new KeyValue(new Key("row1", "cf1", "cq2"), "value2".getBytes());
    KeyValue item3 = new KeyValue(new Key("row1", "cf2", "cq1"), "value1".getBytes());
    KeyValue item4 = new KeyValue(new Key("row2", "cf2", "cq1"), "value1".getBytes());
    List<KeyValue> list = Arrays.asList(item1, item2, item3, item4);
    
    KeyAggregatingTransformIterator it = new KeyAggregatingTransformIterator(PartialKey.ROW_COLFAM, list.iterator(), NOPTransformer.nopTransformer());
    assertTrue(it.hasNext());
    List<Entry<Key,Value>> entries = (List<Entry<Key,Value>>) it.next();
    assertNotNull(entries);
    assertEquals(2, entries.size());
    assertEquals(item1, entries.get(0));
    assertEquals(item2, entries.get(1));
    
    assertTrue(it.hasNext());
    entries = (List<Entry<Key,Value>>) it.next();
    assertEquals(1, entries.size());
    assertEquals(item3, entries.get(0));
    
    assertTrue(it.hasNext());
    entries = (List<Entry<Key,Value>>) it.next();
    assertEquals(1, entries.size());
    assertEquals(item4, entries.get(0));
    
    assertFalse(it.hasNext());
}
 

@Test
public void testSingleValue() throws Exception {
    KeyValue item1 = new KeyValue(new Key("row1"), "".getBytes());
    List<KeyValue> list = Arrays.asList(item1);
    
    KeyAggregatingTransformIterator it = new KeyAggregatingTransformIterator(list.iterator(), NOPTransformer.nopTransformer());
    assertTrue(it.hasNext());
    @SuppressWarnings("unchecked")
    List<Entry<Key,Value>> entries = (List<Entry<Key,Value>>) it.next();
    assertNotNull(entries);
    assertEquals(1, entries.size());
    assertEquals(item1, entries.get(0));
    assertFalse(it.hasNext());
}
 

private static void write(String shardDate, int[] shardIndicies, String dataType, String type, String dateField, String dateValue, String visibility)
                throws ParseException, IOException, InterruptedException {
    ColumnVisibility vis = new ColumnVisibility(visibility);
    KeyValue kv = getDateIndexEntry(shardDate, shardIndicies, dataType, type, dateField, dateValue, vis);
    Mutation m = new Mutation(kv.getKey().getRow());
    m.put(kv.getKey().getColumnFamily(), kv.getKey().getColumnQualifier(), vis, kv.getKey().getTimestamp(), kv.getValue());
    recordWriter.write(new Text(TableName.DATE_INDEX), m);
}
 

/********************** Helper methods ***********************/
private KeyValue replaceEventUidInCQ(KeyValue keyValue, int partIndex, boolean startKey, boolean startKeyInclusive, boolean endKey, boolean endKeyInclusive) {
    Key key = keyValue.getKey();
    String[] replacement = replaceUid(key.getColumnQualifier().toString(), partIndex, startKey, startKeyInclusive, endKey, endKeyInclusive);
    // if no change in the uid, then return the original key
    if (replacement == null) {
        return keyValue;
    }
    
    Key newKey = new Key(key.getRow(), key.getColumnFamily(), new Text(replacement[CQ_INDEX]), key.getColumnVisibility(), key.getTimestamp());
    return new KeyValue(newKey, replacement[ORG_UID_INDEX].getBytes());
}
 

/**
 * 
 */
protected void findTop() throws IOException {
    if (this.source.hasTop()) {
        KeyValue keyValue = mapUid(new KeyValue(this.source.getTopKey(), this.source.getTopValue().get()), false, false, false, false);
        this.topKey = keyValue.getKey();
        this.topValue = keyValue.getValue();
    } else {
        this.topKey = null;
        this.topValue = null;
    }
}
 

/**
 * A method used to purge this error summary from the error processing table
 * 
 * @throws InterruptedException
 * @throws IOException
 */
@SuppressWarnings({"rawtypes", "unchecked"})
public void purge(ContextWriter writer, TaskInputOutputContext context, RawRecordContainer event, Map typeMap) throws IOException, InterruptedException {
    for (KeyValue keyValue : this.keyValues.values()) {
        keyValue.getKey().setDeleted(true);
        BulkIngestKey key = new BulkIngestKey(tableName, keyValue.getKey());
        writer.write(key, keyValue.getValue(), context);
    }
}
 

/**
 * Construct a date index entry
 * 
 * @param shardId
 * @param dataType
 * @param type
 * @param dateField
 * @param dateValue
 * @param visibility
 * @return The key and value
 */
public KeyValue getDateIndexEntry(String shardId, String dataType, String type, String dateField, String dateValue, ColumnVisibility visibility) {
    Date date = null;
    try {
        // get the date to be indexed
        date = dateNormalizer.denormalize(dateValue);
    } catch (Exception e) {
        log.error("Failed to normalize date value (skipping): " + dateValue, e);
        return null;
    }
    
    // set the time to 00:00:00 (for key timestamp)
    date = DateUtils.truncate(date, Calendar.DATE);
    
    // format the date and the shardId date as yyyyMMdd
    String rowDate = DateIndexUtil.format(date);
    String shardDate = ShardIdFactory.getDateString(shardId);
    
    ColumnVisibility biased = new ColumnVisibility(flatten(visibility));
    
    // The row is the date plus the shard partition
    String row = rowDate + '_' + getDateIndexShardPartition(rowDate, type, shardDate, dataType, dateField, new String(biased.getExpression()));
    
    // the colf is the type (e.g. LOAD or ACTIVITY)
    
    // the colq is the event date yyyyMMdd \0 the datatype \0 the field name
    String colq = shardDate + '\0' + dataType + '\0' + dateField;
    
    // the value is a bitset denoting the shard
    Value shardList = createDateIndexValue(ShardIdFactory.getShard(shardId));
    
    // create the key
    Key key = new Key(row, type, colq, biased, date.getTime());
    
    if (log.isTraceEnabled()) {
        log.trace("Dateate index key: " + key + " for shardId " + shardId);
    }
    
    return new KeyValue(key, shardList);
}
 

/**
 * Get the date index ingest keys and merge them into the provided key multimap
 * 
 * @param event
 * @param eventFields
 * @param index
 */
private void getBulkIngestKeys(RawRecordContainer event, Multimap<String,NormalizedContentInterface> eventFields, Multimap<BulkIngestKey,Value> index) {
    if (dataTypeToTypeToFields.containsKey(event.getDataType().typeName()) && null != eventFields && !eventFields.isEmpty()) {
        // date index Table Structure
        // Row: date
        // Colf: type
        // Colq: date\0datatype\0field
        // Value: shard bit set
        
        for (Map.Entry<String,String> entry : dataTypeToTypeToFields.get(event.getDataType().typeName()).entries()) {
            String type = entry.getKey();
            String field = entry.getValue();
            for (NormalizedContentInterface nci : eventFields.get(field)) {
                KeyValue keyValue = getDateIndexEntry(getShardId(event), event.getDataType().outputName(), type, field, nci.getIndexedFieldValue(),
                                event.getVisibility());
                
                if (keyValue != null) {
                    if (log.isDebugEnabled()) {
                        log.debug("Outputting " + keyValue + " to " + getDateIndexTableName());
                    }
                    
                    BulkIngestKey bulkIngestKey = new BulkIngestKey(getDateIndexTableName(), keyValue.getKey());
                    if (index.containsKey(bulkIngestKey)) {
                        index.put(bulkIngestKey, keyValue.getValue());
                        DateIndexDateAggregator aggregator = new DateIndexDateAggregator();
                        Value value = aggregator.reduce(bulkIngestKey.getKey(), index.get(bulkIngestKey).iterator());
                        index.removeAll(bulkIngestKey);
                        index.put(bulkIngestKey, value);
                    } else {
                        index.put(bulkIngestKey, keyValue.getValue());
                    }
                }
            }
        }
    }
}
 

/**
 * Turn a mutation's column update into a key
 *
 * @param m
 *            the Mutation from which KeyValue pairs should be extracted
 * @return a List of KeyValue pairs representing the contents of {@code m}
 */
public static List<KeyValue> getKeyValues(Mutation m) {
    List<KeyValue> values = new ArrayList<>();
    for (ColumnUpdate update : m.getUpdates()) {
        values.add(new KeyValue(new Key(m.getRow(), update.getColumnFamily(), update.getColumnQualifier(), update.getColumnVisibility(), (update
                        .hasTimestamp() ? update.getTimestamp() : -1), update.isDeleted()), update.getValue()));
    }
    return values;
}
 

@Test
@SuppressWarnings("unchecked")
public void testMultipleValues() throws Exception {
    KeyValue item1 = new KeyValue(new Key("row1", "cf1"), "value1".getBytes());
    KeyValue item2 = new KeyValue(new Key("row1", "cf2"), "value2".getBytes());
    KeyValue item3 = new KeyValue(new Key("row2", "cf1"), "value1".getBytes());
    KeyValue item4 = new KeyValue(new Key("row3", "cf1"), "value1".getBytes());
    KeyValue item5 = new KeyValue(new Key("row4", "cf1"), "value1".getBytes());
    KeyValue item6 = new KeyValue(new Key("row4", "cf2"), "value2".getBytes());
    KeyValue item7 = new KeyValue(new Key("row4", "cf3"), "value3".getBytes());
    KeyValue item8 = new KeyValue(new Key("row5", "cf1"), "value1".getBytes());
    KeyValue item9 = new KeyValue(new Key("row5", "cf2"), "value2".getBytes());
    List<KeyValue> list = Arrays.asList(item1, item2, item3, item4, item5, item6, item7, item8, item9);
    
    KeyAggregatingTransformIterator it = new KeyAggregatingTransformIterator(list.iterator(), NOPTransformer.nopTransformer());
    assertTrue(it.hasNext());
    List<Entry<Key,Value>> entries = (List<Entry<Key,Value>>) it.next();
    assertNotNull(entries);
    assertEquals(2, entries.size());
    assertEquals(item1, entries.get(0));
    assertEquals(item2, entries.get(1));
    
    assertTrue(it.hasNext());
    entries = (List<Entry<Key,Value>>) it.next();
    assertEquals(1, entries.size());
    assertEquals(item3, entries.get(0));
    
    assertTrue(it.hasNext());
    entries = (List<Entry<Key,Value>>) it.next();
    assertEquals(1, entries.size());
    assertEquals(item4, entries.get(0));
    
    assertTrue(it.hasNext());
    entries = (List<Entry<Key,Value>>) it.next();
    assertEquals(3, entries.size());
    assertEquals(item5, entries.get(0));
    assertEquals(item6, entries.get(1));
    assertEquals(item7, entries.get(2));
    
    assertTrue(it.hasNext());
    entries = (List<Entry<Key,Value>>) it.next();
    assertEquals(2, entries.size());
    assertEquals(item8, entries.get(0));
    assertEquals(item9, entries.get(1));
    
    assertFalse(it.hasNext());
}
 

public Multimap<Text,KeyValue> getKeyValues() {
    return keyValues;
}
 

public Collection<KeyValue> getErrorFields() {
    return this.keyValues.get(FIELD);
}
 

private static void addData(List<Entry<Key,Value>> data, String row, String cf, String cq,
    String vis, String value) {
  data.add(new KeyValue(new Key(new Text(row), new Text(cf), new Text(cq), new Text(vis)),
      value.getBytes()));
}
 

public Collection<KeyValue> getErrorInfoKV() {
    return this.keyValues.get(INFO);
}
 

static void addData(List<Entry<Key,Value>> data, String row, String cf, String cq, String vis,
    String value) {
  data.add(new KeyValue(new Key(new Text(row), new Text(cf), new Text(cq), new Text(vis)),
      value.getBytes()));
}
 

/**
 * Output some verbose counters
 *
 * @param context
 *            hadoop task context for writing counter values
 * @param tableName
 *            the table name to write in the counter
 * @param mutation
 *            a Mutation containing the key-value pairs to log to counters
 */
@SuppressWarnings("rawtypes")
public static void verboseCounters(TaskInputOutputContext context, String location, Text tableName, Mutation mutation) {
    for (KeyValue keyValue : getKeyValues(mutation)) {
        verboseCounter(context, location, tableName, keyValue.getKey().getRow().getBytes(), keyValue.getKey().getColumnFamily().getBytes(), keyValue
                        .getKey().getColumnQualifier().getBytes(), keyValue.getKey().getColumnVisibility(), keyValue.getValue().get());
    }
}
 

/**
 * Map the uid in the supplied key.
 * 
 * @param key
 *            The key to map. Could be null.
 * @param startKey
 *            true if this is the startKey in a range
 * @param startKeyInclusive
 *            true if the start key was inclusive
 * @param endKey
 *            true if this is the endKey in a range
 * @return the key with the uid mapped appropriately
 */
protected Key mapUid(Key key, boolean startKey, boolean startKeyInclusive, boolean endKey, boolean endKeyInclusive) {
    return mapUid(new KeyValue(key, EMPTY_BYTES), startKey, startKeyInclusive, endKey, endKeyInclusive).getKey();
}
 

/**
 * Map the uid in the supplied key and value.
 * 
 * @param keyValue
 *            The key and value to map. Could be null.
 * @param startKey
 *            true if this is the startKey in a range
 * @param startKeyInclusive
 *            true if the start key was inclusive
 * @param endKey
 *            true if this is the endKey in a range
 * @return the keyValue with the uid mapped appropriately
 */
protected KeyValue mapUid(KeyValue keyValue, boolean startKey, boolean startKeyInclusive, boolean endKey, boolean endKeyInclusive) {
    return keyValue;
}