org.apache.accumulo.core.data.Value Java Examples

@Override
public void init(
    final SortedKeyValueIterator<Key, Value> source,
    final Map<String, String> options,
    final IteratorEnvironment env) throws IOException {
  super.init(source, options, env);
  if ((options == null) || (!options.containsKey(PRIMARY_INDEX_ID))) {
    throw new IllegalArgumentException(
        "Arguments must be set for " + SecondaryIndexQueryFilterIterator.class.getName());
  }
  if (options.containsKey(FILTERS)) {
    final String filterStr = options.get(FILTERS);
    final byte[] filterBytes = ByteArrayUtils.byteArrayFromString(filterStr);
    filter = (QueryFilter) URLClassloaderUtils.fromBinary(filterBytes);
  }
  primaryIndexId = options.get(PRIMARY_INDEX_ID);
}
 

public static Mutation createMutation(TripleRow tripleRow) {
  Mutation mutation = new Mutation(new Text(tripleRow.getRow()));
  byte[] columnVisibility = tripleRow.getColumnVisibility();
  ColumnVisibility cv = columnVisibility == null ? EMPTY_CV : new ColumnVisibility(columnVisibility);
  Long timestamp = tripleRow.getTimestamp();
  boolean hasts = timestamp != null;
  timestamp = timestamp == null ? 0l : timestamp;
  byte[] value = tripleRow.getValue();
  Value v = value == null ? EMPTY_VALUE : new Value(value);
  byte[] columnQualifier = tripleRow.getColumnQualifier();
  Text cqText = columnQualifier == null ? EMPTY_TEXT : new Text(columnQualifier);
  byte[] columnFamily = tripleRow.getColumnFamily();
  Text cfText = columnFamily == null ? EMPTY_TEXT : new Text(columnFamily);

  if (hasts) {
    mutation.put(cfText, cqText, cv, timestamp, v);
  } else {
    mutation.put(cfText, cqText, cv, v);

  }
  return mutation;
}
 

private void runTest(boolean reseek, TreeMap<Key,Value> source, TreeMap<Key,Value> result,
    Collection<ByteSequence> cols) throws IOException {
  MapIterator src = new MapIterator(source);
  SortedKeyValueIterator<Key,Value> iter = new ChunkCombiner();
  iter.init(src, null, null);
  iter = iter.deepCopy(null);
  iter.seek(new Range(), cols, true);

  TreeMap<Key,Value> seen = new TreeMap<>();

  while (iter.hasTop()) {
    assertFalse("already contains " + iter.getTopKey(), seen.containsKey(iter.getTopKey()));
    seen.put(new Key(iter.getTopKey()), new Value(iter.getTopValue()));

    if (reseek)
      iter.seek(new Range(iter.getTopKey().followingKey(PartialKey.ROW_COLFAM_COLQUAL), true,
          null, true), cols, true);
    else
      iter.next();
  }

  assertEquals(result, seen);
}
 

@Override
public boolean hasNext() {
    // Assign the next key. Ideally, the first iteration has already been performed.
    final Entry<Key,Value> next = this.nextSeek;
    
    // If not, however, perform the first iteration to find out if there's at least one applicable key
    if (next == UNINITIALIZED_KEY) {
        try {
            this.seekNext(true);
        } catch (IOException e) {
            QueryException qe = new QueryException(DatawaveErrorCode.HAS_NEXT_ELEMENT_ERROR, e);
            throw new DatawaveFatalQueryException(qe);
        }
    }
    
    return (this.nextSeek != ITERATOR_COMPLETE_KEY);
}
 

@Test
public void testInvalidValueType() throws Exception {
    Logger log = Logger.getLogger(GlobalIndexUidAggregator.class);
    Level origLevel = log.getLevel();
    log.setLevel(Level.FATAL);
    Collection<Value> values = Lists.newArrayList();
    agg.reset();
    Value val = new Value(UUID.randomUUID().toString().getBytes());
    values.add(val);
    Value result = agg.reduce(new Key("key"), values.iterator());
    Uid.List resultList = Uid.List.parseFrom(result.get());
    assertEquals(false, resultList.getIGNORE());
    assertEquals(0, resultList.getUIDCount());
    assertEquals(0, resultList.getCOUNT());
    
    log.setLevel(origLevel);
}
 

@Test
public void assertNexts() throws IOException {
    SourceManager manager = new SourceManager(counter);
    manager.setInitialSize(10);
    
    Collection<SortedKeyValueIterator<Key,Value>> kvList = Lists.newArrayList();
    for (int i = 0; i < 500; i++) {
        kvList.add(manager.deepCopy(null));
    }
    
    for (SortedKeyValueIterator<Key,Value> kv : kvList) {
        kv.seek(new Range(), Collections.emptyList(), false);
        kv.next();
    }
    
    assertEquals(500, counter.nextCalls);
    assertEquals(10, counter.counter);
}
 

/**
 * Applies the table configuration if one is specified.
 * 
 * @param topIter
 * @param conf
 * @return
 * @throws IOException
 */
protected SortedKeyValueIterator<Key,Value> applyTableIterators(SortedKeyValueIterator<Key,Value> topIter, Configuration conf) throws IOException {
    
    if (null != acuTableConf) {
        // don't need to be populated as we'll do this later.
        RFileEnvironment iterEnv = new RFileEnvironment();
        
        List<IterInfo> serverSideIteratorList = Collections.emptyList();
        Map<String,Map<String,String>> serverSideIteratorOptions = Collections.emptyMap();
        
        byte[] defaultSecurityLabel;
        
        ColumnVisibility cv = new ColumnVisibility(acuTableConf.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY));
        defaultSecurityLabel = cv.getExpression();
        
        SortedKeyValueIterator<Key,Value> visFilter = VisibilityFilter.wrap(topIter, auths, defaultSecurityLabel);
        
        return IteratorUtil.loadIterators(IteratorScope.scan, visFilter, null, acuTableConf, serverSideIteratorList, serverSideIteratorOptions, iterEnv,
                        false);
    }
    
    return topIter;
}
 

public static KeyValue getDateIndexEntry(String shardDate, int[] shardIndicies, String dataType, String type, String dateField, String dateValue,
                ColumnVisibility visibility) throws ParseException {
    // The row is the date to index yyyyMMdd
    
    // the colf is the type (e.g. LOAD or ACTIVITY)
    
    // the colq is the event date yyyyMMdd \0 the datatype \0 the field name
    String colq = shardDate + '\0' + dataType + '\0' + dateField;
    
    // the value is a bitset denoting the shard
    BitSet bits = DateIndexUtil.getBits(shardIndicies[0]);
    for (int i = 1; i < shardIndicies.length; i++) {
        bits = DateIndexUtil.merge(bits, DateIndexUtil.getBits(shardIndicies[i]));
    }
    Value shardList = new Value(bits.toByteArray());
    
    // create the key
    Key key = new Key(dateValue, type, colq, visibility, DateIndexUtil.getBeginDate(dateValue).getTime());
    
    return new KeyValue(key, shardList);
}
 

@Override
// delegates partitioning
public int getPartition(BulkIngestKey key, Value value, int numPartitions) {
    Text tableName = key.getTableName();
    
    Partitioner<BulkIngestKey,Value> partitioner = partitionerCache.getPartitioner(tableName);
    
    int partition = partitioner.getPartition(key, value, numPartitions);
    Integer offset = this.tableOffsets.get(tableName);
    
    if (null != offset) {
        return (offset + partition) % numPartitions;
    } else {
        return partition % numPartitions;
    }
}
 

/**
 * Using the row encoded by <code>org.apache.accumulo.core.iterators.user.WholeRowIterator</code> construct an
 * AccumuloItem stored in that row, if one exists.
 *
 * @param encodedRow
 * @return
 * @throws IOException
 */
public static AccumuloItem buildFromEncodedRow(Entry<Key, Value> encodedRow) throws IOException {
  SortedMap<Key, Value> m = buildRowMap(encodedRow);
  Key first = m.firstKey();
  Value val = m.remove(first);
  AccumuloItem i = constructItemFromFirstPair(first, val);
  if (i != null) {
    i.populateFromRow(m);
  }
  return i;
}
 

@Override
public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
    CompositeSeekingIterator to = new CompositeSeekingIterator();
    to.setSource(getSource().deepCopy(env));
    
    Collections.copy(to.fieldNames, fieldNames);
    to.fieldToDiscreteIndexType = new HashMap<>(fieldToDiscreteIndexType);
    to.separator = separator;
    to.seekThreshold = seekThreshold;
    to.compositeSeeker = new ShardIndexCompositeSeeker(to.fieldNames, to.separator, to.fieldToDiscreteIndexType);
    
    return to;
}
 

public static Namespace getNamespace(final Iterator<Entry<Key, Value>> rowResults) {
    for (; rowResults.hasNext(); ) {
        final Entry<Key, Value> next = rowResults.next();
        final Key key = next.getKey();
        final Value val = next.getValue();
        final String cf = key.getColumnFamily().toString();
        final String cq = key.getColumnQualifier().toString();
        return new NamespaceImpl(key.getRow().toString(),
                new String(val.get(), StandardCharsets.UTF_8));
    }
    return null;
}
 

private BasePoolableObjectFactory<SortedKeyValueIterator<Key,Value>> createIvaratorSourceFactory(SourceFactory<Key,Value> sourceFactory) {
    return new BasePoolableObjectFactory<SortedKeyValueIterator<Key,Value>>() {
        @Override
        public SortedKeyValueIterator<Key,Value> makeObject() throws Exception {
            return sourceFactory.getSourceDeepCopy();
        }
    };
}
 

/**
     * This test simulates an analysis batch job from a possible user input form at the OSTMap web application.
     * @throws TableNotFoundException Exception thrown if Table not found.
     */
//    @Test
    public void testTweetsLastNDays() throws TableNotFoundException {
        int days = 5;
        Connector connector = amc.getConnector();
        Authorizations authorizations = new Authorizations(AccumuloIdentifiers.AUTHORIZATION.toString());
        System.out.println(String.format("[AUTHORIZATIONS] %s", authorizations.toString()));
        Scanner scanner = connector.createScanner("GeoTemporalIndex", authorizations);

        for (Map.Entry<Key, Value> entry : scanner) {
            System.out.printf("Key : %-50s  Value : %s\n", entry.getKey(), entry.getValue());
        }
        // tbd ...
    }
 

@Override
public Value getTopValue() {
    if (aggrKey != null) {
        return aggrValue;
    }
    return iterator.getTopValue();
}
 

@Test
public void shouldStillIncludeMetricsWithOnlyASubsetOfEnabledLabels() throws Exception {
    Multimap<String,NormalizedContentInterface> fields = MetricsTestData.createFields("table", TableName.SHARD);
    
    // dataType is not set
    service.collect(Metric.KV_PER_TABLE, labels, fields, 1L);
    service.close();
    
    Multimap<BulkIngestKey,Value> pairs = contextWriter.getWritten();
    
    assertEquals(1, pairs.size());
    assertTrue(pairs.containsEntry(new BulkIngestKey(new Text(metricsTable), new Key(expectedRow, metricName, "table\u0000shard\u0000shard")), new Value(
                    "1".getBytes())));
}
 

/**
 * Assert when EntryParser is set to skip node delays
 */
@Test
public void testParse_skipNodeDelay() throws IOException {
    TreeMap<Key,Value> data = new TreeMap<>();
    List<String> docIds = Arrays.asList("doc1", "doc2", "doc3", "doc4");
    Uid.List.Builder builder = Uid.List.newBuilder();
    builder.addAllUID(docIds);
    builder.setCOUNT(docIds.size());
    builder.setIGNORE(false);
    Value hasDocs = new Value(builder.build().toByteArray());
    
    List<IndexMatch> expected = new LinkedList<>();
    
    data.put(new Key("row", "cf", "20190314\u0000A"), hasDocs);
    addToExpected(expected, "A", docIds, JexlNodeFactory.buildEQNode("hello", "world"));
    
    CreateUidsIterator iterator = new CreateUidsIterator();
    iterator.init(new SortedMapIterator(data), null, null);
    iterator.seek(new Range(), Collections.emptySet(), false);
    
    EntryParser parser = new EntryParser("hello", "world", true);
    Map.Entry<Key,Value> top = Maps.immutableEntry(iterator.getTopKey(), iterator.getTopValue());
    Tuple2<String,IndexInfo> tuple = parser.apply(top);
    assertTrue(iterator.hasTop());
    
    assertNotNull(tuple);
    assertEquals("20190314", tuple.first());
    for (IndexMatch match : tuple.second().uids()) {
        assertTrue(expected.remove(match));
    }
    assertTrue(expected.isEmpty());
}
 

/**
 * {@inheritDoc}
 */
@Override
public void save(Iterable<ValueRange<T>> ranges) {
    checkNotNull(ranges);
    try {
        for (ValueRange<T> range : ranges) {

            checkState(helper.isValid(range), "Invalid Range:" + range.toString());

            String low = helper.encode(range.getStart());
            String high = helper.encode(range.getStop());

            Mutation forwardRange = new Mutation(LOWER_BOUND_INDEX + NULL_BYTE + low + NULL_BYTE + high);
            forwardRange.put(new Text(""), new Text(""), new Value("".getBytes()));

            Mutation reverseRange = new Mutation(UPPER_BOUND_INDEX + NULL_BYTE + high + NULL_BYTE + low);
            reverseRange.put(new Text(""), new Text(""), new Value("".getBytes()));

            String distanceComplement = helper.encodeComplement(helper.distance(range));
            Mutation distanceMut = new Mutation(DISTANCE_INDEX + NULL_BYTE + distanceComplement);
            distanceMut.put(new Text(low), new Text(high), new Value("".getBytes()));

            writer.addMutation(forwardRange);
            writer.addMutation(reverseRange);
            writer.addMutation(distanceMut);
        }

    } catch (RuntimeException re) {
        throw re;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

@Test
public void testDefault() throws Exception {
    SortedMap<Key, Value> table = new TreeMap<>();
    table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME), new byte[0], new byte[0], new byte[0],
            TEST_TIME), EMPTY_VALUE);
    table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 1), new byte[0], new byte[0],
            new byte[0], TEST_TIME + 1), EMPTY_VALUE);
    table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 2), new byte[0], new byte[0],
            new byte[0], TEST_TIME + 2), EMPTY_VALUE);
    table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 3), new byte[0], new byte[0],
            new byte[0], TEST_TIME + 3), EMPTY_VALUE);
    table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 4), new byte[0], new byte[0],
            new byte[0], TEST_TIME + 4), EMPTY_VALUE);
    table.put(new Key(MetricAdapter.encodeRowKey("sys.cpu.user", TEST_TIME + 5), new byte[0], new byte[0],
            new byte[0], TEST_TIME + 5), EMPTY_VALUE);

    SortedKeyValueIterator<Key, Value> source = new SortedMapIterator(table);
    MetricAgeOffIterator iter = new MetricAgeOffIterator();
    HashMap<String, String> options = new HashMap<>();
    options.put(MetricAgeOffIterator.AGE_OFF_PREFIX + "default", Integer.toString(1 * ONE_DAY));
    iter.init(source, options, null);
    iter.seek(new Range(), columnFamilies, true);
    int seen = 0;
    while (iter.hasTop()) {
        Key k = iter.getTopKey();
        Assert.assertTrue(k.getTimestamp() >= TEST_TIME && k.getTimestamp() <= TEST_TIME + 5);
        seen++;
        iter.next();
    }
    Assert.assertEquals(6, seen);
}
 

public ShardIndexQueryTransformer(BaseQueryLogic<Entry<Key,Value>> logic, Query settings, MarkingFunctions markingFunctions,
                ResponseObjectFactory responseObjectFactory, QueryModel qm) {
    super(markingFunctions);
    this.responseObjectFactory = responseObjectFactory;
    this.auths = new Authorizations(settings.getQueryAuthorizations().split(","));
    this.logic = logic;
    this.myQueryModel = qm;
}
 

@Override
public RyaDetails getRyaInstanceDetails() throws NotInitializedException, RyaDetailsRepositoryException {
    // Preconditions.
    if(!isInitialized()) {
        throw new NotInitializedException("Could not fetch the details for the Rya instanced named '" +
                instanceName + "' because it has not been initialized yet.");
    }

    // Read it from the table.
    Scanner scanner = null;
    try {
        // Fetch the value from the table.
        scanner = connector.createScanner(detailsTableName, new Authorizations());
        scanner.fetchColumn(COL_FAMILY, COL_QUALIFIER);
        final Entry<Key, Value> entry = scanner.iterator().next();

        // Deserialize it.
        final byte[] bytes = entry.getValue().get();
        return serializer.deserialize( bytes );

    } catch (final TableNotFoundException e) {
        throw new RyaDetailsRepositoryException("Could not get the details from the table.", e);
    } finally {
        if(scanner != null) {
            scanner.close();
        }
    }
}
 

private void persistTestMutations(int numRows, int entriesPerRow, int numColQs) throws TableNotFoundException, MutationsRejectedException {

        BatchWriter writer = connector.createBatchWriter("test", 1000, 1000, 1);

        for (int j = 0; j < numRows; j++) {
            Mutation m = new Mutation(Integer.toString(j));
            for (int i = 0; i < entriesPerRow; i++) {
                for(int q = 0; q < numColQs;   q++)
                    m.put(new Text(Integer.toString(i)), new Text(Integer.toString(q)), new Value("".getBytes()));
            }

            writer.addMutation(m);
        }
        writer.flush();
    }
 

private boolean popCache() {
    this.topKey = null;
    this.topValue = null;
    
    if (!keyCache.isEmpty()) {
        Map.Entry<Key,Value> entry = keyCache.pollFirstEntry();
        this.topKey = entry.getKey();
        this.topValue = entry.getValue();
        return true;
    }
    
    return false;
}
 

@Test(expected = IllegalArgumentException.class)
public void testDefaultMissing() throws Exception {
    SortedMap<Key, Value> table = new TreeMap<>();
    SortedKeyValueIterator<Key, Value> source = new SortedMapIterator(table);
    MetaAgeOffIterator iter = new MetaAgeOffIterator();
    HashMap<String, String> options = new HashMap<>();
    iter.init(source, options, null);
}
 

/**
 * forces a shard range
 * 
 * @return
 */
private static Value getValueForNuthinAndYourHitsForFree() {
    Uid.List.Builder builder = Uid.List.newBuilder();
    builder.setCOUNT(50); // better not be zero!!!!
    builder.setIGNORE(true); // better be true!!!
    return new Value(builder.build().toByteArray());
}
 

public void init(SortedKeyValueIterator<Key, Value> source, Map<String, String> options, IteratorEnvironment env) throws IOException {
    validateOptions(options);
    this.source = source;
    queue = new ArrayBlockingQueue<QueueElement>(queueSize);
    thread = new ProducerThread(this.source);
    t = new Thread(thread, "ReadAheadIterator-SourceThread");
    t.start();
}
 

/**
 * Add the index which will be managed by the versioning iterator and the data rows to scan from the index
 *
 * @param group
 * @param entry
 */
@Override
public void put(String group, Event entry) {
    checkNotNull(group);
    checkNotNull(entry);

    // first put the main index pointing to the contextId (The column family is prefixed with the NULL_BYTE to guarantee it shows up first
    Mutation indexMutation = new Mutation(group);
    indexMutation.put(NULL_BYTE + "INDEX", "", new ColumnVisibility(), entry.getTimestamp(), new Value((entry.getType() + ONE_BYTE + entry.getId()).getBytes()));

    for (Attribute attribute : entry.getAttributes()) {
        String fam = String.format("%s%s%s%s", END_BYTE, entry.getType(), ONE_BYTE, entry.getId());
        Object value = attribute.getValue();
        try {
            String serialize = typeRegistry.encode(value);
            String aliasForType = typeRegistry.getAlias(value);
            String qual = String.format("%s%s%s%s%s", attribute.getKey(), NULL_BYTE, serialize, NULL_BYTE, aliasForType);
            indexMutation.put(fam, qual, new ColumnVisibility(getVisibility(attribute, "")), entry.getTimestamp(),
                    new Value("".getBytes()));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    try {
        writer.addMutation(indexMutation);
    } catch (MutationsRejectedException ex) {
        throw new RuntimeException("There was an error writing the mutation for [index=" + group + ",entryId=" + entry.getId() + "]", ex);
    }
}
 

public static BasePoolableObjectFactory<SortedKeyValueIterator<Key,Value>> createIvaratorSourceFactory() {
    return new BasePoolableObjectFactory<SortedKeyValueIterator<Key,Value>>() {
        @Override
        public SortedKeyValueIterator<Key,Value> makeObject() throws Exception {
            return new SortedListKeyValueIterator(sourceList);
        }
    };
}
 

@Test
public void visitAnd_ExceededValueThresholdMarkerJexlNode_RangeUpperBoundaryTest() throws Exception {
    ASTJexlScript script = JexlASTHelper.parseJexlQuery("BAZ == 'woot' && ((ExceededValueThresholdMarkerJexlNode = true) && (FOO >= 'e' && FOO <= 'm'))");
    Key hit = new Key("row", "dataType" + Constants.NULL + "123.345.456");
    
    List<Map.Entry<Key,Value>> source = new ArrayList<>();
    source.add(new AbstractMap.SimpleEntry(new Key("row", "fi" + Constants.NULL + "BAZ", "woot" + Constants.NULL + "dataType" + Constants.NULL
                    + "123.345.456"), new Value()));
    source.add(new AbstractMap.SimpleEntry(
                    new Key("row", "tf", "dataType" + Constants.NULL + "123.345.456" + Constants.NULL + "m" + Constants.NULL + "FOO"), new Value()));
    
    Set<String> termFrequencyFields = new HashSet<>();
    termFrequencyFields.add("FOO");
    
    // must have doc to get tf field values are within the bounds
    // aggregation fields are not set so no document is created
    vistAnd_ExceededValueThesholdMarkerJexlNode_termFrequencyTest(script, hit, source, false, null, termFrequencyFields, Collections.EMPTY_SET,
                    Collections.emptySet());
    
    List<String> expected = new ArrayList<>();
    expected.add("m");
    Map<String,List<String>> fooMap = new HashMap<>();
    fooMap.put("FOO", expected);
    
    // turn on aggregation and see the document
    vistAnd_ExceededValueThesholdMarkerJexlNode_termFrequencyTest(script, hit, source, true, fooMap, false);
}
 

/**
 * Set splits on a table based on quantile distribution and fixed number of splits
 */
public static void setSplitsByQuantile(
    final BaseDataStore dataStore,
    final Connector connector,
    final String namespace,
    final Index index,
    final int quantile)
    throws AccumuloException, AccumuloSecurityException, IOException, TableNotFoundException {
  final long count = getEntries(dataStore, connector, namespace, index);

  try (final CloseableIterator<Entry<Key, Value>> iterator =
      getIterator(connector, namespace, index)) {

    if (iterator == null) {
      LOGGER.error("Could not get iterator instance, getIterator returned null");
      throw new IOException("Could not get iterator instance, getIterator returned null");
    }

    long ii = 0;
    final long splitInterval = (long) Math.ceil((double) count / (double) quantile);
    final SortedSet<Text> splits = new TreeSet<>();
    while (iterator.hasNext()) {
      final Entry<Key, Value> entry = iterator.next();
      ii++;
      if (ii >= splitInterval) {
        ii = 0;
        splits.add(entry.getKey().getRow());
      }
    }

    final String tableName = AccumuloUtils.getQualifiedTableName(namespace, index.getName());
    connector.tableOperations().addSplits(tableName, splits);
    connector.tableOperations().compact(tableName, null, null, true, true);
  }
}