Java Code Examples for org.apache.accumulo.core.data.Key

The following examples show how to use org.apache.accumulo.core.data.Key. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: datawave   Source File: SeekingAggregator.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Advance an iterator until skip(...) returns false. May be a combination of seek() and next() calls
 * 
 * @param itr
 * @param pointer
 * @param currentRange
 * @param columnFamilies
 * @param includeColumnFamilies
 * @throws IOException
 */
protected void advanceItr(SortedKeyValueIterator<Key,Value> itr, ByteSequence pointer, Range currentRange, Collection<ByteSequence> columnFamilies,
                boolean includeColumnFamilies) throws IOException {
    Key current = itr.getTopKey();
    Text row = current.getRow();
    int nextCount = 0;
    while (current != null && skip(current, row, pointer)) {
        if (maxNextCount == -1 || nextCount < maxNextCount) {
            itr.next();
            nextCount++;
        } else {
            Key startKey = getSeekStartKey(current, pointer);
            Range newRange = new Range(startKey, false, currentRange.getEndKey(), currentRange.isEndKeyInclusive());
            itr.seek(newRange, columnFamilies, includeColumnFamilies);
            nextCount = 0;
        }
        
        current = itr.hasTop() ? itr.getTopKey() : null;
    }
}
 
Example 2
Source Project: accumulo-recipes   Source File: AttributeIndexKey.java    License: Apache License 2.0 6 votes vote down vote up
public AttributeIndexKey(Key key) {

        String row = key.getRow().toString();
        String parts[] = StringUtils.splitByWholeSeparatorPreserveAllTokens(row, INDEX_SEP);
        int firstNBIdx = parts[3].indexOf(NULL_BYTE);

        if (row.startsWith(INDEX_V)) {
            int lastNBIdx = parts[3].lastIndexOf(NULL_BYTE);
            this.alias = parts[2];
            this.key = parts[3].substring(0, firstNBIdx);
            this.normalizedValue = parts[3].substring(firstNBIdx + 1, lastNBIdx);
            this.shard = parts[3].substring(lastNBIdx + 1, parts[3].length());
        } else if (row.startsWith(INDEX_K)) {
            this.key = parts[2];
            this.alias = parts[3].substring(0, firstNBIdx);
            this.shard = parts[3].substring(firstNBIdx + 1, parts[3].length());
        }
    }
 
Example 3
Source Project: datawave   Source File: TFFactory.java    License: Apache License 2.0 6 votes vote down vote up
public static com.google.common.base.Function<Tuple2<Key,Document>,Tuple3<Key,Document,Map<String,Object>>> getFunction(ASTJexlScript query,
                Set<String> contentExpansionFields, Set<String> termFrequencyFields, TypeMetadata typeMetadata, Equality equality,
                EventDataQueryFilter evaluationFilter, SortedKeyValueIterator<Key,Value> sourceCopy) {
    
    Multimap<String,Class<? extends Type<?>>> fieldMappings = LinkedListMultimap.create();
    for (Entry<String,String> dataType : typeMetadata.fold().entries()) {
        String dataTypeName = dataType.getValue();
        
        try {
            fieldMappings.put(dataType.getKey(), (Class<? extends Type<?>>) Class.forName(dataTypeName).asSubclass(Type.class));
        } catch (ClassNotFoundException e) {
            log.warn("Skipping instantiating a " + dataTypeName + " for " + dataType.getKey() + " because the class was not found.", e);
        }
        
    }
    
    return getFunction(query, contentExpansionFields, termFrequencyFields, fieldMappings, equality, evaluationFilter, sourceCopy);
}
 
Example 4
Source Project: rya   Source File: AndingIterator.java    License: Apache License 2.0 6 votes vote down vote up
public void addSource(final SortedKeyValueIterator<Key, Value> source, final IteratorEnvironment env, final Text term, final boolean notFlag) {
	// Check if we have space for the added Source
	if (sources == null) {
		sources = new TermSource[1];
	} else {
		// allocate space for node, and copy current tree.
		// TODO: Should we change this to an ArrayList so that we can just add() ?
		final TermSource[] localSources = new TermSource[sources.length + 1];
		int currSource = 0;
		for (final TermSource myTerm : sources) {
			// TODO: Do I need to call new here? or can I just re-use the term?
			localSources[currSource] = new TermSource(myTerm);
			currSource++;
		}
		sources = localSources;
	}
	sources[sourcesCount] = new TermSource(source.deepCopy(env), term, notFlag);
	sourcesCount++;
}
 
Example 5
Source Project: datawave   Source File: AncestorIndexBuildingVisitor.java    License: Apache License 2.0 6 votes vote down vote up
private List<String> getMembers() {
    Range wholeDocRange = getWholeDocRange(rangeLimiter);
    final String tld = getTLDId(wholeDocRange.getStartKey());
    final String dataType = getDataType(wholeDocRange.getStartKey());
    List<String> members = familyTreeMap.get(tld);
    
    // use the cached tree if available
    if (members == null) {
        SortedKeyValueIterator<Key,Value> kvIter = source.deepCopy(env);
        members = getMembers(wholeDocRange.getStartKey().getRow().toString(), tld, dataType, kvIter);
        
        // set the members for later use
        familyTreeMap.put(tld, members);
    }
    
    return members;
}
 
Example 6
Source Project: datawave   Source File: DescendantCountFunction.java    License: Apache License 2.0 6 votes vote down vote up
private boolean skipExcessiveNumberOfDescendants(final String childSuffix, final Matcher matcher, final Text row, final String fiRootValue, final Key endKey)
                throws IOException {
    boolean skipped;
    if (matcher.find() && (matcher.start() < childSuffix.length())) {
        // Get the base matching child suffix
        final String baseMatch = childSuffix.substring(0, matcher.start());
        
        // create the skipping range
        final Key skipStartKey = new Key(row, this.indexCf, new Text(fiRootValue + baseMatch + '0'));
        final Range skipRange = new Range(skipStartKey, true, endKey, false);
        
        // seek to the next first-generation child, if one exists
        final Set<ByteSequence> emptyCfs = Collections.emptySet();
        this.source.seek(skipRange, emptyCfs, false);
        
        // Assign the return value
        skipped = true;
    } else {
        skipped = false;
    }
    
    return skipped;
}
 
Example 7
Source Project: datawave   Source File: AbstractFunctionalQuery.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public String parse(Key key, Document document) {
    Attribute<?> attr = document.get(this.documentKey);
    Assert.assertNotNull("document key(" + this.documentKey + ") attribute is null", attr);
    
    Object data = attr.getData();
    Assert.assertNotNull("document key attribute is null: key(" + this.documentKey + ")", data);
    String keyVal = null;
    if (data instanceof Type) {
        keyVal = ((Type<?>) data).getDelegate().toString();
    } else if (data instanceof String) {
        keyVal = (String) data;
    } else {
        Assert.fail("invalid type for document key(" + this.documentKey + "");
    }
    
    return keyVal;
}
 
Example 8
Source Project: datawave   Source File: DocumentAggregatingIterator.java    License: Apache License 2.0 6 votes vote down vote up
public void move(Key pointer) throws IOException {
    // check the current position
    if (nextKey != null && nextKey.compareTo(pointer) >= 0) {
        throw new IllegalStateException("Tried to call move when already at or beyond move point: topkey=" + nextKey + ", movekey=" + pointer);
    }
    
    if (!getSource().hasTop()) {
        // there is nothing beyond the current key
        nextKey = null;
        nextValue = null;
        document = null;
    } else if (getSource().getTopKey().compareTo(pointer) >= 0) {
        // load that into next
        next();
    } else {
        // we have to seek
        seek(new Range(pointer, true, seekRange.getEndKey(), seekRange.isEndKeyInclusive()), seekColumnFamilies, seekInclusive);
    }
}
 
Example 9
/**
 * Creates and writes the BulkIngestKey for the event's field/value to the ContextWriter (instead of the Multimap that the {@link ShardedDataTypeHandler}
 * uses).
 *
 * @param event
 * @param eventFields
 * @param values
 * @param fieldVisibility
 * @param shardId
 * @param fieldVisibility
 * @param nFV
 * @throws IOException
 * @throws InterruptedException
 */
protected void createEventColumn(RawRecordContainer event, Multimap<String,NormalizedContentInterface> eventFields, Multimap<BulkIngestKey,Value> values,
                byte[] shardId, byte[] fieldVisibility, NormalizedContentInterface nFV) throws IOException, InterruptedException {
    
    String fieldName = nFV.getEventFieldName();
    String fieldValue = nFV.getEventFieldValue();
    
    if (StringUtils.isEmpty(fieldValue))
        return;
    
    Text colf = new Text(event.getDataType().outputName());
    TextUtil.textAppend(colf, event.getId().toString(), helper.getReplaceMalformedUTF8());
    
    Text colq = new Text(fieldName);
    TextUtil.textAppend(colq, fieldValue, helper.getReplaceMalformedUTF8());
    Key k = createKey(shardId, colf, colq, fieldVisibility, event.getDate(), helper.getDeleteMode());
    BulkIngestKey bKey = new BulkIngestKey(new Text(this.getShardTableName()), k);
    values.put(bKey, NULL_VALUE);
}
 
Example 10
@Test
public void getSeekRange_filter2InclusiveTest() {
    EventDataQueryFilter mockFilter1 = EasyMock.createMock(EventDataQueryFilter.class);
    EventDataQueryFilter mockFilter2 = EasyMock.createMock(EventDataQueryFilter.class);
    
    Key current = new Key();
    Key end = new Key();
    
    Range filter1Result = new Range(new Key("2"), true, new Key("9999"), true);
    Range filter2Result = new Range(new Key("234"), true, new Key("999"), true);
    
    EasyMock.expect(mockFilter1.getSeekRange(current, end, true)).andReturn(filter1Result);
    EasyMock.expect(mockFilter2.getSeekRange(current, end, true)).andReturn(filter2Result);
    
    EasyMock.replay(mockFilter1, mockFilter2);
    
    filter.addFilter(mockFilter1);
    filter.addFilter(mockFilter2);
    
    Range result = filter.getSeekRange(current, end, true);
    
    EasyMock.verify(mockFilter1, mockFilter2);
    
    Assert.assertFalse(result == null);
    Assert.assertTrue(result.equals(filter2Result));
}
 
Example 11
Source Project: datawave   Source File: FieldAgeOffFilterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void handlesFieldsStartingWithNumber() {
    long oneSecondAgo = System.currentTimeMillis() - (1 * ONE_SEC);
    long tenSecondsAgo = System.currentTimeMillis() - (10 * ONE_SEC);
    
    FieldAgeOffFilter ageOffFilter = new FieldAgeOffFilter();
    FilterOptions filterOptions = createFilterOptionsWithPattern();
    // set the default to 5 seconds
    filterOptions.setTTL(5);
    filterOptions.setTTLUnits(AgeOffTtlUnits.SECONDS);
    // set up ttls for field_y and field_z only, deliberately exclude the ttl for field_y
    filterOptions.setOption("fields", "field_y");
    filterOptions.setOption("field.12_3_4.ttl", "2"); // 2 seconds
    filterOptions.setOption("field.12_3_4.ttlUnits", "s"); // 2 seconds
    ageOffFilter.init(filterOptions, iterEnv);
    // field_y is a match, but its ttl was not defined, so it will use the default one
    Key key = new Key("1234", "myDataType\\x00my-uuid", "field_y\u0000value", VISIBILITY_PATTERN, tenSecondsAgo);
    Assert.assertFalse(ageOffFilter.accept(filterOptions.getAgeOffPeriod(System.currentTimeMillis()), key, new Value()));
    Assert.assertTrue(ageOffFilter.isFilterRuleApplied());
    // field_z is a match, but the key is more recent than its age off period
    Key keyNumeric = new Key("1234", "myDataType\\x00my-uuid", "12_3_4\u0000value", VISIBILITY_PATTERN, oneSecondAgo);
    Assert.assertTrue(ageOffFilter.accept(filterOptions.getAgeOffPeriod(System.currentTimeMillis()), keyNumeric, new Value()));
    Assert.assertTrue(ageOffFilter.isFilterRuleApplied());
}
 
Example 12
Source Project: cognition   Source File: AccumuloItem.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Using the provided PeekingIterator, construct an AccumuloItem from the Key-Value pairs it returns.  If skipBadRow
 * is true, this method will skip rows not containing AccumuloItems until there are now more Key-Value pairs or an
 * Accumulo Item is found, otherwise the method will return null;
 *
 * @param iter
 * @param skipBadRow
 * @return
 */
public static AccumuloItem buildFromIterator(PeekingIterator<Entry<Key, Value>> iter, boolean skipBadRow) {
  while (iter.hasNext()) {
    Entry<Key, Value> pair = iter.peek();
    AccumuloItem i = constructItemFromFirstPair(pair.getKey(), pair.getValue());
    if (i != null) {
      iter.next();
      i.populateFromRow(iter);
      return i;
    } else if (skipBadRow) {
      buildRowMap(iter);
    } else {
      break;
    }
  }
  return null;
}
 
Example 13
Source Project: geowave   Source File: RowMergingVisibilityCombiner.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void init(
    final SortedKeyValueIterator<Key, Value> source,
    final Map<String, String> options,
    final IteratorEnvironment env) throws IOException {
  super.init(source, options, env);
  final String rowTransformStr = options.get(RowMergingAdapterOptionProvider.ROW_TRANSFORM_KEY);
  final byte[] rowTransformBytes = ByteArrayUtils.byteArrayFromString(rowTransformStr);
  rowTransform = (RowTransform<Mergeable>) URLClassloaderUtils.fromBinary(rowTransformBytes);
  rowTransform.initOptions(options);
}
 
Example 14
Source Project: datawave   Source File: QueriesTableAgeOffIterator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean accept(Key k, Value v) {
    // The timestamp in the key of the queries table is the expiration date. If it is less than the
    // current time, then remove the key.
    if (currentTime > k.getTimestamp())
        return false;
    else
        return true;
}
 
Example 15
Source Project: datawave   Source File: QueryIterator.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * A routine which should always be used to create deep copies of the source. This ensures that we are thread safe when doing these copies.
 * 
 * @return
 */
public SortedKeyValueIterator<Key,Value> getSourceDeepCopy() {
    SortedKeyValueIterator<Key,Value> sourceDeepCopy;
    synchronized (sourceForDeepCopies) {
        sourceDeepCopy = sourceForDeepCopies.deepCopy(this.myEnvironment);
    }
    return sourceDeepCopy;
}
 
Example 16
Source Project: datawave   Source File: SortedListKeyValueIterator.java    License: Apache License 2.0 5 votes vote down vote up
public SortedListKeyValueIterator(Iterator<Map.Entry<Key,Value>> sourceIterator) {
    this.sourceList = new ArrayList<>();
    while (sourceIterator.hasNext()) {
        sourceList.add(sourceIterator.next());
    }
    this.sourceList.sort(Map.Entry.comparingByKey());
    currentIndex = 0;
}
 
Example 17
Source Project: datawave   Source File: TLDTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testParseRootPointerFromFI() {
    Key fiKey = buildFiKey(rootId);
    ByteSequence parsed = TLD.parseRootPointerFromFI(fiKey.getColumnQualifierData());
    ByteSequence expected = new ArrayByteSequence(datatype + '\u0000' + rootId);
    assertEquals(expected, parsed);
    
    fiKey = buildFiKey(childId);
    parsed = TLD.parseRootPointerFromFI(fiKey.getColumnQualifierData());
    assertEquals(expected, parsed);
    
    fiKey = buildFiKey(grandchildId);
    parsed = TLD.parseRootPointerFromFI(fiKey.getColumnQualifierData());
    assertEquals(expected, parsed);
}
 
Example 18
Source Project: vertexium   Source File: EdgeIterator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public SortedKeyValueIterator<Key, Value> deepCopy(IteratorEnvironment env) {
    if (getSourceIterator() != null) {
        return new EdgeIterator(getSourceIterator().deepCopy(env), getFetchHints(), isCompressTransfer(), getAuthorizations());
    }
    return new EdgeIterator(getFetchHints(), isCompressTransfer(), getAuthorizations());
}
 
Example 19
Source Project: OSTMap   Source File: GeoTempFlatMap.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void flatMap(Tuple2<Key, Value> value, Collector<Tuple2<Text, Mutation>> out) throws Exception {

    GeoTemporalKey gtk = GeoTemporalKey.buildKey(value.f1.toString());

    if (gtk.rowBytes != null && gtk.columQualifier != null) {
        //create mutations for username and screen name
        Mutation m = new Mutation(gtk.rowBytes);
        m.put(value.f0.getRow().getBytes(), gtk.columQualifier, EMPTY_BYTES);
        out.collect(new Tuple2<>(new Text(outputTableName), m));
    }
}
 
Example 20
Source Project: datawave   Source File: BaseEdgeQueryTest.java    License: Apache License 2.0 5 votes vote down vote up
public void compareResults(BaseQueryLogic<Map.Entry<Key,Value>> logic, List<String> expected) {
    int recordsFound = 0;
    List<Key> foundKeys = new ArrayList<>();
    for (Map.Entry<Key,Value> entry : logic) {
        foundKeys.add(entry.getKey());
        Key k = entry.getKey();
        System.out.println("key = " + k.toStringNoTime());
        Assert.assertTrue(UNEXPECTED_RECORD + " : " + k.toStringNoTime(), expected.contains(k.toStringNoTime()));
        recordsFound++;
    }
    
    Assert.assertEquals(UNEXPECTED_NUM_RECORDS, expected.size(), recordsFound);
}
 
Example 21
Source Project: datawave   Source File: Document.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Given an iterator over {@code Entry<Key, Value>}, and a set of normalizers, this method will merge the attributes scanned over by the supplied iterator
 * into <code>this</code> Document.
 *
 * @param iter
 * @param typeMetadata
 * @return
 */
public Document consumeRawData(Key docKey, Set<Key> docKeys, Iterator<Entry<Key,Value>> iter, TypeMetadata typeMetadata,
                CompositeMetadata compositeMetadata, boolean includeGroupingContext, boolean keepRecordId, EventDataQueryFilter attrFilter) {
    invalidateMetadata();
    // extract the sharded time from the dockey if possible
    try {
        this.shardTimestamp = DateHelper.parseWithGMT(docKey.getRow().toString()).getTime();
    } catch (DateTimeParseException e) {
        log.warn("Unable to parse document key row as a shard id of the form yyyyMMdd...: " + docKey.getRow(), e);
        // leave the shardTimestamp empty
        this.shardTimestamp = Long.MAX_VALUE;
    }
    
    // Extract the fieldName from the Key
    Iterator<Entry<Key,String>> extractedFieldNames = Iterators.transform(iter, new KeyToFieldName(includeGroupingContext));
    
    // Transform the remaining entries back into Attributes
    Iterator<Iterable<Entry<String,Attribute<? extends Comparable<?>>>>> attributes = Iterators.transform(extractedFieldNames, new ValueToAttributes(
                    compositeMetadata, typeMetadata, attrFilter, MarkingFunctions.Factory.createMarkingFunctions()));
    
    // Add all of the String=>Attribute pairs to this Document
    while (attributes.hasNext()) {
        Iterable<Entry<String,Attribute<? extends Comparable<?>>>> entries = attributes.next();
        for (Entry<String,Attribute<? extends Comparable<?>>> entry : entries) {
            this.put(entry, includeGroupingContext);
        }
    }
    
    // now add the dockeys as attributes
    Attribute<?> docKeyAttributes = toDocKeyAttributes(docKeys, keepRecordId);
    if (docKeyAttributes != null) {
        this.put(DOCKEY_FIELD_NAME, docKeyAttributes);
    }
    
    // a little debugging here to track large documents
    debugDocumentSize(docKey);
    
    return this;
}
 
Example 22
Source Project: datawave   Source File: ShardTableQueryMetricHandler.java    License: Apache License 2.0 5 votes vote down vote up
private void writeMetrics(QueryMetric updatedQueryMetric, List<QueryMetric> storedQueryMetrics, Date lastUpdated, boolean delete) throws Exception {
    LiveContextWriter contextWriter = null;
    
    MapContext<Text,RawRecordContainer,Text,Mutation> context = null;
    
    try {
        contextWriter = new LiveContextWriter();
        contextWriter.setup(conf, false);
        
        TaskAttemptID taskId = new TaskAttemptID(new TaskID(new JobID(JOB_ID, 1), TaskType.MAP, 1), 1);
        context = new MapContextImpl<>(conf, taskId, null, recordWriter, null, reporter, null);
        
        for (QueryMetric storedQueryMetric : storedQueryMetrics) {
            AbstractColumnBasedHandler<Key> handler = new ContentQueryMetricsHandler<>();
            handler.setup(context);
            
            Multimap<BulkIngestKey,Value> r = getEntries(handler, updatedQueryMetric, storedQueryMetric, lastUpdated, delete);
            
            try {
                if (r != null) {
                    contextWriter.write(r, context);
                }
                
                if (handler.getMetadata() != null) {
                    contextWriter.write(handler.getMetadata().getBulkMetadata(), context);
                }
            } finally {
                contextWriter.commit(context);
            }
        }
    } finally {
        if (contextWriter != null && context != null) {
            contextWriter.cleanup(context);
        }
    }
}
 
Example 23
Source Project: datawave   Source File: DatawaveKeyTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void parseShardFiKey() {
    Key testKey = new Key("row", "fi\u0000fieldNameA", "fieldValueA\u0000datatype\u0000uid", "viz");
    DatawaveKey key = new DatawaveKey(testKey);
    
    Assert.assertEquals("datatype", key.getDataType());
    Assert.assertEquals("uid", key.getUid());
    Assert.assertEquals("fieldValueA", key.getFieldValue());
    Assert.assertEquals("fieldNameA", key.getFieldName());
    Assert.assertEquals(new Text("row"), key.getRow());
}
 
Example 24
Source Project: datawave   Source File: FinalDocumentTrackingIterator.java    License: Apache License 2.0 5 votes vote down vote up
public static boolean isFinalDocumentKey(Key k) {
    if (k != null && k.getColumnQualifierData() != null) {
        ByteSequence bytes = k.getColumnQualifierData();
        if (bytes.length() >= MARKER_TEXT.getLength()) {
            return (bytes.subSequence(bytes.length() - MARKER_TEXT.getLength(), bytes.length()).compareTo(MARKER_SEQUENCE) == 0);
        }
    }
    return false;
}
 
Example 25
Source Project: accumulo-recipes   Source File: AccumuloRangeStore.java    License: Apache License 2.0 5 votes vote down vote up
private T getMaxDistance(Authorizations auths) throws TableNotFoundException {
    Scanner scanner = connector.createScanner(tableName, auths);
    scanner.setRange(prefix(DISTANCE_INDEX));
    scanner.setBatchSize(1);

    Iterator<Entry<Key, Value>> iterator = scanner.iterator();
    if (!iterator.hasNext())
        return null;

    //Only need the top one, as it should be sorted by size.
    Entry<Key, Value> entry = iterator.next();
    return helper.decodeComplement(entry.getKey().getRow().toString().split(NULL_BYTE)[1]);
}
 
Example 26
Source Project: datawave   Source File: TLDEventDataFilterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void getSeekRange_maxFieldSeekNotEqualToLimit() {
    Map<String,Integer> fieldLimits = new HashMap<>(1);
    fieldLimits.put("field1", 1);
    
    expect(mockScript.jjtGetNumChildren()).andReturn(0).anyTimes();
    expect(mockScript.jjtAccept(isA(EventDataQueryExpressionVisitor.class), eq(""))).andReturn(null);
    
    replayAll();
    
    Key key1 = new Key("row", "column", "field1" + Constants.NULL_BYTE_STRING + "value");
    Key key2 = new Key("row", "column", "field2" + Constants.NULL_BYTE_STRING + "value");
    filter = new TLDEventDataFilter(mockScript, mockAttributeFactory, null, null, 3, -1, fieldLimits, "LIMIT_FIELD", Collections.EMPTY_SET);
    
    assertTrue(filter.keep(key1));
    // increments counts = 1
    assertTrue(filter.apply(new AbstractMap.SimpleEntry<>(key1, null)));
    assertNull(filter.getSeekRange(key1, key1.followingKey(PartialKey.ROW), false));
    // does not increment counts so will still return true
    assertTrue(filter.keep(key1));
    // increments counts = 2 rejected by field count
    assertFalse(filter.apply(new AbstractMap.SimpleEntry<>(key1, null)));
    assertNull(filter.getSeekRange(key1, key1.followingKey(PartialKey.ROW), false));
    
    // now fails
    assertFalse(filter.keep(key1));
    
    // see another key on apply to trigger the seek range
    assertFalse(filter.apply(new AbstractMap.SimpleEntry<>(key1, null)));
    Range seekRange = filter.getSeekRange(key1, key1.followingKey(PartialKey.ROW), false);
    assertNotNull(seekRange);
    assertEquals(seekRange.getStartKey().getRow(), key1.getRow());
    assertEquals(seekRange.getStartKey().getColumnFamily(), key1.getColumnFamily());
    assertEquals(seekRange.getStartKey().getColumnQualifier().toString(), "field1" + "\u0001");
    assertEquals(true, seekRange.isStartKeyInclusive());
    
    verifyAll();
}
 
Example 27
Source Project: OSTMap   Source File: FlinkEnvManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * makes accumulo input accessible by flink DataSet api
 * @param env
 * @return
 * @throws IOException
 * @throws AccumuloSecurityException
 */
public DataSet<Tuple2<Key,Value>> getDataFromAccumulo(ExecutionEnvironment env) throws IOException, AccumuloSecurityException {
    job = Job.getInstance(new Configuration(), jobName);
    AccumuloInputFormat.setConnectorInfo(job, accumuloUser, new PasswordToken(accumuloPassword));
    AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(AccumuloIdentifiers.AUTHORIZATION.toString()));
    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.withInstance(accumuloInstanceName);
    clientConfig.withZkHosts(accumuloZookeeper);
    AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
    AccumuloInputFormat.setInputTableName(job, inTable);
    return env.createHadoopInput(new AccumuloInputFormat(),Key.class,Value.class, job);
}
 
Example 28
Source Project: datawave   Source File: EdgeKey.java    License: Apache License 2.0 5 votes vote down vote up
public Key encodeLegacyAttribute2Key() {
    if (this.getFormat() == EDGE_FORMAT.STATS) {
        return encode(EDGE_VERSION.STATS_ATTRIBUTE2);
    } else if (this.getFormat() == EDGE_FORMAT.STANDARD) {
        return encode(EDGE_VERSION.BASE_ATTRIBUTE2);
    } else {
        // EDGE_FORMAT.UNKNOWN
        throw new IllegalStateException("Can't encode unknown edge key format." + this);
    }
}
 
Example 29
@Override
protected void consume() throws IOException {
    if (finished == true || lastRowFound == null)
        return;
    int count = 0;

    String curPrefix = null;
    String lastPrefix = getPrefix(lastRowFound.toString());

    if(getSource().hasTop())
        curPrefix = getPrefix(getSource().getTopKey().getRow().toString());

    while (getSource().hasTop() && curPrefix.equals(lastPrefix)) {
        // try to efficiently jump to the next matching key
        if (count < numscans) {
            ++count;
            getSource().next(); // scan
            if(getSource().hasTop())
                curPrefix = getPrefix(getSource().getTopKey().getRow().toString());
        } else {
            // too many scans, just seek
            count = 0;

            // determine where to seek to, but don't go beyond the user-specified range

            Key nextKey = new Key(curPrefix + END_BYTE);

            if (!latestRange.afterEndKey(nextKey))
                getSource().seek(new Range(nextKey, true, latestRange.getEndKey(), latestRange.isEndKeyInclusive()), latestColumnFamilies, latestInclusive);
            else {
                finished = true;
                break;
            }
        }
    }
    lastRowFound = getSource().hasTop() ? getSource().getTopKey().getRow(lastRowFound) : null;
}
 
Example 30
Source Project: AccumuloGraph   Source File: VertexInputFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
  if (rowIterator.hasNext()) {
    Iterator<Entry<Key,Value>> it = rowIterator.next();

    MapReduceVertex vertex = new MapReduceVertex(parent);
    while (it.hasNext()) {
      Entry<Key,Value> entry = it.next();
      numKeysRead++;

      currentKey = entry.getKey();
      String vid = currentKey.getRow().toString();
      String colf = currentKey.getColumnFamily().toString();
      switch (colf) {
        case Constants.LABEL:
          currentK.set(vid);
          vertex.prepareId(vid);
          break;
        case Constants.IN_EDGE:
          String[] parts = currentKey.getColumnQualifier().toString().split(Constants.ID_DELIM);
          String label = new String(entry.getValue().get());
          vertex.prepareEdge(parts[1], parts[0], label, vid);
          break;
        case Constants.OUT_EDGE:
          parts = currentKey.getColumnQualifier().toString().split(Constants.ID_DELIM);
          label = new String(entry.getValue().get());
          vertex.prepareEdge(parts[1], vid, label, parts[0]);
          break;
        default:
          String propertyKey = currentKey.getColumnFamily().toString();
          Object propertyValue = AccumuloByteSerializer.deserialize(entry.getValue().get());
          vertex.prepareProperty(propertyKey, propertyValue);
      }
    }
    currentV = vertex;
    return true;
  }
  return false;
}