Java Code Examples for org.apache.accumulo.core.client.Scanner

The following examples show how to use org.apache.accumulo.core.client.Scanner. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: AccumuloGraph   Source File: BaseIndexValuesTableWrapper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get elements with the key/value pair.
 * @param key
 * @param value
 * @return
 */
@SuppressWarnings("unchecked")
public <T extends Element> CloseableIterable<T> readElementsFromIndex(String key, Object value) {
  Scanner scan = getScanner();
  byte[] id = AccumuloByteSerializer.serialize(value);
  scan.setRange(Range.exact(new Text(id)));
  scan.fetchColumnFamily(new Text(key));

  final ElementIndexParser<? extends AccumuloElement> parser =
      Vertex.class.equals(elementType) ? new VertexIndexParser(globals) :
        new EdgeIndexParser(globals);

      return new ScannerIterable<T>(scan) {
        @Override
        public T next(PeekingIterator<Entry<Key,Value>> iterator) {
          return (T) parser.parse(Arrays.asList(iterator.next()));
        }      
      };
}
 
Example 2
Source Project: AccumuloGraph   Source File: ElementTableWrapper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Read the given properties for the given element.
 * If propertyKeys is null, read all properties.
 * If the element has no properties, return an empty Map.
 * If the element does not exist, return null.
 * @param id
 * @param propertyKeys
 * @return
 */
public Map<String, Object> readProperties(Element element, String[] propertyKeys) {
  Scanner s = getScanner();
  s.setRange(Range.exact((String) element.getId()));

  // If propertyKeys is null, we read everything.
  // Otherwise, limit to the given attributes.
  if (propertyKeys != null) {
    s.fetchColumnFamily(new Text(Constants.LABEL));

    for (String key : propertyKeys) {
      s.fetchColumnFamily(new Text(key));
    }
  }

  Map<String, Object> props = new PropertyParser().parse(s);
  s.close();

  return props;
}
 
Example 3
Source Project: datawave   Source File: AccumuloIndexAgeDisplay.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Add one or more column families or column family and qualifier to a scanner.
 * 
 * @param scanner
 *            to add columns to.
 * @return The scanner with columns
 */
private Scanner addColumnsToScanner(Scanner scanner) {
    if ((null != columns) && (!columns.equals(("")))) {
        String[] cols = columns.split(",");
        for (String colStr : cols) {
            String[] parts = colStr.split(":");
            if (parts.length == 1) {
                scanner.fetchColumnFamily(new Text(parts[0]));
            } else if (parts.length == 2) {
                scanner.fetchColumn(new Text(parts[0]), new Text(parts[1]));
            }
        }
    }
    
    return (scanner);
}
 
Example 4
Source Project: datawave   Source File: TermFrequencyQueryTable.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setupQuery(GenericQueryConfiguration configuration) throws Exception {
    if (!configuration.getClass().getName().equals(TermFrequencyQueryConfiguration.class.getName())) {
        throw new QueryException("Did not receive a TermFrequencyQueryConfiguration instance");
    }
    
    TermFrequencyQueryConfiguration tfConfig = (TermFrequencyQueryConfiguration) configuration;
    
    try {
        Scanner scanner = QueryScannerHelper.createScanner(tfConfig.getConnector(), tfConfig.getTableName(), tfConfig.getAuthorizations(),
                        tfConfig.getQuery());
        scanner.setRange(tfConfig.getRange());
        
        this.iterator = scanner.iterator();
        this.scanner = scanner;
    } catch (TableNotFoundException e) {
        throw new RuntimeException("Table not found: " + this.getTableName(), e);
    }
}
 
Example 5
Source Project: fluo   Source File: FailureIT.java    License: Apache License 2.0 6 votes vote down vote up
private boolean wasRolledBackPrimary(long startTs, String rolledBackRow)
    throws TableNotFoundException {
  boolean sawExpected = false;
  Scanner scanner = aClient.createScanner(getCurTableName(), Authorizations.EMPTY);

  for (Entry<Key, Value> entry : scanner) {
    ColumnType colType = ColumnType.from(entry.getKey());
    long ts = entry.getKey().getTimestamp() & ColumnConstants.TIMESTAMP_MASK;
    String row = entry.getKey().getRowData().toString();
    byte[] val = entry.getValue().get();

    if (row.equals(rolledBackRow) && colType == ColumnType.DEL_LOCK && ts == startTs
        && DelLockValue.isPrimary(val)) {
      sawExpected = true;
    }
  }
  return sawExpected;
}
 
Example 6
Source Project: datawave   Source File: PrintUtility.java    License: Apache License 2.0 6 votes vote down vote up
public static void printTable(final Connector conn, final Authorizations authorizations, final String tableName, final PrintStream out)
                throws TableNotFoundException {
    final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd HHmmss");
    
    final StringBuilder sb = new StringBuilder("--Begin entire " + tableName + " table--");
    
    sb.append("\n");
    
    final Scanner scanner = conn.createScanner(tableName, authorizations);
    for (final Entry<Key,Value> e : scanner) {
        sb.append(e.getKey().toStringNoTime());
        sb.append(' ');
        sb.append(dateFormat.format(new Date(e.getKey().getTimestamp())));
        sb.append('\t');
        sb.append(getPrintableValue(e.getValue()));
        sb.append("\n");
    }
    
    sb.append("--End entire ").append(tableName).append(" table--").append("\n");
    
    out.println(sb);
}
 
Example 7
Source Project: datawave   Source File: AbstractFunctionalQuery.java    License: Apache License 2.0 6 votes vote down vote up
protected Multimap<String,Key> removeMetadataEntries(Set<String> fields, Text cf) throws AccumuloSecurityException, AccumuloException,
                TableNotFoundException {
    Multimap<String,Key> metadataEntries = HashMultimap.create();
    MultiTableBatchWriter multiTableWriter = connector.createMultiTableBatchWriter(new BatchWriterConfig());
    BatchWriter writer = multiTableWriter.getBatchWriter(QueryTestTableHelper.METADATA_TABLE_NAME);
    for (String field : fields) {
        Mutation mutation = new Mutation(new Text(field));
        Scanner scanner = connector.createScanner(QueryTestTableHelper.METADATA_TABLE_NAME, new Authorizations());
        scanner.fetchColumnFamily(cf);
        scanner.setRange(new Range(new Text(field)));
        boolean foundEntries = false;
        for (Map.Entry<Key,Value> entry : scanner) {
            foundEntries = true;
            metadataEntries.put(field, entry.getKey());
            mutation.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getKey().getColumnVisibilityParsed());
        }
        scanner.close();
        if (foundEntries) {
            writer.addMutation(mutation);
        }
    }
    writer.close();
    connector.tableOperations().compact(QueryTestTableHelper.METADATA_TABLE_NAME, new Text("\0"), new Text("~"), true, true);
    return metadataEntries;
}
 
Example 8
Source Project: AccumuloGraph   Source File: IndexMetadataTableWrapper.java    License: Apache License 2.0 6 votes vote down vote up
public <T extends Element> Set<String> getIndexedKeys(Class<T> elementClass) {
  if (elementClass == null) {
    throw ExceptionFactory.classForElementCannotBeNull();
  }

  IndexedItemsListParser parser = new IndexedItemsListParser(elementClass);

  Scanner scan = null;
  try {
    scan = getScanner();
    scan.fetchColumnFamily(new Text(IndexMetadataEntryType.__INDEX_KEY__.name()));

    Set<String> keys = new HashSet<String>();
    for (IndexedItem item : parser.parse(scan)) {
      keys.add(item.getKey());
    }

    return keys;

  } finally {
    if (scan != null) {
      scan.close();
    }
  }
}
 
Example 9
Source Project: accumulo-examples   Source File: QueryUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Scans over the index table for files or directories with a given name, prefix, or suffix
 * (indicated by a wildcard '*' at the beginning or end of the term.
 *
 * @param exp
 *          the name a file or directory to search for with an optional wildcard '*' at the
 *          beginning or end
 */
public Iterable<Entry<Key,Value>> singleRestrictedWildCardSearch(String exp) throws Exception {
  if (exp.indexOf("/") >= 0)
    throw new Exception("this method only works with unqualified names");

  Scanner scanner = client.createScanner(tableName, auths);
  if (exp.startsWith("*")) {
    System.out.println("executing beginning wildcard search for " + exp);
    exp = exp.substring(1);
    scanner.setRange(Range.prefix(getReverseIndex(exp)));
  } else if (exp.endsWith("*")) {
    System.out.println("executing ending wildcard search for " + exp);
    exp = exp.substring(0, exp.length() - 1);
    scanner.setRange(Range.prefix(getForwardIndex(exp)));
  } else if (exp.indexOf("*") >= 0) {
    throw new Exception("this method only works for beginning or ending wild cards");
  } else {
    return exactTermSearch(exp);
  }
  return scanner;
}
 
Example 10
Source Project: accumulo-examples   Source File: TracingExample.java    License: Apache License 2.0 6 votes vote down vote up
private void readEntries(Opts opts) throws TableNotFoundException {

    Scanner scanner = client.createScanner(opts.getTableName(), opts.auths);

    // Trace the read operation.
    TraceScope readScope = Trace.startSpan("Client Read", Sampler.ALWAYS);
    System.out.println("TraceID: " + Long.toHexString(readScope.getSpan().getTraceId()));

    int numberOfEntriesRead = 0;
    for (Entry<Key,Value> entry : scanner) {
      System.out.println(entry.getKey().toString() + " -> " + entry.getValue().toString());
      ++numberOfEntriesRead;
    }
    // You can add additional metadata (key, values) to Spans which will be able to be viewed in the
    // Monitor
    readScope.getSpan().addKVAnnotation("Number of Entries Read".getBytes(UTF_8),
        String.valueOf(numberOfEntriesRead).getBytes(UTF_8));

    readScope.close();
  }
 
Example 11
Source Project: rya   Source File: AccumuloTemporalIndexer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Interval after given interval.  Find intervals that begin after the endings of the given interval.
 * Use the special following prefix mechanism to avoid matching the beginning date.
 * Indexing Intervals  will probably change or be removed.
 * Currently predicate and subject and context constraints are filtered on the client.
 */
@Override
public CloseableIteration<Statement, QueryEvaluationException> queryIntervalAfter(
        final TemporalInterval queryInterval, final StatementConstraints constraints)
        throws QueryEvaluationException {

    final Scanner scanner = getScanner();
    if (scanner != null) {
        // get rows where the start date is greater than the queryInterval.getEnd()
        final Range range = new Range(new Key(Range.followingPrefix(new Text(queryInterval.getHasEnd().getAsKeyBytes()))), false, null, true);
        scanner.setRange(range);

        if (constraints.hasContext()) {
            scanner.fetchColumn(new Text(constraints.getContext().toString()), new Text(KeyParts.CQ_BEGIN));
        } else {
            scanner.fetchColumn(new Text(""), new Text(KeyParts.CQ_BEGIN));
        }
    }
    // TODO currently predicate, subject and context constraints are filtered on the clients
    return getIteratorWrapper(scanner);
}
 
Example 12
Source Project: accumulo-examples   Source File: ARS.java    License: Apache License 2.0 6 votes vote down vote up
public List<String> list(String what, String when) throws Exception {
  String row = what + ":" + when;

  // its important to use an isolated scanner so that only whole mutations are seen
  try (
      Scanner scanner = new IsolatedScanner(client.createScanner(rTable, Authorizations.EMPTY))) {
    scanner.setRange(new Range(row));
    scanner.fetchColumnFamily(new Text("res"));

    List<String> reservations = new ArrayList<>();

    for (Entry<Key,Value> entry : scanner) {
      String val = entry.getValue().toString();
      reservations.add(val);
    }

    return reservations;
  }
}
 
Example 13
Source Project: accumulo-examples   Source File: CountIT.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void test() throws Exception {
  Scanner scanner = client.createScanner(tableName, new Authorizations());
  scanner.fetchColumn(new Text("dir"), new Text("counts"));
  assertFalse(scanner.iterator().hasNext());

  ScannerOpts scanOpts = new ScannerOpts();
  BatchWriterOpts bwOpts = new BatchWriterOpts();
  FileCount fc = new FileCount(client, tableName, Authorizations.EMPTY, new ColumnVisibility(),
      scanOpts, bwOpts);
  fc.run();

  ArrayList<Pair<String,String>> expected = new ArrayList<>();
  expected.add(new Pair<>(QueryUtil.getRow("").toString(), "1,0,3,3"));
  expected.add(new Pair<>(QueryUtil.getRow("/local").toString(), "2,1,2,3"));
  expected.add(new Pair<>(QueryUtil.getRow("/local/user1").toString(), "0,2,0,2"));
  expected.add(new Pair<>(QueryUtil.getRow("/local/user2").toString(), "0,0,0,0"));

  int i = 0;
  for (Entry<Key,Value> e : scanner) {
    assertEquals(e.getKey().getRow().toString(), expected.get(i).getFirst());
    assertEquals(e.getValue().toString(), expected.get(i).getSecond());
    i++;
  }
  assertEquals(i, expected.size());
}
 
Example 14
Source Project: vertexium   Source File: AccumuloGraph.java    License: Apache License 2.0 6 votes vote down vote up
private long getRowCountFromTable(String tableName, Text signalColumn, Authorizations authorizations) {
    try {
        LOGGER.debug("BEGIN getRowCountFromTable(%s)", tableName);
        Scanner scanner = createScanner(tableName, null, authorizations);
        try {
            scanner.fetchColumnFamily(signalColumn);

            IteratorSetting countingIterator = new IteratorSetting(
                100,
                CountingIterator.class.getSimpleName(),
                CountingIterator.class
            );
            scanner.addScanIterator(countingIterator);

            GRAPH_LOGGER.logStartIterator(tableName, scanner);

            long count = 0;
            for (Map.Entry<Key, Value> entry : scanner) {
                Long countForKey = LongCombiner.FIXED_LEN_ENCODER.decode(entry.getValue().get());
                LOGGER.debug("getRowCountFromTable(%s): %s: %d", tableName, entry.getKey().getRow(), countForKey);
                count += countForKey;
            }
            LOGGER.debug("getRowCountFromTable(%s): TOTAL: %d", tableName, count);
            return count;
        } finally {
            scanner.close();
        }
    } catch (TableNotFoundException ex) {
        throw new VertexiumException("Could not get count from table: " + tableName, ex);
    }
}
 
Example 15
Source Project: attic-apex-malhar   Source File: AccumuloTestHelper.java    License: Apache License 2.0 6 votes vote down vote up
public static AccumuloTuple getAccumuloTuple(String row, String colFam,
    String colName)
{
  Authorizations auths = new Authorizations();

  Scanner scan = null;
  try {
    scan = con.createScanner("tab1", auths);
  } catch (TableNotFoundException e) {
    logger.error("error in test helper");
    DTThrowable.rethrow(e);
  }

  scan.setRange(new Range(new Text(row)));
  scan.fetchColumn(new Text(colFam), new Text(colName));
  // assuming only one row
  for (Entry<Key, Value> entry : scan) {
    AccumuloTuple tuple = new AccumuloTuple();
    tuple.setRow(entry.getKey().getRow().toString());
    tuple.setColFamily(entry.getKey().getColumnFamily().toString());
    tuple.setColName(entry.getKey().getColumnQualifier().toString());
    tuple.setColValue(entry.getValue().toString());
    return tuple;
  }
  return null;
}
 
Example 16
Source Project: fluo   Source File: SnapshotScanner.java    License: Apache License 2.0 5 votes vote down vote up
private void setUpIterator() {
  Scanner scanner;
  try {
    scanner = env.getAccumuloClient().createScanner(env.getTable(), env.getAuthorizations());
  } catch (TableNotFoundException e) {
    throw new RuntimeException(e);
  }
  scanner.clearColumns();
  scanner.clearScanIterators();
  scanner.setRange(SpanUtil.toRange(snapIterConfig.getSpan()));

  setupScanner(scanner, snapIterConfig.getColumns(), startTs, snapIterConfig.showReadLocks);

  this.iterator = scanner.iterator();
}
 
Example 17
Source Project: AccumuloGraph   Source File: EdgeTableWrapper.java    License: Apache License 2.0 5 votes vote down vote up
public void loadEndpointsAndLabel(AccumuloEdge edge) {
  Scanner s = getScanner();

  try {
    s.setRange(new Range(edge.getId().toString()));
    s.fetchColumnFamily(new Text(Constants.LABEL));
    Iterator<Entry<Key,Value>> iter = s.iterator();
    if (!iter.hasNext()) {
      dump();
      throw new AccumuloGraphException("Unable to find edge row: "+edge);
    }

    Entry<Key, Value> entry = iter.next();

    String cq = entry.getKey().getColumnQualifier().toString();
    String[] ids = cq.split(Constants.ID_DELIM);

    String label = AccumuloByteSerializer.deserialize(entry.getValue().get());

    edge.setVertices(new AccumuloVertex(globals, ids[0]),
        new AccumuloVertex(globals, ids[1]));
    edge.setLabel(label);

  } finally {
    s.close();
  }
}
 
Example 18
@Test
public void testWholeColumnBatched_multipleRows() throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException, IOException {

    persistTestMutations(50, 500);

    Scanner scanner = buildScanner(10);

    assertEquals(50, Iterables.size(scanner));
    for (Map.Entry<Key, Value> entry : scanner) {
        List<Map.Entry<Key, Value>> items = RowEncoderUtil.decodeRow(entry.getKey(), entry.getValue());
        assertEquals(500, items.size());
    }
}
 
Example 19
Source Project: datawave   Source File: QueryScannerHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static Scanner createScanner(Connector connector, String tableName, Collection<Authorizations> authorizations, Query query)
                throws TableNotFoundException {
    Scanner scanner = ScannerHelper.createScanner(connector, tableName, authorizations);
    
    scanner.addScanIterator(getQueryInfoIterator(query, false));
    
    return scanner;
}
 
Example 20
Source Project: AccumuloGraph   Source File: VertexTableWrapper.java    License: Apache License 2.0 5 votes vote down vote up
public CloseableIterable<Vertex> getVerticesInRange(Object fromId, Object toId) {
  Scanner scan = getScanner();
  scan.setRange(new Range(fromId != null ? fromId.toString() : null,
      toId != null ? toId.toString() : null));
  scan.fetchColumnFamily(new Text(Constants.LABEL));

  if (globals.getConfig().getPreloadedProperties() != null) {
    for (String key : globals.getConfig().getPreloadedProperties()) {
      scan.fetchColumnFamily(new Text(key));
    }
  }

  final VertexParser parser = new VertexParser(globals);

  return new ScannerIterable<Vertex>(scan) {
    @Override
    public Vertex next(PeekingIterator<Entry<Key, Value>> iterator) {
      // TODO could also check local cache before creating a new instance?

      String rowId = iterator.peek().getKey().getRow().toString();

      List<Entry<Key, Value>> entries =
          new ArrayList<Entry<Key, Value>>();

      while (iterator.peek() != null && rowId.equals(iterator
          .peek().getKey().getRow().toString())) {
        entries.add(iterator.next());
      }

      AccumuloVertex vertex = parser.parse(rowId, entries);
      globals.getCaches().cache(vertex, Vertex.class);

      return vertex;
    }
  };
}
 
Example 21
Source Project: datawave   Source File: ScannerFactory.java    License: Apache License 2.0 5 votes vote down vote up
public synchronized Scanner newSingleScanner(String tableName, Set<Authorizations> auths, Query query) throws TableNotFoundException {
    if (open) {
        Scanner bs = QueryScannerHelper.createScannerWithoutInfo(cxn, tableName, auths, query);
        log.debug("Created scanner " + System.identityHashCode(bs));
        if (log.isTraceEnabled()) {
            log.trace("Adding instance " + bs.hashCode());
        }
        
        return bs;
    } else {
        throw new IllegalStateException("Factory has been locked. No new scanners can be created.");
    }
}
 
Example 22
Source Project: datawave   Source File: RunningResource.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Initializes the scanner resource
 * 
 * @param auths
 * @param tableName
 * @throws TableNotFoundException
 * 
 */
@Override
protected void init(final String tableName, final Set<Authorizations> auths, Collection<Range> currentRange) throws TableNotFoundException {
    Preconditions.checkNotNull(tableName);
    Preconditions.checkArgument(null != currentRange && !currentRange.isEmpty());
    
    // copy the appropriate variables.
    ranges = Lists.newArrayList(currentRange);
    
    this.tableName = tableName;
    
    this.auths = Sets.newHashSet(auths);
    
    if (log.isTraceEnabled())
        log.trace("Creating scanner resource from " + tableName + " " + auths + " " + currentRange);
    
    internalTimer = new StopWatch();
    internalTimer.start();
    
    // let's pre-compute the hashcode.
    hashCode += new HashCodeBuilder().append(tableName).append(auths).append(ranges).toHashCode();
    
    baseScanner = ScannerHelper.createScanner(getConnector(), tableName, auths);
    
    if (baseScanner != null) {
        ((Scanner) baseScanner).setRange(currentRange.iterator().next());
    }
    
}
 
Example 23
Source Project: fluo   Source File: ReadLockIT.java    License: Apache License 2.0 5 votes vote down vote up
private void dumpRow(String row, Consumer<String> out) throws TableNotFoundException {
  Scanner scanner = aClient.createScanner(getCurTableName(), Authorizations.EMPTY);
  scanner.setRange(Range.exact(row));
  for (Entry<Key, Value> entry : scanner) {
    out.accept(FluoFormatter.toString(entry));
  }
}
 
Example 24
Source Project: fluo   Source File: ScanTask.java    License: Apache License 2.0 5 votes vote down vote up
private ScanCounts scan(Session session, PartitionInfo pi, Range range)
    throws TableNotFoundException {
  try (Scanner scanner =
      env.getAccumuloClient().createScanner(env.getTable(), env.getAuthorizations())) {

    scanner.setRange(range);

    Notification.configureScanner(scanner);

    IteratorSetting iterCfg = new IteratorSetting(30, "nhf", NotificationHashFilter.class);
    NotificationHashFilter.setModulusParams(iterCfg, pi.getMyGroupSize(), pi.getMyIdInGroup());
    scanner.addScanIterator(iterCfg);

    scanner.setExecutionHints(SCAN_EXEC_HINTS);

    ScanCounts counts = new ScanCounts();

    for (Entry<Key, Value> entry : scanner) {
      if (!pi.equals(partitionManager.getPartitionInfo())) {
        throw new PartitionInfoChangedException();
      }

      if (stopped.get()) {
        return counts;
      }

      counts.seen++;

      if (session.addNotification(finder, Notification.from(entry.getKey()))) {
        counts.added++;
      }
    }
    return counts;
  }
}
 
Example 25
Source Project: datawave   Source File: DashboardBean.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create scanner for last 60 minutes of logs.
 *
 * @param c
 *            the {@link Connector} to use when creating scanners
 *
 * @return a {@link Scanner} that will only scan over the last 60 minutes of logs
 *
 * @throws TableNotFoundException
 */
private Scanner createScanner(Connector c) throws TableNotFoundException {
    long start = Instant.now().toEpochMilli() - MS_IN_12_HRS;
    long end = start + (1000 * 60 * 10);// 10 minutes
    Scanner scanner = ScannerHelper.createScanner(c, TABLE_NAME_JMC, getAuths());
    Key startKey = new Key(Long.toString(start));
    Key endKey = new Key(Long.toString(end));
    Range range = new Range(startKey, endKey);
    scanner.setRange(range);
    return scanner;
}
 
Example 26
Source Project: datawave   Source File: QueriesTableAgeOffIteratorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAgeOffIterator() throws Exception {
    InMemoryInstance instance = new InMemoryInstance();
    Connector connector = instance.getConnector("root", new PasswordToken(""));
    
    connector.tableOperations().create(TABLE_NAME);
    IteratorSetting iteratorCfg = new IteratorSetting(19, "ageoff", QueriesTableAgeOffIterator.class);
    connector.tableOperations().attachIterator(TABLE_NAME, iteratorCfg, EnumSet.allOf(IteratorScope.class));
    
    long now = System.currentTimeMillis();
    // Write in a couple of keys with varying timestamps
    BatchWriter writer = connector.createBatchWriter(TABLE_NAME, new BatchWriterConfig().setMaxLatency(30, TimeUnit.MILLISECONDS).setMaxMemory(1024L)
                    .setMaxWriteThreads(1));
    
    Mutation m1 = new Mutation("row1");
    m1.put("colf1", "colq1", now, "");
    writer.addMutation(m1);
    
    Mutation m2 = new Mutation("row2");
    m2.put("colf2", "colq2", (now + 100000), "");
    writer.addMutation(m2);
    
    writer.close();
    
    // Scan the entire table, we should only see keys whose timestamps are greater than or equal to now.
    // Mutation 1 should be expired by now, we should only see Mutation 2;
    boolean sawRow2 = false;
    Scanner scanner = connector.createScanner(TABLE_NAME, new Authorizations());
    for (Entry<Key,Value> entry : scanner) {
        if (entry.getKey().getRow().toString().equals("row1"))
            Assert.fail("We saw row1 when it should be expired.");
        if (entry.getKey().getRow().toString().equals("row2"))
            sawRow2 = true;
    }
    if (!sawRow2)
        Assert.fail("We did not see row2 and we should have");
}
 
Example 27
Source Project: rya   Source File: AccumuloRdfCountToolTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTTL() throws Exception {
    RyaIRI test1 = RdfToRyaConversions.convertIRI(VF.createIRI(litdupsNS, "test1"));
    RyaIRI pred1 = RdfToRyaConversions.convertIRI(VF.createIRI(litdupsNS, "pred1"));
    dao.add(new RyaStatement(test1, pred1, RdfToRyaConversions.convertLiteral(VF.createLiteral(0))));
    dao.add(new RyaStatement(test1, pred1, RdfToRyaConversions.convertLiteral(VF.createLiteral(1))));
    dao.add(new RyaStatement(test1, pred1, RdfToRyaConversions.convertLiteral(VF.createLiteral(2))));
    dao.add(new RyaStatement(test1, pred1, RdfToRyaConversions.convertLiteral(VF.createLiteral(3))));
    dao.add(new RyaStatement(test1, pred1, RdfToRyaConversions.convertLiteral(VF.createLiteral(4))));
    dao.add(new RyaStatement(test1, pred1, RdfToRyaConversions.convertLiteral(VF.createLiteral(5))));
    dao.add(new RyaStatement(test1, pred1, RdfToRyaConversions.convertLiteral(VF.createLiteral(6))));
    dao.add(new RyaStatement(test1, pred1, RdfToRyaConversions.convertLiteral(VF.createLiteral(7))));
    dao.add(new RyaStatement(test1, pred1, RdfToRyaConversions.convertLiteral(VF.createLiteral(8))));
    dao.add(new RyaStatement(test1, pred1, RdfToRyaConversions.convertLiteral(VF.createLiteral(9))));
    dao.add(new RyaStatement(test1, pred1, RdfToRyaConversions.convertLiteral(VF.createLiteral(10))));

    AccumuloRdfCountTool.main(new String[]{
            "-Dac.mock=true",
            "-Dac.instance=" + instance,
            "-Dac.username=" + user,
            "-Dac.pwd=" + pwd,
            "-Dac.ttl=0",
            "-Drdf.tablePrefix=" + tablePrefix,
    });

    Scanner scanner = connector.createScanner(tablePrefix + RdfCloudTripleStoreConstants.TBL_EVAL_SUFFIX, auths);
    scanner.setRange(new Range());
    int count = 0;
    for (Map.Entry<Key, Value> entry : scanner) {
        count++;
    }
    assertEquals(0, count);
}
 
Example 28
Source Project: datawave   Source File: MapReduceStatePersisterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUpdateState() throws Exception {
    // create the initial entry
    testPersistentCreate();
    
    PowerMock.resetAll();
    
    // Get ready to call updateState
    HashMap<String,String> trackingMap = new HashMap<>();
    expect(connectionFactory.getTrackingMap(EasyMock.anyObject())).andReturn(trackingMap);
    expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.ADMIN), EasyMock.eq(trackingMap))).andReturn(connection);
    connectionFactory.returnConnection(connection);
    expect(connectionFactory.getTrackingMap(EasyMock.anyObject())).andReturn(trackingMap);
    expect(connectionFactory.getConnection(EasyMock.eq(AccumuloConnectionFactory.Priority.ADMIN), EasyMock.eq(trackingMap))).andReturn(connection);
    connectionFactory.returnConnection(connection);
    replayAll();
    
    bean.updateState(mapReduceJobId, MapReduceState.FAILED);
    verifyAll();
    
    // Ensure that the new FAILED state made it into the table
    Key failedKey = new Key(id, sid, MapReduceStatePersisterBean.STATE + NULL + mapReduceJobId);
    Value failedValue = new Value(MapReduceState.FAILED.toString().getBytes());
    boolean found = false;
    Scanner s = connection.createScanner(TABLE_NAME, new Authorizations(auths));
    s.setRange(new Range(id));
    s.fetchColumnFamily(new Text(sid));
    
    for (Entry<Key,Value> entry : s) {
        if (entry.getKey().getColumnQualifier().toString().equals(MapReduceStatePersisterBean.STATE + NULL + mapReduceJobId)) {
            if (failedKey.equals(entry.getKey(), PartialKey.ROW_COLFAM_COLQUAL) && failedValue.equals(entry.getValue())) {
                found = true;
            }
        }
    }
    if (!found)
        fail("Updated state not found");
    
}
 
Example 29
Source Project: rya   Source File: AccumuloTemporalIndexer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * An iteration wrapper for a loaded scanner that is returned for partially supported interval queries above.
 *
 * @param scanner  the results to iterate, then close.
 * @param constraints  limit statements returned by next() to those matching the constraints.
 * @return an anonymous object that will iterate the resulting statements from a given scanner.
 * @throws QueryEvaluationException
 */
private static CloseableIteration<Statement, QueryEvaluationException> getConstrainedIteratorWrapper(final Scanner scanner, final StatementConstraints constraints) {
    if (!constraints.hasContext() && !constraints.hasSubject() && !constraints.hasPredicates()) {
        return getIteratorWrapper(scanner);
    }
    return new ConstrainedIteratorWrapper(scanner) {
        @Override
        public boolean allowedBy(final Statement statement) {
            return allowedByConstraints(statement, constraints);
        }
    };
}
 
Example 30
Source Project: geowave   Source File: AccumuloOperations.java    License: Apache License 2.0 5 votes vote down vote up
public CloseableIterator<GeoWaveRow> getDataIndexResults(
    final byte[] startRow,
    final byte[] endRow,
    final short adapterId,
    final String... additionalAuthorizations) {
  final byte[] family = StringUtils.stringToBinary(ByteArrayUtils.shortToString(adapterId));

  // to have backwards compatibility before 1.8.0 we can assume BaseScanner is autocloseable
  final Scanner scanner;
  try {
    scanner = createScanner(DataIndexUtils.DATA_ID_INDEX.getName(), additionalAuthorizations);

    scanner.setRange(
        AccumuloUtils.byteArrayRangeToAccumuloRange(new ByteArrayRange(startRow, endRow)));
    scanner.fetchColumnFamily(new Text(family));
    return new CloseableIteratorWrapper(new Closeable() {
      @Override
      public void close() throws IOException {
        scanner.close();
      }
    },
        Streams.stream(scanner.iterator()).map(
            entry -> DataIndexUtils.deserializeDataIndexRow(
                entry.getKey().getRow().getBytes(),
                adapterId,
                entry.getValue().get(),
                false)).iterator());
  } catch (final TableNotFoundException e) {
    LOGGER.error("unable to find data index table", e);
  }
  return new CloseableIterator.Empty<>();
}