org.apache.accumulo.core.client.BatchWriterConfig Java Examples

private long ingestRandom(long timestampInitial, int decrementMillis) {
    long ts = timestampInitial;
    Collection<Mutation> mutations = new ArrayList<>(TestMetricWriter.DEFAULT_BATCHES);
    int idx = 0;
    while (idx++ < TestMetricWriter.DEFAULT_BATCHES) {
        ts = ts - decrementMillis;
        Metric m = metrics.randomMetric(ts);
        Mutation mutation = MetricAdapter.toMutation(m);
        mutations.add(mutation);
    }

    try {
        BatchWriterConfig cfg = new BatchWriterConfig().setMaxWriteThreads(4);
        try (BatchWriter writer = connector.createBatchWriter(tableName, cfg)) {
            writer.addMutations(mutations);
        }
        connector.tableOperations().flush(tableName, null, null, true);
    } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
        throw new RuntimeException(e);
    }

    return ts;
}
 

private static void writeKeyValues(Connector connector, Multimap<BulkIngestKey,Value> keyValues) throws Exception {
    final TableOperations tops = connector.tableOperations();
    final Set<BulkIngestKey> biKeys = keyValues.keySet();
    for (final BulkIngestKey biKey : biKeys) {
        final String tableName = biKey.getTableName().toString();
        if (!tops.exists(tableName))
            tops.create(tableName);
        
        final BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig());
        for (final Value val : keyValues.get(biKey)) {
            final Mutation mutation = new Mutation(biKey.getKey().getRow());
            mutation.put(biKey.getKey().getColumnFamily(), biKey.getKey().getColumnQualifier(), biKey.getKey().getColumnVisibilityParsed(), biKey.getKey()
                            .getTimestamp(), val);
            writer.addMutation(mutation);
        }
        writer.close();
    }
}
 

private static void writeKeyValues(Connector connector, Multimap<BulkIngestKey,Value> keyValues) throws Exception {
    final TableOperations tops = connector.tableOperations();
    final Set<BulkIngestKey> biKeys = keyValues.keySet();
    for (final BulkIngestKey biKey : biKeys) {
        final String tableName = biKey.getTableName().toString();
        if (!tops.exists(tableName))
            tops.create(tableName);
        
        final BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig());
        for (final Value val : keyValues.get(biKey)) {
            final Mutation mutation = new Mutation(biKey.getKey().getRow());
            mutation.put(biKey.getKey().getColumnFamily(), biKey.getKey().getColumnQualifier(), biKey.getKey().getColumnVisibilityParsed(), val);
            writer.addMutation(mutation);
        }
        writer.close();
    }
}
 

/**
 * @throws java.lang.Exception
 */
@Before
public void setUp() throws Exception {
    conf = new Configuration();
    conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, "triplestore_");
    conf.setBoolean(ConfigUtils.USE_MOCK_INSTANCE, true);
    // The temporal predicates are from http://linkedevents.org/ontology
    // and http://motools.sourceforge.net/event/event.html
    conf.setStrings(ConfigUtils.TEMPORAL_PREDICATES_LIST, ""
            + URI_PROPERTY_AT_TIME + ","
            + URI_PROPERTY_CIRCA + ","
            + URI_PROPERTY_EVENT_TIME);

    tIndexer = new AccumuloTemporalIndexer();
    tIndexer.setConf(conf);
    Connector connector = ConfigUtils.getConnector(conf);
    MultiTableBatchWriter mt_bw = connector.createMultiTableBatchWriter(new BatchWriterConfig());
    tIndexer.setConnector(connector);
    tIndexer.setMultiTableBatchWriter(mt_bw);
    tIndexer.init();
}
 

private static void writeKeyValues(Connector connector, Multimap<BulkIngestKey,Value> keyValues) throws Exception {
    final TableOperations tops = connector.tableOperations();
    final Set<BulkIngestKey> biKeys = keyValues.keySet();
    for (final BulkIngestKey biKey : biKeys) {
        final String tableName = biKey.getTableName().toString();
        if (!tops.exists(tableName))
            tops.create(tableName);
        
        final BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig());
        for (final Value val : keyValues.get(biKey)) {
            final Mutation mutation = new Mutation(biKey.getKey().getRow());
            mutation.put(biKey.getKey().getColumnFamily(), biKey.getKey().getColumnQualifier(), biKey.getKey().getColumnVisibilityParsed(), biKey.getKey()
                            .getTimestamp(), val);
            writer.addMutation(mutation);
        }
        writer.close();
    }
}
 

private static void writeKeyValues(Connector connector, Multimap<BulkIngestKey,Value> keyValues) throws Exception {
    final TableOperations tops = connector.tableOperations();
    final Set<BulkIngestKey> biKeys = keyValues.keySet();
    for (final BulkIngestKey biKey : biKeys) {
        final String tableName = biKey.getTableName().toString();
        if (!tops.exists(tableName))
            tops.create(tableName);
        
        final BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig());
        for (final Value val : keyValues.get(biKey)) {
            final Mutation mutation = new Mutation(biKey.getKey().getRow());
            mutation.put(biKey.getKey().getColumnFamily(), biKey.getKey().getColumnQualifier(), biKey.getKey().getColumnVisibilityParsed(), biKey.getKey()
                            .getTimestamp(), val);
            writer.addMutation(mutation);
        }
        writer.close();
    }
}
 

public Job getExpectedStoreJob(String inst, String zookeepers, String user,
        String password, String table, long maxWriteBufferSize,
        int writeThreads, int maxWriteLatencyMS) throws IOException {

    Job expected = new Job(new Configuration());

    try {
        AccumuloOutputFormat.setConnectorInfo(expected, user,
                new PasswordToken(password));
    } catch (AccumuloSecurityException e) {
        Assert.fail(e.getMessage());
    }

    AccumuloOutputFormat.setZooKeeperInstance(expected, inst, zookeepers);
    AccumuloOutputFormat.setCreateTables(expected, true);

    BatchWriterConfig bwConfig = new BatchWriterConfig();
    bwConfig.setMaxLatency(maxWriteLatencyMS, TimeUnit.MILLISECONDS);
    bwConfig.setMaxMemory(maxWriteBufferSize);
    bwConfig.setMaxWriteThreads(writeThreads);

    AccumuloOutputFormat.setBatchWriterOptions(expected, bwConfig);

    return expected;
}
 

private String seedMetadataTable(Connector connector, long time, int keyVersionNum) throws TableNotFoundException, MutationsRejectedException {
    Value emptyVal = new Value();
    SimpleDateFormat dateFormat = new SimpleDateFormat(DateNormalizer.ISO_8601_FORMAT_STRING);
    String dateString = dateFormat.format(new Date(time));
    try (BatchWriter recordWriter = connector.createBatchWriter(metadataTableName, new BatchWriterConfig())) {
        String normalizedVersionNum = NumericalEncoder.encode(Integer.toString(keyVersionNum));
        String rowID = "edge_key";
        String columnFamily = "version";
        String columnQualifier = normalizedVersionNum + "/" + dateString;
        
        Mutation m = new Mutation(rowID);
        
        m.put(new Text(columnFamily), new Text(columnQualifier), emptyVal);
        
        recordWriter.addMutation(m);
    }
    return dateString;
}
 

public static void clearTable(Connector connector, String table) throws AccumuloException, TableNotFoundException, AccumuloSecurityException {

        Authorizations userAuths = connector.securityOperations().getUserAuthorizations(connector.whoami());

        BatchDeleter batchDelete = connector.createBatchDeleter(table, userAuths, 1, new BatchWriterConfig());
        batchDelete.setRanges(Collections.singleton(new Range()));
        batchDelete.delete();
        batchDelete.close();

        batchDelete = connector.createBatchDeleter(table, userAuths, 1, new BatchWriterConfig());
        batchDelete.setRanges(Collections.singleton(new Range()));
        batchDelete.delete();
        batchDelete.close();

        connector.tableOperations().compact(table, new Text("\u0000"), new Text("\uffff"), true, true);
    }
 

@Before
public void init() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException {

    mock = new MockInstance("accumulo");
    PasswordToken pToken = new PasswordToken("pass".getBytes());
    conn = mock.getConnector("user", pToken);

    config = new BatchWriterConfig();
    config.setMaxMemory(1000);
    config.setMaxLatency(1000, TimeUnit.SECONDS);
    config.setMaxWriteThreads(10);

    if (conn.tableOperations().exists("rya_prospects")) {
        conn.tableOperations().delete("rya_prospects");
    }
    if (conn.tableOperations().exists("rya_selectivity")) {
        conn.tableOperations().delete("rya_selectivity");
    }

    arc = new AccumuloRdfConfiguration();
    res = new ProspectorServiceEvalStatsDAO(conn, arc);
    arc.setTableLayoutStrategy(new TablePrefixLayoutStrategy());
    arc.setMaxRangesForScanner(300);

}
 

private static TemporalIndexer getTemporalIndexer(final Configuration conf) throws IOException {
    if (!conf.getBoolean(ENABLE_TEMPORAL, true)) {
        return null;
    }
    final AccumuloTemporalIndexer temporal = new AccumuloTemporalIndexer();
    temporal.setConf(conf);
    Connector connector;
    try {
        connector = ConfigUtils.getConnector(conf);
    } catch (AccumuloException | AccumuloSecurityException e) {
        throw new IOException("Error when attempting to create a connection for writing the temporal index.", e);
    }
    final MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(new BatchWriterConfig());
    temporal.setConnector(connector);
    temporal.setMultiTableBatchWriter(mtbw);
    temporal.init();
    return temporal;
}
 

private static FreeTextIndexer getFreeTextIndexer(final Configuration conf) throws IOException {
    if (!conf.getBoolean(ENABLE_FREETEXT, true)) {
        return null;
    }
    final AccumuloFreeTextIndexer freeText = new AccumuloFreeTextIndexer();
    freeText.setConf(conf);
    Connector connector;
    try {
        connector = ConfigUtils.getConnector(conf);
    } catch (AccumuloException | AccumuloSecurityException e) {
        throw new IOException("Error when attempting to create a connection for writing the freeText index.", e);
    }
    final MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(new BatchWriterConfig());
    freeText.setConnector(connector);
    freeText.setMultiTableBatchWriter(mtbw);
    freeText.init();

    return freeText;
}
 

@Override
 public void setStoreLocation(final String location, final Job job) throws IOException {
     conf = job.getConfiguration();
     setLocationFromUri(location, job);

     if (!conf.getBoolean(AccumuloOutputFormat.class.getSimpleName() + ".configured", false)) {
         try {
	AccumuloOutputFormat.setConnectorInfo(job, user, new PasswordToken(userP.getBytes(StandardCharsets.UTF_8)));
} catch (final AccumuloSecurityException e) {
	throw new RuntimeException(e);
}
         AccumuloOutputFormat.setDefaultTableName(job, table);
         AccumuloOutputFormat.setZooKeeperInstance(job, inst, zookeepers);
         final BatchWriterConfig config = new BatchWriterConfig();
         config.setMaxLatency(10, TimeUnit.SECONDS);
         config.setMaxMemory(10 * 1000 * 1000);
         config.setMaxWriteThreads(10);
         AccumuloOutputFormat.setBatchWriterOptions(job, config);
     }
 }
 

@Before
public void init() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException {

    mock = new MockInstance("accumulo");
    PasswordToken pToken = new PasswordToken("pass".getBytes());
    conn = mock.getConnector("user", pToken);

    config = new BatchWriterConfig();
    config.setMaxMemory(1000);
    config.setMaxLatency(1000, TimeUnit.SECONDS);
    config.setMaxWriteThreads(10);

    if (conn.tableOperations().exists("rya_prospects")) {
        conn.tableOperations().delete("rya_prospects");
    }
    if (conn.tableOperations().exists("rya_selectivity")) {
        conn.tableOperations().delete("rya_selectivity");
    }

    arc = new AccumuloRdfConfiguration();
    arc.setTableLayoutStrategy(new TablePrefixLayoutStrategy());
    arc.setMaxRangesForScanner(300);

}
 

protected Multimap<String,Key> removeMetadataEntries(Set<String> fields, Text cf) throws AccumuloSecurityException, AccumuloException,
                TableNotFoundException {
    Multimap<String,Key> metadataEntries = HashMultimap.create();
    MultiTableBatchWriter multiTableWriter = connector.createMultiTableBatchWriter(new BatchWriterConfig());
    BatchWriter writer = multiTableWriter.getBatchWriter(QueryTestTableHelper.METADATA_TABLE_NAME);
    for (String field : fields) {
        Mutation mutation = new Mutation(new Text(field));
        Scanner scanner = connector.createScanner(QueryTestTableHelper.METADATA_TABLE_NAME, new Authorizations());
        scanner.fetchColumnFamily(cf);
        scanner.setRange(new Range(new Text(field)));
        boolean foundEntries = false;
        for (Map.Entry<Key,Value> entry : scanner) {
            foundEntries = true;
            metadataEntries.put(field, entry.getKey());
            mutation.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getKey().getColumnVisibilityParsed());
        }
        scanner.close();
        if (foundEntries) {
            writer.addMutation(mutation);
        }
    }
    writer.close();
    connector.tableOperations().compact(QueryTestTableHelper.METADATA_TABLE_NAME, new Text("\0"), new Text("~"), true, true);
    return metadataEntries;
}
 

@Test
public void test() throws Exception {
  client.tableOperations().create(tableName);
  BatchWriter bw = client.createBatchWriter(tableName, new BatchWriterConfig());

  for (Entry<Key,Value> e : data) {
    Key k = e.getKey();
    Mutation m = new Mutation(k.getRow());
    m.put(k.getColumnFamily(), k.getColumnQualifier(),
        new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp(), e.getValue());
    bw.addMutation(m);
  }
  bw.close();

  assertEquals(0, CIFTester.main(tableName, CIFTester.TestMapper.class.getName()));
  assertEquals(1, assertionErrors.get(tableName).size());
}
 

@Test
public void testErrorOnNextWithoutClose() throws Exception {
  client.tableOperations().create(tableName);
  BatchWriter bw = client.createBatchWriter(tableName, new BatchWriterConfig());

  for (Entry<Key,Value> e : data) {
    Key k = e.getKey();
    Mutation m = new Mutation(k.getRow());
    m.put(k.getColumnFamily(), k.getColumnQualifier(),
        new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp(), e.getValue());
    bw.addMutation(m);
  }
  bw.close();

  assertEquals(1, CIFTester.main(tableName, CIFTester.TestNoClose.class.getName()));
  assertEquals(1, assertionErrors.get(tableName).size());
  // this should actually exist, in addition to the dummy entry
  assertEquals(2, assertionErrors.get(tableName + "_map_ioexception").size());
}
 

@Test
public void testInfoWithoutChunks() throws Exception {
  client.tableOperations().create(tableName);
  BatchWriter bw = client.createBatchWriter(tableName, new BatchWriterConfig());
  for (Entry<Key,Value> e : baddata) {
    Key k = e.getKey();
    Mutation m = new Mutation(k.getRow());
    m.put(k.getColumnFamily(), k.getColumnQualifier(),
        new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp(), e.getValue());
    bw.addMutation(m);
  }
  bw.close();

  assertEquals(0, CIFTester.main(tableName, CIFTester.TestBadData.class.getName()));
  assertEquals(1, assertionErrors.get(tableName).size());
}
 

@Before
public void setupInstance() throws Exception {
  tableName = getUniqueNames(1)[0];
  client = Accumulo.newClient().from(getClientProperties()).build();
  client.tableOperations().create(tableName);
  BatchWriter bw = client.createBatchWriter(tableName, new BatchWriterConfig());
  ColumnVisibility cv = new ColumnVisibility();
  // / has 1 dir
  // /local has 2 dirs 1 file
  // /local/user1 has 2 files
  bw.addMutation(Ingest.buildMutation(cv, "/local", true, false, true, 272, 12345, null));
  bw.addMutation(Ingest.buildMutation(cv, "/local/user1", true, false, true, 272, 12345, null));
  bw.addMutation(Ingest.buildMutation(cv, "/local/user2", true, false, true, 272, 12345, null));
  bw.addMutation(Ingest.buildMutation(cv, "/local/file", false, false, false, 1024, 12345, null));
  bw.addMutation(Ingest.buildMutation(cv, "/local/file", false, false, false, 1024, 23456, null));
  bw.addMutation(
      Ingest.buildMutation(cv, "/local/user1/file1", false, false, false, 2024, 12345, null));
  bw.addMutation(
      Ingest.buildMutation(cv, "/local/user1/file2", false, false, false, 1028, 23456, null));
  bw.close();
}
 

@Override
public void clear() {
    log.trace("Clearing Accumulo cache for table {}.", tableName);
    try {
        BatchWriterConfig bwCfg = new BatchWriterConfig();
        BatchDeleter deleter = connector.createBatchDeleter(tableName, authorizations, 10, bwCfg);
        try {
            deleter.setRanges(Collections.singletonList(new Range()));
            deleter.delete();
        } finally {
            deleter.close();
        }
    } catch (MutationsRejectedException | TableNotFoundException e) {
        throw new PersistenceException("Unable to clear Accumulo cache for " + tableName, e);
    }
}
 

@BeforeClass
public static void setUp() throws Exception {
    TimeZone.setDefault(TimeZone.getTimeZone("GMT"));
    System.setProperty("file.encoding", "UTF8");
    
    // Need to set SDFs after setting the timezone to GMT, else dates will be EST and converted to GMT
    simpleFormat = new SimpleDateFormat("yyyyMMdd");
    
    InMemoryInstance i = new InMemoryInstance(BaseEdgeQueryTest.class.toString());
    connector = i.getConnector("root", new PasswordToken(""));
    
    // Create the CB tables
    connector.tableOperations().create(EDGE_TABLE_NAME);
    connector.tableOperations().create(MODEL_TABLE_NAME);
    
    // Create the map of batchwriters to cb tables
    recordWriter = new MockAccumuloRecordWriter();
    BatchWriterConfig bwCfg = new BatchWriterConfig().setMaxLatency(1, TimeUnit.SECONDS).setMaxMemory(1000L).setMaxWriteThreads(1);
    recordWriter.addWriter(new Text(EDGE_TABLE_NAME), connector.createBatchWriter(EDGE_TABLE_NAME, bwCfg));
    
    addEdges();
}
 

private static void writeKeyValues(Connector connector, Multimap<BulkIngestKey,Value> keyValues) throws Exception {
    final TableOperations tops = connector.tableOperations();
    final Set<BulkIngestKey> biKeys = keyValues.keySet();
    tops.create(TableName.DATE_INDEX);
    for (final BulkIngestKey biKey : biKeys) {
        final String tableName = biKey.getTableName().toString();
        if (!tops.exists(tableName))
            tops.create(tableName);
        
        final BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig());
        for (final Value val : keyValues.get(biKey)) {
            final Mutation mutation = new Mutation(biKey.getKey().getRow());
            mutation.put(biKey.getKey().getColumnFamily(), biKey.getKey().getColumnQualifier(), biKey.getKey().getColumnVisibilityParsed(), biKey.getKey()
                            .getTimestamp(), val);
            writer.addMutation(mutation);
        }
        writer.close();
    }
}
 

private static void writeKeyValues(Connector connector, Multimap<BulkIngestKey,Value> keyValues) throws Exception {
    final TableOperations tops = connector.tableOperations();
    final Set<BulkIngestKey> biKeys = keyValues.keySet();
    for (final BulkIngestKey biKey : biKeys) {
        final String tableName = biKey.getTableName().toString();
        if (!tops.exists(tableName))
            tops.create(tableName);
        
        final BatchWriter writer = connector.createBatchWriter(tableName, new BatchWriterConfig());
        for (final Value val : keyValues.get(biKey)) {
            final Mutation mutation = new Mutation(biKey.getKey().getRow());
            mutation.put(biKey.getKey().getColumnFamily(), biKey.getKey().getColumnQualifier(), biKey.getKey().getColumnVisibilityParsed(), biKey.getKey()
                            .getTimestamp(), val);
            writer.addMutation(mutation);
        }
        writer.close();
    }
}
 

@Test
@SuppressWarnings("unchecked")
public void testBatchWriterConfigIsSetToValuesWithParameters() {
    int numThreads = 2;
    long timeout = 3;
    long maxMemory = 5;
    long maxLatency = 7;

    Map configMap = Maps.newHashMap();
    MapUtils.putAll(configMap, new String[]{
        AccumuloGraphConfiguration.BATCHWRITER_MAX_LATENCY, "" + maxLatency,
        AccumuloGraphConfiguration.BATCHWRITER_MAX_MEMORY, "" + maxMemory,
        AccumuloGraphConfiguration.BATCHWRITER_MAX_WRITE_THREADS, "" + numThreads,
        AccumuloGraphConfiguration.BATCHWRITER_TIMEOUT, "" + timeout});
    AccumuloGraphConfiguration accumuloGraphConfiguration = new AccumuloGraphConfiguration(configMap);
    BatchWriterConfig batchWriterConfig = accumuloGraphConfiguration.createBatchWriterConfig();

    assertThat(batchWriterConfig.getMaxLatency(TimeUnit.MILLISECONDS), is(maxLatency));
    assertThat(batchWriterConfig.getTimeout(TimeUnit.MILLISECONDS), is(timeout));
    assertThat(batchWriterConfig.getMaxMemory(), is(maxMemory));
    assertThat(batchWriterConfig.getMaxWriteThreads(), is(numThreads));
}
 

@Before
public void init() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException {

  mock = new MockInstance("accumulo");
  PasswordToken pToken = new PasswordToken("pass".getBytes());
  conn = mock.getConnector("user", pToken);

  config = new BatchWriterConfig();
  config.setMaxMemory(1000);
  config.setMaxLatency(1000, TimeUnit.SECONDS);
  config.setMaxWriteThreads(10);

  if (conn.tableOperations().exists("rya_prospects")) {
    conn.tableOperations().delete("rya_prospects");
  }
  if (conn.tableOperations().exists("rya_selectivity")) {
    conn.tableOperations().delete("rya_selectivity");
  }

  arc = new AccumuloRdfConfiguration();
  arc.setTableLayoutStrategy(new TablePrefixLayoutStrategy());
  arc.setMaxRangesForScanner(300);
  res = new ProspectorServiceEvalStatsDAO(conn, arc);

}
 

@OnScheduled
public void onScheduled(final ProcessContext context) {
    accumuloConnectorService = context.getProperty(ACCUMULO_CONNECTOR_SERVICE).asControllerService(BaseAccumuloService.class);
    final Double maxBytes = context.getProperty(MEMORY_SIZE).asDataSize(DataUnit.B);
    this.client = accumuloConnectorService.getClient();
    BatchWriterConfig writerConfig = new BatchWriterConfig();
    writerConfig.setMaxWriteThreads(context.getProperty(THREADS).asInteger());
    writerConfig.setMaxMemory(maxBytes.longValue());
    tableWriter = client.createMultiTableBatchWriter(writerConfig);
    flushOnEveryFlow = context.getProperty(FLUSH_ON_FLOWFILE).asBoolean();
    if (!flushOnEveryFlow){
        writerConfig.setMaxLatency(60, TimeUnit.SECONDS);
    }

    if (context.getProperty(CREATE_TABLE).asBoolean() && !context.getProperty(TABLE_NAME).isExpressionLanguagePresent()) {
        final Map<String, String> flowAttributes = new HashMap<>();
        final String table = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowAttributes).getValue();
          final TableOperations tableOps = this.client.tableOperations();
          if (!tableOps.exists(table)) {
              getLogger().info("Creating " + table + " table.");
              try {
                  tableOps.create(table);
              } catch (TableExistsException te) {
                  // can safely ignore
              } catch (AccumuloSecurityException | AccumuloException e) {
                  getLogger().info("Accumulo or Security error creating. Continuing... " + table + ". ", e);
              }
          }
    }
}
 

public static BatchDeleter createBatchDeleter(Connector connector, String tableName,
        Collection<Authorizations> authorizations, int numQueryThreads, long maxMemory, long maxLatency,
        int maxWriteThreads) throws TableNotFoundException {
    if (authorizations == null || authorizations.isEmpty())
        throw new IllegalArgumentException("Authorizations must not be empty.");

    Iterator<Authorizations> iter = AuthorizationsMinimizer.minimize(authorizations).iterator();
    BatchWriterConfig bwCfg = new BatchWriterConfig().setMaxLatency(maxLatency, TimeUnit.MILLISECONDS)
            .setMaxMemory(maxMemory).setMaxWriteThreads(maxWriteThreads);
    BatchDeleter batchDeleter = connector.createBatchDeleter(tableName, iter.next(), numQueryThreads, bwCfg);
    addVisibilityFilters(iter, batchDeleter);
    return batchDeleter;
}
 

@Test
public void testTimestampSkippingIterPerformance() throws Exception {

  aClient.tableOperations().create("ttsi",
      new NewTableConfiguration().withoutDefaultIterators()
          .setProperties(Collections.singletonMap(AccumuloProps.TABLE_DELETE_BEHAVIOR,
              AccumuloProps.TABLE_DELETE_BEHAVIOR_VALUE)));

  BatchWriter bw = aClient.createBatchWriter("ttsi", new BatchWriterConfig());
  Mutation m = new Mutation("r1");
  for (int i = 0; i < 100000; i++) {
    m.put("f1", "q1", i, "v" + i);
  }

  bw.addMutation(m);
  bw.close();

  long t2 = System.currentTimeMillis();

  Scanner scanner = aClient.createScanner("ttsi", Authorizations.EMPTY);
  scanner.addScanIterator(new IteratorSetting(10, Skip100StampsIterator.class));

  Assert.assertEquals("999", Iterables.getOnlyElement(scanner).getValue().toString());
  long t3 = System.currentTimeMillis();

  if (t3 - t2 > 3000) {
    log.error("Timestamp skipping iterator took longer than expected " + (t3 - t2));
  }

  aClient.tableOperations().flush("ttsi", null, null, true);

  long t4 = System.currentTimeMillis();
  Assert.assertEquals("999", Iterables.getOnlyElement(scanner).getValue().toString());
  long t5 = System.currentTimeMillis();

  if (t5 - t4 > 3000) {
    log.error("Timestamp skipping iterator took longer than expected " + (t5 - t4));
  }
}
 

public SharedResources(Environment env) throws TableNotFoundException {
  this.env = env;
  curator = CuratorUtil.newAppCurator(env.getConfiguration());
  curator.start();

  int numTservers = env.getAccumuloClient().instanceOperations().getTabletServers().size();
  int numBWThreads = FluoConfigurationImpl.getNumBWThreads(env.getConfiguration(), numTservers);
  bw = env.getAccumuloClient().createBatchWriter(env.getTable(),
      new BatchWriterConfig().setMaxWriteThreads(numBWThreads));
  sbw = new SharedBatchWriter(bw);

  int numCWThreads = FluoConfigurationImpl.getNumCWThreads(env.getConfiguration(), numTservers);
  cw = env.getAccumuloClient().createConditionalWriter(env.getTable(),
      new ConditionalWriterConfig().setAuthorizations(env.getAuthorizations())
          .setMaxWriteThreads(numCWThreads));
  bulkCw = env.getAccumuloClient().createConditionalWriter(env.getTable(),
      new ConditionalWriterConfig().setAuthorizations(env.getAuthorizations())
          .setMaxWriteThreads(numCWThreads));

  txInfoCache = new TxInfoCache(env);
  visCache = new VisibilityCache(env.getConfiguration());
  metricRegistry = new MetricRegistry();

  int commitThreads = env.getConfiguration().getInt(FluoConfigurationImpl.ASYNC_COMMIT_THREADS,
      FluoConfigurationImpl.ASYNC_COMMIT_THREADS_DEFAULT);
  asyncCommitExecutor = FluoExecutors.newFixedThreadPool(commitThreads, "async-commits");

  commitThreads = env.getConfiguration().getInt(FluoConfigurationImpl.SYNC_COMMIT_THREADS,
      FluoConfigurationImpl.SYNC_COMMIT_THREADS_DEFAULT);
  syncCommitExecutor = FluoExecutors.newFixedThreadPool(commitThreads, "sync-commits");

  acw = new AsyncConditionalWriter(env, cw);
  bulkAcw = new AsyncConditionalWriter(env, bulkCw);
}
 

public BatchWriterConfig getBatchWriterConfig() {
  BatchWriterConfig config = new BatchWriterConfig();
  config.setMaxWriteThreads(this.batchThreads);
  config.setMaxLatency(this.batchLatency, TimeUnit.MILLISECONDS);
  config.setMaxMemory(this.batchMemory);
  config.setTimeout(this.batchTimeout, TimeUnit.MILLISECONDS);
  return config;
}