Java Code Examples for org.apache.accumulo.core.client.Connector#createMultiTableBatchWriter()

private static FreeTextIndexer getFreeTextIndexer(final Configuration conf) throws IOException {
    if (!conf.getBoolean(ENABLE_FREETEXT, true)) {
        return null;
    }
    final AccumuloFreeTextIndexer freeText = new AccumuloFreeTextIndexer();
    freeText.setConf(conf);
    Connector connector;
    try {
        connector = ConfigUtils.getConnector(conf);
    } catch (AccumuloException | AccumuloSecurityException e) {
        throw new IOException("Error when attempting to create a connection for writing the freeText index.", e);
    }
    final MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(new BatchWriterConfig());
    freeText.setConnector(connector);
    freeText.setMultiTableBatchWriter(mtbw);
    freeText.init();

    return freeText;
}
 

private static TemporalIndexer getTemporalIndexer(final Configuration conf) throws IOException {
    if (!conf.getBoolean(ENABLE_TEMPORAL, true)) {
        return null;
    }
    final AccumuloTemporalIndexer temporal = new AccumuloTemporalIndexer();
    temporal.setConf(conf);
    Connector connector;
    try {
        connector = ConfigUtils.getConnector(conf);
    } catch (AccumuloException | AccumuloSecurityException e) {
        throw new IOException("Error when attempting to create a connection for writing the temporal index.", e);
    }
    final MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(new BatchWriterConfig());
    temporal.setConnector(connector);
    temporal.setMultiTableBatchWriter(mtbw);
    temporal.init();
    return temporal;
}
 

/**
 * @throws java.lang.Exception
 */
@Before
public void setUp() throws Exception {
    conf = new Configuration();
    conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, "triplestore_");
    conf.setBoolean(ConfigUtils.USE_MOCK_INSTANCE, true);
    // The temporal predicates are from http://linkedevents.org/ontology
    // and http://motools.sourceforge.net/event/event.html
    conf.setStrings(ConfigUtils.TEMPORAL_PREDICATES_LIST, ""
            + URI_PROPERTY_AT_TIME + ","
            + URI_PROPERTY_CIRCA + ","
            + URI_PROPERTY_EVENT_TIME);

    tIndexer = new AccumuloTemporalIndexer();
    tIndexer.setConf(conf);
    Connector connector = ConfigUtils.getConnector(conf);
    MultiTableBatchWriter mt_bw = connector.createMultiTableBatchWriter(new BatchWriterConfig());
    tIndexer.setConnector(connector);
    tIndexer.setMultiTableBatchWriter(mt_bw);
    tIndexer.init();
}
 

public static MultiTableBatchWriter createMultitableBatchWriter(final Configuration conf)
        throws AccumuloException, AccumuloSecurityException {
    final Long DEFAULT_MAX_MEMORY = getWriterMaxMemory(conf);
    final Long DEFAULT_MAX_LATENCY = getWriterMaxLatency(conf);
    final Integer DEFAULT_MAX_WRITE_THREADS = getWriterMaxWriteThreads(conf);
    final Connector connector = ConfigUtils.getConnector(conf);
    return connector.createMultiTableBatchWriter(DEFAULT_MAX_MEMORY, DEFAULT_MAX_LATENCY, DEFAULT_MAX_WRITE_THREADS);
}
 

/**
 * Constructor.
 * @param conf Configuration containing any relevant options.
 * @throws  IOException if the core Rya indexer or entity indexer can't
 *          be initialized
 */
public RyaRecordWriter(final Configuration conf) throws IOException {
    // set the visibility
    final String visibility = conf.get(CV_PROPERTY);
    if (visibility != null) {
        cv = visibility.getBytes(StandardCharsets.UTF_8);
    }
    // set the default context
    final String context = conf.get(CONTEXT_PROPERTY, "");
    if (context != null && !context.isEmpty()) {
        defaultContext = new RyaIRI(context);
    }

    // set up the buffer
    bufferSizeLimit = conf.getLong(MAX_MUTATION_BUFFER_SIZE, ONE_MEGABYTE);
    final int bufferCapacity = (int) (bufferSizeLimit / AVE_STATEMENT_SIZE);
    buffer = new ArrayList<RyaStatement>(bufferCapacity);

    // set up the indexers
    freeTextIndexer = getFreeTextIndexer(conf);
    temporalIndexer = getTemporalIndexer(conf);
    entityIndexer = getEntityIndexer(conf);
    ryaIndexer = getRyaIndexer(conf);

    // The entity index needs a batch writer -- typically it uses the DAO's, but decoupling
    // them lets it be used with or without the core tables, like the other indexers.
    if (entityIndexer != null) {
        Connector conn;
        try {
            conn = ConfigUtils.getConnector(conf);
        } catch (AccumuloException | AccumuloSecurityException e) {
            throw new IOException("Error connecting to Accumulo for entity index output", e);
        }
        final BatchWriterConfig batchWriterConfig = new BatchWriterConfig();
        batchWriterConfig.setMaxMemory(RdfCloudTripleStoreConstants.MAX_MEMORY);
        batchWriterConfig.setTimeout(RdfCloudTripleStoreConstants.MAX_TIME, TimeUnit.MILLISECONDS);
        batchWriterConfig.setMaxWriteThreads(RdfCloudTripleStoreConstants.NUM_THREADS);
        writer = conn.createMultiTableBatchWriter(batchWriterConfig);
        entityIndexer.setMultiTableBatchWriter(writer);
    }

    // update fields used for metrics
    startTime = System.currentTimeMillis();
    lastCommitFinishTime = startTime;

    // set up the triple context
    tripleContext = RyaTripleContext.getInstance(new AccumuloRdfConfiguration(conf));
}
 

@Test
public void testTemporalIndexing() throws Exception {
    final TemporalInstant[] instants = {
            new TemporalInstantRfc3339(2015, 12, 30, 12, 00, 01),
            new TemporalInstantRfc3339(2015, 12, 30, 12, 00, 02),
            new TemporalInstantRfc3339(2015, 12, 30, 12, 00, 03),
            new TemporalInstantRfc3339(2015, 12, 30, 12, 00, 03)
    };
    final Statement[] statements = new Statement[instants.length];
    RyaOutputFormat.setCoreTablesEnabled(job, false);
    RyaOutputFormat.setFreeTextEnabled(job, false);
    RyaOutputFormat.setTemporalEnabled(job, true);
    RyaOutputFormat.setEntityEnabled(job, false);
    final ValueFactory vf = SimpleValueFactory.getInstance();
    for (int i = 0; i < instants.length; i++) {
        final RyaType time = RdfToRyaConversions.convertLiteral(vf.createLiteral(instants[i].toString()));
        final RyaStatement input = RyaStatement.builder()
                .setSubject(new RyaIRI(GRAPH + ":s"))
                .setPredicate(new RyaIRI(GRAPH + ":p"))
                .setObject(time)
                .build();
        write(input);
        statements[i] = RyaToRdfConversions.convertStatement(input);
    }
    final AccumuloTemporalIndexer temporal = new AccumuloTemporalIndexer();
    temporal.setConf(conf);
    Connector connector = ConfigUtils.getConnector(conf);
    MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(new BatchWriterConfig());
    temporal.setConnector(connector);
    temporal.setMultiTableBatchWriter(mtbw);
    temporal.init();
    final Set<Statement> empty = new HashSet<>();
    final Set<Statement> head = new HashSet<>();
    final Set<Statement> tail = new HashSet<>();
    head.add(statements[0]);
    tail.add(statements[2]);
    tail.add(statements[3]);
    Assert.assertEquals(empty, getSet(temporal.queryInstantBeforeInstant(instants[0], new StatementConstraints())));
    Assert.assertEquals(empty, getSet(temporal.queryInstantAfterInstant(instants[3], new StatementConstraints())));
    Assert.assertEquals(head, getSet(temporal.queryInstantBeforeInstant(instants[1], new StatementConstraints())));
    Assert.assertEquals(tail, getSet(temporal.queryInstantAfterInstant(instants[1], new StatementConstraints())));
    temporal.close();
}