Java Code Examples for org.apache.accumulo.core.client.BatchWriter#flush()

public void testSimpleOutput() throws Exception {
    BatchWriter batchWriter = connector.createBatchWriter(table, 10l, 10l, 2);
    Mutation row = new Mutation("row");
    row.put("cf", "cq", new Value(new byte[0]));
    batchWriter.addMutation(row);
    batchWriter.flush();
    batchWriter.close();

    String location = "accumulo://" + table + "?instance=" + instance + "&user=" + user + "&password=" + pwd + "&range=a|z&mock=true";
    AccumuloStorage storage = createAccumuloStorage(location);
    int count = 0;
    while (true) {
        Tuple next = storage.getNext();
        if (next == null)
            break;
        assertEquals(6, next.size());
        count++;
    }
    assertEquals(1, count);
}
 

public void testColumns() throws Exception {
    BatchWriter batchWriter = connector.createBatchWriter(table, 10l, 10l, 2);
    Mutation row = new Mutation("a");
    row.put("cf1", "cq", new Value(new byte[0]));
    row.put("cf2", "cq", new Value(new byte[0]));
    row.put("cf3", "cq1", new Value(new byte[0]));
    row.put("cf3", "cq2", new Value(new byte[0]));
    batchWriter.addMutation(row);
    batchWriter.flush();
    batchWriter.close();

    String location = "accumulo://" + table + "?instance=" + instance + "&user=" + user + "&password=" + pwd + "&range=a|c&columns=cf1,cf3|cq1&mock=true";
    AccumuloStorage storage = createAccumuloStorage(location);
    int count = 0;
    while (true) {
        Tuple next = storage.getNext();
        if (next == null)
            break;
        assertEquals(6, next.size());
        count++;
    }
    assertEquals(2, count);
}
 

public void testWholeRowRange() throws Exception {
    BatchWriter batchWriter = connector.createBatchWriter(table, 10l, 10l, 2);
    Mutation row = new Mutation("a");
    row.put("cf1", "cq", new Value(new byte[0]));
    row.put("cf2", "cq", new Value(new byte[0]));
    row.put("cf3", "cq1", new Value(new byte[0]));
    row.put("cf3", "cq2", new Value(new byte[0]));
    batchWriter.addMutation(row);
    batchWriter.flush();
    batchWriter.close();

    String location = "accumulo://" + table + "?instance=" + instance + "&user=" + user + "&password=" + pwd + "&range=a&mock=true";
    AccumuloStorage storage = createAccumuloStorage(location);
    int count = 0;
    while (true) {
        Tuple next = storage.getNext();
        if (next == null)
            break;
        assertEquals(6, next.size());
        count++;
    }
    assertEquals(4, count);
}
 

@Override
public void close(TaskAttemptContext context) throws IOException, InterruptedException {
    try {
        for (BatchWriter w : writerMap.values()) {
            w.flush();
            w.close();
        }
    } catch (MutationsRejectedException e) {
        throw new IOException("Error closing Batch Writer", e);
    }
}
 

public void run() throws Exception {

    entriesScanned = 0;
    inserts = 0;

    Scanner scanner = client.createScanner(tableName, auths);
    scanner.setBatchSize(scanOpts.scanBatchSize);
    BatchWriter bw = client.createBatchWriter(tableName, bwOpts.getBatchWriterConfig());

    long t1 = System.currentTimeMillis();

    int depth = findMaxDepth(scanner);

    long t2 = System.currentTimeMillis();

    for (int d = depth; d > 0; d--) {
      calculateCounts(scanner, d, bw);
      // must flush so next depth can read what prev depth wrote
      bw.flush();
    }

    bw.close();

    long t3 = System.currentTimeMillis();

    System.out.printf("Max depth              : %d%n", depth);
    System.out.printf("Time to find max depth : %,d ms%n", (t2 - t1));
    System.out.printf("Time to compute counts : %,d ms%n", (t3 - t2));
    System.out.printf("Entries scanned        : %,d %n", entriesScanned);
    System.out.printf("Counts inserted        : %,d %n", inserts);
  }
 

private static void deleteRow(String row, AccumuloClient client, BatchWriter bw)
    throws MutationsRejectedException, TableNotFoundException {
  Mutation mut = new Mutation(row);
  try (Scanner scanner = client.createScanner(table, Authorizations.EMPTY)) {
    scanner.setRange(Range.exact(row));
    for (Entry<Key,Value> entry : scanner) {
      mut.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
    }
  }
  bw.addMutation(mut);
  bw.flush();
}
 

public static void writeMutations(final Connector connector, final String tableName, final Collection<Mutation> mutations) throws TableNotFoundException, MutationsRejectedException {
    final BatchWriter bw = connector.createBatchWriter(tableName, 10000l, 10000l, 4);
    for(final Mutation mutation : mutations) {
        bw.addMutation(mutation);
    }
    bw.flush();
    bw.close();
}
 

private void persistTestMutations(int numRows, int entriesPerRow) throws TableNotFoundException, MutationsRejectedException {

        BatchWriter writer = connector.createBatchWriter("test", 1000, 1000, 1);

        for (int j = 0; j < numRows; j++) {
            Mutation m = new Mutation(Integer.toString(j));
            for (int i = 0; i < entriesPerRow; i++)
                m.put(new Text(Integer.toString(i)), new Text(""), new Value("".getBytes()));

            writer.addMutation(m);
        }
        writer.flush();
    }
 

private void persistTestMutations(int numRows, int entriesPerRow) throws TableNotFoundException, MutationsRejectedException {

        BatchWriter writer = connector.createBatchWriter("test", 1000, 1000, 1);

        for (int j = 0; j < numRows; j++) {
            Mutation m = new Mutation(Integer.toString(j));
            for (int i = 0; i < entriesPerRow; i++)
                m.put(new Text(Integer.toString(j)), new Text(String.valueOf(i)), new Value("".getBytes()));

            writer.addMutation(m);
        }
        writer.flush();
    }
 

private void persistTestMutations(int numRows) throws TableNotFoundException, MutationsRejectedException {

    BatchWriter writer = connector.createBatchWriter("test", 1000, 1000, 1);

    for (int j = 0; j < numRows; j++) {
      Mutation m = new Mutation(Integer.toString(j) + "|suffix");
      m.put(new Text(), new Text(), new Value("".getBytes()));

      for(int i = 0; i < numRows; i++)
        m.put(new Text("" +i), new Text(), new Value("".getBytes()));
      writer.addMutation(m);
    }
    writer.flush();
  }
 

private void doScan(WebSocketClient client) throws Exception {
    long now = System.currentTimeMillis();
    String tableName = "qonduit.scanTest";
    Connector con = mac.getConnector(MAC_ROOT_USER, MAC_ROOT_PASSWORD);
    con.namespaceOperations().create("qonduit");
    con.tableOperations().create(tableName);
    BatchWriterConfig bwc = new BatchWriterConfig();
    bwc.setMaxLatency(2, TimeUnit.SECONDS);
    BatchWriter writer = con.createBatchWriter(tableName, bwc);

    ColumnVisibility cv = new ColumnVisibility();
    for (int i = 0; i < 10; i++) {
        Mutation m = new Mutation("m" + i);
        m.put("cf" + i, "cq" + i, cv, now + i, Integer.toString(i));
        writer.addMutation(m);
    }
    writer.flush();
    writer.close();
    sleepUninterruptibly(2, TimeUnit.SECONDS);
    List<byte[]> responses = new ArrayList<>();
    String id = UUID.randomUUID().toString();
    ScanRequest request = new ScanRequest();
    request.setRequestId(id);
    request.setTableName(tableName);
    request.setResultBatchSize(5);
    doIt(client, request, responses, 3);
    Assert.assertEquals(11, responses.size());
    for (byte[] b : responses) {
        KVPair kv = JsonSerializer.getObjectMapper().readValue(b, KVPair.class);
        Value val = kv.getValue();
        if (null != val) {
            int num = Integer.parseInt(new String(val.getValue()));
            Key key = kv.getKey().toKey();
            Assert.assertEquals("m" + num, key.getRow().toString());
            Assert.assertEquals("cf" + num, key.getColumnFamily().toString());
            Assert.assertEquals("cq" + num, key.getColumnQualifier().toString());
            Assert.assertEquals(now + num, key.getTimestamp());
            Assert.assertEquals(id, kv.getRequestId());
        } else {
            Assert.assertTrue(kv.isEndOfResults());
        }
    }
}
 

public void execute(Options options, Authorizations authorizations) {
    try {
        org.apache.accumulo.core.security.Authorizations accumuloAuthorizations = graph.toAccumuloAuthorizations(authorizations);
        Scanner scanner = graph.getConnector().createScanner(graph.getDataTableName(), accumuloAuthorizations);
        BatchWriter writer = graph.getConnector().createBatchWriter(
            graph.getDataTableName(),
            graph.getConfiguration().createBatchWriterConfig()
        );
        String lastRowIdPrefix = null;
        List<Key> rowsToDelete = new ArrayList<>();
        try {
            int rowCount = 0;
            for (Map.Entry<Key, Value> row : scanner) {
                if (rowCount % 10000 == 0) {
                    writer.flush();
                    LOGGER.debug("looking at row: %s (row count: %d)", row.getKey().getRow().toString(), rowCount);
                }
                rowCount++;
                if (!EMPTY_TEXT.equals(row.getKey().getColumnFamily())) {
                    continue;
                }
                if (!EMPTY_TEXT.equals(row.getKey().getColumnQualifier())) {
                    continue;
                }

                String rowId = row.getKey().getRow().toString();
                String[] rowIdParts = rowId.split("" + DataTableRowKey.VALUE_SEPARATOR);
                if (rowIdParts.length < 3) {
                    continue;
                }

                if (lastRowIdPrefix == null || !isSameProperty(lastRowIdPrefix, rowId)) {
                    deleteRows(writer, rowsToDelete, options);
                    rowsToDelete.clear();
                    lastRowIdPrefix = rowIdParts[0]
                        + DataTableRowKey.VALUE_SEPARATOR
                        + rowIdParts[1]
                        + DataTableRowKey.VALUE_SEPARATOR
                        + rowIdParts[2];
                }
                rowsToDelete.add(row.getKey());
            }
            deleteRows(writer, rowsToDelete, options);
        } finally {
            writer.flush();
            scanner.close();
        }
    } catch (Exception ex) {
        throw new VertexiumException("Could not delete old SPV data", ex);
    }
}