Java Code Examples for org.apache.accumulo.core.data.Key

The following are top voted examples for showing how to use org.apache.accumulo.core.data.Key. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: PACE   File: Matchers.java   View source code 6 votes vote down vote up
/**
 * Matches the set of entries against the given set of values. The full combinatorial of values passed in is expected in the output set.
 *
 * @param rows
 *          Rows to check for.
 * @param colFs
 *          Column families to check for.
 * @param colQs
 *          Column qualifiers to check for.
 * @param colVs
 *          Column visibilities to check for.
 * @param values
 *          Values to check for.
 * @return Hamcrest matcher.
 */
@Factory
@SuppressWarnings("unchecked")
public static Matcher<Iterable<Entry<Key,Value>>> hasData(Collection<String> rows, Collection<String> colFs, Collection<String> colQs,
    Collection<String> colVs, Collection<String> values) {
  int size = rows.size() * colFs.size() * colQs.size() * colVs.size() * values.size();
  ArrayList<Matcher<? super Iterable<Entry<Key,Value>>>> matchers = new ArrayList<>(size + 1);

  matchers.add(IsIterableWithSize.iterableWithSize(size));

  for (String row : rows) {
    for (String colF : colFs) {
      for (String colQ : colQs) {
        for (String colV : colVs) {
          for (String value : values) {
            matchers.add(hasItems(equalToRow(row, colF, colQ, colV, value)));
          }
        }
      }
    }
  }

  return allOf(matchers);
}
 
Example 2
Project: PACE   File: GenerateRandomEntries.java   View source code 6 votes vote down vote up
/**
 * Get random data.
 *
 * @param count
 *          Number of entries to create.
 * @param rowSize
 *          Size (in bytes) of the row.
 * @param colFSize
 *          Size (in bytes) of the column family.
 * @param colQSize
 *          Size (in bytes) of the column qualifier.
 * @param colVisValue
 *          Value to use for colVis.
 * @param valueSize
 *          Size (in bytes) of the value.
 * @return Random data.
 */
static SortedMap<Key,Value> getRandomData(int count, int rowSize, int colFSize, int colQSize, byte[] colVisValue, int valueSize) {
  SortedMap<Key,Value> data = new TreeMap<>();

  for (int i = 0; i < count; i++) {
    byte[] row = new byte[rowSize];
    byte[] colF = new byte[colFSize];
    byte[] colQ = new byte[colQSize];
    byte[] colVis = colVisValue.clone();
    byte[] value = new byte[valueSize];

    random.nextBytes(row);
    random.nextBytes(colF);
    random.nextBytes(colQ);
    random.nextBytes(value);
    long timestamp = random.nextLong();

    data.put(new Key(row, colF, colQ, colVis, timestamp, false, false), new Value(value));
  }

  return data;
}
 
Example 3
Project: PACE   File: VersioningIT.java   View source code 6 votes vote down vote up
/**
 * Run a specific test for a given version and configuration.
 *
 * @param dataScanner
 *          Scanner for the unsigned data.
 * @param signedScanner
 *          Scanner for the signed data.
 * @param verifier
 *          Verifier for the signed data.
 * @param signatureConfig
 *          Configuration for the verifier.
 */
private void runTest(Scanner dataScanner, Scanner signedScanner, EntrySigner verifier, SignatureConfig signatureConfig) {
  Iterator<Entry<Key,Value>> iterator = signedScanner.iterator();
  for (Entry<Key,Value> entry : dataScanner) {
    assertThat("should have more entries", iterator.hasNext(), is(true));
    Entry<Key,Value> signedEntry = iterator.next();

    Entry<Key,Value> verifiedEntry;
    if (signatureConfig.isSignatureInSeparateTable()) {
      assertThat("keys should match", signedEntry.getKey(), equalTo(entry.getKey()));
      assertThat("values should not match", signedEntry.getValue().get(), not(equalTo(entry.getValue().get())));
      verifiedEntry = verifier.verify(entry, signedEntry);
    } else {
      verifiedEntry = verifier.verify(signedEntry);
    }

    assertThat("entries should match", verifiedEntry, Matchers.equalTo(entry));
  }

  assertThat("should have no more entries", iterator.hasNext(), is(false));
}
 
Example 4
Project: PACE   File: BatchWriteReadIT.java   View source code 6 votes vote down vote up
public void check(SignatureConfig config) throws Exception {
  Scanner scanner = AccumuloInstance.getConnector(USER).createScanner(UNSIGNED_TEST_TABLE, AccumuloInstance.getUser(USER).authorizations);
  Scanner signedScanner;

  if (!config.isSignatureInSeparateTable()) {
    signedScanner = new SignedScanner(AccumuloInstance.getConnector(USER), SIGNED_TEST_TABLE, AccumuloInstance.getUser(USER).authorizations, config,
        AccumuloInstance.getUser(USER).signatureKeys.get(config.getAlgorithm()));
  } else {
    signedScanner = new SignedScanner(AccumuloInstance.getConnector(USER), UNSIGNED_TEST_TABLE, AccumuloInstance.getUser(USER).authorizations, config,
        AccumuloInstance.getUser(USER).signatureKeys.get(config.getAlgorithm()));
  }

  Iterator<Entry<Key,Value>> iterator = signedScanner.iterator();
  for (Entry<Key,Value> entry : scanner) {
    assertThat("should have an entry that matches", iterator.hasNext(), is(true));
    assertThat("entries match", iterator.next(), equalTo(entry));
  }

  assertThat("should have no more entries", iterator.hasNext(), is(false));
}
 
Example 5
Project: PACE   File: FilteringIT.java   View source code 6 votes vote down vote up
private void setRangesTest(String configuration) throws Exception {
  List<String> rows = Arrays.asList("row1", "row2");
  List<String> colFs = Arrays.asList("colF1", "colF2");
  List<String> colQs = Arrays.asList("colQ1", "colQ2");
  List<String> colVs = Collections.singletonList("");
  List<String> values = Collections.singletonList("value");

  clearTable();
  EncryptedBatchWriter writer = getEncryptedWriter(CHARLIE, configuration);
  writeData(writer, rows, colFs, colQs, colVs, values);
  writer.close();

  EncryptedBatchScanner scanner = getEncryptedScanner(CHARLIE, configuration);
  scanner.setRanges(Collections.singletonList(new Range(new Key("row1", "colF1", "colQ1"), true, new Key("row1", "colF1", "colQ2")
      .followingKey(PartialKey.ROW_COLFAM_COLQUAL), false)));
  assertThat("contains the filtered data", scanner, hasData(Collections.singletonList("row1"), Collections.singletonList("colF1"), colQs, colVs, values));
}
 
Example 6
Project: PACE   File: EntryEncryptor.java   View source code 6 votes vote down vote up
/**
 * Decrypt the given entry.
 *
 * @param entry
 *          entry to decrypt.
 * @return Decrypted entry.
 */
public Entry<Key,Value> decrypt(Entry<Key,Value> entry) {
  checkArgument(entry != null, "entry is null");

  MutableEntry wrapped = new MutableEntry(entry);
  MutableEntry result = new MutableEntry(entry);
  ColumnVisibility visibility = entry.getKey().getColumnVisibilityParsed();

  // Decrypt the various fields.
  try {
    for (FieldEncryptor fieldEncryptor : encryptors) {
      fieldEncryptor.decrypt(wrapped, result, visibility);
    }
  } catch (IOException e) { // IO exceptions won't be thrown in practice as we are operating on in-memory streams.
    throw new EncryptionException(e);
  }

  return result.toEntry();
}
 
Example 7
Project: PACE   File: EntryEncryptor.java   View source code 6 votes vote down vote up
/**
 * Given a key for deletion, generate the appropriate server side keys to delete.
 *
 * @param key
 *          Key to delete.
 * @return Set of encrypted keys to delete.
 */
public Collection<Key> getDeleteKeys(Key key) {
  checkArgument(canBeDeleteServerSide(), "canBeDeleteServerSide is false");
  checkArgument(key != null, "key is null");
  MutableEntry searchKey = new MutableEntry(key);

  // Pair up the correct answers.
  Collection<byte[]> rowValues = getDeleteValues(searchKey, EntryField.ROW);
  Collection<byte[]> colFValues = getDeleteValues(searchKey, EntryField.COLUMN_FAMILY);
  Collection<byte[]> colQValues = getDeleteValues(searchKey, EntryField.COLUMN_QUALIFIER);

  return rowValues
      .stream()
      .flatMap(
          r -> colFValues.stream().flatMap(
              cf -> colQValues.stream().map(cq -> new Key(r, cf, cq, key.getColumnVisibilityData().getBackingArray(), key.getTimestamp(), true, true))))
      .collect(Collectors.toList());
}
 
Example 8
Project: PACE   File: EntrySignerTest.java   View source code 6 votes vote down vote up
@Test
public void signVerifyInValueTest() throws Exception {
  MutableEntry entry = new MutableEntry(new SimpleImmutableEntry<>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3},
      "secret".getBytes(VISIBILITY_CHARSET), (long) 5, false), new Value(new byte[] {6})));

  MutableEntry signed;
  Entry<Key,Value> verified;

  EntrySigner signer = getSigner("config1.ini", aliceKeyContainers.get(ValueSigner.RSA_PSS));
  EntrySigner verifier = getSigner("config1.ini", bobKeyContainers.get(ValueSigner.RSA_PSS));

  signed = new MutableEntry(signer.sign(entry.toEntry(), true));
  assertThat("row should not have changed", signed.row, is(entry.row));
  assertThat("colFamily should not have changed", signed.colF, is(entry.colF));
  assertThat("colQualifier should not have changed", signed.colQ, is(entry.colQ));
  assertThat("colVisibility should not have changed", signed.colVis, is(entry.colVis));
  assertThat("timestamp should not have changed", signed.timestamp, is(entry.timestamp));
  assertThat("delete should not have changed", signed.delete, is(entry.delete));
  assertThat("value should have changed", signed.value, is(not(entry.value)));

  verified = verifier.verify(signed.toEntry());
  assertThat("original and verified records are the same.", verified, Matchers.equalTo(entry.toEntry()));
}
 
Example 9
Project: PACE   File: EntrySignerTest.java   View source code 6 votes vote down vote up
@Test
public void signVerifyInSeparateTableTest() throws Exception {
  MutableEntry entry = new MutableEntry(new SimpleImmutableEntry<>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3},
      "secret".getBytes(VISIBILITY_CHARSET), (long) 5, false), new Value(new byte[] {6})));

  MutableEntry signed;
  Entry<Key,Value> verified;

  EntrySigner signer = getSigner("config3.ini", aliceKeyContainers.get(ValueSigner.ECDSA));
  EntrySigner verifier = getSigner("config3.ini", bobKeyContainers.get(ValueSigner.ECDSA));

  signed = new MutableEntry(signer.sign(entry.toEntry(), true));
  assertThat("row should not have changed", signed.row, is(entry.row));
  assertThat("colFamily should not have changed", signed.colF, is(entry.colF));
  assertThat("colQualifier should not have changed", signed.colQ, is(entry.colQ));
  assertThat("colVisibility should not have changed", signed.colVis, is(entry.colVis));
  assertThat("timestamp should not have changed", signed.timestamp, is(entry.timestamp));
  assertThat("delete should not have changed", signed.delete, is(entry.delete));
  assertThat("value should have changed", signed.value, is(not(entry.value)));

  verified = verifier.verify(entry.toEntry(), signed.toEntry());
  assertThat("original and verified records are the same.", verified, Matchers.equalTo(entry.toEntry()));
}
 
Example 10
Project: PACE   File: EntrySignerTest.java   View source code 6 votes vote down vote up
@Test
public void hasNoTimestampTest() throws Exception {
  MutableEntry entry = new MutableEntry(new SimpleImmutableEntry<>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3},
      "secret".getBytes(VISIBILITY_CHARSET), (long) 0, false), new Value(new byte[] {6})));
  MutableEntry signed;

  EntrySigner signer = getSigner("config1.ini", aliceKeyContainers.get(ValueSigner.RSA_PSS));
  EntrySigner verifier = getSigner("config1.ini", bobKeyContainers.get(ValueSigner.RSA_PSS));

  try {
    signed = new MutableEntry(signer.sign(entry.toEntry(), true));
    signed.timestamp = 1000L;
    verifier.verify(signed.toEntry());
    fail("changing the timestamp should cause the signature to fail");
  } catch (SignatureException e) { /* expected */}

  signed = new MutableEntry(signer.sign(entry.toEntry(), false));
  signed.timestamp = 1000L;
  verifier.verify(signed.toEntry());
}
 
Example 11
Project: PACE   File: EntrySignerTest.java   View source code 6 votes vote down vote up
@Test
public void badSignatureTest() throws Exception {
  MutableEntry entry = new MutableEntry(new SimpleImmutableEntry<>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3},
      "secret".getBytes(VISIBILITY_CHARSET), (long) 0, false), new Value(new byte[] {6})));
  Entry<Key,Value> signed;

  // Sign to value;
  EntrySigner signer = getSigner("config3.ini", aliceKeyContainers.get(ValueSigner.ECDSA));
  EntrySigner verifier = getSigner("config3.ini", bobKeyContainers.get(ValueSigner.ECDSA));

  signed = signer.sign(entry.toEntry(), true);
  entry.value = new byte[] {7};

  try {
    verifier.verify(entry.toEntry(), signed);
    fail("bad signature should thrown an exception");
  } catch (SignatureException e) { /* expected */}
}
 
Example 12
Project: PACE   File: SignedIteratorTest.java   View source code 6 votes vote down vote up
@Test
public void internalUnprocessedTest() throws Exception {
  EntrySigner signer = getSigner("config1.ini", aliceKeyContainers.get(ValueSigner.RSA_PSS));
  EntrySigner verifier = getSigner("config1.ini", bobKeyContainers.get(ValueSigner.RSA_PSS));

  List<Entry<Key,Value>> entries = new ArrayList<>();
  byte[] row = new byte[] {1};
  Entry<Key,Value> entry = new SimpleImmutableEntry<>(new Key(row, new byte[] {2}, new byte[] {3}, "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false,
      false), new Value(new byte[] {4}));

  entries.add(signer.sign(entry, true));
  SignedInlineScannerIterator iterator = new SignedInlineScannerIterator(entries.iterator(), verifier);

  iterator.next();
  assertThat("unprocessed item is correct", iterator.unprocessed(), Matchers.equalTo(entries.get(0)));
}
 
Example 13
Project: PACE   File: SignedIteratorTest.java   View source code 6 votes vote down vote up
@Test
public void externalUnprocessedTest() throws Exception {
  EntrySigner signer = getSigner("config3.ini", aliceKeyContainers.get(ValueSigner.ECDSA));
  EntrySigner verifier = getSigner("config3.ini", bobKeyContainers.get(ValueSigner.ECDSA));

  List<Entry<Key,Value>> entries = new ArrayList<>();
  List<Entry<Key,Value>> signedEntries = new ArrayList<>();

  byte[] row = new byte[] {1};
  Entry<Key,Value> entry = new SimpleImmutableEntry<>(new Key(row, new byte[] {2}, new byte[] {3}, "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false,
      false), new Value(new byte[] {4}));

  entries.add(entry);
  signedEntries.add(signer.sign(entry, true));

  SignedExternalScannerIterator iterator = new SignedExternalScannerIterator(entries.iterator(), signedEntries.iterator(), verifier, true);

  iterator.next();
  assertThat("unprocessed item is correct", iterator.unprocessed(), Matchers.equalTo(entries.get(0)));
}
 
Example 14
Project: PACE   File: EntryEncryptorTest.java   View source code 6 votes vote down vote up
@Test
public void getDeleteKeysTest() throws Exception {
  EntryEncryptor encryptor = getEncryptor("deterministic.ini");
  MutableEntry searchKey = new MutableEntry(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3}, "secret".getBytes(VISIBILITY_CHARSET), (long) 5, false));

  Collection<Key> keys = encryptor.getDeleteKeys(searchKey.toKey());
  assertThat("has correct number of delete keys", keys, hasSize(2));

  for (Key deleteKey : keys) {
    MutableEntry key = new MutableEntry(deleteKey);
    assertThat("row is encrypted", key.row, not(equalTo(searchKey.row)));
    assertThat("colF is zeroes", key.colF, equalTo(EMPTY));
    assertThat("colQ is plaintext", key.colQ, equalTo(searchKey.colQ));
    assertThat("colQ is plaintext", key.colVis, equalTo(searchKey.colVis));
    assertThat("colQ is plaintext", key.timestamp, equalTo(searchKey.timestamp));
    assertThat("delete is true", key.delete, is(true));
  }
}
 
Example 15
Project: PACE   File: EncryptedScannerIteratorTest.java   View source code 6 votes vote down vote up
@Test
public void hasNextTest() throws Exception {
  EntryEncryptor encryptor = new EntryEncryptor(getConfig("config.ini"), KEYS);

  List<Entry<Key,Value>> entries = new ArrayList<>();
  Entry<Key,Value> entry = new SimpleImmutableEntry<Key,Value>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {4}));
  entries.add(encryptor.encrypt(entry));

  EncryptedScannerIterator iterator = new EncryptedScannerIterator(entries.iterator(), encryptor, Collections.singletonList(new Range()),
      new TreeSet<Column>());

  assertThat("has next item", iterator.hasNext(), is(true));
  assertThat("has next item", iterator.hasNext(), is(true));
  iterator.next();
  assertThat("does not have a next item", iterator.hasNext(), is(false));
}
 
Example 16
Project: PACE   File: EncryptedScannerIteratorTest.java   View source code 6 votes vote down vote up
@Test
public void nextTest() throws Exception {
  EntryEncryptor encryptor = new EntryEncryptor(getConfig("config.ini"), KEYS);

  List<Entry<Key,Value>> entries = new ArrayList<>();
  Entry<Key,Value> entry = new SimpleImmutableEntry<Key,Value>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {4}));
  entries.add(encryptor.encrypt(entry));

  EncryptedScannerIterator iterator = new EncryptedScannerIterator(entries.iterator(), encryptor, Collections.singletonList(new Range()),
      new TreeSet<Column>());
  assertThat("next item is correct", iterator.next(), Matchers.equalTo(entry));

  try {
    iterator.next();
    fail("no items should be left");
  } catch (NoSuchElementException e) { /* expected */}
}
 
Example 17
Project: PACE   File: EncryptedScannerIteratorTest.java   View source code 6 votes vote down vote up
@Test
public void matchRangeTest() throws Exception {
  EntryEncryptor encryptor = new EntryEncryptor(getConfig("config.ini"), KEYS);

  List<Entry<Key,Value>> entries = new ArrayList<>();
  Entry<Key,Value> entry = new SimpleImmutableEntry<>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3}, "secret".getBytes(Utils.VISIBILITY_CHARSET), 0,
      false, false), new Value(new byte[] {4}));
  Entry<Key,Value> entry2 = new SimpleImmutableEntry<>(new Key(new byte[] {5}, new byte[] {6}, new byte[] {7}, "secret".getBytes(Utils.VISIBILITY_CHARSET),
      0, false, false), new Value(new byte[] {8}));
  entries.add(encryptor.encrypt(entry));
  entries.add(encryptor.encrypt(entry2));

  EncryptedScannerIterator iterator = new EncryptedScannerIterator(entries.iterator(), new EntryEncryptor(getConfig("config.ini"), KEYS),
      Collections.singletonList(new Range()), new TreeSet<>());
  assertThat("correct number of items", Lists.newArrayList(iterator), hasSize(2));

  iterator = new EncryptedScannerIterator(entries.iterator(), new EntryEncryptor(getConfig("config.ini"), KEYS), new ArrayList<Range>(),
      new TreeSet<Column>());
  assertThat("correct number of items", Lists.newArrayList(iterator), hasSize(2));

  iterator = getIteratorForRange(entries, (byte) 1);
  assertThat("correct number of items", Lists.newArrayList(iterator), hasSize(1));

  iterator = getIteratorForRange(entries, (byte) 3);
  assertThat("correct number of items", Lists.newArrayList(iterator), hasSize(0));
}
 
Example 18
Project: PACE   File: EncryptedScannerIteratorTest.java   View source code 6 votes vote down vote up
@Test
public void matchColumnFilters() throws Exception {
  EntryEncryptor encryptor = new EntryEncryptor(getConfig("config.ini"), KEYS);

  List<Entry<Key,Value>> entries = new ArrayList<>();
  Entry<Key,Value> entry = new SimpleImmutableEntry<>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3}, "secret".getBytes(Utils.VISIBILITY_CHARSET), 0,
      false, false), new Value(new byte[] {4}));
  Entry<Key,Value> entry2 = new SimpleImmutableEntry<>(new Key(new byte[] {5}, new byte[] {2}, new byte[] {7}, "secret".getBytes(Utils.VISIBILITY_CHARSET),
      0, false, false), new Value(new byte[] {8}));
  entries.add(encryptor.encrypt(entry));
  entries.add(encryptor.encrypt(entry2));

  EncryptedScannerIterator iterator = new EncryptedScannerIterator(entries.iterator(), new EntryEncryptor(getConfig("config.ini"), KEYS),
      Collections.singletonList(new Range()), new TreeSet<>());
  assertThat("correct number of items", Lists.newArrayList(iterator), hasSize(2));

  iterator = getIteratorForColumn(entries, new byte[] {2}, null);
  assertThat("correct number of items", Lists.newArrayList(iterator), hasSize(2));

  iterator = getIteratorForColumn(entries, new byte[] {3}, null);
  assertThat("correct number of items", Lists.newArrayList(iterator), hasSize(0));

  iterator = getIteratorForColumn(entries, new byte[] {2}, new byte[] {7});
  assertThat("correct number of items", Lists.newArrayList(iterator), hasSize(1));
}
 
Example 19
Project: PACE   File: EncryptedScannerIteratorTest.java   View source code 6 votes vote down vote up
@Test
public void removeException() throws Exception {
  EntryEncryptor encryptor = new EntryEncryptor(getConfig("config.ini"), KEYS);

  List<Entry<Key,Value>> entries = new ArrayList<>();
  Entry<Key,Value> entry = new SimpleImmutableEntry<Key,Value>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {4}));
  entries.add(encryptor.encrypt(entry));

  EncryptedScannerIterator iterator = new EncryptedScannerIterator(entries.iterator(), encryptor, Collections.singletonList(new Range()), new TreeSet<>());

  try {
    iterator.remove();
    fail("remove not supported");
  } catch (UnsupportedOperationException e) { /* expected */}
}
 
Example 20
Project: PACE   File: EncryptedBatchScannerTest.java   View source code 6 votes vote down vote up
@Test
public void setRangesSearchableTest() throws Exception {
  when(mockConnector.createBatchScanner(TEST_TABLE, authorizations, 1)).thenReturn(mockScanner);

  EntryEncryptor encryptor = new EntryEncryptor(getConfig("searchable-row.ini"), KEYS);
  List<Map.Entry<Key,Value>> entries = new ArrayList<>();
  Map.Entry<Key,Value> entry = new SimpleImmutableEntry<>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {4}));
  Map.Entry<Key,Value> entry2 = new SimpleImmutableEntry<>(new Key(new byte[] {5}, new byte[] {6}, new byte[] {7},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {8}));
  entries.add(encryptor.encrypt(entry));
  entries.add(encryptor.encrypt(entry2));
  when(mockScanner.iterator()).thenReturn(entries.iterator()).thenReturn(entries.iterator()).thenReturn(entries.iterator());

  BatchScanner scanner = new EncryptedBatchScanner(mockConnector, TEST_TABLE, authorizations, 1, getConfig("searchable-row.ini"), KEYS);
  scanner.setRanges(Collections.singletonList(new Range(new Text(new byte[] {1}))));
  assertThat("has correct number of elements", scanner, iterableWithSize(2));

  verify(mockScanner).setRanges(captor.capture());
  assertThat("correct number of ranges", captor.getValue(), hasSize(2));
}
 
Example 21
Project: PACE   File: EncryptedBatchScannerTest.java   View source code 6 votes vote down vote up
@Test
public void fetchColumnFamilySemanticEncryptionTest() throws Exception {
  when(mockConnector.createBatchScanner(TEST_TABLE, authorizations, 1)).thenReturn(mockScanner);

  EntryEncryptor encryptor = new EntryEncryptor(getConfig("encrypt-key.ini"), KEYS);
  List<Map.Entry<Key,Value>> entries = new ArrayList<>();
  Map.Entry<Key,Value> entry = new SimpleImmutableEntry<>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {4}));
  Map.Entry<Key,Value> entry2 = new SimpleImmutableEntry<>(new Key(new byte[] {5}, new byte[] {6}, new byte[] {7},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {8}));
  entries.add(encryptor.encrypt(entry));
  entries.add(encryptor.encrypt(entry2));
  when(mockScanner.iterator()).thenReturn(entries.iterator()).thenReturn(entries.iterator()).thenReturn(entries.iterator());

  BatchScanner scanner = new EncryptedBatchScanner(mockConnector, TEST_TABLE, authorizations, 1, getConfig("encrypt-key.ini"), KEYS);
  assertThat("has correct number of elements", scanner, iterableWithSize(2));

  scanner.fetchColumnFamily(new Text(new byte[] {2}));
  assertThat("has correct number of elements", scanner, iterableWithSize(1));

  scanner.fetchColumnFamily(new Text(new byte[] {6}));
  assertThat("has correct number of elements", scanner, iterableWithSize(2));

  // Should not have been handled server side.
  verify(mockScanner, never()).fetchColumn(any());
}
 
Example 22
Project: PACE   File: EncryptedBatchScannerTest.java   View source code 6 votes vote down vote up
@Test
public void fetchColumnFamilySearchableTest() throws Exception {
  when(mockConnector.createBatchScanner(TEST_TABLE, authorizations, 1)).thenReturn(mockScanner);

  EntryEncryptor encryptor = new EntryEncryptor(getConfig("searchable.ini"), KEYS);
  List<Map.Entry<Key,Value>> entries = new ArrayList<>();
  Map.Entry<Key,Value> entry = new SimpleImmutableEntry<>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {4}));
  Map.Entry<Key,Value> entry2 = new SimpleImmutableEntry<>(new Key(new byte[] {5}, new byte[] {6}, new byte[] {7},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {8}));
  entries.add(encryptor.encrypt(entry));
  entries.add(encryptor.encrypt(entry2));
  when(mockScanner.iterator()).thenReturn(entries.iterator()).thenReturn(entries.iterator()).thenReturn(entries.iterator());

  BatchScanner scanner = new EncryptedBatchScanner(mockConnector, TEST_TABLE, authorizations, 1, getConfig("searchable.ini"), KEYS);
  scanner.fetchColumnFamily(new Text(new byte[] {2}));
  assertThat("filtering is not happening client-side", scanner, iterableWithSize(2));
  verify(mockScanner, times(2)).fetchColumnFamily(any());
}
 
Example 23
Project: PACE   File: EncryptedBatchScannerTest.java   View source code 6 votes vote down vote up
@Test
public void fetchColumnSemanticEncryptionTest() throws Exception {
  when(mockConnector.createBatchScanner(TEST_TABLE, authorizations, 1)).thenReturn(mockScanner);

  EntryEncryptor encryptor = new EntryEncryptor(getConfig("encrypt-key.ini"), KEYS);
  List<Map.Entry<Key,Value>> entries = new ArrayList<>();
  Map.Entry<Key,Value> entry = new SimpleImmutableEntry<>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {4}));
  Map.Entry<Key,Value> entry2 = new SimpleImmutableEntry<>(new Key(new byte[] {5}, new byte[] {6}, new byte[] {7},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {8}));
  entries.add(encryptor.encrypt(entry));
  entries.add(encryptor.encrypt(entry2));
  when(mockScanner.iterator()).thenReturn(entries.iterator()).thenReturn(entries.iterator()).thenReturn(entries.iterator());

  BatchScanner scanner = new EncryptedBatchScanner(mockConnector, TEST_TABLE, authorizations, 1, getConfig("encrypt-key.ini"), KEYS);
  assertThat("has correct number of elements", scanner, iterableWithSize(2));

  scanner.fetchColumn(new IteratorSetting.Column(new Text(new byte[] {2}), new Text(new byte[] {3})));
  assertThat("has correct number of elements", scanner, iterableWithSize(1));

  scanner.fetchColumn(new IteratorSetting.Column(new Text(new byte[] {6}), new Text(new byte[] {7})));
  assertThat("has correct number of elements", scanner, iterableWithSize(2));

  // Should not have been handled server side.
  verify(mockScanner, never()).fetchColumn(any());
}
 
Example 24
Project: PACE   File: EncryptedBatchScannerTest.java   View source code 6 votes vote down vote up
@Test
public void fetchColumnSearchableTest() throws Exception {
  when(mockConnector.createBatchScanner(TEST_TABLE, authorizations, 1)).thenReturn(mockScanner);

  EntryEncryptor encryptor = new EntryEncryptor(getConfig("searchable.ini"), KEYS);
  List<Map.Entry<Key,Value>> entries = new ArrayList<>();
  Map.Entry<Key,Value> entry = new SimpleImmutableEntry<>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {4}));
  Map.Entry<Key,Value> entry2 = new SimpleImmutableEntry<>(new Key(new byte[] {5}, new byte[] {6}, new byte[] {7},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {8}));
  entries.add(encryptor.encrypt(entry));
  entries.add(encryptor.encrypt(entry2));
  when(mockScanner.iterator()).thenReturn(entries.iterator()).thenReturn(entries.iterator()).thenReturn(entries.iterator());

  BatchScanner scanner = new EncryptedBatchScanner(mockConnector, TEST_TABLE, authorizations, 1, getConfig("searchable.ini"), KEYS);
  scanner.fetchColumn(new Text(new byte[] {2}), new Text(new byte[] {3}));
  assertThat("filtering is not happening client-side", scanner, iterableWithSize(2));
  verify(mockScanner, times(4)).fetchColumn(any(), any());
}
 
Example 25
Project: PACE   File: EncryptedBatchScannerTest.java   View source code 6 votes vote down vote up
@Test
public void fetchColumnPartiallySearchableTest() throws Exception {
  when(mockConnector.createBatchScanner(TEST_TABLE, authorizations, 1)).thenReturn(mockScanner);

  EntryEncryptor encryptor = new EntryEncryptor(getConfig("partially-searchable.ini"), KEYS);
  List<Map.Entry<Key,Value>> entries = new ArrayList<>();
  Map.Entry<Key,Value> entry = new SimpleImmutableEntry<>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {4}));
  Map.Entry<Key,Value> entry2 = new SimpleImmutableEntry<>(new Key(new byte[] {5}, new byte[] {6}, new byte[] {7},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {8}));
  entries.add(encryptor.encrypt(entry));
  entries.add(encryptor.encrypt(entry2));
  when(mockScanner.iterator()).thenReturn(entries.iterator()).thenReturn(entries.iterator()).thenReturn(entries.iterator());

  BatchScanner scanner = new EncryptedBatchScanner(mockConnector, TEST_TABLE, authorizations, 1, getConfig("partially-searchable.ini"), KEYS);
  scanner.fetchColumn(new Text(new byte[] {2}), new Text(new byte[] {3}));
  assertThat("filtering is partially happening client-side", scanner, iterableWithSize(1));
  verify(mockScanner, times(2)).fetchColumnFamily(any());
}
 
Example 26
Project: PACE   File: EncryptedBatchScannerTest.java   View source code 6 votes vote down vote up
@Test
public void clearColumnsTest() throws Exception {
  when(mockConnector.createBatchScanner(TEST_TABLE, authorizations, 1)).thenReturn(mockScanner);

  EntryEncryptor encryptor = new EntryEncryptor(getConfig("encrypt-key.ini"), KEYS);
  List<Map.Entry<Key,Value>> entries = new ArrayList<>();
  Map.Entry<Key,Value> entry = new SimpleImmutableEntry<>(new Key(new byte[] {1}, new byte[] {2}, new byte[] {3},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {4}));
  Map.Entry<Key,Value> entry2 = new SimpleImmutableEntry<>(new Key(new byte[] {5}, new byte[] {6}, new byte[] {7},
      "secret".getBytes(Utils.VISIBILITY_CHARSET), 0, false, false), new Value(new byte[] {8}));
  entries.add(encryptor.encrypt(entry));
  entries.add(encryptor.encrypt(entry2));
  when(mockScanner.iterator()).thenReturn(entries.iterator()).thenReturn(entries.iterator());

  BatchScanner scanner = new EncryptedBatchScanner(mockConnector, TEST_TABLE, authorizations, 1, getConfig("encrypt-key.ini"), KEYS);
  scanner.fetchColumn(new Text(new byte[] {2}), new Text(new byte[] {2}));
  assertThat("has correct number of elements", scanner, iterableWithSize(0));

  scanner.clearColumns();
  assertThat("has correct number of elements", scanner, iterableWithSize(2));
}
 
Example 27
Project: accumulo-wikisearch   File: BooleanLogicIterator.java   View source code 6 votes vote down vote up
private OrIterator createOrIterator(BooleanLogicTreeNode node) throws IOException {
  if (log.isDebugEnabled()) {
    log.debug("createOrIterator(node)");
    log.debug("fName: " + node.getFieldName() + " , fValue: " + node.getFieldValue() + " , operator: " + node.getFieldOperator());
  }
  
  Enumeration<?> children = node.children();
  ArrayList<Text> fams = new ArrayList<Text>();
  ArrayList<Text> quals = new ArrayList<Text>();
  while (children.hasMoreElements()) {
    BooleanLogicTreeNode child = (BooleanLogicTreeNode) children.nextElement();
    fams.add(child.getFieldName());
    quals.add(child.getFieldValue());
  }
  
  OrIterator iter = new OrIterator();
  SortedKeyValueIterator<Key,Value> source = sourceIterator.deepCopy(env);
  for (int i = 0; i < fams.size(); i++) {
    iter.addTerm(source, fams.get(i), quals.get(i), env);
  }
  
  return iter;
}
 
Example 28
Project: accumulo-examples   File: ChunkCombinerTest.java   View source code 6 votes vote down vote up
private void runTest(boolean reseek, TreeMap<Key,Value> source, TreeMap<Key,Value> result, Collection<ByteSequence> cols) throws IOException {
  MapIterator src = new MapIterator(source);
  SortedKeyValueIterator<Key,Value> iter = new ChunkCombiner();
  iter.init(src, null, null);
  iter = iter.deepCopy(null);
  iter.seek(new Range(), cols, true);

  TreeMap<Key,Value> seen = new TreeMap<>();

  while (iter.hasTop()) {
    assertFalse("already contains " + iter.getTopKey(), seen.containsKey(iter.getTopKey()));
    seen.put(new Key(iter.getTopKey()), new Value(iter.getTopValue()));

    if (reseek)
      iter.seek(new Range(iter.getTopKey().followingKey(PartialKey.ROW_COLFAM_COLQUAL), true, null, true), cols, true);
    else
      iter.next();
  }

  assertEquals(result, seen);
}
 
Example 29
Project: timely   File: TimelyMetricsFormatter.java   View source code 6 votes vote down vote up
@Override
public String next() {
    Entry<Key, Value> e = getScannerIterator().next();
    StringBuilder b = new StringBuilder();
    ComparablePair<String, Long> p = rowCoder.decode(e.getKey().getRow().copyBytes());
    b.append(p.getFirst()).append("\\x00").append(p.getSecond());
    b.append(" ");
    b.append(e.getKey().getColumnFamily().toString());
    b.append(":");
    b.append(e.getKey().getColumnQualifier().toString());
    b.append(" ");
    b.append(e.getKey().getColumnVisibility().toString());
    if (super.isDoTimestamps()) {
        b.append(" ");
        b.append(e.getKey().getTimestamp());
    }
    b.append(" ");
    ByteBuffer.wrap(e.getValue().get());
    b.append(ByteBuffer.wrap(e.getValue().get()).getDouble());
    return b.toString();
}
 
Example 30
Project: timely   File: GetMetricTableSplitPoints.java   View source code 6 votes vote down vote up
public static void main(String[] args) throws Exception {

        try (ConfigurableApplicationContext ctx = new SpringApplicationBuilder(SpringBootstrap.class)
                .bannerMode(Mode.OFF).web(false).run(args)) {
            Configuration conf = ctx.getBean(Configuration.class);

            final BaseConfiguration apacheConf = new BaseConfiguration();
            Configuration.Accumulo accumuloConf = conf.getAccumulo();
            apacheConf.setProperty("instance.name", accumuloConf.getInstanceName());
            apacheConf.setProperty("instance.zookeeper.host", accumuloConf.getZookeepers());
            final ClientConfiguration aconf = new ClientConfiguration(Collections.singletonList(apacheConf));
            final Instance instance = new ZooKeeperInstance(aconf);
            Connector con = instance.getConnector(accumuloConf.getUsername(),
                    new PasswordToken(accumuloConf.getPassword()));
            Scanner s = con.createScanner(conf.getMetaTable(),
                    con.securityOperations().getUserAuthorizations(con.whoami()));
            try {
                s.setRange(new Range(Meta.METRIC_PREFIX, true, Meta.TAG_PREFIX, false));
                for (Entry<Key, Value> e : s) {
                    System.out.println(e.getKey().getRow().toString().substring(Meta.METRIC_PREFIX.length()));
                }
            } finally {
                s.close();
            }
        }
    }
 
Example 31
Project: timely   File: MetricAgeOffFilter.java   View source code 6 votes vote down vote up
@Override
public void init(SortedKeyValueIterator<Key, Value> source, Map<String, String> options, IteratorEnvironment env)
        throws IOException {
    super.init(source, options, env);
    validateOptions(options);
    ageoffs = new PatriciaTrie<>();
    options.forEach((k, v) -> {
        if (k.startsWith(AGE_OFF_PREFIX)) {
            String name = k.substring(AGE_OFF_PREFIX.length());
            LOG.trace("Adding {} to Trie with value", name, Long.parseLong(v));
            long ageoff = Long.parseLong(v);
            this.minAgeOff = Math.min(this.minAgeOff, ageoff);
            this.maxAgeOff = Math.max(this.maxAgeOff, ageoff);
            ageoffs.put(name, ageoff);
        }
    });
    defaultAgeOff = ageoffs.get(DEFAULT_AGEOFF_KEY);
    currentTime = System.currentTimeMillis();
}
 
Example 32
Project: timely   File: MetaKeySetTest.java   View source code 6 votes vote down vote up
@Test
public void testContents() {
    Meta one = new Meta("sys.cpu.user", "tag1", "value1");
    Meta two = new Meta("sys.cpu.user", "tag2", "value2");
    Meta three = new Meta("sys.cpu.user", "tag3", "value3");
    MetaKeySet mks = new MetaKeySet();
    mks.addAll(one.toKeys());
    mks.addAll(two.toKeys());
    mks.addAll(three.toKeys());
    Assert.assertEquals(7, mks.size());
    Assert.assertTrue(mks.contains(new Key("m:sys.cpu.user")));
    Assert.assertTrue(mks.contains(new Key("t:sys.cpu.user", "tag1")));
    Assert.assertTrue(mks.contains(new Key("t:sys.cpu.user", "tag2")));
    Assert.assertTrue(mks.contains(new Key("t:sys.cpu.user", "tag3")));
    Assert.assertTrue(mks.contains(new Key("v:sys.cpu.user", "tag1", "value1")));
    Assert.assertTrue(mks.contains(new Key("v:sys.cpu.user", "tag2", "value2")));
    Assert.assertTrue(mks.contains(new Key("v:sys.cpu.user", "tag3", "value3")));
}
 
Example 33
Project: accumulo-wikisearch   File: GlobalIndexUidTest.java   View source code 6 votes vote down vote up
@Test
public void testLessThanMax() throws Exception {
  List<String> savedUUIDs = new ArrayList<String>();
  for (int i = 0; i < GlobalIndexUidCombiner.MAX - 1; i++) {
    Builder b = createNewUidList();
    b.setIGNORE(false);
    String uuid = UUID.randomUUID().toString();
    savedUUIDs.add(uuid);
    b.setCOUNT(i);
    b.addUID(uuid);
    Uid.List uidList = b.build();
    Value val = new Value(uidList.toByteArray());
    values.add(val);
  }
  Value result = combiner.reduce(new Key(), values.iterator());
  Uid.List resultList = Uid.List.parseFrom(result.get());
  assertTrue(resultList.getIGNORE() == false);
  assertTrue(resultList.getUIDCount() == (GlobalIndexUidCombiner.MAX - 1));
  List<String> resultListUUIDs = resultList.getUIDList();
  for (String s : savedUUIDs)
    assertTrue(resultListUUIDs.contains(s));
}
 
Example 34
Project: accumulo-wikisearch   File: AbstractEvaluatingIterator.java   View source code 6 votes vote down vote up
public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
  validateOptions(options);
  event = new EventFields();
  this.comparator = getKeyComparator();
  this.iterator = source;
  try {
    // Replace any expressions that we should not evaluate.
    if (null != this.skipExpressions && this.skipExpressions.size() != 0) {
      for (String skip : this.skipExpressions) {
        // Expression should have form: field<sp>operator<sp>literal.
        // We are going to replace the expression with field == null.
        String field = skip.substring(0, skip.indexOf(" ") - 1);
        this.expression = this.expression.replaceAll(skip, field + " == null");
      }
    }
    this.evaluator = new QueryEvaluator(this.expression);
  } catch (ParseException e) {
    throw new IllegalArgumentException("Failed to parse query", e);
  }
  EventFields.initializeKryo(kryo);
}
 
Example 35
Project: accumulo-examples   File: ChunkInputFormatIT.java   View source code 6 votes vote down vote up
@Test
public void testErrorOnNextWithoutClose() throws Exception {
  conn.tableOperations().create(tableName);
  BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());

  for (Entry<Key,Value> e : data) {
    Key k = e.getKey();
    Mutation m = new Mutation(k.getRow());
    m.put(k.getColumnFamily(), k.getColumnQualifier(), new ColumnVisibility(k.getColumnVisibility()), k.getTimestamp(), e.getValue());
    bw.addMutation(m);
  }
  bw.close();

  assertEquals(1, CIFTester.main(tableName, CIFTester.TestNoClose.class.getName()));
  assertEquals(1, assertionErrors.get(tableName).size());
  // this should actually exist, in addition to the dummy entry
  assertEquals(2, assertionErrors.get(tableName + "_map_ioexception").size());
}
 
Example 36
Project: accumulo-wikisearch   File: BooleanLogicTreeNode.java   View source code 6 votes vote down vote up
public static Key buildKey(Key key) {
  if (key == null) {
    log.error("Problem in BooleanLogicTreeNodeJexl.buildKey");
    return null;
  }
  // Build Key(Text row, Text colfam) where colFam is dataype\0uid
  String[] cq = key.getColumnQualifier().toString().split("\0");
  Text uuid = new Text(cq[cq.length - 2] + "\0" + cq[cq.length - 1]);
  Text row = key.getRow();
  if (log.isDebugEnabled()) {
    log.debug("Key-> r:" + row + "  fam:" + uuid);
  }
  // System.out.println("Key-> r:"+row+"  fam:"+uuid);
  
  Key k = new Key(row, uuid);
  // System.out.println("Key: "+k);
  return k;
}
 
Example 37
Project: OSTMap   File: PathGeoExtrationFlatMap.java   View source code 6 votes vote down vote up
/**
 * Extracts the Coordinates and the timestamp from any given tweet
 *
 * @param in  Entries from the Rawtwittertable
 * @param out extracted data
 */
@Override
public void flatMap(Tuple2<Key, Value> in, Collector<Tuple2<String, String>> out) {
    try {
        JSONObject obj = new JSONObject(in.f1.toString());
        ;
        JSONObject toReturn = null;
        String userName = "";
        Double[] coordinates = Extractor.extractLocation(obj);
        Long timestamp = 0L;

        userName = obj.getJSONObject("user").getString("screen_name");
        timestamp = obj.getLong("timestamp_ms");
        toReturn = new JSONObject();
        toReturn.append("timestamp_ms", timestamp);
        JSONArray coords = new JSONArray();
        coords.put(coordinates[0]);
        coords.put(coordinates[1]);
        toReturn.append("coordinates", coords);
        out.collect(new Tuple2<>(userName, toReturn.toString()));
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 38
Project: OSTMap   File: TimeFilterTest.java   View source code 6 votes vote down vote up
@Test
public void testTimeFilter() throws Exception{
    TimeFilter tfAll = new TimeFilter(Long.MIN_VALUE,Long.MAX_VALUE);
    TimeFilter tf = new TimeFilter(0,1000);
    TimeFilter tfNone = new TimeFilter(5,0);

    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    ByteBuffer bb = ByteBuffer.allocate(Long.BYTES +Integer.BYTES);
    bb.putLong(100).putInt(123);
    Key k1 = new Key(new Text(bb.array()));

    ByteBuffer bb2 = ByteBuffer.allocate(Long.BYTES +Integer.BYTES);
    bb2.putLong(2000).putInt(1234);
    Key k2 = new Key(new Text(bb2.array()));

    DataSet<Tuple2<Key,Value>> data = env.fromElements(new Tuple2<>(k1, new Value()),
            new Tuple2<>(k2, new Value()));

    assertEquals(2,data.count());
    assertEquals(2,data.filter(tfAll).count());
    assertEquals(1,data.filter(tf).count());
    assertEquals(0,data.filter(tfNone).count());

}
 
Example 39
Project: accumulo-examples   File: Reverse.java   View source code 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  Opts opts = new Opts();
  ScannerOpts scanOpts = new ScannerOpts();
  BatchWriterOpts bwOpts = new BatchWriterOpts();
  opts.parseArgs(Reverse.class.getName(), args, scanOpts, bwOpts);

  Connector conn = opts.getConnector();

  Scanner scanner = conn.createScanner(opts.shardTable, opts.auths);
  scanner.setBatchSize(scanOpts.scanBatchSize);
  BatchWriter bw = conn.createBatchWriter(opts.doc2TermTable, bwOpts.getBatchWriterConfig());

  for (Entry<Key,Value> entry : scanner) {
    Key key = entry.getKey();
    Mutation m = new Mutation(key.getColumnQualifier());
    m.put(key.getColumnFamily(), new Text(), new Value(new byte[0]));
    bw.addMutation(m);
  }

  bw.close();

}
 
Example 40
Project: accumulo-examples   File: CountingVerifyingReceiver.java   View source code 6 votes vote down vote up
public void receive(Key key, Value value) {

    String row = key.getRow().toString();
    long rowid = Integer.parseInt(row.split("_")[1]);

    byte expectedValue[] = RandomBatchWriter.createValue(rowid, expectedValueSize);

    if (!Arrays.equals(expectedValue, value.get())) {
      log.error("Got unexpected value for " + key + " expected : " + new String(expectedValue, UTF_8) + " got : " + new String(value.get(), UTF_8));
    }

    if (!expectedRows.containsKey(key.getRow())) {
      log.error("Got unexpected key " + key);
    } else {
      expectedRows.put(key.getRow(), true);
    }

    count++;
  }
 
Example 41
Project: accumulo-wikisearch   File: GlobalIndexUidTest.java   View source code 6 votes vote down vote up
@Test
public void testMoreThanMax() throws Exception {
  List<String> savedUUIDs = new ArrayList<String>();
  for (int i = 0; i < GlobalIndexUidCombiner.MAX + 10; i++) {
    Builder b = createNewUidList();
    b.setIGNORE(false);
    String uuid = UUID.randomUUID().toString();
    savedUUIDs.add(uuid);
    b.setCOUNT(1);
    b.addUID(uuid);
    Uid.List uidList = b.build();
    Value val = new Value(uidList.toByteArray());
    values.add(val);
  }
  Value result = combiner.reduce(new Key(), values.iterator());
  Uid.List resultList = Uid.List.parseFrom(result.get());
  assertTrue(resultList.getIGNORE() == true);
  assertTrue(resultList.getUIDCount() == 0);
  assertTrue(resultList.getCOUNT() == (GlobalIndexUidCombiner.MAX + 10));
}
 
Example 42
Project: PACE   File: SignedReadWriteExample.java   View source code 5 votes vote down vote up
private void execute(Opts opts, ScannerOpts scanOpts) throws Exception {
  conn = opts.getConnector();

  // add the authorizations to the user
  Authorizations userAuthorizations = conn.securityOperations().getUserAuthorizations(opts.getPrincipal());
  ByteArraySet auths = new ByteArraySet(userAuthorizations.getAuthorizations());
  auths.addAll(opts.auths.getAuthorizations());
  if (!auths.isEmpty())
    conn.securityOperations().changeUserAuthorizations(opts.getPrincipal(), new Authorizations(auths));

  // create table
  if (opts.createtable) {
    SortedSet<Text> partitionKeys = new TreeSet<>();
    for (int i = Byte.MIN_VALUE; i < Byte.MAX_VALUE; i++)
      partitionKeys.add(new Text(new byte[] {(byte) i}));
    conn.tableOperations().create(opts.getTableName());
    conn.tableOperations().addSplits(opts.getTableName(), partitionKeys);
  }

  // send mutations
  createEntries(opts);

  // read entries
  if (opts.readEntries) {
    // Note that the user needs to have the authorizations for the specified scan authorizations
    // by an administrator first
    Scanner scanner = new SignedScanner(conn, opts.getTableName(), opts.auths, opts.signatureConfig, opts.signatureKeys);
    scanner.setBatchSize(scanOpts.scanBatchSize);
    for (Entry<Key,Value> entry : scanner)
      System.out.println(entry.getKey().toString() + " -> " + entry.getValue().toString());
  }

  // delete table
  if (opts.deletetable)
    conn.tableOperations().delete(opts.getTableName());
}
 
Example 43
Project: PACE   File: EncryptedReadWriteExample.java   View source code 5 votes vote down vote up
private void execute(Opts opts) throws Exception {
  conn = opts.getConnector();

  // add the authorizations to the user
  Authorizations userAuthorizations = conn.securityOperations().getUserAuthorizations(opts.getPrincipal());
  ByteArraySet auths = new ByteArraySet(userAuthorizations.getAuthorizations());
  auths.addAll(opts.auths.getAuthorizations());
  if (!auths.isEmpty())
    conn.securityOperations().changeUserAuthorizations(opts.getPrincipal(), new Authorizations(auths));

  // create table
  if (opts.createtable) {
    SortedSet<Text> partitionKeys = new TreeSet<>();
    for (int i = Byte.MIN_VALUE; i < Byte.MAX_VALUE; i++)
      partitionKeys.add(new Text(new byte[] {(byte) i}));
    conn.tableOperations().create(opts.getTableName());
    conn.tableOperations().addSplits(opts.getTableName(), partitionKeys);
  }

  // send mutations
  createEntries(opts);

  // read entries
  if (opts.readEntries) {
    // Note that the user needs to have the authorizations for the specified scan authorizations
    // by an administrator first
    BatchScanner scanner = new EncryptedBatchScanner(conn, opts.getTableName(), opts.auths, 1, opts.encryptionConfig, opts.encryptionKeys);
    scanner.setRanges(Collections.singletonList(new Range()));
    for (Entry<Key,Value> entry : scanner)
      System.out.println(entry.getKey().toString() + " -> " + entry.getValue().toString());
  }

  // delete table
  if (opts.deletetable)
    conn.tableOperations().delete(opts.getTableName());
}
 
Example 44
Project: PACE   File: SignatureReadBenchmark.java   View source code 5 votes vote down vote up
/**
 * Benchmark creating the scanner, reading the entries, and closing the scanner.
 */
@Benchmark
public void benchmark() throws Exception {
  Scanner scanner = getSignedScanner(configFile, TEST_TABLE);
  Iterator<Entry<Key,Value>> iterator = scanner.iterator();
  while (iterator.hasNext()) {
    iterator.next();
  }
  scanner.close();
}
 
Example 45
Project: PACE   File: EncryptionReadBenchmark.java   View source code 5 votes vote down vote up
/**
 * Benchmark creating the scanner, reading the entries, and closing the scanner.
 */
@Benchmark
public void benchmark() throws Exception {
  BatchScanner scanner = getEncryptedBatchScanner(configFile, TEST_TABLE);
  Iterator<Entry<Key,Value>> iterator = scanner.iterator();
  while (iterator.hasNext()) {
    iterator.next();
  }
  scanner.close();
}
 
Example 46
Project: PACE   File: GenerateSignatureVersioningFiles.java   View source code 5 votes vote down vote up
/**
 * Generate files for the versioning test.
 *
 * @param config
 *          Configuration file describing how the data is generated.
 * @param destination
 *          Directory data will be placed in.
 * @param entryCount
 *          Number of entries to create in each file.
 * @param entryFieldSize
 *          Size of each field in the generated entries.
 */
private static void generateFiles(File config, File destination, int entryCount, int entryFieldSize) throws Exception {
  String parentDirectory = config.getParent();
  JsonParser parser = new JsonParser();
  JsonObject configJson = parser.parse(new FileReader(config)).getAsJsonObject();

  // Generate the signing keys.
  File keyManifest = Paths.get(parentDirectory, configJson.getAsJsonPrimitive("key-manifest").getAsString()).toFile();
  GenerateKeys.generateKeys(keyManifest, destination, false, true);

  // Generate the data and write it to a file.
  byte[] visibility = new ColumnVisibility(configJson.getAsJsonPrimitive("visibility").getAsString()).getExpression();
  SortedMap<Key,Value> data = GenerateRandomEntries.getRandomData(entryCount, entryFieldSize, visibility);

  LocalFileSystem fs = FileSystem.getLocal(new Configuration());
  RFileWriter writer = RFile.newWriter().to(Paths.get(parentDirectory, configJson.getAsJsonPrimitive("data-table").getAsString()).toString())
      .withFileSystem(fs).build();
  writer.append(data.entrySet());
  writer.close();

  // Generate the files for each configuration.
  for (JsonElement testElem : configJson.getAsJsonArray("tests")) {
    JsonObject test = testElem.getAsJsonObject();
    SignatureConfig signatureConfig = new SignatureConfigBuilder().readFromFile(
        new FileReader(Paths.get(parentDirectory, test.getAsJsonPrimitive("config").getAsString()).toFile())).build();
    SignatureKeyContainer keyContainer = LocalSignatureKeyContainer.read(new FileReader(Paths.get(parentDirectory,
        test.getAsJsonPrimitive("keys").getAsString()).toFile()));

    EntrySigner signer = new EntrySigner(signatureConfig, keyContainer);
    RFileWriter signedWriter = RFile.newWriter().to(Paths.get(parentDirectory, test.getAsJsonPrimitive("table").getAsString()).toString()).withFileSystem(fs)
        .build();

    for (Entry<Key,Value> entry : data.entrySet()) {
      Entry<Key,Value> signedEntry = signer.sign(entry, true);
      signedWriter.append(signedEntry.getKey(), signedEntry.getValue());
    }

    signedWriter.close();
  }
}
 
Example 47
Project: PACE   File: VersioningIT.java   View source code 5 votes vote down vote up
/**
 * Run a specific test for a given version and configuration.
 *
 * @param dataScanner
 *          Scanner for the unsigned data.
 * @param encryptedScanner
 *          Scanner for the encryption data.
 * @param decryptor
 *          Decryptor for the data.
 */
private void runTest(Scanner dataScanner, Scanner encryptedScanner, EntryEncryptor decryptor) {
  // Read the encrypted data into memory.
  SortedMap<Key,Value> decryptedData = new TreeMap<>();
  for (Entry<Key,Value> encryptedEntry : encryptedScanner) {
    Entry<Key,Value> decryptedEntry = decryptor.decrypt(encryptedEntry);
    decryptedData.put(decryptedEntry.getKey(), decryptedEntry.getValue());
  }

  for (Entry<Key,Value> entry : dataScanner) {
    assertThat("there is a matching decrypted value", decryptedData.containsKey(entry.getKey()), is(true));
    assertThat("the value also matches", decryptedData.get(entry.getKey()).get(), equalTo(entry.getValue().get()));
  }
}
 
Example 48
Project: PACE   File: BatchWriteReadIT.java   View source code 5 votes vote down vote up
public void change(EncryptionConfig config) throws Exception {
  Random random = new Random();
  List<Entry<Key,Value>> entries = new ArrayList<>();

  for (int i = 0; i < 1000; i++) {
    byte[] bytes = new byte[32 * 4];
    random.nextBytes(bytes);
    entries.add(new AbstractMap.SimpleImmutableEntry<>(new Key(Arrays.copyOfRange(bytes, 0, 32), Arrays.copyOfRange(bytes, 32, 64), Arrays.copyOfRange(bytes,
        64, 96), "secret".getBytes(VISIBILITY_CHARSET), (long) 0, false), new Value(Arrays.copyOfRange(bytes, 96, 128))));
  }

  // Write the entries to Accumulo.
  BatchWriter writer = AccumuloInstance.getConnector(USER).createBatchWriter(UNENCRYPTED_TEST_TABLE, null);
  BatchWriter encryptedWriter = new EncryptedBatchWriter(AccumuloInstance.getConnector(USER), ENCRYPTED_TEST_TABLE, null, config,
      AccumuloInstance.getUser(USER).encryptionKeys);

  for (Entry<Key,Value> entry : entries) {
    Key key = entry.getKey();
    Mutation mutation = new Mutation(entry.getKey().getRow());

    if (key.isDeleted()) {
      mutation.putDelete(key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibilityParsed(), key.getTimestamp());
    } else {
      mutation.put(key.getColumnFamily(), key.getColumnQualifier(), key.getColumnVisibilityParsed(), key.getTimestamp(), entry.getValue());
    }

    writer.addMutation(mutation);
    encryptedWriter.addMutation(mutation);
  }

  writer.close();
  encryptedWriter.close();
}
 
Example 49
Project: PACE   File: BatchWriteReadIT.java   View source code 5 votes vote down vote up
public void check(EncryptionConfig config) throws Exception {
  Scanner scanner = AccumuloInstance.getConnector(USER).createScanner(UNENCRYPTED_TEST_TABLE, AccumuloInstance.getUser(USER).authorizations);
  EncryptedBatchScanner encryptedScanner = new EncryptedBatchScanner(AccumuloInstance.getConnector(USER), ENCRYPTED_TEST_TABLE,
      AccumuloInstance.getUser(USER).authorizations, 1, config, AccumuloInstance.getUser(USER).encryptionKeys);

  for (Entry<Key,Value> entry : scanner) {
    // Search for the entry.
    encryptedScanner.setRanges(Collections.singletonList(new Range(entry.getKey(), entry.getKey())));
    Iterator<Entry<Key,Value>> iterator = encryptedScanner.iterator();

    assertThat("should have an entry that matches", iterator.hasNext(), is(true));
    assertThat("entries match", iterator.next(), equalTo(entry));
    assertThat("should have no more entries", iterator.hasNext(), is(false));
  }
}
 
Example 50
Project: PACE   File: FilteringIT.java   View source code 5 votes vote down vote up
@Test
public void simpleWriteReadTest() throws Exception {
  // Write the data.
  EncryptedBatchWriter writer = getEncryptedWriter(CHARLIE, "encrypt-value.ini");

  Mutation mutation = new Mutation("hello");
  mutation.put("world", "core", "works!");

  writer.addMutation(mutation);
  writer.flush();
  writer.close();

  // Verify that the data has been encrypted.
  Scanner plaintextScanner = AccumuloInstance.getConnector(ALICE).createScanner(TEST_TABLE, AccumuloInstance.getUser(ALICE).authorizations);

  List<Entry<Key,Value>> entries = ImmutableList.copyOf(plaintextScanner.iterator());
  assertThat("should have one entry", entries, hasSize(1));

  MutableEntry entry = new MutableEntry(entries.get(0));
  assertThat("should have correct row", entry.row, equalTo("hello".getBytes(CHARSET)));
  assertThat("should have correct row", entry.colF, equalTo("world".getBytes(CHARSET)));
  assertThat("should have correct row", entry.colQ, equalTo("core".getBytes(CHARSET)));
  assertThat("should have correct row", entry.value, not(equalTo("works!".getBytes(CHARSET))));

  // Read the encrypted data.
  EncryptedBatchScanner scanner = getEncryptedScanner(ALICE, "encrypt-value.ini");
  scanner.setRanges(Collections.singletonList(new Range("hello")));

  entries = ImmutableList.copyOf(scanner.iterator());
  assertThat("should have one entry", entries, hasSize(1));
  assertThat("should have correct row", entries.get(0), equalToRow("hello", "world", "core", "", "works!"));
}