Java Code Examples for org.apache.hadoop.hbase.client.Put#setDurability()

The following examples show how to use org.apache.hadoop.hbase.client.Put#setDurability() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private Put transactionalizeAction(Put put) throws IOException {
  Put txPut = new Put(put.getRow(), tx.getWritePointer());
  Set<Map.Entry<byte[], List<Cell>>> familyMap = put.getFamilyCellMap().entrySet();
  if (!familyMap.isEmpty()) {
    for (Map.Entry<byte[], List<Cell>> family : familyMap) {
      List<Cell> familyValues = family.getValue();
      if (!familyValues.isEmpty()) {
        for (Cell value : familyValues) {
          txPut.add(value.getFamily(), value.getQualifier(), tx.getWritePointer(), value.getValue());
          addToChangeSet(txPut.getRow(), value.getFamily(), value.getQualifier());
        }
      }
    }
  }
  for (Map.Entry<String, byte[]> entry : put.getAttributesMap().entrySet()) {
    txPut.setAttribute(entry.getKey(), entry.getValue());
  }
  txPut.setDurability(put.getDurability());
  addToOperation(txPut, tx);
  return txPut;
}
 
Example 2
Source Project: hbase   File: TestScannersWithLabels.java    License: Apache License 2.0 6 votes vote down vote up
private static int insertData(TableName tableName, String column, double prob)
    throws IOException {
  byte[] k = new byte[3];
  byte[][] famAndQf = CellUtil.parseColumn(Bytes.toBytes(column));

  List<Put> puts = new ArrayList<>(9);
  for (int i = 0; i < 9; i++) {
    Put put = new Put(Bytes.toBytes("row" + i));
    put.setDurability(Durability.SKIP_WAL);
    put.addColumn(famAndQf[0], famAndQf[1], k);
    put.setCellVisibility(new CellVisibility("(" + SECRET + "|" + CONFIDENTIAL + ")" + "&" + "!"
        + TOPSECRET));
    puts.add(put);
  }
  try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
    table.put(puts);
  }
  return puts.size();
}
 
Example 3
private Put transactionalizeAction(Put put) throws IOException {
  Put txPut = new Put(put.getRow(), tx.getWritePointer());
  Set<Map.Entry<byte[], List<Cell>>> familyMap = put.getFamilyCellMap().entrySet();
  if (!familyMap.isEmpty()) {
    for (Map.Entry<byte[], List<Cell>> family : familyMap) {
      List<Cell> familyValues = family.getValue();
      if (!familyValues.isEmpty()) {
        for (Cell value : familyValues) {
          txPut.add(value.getFamily(), value.getQualifier(), tx.getWritePointer(), value.getValue());
          addToChangeSet(txPut.getRow(), value.getFamily(), value.getQualifier());
        }
      }
    }
  }
  for (Map.Entry<String, byte[]> entry : put.getAttributesMap().entrySet()) {
    txPut.setAttribute(entry.getKey(), entry.getValue());
  }
  txPut.setDurability(put.getDurability());
  addToOperation(txPut, tx);
  return txPut;
}
 
Example 4
private Put transactionalizeAction(Put put) throws IOException {
  Put txPut = new Put(put.getRow(), tx.getWritePointer());
  Set<Map.Entry<byte[], List<Cell>>> familyMap = put.getFamilyCellMap().entrySet();
  if (!familyMap.isEmpty()) {
    for (Map.Entry<byte[], List<Cell>> family : familyMap) {
      List<Cell> familyValues = family.getValue();
      if (!familyValues.isEmpty()) {
        for (Cell value : familyValues) {
          txPut.add(value.getFamily(), value.getQualifier(), tx.getWritePointer(), value.getValue());
          addToChangeSet(txPut.getRow(), value.getFamily(), value.getQualifier());
        }
      }
    }
  }
  for (Map.Entry<String, byte[]> entry : put.getAttributesMap().entrySet()) {
    txPut.setAttribute(entry.getKey(), entry.getValue());
  }
  txPut.setDurability(put.getDurability());
  addToOperation(txPut, tx);
  return txPut;
}
 
Example 5
Source Project: hbase   File: PerformanceEvaluation.java    License: Apache License 2.0 6 votes vote down vote up
@Override
void testRow(final int i) throws IOException {
  byte[] row = format(i);
  Put put = new Put(row);
  byte[] value = generateData(this.rand, ROW_LENGTH);
  if (useTags) {
    byte[] tag = generateData(this.rand, TAG_LENGTH);
    Tag[] tags = new Tag[noOfTags];
    for (int n = 0; n < noOfTags; n++) {
      Tag t = new ArrayBackedTag((byte) n, tag);
      tags[n] = t;
    }
    KeyValue kv = new KeyValue(row, FAMILY_NAME, QUALIFIER_NAME, HConstants.LATEST_TIMESTAMP,
        value, tags);
    put.add(kv);
  } else {
    put.addColumn(FAMILY_NAME, QUALIFIER_NAME, value);
  }
  put.setDurability(writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL);
  mutator.mutate(put);
}
 
Example 6
Source Project: hbase   File: HBaseTestingUtility.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Load table of multiple column families with rows from 'aaa' to 'zzz'.
 * @param t Table
 * @param f Array of Families to load
 * @param value the values of the cells. If null is passed, the row key is used as value
 * @return Count of rows loaded.
 * @throws IOException
 */
public int loadTable(final Table t, final byte[][] f, byte[] value,
    boolean writeToWAL) throws IOException {
  List<Put> puts = new ArrayList<>();
  for (byte[] row : HBaseTestingUtility.ROWS) {
    Put put = new Put(row);
    put.setDurability(writeToWAL ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
    for (int i = 0; i < f.length; i++) {
      byte[] value1 = value != null ? value : row;
      put.addColumn(f[i], f[i], value1);
    }
    puts.add(put);
  }
  t.put(puts);
  return puts.size();
}
 
Example 7
public void loadTable(final Table table, int numRows) throws IOException {
  List<Put> puts = new ArrayList<>(numRows);
  for (int i = 0; i < numRows; ++i) {
    byte[] row = Bytes.toBytes(String.format("%09d", i));
    Put put = new Put(row);
    put.setDurability(Durability.SKIP_WAL);
    put.addColumn(FAMILY_NAME, null, row);
    table.put(put);
  }
}
 
Example 8
Source Project: hbase   File: TestScannerResource.java    License: Apache License 2.0 5 votes vote down vote up
static int insertData(Configuration conf, TableName tableName, String column, double prob)
    throws IOException {
  Random rng = new Random();
  byte[] k = new byte[3];
  byte [][] famAndQf = CellUtil.parseColumn(Bytes.toBytes(column));
  List<Put> puts = new ArrayList<>();
  for (byte b1 = 'a'; b1 < 'z'; b1++) {
    for (byte b2 = 'a'; b2 < 'z'; b2++) {
      for (byte b3 = 'a'; b3 < 'z'; b3++) {
        if (rng.nextDouble() < prob) {
          k[0] = b1;
          k[1] = b2;
          k[2] = b3;
          Put put = new Put(k);
          put.setDurability(Durability.SKIP_WAL);
          put.addColumn(famAndQf[0], famAndQf[1], k);
          puts.add(put);
        }
      }
    }
  }
  try (Connection conn = ConnectionFactory.createConnection(conf);
      Table table = conn.getTable(tableName)) {
    table.put(puts);
  }
  return puts.size();
}
 
Example 9
private void loadTable(Table table, int numRows) throws IOException {
  for (int i = 0; i < numRows; ++i) {
    byte[] row = Bytes.toBytes(i);
    Put put = new Put(row);
    put.setDurability(Durability.SKIP_WAL);
    put.addColumn(FAMILY_NAME, null, row);
    table.put(put);
  }
}
 
Example 10
private void generateRows(int numberOfRows, Table ht, byte[] family, byte[] qf, byte[] value)
    throws IOException {
  for (int i = 0; i < numberOfRows; i++) {
    byte[] row = Bytes.toBytes(i);
    Put p = new Put(row);
    p.addColumn(family, qf, value);
    p.setDurability(Durability.SKIP_WAL);
    ht.put(p);
  }
  TEST_UTIL.flush();
}
 
Example 11
Source Project: hbase   File: HBaseTestingUtility.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Load region with rows from 'aaa' to 'zzz'.
 * @param r Region
 * @param f Family
 * @param flush flush the cache if true
 * @return Count of rows loaded.
 * @throws IOException
 */
public int loadRegion(final HRegion r, final byte[] f, final boolean flush)
throws IOException {
  byte[] k = new byte[3];
  int rowCount = 0;
  for (byte b1 = 'a'; b1 <= 'z'; b1++) {
    for (byte b2 = 'a'; b2 <= 'z'; b2++) {
      for (byte b3 = 'a'; b3 <= 'z'; b3++) {
        k[0] = b1;
        k[1] = b2;
        k[2] = b3;
        Put put = new Put(k);
        put.setDurability(Durability.SKIP_WAL);
        put.addColumn(f, null, k);
        if (r.getWAL() == null) {
          put.setDurability(Durability.SKIP_WAL);
        }
        int preRowCount = rowCount;
        int pause = 10;
        int maxPause = 1000;
        while (rowCount == preRowCount) {
          try {
            r.put(put);
            rowCount++;
          } catch (RegionTooBusyException e) {
            pause = (pause * 2 >= maxPause) ? maxPause : pause * 2;
            Threads.sleep(pause);
          }
        }
      }
    }
    if (flush) {
      r.flush(true);
    }
  }
  return rowCount;
}
 
Example 12
private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) {
  byte[] q = Bytes.toBytes("q");
  Put put = new Put(key);
  put.setDurability(Durability.SKIP_WAL);
  for (byte[] family: families) {
    put.addColumn(family, q, value);
  }
  return put;
}
 
Example 13
Source Project: hbase   File: TestHRegionReplayEvents.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSeqIdsFromReplay() throws IOException {
  // test the case where seqId's coming from replayed WALEdits are made persisted with their
  // original seqIds and they are made visible through mvcc read point upon replay
  String method = name.getMethodName();
  byte[] tableName = Bytes.toBytes(method);
  byte[] family = Bytes.toBytes("family");

  HRegion region = initHRegion(tableName, family);
  try {
    // replay an entry that is bigger than current read point
    long readPoint = region.getMVCC().getReadPoint();
    long origSeqId = readPoint + 100;

    Put put = new Put(row).addColumn(family, row, row);
    put.setDurability(Durability.SKIP_WAL); // we replay with skip wal
    replay(region, put, origSeqId);

    // read point should have advanced to this seqId
    assertGet(region, family, row);

    // region seqId should have advanced at least to this seqId
    assertEquals(origSeqId, region.getReadPoint(null));

    // replay an entry that is smaller than current read point
    // caution: adding an entry below current read point might cause partial dirty reads. Normal
    // replay does not allow reads while replay is going on.
    put = new Put(row2).addColumn(family, row2, row2);
    put.setDurability(Durability.SKIP_WAL);
    replay(region, put, origSeqId - 50);

    assertGet(region, family, row2);
  } finally {
    region.close();
  }
}
 
Example 14
Source Project: hbase   File: TestBlocksRead.java    License: Apache License 2.0 5 votes vote down vote up
private void putData(byte[] cf, String row, String col, long versionStart,
    long versionEnd) throws IOException {
  byte [] columnBytes = Bytes.toBytes(col);
  Put put = new Put(Bytes.toBytes(row));
  put.setDurability(Durability.SKIP_WAL);

  for (long version = versionStart; version <= versionEnd; version++) {
    put.addColumn(cf, columnBytes, version, genValue(row, col, version));
  }
  region.put(put);
}
 
Example 15
@Test
public void testMultipleColumnPrefixFilterWithColumnPrefixFilter() throws IOException {
  String family = "Family";
  TableDescriptorBuilder tableDescriptorBuilder =
    TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
  ColumnFamilyDescriptor columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
  RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.
    getDataTestDir(), TEST_UTIL.getConfiguration(), tableDescriptor);

  List<String> rows = generateRandomWords(100, "row");
  List<String> columns = generateRandomWords(10000, "column");
  long maxTimestamp = 2;

  String valueString = "ValueString";

  for (String row: rows) {
    Put p = new Put(Bytes.toBytes(row));
    p.setDurability(Durability.SKIP_WAL);
    for (String column: columns) {
      for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
        KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
            valueString);
        p.add(kv);
      }
    }
    region.put(p);
  }

  MultipleColumnPrefixFilter multiplePrefixFilter;
  Scan scan1 = new Scan();
  scan1.readAllVersions();
  byte [][] filter_prefix = new byte [1][];
  filter_prefix[0] = new byte [] {'p'};

  multiplePrefixFilter = new MultipleColumnPrefixFilter(filter_prefix);
  scan1.setFilter(multiplePrefixFilter);
  List<Cell> results1 = new ArrayList<>();
  InternalScanner scanner1 = region.getScanner(scan1);
  while (scanner1.next(results1))
    ;

  ColumnPrefixFilter singlePrefixFilter;
  Scan scan2 = new Scan();
  scan2.readAllVersions();
  singlePrefixFilter = new ColumnPrefixFilter(Bytes.toBytes("p"));

  scan2.setFilter(singlePrefixFilter);
  List<Cell> results2 = new ArrayList<>();
  InternalScanner scanner2 = region.getScanner(scan1);
  while (scanner2.next(results2))
    ;

  assertEquals(results1.size(), results2.size());

  HBaseTestingUtility.closeRegionAndWAL(region);
}
 
Example 16
Source Project: hbase   File: TestMutateRowsRecovery.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void MutateRowsAndCheckPostKill() throws IOException, InterruptedException {
  final TableName tableName = TableName.valueOf("test");
  Admin admin = null;
  Table hTable = null;
  try {
    admin = connection.getAdmin();
    hTable = connection.getTable(tableName);
    TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
      new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
    tableDescriptor.setColumnFamily(
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam1));
    admin.createTable(tableDescriptor);

    // Add a multi
    RowMutations rm = new RowMutations(row1);
    Put p1 = new Put(row1);
    p1.addColumn(fam1, qual1, value1);
    p1.setDurability(Durability.SYNC_WAL);
    rm.add(p1);
    hTable.mutateRow(rm);

    // Add a put
    Put p2 = new Put(row1);
    p2.addColumn(fam1, qual2, value2);
    p2.setDurability(Durability.SYNC_WAL);
    hTable.put(p2);

    HRegionServer rs1 = TESTING_UTIL.getRSForFirstRegionInTable(tableName);
    long now = EnvironmentEdgeManager.currentTime();
    // Send the RS Load to ensure correct lastflushedseqid for stores
    rs1.tryRegionServerReport(now - 30000, now);
    // Kill the RS to trigger wal replay
    cluster.killRegionServer(rs1.serverName);

    // Ensure correct data exists
    Get g1 = new Get(row1);
    Result result = hTable.get(g1);
    assertTrue(result.getValue(fam1, qual1) != null);
    assertEquals(0, Bytes.compareTo(result.getValue(fam1, qual1), value1));
    assertTrue(result.getValue(fam1, qual2) != null);
    assertEquals(0, Bytes.compareTo(result.getValue(fam1, qual2), value2));
  } finally {
    if (admin != null) {
      admin.close();
    }
    if (hTable != null) {
      hTable.close();
    }
  }
}
 
Example 17
@Test
public void testMultipleColumnPrefixFilter() throws IOException {
  String family = "Family";
  TableDescriptorBuilder tableDescriptorBuilder =
    TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
  ColumnFamilyDescriptor columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder
      .newBuilder(Bytes.toBytes(family))
      .setMaxVersions(3)
      .build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
  // HRegionInfo info = new HRegionInfo(htd, null, null, false);
  RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  HRegion region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.
      getDataTestDir(), TEST_UTIL.getConfiguration(), tableDescriptor);

  List<String> rows = generateRandomWords(100, "row");
  List<String> columns = generateRandomWords(10000, "column");
  long maxTimestamp = 2;

  List<Cell> kvList = new ArrayList<>();

  Map<String, List<Cell>> prefixMap = new HashMap<>();

  prefixMap.put("p", new ArrayList<>());
  prefixMap.put("q", new ArrayList<>());
  prefixMap.put("s", new ArrayList<>());

  String valueString = "ValueString";

  for (String row: rows) {
    Put p = new Put(Bytes.toBytes(row));
    p.setDurability(Durability.SKIP_WAL);
    for (String column: columns) {
      for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
        KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
            valueString);
        p.add(kv);
        kvList.add(kv);
        for (String s: prefixMap.keySet()) {
          if (column.startsWith(s)) {
            prefixMap.get(s).add(kv);
          }
        }
      }
    }
    region.put(p);
  }

  MultipleColumnPrefixFilter filter;
  Scan scan = new Scan();
  scan.readAllVersions();
  byte [][] filter_prefix = new byte [2][];
  filter_prefix[0] = new byte [] {'p'};
  filter_prefix[1] = new byte [] {'q'};

  filter = new MultipleColumnPrefixFilter(filter_prefix);
  scan.setFilter(filter);
  List<Cell> results = new ArrayList<>();
  InternalScanner scanner = region.getScanner(scan);
  while (scanner.next(results))
    ;
  assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());

  HBaseTestingUtility.closeRegionAndWAL(region);
}
 
Example 18
Source Project: hbase   File: TestFilter.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testColumnPaginationFilter() throws Exception {
  // Test that the filter skips multiple column versions.
  Put p = new Put(ROWS_ONE[0]);
  p.setDurability(Durability.SKIP_WAL);
  p.addColumn(FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]);
  this.region.put(p);
    this.region.flush(true);

    // Set of KVs (page: 1; pageSize: 1) - the first set of 1 column per row
    KeyValue [] expectedKVs = {
      // testRowOne-0
      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
      // testRowOne-2
      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
      // testRowOne-3
      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
      // testRowTwo-0
      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
      // testRowTwo-2
      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
      // testRowTwo-3
      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1])
    };

    // Set of KVs (page: 3; pageSize: 1)  - the third set of 1 column per row
    KeyValue [] expectedKVs2 = {
      // testRowOne-0
      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
      // testRowOne-2
      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
      // testRowOne-3
      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
      // testRowTwo-0
      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
      // testRowTwo-2
      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
      // testRowTwo-3
      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
    };

    // Set of KVs (page: 2; pageSize 2)  - the 2nd set of 2 columns per row
    KeyValue [] expectedKVs3 = {
      // testRowOne-0
      new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
      new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
      // testRowOne-2
      new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
      new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
      // testRowOne-3
      new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
      new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
      // testRowTwo-0
      new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
      new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
      // testRowTwo-2
      new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
      new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
      // testRowTwo-3
      new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
      new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
    };


    // Set of KVs (page: 2; pageSize 2)  - the 2nd set of 2 columns per row
    KeyValue [] expectedKVs4 = {

    };

    long expectedRows = this.numRows;
    long expectedKeys = 1;
    Scan s = new Scan();


    // Page 1; 1 Column per page  (Limit 1, Offset 0)
    s.setFilter(new ColumnPaginationFilter(1,0));
    verifyScan(s, expectedRows, expectedKeys);
    this.verifyScanFull(s, expectedKVs);

    // Page 3; 1 Result per page  (Limit 1, Offset 2)
    s.setFilter(new ColumnPaginationFilter(1,2));
    verifyScan(s, expectedRows, expectedKeys);
    this.verifyScanFull(s, expectedKVs2);

    // Page 2; 2 Results per page (Limit 2, Offset 2)
    s.setFilter(new ColumnPaginationFilter(2,2));
    expectedKeys = 2;
    verifyScan(s, expectedRows, expectedKeys);
    this.verifyScanFull(s, expectedKVs3);

    // Page 8; 20 Results per page (no results) (Limit 20, Offset 140)
    s.setFilter(new ColumnPaginationFilter(20,140));
    expectedKeys = 0;
    expectedRows = 0;
    verifyScan(s, expectedRows, 0);
    this.verifyScanFull(s, expectedKVs4);
}
 
Example 19
Source Project: hbase   File: TestCompaction.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Verify that you can stop a long-running compaction
 * (used during RS shutdown)
 * @throws Exception
 */
@Test
public void testInterruptCompactionBySize() throws Exception {
  assertEquals(0, count());

  // lower the polling interval for this test
  conf.setInt(SIZE_LIMIT_KEY, 10 * 1000 /* 10 KB */);

  try {
    // Create a couple store files w/ 15KB (over 10KB interval)
    int jmax = (int) Math.ceil(15.0/compactionThreshold);
    byte [] pad = new byte[1000]; // 1 KB chunk
    for (int i = 0; i < compactionThreshold; i++) {
      Table loader = new RegionAsTable(r);
      Put p = new Put(Bytes.add(STARTROW, Bytes.toBytes(i)));
      p.setDurability(Durability.SKIP_WAL);
      for (int j = 0; j < jmax; j++) {
        p.addColumn(COLUMN_FAMILY, Bytes.toBytes(j), pad);
      }
      HTestConst.addContent(loader, Bytes.toString(COLUMN_FAMILY));
      loader.put(p);
      r.flush(true);
    }

    HRegion spyR = spy(r);
    doAnswer(new Answer() {
      @Override
      public Object answer(InvocationOnMock invocation) throws Throwable {
        r.writestate.writesEnabled = false;
        return invocation.callRealMethod();
      }
    }).when(spyR).doRegionCompactionPrep();

    // force a minor compaction, but not before requesting a stop
    spyR.compactStores();

    // ensure that the compaction stopped, all old files are intact,
    HStore s = r.getStore(COLUMN_FAMILY);
    assertEquals(compactionThreshold, s.getStorefilesCount());
    assertTrue(s.getStorefilesSize() > 15*1000);
    // and no new store files persisted past compactStores()
    // only one empty dir exists in temp dir
    FileStatus[] ls = r.getFilesystem().listStatus(r.getRegionFileSystem().getTempDir());
    assertEquals(1, ls.length);
    Path storeTempDir = new Path(r.getRegionFileSystem().getTempDir(), Bytes.toString(COLUMN_FAMILY));
    assertTrue(r.getFilesystem().exists(storeTempDir));
    ls = r.getFilesystem().listStatus(storeTempDir);
    assertEquals(0, ls.length);
  } finally {
    // don't mess up future tests
    r.writestate.writesEnabled = true;
    conf.setInt(SIZE_LIMIT_KEY, 10 * 1000 * 1000 /* 10 MB */);

    // Delete all Store information once done using
    for (int i = 0; i < compactionThreshold; i++) {
      Delete delete = new Delete(Bytes.add(STARTROW, Bytes.toBytes(i)));
      byte [][] famAndQf = {COLUMN_FAMILY, null};
      delete.addFamily(famAndQf[0]);
      r.delete(delete);
    }
    r.flush(true);

    // Multiple versions allowed for an entry, so the delete isn't enough
    // Lower TTL and expire to ensure that all our entries have been wiped
    final int ttl = 1000;
    for (HStore store : this.r.stores.values()) {
      ScanInfo old = store.getScanInfo();
      ScanInfo si = old.customize(old.getMaxVersions(), ttl, old.getKeepDeletedCells());
      store.setScanInfo(si);
    }
    Thread.sleep(ttl);

    r.compact(true);
    assertEquals(0, count());
  }
}
 
Example 20
Source Project: metron   File: HBaseClient.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Creates an HBase Put.
 *
 * @param rowKey     The row key.
 * @param cols       The columns to put.
 * @param durability The durability of the put.
 */
private Put createPut(byte[] rowKey, ColumnList cols, Durability durability) {
  Put put = new Put(rowKey);
  put.setDurability(durability);
  addColumns(cols, put);
  return put;
}