Java Code Examples for org.apache.hadoop.hbase.client.Put.setDurability()

The following are Jave code examples for showing how to use setDurability() of the org.apache.hadoop.hbase.client.Put class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
Example 1
Project: ditb   File: TestTimeRangeMapRed.java   Source Code and License Vote up 8 votes
@Override
public void map(ImmutableBytesWritable key, Result result,
    Context context)
throws IOException {
  List<Long> tsList = new ArrayList<Long>();
  for (Cell kv : result.listCells()) {
    tsList.add(kv.getTimestamp());
  }

  List<Put> puts = new ArrayList<>();
  for (Long ts : tsList) {
    Put put = new Put(key.get());
    put.setDurability(Durability.SKIP_WAL);
    put.add(FAMILY_NAME, COLUMN_NAME, ts, Bytes.toBytes(true));
    puts.add(put);
  }
  table.put(puts);
}
 
Example 2
Project: ditb   File: MultiTableOutputFormat.java   Source Code and License Vote up 6 votes
/**
 * Writes an action (Put or Delete) to the specified table.
 *
 * @param tableName
 *          the table being updated.
 * @param action
 *          the update, either a put or a delete.
 * @throws IllegalArgumentException
 *          if the action is not a put or a delete.
 */
@Override
public void write(ImmutableBytesWritable tableName, Mutation action) throws IOException {
  BufferedMutator mutator = getBufferedMutator(tableName);
  // The actions are not immutable, so we defensively copy them
  if (action instanceof Put) {
    Put put = new Put((Put) action);
    put.setDurability(useWriteAheadLogging ? Durability.SYNC_WAL
        : Durability.SKIP_WAL);
    mutator.mutate(put);
  } else if (action instanceof Delete) {
    Delete delete = new Delete((Delete) action);
    mutator.mutate(delete);
  } else
    throw new IllegalArgumentException(
        "action must be either Delete or Put");
}
 
Example 3
Project: ditb   File: PerformanceEvaluation.java   Source Code and License Vote up 6 votes
@Override
void testRow(final int i) throws IOException {
  byte[] row = getRandomRow(this.rand, opts.totalRows);
  Put put = new Put(row);
  for (int column = 0; column < opts.columns; column++) {
    byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
    byte[] value = generateData(this.rand, getValueLength(this.rand));
    if (opts.useTags) {
      byte[] tag = generateData(this.rand, TAG_LENGTH);
      Tag[] tags = new Tag[opts.noOfTags];
      for (int n = 0; n < opts.noOfTags; n++) {
        Tag t = new Tag((byte) n, tag);
        tags[n] = t;
      }
      KeyValue kv = new KeyValue(row, FAMILY_NAME, qualifier, HConstants.LATEST_TIMESTAMP,
          value, tags);
      put.add(kv);
      updateValueSize(kv.getValueLength());
    } else {
      put.add(FAMILY_NAME, qualifier, value);
      updateValueSize(value.length);
    }
  }
  put.setDurability(opts.writeToWAL ? Durability.SYNC_WAL : Durability.SKIP_WAL);
  mutator.mutate(put);
}
 
Example 4
Project: ditb   File: TestTimeRangeMapRed.java   Source Code and License Vote up 6 votes
@Test
public void testTimeRangeMapRed()
throws IOException, InterruptedException, ClassNotFoundException {
  final HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
  final HColumnDescriptor col = new HColumnDescriptor(FAMILY_NAME);
  col.setMaxVersions(Integer.MAX_VALUE);
  desc.addFamily(col);
  admin.createTable(desc);
  List<Put> puts = new ArrayList<Put>();
  for (Map.Entry<Long, Boolean> entry : TIMESTAMP.entrySet()) {
    Put put = new Put(KEY);
    put.setDurability(Durability.SKIP_WAL);
    put.add(FAMILY_NAME, COLUMN_NAME, entry.getKey(), Bytes.toBytes(false));
    puts.add(put);
  }
  Table table = new HTable(UTIL.getConfiguration(), desc.getTableName());
  table.put(puts);
  runTestOnTable();
  verify(table);
  table.close();
}
 
Example 5
Project: ditb   File: TestHRegion.java   Source Code and License Vote up 5 votes
static void putData(HRegion region, Durability durability,
    int startRow, int numRows, byte[] qf, byte[]... families) throws IOException {
  for (int i = startRow; i < startRow + numRows; i++) {
    Put put = new Put(Bytes.toBytes("" + i));
    put.setDurability(durability);
    for (byte[] family : families) {
      put.add(family, qf, null);
    }
    region.put(put);
  }
}
 
Example 6
Project: ditb   File: TestMetaMigrationConvertingToPB.java   Source Code and License Vote up 5 votes
/**
 * Inserts multiple regions into hbase:meta using Writable serialization instead of PB
 */
public int createMultiRegionsWithWritableSerialization(final Configuration c,
    final TableName tableName, byte [][] startKeys)
throws IOException {
  Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
  HTable meta = new HTable(c, TableName.META_TABLE_NAME);

  List<HRegionInfo> newRegions
      = new ArrayList<HRegionInfo>(startKeys.length);
  int count = 0;
  for (int i = 0; i < startKeys.length; i++) {
    int j = (i + 1) % startKeys.length;
    HRegionInfo hri = new HRegionInfo(tableName, startKeys[i], startKeys[j]);
    Put put = new Put(hri.getRegionName());
    put.setDurability(Durability.SKIP_WAL);
    put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
      getBytes(hri)); //this is the old Writable serialization

    //also add the region as it's daughters
    put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER,
        getBytes(hri)); //this is the old Writable serialization

    put.add(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER,
        getBytes(hri)); //this is the old Writable serialization

    meta.put(put);
    LOG.info("createMultiRegionsWithWritableSerialization: PUT inserted " + hri.toString());

    newRegions.add(hri);
    count++;
  }
  meta.close();
  return count;
}
 
Example 7
Project: ditb   File: TestRegionMergeTransaction.java   Source Code and License Vote up 5 votes
/**
 * Load region with rows from 'aaa' to 'zzz', skip the rows which are out of
 * range of the region
 * @param r Region
 * @param f Family
 * @param flush flush the cache if true
 * @return Count of rows loaded.
 * @throws IOException
 */
private int loadRegion(final HRegion r, final byte[] f, final boolean flush)
    throws IOException {
  byte[] k = new byte[3];
  int rowCount = 0;
  for (byte b1 = 'a'; b1 <= 'z'; b1++) {
    for (byte b2 = 'a'; b2 <= 'z'; b2++) {
      for (byte b3 = 'a'; b3 <= 'z'; b3++) {
        k[0] = b1;
        k[1] = b2;
        k[2] = b3;
        if (!HRegion.rowIsInRange(r.getRegionInfo(), k)) {
          continue;
        }
        Put put = new Put(k);
        put.add(f, null, k);
        if (r.getWAL() == null)
          put.setDurability(Durability.SKIP_WAL);
        r.put(put);
        rowCount++;
      }
    }
    if (flush) {
      r.flush(true);
    }
  }
  return rowCount;
}
 
Example 8
Project: ditb   File: HBaseTestingUtility.java   Source Code and License Vote up 5 votes
/**
 * Load region with rows from 'aaa' to 'zzz'.
 * @param r Region
 * @param f Family
 * @param flush flush the cache if true
 * @return Count of rows loaded.
 * @throws IOException
 */
public int loadRegion(final HRegion r, final byte[] f, final boolean flush)
throws IOException {
  byte[] k = new byte[3];
  int rowCount = 0;
  for (byte b1 = 'a'; b1 <= 'z'; b1++) {
    for (byte b2 = 'a'; b2 <= 'z'; b2++) {
      for (byte b3 = 'a'; b3 <= 'z'; b3++) {
        k[0] = b1;
        k[1] = b2;
        k[2] = b3;
        Put put = new Put(k);
        put.setDurability(Durability.SKIP_WAL);
        put.add(f, null, k);
        if (r.getWAL() == null) {
          put.setDurability(Durability.SKIP_WAL);
        }
        int preRowCount = rowCount;
        int pause = 10;
        int maxPause = 1000;
        while (rowCount == preRowCount) {
          try {
            r.put(put);
            rowCount++;
          } catch (RegionTooBusyException e) {
            pause = (pause * 2 >= maxPause) ? maxPause : pause * 2;
            Threads.sleep(pause);
          }
        }
      }
    }
    if (flush) {
      r.flush(true);
    }
  }
  return rowCount;
}
 
Example 9
Project: ditb   File: TestEncodedSeekers.java   Source Code and License Vote up 5 votes
private void doPuts(Region region) throws IOException{
  LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(MIN_VALUE_SIZE, MAX_VALUE_SIZE);
   for (int i = 0; i < NUM_ROWS; ++i) {
    byte[] key = LoadTestKVGenerator.md5PrefixedKey(i).getBytes();
    for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
      Put put = new Put(key);
      put.setDurability(Durability.ASYNC_WAL);
      byte[] col = Bytes.toBytes(String.valueOf(j));
      byte[] value = dataGenerator.generateRandomSizeValue(key, col);
      if (includeTags) {
        Tag[] tag = new Tag[1];
        tag[0] = new Tag((byte) 1, "Visibility");
        KeyValue kv = new KeyValue(key, CF_BYTES, col, HConstants.LATEST_TIMESTAMP, value, tag);
        put.add(kv);
      } else {
        put.add(CF_BYTES, col, value);
      }
      if(VERBOSE){
        KeyValue kvPut = new KeyValue(key, CF_BYTES, col, value);
        System.err.println(Strings.padFront(i+"", ' ', 4)+" "+kvPut);
      }
      region.put(put);
    }
    if (i % NUM_ROWS_PER_FLUSH == 0) {
      region.flush(true);
    }
  }
}
 
Example 10
Project: ditb   File: TestMasterTransitions.java   Source Code and License Vote up 5 votes
private static int addToEachStartKey(final int expected) throws IOException {
  Table t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
  Table meta = new HTable(TEST_UTIL.getConfiguration(),
      TableName.META_TABLE_NAME);
  int rows = 0;
  Scan scan = new Scan();
  scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
  ResultScanner s = meta.getScanner(scan);
  for (Result r = null; (r = s.next()) != null;) {
    HRegionInfo hri = HRegionInfo.getHRegionInfo(r);
    if (hri == null) break;
    if (!hri.getTable().equals(TABLENAME)) {
      continue;
    }

    // If start key, add 'aaa'.
    if(!hri.getTable().equals(TABLENAME)) {
      continue;
    }
    byte [] row = getStartKey(hri);
    Put p = new Put(row);
    p.setDurability(Durability.SKIP_WAL);
    p.add(getTestFamily(), getTestQualifier(), row);
    t.put(p);
    rows++;
  }
  s.close();
  Assert.assertEquals(expected, rows);
  t.close();
  meta.close();
  return rows;
}
 
Example 11
Project: ditb   File: TsvImporterCustomTestMapper.java   Source Code and License Vote up 5 votes
/**
 * Convert a line of TSV text into an HBase table row after transforming the
 * values by multiplying them by 3.
 */
@Override
public void map(LongWritable offset, Text value, Context context)
      throws IOException {
  byte[] family = Bytes.toBytes("FAM");
  final byte[][] qualifiers = { Bytes.toBytes("A"), Bytes.toBytes("B") };

  // do some basic line parsing
  byte[] lineBytes = value.getBytes();
  String[] valueTokens = new String(lineBytes, "UTF-8").split("\u001b");

  // create the rowKey and Put
  ImmutableBytesWritable rowKey =
    new ImmutableBytesWritable(Bytes.toBytes(valueTokens[0]));
  Put put = new Put(rowKey.copyBytes());
  put.setDurability(Durability.SKIP_WAL);

  //The value should look like this: VALUE1 or VALUE2. Let's multiply
  //the integer by 3
  for(int i = 1; i < valueTokens.length; i++) {
    String prefix = valueTokens[i].substring(0, "VALUE".length());
    String suffix = valueTokens[i].substring("VALUE".length());
    String newValue = prefix + Integer.parseInt(suffix) * 3;

    KeyValue kv = new KeyValue(rowKey.copyBytes(), family,
        qualifiers[i-1], Bytes.toBytes(newValue));
    put.add(kv);
  }

  try {
    context.write(rowKey, put);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }
}
 
Example 12
Project: ditb   File: TestBigDecimalColumnInterpreter.java   Source Code and License Vote up 5 votes
/**
 * A set up method to start the test cluster. AggregateProtocolImpl is registered and will be
 * loaded during region startup.
 * @throws Exception
 */
@BeforeClass
public static void setupBeforeClass() throws Exception {

  conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
    "org.apache.hadoop.hbase.coprocessor.AggregateImplementation");

  util.startMiniCluster(2);
  final byte[][] SPLIT_KEYS = new byte[][] { ROWS[rowSeperator1], ROWS[rowSeperator2] };
  HTable table = util.createTable(TEST_TABLE, TEST_FAMILY, SPLIT_KEYS);
  /**
   * The testtable has one CQ which is always populated and one variable CQ for each row rowkey1:
   * CF:CQ CF:CQ1 rowKey2: CF:CQ CF:CQ2
   */
  for (int i = 0; i < ROWSIZE; i++) {
    Put put = new Put(ROWS[i]);
    put.setDurability(Durability.SKIP_WAL);
    BigDecimal bd = new BigDecimal(i);
    put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(bd));
    table.put(put);
    Put p2 = new Put(ROWS[i]);
    put.setDurability(Durability.SKIP_WAL);
    p2.add(TEST_FAMILY, Bytes.add(TEST_MULTI_CQ, Bytes.toBytes(bd)),
      Bytes.toBytes(bd.multiply(new BigDecimal("0.10"))));
    table.put(p2);
  }
  table.close();
}
 
Example 13
Project: ditb   File: TestMergeTable.java   Source Code and License Vote up 5 votes
private HRegion createRegion(final HTableDescriptor desc,
    byte [] startKey, byte [] endKey, int firstRow, int nrows, Path rootdir)
throws IOException {
  HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
  HRegion region = HRegion.createHRegion(hri, rootdir, UTIL.getConfiguration(), desc);
  LOG.info("Created region " + region.getRegionInfo().getRegionNameAsString());
  for(int i = firstRow; i < firstRow + nrows; i++) {
    Put put = new Put(Bytes.toBytes("row_" + String.format("%1$05d", i)));
    put.setDurability(Durability.SKIP_WAL);
    put.add(COLUMN_NAME, null,  VALUE);
    region.put(put);
    if (i % 10000 == 0) {
      LOG.info("Flushing write #" + i);
      region.flush(true);
    }
  }
  HRegion.closeHRegion(region);
  return region;
}
 
Example 14
Project: ditb   File: TestFuzzyRowAndColumnRangeFilter.java   Source Code and License Vote up 5 votes
@Test
public void Test() throws Exception {
  String cf = "f";
  String table = "TestFuzzyAndColumnRangeFilterClient";
  Table ht = TEST_UTIL.createTable(TableName.valueOf(table),
          Bytes.toBytes(cf), Integer.MAX_VALUE);

  // 10 byte row key - (2 bytes 4 bytes 4 bytes)
  // 4 byte qualifier
  // 4 byte value

  for (int i1 = 0; i1 < 2; i1++) {
    for (int i2 = 0; i2 < 5; i2++) {
      byte[] rk = new byte[10];

      ByteBuffer buf = ByteBuffer.wrap(rk);
      buf.clear();
      buf.putShort((short) 2);
      buf.putInt(i1);
      buf.putInt(i2);

      for (int c = 0; c < 5; c++) {
        byte[] cq = new byte[4];
        Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);

        Put p = new Put(rk);
        p.setDurability(Durability.SKIP_WAL);
        p.add(cf.getBytes(), cq, Bytes.toBytes(c));
        ht.put(p);
        LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
                + Bytes.toStringBinary(cq));
      }
    }
  }

  TEST_UTIL.flush();

  // test passes
  runTest(ht, 0, 10);

  // test fails
  runTest(ht, 1, 8);
}
 
Example 15
Project: ditb   File: IndexPut.java   Source Code and License Vote up 4 votes
public void setWAL(Durability wal) {
	for (Put put : puts.values()) {
		put.setDurability(wal);
	}
}
 
Example 16
Project: ditb   File: TestHRegionReplayEvents.java   Source Code and License Vote up 4 votes
private void replay(HRegion region, Put put, long replaySeqId) throws IOException {
  put.setDurability(Durability.SKIP_WAL);
  MutationReplay mutation = new MutationReplay(MutationType.PUT, put, 0, 0);
  region.batchReplay(new MutationReplay[] {mutation}, replaySeqId);
}
 
Example 17
Project: ditb   File: TestMultipleColumnPrefixFilter.java   Source Code and License Vote up 4 votes
@Test
public void testMultipleColumnPrefixFilterWithManyFamilies() throws IOException {
  String family1 = "Family1";
  String family2 = "Family2";
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter"));
  HColumnDescriptor hcd1 = new HColumnDescriptor(family1);
  hcd1.setMaxVersions(3);
  htd.addFamily(hcd1);
  HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
  hcd2.setMaxVersions(3);
  htd.addFamily(hcd2);
  HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
  HRegion region = HRegion.createHRegion(info, TEST_UTIL.
    getDataTestDir(), TEST_UTIL.getConfiguration(), htd);

  List<String> rows = generateRandomWords(100, "row");
  List<String> columns = generateRandomWords(10000, "column");
  long maxTimestamp = 3;

  List<Cell> kvList = new ArrayList<Cell>();

  Map<String, List<Cell>> prefixMap = new HashMap<String,
      List<Cell>>();

  prefixMap.put("p", new ArrayList<Cell>());
  prefixMap.put("q", new ArrayList<Cell>());
  prefixMap.put("s", new ArrayList<Cell>());

  String valueString = "ValueString";

  for (String row: rows) {
    Put p = new Put(Bytes.toBytes(row));
    p.setDurability(Durability.SKIP_WAL);
    for (String column: columns) {
      for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
        double rand = Math.random();
        Cell kv;
        if (rand < 0.5) 
          kv = KeyValueTestUtil.create(row, family1, column, timestamp,
              valueString);
        else 
          kv = KeyValueTestUtil.create(row, family2, column, timestamp,
              valueString);
        p.add(kv);
        kvList.add(kv);
        for (String s: prefixMap.keySet()) {
          if (column.startsWith(s)) {
            prefixMap.get(s).add(kv);
          }
        }
      }
    }
    region.put(p);
  }

  MultipleColumnPrefixFilter filter;
  Scan scan = new Scan();
  scan.setMaxVersions();
  byte [][] filter_prefix = new byte [2][];
  filter_prefix[0] = new byte [] {'p'};
  filter_prefix[1] = new byte [] {'q'};
  
  filter = new MultipleColumnPrefixFilter(filter_prefix);
  scan.setFilter(filter);
  List<Cell> results = new ArrayList<Cell>();  
  InternalScanner scanner = region.getScanner(scan);
  while (scanner.next(results))
    ;
  assertEquals(prefixMap.get("p").size() + prefixMap.get("q").size(), results.size());

  HRegion.closeHRegion(region);
}
 
Example 18
Project: ditb   File: TestTableResource.java   Source Code and License Vote up 4 votes
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  TEST_UTIL.startMiniCluster(3);
  REST_TEST_UTIL.startServletContainer(TEST_UTIL.getConfiguration());
  client = new Client(new Cluster().add("localhost",
    REST_TEST_UTIL.getServletPort()));
  context = JAXBContext.newInstance(
      TableModel.class,
      TableInfoModel.class,
      TableListModel.class,
      TableRegionModel.class);
  Admin admin = TEST_UTIL.getHBaseAdmin();
  if (admin.tableExists(TABLE)) {
    return;
  }
  HTableDescriptor htd = new HTableDescriptor(TABLE);
  htd.addFamily(new HColumnDescriptor(COLUMN_FAMILY));
  admin.createTable(htd);
  byte[] k = new byte[3];
  byte [][] famAndQf = KeyValue.parseColumn(Bytes.toBytes(COLUMN));
  List<Put> puts = new ArrayList<>();
  for (byte b1 = 'a'; b1 < 'z'; b1++) {
    for (byte b2 = 'a'; b2 < 'z'; b2++) {
      for (byte b3 = 'a'; b3 < 'z'; b3++) {
        k[0] = b1;
        k[1] = b2;
        k[2] = b3;
        Put put = new Put(k);
        put.setDurability(Durability.SKIP_WAL);
        put.add(famAndQf[0], famAndQf[1], k);
        puts.add(put);
      }
    }
  }
  Connection connection = TEST_UTIL.getConnection();
  
  Table table =  connection.getTable(TABLE);
  table.put(puts);
  table.close();
  // get the initial layout (should just be one region)
  
  RegionLocator regionLocator = connection.getRegionLocator(TABLE);
  List<HRegionLocation> m = regionLocator.getAllRegionLocations();
  assertEquals(m.size(), 1);
  // tell the master to split the table
  admin.split(TABLE);
  // give some time for the split to happen

  long timeout = System.currentTimeMillis() + (15 * 1000);
  while (System.currentTimeMillis() < timeout && m.size()!=2){
    try {
      Thread.sleep(250);
    } catch (InterruptedException e) {
      LOG.warn(StringUtils.stringifyException(e));
    }
    // check again
    m = regionLocator.getAllRegionLocations();
  }

  // should have two regions now
  assertEquals(m.size(), 2);
  regionMap = m;
  LOG.info("regions: " + regionMap);
  regionLocator.close();
}
 
Example 19
Project: ditb   File: TestCompaction.java   Source Code and License Vote up 4 votes
/**
 * Verify that you can stop a long-running compaction (used during RS shutdown)
 * @throws Exception
 */
@Test
public void testInterruptCompaction() throws Exception {
  assertEquals(0, count());

  // lower the polling interval for this test
  int origWI = HStore.closeCheckInterval;
  HStore.closeCheckInterval = 10 * 1000; // 10 KB

  try {
    // Create a couple store files w/ 15KB (over 10KB interval)
    int jmax = (int) Math.ceil(15.0 / compactionThreshold);
    byte[] pad = new byte[1000]; // 1 KB chunk
    for (int i = 0; i < compactionThreshold; i++) {
      HRegionIncommon loader = new HRegionIncommon(r);
      Put p = new Put(Bytes.add(STARTROW, Bytes.toBytes(i)));
      p.setDurability(Durability.SKIP_WAL);
      for (int j = 0; j < jmax; j++) {
        p.add(COLUMN_FAMILY, Bytes.toBytes(j), pad);
      }
      HBaseTestCase.addContent(loader, Bytes.toString(COLUMN_FAMILY));
      loader.put(p);
      loader.flushcache();
    }

    HRegion spyR = spy(r);
    doAnswer(new Answer() {
      public Object answer(InvocationOnMock invocation) throws Throwable {
        r.writestate.writesEnabled = false;
        return invocation.callRealMethod();
      }
    }).when(spyR).doRegionCompactionPrep();

    // force a minor compaction, but not before requesting a stop
    spyR.compactStores();

    // ensure that the compaction stopped, all old files are intact,
    Store s = r.stores.get(COLUMN_FAMILY);
    assertEquals(compactionThreshold, s.getStorefilesCount());
    assertTrue(s.getStorefilesSize() > 15 * 1000);
    // and no new store files persisted past compactStores()
    FileStatus[] ls = r.getFilesystem().listStatus(r.getRegionFileSystem().getTempDir());
    assertEquals(0, ls.length);

  } finally {
    // don't mess up future tests
    r.writestate.writesEnabled = true;
    HStore.closeCheckInterval = origWI;

    // Delete all Store information once done using
    for (int i = 0; i < compactionThreshold; i++) {
      Delete delete = new Delete(Bytes.add(STARTROW, Bytes.toBytes(i)));
      byte[][] famAndQf = { COLUMN_FAMILY, null };
      delete.deleteFamily(famAndQf[0]);
      r.delete(delete);
    }
    r.flush(true);

    // Multiple versions allowed for an entry, so the delete isn't enough
    // Lower TTL and expire to ensure that all our entries have been wiped
    final int ttl = 1000;
    for (Store hstore : this.r.stores.values()) {
      HStore store = (HStore) hstore;
      ScanInfo old = store.getScanInfo();
      ScanInfo si =
          new ScanInfo(old.getConfiguration(), old.getFamily(), old.getMinVersions(),
              old.getMaxVersions(), ttl, old.getKeepDeletedCells(), 0, old.getComparator());
      store.setScanInfo(si);
    }
    Thread.sleep(ttl);

    r.compact(true);
    assertEquals(0, count());
  }
}
 
Example 20
Project: ditb   File: TestHBaseFsck.java   Source Code and License Vote up 4 votes
@Test (timeout=180000)
public void testHBaseFsck() throws Exception {
  assertNoErrors(doFsck(conf, false));
  TableName table = TableName.valueOf("tableBadMetaAssign");
  HTableDescriptor desc = new HTableDescriptor(table);
  HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
  desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
  createTable(TEST_UTIL, desc, null);

  // We created 1 table, should be fine
  assertNoErrors(doFsck(conf, false));

  // Now let's mess it up and change the assignment in hbase:meta to
  // point to a different region server
  Table meta = connection.getTable(TableName.META_TABLE_NAME, tableExecutorService);
  Scan scan = new Scan();
  scan.setStartRow(Bytes.toBytes(table+",,"));
  ResultScanner scanner = meta.getScanner(scan);
  HRegionInfo hri = null;

  Result res = scanner.next();
  ServerName currServer =
    ServerName.parseFrom(res.getValue(HConstants.CATALOG_FAMILY,
        HConstants.SERVER_QUALIFIER));
  long startCode = Bytes.toLong(res.getValue(HConstants.CATALOG_FAMILY,
      HConstants.STARTCODE_QUALIFIER));

  for (JVMClusterUtil.RegionServerThread rs :
      TEST_UTIL.getHBaseCluster().getRegionServerThreads()) {

    ServerName sn = rs.getRegionServer().getServerName();

    // When we find a diff RS, change the assignment and break
    if (!currServer.getHostAndPort().equals(sn.getHostAndPort()) ||
        startCode != sn.getStartcode()) {
      Put put = new Put(res.getRow());
      put.setDurability(Durability.SKIP_WAL);
      put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
        Bytes.toBytes(sn.getHostAndPort()));
      put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
        Bytes.toBytes(sn.getStartcode()));
      meta.put(put);
      hri = MetaTableAccessor.getHRegionInfo(res);
      break;
    }
  }

  // Try to fix the data
  assertErrors(doFsck(conf, true), new ERROR_CODE[]{
      ERROR_CODE.SERVER_DOES_NOT_MATCH_META});

  TEST_UTIL.getHBaseCluster().getMaster()
    .getAssignmentManager().waitForAssignment(hri);

  // Should be fixed now
  assertNoErrors(doFsck(conf, false));

  // comment needed - what is the purpose of this line
  Table t = connection.getTable(table, tableExecutorService);
  ResultScanner s = t.getScanner(new Scan());
  s.close();
  t.close();

  scanner.close();
  meta.close();
}