Java Code Examples for org.apache.hadoop.hbase.client.TableDescriptor#getColumnFamilyNames()

/**
 * Removes from hdfs the families that are not longer present in the new table descriptor.
 * @param env MasterProcedureEnv
 * @throws IOException
 */
private void deleteFromFs(final MasterProcedureEnv env,
    final TableDescriptor oldTableDescriptor, final TableDescriptor newTableDescriptor)
    throws IOException {
  final Set<byte[]> oldFamilies = oldTableDescriptor.getColumnFamilyNames();
  final Set<byte[]> newFamilies = newTableDescriptor.getColumnFamilyNames();
  for (byte[] familyName : oldFamilies) {
    if (!newFamilies.contains(familyName)) {
      MasterDDLOperationHelper.deleteColumnFamilyFromFileSystem(
        env,
        getTableName(),
        getRegionInfoList(env),
        familyName, oldTableDescriptor.getColumnFamily(familyName).isMobEnabled());
    }
  }
}
 

/*********************************** Observer implementations ***********************************/

  @Override
  public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> c,
      TableDescriptor desc, RegionInfo[] regions) throws IOException {
    Set<byte[]> families = desc.getColumnFamilyNames();
    Map<byte[], Set<byte[]>> familyMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] family: families) {
      familyMap.put(family, null);
    }
    requireNamespacePermission(c, "createTable",
        desc.getTableName().getNamespaceAsString(), desc.getTableName(), familyMap, Action.ADMIN,
        Action.CREATE);
  }
 

/**
 * Find out whether all column families in unmodifiedTableDescriptor also exists in
 * the modifiedTableDescriptor.
 * @return True if we are deleting a column family.
 */
private static boolean isDeleteColumnFamily(TableDescriptor originalDescriptor,
    TableDescriptor newDescriptor) {
  boolean result = false;
  final Set<byte[]> originalFamilies = originalDescriptor.getColumnFamilyNames();
  final Set<byte[]> newFamilies = newDescriptor.getColumnFamilyNames();
  for (byte[] familyName : originalFamilies) {
    if (!newFamilies.contains(familyName)) {
      result = true;
      break;
    }
  }
  return result;
}
 

WALPutBenchmark(final HRegion region, final TableDescriptor htd,
    final long numIterations, final boolean noSync, final int syncInterval,
    final double traceFreq) {
  this.numIterations = numIterations;
  this.noSync = noSync;
  this.syncInterval = syncInterval;
  this.numFamilies = htd.getColumnFamilyCount();
  this.region = region;
  scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for(byte[] fam : htd.getColumnFamilyNames()) {
    scopes.put(fam, 0);
  }
  String spanReceivers = getConf().get("hbase.trace.spanreceiver.classes");
  if (spanReceivers == null || spanReceivers.isEmpty()) {
    loopSampler = Sampler.NEVER;
  } else {
    if (traceFreq <= 0.0) {
      LOG.warn("Tracing enabled but traceFreq=0.");
      loopSampler = Sampler.NEVER;
    } else if (traceFreq >= 1.0) {
      loopSampler = Sampler.ALWAYS;
      if (numIterations > 1000) {
        LOG.warn("Full tracing of all iterations will produce a lot of data. Be sure your"
          + " SpanReceiver can keep up.");
      }
    } else {
      getConf().setDouble("hbase.sampler.fraction", traceFreq);
      loopSampler = new ProbabilitySampler(new HBaseHTraceConfiguration(getConf()));
    }
  }
}
 

/**
 * Returns a {@link Predicate} for checking that table is enabled
 */
public Waiter.Predicate<IOException> predicateTableAvailable(final TableName tableName) {
  return new ExplainingPredicate<IOException>() {
    @Override
    public String explainFailure() throws IOException {
      return explainTableAvailability(tableName);
    }

    @Override
    public boolean evaluate() throws IOException {
      boolean tableAvailable = getAdmin().isTableAvailable(tableName);
      if (tableAvailable) {
        try (Table table = getConnection().getTable(tableName)) {
          TableDescriptor htd = table.getDescriptor();
          for (HRegionLocation loc : getConnection().getRegionLocator(tableName)
              .getAllRegionLocations()) {
            Scan scan = new Scan().withStartRow(loc.getRegion().getStartKey())
                .withStopRow(loc.getRegion().getEndKey()).setOneRowLimit()
                .setMaxResultsPerColumnFamily(1).setCacheBlocks(false);
            for (byte[] family : htd.getColumnFamilyNames()) {
              scan.addFamily(family);
            }
            try (ResultScanner scanner = table.getScanner(scan)) {
              scanner.next();
            }
          }
        }
      }
      return tableAvailable;
    }
  };
}
 

private void verifyTableDescriptor(final TableDescriptor htd,
    final TableName tableName, final byte[]... families) {
  Set<byte[]> htdFamilies = htd.getColumnFamilyNames();
  assertEquals(tableName, htd.getTableName());
  assertEquals(families.length, htdFamilies.size());
  for (byte[] familyName: families) {
    assertTrue("Expected family " + Bytes.toString(familyName), htdFamilies.contains(familyName));
  }
}
 

/**
 * Coprocessors shouldn't get notice of empty waledits.
 */
@Test
public void testEmptyWALEditAreNotSeen() throws Exception {
  RegionInfo hri = createBasicHRegionInfo(Bytes.toString(TEST_TABLE));
  TableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE));
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for(byte[] fam : htd.getColumnFamilyNames()) {
    scopes.put(fam, 0);
  }
  WAL log = wals.getWAL(null);
  try {
    SampleRegionWALCoprocessor cp = getCoprocessor(log, SampleRegionWALCoprocessor.class);

    cp.setTestValues(TEST_TABLE, null, null, null, null, null, null, null);

    assertFalse(cp.isPreWALWriteCalled());
    assertFalse(cp.isPostWALWriteCalled());

    final long now = EnvironmentEdgeManager.currentTime();
    long txid = log.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc, scopes),
      new WALEdit());
    log.sync(txid);

    assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPreWALWriteCalled());
    assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPostWALWriteCalled());
  } finally {
    log.close();
  }
}
 

@Test
public void testWriteEntryCanBeNull() throws IOException {
  String testName = currentTest.getMethodName();
  AbstractFSWAL<?> wal = newWAL(FS, CommonFSUtils.getWALRootDir(CONF), DIR.toString(), testName,
    CONF, null, true, null, null);
  wal.close();
  TableDescriptor td = TableDescriptorBuilder.newBuilder(TableName.valueOf("table"))
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();
  RegionInfo ri = RegionInfoBuilder.newBuilder(td.getTableName()).build();
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for (byte[] fam : td.getColumnFamilyNames()) {
    scopes.put(fam, 0);
  }
  long timestamp = System.currentTimeMillis();
  byte[] row = Bytes.toBytes("row");
  WALEdit cols = new WALEdit();
  cols.add(new KeyValue(row, row, row, timestamp, row));
  WALKeyImpl key =
      new WALKeyImpl(ri.getEncodedNameAsBytes(), td.getTableName(), SequenceId.NO_SEQUENCE_ID,
        timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE, HConstants.NO_NONCE, mvcc, scopes);
  try {
    wal.append(ri, key, cols, true);
    fail("Should fail since the wal has already been closed");
  } catch (IOException e) {
    // expected
    assertThat(e.getMessage(), containsString("log is closed"));
    // the WriteEntry should be null since we fail before setting it.
    assertNull(key.getWriteEntry());
  }
}
 

@Test
public void testSyncRunnerIndexOverflow() throws IOException, NoSuchFieldException,
    SecurityException, IllegalArgumentException, IllegalAccessException {
  final String name = this.name.getMethodName();
  FSHLog log = new FSHLog(FS, CommonFSUtils.getRootDir(CONF), name,
    HConstants.HREGION_OLDLOGDIR_NAME, CONF, null, true, null, null);
  log.init();
  try {
    Field ringBufferEventHandlerField = FSHLog.class.getDeclaredField("ringBufferEventHandler");
    ringBufferEventHandlerField.setAccessible(true);
    FSHLog.RingBufferEventHandler ringBufferEventHandler =
        (FSHLog.RingBufferEventHandler) ringBufferEventHandlerField.get(log);
    Field syncRunnerIndexField =
        FSHLog.RingBufferEventHandler.class.getDeclaredField("syncRunnerIndex");
    syncRunnerIndexField.setAccessible(true);
    syncRunnerIndexField.set(ringBufferEventHandler, Integer.MAX_VALUE - 1);
    TableDescriptor htd =
        TableDescriptorBuilder.newBuilder(TableName.valueOf(this.name.getMethodName()))
          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("row")).build();
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    for (byte[] fam : htd.getColumnFamilyNames()) {
      scopes.put(fam, 0);
    }
    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
    for (int i = 0; i < 10; i++) {
      addEdits(log, hri, htd, 1, mvcc, scopes, "row");
    }
  } finally {
    log.close();
  }
}
 

public void invalidateAll(TableDescriptor htableDesc) {
    Preconditions.checkNotNull(htableDesc);
    byte[] tableName = htableDesc.getTableName().getName();
    for (byte[] fam : htableDesc.getColumnFamilyNames()) {
        invalidate(new GuidePostsKey(tableName, fam));
    }
}
 

@Test
public void testLogRoll() throws Exception {
  long baseline = 1000;
  long time = baseline;
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  KeyValue kv = new KeyValue(r1, f1, r1);
  WALEdit edit = new WALEdit();
  edit.add(kv);

  WALFactory wals =
    new WALFactory(utility.getConfiguration(), URLEncoder.encode("regionserver:60020", "UTF8"));
  ReplicationSourceManager replicationManager = replication.getReplicationManager();
  wals.getWALProvider()
    .addWALActionsListener(new ReplicationSourceWALActionListener(conf, replicationManager));
  final WAL wal = wals.getWAL(hri);
  manager.init();
  TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf("tableame"))
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of(f1)).build();
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for(byte[] fam : htd.getColumnFamilyNames()) {
    scopes.put(fam, 0);
  }
  // Testing normal log rolling every 20
  for(long i = 1; i < 101; i++) {
    if(i > 1 && i % 20 == 0) {
      wal.rollWriter();
    }
    LOG.info(Long.toString(i));
    final long txid = wal.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes),
      edit);
    wal.sync(txid);
  }

  // Simulate a rapid insert that's followed
  // by a report that's still not totally complete (missing last one)
  LOG.info(baseline + " and " + time);
  baseline += 101;
  time = baseline;
  LOG.info(baseline + " and " + time);

  for (int i = 0; i < 3; i++) {
    wal.appendData(hri,
      new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes),
      edit);
  }
  wal.sync();

  int logNumber = 0;
  for (Map.Entry<String, NavigableSet<String>> entry : manager.getWALs().get(slaveId)
    .entrySet()) {
    logNumber += entry.getValue().size();
  }
  assertEquals(6, logNumber);

  wal.rollWriter();

  ReplicationSourceInterface source = mock(ReplicationSourceInterface.class);
  when(source.getQueueId()).thenReturn("1");
  when(source.isRecovered()).thenReturn(false);
  when(source.isSyncReplication()).thenReturn(false);
  manager.logPositionAndCleanOldLogs(source,
    new WALEntryBatch(0, manager.getSources().get(0).getCurrentPath()));

  wal.appendData(hri,
    new WALKeyImpl(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc, scopes),
    edit);
  wal.sync();

  assertEquals(1, manager.getWALs().size());


  // TODO Need a case with only 2 WALs and we only want to delete the first one
}
 

/**
 * Tests that we can write out an edit, close, and then read it back in again.
 */
@Test
public void testEditAdd() throws IOException {
  int colCount = 10;
  TableDescriptor htd =
      TableDescriptorBuilder.newBuilder(TableName.valueOf(currentTest.getMethodName()))
          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("column")).build();
  NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
  for (byte[] fam : htd.getColumnFamilyNames()) {
    scopes.put(fam, 0);
  }
  byte[] row = Bytes.toBytes("row");
  WAL.Reader reader = null;
  try {
    final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);

    // Write columns named 1, 2, 3, etc. and then values of single byte
    // 1, 2, 3...
    long timestamp = System.currentTimeMillis();
    WALEdit cols = new WALEdit();
    for (int i = 0; i < colCount; i++) {
      cols.add(new KeyValue(row, Bytes.toBytes("column"),
          Bytes.toBytes(Integer.toString(i)),
        timestamp, new byte[] { (byte)(i + '0') }));
    }
    RegionInfo info = RegionInfoBuilder.newBuilder(htd.getTableName()).setStartKey(row)
        .setEndKey(Bytes.toBytes(Bytes.toString(row) + "1")).build();
    final WAL log = wals.getWAL(info);

    final long txid = log.appendData(info, new WALKeyImpl(info.getEncodedNameAsBytes(),
      htd.getTableName(), System.currentTimeMillis(), mvcc, scopes), cols);
    log.sync(txid);
    log.startCacheFlush(info.getEncodedNameAsBytes(), htd.getColumnFamilyNames());
    log.completeCacheFlush(info.getEncodedNameAsBytes());
    log.shutdown();
    Path filename = AbstractFSWALProvider.getCurrentFileName(log);
    // Now open a reader on the log and assert append worked.
    reader = wals.createReader(fs, filename);
    // Above we added all columns on a single row so we only read one
    // entry in the below... thats why we have '1'.
    for (int i = 0; i < 1; i++) {
      WAL.Entry entry = reader.next(null);
      if (entry == null) break;
      WALKey key = entry.getKey();
      WALEdit val = entry.getEdit();
      assertTrue(Bytes.equals(info.getEncodedNameAsBytes(), key.getEncodedRegionName()));
      assertTrue(htd.getTableName().equals(key.getTableName()));
      Cell cell = val.getCells().get(0);
      assertTrue(Bytes.equals(row, 0, row.length, cell.getRowArray(), cell.getRowOffset(),
        cell.getRowLength()));
      assertEquals((byte)(i + '0'), CellUtil.cloneValue(cell)[0]);
      System.out.println(key + " " + val);
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
}
 

@Test
public void testAppend() throws IOException {
  int colCount = 10;
  TableDescriptor htd =
      TableDescriptorBuilder.newBuilder(TableName.valueOf(currentTest.getMethodName()))
          .setColumnFamily(ColumnFamilyDescriptorBuilder.of("column")).build();
  NavigableMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
  for (byte[] fam : htd.getColumnFamilyNames()) {
    scopes.put(fam, 0);
  }
  byte[] row = Bytes.toBytes("row");
  WAL.Reader reader = null;
  final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);
  try {
    // Write columns named 1, 2, 3, etc. and then values of single byte
    // 1, 2, 3...
    long timestamp = System.currentTimeMillis();
    WALEdit cols = new WALEdit();
    for (int i = 0; i < colCount; i++) {
      cols.add(new KeyValue(row, Bytes.toBytes("column"),
        Bytes.toBytes(Integer.toString(i)),
        timestamp, new byte[] { (byte)(i + '0') }));
    }
    RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
    final WAL log = wals.getWAL(hri);
    final long txid = log.appendData(hri, new WALKeyImpl(hri.getEncodedNameAsBytes(),
      htd.getTableName(), System.currentTimeMillis(), mvcc, scopes), cols);
    log.sync(txid);
    log.startCacheFlush(hri.getEncodedNameAsBytes(), htd.getColumnFamilyNames());
    log.completeCacheFlush(hri.getEncodedNameAsBytes());
    log.shutdown();
    Path filename = AbstractFSWALProvider.getCurrentFileName(log);
    // Now open a reader on the log and assert append worked.
    reader = wals.createReader(fs, filename);
    WAL.Entry entry = reader.next();
    assertEquals(colCount, entry.getEdit().size());
    int idx = 0;
    for (Cell val : entry.getEdit().getCells()) {
      assertTrue(Bytes.equals(hri.getEncodedNameAsBytes(),
        entry.getKey().getEncodedRegionName()));
      assertTrue(htd.getTableName().equals(entry.getKey().getTableName()));
      assertTrue(Bytes.equals(row, 0, row.length, val.getRowArray(), val.getRowOffset(),
        val.getRowLength()));
      assertEquals((byte) (idx + '0'), CellUtil.cloneValue(val)[0]);
      System.out.println(entry.getKey() + " " + val);
      idx++;
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
}