Java Code Examples for org.apache.hadoop.hbase.client.ColumnFamilyDescriptor

The following examples show how to use org.apache.hadoop.hbase.client.ColumnFamilyDescriptor. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hbase   Source File: TestTimestampFilterSeekHint.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void prepareRegion() throws IOException {
  ColumnFamilyDescriptor columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder
      .newBuilder(Bytes.toBytes(FAMILY))
      .setBlocksize(1024)
      .setMaxVersions(MAX_VERSIONS)
      .build();
  region = TEST_UTIL
    .createTestRegion("TestTimestampFilterSeekHint" + regionCount++, columnFamilyDescriptor);

  for (long i = 0; i <MAX_VERSIONS - 2; i++) {
    Put p = new Put(RK_BYTES, i);
    p.addColumn(FAMILY_BYTES, QUAL_BYTES, Bytes.toBytes(RandomStringUtils.randomAlphabetic(255)));
    region.put(p);
  }
  region.flush(true);
}
 
Example 2
Source Project: hbase   Source File: TestCanaryTool.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBasicCanaryWorks() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  Table table = testingUtility.createTable(tableName, new byte[][] { FAMILY });
  // insert some test rows
  for (int i=0; i<1000; i++) {
    byte[] iBytes = Bytes.toBytes(i);
    Put p = new Put(iBytes);
    p.addColumn(FAMILY, COLUMN, iBytes);
    table.put(p);
  }
  ExecutorService executor = new ScheduledThreadPoolExecutor(1);
  CanaryTool.RegionStdOutSink sink = spy(new CanaryTool.RegionStdOutSink());
  CanaryTool canary = new CanaryTool(executor, sink);
  String[] args = { "-writeSniffing", "-t", "10000", tableName.getNameAsString() };
  assertEquals(0, ToolRunner.run(testingUtility.getConfiguration(), canary, args));
  assertEquals("verify no read error count", 0, canary.getReadFailures().size());
  assertEquals("verify no write error count", 0, canary.getWriteFailures().size());
  verify(sink, atLeastOnce()).publishReadTiming(isA(ServerName.class), isA(RegionInfo.class),
    isA(ColumnFamilyDescriptor.class), anyLong());
}
 
Example 3
Source Project: hbase   Source File: AddColumnAction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void perform() throws Exception {
  TableDescriptor tableDescriptor = admin.getDescriptor(tableName);
  ColumnFamilyDescriptor columnDescriptor = null;

  while (columnDescriptor == null
      || tableDescriptor.getColumnFamily(columnDescriptor.getName()) != null) {
    columnDescriptor = ColumnFamilyDescriptorBuilder.of(RandomStringUtils.randomAlphabetic(5));
  }

  // Don't try the modify if we're stopping
  if (context.isStopping()) {
    return;
  }

  getLogger().debug("Performing action: Adding " + columnDescriptor + " to " + tableName);

  TableDescriptor modifiedTable = TableDescriptorBuilder.newBuilder(tableDescriptor)
      .setColumnFamily(columnDescriptor).build();
  admin.modifyTable(modifiedTable);
}
 
Example 4
Source Project: hbase   Source File: TestMinVersions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception {
  int ttl = 4;
  ColumnFamilyDescriptor cfd =
    ColumnFamilyDescriptorBuilder.newBuilder(c0)
      .setVersionsWithTimeToLive(ttl, 2).build();
  verifyVersionedCellKeyValues(ttl, cfd);

  cfd = ColumnFamilyDescriptorBuilder.newBuilder(c0)
    .setMinVersions(2)
    .setMaxVersions(Integer.MAX_VALUE)
    .setTimeToLive(ttl)
    .setKeepDeletedCells(KeepDeletedCells.TTL)
    .build();
  verifyVersionedCellKeyValues(ttl, cfd);
}
 
Example 5
Source Project: atlas   Source File: HBaseOperationContext.java    License: Apache License 2.0 6 votes vote down vote up
public HBaseOperationContext(NamespaceDescriptor namespaceDescriptor, String nameSpace, TableDescriptor tableDescriptor, TableName tableName, ColumnFamilyDescriptor[] columnFamilyDescriptors,
                             ColumnFamilyDescriptor columnFamilyDescriptor, String columnFamily, HBaseAtlasHook.OPERATION operation, UserGroupInformation ugi , String user, String owner,
                             Map<String, String> hbaseConf) {
    this.namespaceDescriptor     = namespaceDescriptor;
    this.nameSpace               = nameSpace;
    this.tableDescriptor         = tableDescriptor;
    this.tableName               = tableName;
    this.columnFamilyDescriptors = columnFamilyDescriptors;
    this.columnFamilyDescriptor  = columnFamilyDescriptor;
    this.columnFamily            = columnFamily;
    this.operation               = operation;
    this.ugi                     = ugi;
    this.user                    = user;
    this.owner                   = owner;
    this.hbaseConf               = hbaseConf;
}
 
Example 6
Source Project: hbase   Source File: IntegrationTestManyRegions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateTableWithRegions() throws Exception {
  ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
    .newBuilder(Bytes.toBytes("cf"))
    .build();
  TableDescriptor tableDescriptor =
    TableDescriptorBuilder.newBuilder(TABLE_NAME)
      .setColumnFamily(columnFamilyDescriptor)
      .build();

  SplitAlgorithm algo = new RegionSplitter.HexStringSplit();
  byte[][] splits = algo.split(REGION_COUNT);

  LOG.info(String.format("Creating table %s with %d splits.", TABLE_NAME, REGION_COUNT));
  long startTime = System.currentTimeMillis();
  try {
    admin.createTable(tableDescriptor, splits);
    LOG.info(String.format("Pre-split table created successfully in %dms.",
        (System.currentTimeMillis() - startTime)));
  } catch (IOException e) {
    LOG.error("Failed to create table", e);
  }
}
 
Example 7
Source Project: hbase   Source File: BackupSystemTable.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get backup system table descriptor
 * @return table's descriptor
 */
public static TableDescriptor getSystemTableDescriptor(Configuration conf) {
  TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(getTableName(conf));

  ColumnFamilyDescriptorBuilder colBuilder =
      ColumnFamilyDescriptorBuilder.newBuilder(SESSIONS_FAMILY);

  colBuilder.setMaxVersions(1);
  Configuration config = HBaseConfiguration.create();
  int ttl = config.getInt(BackupRestoreConstants.BACKUP_SYSTEM_TTL_KEY,
    BackupRestoreConstants.BACKUP_SYSTEM_TTL_DEFAULT);
  colBuilder.setTimeToLive(ttl);

  ColumnFamilyDescriptor colSessionsDesc = colBuilder.build();
  builder.setColumnFamily(colSessionsDesc);

  colBuilder = ColumnFamilyDescriptorBuilder.newBuilder(META_FAMILY);
  colBuilder.setTimeToLive(ttl);
  builder.setColumnFamily(colBuilder.build());
  return builder.build();
}
 
Example 8
Source Project: phoenix   Source File: UpgradeUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Synchronize column family properties using the default cf properties for a given table
 * @param tableDesc table descriptor of table to modify
 * @param defaultColFam default column family used as the baseline for property synchronization
 * @param syncedProps Map of properties to be kept in sync as read from the default column family descriptor
 * @return modified table descriptor builder
 */
private static TableDescriptorBuilder syncColFamProperties(TableDescriptor tableDesc, ColumnFamilyDescriptor defaultColFam,
        Map<String, Object> syncedProps) {
    TableDescriptorBuilder tableDescBuilder = TableDescriptorBuilder.newBuilder(tableDesc);
    // Ensure that all column families have necessary properties in sync (including local index cf if present)
    for (ColumnFamilyDescriptor currentColFam: tableDesc.getColumnFamilies()) {
        if (!currentColFam.equals(defaultColFam)) {
            ColumnFamilyDescriptorBuilder colFamDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(currentColFam);
            for (String prop: MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
                String existingPropVal = Bytes.toString(currentColFam.getValue(Bytes.toBytes(prop)));
                String expectedPropVal = syncedProps.get(prop).toString();
                if (existingPropVal == null || !existingPropVal.toLowerCase().equals(expectedPropVal.toLowerCase())) {
                    // Need to synchronize this property for the current column family descriptor
                    colFamDescBuilder.setValue(prop, expectedPropVal);
                }
            }
            if (!colFamDescBuilder.equals(ColumnFamilyDescriptorBuilder.newBuilder(currentColFam))) {
                tableDescBuilder.modifyColumnFamily(colFamDescBuilder.build());
            }
        }
    }
    return tableDescBuilder;
}
 
Example 9
Source Project: hbase   Source File: TestCoprocessorTableEndpoint.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDynamicCoprocessorTableEndpoint() throws Throwable {
  final TableName tableName = TableName.valueOf(name.getMethodName());

  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  ColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(TEST_FAMILY);

  tableDescriptor.setColumnFamily(familyDescriptor);
  createTable(tableDescriptor);

  tableDescriptor.setCoprocessor(ColumnAggregationEndpoint.class.getName());
  updateTable(tableDescriptor);

  verifyTable(tableName);
}
 
Example 10
Source Project: hbase   Source File: TestExpiredMobFileCleaner.java    License: Apache License 2.0 6 votes vote down vote up
private void init() throws Exception {
  TableDescriptorBuilder tableDescriptorBuilder =
    TableDescriptorBuilder.newBuilder(tableName);
  ColumnFamilyDescriptor columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder
      .newBuilder(Bytes.toBytes(family))
      .setMobEnabled(true)
      .setMobThreshold(3L)
      .setMaxVersions(4)
      .build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);

  admin = TEST_UTIL.getAdmin();
  admin.createTable(tableDescriptorBuilder.build());
  table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())
          .getBufferedMutator(tableName);
}
 
Example 11
Source Project: phoenix   Source File: MultiHfileOutputFormat.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Serialize column family to compression algorithm map to configuration.
 * Invoked while configuring the MR job for incremental load.
 *
 * @param table to read the properties from
 * @param conf to persist serialized values into
 * @throws IOException
 *           on failure to read column family descriptors
 */
@edu.umd.cs.findbugs.annotations.SuppressWarnings(
    value="RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE")
@VisibleForTesting
static String configureCompression(TableDescriptor tableDescriptor)
    throws UnsupportedEncodingException {

    StringBuilder compressionConfigValue = new StringBuilder();
    if(tableDescriptor == null){
        // could happen with mock table instance
        return compressionConfigValue.toString();
    }
    ColumnFamilyDescriptor[] families = tableDescriptor.getColumnFamilies();
    int i = 0;
    for (ColumnFamilyDescriptor familyDescriptor : families) {
        if (i++ > 0) {
            compressionConfigValue.append('&');
        }
        compressionConfigValue.append(URLEncoder.encode(
                familyDescriptor.getNameAsString(), "UTF-8"));
        compressionConfigValue.append('=');
        compressionConfigValue.append(URLEncoder.encode(
                familyDescriptor.getCompressionType().getName(), "UTF-8"));
    }
    return compressionConfigValue.toString();
}
 
Example 12
Source Project: hbase   Source File: TestHRegionServerBulkLoad.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a table with given table name and specified number of column
 * families if the table does not already exist.
 */
public void setupTable(TableName table, int cfs) throws IOException {
  try {
    LOG.info("Creating table " + table);
    TableDescriptorBuilder tableDescriptorBuilder =
      TableDescriptorBuilder.newBuilder(table);

    tableDescriptorBuilder.setCoprocessor(MyObserver.class.getName());
    MyObserver.sleepDuration = this.sleepDuration;
    for (int i = 0; i < 10; i++) {
      ColumnFamilyDescriptor columnFamilyDescriptor =
        ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family(i))).build();
      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
    }

    UTIL.getAdmin().createTable(tableDescriptorBuilder.build());
  } catch (TableExistsException tee) {
    LOG.info("Table " + table + " already exists");
  }
}
 
Example 13
Source Project: hbase   Source File: TestAccessController.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTableModify() throws Exception {
  AccessTestAction modifyTable = new AccessTestAction() {
    @Override
    public Object run() throws Exception {
      TableDescriptorBuilder tableDescriptorBuilder =
        TableDescriptorBuilder.newBuilder(TEST_TABLE);
      ColumnFamilyDescriptor columnFamilyDescriptor =
        ColumnFamilyDescriptorBuilder.newBuilder(TEST_FAMILY).build();
      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
      columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
        .newBuilder(Bytes.toBytes("fam_" + User.getCurrent().getShortName())).build();
      tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
      ACCESS_CONTROLLER.preModifyTable(ObserverContextImpl.createAndPrepare(CP_ENV),
          TEST_TABLE,
          null,  // not needed by AccessController
          tableDescriptorBuilder.build());
      return null;
    }
  };

  verifyAllowed(modifyTable, SUPERUSER, USER_ADMIN, USER_CREATE, USER_OWNER, USER_GROUP_CREATE,
    USER_GROUP_ADMIN);
  verifyDenied(modifyTable, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE);
}
 
Example 14
Source Project: hbase   Source File: HMaster.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public long modifyColumn(final TableName tableName, final ColumnFamilyDescriptor descriptor,
    final long nonceGroup, final long nonce) throws IOException {
  checkInitialized();
  checkTableExists(tableName);
  return modifyTable(tableName, new TableDescriptorGetter() {

    @Override
    public TableDescriptor get() throws IOException {
      TableDescriptor old = getTableDescriptors().get(tableName);
      if (!old.hasColumnFamily(descriptor.getName())) {
        throw new InvalidFamilyOperationException("Family '" + descriptor.getNameAsString()
            + "' does not exist, so it cannot be modified");
      }

      return TableDescriptorBuilder.newBuilder(old).modifyColumnFamily(descriptor).build();
    }
  }, nonceGroup, nonce, true);
}
 
Example 15
Source Project: phoenix   Source File: SetPropertyIT.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSetPropertyAndAddColumnForDefaultColumnFamily() throws Exception {
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    Connection conn = DriverManager.getConnection(getUrl(), props);
    conn.setAutoCommit(false);
    String ddl = "CREATE TABLE " + dataTableFullName +
            "  (a_string varchar not null, col1 integer" +
            "  CONSTRAINT pk PRIMARY KEY (a_string)) " + tableDDLOptions;
    try {
        conn.createStatement().execute(ddl);
        conn.createStatement().execute("ALTER TABLE " + dataTableFullName + " ADD col2 integer IN_MEMORY=true");
        try (Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) {
            ColumnFamilyDescriptor[] columnFamilies = admin.getDescriptor(TableName.valueOf(dataTableFullName))
                    .getColumnFamilies();
            assertEquals(1, columnFamilies.length);
            assertEquals("0", columnFamilies[0].getNameAsString());
            assertTrue(columnFamilies[0].isInMemory());
        }
    } finally {
        conn.close();
    }
}
 
Example 16
Source Project: hbase   Source File: TestHMobStore.java    License: Apache License 2.0 6 votes vote down vote up
private void init(String methodName, Configuration conf, ColumnFamilyDescriptor cfd,
    boolean testStore) throws IOException {
  TableDescriptor td =
      TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setColumnFamily(cfd).build();

  //Setting up tje Region and Store
  Path basedir = new Path(DIR + methodName);
  Path tableDir = CommonFSUtils.getTableDir(basedir, td.getTableName());
  String logName = "logs";
  Path logdir = new Path(basedir, logName);
  FileSystem fs = FileSystem.get(conf);
  fs.delete(logdir, true);

  RegionInfo info = RegionInfoBuilder.newBuilder(td.getTableName()).build();
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  final Configuration walConf = new Configuration(conf);
  CommonFSUtils.setRootDir(walConf, basedir);
  final WALFactory wals = new WALFactory(walConf, methodName);
  region = new HRegion(tableDir, wals.getWAL(info), fs, conf, info, td, null);
  region.setMobFileCache(new MobFileCache(conf));
  store = new HMobStore(region, cfd, conf, false);
  if (testStore) {
    init(conf, cfd);
  }
}
 
Example 17
Source Project: hbase   Source File: HMobStore.java    License: Apache License 2.0 6 votes vote down vote up
public HMobStore(final HRegion region, final ColumnFamilyDescriptor family,
    final Configuration confParam, boolean warmup) throws IOException {
  super(region, family, confParam, warmup);
  this.family = family;
  this.mobFileCache = region.getMobFileCache();
  this.homePath = MobUtils.getMobHome(conf);
  this.mobFamilyPath = MobUtils.getMobFamilyPath(conf, this.getTableName(),
      family.getNameAsString());
  List<Path> locations = new ArrayList<>(2);
  locations.add(mobFamilyPath);
  TableName tn = region.getTableDescriptor().getTableName();
  locations.add(HFileArchiveUtil.getStoreArchivePath(conf, tn, MobUtils.getMobRegionInfo(tn)
      .getEncodedName(), family.getNameAsString()));
  map.put(tn, locations);
  List<Tag> tags = new ArrayList<>(2);
  tags.add(MobConstants.MOB_REF_TAG);
  Tag tableNameTag = new ArrayBackedTag(TagType.MOB_TABLE_NAME_TAG_TYPE,
      getTableName().getName());
  tags.add(tableNameTag);
  this.refCellTags = TagUtil.fromList(tags);
}
 
Example 18
Source Project: phoenix   Source File: SetPropertyIT.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSetPropertyAndAddColumnForDifferentColumnFamilies() throws Exception {
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    Connection conn = DriverManager.getConnection(getUrl(), props);
    String ddl = "CREATE TABLE " + dataTableFullName
            +
            "  (a_string varchar not null, col1 integer, CF1.col2 integer, CF2.col3 integer" +
            "  CONSTRAINT pk PRIMARY KEY (a_string)) " + generateDDLOptions("DEFAULT_COLUMN_FAMILY = 'XYZ' ");
    try {
        conn.createStatement().execute(ddl);
        conn.createStatement()
                .execute(
                        "ALTER TABLE "
                                + dataTableFullName
                                + " ADD col4 integer, CF1.col5 integer, CF2.col6 integer, CF3.col7 integer CF1.BLOCKSIZE=50000, CF1.IN_MEMORY=false, IN_MEMORY=true ");
        try (Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) {
            ColumnFamilyDescriptor[] columnFamilies = admin.getDescriptor(TableName.valueOf(dataTableFullName))
                    .getColumnFamilies();
            assertEquals(4, columnFamilies.length);
            assertEquals("CF1", columnFamilies[0].getNameAsString());
            assertFalse(columnFamilies[0].isInMemory());
            assertEquals(0, columnFamilies[0].getScope());
            assertEquals(50000, columnFamilies[0].getBlocksize());
            assertEquals("CF2", columnFamilies[1].getNameAsString());
            assertTrue(columnFamilies[1].isInMemory());
            assertEquals(0, columnFamilies[1].getScope());
            assertEquals("CF3", columnFamilies[2].getNameAsString());
            assertTrue(columnFamilies[2].isInMemory());
            assertEquals(0, columnFamilies[2].getScope());
            assertEquals("XYZ", columnFamilies[3].getNameAsString());
            assertTrue(columnFamilies[3].isInMemory());
            assertEquals(0, columnFamilies[3].getScope());
        }
    } finally {
        conn.close();
    }
}
 
Example 19
Source Project: hbase   Source File: ProtobufUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Converts an TableDescriptor to TableSchema
 * @param htd the TableDescriptor
 * @return Convert the current {@link TableDescriptor} into a pb TableSchema instance.
 */
public static TableSchema toTableSchema(TableDescriptor htd) {
  TableSchema.Builder builder = TableSchema.newBuilder();
  builder.setTableName(toProtoTableName(htd.getTableName()));
  for (Map.Entry<Bytes, Bytes> e : htd.getValues().entrySet()) {
    BytesBytesPair.Builder aBuilder = BytesBytesPair.newBuilder();
    aBuilder.setFirst(UnsafeByteOperations.unsafeWrap(e.getKey().get()));
    aBuilder.setSecond(UnsafeByteOperations.unsafeWrap(e.getValue().get()));
    builder.addAttributes(aBuilder.build());
  }
  for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
    builder.addColumnFamilies(toColumnFamilySchema(hcd));
  }
  return builder.build();
}
 
Example 20
Source Project: hbase   Source File: TestAtomicOperation.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test written as a verifier for HBASE-7051, CheckAndPut should properly read
 * MVCC.
 *
 * Moved into TestAtomicOperation from its original location, TestHBase7051
 */
@Test
public void testPutAndCheckAndPutInParallel() throws Exception {
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setClass(HConstants.REGION_IMPL, MockHRegion.class, HeapSize.class);
  TableDescriptorBuilder tableDescriptorBuilder =
    TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
  ColumnFamilyDescriptor columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family)).build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  this.region = TEST_UTIL.createLocalHRegion(tableDescriptorBuilder.build(), null, null);
  Put[] puts = new Put[1];
  Put put = new Put(Bytes.toBytes("r1"));
  put.addColumn(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("10"));
  puts[0] = put;

  region.batchMutate(puts, HConstants.NO_NONCE, HConstants.NO_NONCE);
  MultithreadedTestUtil.TestContext ctx =
    new MultithreadedTestUtil.TestContext(conf);
  ctx.addThread(new PutThread(ctx, region));
  ctx.addThread(new CheckAndPutThread(ctx, region));
  ctx.startThreads();
  while (testStep != TestStep.CHECKANDPUT_COMPLETED) {
    Thread.sleep(100);
  }
  ctx.stop();
  Scan s = new Scan();
  RegionScanner scanner = region.getScanner(s);
  List<Cell> results = new ArrayList<>();
  ScannerContext scannerContext = ScannerContext.newBuilder().setBatchLimit(2).build();
  scanner.next(results, scannerContext);
  for (Cell keyValue : results) {
    assertEquals("50",Bytes.toString(CellUtil.cloneValue(keyValue)));
  }
}
 
Example 21
Source Project: atlas   Source File: HBaseAtlasHook.java    License: Apache License 2.0 5 votes vote down vote up
private HBaseOperationContext handleHBaseTableOperation(TableDescriptor tableDescriptor, TableName tableName, OPERATION operation, UserGroupInformation ugi, String userName) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("==> HBaseAtlasHook.handleHBaseTableOperation()");
    }

    Map<String, String>  hbaseConf          = null;
    String               owner              = null;
    String               tableNameSpace     = null;
    TableName            hbaseTableName     = null;
    ColumnFamilyDescriptor[]  columnFamilyDescriptors = null;

    if (tableDescriptor != null) {
        owner = tableDescriptor.getOwnerString();
        hbaseConf = null;
        hbaseTableName = tableDescriptor.getTableName();
        if (hbaseTableName != null) {
            tableNameSpace = hbaseTableName.getNamespaceAsString();
            if (tableNameSpace == null) {
                tableNameSpace = hbaseTableName.getNameWithNamespaceInclAsString();
            }
        }
    }

    if (owner == null) {
        owner = userName;
    }

    if (tableDescriptor != null) {
        columnFamilyDescriptors = tableDescriptor.getColumnFamilies();
    }

    HBaseOperationContext hbaseOperationContext = new HBaseOperationContext(tableNameSpace, tableDescriptor, tableName, columnFamilyDescriptors, operation, ugi, userName, owner, hbaseConf);
    createAtlasInstances(hbaseOperationContext);

    if (LOG.isDebugEnabled()) {
        LOG.debug("<== HBaseAtlasHook.handleHBaseTableOperation(): {}",  hbaseOperationContext);
    }
    return hbaseOperationContext;
}
 
Example 22
Source Project: hbase   Source File: TestNamespaceAuditor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCloneSnapshot() throws Exception {
  String nsp = prefix + "_testCloneSnapshot";
  NamespaceDescriptor nspDesc =
      NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2")
          .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build();
  ADMIN.createNamespace(nspDesc);
  assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp));
  TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1");
  TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2");

  ColumnFamilyDescriptor columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("fam1")).build();
  TableDescriptorBuilder tableDescOne = TableDescriptorBuilder
    .newBuilder(tableName);
  tableDescOne.setColumnFamily(columnFamilyDescriptor);

  ADMIN.createTable(tableDescOne.build(), Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4);
  String snapshot = "snapshot_testCloneSnapshot";
  ADMIN.snapshot(snapshot, tableName);
  ADMIN.cloneSnapshot(snapshot, cloneTableName);

  int tableLength;
  try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) {
    tableLength = locator.getStartKeys().length;
  }
  assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength);

  try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) {
    tableLength = locator.getStartKeys().length;
  }
  assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength);

  NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp);
  assertEquals("Total tables count should be 2.", 2, nstate.getTables().size());
  assertEquals("Total regions count should be.", 8, nstate.getRegionCount());

  ADMIN.deleteSnapshot(snapshot);
}
 
Example 23
Source Project: phoenix-omid   Source File: HBaseShims.java    License: Apache License 2.0 5 votes vote down vote up
public static boolean OmidCompactionEnabled(ObserverContext<RegionCoprocessorEnvironment> env,
                              Store store,
                              String cfFlagValue) {
    TableDescriptor desc = env.getEnvironment().getRegion().getTableDescriptor();
    ColumnFamilyDescriptor famDesc = desc.getColumnFamily(Bytes.toBytes(store.getColumnFamilyName()));
    return Boolean.valueOf(Bytes.toString(famDesc.getValue(Bytes.toBytes(cfFlagValue))));
}
 
Example 24
Source Project: phoenix-omid   Source File: HBaseShims.java    License: Apache License 2.0 5 votes vote down vote up
public static void setCompaction(Connection conn, TableName table, byte[] columnFamily, String key, String value)
        throws IOException {
    try(Admin admin = conn.getAdmin()) {
        TableDescriptor desc = admin.getDescriptor(table);
        ColumnFamilyDescriptor cfDesc = desc.getColumnFamily(columnFamily);
        ColumnFamilyDescriptorBuilder cfBuilder = ColumnFamilyDescriptorBuilder.newBuilder(cfDesc);
        cfBuilder.setValue(Bytes.toBytes(key),Bytes.toBytes(value));
        admin.modifyColumnFamily(table, cfBuilder.build());
    }
}
 
Example 25
Source Project: phoenix-tephra   Source File: TransactionProcessor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void start(CoprocessorEnvironment e) throws IOException {
  if (e instanceof RegionCoprocessorEnvironment) {
    RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
    this.cacheSupplier = getTransactionStateCacheSupplier(env);
    this.cache = cacheSupplier.get();

    TableDescriptor tableDesc = env.getRegion().getTableDescriptor();
    for (ColumnFamilyDescriptor columnDesc : tableDesc.getColumnFamilies()) {
      byte[] columnTTL = columnDesc.getValue(Bytes.toBytes(TxConstants.PROPERTY_TTL));
      long ttl = 0;
      if (columnTTL != null) {
        try {
          ttl = Long.parseLong(Bytes.toString(columnTTL));
          LOG.info("Family " + columnDesc.getNameAsString() + " has TTL of " + ttl);
        } catch (NumberFormatException nfe) {
          LOG.warn("Invalid TTL value configured for column family "
              + columnDesc.getNameAsString() + ", value = " + Bytes.toString(columnTTL));
        }
      }
      ttlByFamily.put(columnDesc.getName(), ttl);
    }

    this.allowEmptyValues = getAllowEmptyValues(env, tableDesc);
    this.txMaxLifetimeMillis = getTxMaxLifetimeMillis(env);
    this.readNonTxnData = Boolean.valueOf(tableDesc.getValue(TxConstants.READ_NON_TX_DATA));
    if (readNonTxnData) {
      LOG.info("Reading pre-existing data enabled for table "
          + tableDesc.getTableName().getNameAsString());
    }
    initializePruneState(env);
  }
}
 
Example 26
Source Project: phoenix   Source File: UpgradeUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Synchronize certain properties across column families of view index tables for a given base table
 * @param cqs CQS object to get table descriptor from PTable
 * @param baseTable base table
 * @param defaultColFam column family to be used for synchronizing properties
 * @param syncedProps Map of properties to be kept in sync as read from the default column family descriptor
 * @param tableDescsToSync set of modified table descriptors
 */
private static void syncViewIndexTable(ConnectionQueryServices cqs, PTable baseTable, ColumnFamilyDescriptor defaultColFam,
        Map<String, Object> syncedProps, Set<TableDescriptor> tableDescsToSync) throws SQLException {
    String viewIndexName = MetaDataUtil.getViewIndexPhysicalName(baseTable.getPhysicalName().getString());
    if (!Strings.isNullOrEmpty(viewIndexName)) {
        try {
            addTableDescIfPropsChanged(cqs.getTableDescriptor(Bytes.toBytes(viewIndexName)),
                    defaultColFam, syncedProps, tableDescsToSync);
        } catch (TableNotFoundException ignore) {
            // Ignore since this means that a view index table does not exist for this table
        }
    }
}
 
Example 27
Source Project: hbase   Source File: ProtobufUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Converts a ColumnFamilySchema to ColumnFamilyDescriptor
 * @param cfs the ColumnFamilySchema
 * @return An {@link ColumnFamilyDescriptor} made from the passed in <code>cfs</code>
 */
public static ColumnFamilyDescriptor toColumnFamilyDescriptor(final ColumnFamilySchema cfs) {
  // Use the empty constructor so we preserve the initial values set on construction for things
  // like maxVersion.  Otherwise, we pick up wrong values on deserialization which makes for
  // unrelated-looking test failures that are hard to trace back to here.
  ColumnFamilyDescriptorBuilder builder
    = ColumnFamilyDescriptorBuilder.newBuilder(cfs.getName().toByteArray());
  cfs.getAttributesList().forEach(a -> builder.setValue(a.getFirst().toByteArray(), a.getSecond().toByteArray()));
  cfs.getConfigurationList().forEach(a -> builder.setConfiguration(a.getName(), a.getValue()));
  return builder.build();
}
 
Example 28
@Override
public TableDescriptor preModifyTable(ObserverContext<MasterCoprocessorEnvironment> env,
    TableName tableName, final TableDescriptor currentDescriptor,
    final TableDescriptor newDescriptor) throws IOException {
  TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(newDescriptor);
  for (ColumnFamilyDescriptor cfd : newDescriptor.getColumnFamilies()) {
    builder.modifyColumnFamily(
        ColumnFamilyDescriptorBuilder.newBuilder(cfd).setMaxVersions(1).build());
  }
  return builder.build();
}
 
Example 29
Source Project: hbase   Source File: TestPrefetch.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPrefetchSetInHCDWorks() {
  ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
      .newBuilder(Bytes.toBytes("f")).setPrefetchBlocksOnOpen(true).build();
  Configuration c = HBaseConfiguration.create();
  assertFalse(c.getBoolean(CacheConfig.PREFETCH_BLOCKS_ON_OPEN_KEY, false));
  CacheConfig cc =
      new CacheConfig(c, columnFamilyDescriptor, blockCache, ByteBuffAllocator.HEAP);
  assertTrue(cc.shouldPrefetchOnOpen());
}
 
Example 30
Source Project: hbase   Source File: ThriftAdmin.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void modifyColumnFamily(TableName tableName, ColumnFamilyDescriptor columnFamily)
    throws IOException {
  TTableName tTableName = ThriftUtilities.tableNameFromHBase(tableName);
  TColumnFamilyDescriptor tColumnFamilyDescriptor = ThriftUtilities
      .columnFamilyDescriptorFromHBase(columnFamily);
  try {
    client.modifyColumnFamily(tTableName, tColumnFamilyDescriptor);
  } catch (TException e) {
    throw new IOException(e);
  }
}