Java Code Examples for org.apache.hadoop.hbase.regionserver.HRegion

The following examples show how to use org.apache.hadoop.hbase.regionserver.HRegion. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hbase   Source File: HbckTableInfo.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException {
  errors.reportError(HbckErrorReporter.ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY,
      "Last region should end with an empty key.  Creating a new "
          + "region and regioninfo in HDFS to plug the hole.", getTableInfo());
  TableDescriptor htd = getTableInfo().getTableDescriptor();
  // from curEndKey to EMPTY_START_ROW
  RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName())
      .setStartKey(curEndKey)
      .setEndKey(HConstants.EMPTY_START_ROW)
      .build();

  HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd);
  LOG.info("Table region end key was not empty.  Created new empty region: " + newRegion
      + " " + region);
  hbck.fixes++;
}
 
Example 2
Source Project: hbase   Source File: TestCoprocessorConfiguration.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRegionCoprocessorHostDefaults() throws Exception {
  Configuration conf = new Configuration(CONF);
  HRegion region = mock(HRegion.class);
  when(region.getRegionInfo()).thenReturn(REGIONINFO);
  when(region.getTableDescriptor()).thenReturn(TABLEDESC);
  RegionServerServices rsServices = mock(RegionServerServices.class);
  systemCoprocessorLoaded.set(false);
  tableCoprocessorLoaded.set(false);
  new RegionCoprocessorHost(region, rsServices, conf);
  assertEquals("System coprocessors loading default was not honored",
      CoprocessorHost.DEFAULT_COPROCESSORS_ENABLED, systemCoprocessorLoaded.get());
  assertEquals("Table coprocessors loading default was not honored",
      CoprocessorHost.DEFAULT_COPROCESSORS_ENABLED &&
      CoprocessorHost.DEFAULT_USER_COPROCESSORS_ENABLED, tableCoprocessorLoaded.get());
}
 
Example 3
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 4
Source Project: hbase   Source File: TestCoprocessorConfiguration.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRegionCoprocessorHostAllDisabled() throws Exception {
  Configuration conf = new Configuration(CONF);
  conf.setBoolean(CoprocessorHost.COPROCESSORS_ENABLED_CONF_KEY, false);
  HRegion region = mock(HRegion.class);
  when(region.getRegionInfo()).thenReturn(REGIONINFO);
  when(region.getTableDescriptor()).thenReturn(TABLEDESC);
  RegionServerServices rsServices = mock(RegionServerServices.class);
  systemCoprocessorLoaded.set(false);
  tableCoprocessorLoaded.set(false);
  new RegionCoprocessorHost(region, rsServices, conf);
  assertFalse("System coprocessors should not have been loaded",
    systemCoprocessorLoaded.get());
  assertFalse("Table coprocessors should not have been loaded",
    tableCoprocessorLoaded.get());
}
 
Example 5
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion createRegion(String tableName, byte[] family, long ttl) throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
  HColumnDescriptor cfd = new HColumnDescriptor(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  htd.addFamily(cfd);
  htd.addCoprocessor(TransactionProcessor.class.getName());
  Path tablePath = FSUtils.getTableDir(FSUtils.getRootDir(conf), htd.getTableName());
  FileSystem fs = FileSystem.get(conf);
  assertTrue(fs.mkdirs(tablePath));
  WALFactory walFactory = new WALFactory(conf, null, tableName + ".hlog");
  WAL hLog = walFactory.getWAL(new byte[]{1}, null);
  HRegionInfo regionInfo = new HRegionInfo(TableName.valueOf(tableName));
  HRegionFileSystem regionFS = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tablePath, regionInfo);
  return new HRegion(regionFS, hLog, conf, htd,
      new LocalRegionServerServices(conf, ServerName.valueOf(
          InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())));
}
 
Example 6
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 6 votes vote down vote up
private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
  region.close();
  TableDescriptorBuilder tableBuilder =
      TableDescriptorBuilder.newBuilder(region.getTableDescriptor());
  ColumnFamilyDescriptorBuilder cfd =
      ColumnFamilyDescriptorBuilder.newBuilder(tableBuilder.build().getColumnFamily(family));
  if (ttl > 0) {
    cfd.setValue(Bytes.toBytes(TxConstants.PROPERTY_TTL), Bytes.toBytes(String.valueOf(ttl)));
  }
  cfd.setMaxVersions(10);
  tableBuilder.removeColumnFamily(family);
  tableBuilder.addColumnFamily(cfd.build());
  return HRegion
      .openHRegion(region.getRegionInfo(), tableBuilder.build(), region.getWAL(), conf,
        new LocalRegionServerServices(conf, ServerName
            .valueOf(InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())),
        null);
}
 
Example 7
Source Project: hbase   Source File: TestRegionStateStore.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testVisitMetaForRegionExistingRegion() throws Exception {
  final TableName tableName = TableName.valueOf("testVisitMetaForRegion");
  UTIL.createTable(tableName, "cf");
  final List<HRegion> regions = UTIL.getHBaseCluster().getRegions(tableName);
  final String encodedName = regions.get(0).getRegionInfo().getEncodedName();
  final RegionStateStore regionStateStore = UTIL.getHBaseCluster().getMaster().
    getAssignmentManager().getRegionStateStore();
  final AtomicBoolean visitorCalled = new AtomicBoolean(false);
  regionStateStore.visitMetaForRegion(encodedName, new RegionStateStore.RegionStateVisitor() {
    @Override
    public void visitRegionState(Result result, RegionInfo regionInfo, RegionState.State state,
      ServerName regionLocation, ServerName lastHost, long openSeqNum) {
      assertEquals(encodedName, regionInfo.getEncodedName());
      visitorCalled.set(true);
    }
  });
  assertTrue("Visitor has not been called.", visitorCalled.get());
}
 
Example 8
Source Project: hbase   Source File: HBaseTestingUtility.java    License: Apache License 2.0 6 votes vote down vote up
public HRegion createLocalHRegionWithInMemoryFlags(TableName tableName, byte[] startKey,
  byte[] stopKey, boolean isReadOnly, Durability durability, WAL wal, boolean[] compactedMemStore,
  byte[]... families) throws IOException {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  tableDescriptor.setReadOnly(isReadOnly);
  int i = 0;
  for (byte[] family : families) {
    ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family);
    if (compactedMemStore != null && i < compactedMemStore.length) {
      familyDescriptor.setInMemoryCompaction(MemoryCompactionPolicy.BASIC);
    } else {
      familyDescriptor.setInMemoryCompaction(MemoryCompactionPolicy.NONE);

    }
    i++;
    // Set default to be three versions.
    familyDescriptor.setMaxVersions(Integer.MAX_VALUE);
    tableDescriptor.setColumnFamily(familyDescriptor);
  }
  tableDescriptor.setDurability(durability);
  RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName())
    .setStartKey(startKey).setEndKey(stopKey).build();
  return createLocalHRegion(info, tableDescriptor, wal);
}
 
Example 9
Source Project: hbase   Source File: HBaseTestingUtility.java    License: Apache License 2.0 6 votes vote down vote up
public void verifyNumericRows(HRegion region, final byte[] f, int startRow, int endRow,
    final boolean present) throws IOException {
  for (int i = startRow; i < endRow; i++) {
    String failMsg = "Failed verification of row :" + i;
    byte[] data = Bytes.toBytes(String.valueOf(i));
    Result result = region.get(new Get(data));

    boolean hasResult = result != null && !result.isEmpty();
    assertEquals(failMsg + result, present, hasResult);
    if (!present) continue;

    assertTrue(failMsg, result.containsColumn(f, null));
    assertEquals(failMsg, 1, result.getColumnCells(f, null).size());
    Cell cell = result.getColumnLatestCell(f, null);
    assertTrue(failMsg,
      Bytes.equals(data, 0, data.length, cell.getValueArray(), cell.getValueOffset(),
        cell.getValueLength()));
  }
}
 
Example 10
Source Project: phoenix-tephra   Source File: TransactionProcessor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void postFlush(ObserverContext<RegionCoprocessorEnvironment> e) throws IOException {
  // Record whether the region is empty after a flush
  HRegion region = e.getEnvironment().getRegion();
  // After a flush, if the memstore size is zero and there are no store files for any stores in the region
  // then the region must be empty
  long numStoreFiles = numStoreFilesForRegion(e);
  long memstoreSize = region.getMemstoreSize().get();
  LOG.debug(String.format("Region %s: memstore size = %s, num store files = %s",
                          region.getRegionInfo().getRegionNameAsString(), memstoreSize, numStoreFiles));
  if (memstoreSize == 0 && numStoreFiles == 0) {
    if (compactionState != null) {
      compactionState.persistRegionEmpty(System.currentTimeMillis());
    }
  }
}
 
Example 11
Source Project: hbase   Source File: AbstractTestLogRolling.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that logs are deleted
 */
@Test
public void testLogRolling() throws Exception {
  this.tableName = getName();
  // TODO: Why does this write data take for ever?
  startAndWriteData();
  RegionInfo region = server.getRegions(TableName.valueOf(tableName)).get(0).getRegionInfo();
  final WAL log = server.getWAL(region);
  LOG.info("after writing there are " + AbstractFSWALProvider.getNumRolledLogFiles(log) + " log files");
  assertLogFileSize(log);

  // flush all regions
  for (HRegion r : server.getOnlineRegionsLocalContext()) {
    r.flush(true);
  }

  // Now roll the log
  log.rollWriter();

  int count = AbstractFSWALProvider.getNumRolledLogFiles(log);
  LOG.info("after flushing all regions and rolling logs there are " + count + " log files");
  assertTrue(("actual count: " + count), count <= 2);
  assertLogFileSize(log);
}
 
Example 12
Source Project: phoenix   Source File: WALRecoveryRegionPostOpenIT.java    License: Apache License 2.0 6 votes vote down vote up
private void moveRegionAndWait(MiniHBaseCluster miniHBaseCluster,HRegion destRegion, HRegionServer destRegionServer) throws IOException, InterruptedException {
    HMaster master = miniHBaseCluster.getMaster();
    getUtility().getHBaseAdmin().move(
            destRegion.getRegionInfo().getEncodedNameAsBytes(),
            Bytes.toBytes(destRegionServer.getServerName().getServerName()));
    while (true) {
        ServerName currentRegionServerName =
                master.getAssignmentManager().getRegionStates().getRegionServerOfRegion(destRegion.getRegionInfo());
        if (currentRegionServerName != null && currentRegionServerName.equals(destRegionServer.getServerName())) {
            getUtility().assertRegionOnServer(
                    destRegion.getRegionInfo(), currentRegionServerName, 200);
            break;
        }
        Thread.sleep(10);
    }
}
 
Example 13
Source Project: hbase   Source File: TestRegionObserverScannerOpenHook.java    License: Apache License 2.0 6 votes vote down vote up
HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
    byte[]... families) throws IOException {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(tableName));
  for (byte[] family : families) {
    tableDescriptor.setColumnFamily(
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family));
  }
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  Path path = new Path(DIR + callingMethod);
  WAL wal = HBaseTestingUtility.createWal(conf, path, info);
  HRegion r = HRegion.createHRegion(info, path, conf, tableDescriptor, wal);
  // this following piece is a hack. currently a coprocessorHost
  // is secretly loaded at OpenRegionHandler. we don't really
  // start a region server here, so just manually create cphost
  // and set it to region.
  RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf);
  r.setCoprocessorHost(host);
  return r;
}
 
Example 14
Source Project: hbase   Source File: TestMasterRegionFlush.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws IOException {
  conf = HBaseConfiguration.create();
  region = mock(HRegion.class);
  HStore store = mock(HStore.class);
  when(store.getStorefilesCount()).thenReturn(1);
  when(region.getStores()).thenReturn(Collections.singletonList(store));
  when(region.getRegionInfo())
    .thenReturn(RegionInfoBuilder.newBuilder(TableName.valueOf("hbase:local")).build());
  flushCalled = new AtomicInteger(0);
  memstoreHeapSize = new AtomicLong(0);
  memstoreOffHeapSize = new AtomicLong(0);
  when(region.getMemStoreHeapSize()).thenAnswer(invocation -> memstoreHeapSize.get());
  when(region.getMemStoreOffHeapSize()).thenAnswer(invocation -> memstoreOffHeapSize.get());
  when(region.flush(anyBoolean())).thenAnswer(invocation -> {
    assertTrue(invocation.getArgument(0));
    memstoreHeapSize.set(0);
    memstoreOffHeapSize.set(0);
    flushCalled.incrementAndGet();
    return null;
  });
}
 
Example 15
Source Project: phoenix   Source File: StatisticsCollector.java    License: Apache License 2.0 6 votes vote down vote up
public void splitStats(HRegion parent, HRegion left, HRegion right) {
    try {
        if (logger.isDebugEnabled()) {
            logger.debug("Collecting stats for split of " + parent.getRegionInfo() + " into " + left.getRegionInfo() + " and " + right.getRegionInfo());
        }
        List<Mutation> mutations = Lists.newArrayListWithExpectedSize(3);
        for (byte[] fam : parent.getStores().keySet()) {
        	statsTable.splitStats(parent, left, right, this, new ImmutableBytesPtr(fam), mutations);
        }
        if (logger.isDebugEnabled()) {
            logger.debug("Committing stats for the daughter regions as part of split " + parent.getRegionInfo());
        }
        commitStats(mutations);
    } catch (IOException e) {
        logger.error("Error while capturing stats after split of region "
                + parent.getRegionInfo().getRegionNameAsString(), e);
    }
}
 
Example 16
private PTable buildDeletedTable(byte[] key, ImmutableBytesPtr cacheKey, HRegion region, long clientTimeStamp) throws IOException {
    if (clientTimeStamp == HConstants.LATEST_TIMESTAMP) {
        return null;
    }
    
    Scan scan = newTableRowsScan(key, clientTimeStamp, HConstants.LATEST_TIMESTAMP);
    scan.setFilter(new FirstKeyOnlyFilter());
    scan.setRaw(true);
    RegionScanner scanner = region.getScanner(scan);
    List<KeyValue> results = Lists.<KeyValue>newArrayList();
    scanner.next(results);
    // HBase ignores the time range on a raw scan (HBASE-7362)
    if (!results.isEmpty() && results.get(0).getTimestamp() > clientTimeStamp) {
        KeyValue kv = results.get(0);
        if (kv.isDelete()) {
            Map<ImmutableBytesPtr,PTable> metaDataCache = GlobalCache.getInstance(this.getEnvironment()).getMetaDataCache();
            PTable table = newDeletedTableMarker(kv.getTimestamp());
            metaDataCache.put(cacheKey, table);
            return table;
        }
    }
    return null;
}
 
Example 17
Source Project: phoenix   Source File: MetaDataEndpointImpl.java    License: Apache License 2.0 6 votes vote down vote up
private PTable loadTable(RegionCoprocessorEnvironment env, byte[] key,
    ImmutableBytesPtr cacheKey, long clientTimeStamp, long asOfTimeStamp)
    throws IOException, SQLException {
    HRegion region = env.getRegion();
    Cache<ImmutableBytesPtr,PTable> metaDataCache = GlobalCache.getInstance(this.env).getMetaDataCache();
    PTable table = metaDataCache.getIfPresent(cacheKey);
    // We always cache the latest version - fault in if not in cache
    if (table != null || (table = buildTable(key, cacheKey, region, asOfTimeStamp)) != null) {
        return table;
    }
    // if not found then check if newer table already exists and add delete marker for timestamp
    // found
    if (table == null
            && (table = buildDeletedTable(key, cacheKey, region, clientTimeStamp)) != null) {
        return table;
    }
    return null;
}
 
Example 18
Source Project: hbase   Source File: TestVisibilityLabels.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFlushedFileWithVisibilityTags() throws Exception {
  final byte[] qual2 = Bytes.toBytes("qual2");
  TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam);
  tableDescriptor.setColumnFamily(familyDescriptor);
  TEST_UTIL.getAdmin().createTable(tableDescriptor);
  try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
    Put p1 = new Put(row1);
    p1.addColumn(fam, qual, value);
    p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));

    Put p2 = new Put(row1);
    p2.addColumn(fam, qual2, value);
    p2.setCellVisibility(new CellVisibility(SECRET));

    RowMutations rm = new RowMutations(row1);
    rm.add(p1);
    rm.add(p2);

    table.mutateRow(rm);
  }
  TEST_UTIL.getAdmin().flush(tableName);
  List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
  HStore store = regions.get(0).getStore(fam);
  Collection<HStoreFile> storefiles = store.getStorefiles();
  assertTrue(storefiles.size() > 0);
  for (HStoreFile storeFile : storefiles) {
    assertTrue(storeFile.getReader().getHFileReader().getFileContext().isIncludesTags());
  }
}
 
Example 19
Source Project: hbase   Source File: SyncReplicationTestBase.java    License: Apache License 2.0 5 votes vote down vote up
protected final void verifyNotReplicatedThroughRegion(HBaseTestingUtility util, int start,
    int end) throws IOException {
  HRegion region = util.getMiniHBaseCluster().getRegions(TABLE_NAME).get(0);
  for (int i = start; i < end; i++) {
    assertTrue(region.get(new Get(Bytes.toBytes(i))).isEmpty());
  }
}
 
Example 20
Source Project: Kylin   Source File: AggregateRegionObserver.java    License: Apache License 2.0 5 votes vote down vote up
private RegionScanner doPostScannerObserver(final ObserverContext<RegionCoprocessorEnvironment> ctxt, final Scan scan, final RegionScanner innerScanner) throws IOException {
    byte[] coprocessorEnableBytes = scan.getAttribute(COPROCESSOR_ENABLE);
    if (coprocessorEnableBytes == null || coprocessorEnableBytes.length == 0 || coprocessorEnableBytes[0] == 0) {
        return innerScanner;
    }

    byte[] typeBytes = scan.getAttribute(TYPE);
    CoprocessorRowType type = CoprocessorRowType.deserialize(typeBytes);

    byte[] projectorBytes = scan.getAttribute(PROJECTOR);
    CoprocessorProjector projector = CoprocessorProjector.deserialize(projectorBytes);

    byte[] aggregatorBytes = scan.getAttribute(AGGREGATORS);
    ObserverAggregators aggregators = ObserverAggregators.deserialize(aggregatorBytes);

    byte[] filterBytes = scan.getAttribute(FILTER);
    CoprocessorFilter filter = CoprocessorFilter.deserialize(filterBytes);

    // start/end region operation & sync on scanner is suggested by the
    // javadoc of RegionScanner.nextRaw()
    // FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly attribute helping here? by mhb
    HRegion region = ctxt.getEnvironment().getRegion();
    region.startRegionOperation();
    try {
        synchronized (innerScanner) {
            return new AggregationScanner(type, filter, projector, aggregators, innerScanner);
        }
    } finally {
        region.closeRegionOperation();
    }

}
 
Example 21
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 5 votes vote down vote up
private HRegion updateTtl(HRegion region, byte[] family, long ttl) throws Exception {
  region.close();
  HTableDescriptor htd = region.getTableDesc();
  HColumnDescriptor cfd = htd.getFamily(family);
  if (ttl > 0) {
    cfd.setValue(TxConstants.PROPERTY_TTL, String.valueOf(ttl));
  }
  cfd.setMaxVersions(10);
  return HRegion.openHRegion(region.getRegionInfo(), htd, region.getWAL(), conf,
                             new LocalRegionServerServices(conf, ServerName.valueOf(
                               InetAddress.getLocalHost().getHostName(), 0, System.currentTimeMillis())), null);
}
 
Example 22
Source Project: hbase   Source File: TestRefreshHFilesEndpoint.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRefreshRegionHFilesEndpoint() throws Exception {
  setUp(HRegion.class.getName());
  addHFilesToRegions();
  assertEquals(2, HTU.getNumHFiles(TABLE_NAME, FAMILY));
  callRefreshRegionHFilesEndPoint();
  assertEquals(4, HTU.getNumHFiles(TABLE_NAME, FAMILY));
}
 
Example 23
@Override
public void batchStarted(MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatchOp) throws IOException {
    // The entire purpose of this method impl is to get the existing rows for the
    // table rows being indexed into the block cache, as the index maintenance code
    // does a point scan per row
    List<KeyRange> keys = Lists.newArrayListWithExpectedSize(miniBatchOp.size());
    List<IndexMaintainer> maintainers = new ArrayList<IndexMaintainer>();
    for (int i = 0; i < miniBatchOp.size(); i++) {
        Mutation m = miniBatchOp.getOperation(i).getFirst();
        keys.add(PDataType.VARBINARY.getKeyRange(m.getRow()));
        maintainers.addAll(getCodec().getIndexMaintainers(m.getAttributesMap()));
    }
    Scan scan = IndexManagementUtil.newLocalStateScan(maintainers);
    ScanRanges scanRanges = ScanRanges.create(Collections.singletonList(keys), SchemaUtil.VAR_BINARY_SCHEMA);
    scanRanges.setScanStartStopRow(scan);
    scan.setFilter(scanRanges.getSkipScanFilter());
    HRegion region = this.env.getRegion();
    RegionScanner scanner = region.getScanner(scan);
    // Run through the scanner using internal nextRaw method
    MultiVersionConsistencyControl.setThreadReadPoint(scanner.getMvccReadPoint());
    region.startRegionOperation();
    try {
        boolean hasMore;
        do {
            List<KeyValue> results = Lists.newArrayList();
            // Results are potentially returned even when the return value of s.next is false
            // since this is an indication of whether or not there are more values after the
            // ones returned
            hasMore = scanner.nextRaw(results, null);
        } while (hasMore);
    } finally {
        try {
            scanner.close();
        } finally {
            region.closeRegionOperation();
        }
    }
}
 
Example 24
/**
 * If the end key from the request equals to the region start key, the request is meant for the previous region.
 * Ignore shuch requests.
 * @param request
 * @param region
 * @return
 */
public static boolean regionKeysMatch(SpliceMessage.PrepareBackupRequest request, HRegion region) {
    byte[] requestStartKey = request.hasStartKey() ? request.getStartKey().toByteArray() : new byte[0];
    byte[] requestEndKey = request.hasEndKey() ? request.getEndKey().toByteArray() : new byte[0];

    byte[] regionStartKey = region.getRegionInfo().getStartKey() != null? region.getRegionInfo().getStartKey() : new byte[0];
    byte[] regionEndKey = region.getRegionInfo().getEndKey() != null ? region.getRegionInfo().getEndKey() : new byte[0];

    return Bytes.compareTo(requestStartKey, regionStartKey) ==0 &&
            Bytes.compareTo(requestEndKey, regionEndKey) == 0;
}
 
Example 25
@SuppressWarnings("unchecked")
@Test
public void testCorrectOrderingWithLazyLoadingColumns() throws Exception {
  Put m = new Put(row);
  m.add(fam, qual, ts, val);
  // setup mocks
  Configuration conf = new Configuration(false);
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);
  Mockito.when(env.getConfiguration()).thenReturn(conf);

  HRegion region = Mockito.mock(HRegion.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  RegionScanner scanner = Mockito.mock(RegionScanner.class);
  Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner);
  final byte[] stored = Bytes.toBytes("stored-value");
  Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() {
    @Override
    public Boolean answer(InvocationOnMock invocation) throws Throwable {
      List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0];
      KeyValue kv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
      kv.setMemstoreTS(0);
      list.add(kv);
      return false;
    }
  });


  LocalHBaseState state = new LocalTable(env);
  LocalTableState table = new LocalTableState(env, state, m);
  //add the kvs from the mutation
  table.addPendingUpdates(m.get(fam, qual));

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  //check that our value still shows up first on scan, even though this is a lazy load
  Pair<Scanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col));
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", m.get(fam, qual).get(0), s.next());
}
 
Example 26
Source Project: phoenix   Source File: SequenceRegionObserver.java    License: Apache License 2.0 5 votes vote down vote up
private static void acquireLock(HRegion region, byte[] key, List<RowLock> locks)
    throws IOException {
    RowLock rowLock = region.getRowLock(key);
    if (rowLock == null) {
        throw new IOException("Failed to acquire lock on " + Bytes.toStringBinary(key));
    }
    locks.add(rowLock);
}
 
Example 27
Source Project: hbase   Source File: TestSimpleRegionNormalizerOnCluster.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * create a table with 5 regions, having region sizes so as to provoke a merge
 * of the smallest regions.
 * <ul>
 *   <li>total table size: 13</li>
 *   <li>average region size: 2.6</li>
 *   <li>sum of sizes of first two regions < average</li>
 * </ul>
 */
private static int createTableBegsMerge(final TableName tableName) throws IOException {
  // create 5 regions with sizes to trigger merge of small regions
  final List<HRegion> generatedRegions = generateTestData(tableName, 1, 1, 3, 3, 5);
  assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName));
  admin.flush(tableName);

  final TableDescriptor td = TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName))
    .setNormalizationEnabled(true)
    .build();
  admin.modifyTable(td);

  // make sure relatively accurate region statistics are available for the test table. use
  // the last/largest region as clue.
  LOG.debug("waiting for region statistics to settle.");
  TEST_UTIL.waitFor(TimeUnit.MINUTES.toMillis(1), new ExplainingPredicate<IOException>() {
    @Override public String explainFailure() {
      return "expected largest region to be >= 4mb.";
    }
    @Override public boolean evaluate() {
      return generatedRegions.stream()
        .mapToDouble(val -> getRegionSizeMB(master, val.getRegionInfo()))
        .allMatch(val -> val > 0)
        && getRegionSizeMB(master, generatedRegions.get(4).getRegionInfo()) >= 4.0;
    }
  });
  return 5;
}
 
Example 28
private PTable buildTable(byte[] key, ImmutableBytesPtr cacheKey, HRegion region, long clientTimeStamp) throws IOException, SQLException {
    Scan scan = newTableRowsScan(key, MIN_TABLE_TIMESTAMP, clientTimeStamp);
    RegionScanner scanner = region.getScanner(scan);
    Map<ImmutableBytesPtr,PTable> metaDataCache = GlobalCache.getInstance(this.getEnvironment()).getMetaDataCache();
    try {
        PTable oldTable = metaDataCache.get(cacheKey);
        long tableTimeStamp = oldTable == null ? MIN_TABLE_TIMESTAMP-1 : oldTable.getTimeStamp();
        PTable newTable;
        newTable = getTable(scanner, clientTimeStamp, tableTimeStamp);
        if (newTable == null) {
            return null;
        }
        if (oldTable == null || tableTimeStamp < newTable.getTimeStamp()) {
            if (logger.isDebugEnabled()) {
                logger.debug("Caching table " + Bytes.toStringBinary(cacheKey.get(), cacheKey.getOffset(), cacheKey.getLength()) + " at seqNum " + newTable.getSequenceNumber() + " with newer timestamp " + newTable.getTimeStamp() + " versus " + tableTimeStamp);
            }
            oldTable = metaDataCache.put(cacheKey, newTable);
            if (logger.isDebugEnabled()) {
                if (oldTable == null) {
                    logger.debug("No previously cached table " + Bytes.toStringBinary(cacheKey.get(), cacheKey.getOffset(), cacheKey.getLength()));
                } else {
                    logger.debug("Previously cached table " + Bytes.toStringBinary(cacheKey.get(), cacheKey.getOffset(), cacheKey.getLength()) + " was at seqNum " + oldTable.getSequenceNumber() + " with timestamp " + oldTable.getTimeStamp());
                }
            }
        }
        return newTable;
    } finally {
        scanner.close();
    }
}
 
Example 29
Source Project: hbase   Source File: WALPerformanceEvaluation.java    License: Apache License 2.0 5 votes vote down vote up
private void closeRegion(final HRegion region) throws IOException {
  if (region != null) {
    region.close();
    WAL wal = region.getWAL();
    if (wal != null) {
      wal.shutdown();
    }
  }
}
 
Example 30
Source Project: hbase   Source File: SpaceQuotaHelperForTests.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean evaluate() throws Exception {
  for (HRegion region : cluster.getRegions(tn)) {
    for (HStore store : region.getStores()) {
      Collection<HStoreFile> files =
          store.getStoreEngine().getStoreFileManager().getCompactedfiles();
      if (null != files && !files.isEmpty()) {
        LOG.debug(region.getRegionInfo().getEncodedName() + " still has compacted files");
        return false;
      }
    }
  }
  return true;
}