org.apache.hadoop.hbase.util.EnvironmentEdgeManager Java Examples

The following examples show how to use org.apache.hadoop.hbase.util.EnvironmentEdgeManager. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestWALSplit.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static void appendCompactionEvent(Writer w, RegionInfo hri, String[] inputs,
    String output) throws IOException {
  WALProtos.CompactionDescriptor.Builder desc = WALProtos.CompactionDescriptor.newBuilder();
  desc.setTableName(ByteString.copyFrom(hri.getTable().toBytes()))
      .setEncodedRegionName(ByteString.copyFrom(hri.getEncodedNameAsBytes()))
      .setRegionName(ByteString.copyFrom(hri.getRegionName()))
      .setFamilyName(ByteString.copyFrom(FAMILY))
      .setStoreHomeDir(hri.getEncodedName() + "/" + Bytes.toString(FAMILY))
      .addAllCompactionInput(Arrays.asList(inputs))
      .addCompactionOutput(output);

  WALEdit edit = WALEdit.createCompaction(hri, desc.build());
  WALKeyImpl key = new WALKeyImpl(hri.getEncodedNameAsBytes(), TABLE_NAME, 1,
      EnvironmentEdgeManager.currentTime(), HConstants.DEFAULT_CLUSTER_ID);
  w.append(new Entry(key, edit));
  w.sync(false);
}
 
Example #2
Source File: CoveredColumnsIndexBuilder.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
/**
 * Batch all the {@link KeyValue}s in a collection of kvs by timestamp. Updates any
 * {@link KeyValue} with a timestamp == {@link HConstants#LATEST_TIMESTAMP} to the timestamp at
 * the time the method is called.
 * @param kvs {@link KeyValue}s to break into batches
 * @param batches to update with the given kvs
 */
protected void createTimestampBatchesFromKeyValues(Collection<KeyValue> kvs,
    Map<Long, Batch> batches) {
  long now = EnvironmentEdgeManager.currentTimeMillis();
  byte[] nowBytes = Bytes.toBytes(now);

  // batch kvs by timestamp
  for (KeyValue kv : kvs) {
    long ts = kv.getTimestamp();
    // override the timestamp to the current time, so the index and primary tables match
    // all the keys with LATEST_TIMESTAMP will then be put into the same batch
    if (kv.updateLatestStamp(nowBytes)) {
      ts = now;
    }
    Batch batch = batches.get(ts);
    if (batch == null) {
      batch = new Batch(ts);
      batches.put(ts, batch);
    }
    batch.add(kv);
  }
}
 
Example #3
Source File: MetaTableAccessor.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Deletes some replica columns corresponding to replicas for the passed rows
 * @param metaRows rows in hbase:meta
 * @param replicaIndexToDeleteFrom the replica ID we would start deleting from
 * @param numReplicasToRemove how many replicas to remove
 * @param connection connection we're using to access meta table
 */
public static void removeRegionReplicasFromMeta(Set<byte[]> metaRows,
  int replicaIndexToDeleteFrom, int numReplicasToRemove, Connection connection)
  throws IOException {
  int absoluteIndex = replicaIndexToDeleteFrom + numReplicasToRemove;
  for (byte[] row : metaRows) {
    long now = EnvironmentEdgeManager.currentTime();
    Delete deleteReplicaLocations = new Delete(row);
    for (int i = replicaIndexToDeleteFrom; i < absoluteIndex; i++) {
      deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY,
        CatalogFamilyFormat.getServerColumn(i), now);
      deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY,
        CatalogFamilyFormat.getSeqNumColumn(i), now);
      deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY,
        CatalogFamilyFormat.getStartCodeColumn(i), now);
      deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY,
        CatalogFamilyFormat.getServerNameColumn(i), now);
      deleteReplicaLocations.addColumns(HConstants.CATALOG_FAMILY,
        CatalogFamilyFormat.getRegionStateColumn(i), now);
    }

    deleteFromMetaTable(connection, deleteReplicaLocations);
  }
}
 
Example #4
Source File: MobFileCompactionChore.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void startCompaction(Admin admin, TableName table, RegionInfo region, byte[] cf)
    throws IOException, InterruptedException {

  LOG.info("Started major compaction: table={} cf={} region={}", table,
    Bytes.toString(cf), region.getRegionNameAsString());
  admin.majorCompactRegion(region.getRegionName(), cf);
  // Wait until it really starts
  // but with finite timeout
  long waitTime = 300000; // 5 min
  long startTime = EnvironmentEdgeManager.currentTime();
  while (admin.getCompactionStateForRegion(region.getRegionName()) == CompactionState.NONE) {
    // Is 1 second too aggressive?
    Thread.sleep(1000);
    if (EnvironmentEdgeManager.currentTime() - startTime > waitTime) {
      LOG.warn("Waited for {} ms to start major MOB compaction on table={} cf={} region={}."+
        " Stopped waiting for request confirmation. This is not an ERROR, continue next region."
        , waitTime, table.getNameAsString(), Bytes.toString(cf),region.getRegionNameAsString());
      break;
    }
  }
}
 
Example #5
Source File: MetaDataEndpointImpl.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Override
public MetaDataMutationResult getTable(byte[] tenantId, byte[] schemaName, byte[] tableName, long tableTimeStamp, long clientTimeStamp) throws IOException {
    try {
        byte[] key = SchemaUtil.getTableKey(tenantId, schemaName, tableName);
        
        // get the co-processor environment
        RegionCoprocessorEnvironment env = getEnvironment();
        // TODO: check that key is within region.getStartKey() and region.getEndKey()
        // and return special code to force client to lookup region from meta.
        HRegion region = env.getRegion();
        MetaDataMutationResult result = checkTableKeyInRegion(key, region);
        if (result != null) {
            return result; 
        }
        
        long currentTime = EnvironmentEdgeManager.currentTimeMillis();
        PTable table = doGetTable(key, clientTimeStamp);
        if (table == null) {
            return new MetaDataMutationResult(MutationCode.TABLE_NOT_FOUND, currentTime, null);
        }
        return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, currentTime, table.getTimeStamp() != tableTimeStamp ? table : null);
    } catch (Throwable t) {
        ServerUtil.throwIOException(SchemaUtil.getTableName(schemaName, tableName), t);
        return null; // impossible
    }
}
 
Example #6
Source File: AbstractRpcClient.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void cleanupIdleConnections() {
  long closeBeforeTime = EnvironmentEdgeManager.currentTime() - minIdleTimeBeforeClose;
  synchronized (connections) {
    for (T conn : connections.values()) {
      // Remove connection if it has not been chosen by anyone for more than maxIdleTime, and the
      // connection itself has already shutdown. The latter check is because we may still
      // have some pending calls on connection so we should not shutdown the connection outside.
      // The connection itself will disconnect if there is no pending call for maxIdleTime.
      if (conn.getLastTouched() < closeBeforeTime && !conn.isActive()) {
        if (LOG.isTraceEnabled()) {
          LOG.trace("Cleanup idle connection to {}", conn.remoteId().address);
        }
        connections.removeValue(conn.remoteId(), conn);
        conn.cleanupConnection();
      }
    }
  }
}
 
Example #7
Source File: MetaTableAccessor.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Overwrites the specified regions from hbase:meta. Deletes old rows for the given regions and
 * adds new ones. Regions added back have state CLOSED.
 * @param connection connection we're using
 * @param regionInfos list of regions to be added to META
 */
public static void overwriteRegions(Connection connection, List<RegionInfo> regionInfos,
  int regionReplication) throws IOException {
  // use master time for delete marker and the Put
  long now = EnvironmentEdgeManager.currentTime();
  deleteRegionInfos(connection, regionInfos, now);
  // Why sleep? This is the easiest way to ensure that the previous deletes does not
  // eclipse the following puts, that might happen in the same ts from the server.
  // See HBASE-9906, and HBASE-9879. Once either HBASE-9879, HBASE-8770 is fixed,
  // or HBASE-9905 is fixed and meta uses seqIds, we do not need the sleep.
  //
  // HBASE-13875 uses master timestamp for the mutations. The 20ms sleep is not needed
  addRegionsToMeta(connection, regionInfos, regionReplication, now + 1);
  LOG.info("Overwritten " + regionInfos.size() + " regions to Meta");
  LOG.debug("Overwritten regions: {} ", regionInfos);
}
 
Example #8
Source File: FIFOCompactionPolicy.java    From hbase with Apache License 2.0 6 votes vote down vote up
private Collection<HStoreFile> getExpiredStores(Collection<HStoreFile> files,
    Collection<HStoreFile> filesCompacting) {
  long currentTime = EnvironmentEdgeManager.currentTime();
  Collection<HStoreFile> expiredStores = new ArrayList<>();
  for (HStoreFile sf : files) {
    if (isEmptyStoreFile(sf) && !filesCompacting.contains(sf)) {
      expiredStores.add(sf);
      continue;
    }
    // Check MIN_VERSIONS is in HStore removeUnneededFiles
    long maxTs = sf.getReader().getMaxTimestamp();
    long maxTtl = storeConfigInfo.getStoreFileTtl();
    if (maxTtl == Long.MAX_VALUE || (currentTime - maxTtl < maxTs)) {
      continue;
    } else if (filesCompacting == null || !filesCompacting.contains(sf)) {
      expiredStores.add(sf);
    }
  }
  return expiredStores;
}
 
Example #9
Source File: HFileArchiver.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Archive the store file
 * @param fs the filesystem where the store files live
 * @param regionInfo region hosting the store files
 * @param conf {@link Configuration} to examine to determine the archive directory
 * @param tableDir {@link Path} to where the table is being stored (for building the archive path)
 * @param family the family hosting the store files
 * @param storeFile file to be archived
 * @throws IOException if the files could not be correctly disposed.
 */
public static void archiveStoreFile(Configuration conf, FileSystem fs, RegionInfo regionInfo,
    Path tableDir, byte[] family, Path storeFile) throws IOException {
  Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, regionInfo, tableDir, family);
  // make sure we don't archive if we can't and that the archive dir exists
  if (!fs.mkdirs(storeArchiveDir)) {
    throw new IOException("Could not make archive directory (" + storeArchiveDir + ") for store:"
        + Bytes.toString(family) + ", deleting compacted files instead.");
  }

  // do the actual archive
  long start = EnvironmentEdgeManager.currentTime();
  File file = new FileablePath(fs, storeFile);
  if (!resolveAndArchiveFile(storeArchiveDir, file, Long.toString(start))) {
    throw new IOException("Failed to archive/delete the file for region:"
        + regionInfo.getRegionNameAsString() + ", family:" + Bytes.toString(family)
        + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.");
  }
}
 
Example #10
Source File: AbstractTestDateTieredCompactionPolicy.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected ArrayList<HStoreFile> sfCreate(long[] minTimestamps, long[] maxTimestamps, long[] sizes)
    throws IOException {
  ManualEnvironmentEdge timeMachine = new ManualEnvironmentEdge();
  EnvironmentEdgeManager.injectEdge(timeMachine);
  // Has to be > 0 and < now.
  timeMachine.setValue(1);
  ArrayList<Long> ageInDisk = new ArrayList<>();
  for (int i = 0; i < sizes.length; i++) {
    ageInDisk.add(0L);
  }

  ArrayList<HStoreFile> ret = Lists.newArrayList();
  for (int i = 0; i < sizes.length; i++) {
    MockHStoreFile msf =
        new MockHStoreFile(TEST_UTIL, TEST_FILE, sizes[i], ageInDisk.get(i), false, i);
    msf.setTimeRangeTracker(TimeRangeTracker.create(TimeRangeTracker.Type.SYNC, minTimestamps[i], maxTimestamps[i]));
    ret.add(msf);
  }
  return ret;
}
 
Example #11
Source File: TestRateLimiter.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testCanExecuteOfFixedIntervalRateLimiter() throws InterruptedException {
  RateLimiter limiter = new FixedIntervalRateLimiter();
  // when set limit is 100 per sec, this FixedIntervalRateLimiter will support at max 100 per sec
  limiter.set(100, TimeUnit.SECONDS);
  limiter.setNextRefillTime(EnvironmentEdgeManager.currentTime());
  assertEquals(50, testCanExecuteByRate(limiter, 50));

  // refill the avail to limit
  limiter.set(100, TimeUnit.SECONDS);
  limiter.setNextRefillTime(EnvironmentEdgeManager.currentTime());
  assertEquals(100, testCanExecuteByRate(limiter, 100));

  // refill the avail to limit
  limiter.set(100, TimeUnit.SECONDS);
  limiter.setNextRefillTime(EnvironmentEdgeManager.currentTime());
  assertEquals(100, testCanExecuteByRate(limiter, 200));
}
 
Example #12
Source File: AuthenticationTokenSecretManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
synchronized void removeExpiredKeys() {
  if (!leaderElector.isMaster()) {
    LOG.info("Skipping removeExpiredKeys() because not running as master.");
    return;
  }

  long now = EnvironmentEdgeManager.currentTime();
  Iterator<AuthenticationKey> iter = allKeys.values().iterator();
  while (iter.hasNext()) {
    AuthenticationKey key = iter.next();
    if (key.getExpiration() < now) {
      LOG.debug("Removing expired key {}", key);
      iter.remove();
      zkWatcher.removeKeyFromZK(key);
    }
  }
}
 
Example #13
Source File: TableOverAsyncTable.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public void batch(List<? extends Row> actions, Object[] results) throws IOException {
  if (ArrayUtils.isEmpty(results)) {
    FutureUtils.get(table.batchAll(actions));
    return;
  }
  List<ThrowableWithExtraContext> errors = new ArrayList<>();
  List<CompletableFuture<Object>> futures = table.batch(actions);
  for (int i = 0, n = results.length; i < n; i++) {
    try {
      results[i] = FutureUtils.get(futures.get(i));
    } catch (IOException e) {
      results[i] = e;
      errors.add(new ThrowableWithExtraContext(e, EnvironmentEdgeManager.currentTime(),
        "Error when processing " + actions.get(i)));
    }
  }
  if (!errors.isEmpty()) {
    throw new RetriesExhaustedException(errors.size(), errors);
  }
}
 
Example #14
Source File: HFileArchiver.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Removes from the specified region the store files of the specified column family,
 * either by archiving them or outright deletion
 * @param fs the filesystem where the store files live
 * @param conf {@link Configuration} to examine to determine the archive directory
 * @param parent Parent region hosting the store files
 * @param familyDir {@link Path} to where the family is being stored
 * @param family the family hosting the store files
 * @throws IOException if the files could not be correctly disposed.
 */
public static void archiveFamilyByFamilyDir(FileSystem fs, Configuration conf,
    RegionInfo parent, Path familyDir, byte[] family) throws IOException {
  FileStatus[] storeFiles = CommonFSUtils.listStatus(fs, familyDir);
  if (storeFiles == null) {
    LOG.debug("No files to dispose of in {}, family={}", parent.getRegionNameAsString(),
        Bytes.toString(family));
    return;
  }

  FileStatusConverter getAsFile = new FileStatusConverter(fs);
  Collection<File> toArchive = Stream.of(storeFiles).map(getAsFile).collect(Collectors.toList());
  Path storeArchiveDir = HFileArchiveUtil.getStoreArchivePath(conf, parent, family);

  // do the actual archive
  List<File> failedArchive = resolveAndArchive(fs, storeArchiveDir, toArchive,
      EnvironmentEdgeManager.currentTime());
  if (!failedArchive.isEmpty()){
    throw new FailedArchiveException("Failed to archive/delete all the files for region:"
        + Bytes.toString(parent.getRegionName()) + ", family:" + Bytes.toString(family)
        + " into " + storeArchiveDir + ". Something is probably awry on the filesystem.",
        failedArchive.stream().map(FUNC_FILE_TO_PATH).collect(Collectors.toList()));
  }
}
 
Example #15
Source File: TestDeadServer.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testSortExtract(){
  ManualEnvironmentEdge mee = new ManualEnvironmentEdge();
  EnvironmentEdgeManager.injectEdge(mee);
  mee.setValue(1);

  DeadServer d = new DeadServer();

  d.putIfAbsent(hostname123);
  mee.incValue(1);
  d.putIfAbsent(hostname1234);
  mee.incValue(1);
  d.putIfAbsent(hostname12345);

  List<Pair<ServerName, Long>> copy = d.copyDeadServersSince(2L);
  Assert.assertEquals(2, copy.size());

  Assert.assertEquals(hostname1234, copy.get(0).getFirst());
  Assert.assertEquals(new Long(2L), copy.get(0).getSecond());

  Assert.assertEquals(hostname12345, copy.get(1).getFirst());
  Assert.assertEquals(new Long(3L), copy.get(1).getSecond());

  EnvironmentEdgeManager.reset();
}
 
Example #16
Source File: MockHStoreFile.java    From hbase with Apache License 2.0 5 votes vote down vote up
MockHStoreFile(HBaseTestingUtility testUtil, Path testPath,
    long length, long ageInDisk, boolean isRef, long sequenceid) throws IOException {
  super(testUtil.getTestFileSystem(), testPath, testUtil.getConfiguration(),
      new CacheConfig(testUtil.getConfiguration()), BloomType.NONE, true);
  this.length = length;
  this.isRef = isRef;
  this.ageInDisk = ageInDisk;
  this.sequenceid = sequenceid;
  this.isMajor = false;
  hdfsBlocksDistribution = new HDFSBlocksDistribution();
  hdfsBlocksDistribution.addHostsAndBlockWeight(new String[]
    { DNS.getHostname(testUtil.getConfiguration(), DNS.ServerType.REGIONSERVER) }, 1);
  modificationTime = EnvironmentEdgeManager.currentTime();
}
 
Example #17
Source File: SplitLogManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Add a task entry to coordination if it is not already there.
 * @param taskname the path of the log to be split
 * @param batch the batch this task belongs to
 * @return true if a new entry is created, false if it is already there.
 */
boolean enqueueSplitTask(String taskname, TaskBatch batch) {
  lastTaskCreateTime = EnvironmentEdgeManager.currentTime();
  String task = getSplitLogManagerCoordination().prepareTask(taskname);
  Task oldtask = createTaskIfAbsent(task, batch);
  if (oldtask == null) {
    // publish the task in the coordination engine
    getSplitLogManagerCoordination().submitTask(task);
    return true;
  }
  return false;
}
 
Example #18
Source File: TestAppendTimeRange.java    From hbase with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setupBeforeClass() throws Exception {
  util.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
      MyObserver.class.getName());
  // Make general delay zero rather than default. Timing is off in this
  // test that depends on an evironment edge that is manually moved forward.
  util.getConfiguration().setInt(RemoteProcedureDispatcher.DISPATCH_DELAY_CONF_KEY, 0);
  util.startMiniCluster();
  EnvironmentEdgeManager.injectEdge(mee);
}
 
Example #19
Source File: RemoteProcedureDispatcher.java    From hbase with Apache License 2.0 5 votes vote down vote up
public void awaitTermination() {
  try {
    final long startTime = EnvironmentEdgeManager.currentTime();
    for (int i = 0; isAlive(); ++i) {
      sendStopSignal();
      join(250);
      if (i > 0 && (i % 8) == 0) {
        LOG.warn("Waiting termination of thread " + getName() + ", " +
          StringUtils.humanTimeDiff(EnvironmentEdgeManager.currentTime() - startTime));
      }
    }
  } catch (InterruptedException e) {
    LOG.warn(getName() + " join wait got interrupted", e);
  }
}
 
Example #20
Source File: SplitTableRegionProcedure.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Calculate daughter regionid to use.
 * @param hri Parent {@link RegionInfo}
 * @return Daughter region id (timestamp) to use.
 */
private static long getDaughterRegionIdTimestamp(final RegionInfo hri) {
  long rid = EnvironmentEdgeManager.currentTime();
  // Regionid is timestamp.  Can't be less than that of parent else will insert
  // at wrong location in hbase:meta (See HBASE-710).
  if (rid < hri.getRegionId()) {
    LOG.warn("Clock skew; parent regions id is " + hri.getRegionId() +
      " but current time here is " + rid);
    rid = hri.getRegionId() + 1;
  }
  return rid;
}
 
Example #21
Source File: TestUserScanQueryMatcher.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testMatchWhenFilterReturnsIncludeAndSeekNextRow() throws IOException {
  List<MatchCode> expected = new ArrayList<>();
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
  expected.add(ScanQueryMatcher.MatchCode.DONE);

  Scan scanWithFilter = new Scan(scan).setFilter(new AlwaysIncludeAndSeekNextRowFilter());

  long now = EnvironmentEdgeManager.currentTime();

  // scan with column 2,4,5
  UserScanQueryMatcher qm = UserScanQueryMatcher.create(
    scanWithFilter, new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE,
        HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
    get.getFamilyMap().get(fam2), now - ttl, now, null);

  List<KeyValue> memstore = new ArrayList<>();
  // ColumnTracker will return INCLUDE_AND_SEEK_NEXT_COL , and filter will return
  // INCLUDE_AND_SEEK_NEXT_ROW, so final match code will be INCLUDE_AND_SEEK_NEXT_ROW.
  memstore.add(new KeyValue(row1, fam2, col2, 1, data));
  memstore.add(new KeyValue(row2, fam1, col1, data));

  List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(memstore.size());
  KeyValue k = memstore.get(0);
  qm.setToNewRow(k);

  for (KeyValue kv : memstore) {
    actual.add(qm.match(kv));
  }

  assertEquals(expected.size(), actual.size());
  for (int i = 0; i < expected.size(); i++) {
    LOG.debug("expected " + expected.get(i) + ", actual " + actual.get(i));
    assertEquals(expected.get(i), actual.get(i));
  }
}
 
Example #22
Source File: TestIncrementTimeRange.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void checkHTableInterfaceMethods() throws Exception {
  long time = EnvironmentEdgeManager.currentTime();
  mee.setValue(time);
  hTableInterface.put(new Put(ROW_A).addColumn(TEST_FAMILY, qualifierCol1, Bytes.toBytes(1L)));
  checkRowValue(ROW_A, Bytes.toBytes(1L));

  time = EnvironmentEdgeManager.currentTime();
  mee.setValue(time);
  TimeRange range10 = new TimeRange(1, time+10);
  hTableInterface.increment(new Increment(ROW_A).addColumn(TEST_FAMILY, qualifierCol1, 10L)
      .setTimeRange(range10.getMin(), range10.getMax()));
  checkRowValue(ROW_A, Bytes.toBytes(11L));
  assertEquals(MyObserver.tr10.getMin(), range10.getMin());
  assertEquals(MyObserver.tr10.getMax(), range10.getMax());

  time = EnvironmentEdgeManager.currentTime();
  mee.setValue(time);
  TimeRange range2 = new TimeRange(1, time+20);
  List<Row> actions =
      Arrays.asList(new Row[] { new Increment(ROW_A).addColumn(TEST_FAMILY, qualifierCol1, 2L)
          .setTimeRange(range2.getMin(), range2.getMax()),
          new Increment(ROW_A).addColumn(TEST_FAMILY, qualifierCol1, 2L)
          .setTimeRange(range2.getMin(), range2.getMax()) });
  Object[] results3 = new Object[actions.size()];
  Object[] results1 = results3;
  hTableInterface.batch(actions, results1);
  assertEquals(MyObserver.tr2.getMin(), range2.getMin());
  assertEquals(MyObserver.tr2.getMax(), range2.getMax());
  for (Object r2 : results1) {
    assertTrue(r2 instanceof Result);
  }
  checkRowValue(ROW_A, Bytes.toBytes(15L));

  hTableInterface.close();
}
 
Example #23
Source File: TestDefaultMemStore.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the timeOfOldestEdit is updated correctly for the
 * various edit operations in memstore.
 * @throws Exception
 */
@Test
public void testUpdateToTimeOfOldestEdit() throws Exception {
  try {
    EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
    EnvironmentEdgeManager.injectEdge(edge);
    DefaultMemStore memstore = new DefaultMemStore();
    long t = memstore.timeOfOldestEdit();
    assertEquals(Long.MAX_VALUE, t);

    // test the case that the timeOfOldestEdit is updated after a KV add
    memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"), null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
    // snapshot() will reset timeOfOldestEdit. The method will also assert the
    // value is reset to Long.MAX_VALUE
    t = runSnapshot(memstore);

    // test the case that the timeOfOldestEdit is updated after a KV delete
    memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, KeyValue.Type.Delete, "v"), null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
    t = runSnapshot(memstore);

    // test the case that the timeOfOldestEdit is updated after a KV upsert
    List<Cell> l = new ArrayList<>();
    KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
    kv1.setSequenceId(100);
    l.add(kv1);
    memstore.upsert(l, 1000, null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
  } finally {
    EnvironmentEdgeManager.reset();
  }
}
 
Example #24
Source File: RegionLocationFinder.java    From hbase with Apache License 2.0 5 votes vote down vote up
public void setClusterMetrics(ClusterMetrics status) {
  long currentTime = EnvironmentEdgeManager.currentTime();
  this.status = status;
  if (currentTime > lastFullRefresh + (CACHE_TIME / 2)) {
    // Only count the refresh if it includes user tables ( eg more than meta and namespace ).
    lastFullRefresh = scheduleFullRefresh()?currentTime:lastFullRefresh;
  }

}
 
Example #25
Source File: WALEdit.java    From hbase with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
public static WALEdit createRegionEventWALEdit(byte [] rowForRegion,
    RegionEventDescriptor regionEventDesc) {
  KeyValue kv = new KeyValue(rowForRegion, METAFAMILY,
      createRegionEventDescriptorQualifier(regionEventDesc.getEventType()),
      EnvironmentEdgeManager.currentTime(), regionEventDesc.toByteArray());
  return new WALEdit().add(kv, METAFAMILY);
}
 
Example #26
Source File: ExpiredMobFileCleaner.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Cleans the MOB files when they're expired and their min versions are 0.
 * If the latest timestamp of Cells in a MOB file is older than the TTL in the column family,
 * it's regarded as expired. This cleaner deletes them.
 * At a time T0, the cells in a mob file M0 are expired. If a user starts a scan before T0, those
 * mob cells are visible, this scan still runs after T0. At that time T1, this mob file M0
 * is expired, meanwhile a cleaner starts, the M0 is archived and can be read in the archive
 * directory.
 * @param tableName The current table name.
 * @param family The current family.
 */
public void cleanExpiredMobFiles(String tableName, ColumnFamilyDescriptor family)
    throws IOException {
  Configuration conf = getConf();
  TableName tn = TableName.valueOf(tableName);
  FileSystem fs = FileSystem.get(conf);
  LOG.info("Cleaning the expired MOB files of " + family.getNameAsString() + " in " + tableName);
  // disable the block cache.
  Configuration copyOfConf = new Configuration(conf);
  copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
  CacheConfig cacheConfig = new CacheConfig(copyOfConf);
  MobUtils.cleanExpiredMobFiles(fs, conf, tn, family, cacheConfig,
    EnvironmentEdgeManager.currentTime());
}
 
Example #27
Source File: MetaDataEndpointImpl.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
private static MetaDataMutationResult checkTableKeyInRegion(byte[] key, HRegion region) {
    byte[] startKey = region.getStartKey();
    byte[] endKey = region.getEndKey();
    if (Bytes.compareTo(startKey, key) <= 0 && (Bytes.compareTo(HConstants.LAST_ROW, endKey) == 0 || Bytes.compareTo(key, endKey) < 0)) {
        return null; // normal case;
    }
    return new MetaDataMutationResult(MutationCode.TABLE_NOT_IN_REGION, EnvironmentEdgeManager.currentTimeMillis(), null);
}
 
Example #28
Source File: TestWALSplit.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static void appendRegionEvent(Writer w, String region) throws IOException {
  WALProtos.RegionEventDescriptor regionOpenDesc = ProtobufUtil.toRegionEventDescriptor(
      WALProtos.RegionEventDescriptor.EventType.REGION_OPEN,
      TABLE_NAME.toBytes(),
      Bytes.toBytes(region),
      Bytes.toBytes(String.valueOf(region.hashCode())),
      1,
      ServerName.parseServerName("ServerName:9099"), ImmutableMap.<byte[], List<Path>>of());
  final long time = EnvironmentEdgeManager.currentTime();
  final WALKeyImpl walKey = new WALKeyImpl(Bytes.toBytes(region), TABLE_NAME, 1, time,
      HConstants.DEFAULT_CLUSTER_ID);
  WALEdit we = WALEdit.createRegionEventWALEdit(Bytes.toBytes(region), regionOpenDesc);
  w.append(new Entry(walKey, we));
  w.sync(false);
}
 
Example #29
Source File: DateTieredCompactionPolicy.java    From hbase with Apache License 2.0 5 votes vote down vote up
public CompactionRequestImpl selectMajorCompaction(ArrayList<HStoreFile> candidateSelection) {
  long now = EnvironmentEdgeManager.currentTime();
  List<Long> boundaries = getCompactBoundariesForMajor(candidateSelection, now);
  Map<Long, String> boundariesPolicies = getBoundariesStoragePolicyForMajor(boundaries, now);
  return new DateTieredCompactionRequest(candidateSelection,
    boundaries, boundariesPolicies);
}
 
Example #30
Source File: HBCKMetaTableAccessor.java    From hbase-operator-tools with Apache License 2.0 5 votes vote down vote up
/**
 * Updates state in META
 * @param conn connection to use
 * @param tableName table to look for
 */
public static void updateTableState(Connection conn, TableName tableName,
  TableState.State actual) throws IOException {
  Put put = makePutFromTableState(new TableState(tableName, actual),
    EnvironmentEdgeManager.currentTime());
  conn.getTable(TableName.META_TABLE_NAME).put(put);
}