Java Code Examples for org.apache.hadoop.hbase.client.SnapshotDescription

The following examples show how to use org.apache.hadoop.hbase.client.SnapshotDescription. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hbase   Source File: TestFlushSnapshotFromClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Helper method for testing async snapshot operations. Just waits for the given snapshot to
 * complete on the server by repeatedly checking the master.
 * @param master the master running the snapshot
 * @param snapshot the snapshot to check
 * @param timeoutNanos the timeout in nano between checks to see if the snapshot is done
 */
private static void waitForSnapshotToComplete(HMaster master,
    SnapshotProtos.SnapshotDescription snapshot, long timeoutNanos) throws Exception {
  final IsSnapshotDoneRequest request =
    IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot).build();
  long start = System.nanoTime();
  while (System.nanoTime() - start < timeoutNanos) {
    try {
      IsSnapshotDoneResponse done = master.getMasterRpcServices().isSnapshotDone(null, request);
      if (done.getDone()) {
        return;
      }
    } catch (ServiceException e) {
      // ignore UnknownSnapshotException, this is possible as for AsyncAdmin, the method will
      // return immediately after sending out the request, no matter whether the master has
      // processed the request or not.
      if (!(e.getCause() instanceof UnknownSnapshotException)) {
        throw e;
      }
    }

    Thread.sleep(200);
  }
  throw new TimeoutException("Timeout waiting for snapshot " + snapshot + " to complete");
}
 
Example 2
Source Project: atlas   Source File: HBaseAtlasCoprocessor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void postCloneSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("==> HBaseAtlasCoprocessor.postCloneSnapshot()");
    }

    try {
        activatePluginClassLoader();
        implMasterObserver.postCloneSnapshot(observerContext,snapshot,tableDescriptor);
    } finally {
        deactivatePluginClassLoader();
    }

    if (LOG.isDebugEnabled()) {
        LOG.debug("<== HBaseAtlasCoprocessor.postCloneSnapshot()");
    }
}
 
Example 3
Source Project: hbase   Source File: AccessController.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor)
    throws IOException {
  User user = getActiveUser(ctx);
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)
      && hTableDescriptor.getTableName().getNameAsString()
      .equals(snapshot.getTableNameAsString())) {
    // Snapshot owner is allowed to create a table with the same name as the snapshot he took
    AuthResult result = AuthResult.allow("cloneSnapshot " + snapshot.getName(),
      "Snapshot owner check allowed", user, null, hTableDescriptor.getTableName(), null);
    AccessChecker.logResult(result);
  } else {
    accessChecker.requirePermission(user, "cloneSnapshot " + snapshot.getName(), null,
      Action.ADMIN);
  }
}
 
Example 4
Source Project: hbase   Source File: SnapshotInfo.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns the list of available snapshots in the specified location
 * @param conf the {@link Configuration} to use
 * @return the list of snapshots
 */
public static List<SnapshotDescription> getSnapshotList(final Configuration conf)
    throws IOException {
  Path rootDir = CommonFSUtils.getRootDir(conf);
  FileSystem fs = FileSystem.get(rootDir.toUri(), conf);
  Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
  FileStatus[] snapshots = fs.listStatus(snapshotDir,
      new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
  List<SnapshotDescription> snapshotLists = new ArrayList<>(snapshots.length);
  for (FileStatus snapshotDirStat: snapshots) {
    SnapshotProtos.SnapshotDescription snapshotDesc =
        SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath());
    snapshotLists.add(ProtobufUtil.createSnapshotDesc(snapshotDesc));
  }
  return snapshotLists;
}
 
Example 5
Source Project: hbase   Source File: SnapshotTestingUtils.java    License: Apache License 2.0 6 votes vote down vote up
private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
    final int numRegions, final int version) throws IOException {
  TableDescriptor htd = createHtd(tableName);
  RegionData[] regions = createTable(htd, numRegions);

  SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder()
    .setTable(htd.getTableName().getNameAsString())
    .setName(snapshotName)
    .setVersion(version)
    .build();

  Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
  FileSystem workingFs = workingDir.getFileSystem(conf);
  SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, workingFs);
  return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
}
 
Example 6
Source Project: hbase   Source File: TestFileArchiverNotifierImpl.java    License: Apache License 2.0 6 votes vote down vote up
private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
  HashSet<String> files = new HashSet<>();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
      snapshotName, CommonFSUtils.getRootDir(conf));
  SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(
      fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
  // For each region referenced by the snapshot
  for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
    // For each column family in this region
    for (FamilyFiles ff : rm.getFamilyFilesList()) {
      // And each store file in that family
      for (StoreFile sf : ff.getStoreFilesList()) {
        files.add(sf.getName());
      }
    }
  }
  return files;
}
 
Example 7
Source Project: hbase   Source File: TestCloneSnapshotProcedure.java    License: Apache License 2.0 6 votes vote down vote up
private SnapshotProtos.SnapshotDescription getSnapshot() throws Exception {
  if (snapshot == null) {
    final TableName snapshotTableName = TableName.valueOf("testCloneSnapshot");
    long tid = System.currentTimeMillis();
    final String snapshotName = "snapshot-" + tid;

    Admin admin = UTIL.getAdmin();
    // create Table
    SnapshotTestingUtils.createTable(UTIL, snapshotTableName, getNumReplicas(), CF);
    // Load data
    SnapshotTestingUtils.loadData(UTIL, snapshotTableName, 500, CF);
    admin.disableTable(snapshotTableName);
    // take a snapshot
    admin.snapshot(snapshotName, snapshotTableName);
    admin.enableTable(snapshotTableName);

    List<SnapshotDescription> snapshotList = admin.listSnapshots();
    snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotList.get(0));
  }
  return snapshot;
}
 
Example 8
Source Project: hbase   Source File: TestCloneSnapshotProcedure.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCloneSnapshot() throws Exception {
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  final TableName clonedTableName = TableName.valueOf("testCloneSnapshot2");
  final TableDescriptor htd = createTableDescriptor(clonedTableName, CF);

  // take the snapshot
  SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();

  long procId = ProcedureTestingUtility.submitAndWait(
    procExec, new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc));
  ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
  MasterProcedureTestingUtility.validateTableIsEnabled(
    UTIL.getHBaseCluster().getMaster(),
    clonedTableName);
}
 
Example 9
Source Project: hbase   Source File: TestCloneSnapshotProcedure.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCloneSnapshotToSameTable() throws Exception {
  // take the snapshot
  SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot();

  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  final TableName clonedTableName = TableName.valueOf(snapshotDesc.getTable());
  final TableDescriptor htd = createTableDescriptor(clonedTableName, CF);

  long procId = ProcedureTestingUtility.submitAndWait(
    procExec, new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc));
  Procedure<?> result = procExec.getResult(procId);
  assertTrue(result.isFailed());
  LOG.debug("Clone snapshot failed with exception: " + result.getException());
  assertTrue(
    ProcedureTestingUtility.getExceptionCause(result) instanceof TableExistsException);
}
 
Example 10
Source Project: phoenix   Source File: TableSnapshotReadsMapReduceIT.java    License: Apache License 2.0 6 votes vote down vote up
private void upsertAndSnapshot(String tableName, boolean shouldSplit) throws Exception {
  upsertData(tableName);

  TableName hbaseTableName = TableName.valueOf(tableName);
  Connection conn = DriverManager.getConnection(getUrl());
  Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();

  if (shouldSplit) {
    splitTableSync(admin, hbaseTableName, "BBBB".getBytes(), 2);
  }

  admin.snapshot(SNAPSHOT_NAME, hbaseTableName);

  List<SnapshotDescription> snapshots = admin.listSnapshots();
  Assert.assertEquals(tableName, snapshots.get(0).getTable());

  // Capture the snapshot timestamp to use as SCN while reading the table later
  // Assigning the timestamp value here will make tests less flaky
  timestamp = System.currentTimeMillis();

  // upsert data after snapshot
  PreparedStatement stmt = conn.prepareStatement(String.format(UPSERT, tableName));
  upsertData(stmt, "DDDD", "SNFB", 45);
  conn.commit();
}
 
Example 11
Source Project: hbase   Source File: SnapshotTestingUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Make sure that there is only one snapshot returned from the master and its
 * name and table match the passed in parameters.
 */
public static List<SnapshotDescription> assertExistsMatchingSnapshot(
    Admin admin, String snapshotName, TableName tableName)
    throws IOException {
  // list the snapshot
  List<SnapshotDescription> snapshots = admin.listSnapshots();

  List<SnapshotDescription> returnedSnapshots = new ArrayList<>();
  for (SnapshotDescription sd : snapshots) {
    if (snapshotName.equals(sd.getName()) && tableName.equals(sd.getTableName())) {
      returnedSnapshots.add(sd);
    }
  }

  Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
  return returnedSnapshots;
}
 
Example 12
Source Project: atlas   Source File: HBaseAtlasCoprocessor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void postRestoreSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException {
    LOG.info("==> HBaseAtlasCoprocessor.postRestoreSnapshot()");

    hbaseAtlasHook.sendHBaseTableOperation(tableDescriptor, snapshot.getTableName(), HBaseAtlasHook.OPERATION.ALTER_TABLE, observerContext);

    if (LOG.isDebugEnabled()) {
        LOG.debug("<== HBaseAtlasCoprocessor.postRestoreSnapshot()");
    }
}
 
Example 13
Source Project: hbase   Source File: BackupSystemTable.java    License: Apache License 2.0 5 votes vote down vote up
protected static boolean snapshotExists(Admin admin, String snapshotName) throws IOException {
  List<SnapshotDescription> list = admin.listSnapshots();
  for (SnapshotDescription desc : list) {
    if (desc.getName().equals(snapshotName)) {
      return true;
    }
  }
  return false;
}
 
Example 14
Source Project: hbase   Source File: TestBackupDeleteWithFailures.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor)
    throws IOException {
  if (failures.contains(Failure.PRE_SNAPSHOT_FAILURE)) {
    throw new IOException("preSnapshot");
  }
}
 
Example 15
Source Project: hbase   Source File: TestSnapshotClientRetries.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor)
    throws IOException {
  if (cloneCount != null) {
    cloneCount.incrementAndGet();
  }
}
 
Example 16
Source Project: hbase   Source File: TestBackupDeleteWithFailures.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void postDeleteSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
    SnapshotDescription snapshot) throws IOException {
  if (failures.contains(Failure.POST_DELETE_SNAPSHOT_FAILURE)) {
    throw new IOException("postDeleteSnapshot");
  }
}
 
Example 17
Source Project: hbase   Source File: SnapshotTestingUtils.java    License: Apache License 2.0 5 votes vote down vote up
private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
    final int numRegions, final int version, final long ttl) throws IOException {
  TableDescriptor htd = createHtd(tableName);
  RegionData[] regions = createTable(htd, numRegions);
  SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder()
      .setTable(htd.getTableName().getNameAsString())
      .setName(snapshotName)
      .setVersion(version)
      .setCreationTime(EnvironmentEdgeManager.currentTime())
      .setTtl(ttl)
      .build();
  Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf);
  SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
  return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
}
 
Example 18
Source Project: hbase   Source File: AccessController.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preListSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot) throws IOException {
  User user = getActiveUser(ctx);
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)) {
    // list it, if user is the owner of snapshot
    AuthResult result = AuthResult.allow("listSnapshot " + snapshot.getName(),
        "Snapshot owner check allowed", user, null, null, null);
    AccessChecker.logResult(result);
  } else {
    accessChecker.requirePermission(user, "listSnapshot " + snapshot.getName(), null,
      Action.ADMIN);
  }
}
 
Example 19
Source Project: hbase   Source File: AccessController.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot) throws IOException {
  User user = getActiveUser(ctx);
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)) {
    // Snapshot owner is allowed to delete the snapshot
    AuthResult result = AuthResult.allow("deleteSnapshot " + snapshot.getName(),
        "Snapshot owner check allowed", user, null, null, null);
    AccessChecker.logResult(result);
  } else {
    accessChecker.requirePermission(user, "deleteSnapshot " + snapshot.getName(), null,
      Action.ADMIN);
  }
}
 
Example 20
Source Project: hbase   Source File: SnapshotScannerHDFSAclController.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void postCompletedSnapshotAction(ObserverContext<MasterCoprocessorEnvironment> c,
    SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException {
  if (needHandleTableHdfsAcl(tableDescriptor, "snapshot " + snapshot.getName())) {
    // Add HDFS acls of users with table read permission to snapshot files
    hdfsAclHelper.snapshotAcl(snapshot);
  }
}
 
Example 21
Source Project: hbase   Source File: SnapshotTestingUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Take a snapshot of the specified table and verify the given families.
 * Note that this will leave the table disabled in the case of an offline snapshot.
 */
public static void createSnapshotAndValidate(Admin admin,
    TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
    String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
      throws Exception {
  if (!onlineSnapshot) {
    try {
      LOG.info("prepping for offline snapshot.");
      admin.disableTable(tableName);
    } catch (TableNotEnabledException tne) {
      LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
          "already disabled.");
    }
  }
  LOG.info("taking snapshot.");
  admin.snapshot(snapshotNameString, tableName);

  LOG.info("Confirming snapshot exists.");
  List<SnapshotDescription> snapshots =
      SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName);
  if (snapshots == null || snapshots.size() != 1) {
    Assert.fail("Incorrect number of snapshots for table " + tableName);
  }

  LOG.info("validating snapshot.");
  SnapshotTestingUtils.confirmSnapshotValid(
    ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames,
    emptyFamilyNames, rootDir, admin, fs);
}
 
Example 22
Source Project: hbase   Source File: MasterCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
public void preSnapshot(final SnapshotDescription snapshot,
    final TableDescriptor hTableDescriptor) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.preSnapshot(this, snapshot, hTableDescriptor);
    }
  });
}
 
Example 23
Source Project: hbase   Source File: MasterCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
public void postSnapshot(final SnapshotDescription snapshot,
    final TableDescriptor hTableDescriptor) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.postSnapshot(this, snapshot, hTableDescriptor);
    }
  });
}
 
Example 24
Source Project: hbase   Source File: MasterCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
public void postCompletedSnapshotAction(SnapshotDescription snapshot,
    TableDescriptor hTableDescriptor) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.postCompletedSnapshotAction(this, snapshot, hTableDescriptor);
    }
  });
}
 
Example 25
Source Project: hbase   Source File: MasterCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
public void preListSnapshot(final SnapshotDescription snapshot) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.preListSnapshot(this, snapshot);
    }
  });
}
 
Example 26
Source Project: hbase   Source File: TestFlushSnapshotFromClient.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test snapshotting a table that is online without flushing
 */
@Test
public void testSkipFlushTableSnapshot() throws Exception {
  // make sure we don't fail on listing snapshots
  SnapshotTestingUtils.assertNoSnapshots(admin);

  // put some stuff in the table
  Table table = UTIL.getConnection().getTable(TABLE_NAME);
  UTIL.loadTable(table, TEST_FAM);
  UTIL.flush(TABLE_NAME);

  LOG.debug("FS state before snapshot:");
  UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);

  // take a snapshot of the enabled table
  String snapshotString = "skipFlushTableSnapshot";
  String snapshot = snapshotString;
  admin.snapshot(snapshotString, TABLE_NAME, SnapshotType.SKIPFLUSH);
  LOG.debug("Snapshot completed.");

  // make sure we have the snapshot
  List<SnapshotDescription> snapshots =
      SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);

  // make sure its a valid snapshot
  LOG.debug("FS state after snapshot:");
  UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);

  SnapshotTestingUtils.confirmSnapshotValid(UTIL,
    ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);

  admin.deleteSnapshot(snapshot);
  snapshots = admin.listSnapshots();
  SnapshotTestingUtils.assertNoSnapshots(admin);
}
 
Example 27
Source Project: hbase   Source File: MasterCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
public void preCloneSnapshot(final SnapshotDescription snapshot,
    final TableDescriptor hTableDescriptor) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.preCloneSnapshot(this, snapshot, hTableDescriptor);
    }
  });
}
 
Example 28
Source Project: hbase   Source File: MasterCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
public void preRestoreSnapshot(final SnapshotDescription snapshot,
    final TableDescriptor hTableDescriptor) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.preRestoreSnapshot(this, snapshot, hTableDescriptor);
    }
  });
}
 
Example 29
Source Project: hbase   Source File: MasterCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
public void postRestoreSnapshot(final SnapshotDescription snapshot,
    final TableDescriptor hTableDescriptor) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.postRestoreSnapshot(this, snapshot, hTableDescriptor);
    }
  });
}
 
Example 30
Source Project: hbase   Source File: MasterCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
public void postDeleteSnapshot(final SnapshotDescription snapshot) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.postDeleteSnapshot(this, snapshot);
    }
  });
}