org.apache.hadoop.hbase.client.SnapshotDescription Java Examples
The following examples show how to use
org.apache.hadoop.hbase.client.SnapshotDescription.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestFlushSnapshotFromClient.java From hbase with Apache License 2.0 | 6 votes |
/** * Helper method for testing async snapshot operations. Just waits for the given snapshot to * complete on the server by repeatedly checking the master. * @param master the master running the snapshot * @param snapshot the snapshot to check * @param timeoutNanos the timeout in nano between checks to see if the snapshot is done */ private static void waitForSnapshotToComplete(HMaster master, SnapshotProtos.SnapshotDescription snapshot, long timeoutNanos) throws Exception { final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder().setSnapshot(snapshot).build(); long start = System.nanoTime(); while (System.nanoTime() - start < timeoutNanos) { try { IsSnapshotDoneResponse done = master.getMasterRpcServices().isSnapshotDone(null, request); if (done.getDone()) { return; } } catch (ServiceException e) { // ignore UnknownSnapshotException, this is possible as for AsyncAdmin, the method will // return immediately after sending out the request, no matter whether the master has // processed the request or not. if (!(e.getCause() instanceof UnknownSnapshotException)) { throw e; } } Thread.sleep(200); } throw new TimeoutException("Timeout waiting for snapshot " + snapshot + " to complete"); }
Example #2
Source File: SnapshotInfo.java From hbase with Apache License 2.0 | 6 votes |
/** * Returns the list of available snapshots in the specified location * @param conf the {@link Configuration} to use * @return the list of snapshots */ public static List<SnapshotDescription> getSnapshotList(final Configuration conf) throws IOException { Path rootDir = CommonFSUtils.getRootDir(conf); FileSystem fs = FileSystem.get(rootDir.toUri(), conf); Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); FileStatus[] snapshots = fs.listStatus(snapshotDir, new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs)); List<SnapshotDescription> snapshotLists = new ArrayList<>(snapshots.length); for (FileStatus snapshotDirStat: snapshots) { SnapshotProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()); snapshotLists.add(ProtobufUtil.createSnapshotDesc(snapshotDesc)); } return snapshotLists; }
Example #3
Source File: AccessController.java From hbase with Apache License 2.0 | 6 votes |
@Override public void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx, final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor) throws IOException { User user = getActiveUser(ctx); if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user) && hTableDescriptor.getTableName().getNameAsString() .equals(snapshot.getTableNameAsString())) { // Snapshot owner is allowed to create a table with the same name as the snapshot he took AuthResult result = AuthResult.allow("cloneSnapshot " + snapshot.getName(), "Snapshot owner check allowed", user, null, hTableDescriptor.getTableName(), null); AccessChecker.logResult(result); } else { accessChecker.requirePermission(user, "cloneSnapshot " + snapshot.getName(), null, Action.ADMIN); } }
Example #4
Source File: SnapshotTestingUtils.java From hbase with Apache License 2.0 | 6 votes |
private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName, final int numRegions, final int version) throws IOException { TableDescriptor htd = createHtd(tableName); RegionData[] regions = createTable(htd, numRegions); SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder() .setTable(htd.getTableName().getNameAsString()) .setName(snapshotName) .setVersion(version) .build(); Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf); FileSystem workingFs = workingDir.getFileSystem(conf); SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, workingFs); return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions); }
Example #5
Source File: TestFileArchiverNotifierImpl.java From hbase with Apache License 2.0 | 6 votes |
private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException { HashSet<String> files = new HashSet<>(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir( snapshotName, CommonFSUtils.getRootDir(conf)); SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo( fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd); // For each region referenced by the snapshot for (SnapshotRegionManifest rm : manifest.getRegionManifests()) { // For each column family in this region for (FamilyFiles ff : rm.getFamilyFilesList()) { // And each store file in that family for (StoreFile sf : ff.getStoreFilesList()) { files.add(sf.getName()); } } } return files; }
Example #6
Source File: TestCloneSnapshotProcedure.java From hbase with Apache License 2.0 | 6 votes |
private SnapshotProtos.SnapshotDescription getSnapshot() throws Exception { if (snapshot == null) { final TableName snapshotTableName = TableName.valueOf("testCloneSnapshot"); long tid = System.currentTimeMillis(); final String snapshotName = "snapshot-" + tid; Admin admin = UTIL.getAdmin(); // create Table SnapshotTestingUtils.createTable(UTIL, snapshotTableName, getNumReplicas(), CF); // Load data SnapshotTestingUtils.loadData(UTIL, snapshotTableName, 500, CF); admin.disableTable(snapshotTableName); // take a snapshot admin.snapshot(snapshotName, snapshotTableName); admin.enableTable(snapshotTableName); List<SnapshotDescription> snapshotList = admin.listSnapshots(); snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotList.get(0)); } return snapshot; }
Example #7
Source File: TestCloneSnapshotProcedure.java From hbase with Apache License 2.0 | 6 votes |
@Test public void testCloneSnapshot() throws Exception { final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); final TableName clonedTableName = TableName.valueOf("testCloneSnapshot2"); final TableDescriptor htd = createTableDescriptor(clonedTableName, CF); // take the snapshot SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot(); long procId = ProcedureTestingUtility.submitAndWait( procExec, new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc)); ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); MasterProcedureTestingUtility.validateTableIsEnabled( UTIL.getHBaseCluster().getMaster(), clonedTableName); }
Example #8
Source File: TestCloneSnapshotProcedure.java From hbase with Apache License 2.0 | 6 votes |
@Test public void testCloneSnapshotToSameTable() throws Exception { // take the snapshot SnapshotProtos.SnapshotDescription snapshotDesc = getSnapshot(); final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); final TableName clonedTableName = TableName.valueOf(snapshotDesc.getTable()); final TableDescriptor htd = createTableDescriptor(clonedTableName, CF); long procId = ProcedureTestingUtility.submitAndWait( procExec, new CloneSnapshotProcedure(procExec.getEnvironment(), htd, snapshotDesc)); Procedure<?> result = procExec.getResult(procId); assertTrue(result.isFailed()); LOG.debug("Clone snapshot failed with exception: " + result.getException()); assertTrue( ProcedureTestingUtility.getExceptionCause(result) instanceof TableExistsException); }
Example #9
Source File: SnapshotTestingUtils.java From hbase with Apache License 2.0 | 6 votes |
/** * Make sure that there is only one snapshot returned from the master and its * name and table match the passed in parameters. */ public static List<SnapshotDescription> assertExistsMatchingSnapshot( Admin admin, String snapshotName, TableName tableName) throws IOException { // list the snapshot List<SnapshotDescription> snapshots = admin.listSnapshots(); List<SnapshotDescription> returnedSnapshots = new ArrayList<>(); for (SnapshotDescription sd : snapshots) { if (snapshotName.equals(sd.getName()) && tableName.equals(sd.getTableName())) { returnedSnapshots.add(sd); } } Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0); return returnedSnapshots; }
Example #10
Source File: HBaseAtlasCoprocessor.java From atlas with Apache License 2.0 | 6 votes |
@Override public void postCloneSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("==> HBaseAtlasCoprocessor.postCloneSnapshot()"); } try { activatePluginClassLoader(); implMasterObserver.postCloneSnapshot(observerContext,snapshot,tableDescriptor); } finally { deactivatePluginClassLoader(); } if (LOG.isDebugEnabled()) { LOG.debug("<== HBaseAtlasCoprocessor.postCloneSnapshot()"); } }
Example #11
Source File: TableSnapshotReadsMapReduceIT.java From phoenix with Apache License 2.0 | 6 votes |
private void upsertAndSnapshot(String tableName, boolean shouldSplit) throws Exception { upsertData(tableName); TableName hbaseTableName = TableName.valueOf(tableName); Connection conn = DriverManager.getConnection(getUrl()); Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin(); if (shouldSplit) { splitTableSync(admin, hbaseTableName, "BBBB".getBytes(), 2); } admin.snapshot(SNAPSHOT_NAME, hbaseTableName); List<SnapshotDescription> snapshots = admin.listSnapshots(); Assert.assertEquals(tableName, snapshots.get(0).getTable()); // Capture the snapshot timestamp to use as SCN while reading the table later // Assigning the timestamp value here will make tests less flaky timestamp = System.currentTimeMillis(); // upsert data after snapshot PreparedStatement stmt = conn.prepareStatement(String.format(UPSERT, tableName)); upsertData(stmt, "DDDD", "SNFB", 45); conn.commit(); }
Example #12
Source File: SnapshotInfo.java From hbase with Apache License 2.0 | 5 votes |
/** * Load snapshot info and table descriptor for the specified snapshot * @param snapshotName name of the snapshot to load * @return false if snapshot is not found */ private boolean loadSnapshotInfo(final String snapshotName) throws IOException { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); if (!fs.exists(snapshotDir)) { LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir); return false; } SnapshotProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc); return true; }
Example #13
Source File: MasterCoprocessorHost.java From hbase with Apache License 2.0 | 5 votes |
public void preSnapshot(final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor) throws IOException { execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { @Override public void call(MasterObserver observer) throws IOException { observer.preSnapshot(this, snapshot, hTableDescriptor); } }); }
Example #14
Source File: SnapshotTestingUtils.java From hbase with Apache License 2.0 | 5 votes |
/** * Take a snapshot of the specified table and verify the given families. * Note that this will leave the table disabled in the case of an offline snapshot. */ public static void createSnapshotAndValidate(Admin admin, TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames, String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot) throws Exception { if (!onlineSnapshot) { try { LOG.info("prepping for offline snapshot."); admin.disableTable(tableName); } catch (TableNotEnabledException tne) { LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " + "already disabled."); } } LOG.info("taking snapshot."); admin.snapshot(snapshotNameString, tableName); LOG.info("Confirming snapshot exists."); List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName); if (snapshots == null || snapshots.size() != 1) { Assert.fail("Incorrect number of snapshots for table " + tableName); } LOG.info("validating snapshot."); SnapshotTestingUtils.confirmSnapshotValid( ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames, emptyFamilyNames, rootDir, admin, fs); }
Example #15
Source File: MasterCoprocessorHost.java From hbase with Apache License 2.0 | 5 votes |
public void postSnapshot(final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor) throws IOException { execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { @Override public void call(MasterObserver observer) throws IOException { observer.postSnapshot(this, snapshot, hTableDescriptor); } }); }
Example #16
Source File: MasterCoprocessorHost.java From hbase with Apache License 2.0 | 5 votes |
public void preRestoreSnapshot(final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor) throws IOException { execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { @Override public void call(MasterObserver observer) throws IOException { observer.preRestoreSnapshot(this, snapshot, hTableDescriptor); } }); }
Example #17
Source File: MasterCoprocessorHost.java From hbase with Apache License 2.0 | 5 votes |
public void postRestoreSnapshot(final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor) throws IOException { execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { @Override public void call(MasterObserver observer) throws IOException { observer.postRestoreSnapshot(this, snapshot, hTableDescriptor); } }); }
Example #18
Source File: MasterCoprocessorHost.java From hbase with Apache License 2.0 | 5 votes |
public void postDeleteSnapshot(final SnapshotDescription snapshot) throws IOException { execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() { @Override public void call(MasterObserver observer) throws IOException { observer.postDeleteSnapshot(this, snapshot); } }); }
Example #19
Source File: SnapshotTestingUtils.java From hbase with Apache License 2.0 | 5 votes |
public static void deleteAllSnapshots(final Admin admin) throws IOException { // Delete all the snapshots for (SnapshotDescription snapshot: admin.listSnapshots()) { admin.deleteSnapshot(snapshot.getName()); } SnapshotTestingUtils.assertNoSnapshots(admin); }
Example #20
Source File: SnapshotTestingUtils.java From hbase with Apache License 2.0 | 5 votes |
/** * Confirm that the snapshot contains references to all the files that should * be in the snapshot. */ public static void confirmSnapshotValid(SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] testFamily, Path rootDir, Admin admin, FileSystem fs) throws IOException { ArrayList nonEmptyTestFamilies = new ArrayList(1); nonEmptyTestFamilies.add(testFamily); confirmSnapshotValid(snapshotDescriptor, tableName, nonEmptyTestFamilies, null, rootDir, admin, fs); }
Example #21
Source File: SnapshotInfo.java From hbase with Apache License 2.0 | 5 votes |
SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotProtos.SnapshotDescription snapshot) { this.snapshot = snapshot; this.snapshotTable = TableName.valueOf(snapshot.getTable()); this.conf = conf; this.fs = fs; }
Example #22
Source File: SnapshotInfo.java From hbase with Apache License 2.0 | 5 votes |
@Override public int doWork() throws IOException, InterruptedException { if (remoteDir != null) { URI defaultFs = remoteDir.getFileSystem(conf).getUri(); CommonFSUtils.setFsDefault(conf, new Path(defaultFs)); CommonFSUtils.setRootDir(conf, remoteDir); } // List Available Snapshots if (listSnapshots) { SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); System.out.printf("%-20s | %-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TTL IN SEC", "TABLE NAME"); for (SnapshotDescription desc: getSnapshotList(conf)) { System.out.printf("%-20s | %20s | %20s | %s%n", desc.getName(), df.format(new Date(desc.getCreationTime())), desc.getTtl(), desc.getTableNameAsString()); } return 0; } rootDir = CommonFSUtils.getRootDir(conf); fs = FileSystem.get(rootDir.toUri(), conf); LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir); // Load snapshot information if (!loadSnapshotInfo(snapshotName)) { System.err.println("Snapshot '" + snapshotName + "' not found!"); return 1; } printInfo(); if (showSchema) printSchema(); printFiles(showFiles, showStats); return 0; }
Example #23
Source File: SnapshotInfo.java From hbase with Apache License 2.0 | 5 votes |
/** * Dump the {@link SnapshotDescription} */ private void printInfo() { SnapshotProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription(); SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); System.out.println("Snapshot Info"); System.out.println("----------------------------------------"); System.out.println(" Name: " + snapshotDesc.getName()); System.out.println(" Type: " + snapshotDesc.getType()); System.out.println(" Table: " + snapshotDesc.getTable()); System.out.println(" Format: " + snapshotDesc.getVersion()); System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime()))); System.out.println(" Ttl: " + snapshotDesc.getTtl()); System.out.println(" Owner: " + snapshotDesc.getOwner()); System.out.println(); }
Example #24
Source File: SnapshotInfo.java From hbase with Apache License 2.0 | 5 votes |
/** * Returns the snapshot stats * @param conf the {@link Configuration} to use * @param snapshot {@link SnapshotDescription} to get stats from * @return the snapshot stats */ public static SnapshotStats getSnapshotStats(final Configuration conf, final SnapshotDescription snapshot) throws IOException { SnapshotProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot); return getSnapshotStats(conf, snapshotDesc, null); }
Example #25
Source File: HBaseAtlasCoprocessor.java From atlas with Apache License 2.0 | 5 votes |
@Override public void postRestoreSnapshot(ObserverContext<MasterCoprocessorEnvironment> observerContext, SnapshotDescription snapshot, TableDescriptor tableDescriptor) throws IOException { LOG.info("==> HBaseAtlasCoprocessor.postRestoreSnapshot()"); hbaseAtlasHook.sendHBaseTableOperation(tableDescriptor, snapshot.getTableName(), HBaseAtlasHook.OPERATION.ALTER_TABLE, observerContext); if (LOG.isDebugEnabled()) { LOG.debug("<== HBaseAtlasCoprocessor.postRestoreSnapshot()"); } }
Example #26
Source File: SnapshotTestingUtils.java From hbase with Apache License 2.0 | 5 votes |
public static void confirmSnapshotValid(HBaseTestingUtility testUtil, SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family) throws IOException { MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem(); confirmSnapshotValid(snapshotDescriptor, tableName, family, mfs.getRootDir(), testUtil.getAdmin(), mfs.getFileSystem()); }
Example #27
Source File: TestFlushSnapshotFromClient.java From hbase with Apache License 2.0 | 5 votes |
/** * Test simple flush snapshotting a table that is online */ @Test public void testFlushTableSnapshot() throws Exception { // make sure we don't fail on listing snapshots SnapshotTestingUtils.assertNoSnapshots(admin); // put some stuff in the table SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); LOG.debug("FS state before snapshot:"); UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); // take a snapshot of the enabled table String snapshotString = "offlineTableSnapshot"; byte[] snapshot = Bytes.toBytes(snapshotString); admin.snapshot(snapshotString, TABLE_NAME, SnapshotType.FLUSH); LOG.debug("Snapshot completed."); // make sure we have the snapshot List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME); // make sure its a valid snapshot LOG.debug("FS state after snapshot:"); UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); SnapshotTestingUtils.confirmSnapshotValid(UTIL, ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM); }
Example #28
Source File: TestSnapshotClientRetries.java From hbase with Apache License 2.0 | 5 votes |
@Override public void preSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx, final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor) throws IOException { if (snapshotCount != null) { snapshotCount.incrementAndGet(); } }
Example #29
Source File: SnapshotTestingUtils.java From hbase with Apache License 2.0 | 5 votes |
/** * Confirm that the snapshot has no references files but only metadata. */ public static void confirmEmptySnapshotValid( SnapshotProtos.SnapshotDescription snapshotDescriptor, TableName tableName, byte[] testFamily, Path rootDir, Admin admin, FileSystem fs) throws IOException { ArrayList emptyTestFamilies = new ArrayList(1); emptyTestFamilies.add(testFamily); confirmSnapshotValid(snapshotDescriptor, tableName, null, emptyTestFamilies, rootDir, admin, fs); }
Example #30
Source File: SnapshotTestingUtils.java From hbase with Apache License 2.0 | 5 votes |
/** * Make sure that there is only one snapshot returned from the master and its * name and table match the passed in parameters. */ public static List<SnapshotDescription> assertOneSnapshotThatMatches( Admin admin, String snapshotName, TableName tableName) throws IOException { // list the snapshot List<SnapshotDescription> snapshots = admin.listSnapshots(); assertEquals("Should only have 1 snapshot", 1, snapshots.size()); assertEquals(snapshotName, snapshots.get(0).getName()); assertEquals(tableName, snapshots.get(0).getTableName()); return snapshots; }