/* * Licensed to Crate under one or more contributor license agreements. * See the NOTICE file distributed with this work for additional * information regarding copyright ownership. Crate licenses this file * to you under the Apache License, Version 2.0 (the "License"); you may * not use this file except in compliance with the License. You may * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. * * However, if you have executed another commercial license agreement * with Crate these terms will supersede the license and you may use the * software solely pursuant to the terms of the relevant commercial * agreement. */ package io.crate.integrationtests; import com.google.common.base.Joiner; import io.crate.action.sql.SQLActionException; import io.crate.testing.SQLResponse; import io.crate.testing.TestingHelpers; import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.cluster.SnapshotsInProgress; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import io.crate.common.unit.TimeValue; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.snapshots.SnapshotInfo; import org.elasticsearch.snapshots.SnapshotState; import org.elasticsearch.threadpool.ThreadPool; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import java.io.File; import java.nio.file.Files; import java.nio.file.StandardOpenOption; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.concurrent.CountDownLatch; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; public class SnapshotRestoreIntegrationTest extends SQLTransportIntegrationTest { private static final String REPOSITORY_NAME = "my_repo"; private static final String SNAPSHOT_NAME = "my_snapshot"; @ClassRule public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder(); private File defaultRepositoryLocation; @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder().put(super.nodeSettings(nodeOrdinal)) .put("path.repo", TEMPORARY_FOLDER.getRoot().getAbsolutePath()) .build(); } @Before public void createRepository() throws Exception { defaultRepositoryLocation = TEMPORARY_FOLDER.newFolder(); execute("CREATE REPOSITORY " + REPOSITORY_NAME + " TYPE \"fs\" with (location=?, compress=True)", new Object[]{defaultRepositoryLocation.getAbsolutePath()}); assertThat(response.rowCount(), is(1L)); execute( "CREATE REPOSITORY my_repo_ro TYPE \"fs\" with (location=?, compress=true, readonly=true)", new Object[]{defaultRepositoryLocation.getAbsolutePath()} ); } private void createTableAndSnapshot(String tableName, String snapshotName) { createTableAndSnapshot(tableName, snapshotName, false); } private void createTableAndSnapshot(String tableName, String snapshotName, boolean partitioned) { createTable(tableName, partitioned); createSnapshot(snapshotName, tableName); } private void createTable(String tableName, boolean partitioned) { execute("CREATE TABLE " + tableName + " (" + " id long primary key, " + " name string, " + " date timestamp with time zone " + (partitioned ? "primary key," : ",") + " ft string index using fulltext with (analyzer='default')" + ") " + (partitioned ? "partitioned by (date) " : "") + "clustered into 1 shards with (number_of_replicas=0)"); execute("INSERT INTO " + tableName + " (id, name, date, ft) VALUES (?, ?, ?, ?)", new Object[][]{ {1L, "foo", "1970-01-01", "The quick brown fox jumps over the lazy dog."}, {2L, "bar", "2015-10-27T11:29:00+01:00", "Morgenstund hat Gold im Mund."}, {3L, "baz", "1989-11-09", "Reden ist Schweigen. Silber ist Gold."}, }); execute("REFRESH TABLE " + tableName); } private void createSnapshot(String snapshotName, String... tables) { execute("CREATE SNAPSHOT " + REPOSITORY_NAME + "." + snapshotName + " TABLE " + Joiner.on(", ").join(tables) + " WITH (wait_for_completion=true)"); assertThat(response.rowCount(), is(1L)); } private static String snapshotName() { return String.format(Locale.ENGLISH, "%s.%s", REPOSITORY_NAME, SNAPSHOT_NAME); } @Test public void testDropSnapshot() throws Exception { String snapshotName = "my_snap_1"; createTableAndSnapshot("my_table", snapshotName); execute("drop snapshot " + REPOSITORY_NAME + "." + snapshotName); assertThat(response.rowCount(), is(1L)); execute("select * from sys.snapshots where name = ?", new Object[]{snapshotName}); assertThat(response.rowCount(), is(0L)); assertAllRepoSnapshotFilesAreDeleted(defaultRepositoryLocation); } @Test public void testDropUnknownSnapshot() throws Exception { String snapshot = "unknown_snap"; expectedException.expect(SQLActionException.class); expectedException.expectMessage(String.format(Locale.ENGLISH, "Snapshot '%s.%s' unknown", REPOSITORY_NAME, snapshot)); execute("drop snapshot " + REPOSITORY_NAME + "." + snapshot); } @Test public void testDropSnapshotUnknownRepository() throws Exception { String repository = "unknown_repo"; String snapshot = "unknown_snap"; expectedException.expect(SQLActionException.class); expectedException.expectMessage(String.format(Locale.ENGLISH, "Repository '%s' unknown", repository)); execute("drop snapshot " + repository + "." + snapshot); } @Test public void testCreateSnapshot() throws Exception { createTable("backmeup", false); execute("CREATE SNAPSHOT " + snapshotName() + " TABLE backmeup WITH (wait_for_completion=true)"); assertThat(response.rowCount(), is(1L)); execute("select name, \"repository\", concrete_indices, state from sys.snapshots order by 2"); assertThat(TestingHelpers.printedTable(response.rows()), is(String.format( "my_snapshot| my_repo| [%s.backmeup]| SUCCESS\n" + // shows up twice because both repos have the same data path "my_snapshot| my_repo_ro| [%s.backmeup]| SUCCESS\n", sqlExecutor.getCurrentSchema(), sqlExecutor.getCurrentSchema()))); } @Test public void testCreateSnapshotWithoutWaitForCompletion() throws Exception { // this test just verifies that no exception is thrown if wait_for_completion is false execute("CREATE SNAPSHOT my_repo.snapshot_no_wait ALL WITH (wait_for_completion=false)"); assertThat(response.rowCount(), is(1L)); waitForCompletion(REPOSITORY_NAME, "snapshot_no_wait", TimeValue.timeValueSeconds(20)); } private SnapshotInfo waitForCompletion(String repository, String snapshotName, TimeValue timeout) throws InterruptedException { long start = System.currentTimeMillis(); Snapshot snapshot = new Snapshot(repository, new SnapshotId(repository, snapshotName)); while (System.currentTimeMillis() - start < timeout.millis()) { List<SnapshotInfo> snapshotInfos = client().admin().cluster().prepareGetSnapshots(repository).setSnapshots(snapshotName).get().getSnapshots(); assertThat(snapshotInfos.size(), equalTo(1)); if (snapshotInfos.get(0).state().completed()) { // Make sure that snapshot clean up operations are finished ClusterStateResponse stateResponse = client().admin().cluster().prepareState().get(); SnapshotsInProgress snapshotsInProgress = stateResponse.getState().custom(SnapshotsInProgress.TYPE); if (snapshotsInProgress == null || snapshotsInProgress.snapshot(snapshot) == null) { return snapshotInfos.get(0); } } Thread.sleep(100); } fail("Timeout waiting for snapshot completion!"); return null; } @Test public void testCreateSnapshotFromPartition() throws Exception { createTable("custom.backmeup", true); execute("CREATE SNAPSHOT " + snapshotName() + " TABLE custom.backmeup PARTITION (date='1970-01-01') WITH (wait_for_completion=true)"); assertThat(response.rowCount(), is(1L)); execute("select name, \"repository\", concrete_indices, state from sys.snapshots order by 2"); assertThat(TestingHelpers.printedTable(response.rows()), is("my_snapshot| my_repo| [custom..partitioned.backmeup.04130]| SUCCESS\n" + // shows up twice because the repos have the same fs path. "my_snapshot| my_repo_ro| [custom..partitioned.backmeup.04130]| SUCCESS\n")); } @Test public void testCreateSnapshotAllBlobsExcluded() throws Exception { execute("CREATE TABLE t1 (id INTEGER, name STRING)"); execute("CREATE BLOB TABLE b1"); ensureYellow(); execute("CREATE SNAPSHOT " + snapshotName() + " ALL WITH (wait_for_completion=true)"); assertThat(response.rowCount(), is(1L)); execute("select concrete_indices from sys.snapshots"); assertThat(response.rows()[0][0], is(List.of(getFqn("t1")))); } @Test public void testCreateExistingSnapshot() throws Exception { createTable("backmeup", randomBoolean()); execute("CREATE SNAPSHOT " + snapshotName() + " ALL WITH (wait_for_completion=true)"); assertThat(response.rowCount(), is(1L)); expectedException.expect(SQLActionException.class); expectedException.expectMessage("Invalid snapshot name [my_snapshot], snapshot with the same name already exists"); execute("CREATE SNAPSHOT " + snapshotName() + " ALL WITH (wait_for_completion=true)"); } @Test public void testCreateSnapshotUnknownRepo() throws Exception { expectedException.expect(SQLActionException.class); expectedException.expectMessage("Repository 'unknown_repo' unknown"); execute("CREATE SNAPSHOT unknown_repo.my_snapshot ALL WITH (wait_for_completion=true)"); } @Test public void testInvalidSnapshotName() throws Exception { expectedException.expect(SQLActionException.class); expectedException.expectMessage("Invalid snapshot name [MY_UPPER_SNAPSHOT], must be lowercase"); execute("CREATE SNAPSHOT my_repo.\"MY_UPPER_SNAPSHOT\" ALL WITH (wait_for_completion=true)"); } @Test public void testSnapshotWithMetadataDoesNotDeleteExistingStuff() throws Exception { createTable("my_other", true); execute("CREATE SNAPSHOT " + snapshotName() + " TABLE my_other with (wait_for_completion=true)"); execute("alter table my_other add column x double"); waitForMappingUpdateOnAll("my_other", "x"); execute("delete from my_other"); execute("CREATE TABLE survivor (bla string, blubb float) partitioned by (blubb) with (number_of_replicas=0)"); ensureYellow(); execute("insert into survivor (bla, blubb) values (?, ?)", new Object[][]{ {"foo", 1.2}, {"bar", 1.4}, {"baz", 1.2} }); execute("refresh table survivor"); execute("restore snapshot " + snapshotName() + " ALL with (wait_for_completion=true)"); execute("select * from survivor order by bla"); assertThat(TestingHelpers.printedTable(response.rows()), is( "bar| 1.4\n" + "baz| 1.2\n" + "foo| 1.2\n")); } @Test public void testSnapshotWithMetadataConcurrentlyModified() throws Exception { int shards = randomFrom(1, 3, 5); int replicas = randomIntBetween(2, 10); long documents = randomLongBetween(2, 100); execute("CREATE TABLE test (" + " id long primary key)" + "clustered into " + shards + " shards with (column_policy = 'dynamic', number_of_replicas=" + replicas + ")"); ensureYellow(); ActionFuture<SQLResponse> createSnapshot = null; // Insert data with dynamic column creation so we trigger a dynamic mapping update for each of them for (var i = 0; i < documents; i++) { execute("INSERT INTO test (id, field_" + i + ") VALUES (?, ?)", new Object[][]{{i, "value_" + i},}); execute("REFRESH TABLE test"); if (createSnapshot == null) { createSnapshot = sqlExecutor.execute( "CREATE SNAPSHOT " + snapshotName() + " TABLE test with (wait_for_completion=true)", null); } } if (createSnapshot != null) { createSnapshot.actionGet(); } execute("DROP table test"); execute("select state from sys.snapshots where name=?", new Object[]{SNAPSHOT_NAME}); assertThat(response.rows()[0][0], is("SUCCESS")); execute("RESTORE SNAPSHOT " + snapshotName() + " ALL with (wait_for_completion=true)"); waitNoPendingTasksOnAll(); SnapshotsInProgress finalSnapshotsInProgress = clusterService().state().custom(SnapshotsInProgress.TYPE); assertFalse(finalSnapshotsInProgress.entries().stream().anyMatch(entry -> entry.state().completed() == false)); ImmutableOpenMap<String, IndexMetaData> state = clusterService().state().metaData().indices(); IndexMetaData indexMetaData = state.values().iterator().next().value; int sizeOfProperties = ((Map<?, ?>) indexMetaData.getMappings().values().iterator().next().value.sourceAsMap().get("properties")).size(); execute("select count(*) from test"); assertThat( "Documents were restored but the restored index mapping was older than some documents and misses some of their fields", (Long)response.rows()[0][0], lessThanOrEqualTo((long) sizeOfProperties)); } @Test public void testRestoreSnapshotAll() throws Exception { createTableAndSnapshot("my_table", SNAPSHOT_NAME); execute("drop table my_table"); execute("RESTORE SNAPSHOT " + snapshotName() + " ALL with (" + "ignore_unavailable=false, " + "wait_for_completion=true)"); ensureGreen(); execute("select * from my_table order by id"); assertThat(response.rowCount(), is(3L)); } @Test public void testRestoreSnapshotSinglePartition() throws Exception { createTableAndSnapshot("my_parted_table", SNAPSHOT_NAME, true); waitNoPendingTasksOnAll(); execute("delete from my_parted_table"); waitNoPendingTasksOnAll(); execute("RESTORE SNAPSHOT " + snapshotName() + " TABLE my_parted_table PARTITION (date='1970-01-01') with (" + "ignore_unavailable=false, " + "wait_for_completion=true)"); execute("select date from my_parted_table"); assertThat(TestingHelpers.printedTable(response.rows()), is("0\n")); } @Test public void testRestoreSinglePartitionSnapshotIntoDroppedPartition() throws Exception { createTable("parted_table", true); execute("CREATE SNAPSHOT " + snapshotName() + " TABLE parted_table PARTITION (date=0) WITH (wait_for_completion=true)"); execute("delete from parted_table where date=0"); waitNoPendingTasksOnAll(); execute("RESTORE SNAPSHOT " + snapshotName() + " TABLE parted_table PARTITION (date=0) with (" + "ignore_unavailable=false, " + "wait_for_completion=true)"); execute("select date from parted_table order by id"); assertThat(TestingHelpers.printedTable(response.rows()), is("0\n1445941740000\n626572800000\n")); } @Test public void testRestoreSinglePartitionSnapshotIntoDroppedTable() throws Exception { createTable("parted_table", true); execute("CREATE SNAPSHOT " + snapshotName() + " TABLE parted_table PARTITION (date=0) WITH (wait_for_completion=true)"); execute("drop table parted_table"); waitNoPendingTasksOnAll(); execute("RESTORE SNAPSHOT " + snapshotName() + " TABLE parted_table PARTITION (date=0) with (" + "ignore_unavailable=false, " + "wait_for_completion=true)"); execute("select date from parted_table order by id"); assertThat(TestingHelpers.printedTable(response.rows()), is("0\n")); } @Test public void testRestoreFullPartedTableSnapshotSinglePartitionIntoDroppedTable() throws Exception { createTableAndSnapshot("my_parted_table", SNAPSHOT_NAME, true); execute("drop table my_parted_table"); waitNoPendingTasksOnAll(); execute("RESTORE SNAPSHOT " + snapshotName() + " TABLE my_parted_table PARTITION (date=0) with (" + "ignore_unavailable=false, " + "wait_for_completion=true)"); execute("select date from my_parted_table"); assertThat(TestingHelpers.printedTable(response.rows()), is("0\n")); } @Test public void testRestoreSnapshotIgnoreUnavailable() throws Exception { createTableAndSnapshot("my_table", SNAPSHOT_NAME, true); execute("drop table my_table"); execute("RESTORE SNAPSHOT " + snapshotName() + " TABLE my_table, not_my_table with (" + "ignore_unavailable=true, " + "wait_for_completion=true)"); execute("select table_schema || '.' || table_name from information_schema.tables where table_schema = ?", new Object[]{sqlExecutor.getCurrentSchema()}); assertThat(TestingHelpers.printedTable(response.rows()), is(getFqn("my_table") + "\n")); } @Test public void testRestoreOnlyOneTable() throws Exception { createTable("my_table_1", false); createTable("my_table_2", false); createSnapshot(SNAPSHOT_NAME, "my_table_1", "my_table_2"); waitNoPendingTasksOnAll(); execute("drop table my_table_1"); execute("RESTORE SNAPSHOT " + snapshotName() + " TABLE my_table_1 with (" + "wait_for_completion=true)"); execute("select table_schema || '.' || table_name from information_schema.tables where table_schema = ? order by 1", new Object[]{sqlExecutor.getCurrentSchema()}); assertThat(TestingHelpers.printedTable(response.rows()), is(getFqn("my_table_1") + "\n" + getFqn("my_table_2") + "\n")); } /** * Test to restore a concrete partitioned table. * <p> * This requires a patch in ES in order to restore templates when concrete tables are passed as an restore argument: * https://github.com/crate/elasticsearch/commit/3c14e74a3e50ea7d890f436db72ff18c2953ebc4 */ @Test public void testRestoreOnlyOnePartitionedTable() throws Exception { createTable("my_parted_1", true); createTable("my_parted_2", true); createSnapshot(SNAPSHOT_NAME, "my_parted_1", "my_parted_2"); execute("drop table my_parted_1"); execute("drop table my_parted_2"); execute("RESTORE SNAPSHOT " + snapshotName() + " TABLE my_parted_1 with (" + "wait_for_completion=true)"); execute("select table_schema || '.' || table_name from information_schema.tables where table_schema = ?", new Object[]{sqlExecutor.getCurrentSchema()}); assertThat(TestingHelpers.printedTable(response.rows()), is(getFqn("my_parted_1") + "\n")); } @Test public void testRestoreEmptyPartitionedTableUsingALL() throws Exception { execute("create table employees(section integer, name string) partitioned by (section)"); ensureYellow(); execute("CREATE SNAPSHOT " + snapshotName() + " ALL WITH (wait_for_completion=true)"); execute("drop table employees"); ensureYellow(); execute("RESTORE SNAPSHOT " + snapshotName() + " ALL with (wait_for_completion=true)"); ensureYellow(); execute("select table_schema || '.' || table_name from information_schema.tables where table_schema = ?", new Object[]{sqlExecutor.getCurrentSchema()}); assertThat(TestingHelpers.printedTable(response.rows()), is(getFqn("employees") + "\n")); } @Test public void testRestoreEmptyPartitionedTable() throws Exception { execute("create table employees(section integer, name string) partitioned by (section)"); ensureYellow(); execute("CREATE SNAPSHOT " + snapshotName() + " ALL WITH (wait_for_completion=true)"); execute("drop table employees"); ensureYellow(); execute("RESTORE SNAPSHOT " + snapshotName() + " TABLE employees with (wait_for_completion=true)"); ensureYellow(); execute("select table_schema || '.' || table_name from information_schema.tables where table_schema = ?", new Object[]{sqlExecutor.getCurrentSchema()}); assertThat(TestingHelpers.printedTable(response.rows()), is(getFqn("employees") + "\n")); } @Test public void testResolveUnknownTableFromSnapshot() throws Exception { expectedException.expect(SQLActionException.class); expectedException.expectMessage(String.format("ResourceNotFoundException: [%s..partitioned.employees.] template not found", sqlExecutor.getCurrentSchema())); execute("CREATE SNAPSHOT " + snapshotName() + " ALL WITH (wait_for_completion=true)"); ensureYellow(); execute("RESTORE SNAPSHOT " + snapshotName() + " TABLE employees with (wait_for_completion=true)"); } @Test public void test_cannot_create_snapshot_in_read_only_repo() { expectedException.expectMessage("cannot create snapshot in a readonly repository"); execute("create snapshot my_repo_ro.s1 ALL WITH (wait_for_completion=true)"); } public void test_snapshot_with_corrupted_shard_index_file() throws Exception { execute("CREATE TABLE t1 (x int)"); var numberOfDocs = randomLongBetween(0, 10); for (int i = 0; i < numberOfDocs; i++) { execute("INSERT INTO t1 (x) VALUES (?)", new Object[]{randomInt()}); } execute("REFRESH TABLE t1"); var snapShotName1 = "s1"; var fullSnapShotName1 = REPOSITORY_NAME + "." + snapShotName1; execute("CREATE SNAPSHOT " + fullSnapShotName1 + " ALL WITH (wait_for_completion=true)"); var repositoryData = getRepositoryData(); var indexIds = repositoryData.getIndices(); assertThat(indexIds.size(), equalTo(1)); var corruptedIndex = indexIds.entrySet().iterator().next().getValue(); var shardIndexFile = defaultRepositoryLocation.toPath().resolve("indices") .resolve(corruptedIndex.getId()).resolve("0") .resolve("index-" + repositoryData.shardGenerations().getShardGen(corruptedIndex, 0)); // Truncating shard index file try (var outChan = Files.newByteChannel(shardIndexFile, StandardOpenOption.WRITE)) { outChan.truncate(randomInt(10)); } assertSnapShotState(snapShotName1, SnapshotState.SUCCESS); execute("drop table t1"); execute("RESTORE SNAPSHOT " + fullSnapShotName1 + " TABLE t1 with (wait_for_completion=true)"); ensureYellow(); execute("SELECT COUNT(*) FROM t1"); assertThat(response.rows()[0][0], is(numberOfDocs)); var numberOfAdditionalDocs = randomLongBetween(0, 10); for (int i = 0; i < numberOfAdditionalDocs; i++) { execute("INSERT INTO t1 (x) VALUES (?)", new Object[]{randomInt()}); } execute("REFRESH TABLE t1"); var snapShotName2 = "s2"; var fullSnapShotName2 = REPOSITORY_NAME + ".s2"; execute("CREATE SNAPSHOT " + fullSnapShotName2 + " ALL WITH (wait_for_completion=true)"); assertSnapShotState(snapShotName2, SnapshotState.PARTIAL); } private void assertSnapShotState(String snapShotName, SnapshotState state) { execute( "SELECT state, array_length(concrete_indices, 1) FROM sys.snapshots where name = ? and repository = ?", new Object[]{snapShotName, REPOSITORY_NAME}); assertThat(response.rows()[0][0], is(state.name())); assertThat(response.rows()[0][1], is(1)); } private static void assertAllRepoSnapshotFilesAreDeleted(File location) { //Make sure the file location does not consist of any .dat file for (var file : location.listFiles()) { assertThat(file.getName().endsWith(".dat"), is(false)); } } private RepositoryData getRepositoryData() throws Exception { RepositoriesService service = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName()); Repository repository = service.repository(REPOSITORY_NAME); ThreadPool threadPool = internalCluster().getInstance(ThreadPool.class, internalCluster().getMasterName()); final SetOnce<RepositoryData> repositoryData = new SetOnce<>(); final CountDownLatch latch = new CountDownLatch(1); threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> { repositoryData.set(repository.getRepositoryData()); latch.countDown(); }); latch.await(); return repositoryData.get(); } }