Java Code Examples for org.apache.hadoop.hbase.MetaTableAccessor#getTableRegions()

/**
 * Initialize namespace state cache by scanning meta table.
 */
private void initialize() throws IOException {
  List<NamespaceDescriptor> namespaces = this.master.getClusterSchema().getNamespaces();
  for (NamespaceDescriptor namespace : namespaces) {
    addNamespace(namespace.getName());
    List<TableName> tables = this.master.listTableNamesByNamespace(namespace.getName());
    for (TableName table : tables) {
      if (table.isSystemTable()) {
        continue;
      }
      List<RegionInfo> regions =
          MetaTableAccessor.getTableRegions(this.master.getConnection(), table, true);
      addTable(table, regions.size());
    }
  }
  LOG.info("Finished updating state of " + nsStateCache.size() + " namespaces. ");
  initialized = true;
}
 

@Test
public void testSplitShouldNotHappenIfSplitIsDisabledForTable() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
    .setRegionSplitPolicyClassName(DisabledRegionSplitPolicy.class.getName())
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
  Table table = TEST_UTIL.createTable(htd, null);
  for (int i = 0; i < 10; i++) {
    Put p = new Put(Bytes.toBytes("row" + i));
    byte[] q1 = Bytes.toBytes("q1");
    byte[] v1 = Bytes.toBytes("v1");
    p.addColumn(Bytes.toBytes("f"), q1, v1);
    table.put(p);
  }
  ADMIN.flush(tableName);
  try {
    ADMIN.split(tableName, Bytes.toBytes("row5"));
    Threads.sleep(10000);
  } catch (Exception e) {
    // Nothing to do.
  }
  // Split should not happen.
  List<RegionInfo> allRegions =
    MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName, true);
  assertEquals(1, allRegions.size());
}
 

/**
 * Just make sure running fixMeta does right thing for the case
 * of a single-region Table where the region gets dropped.
 * There is nothing much we can do. We can't restore what
 * we don't know about (at least from a read of hbase:meta).
 */
@Test
public void testOneRegionTable() throws IOException {
  TableName tn = TableName.valueOf(this.name.getMethodName());
  TEST_UTIL.createTable(tn, HConstants.CATALOG_FAMILY);
  List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
  services.getCatalogJanitor().scan();
  deleteRegion(services, ris.get(0));
  services.getCatalogJanitor().scan();
  CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
  ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertTrue(ris.isEmpty());
  MetaFixer fixer = new MetaFixer(services);
  fixer.fixHoles(report);
  report = services.getCatalogJanitor().getLastReport();
  assertTrue(report.isEmpty());
  ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertEquals(0, ris.size());
}
 

@Test
public void testCreateTableWithSingleReplica() throws Exception {
  final int numRegions = 3;
  final int numReplica = 1;
  final TableName tableName = TableName.valueOf(name.getMethodName());
  try {
    TableDescriptor desc =
      TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
        .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
    ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
    TEST_UTIL.waitUntilNoRegionsInTransition();

    validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
    List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
    assertEquals(numRegions * numReplica, hris.size());
  } finally {
    ADMIN.disableTable(tableName);
    ADMIN.deleteTable(tableName);
  }
}
 

@Test
public void testPlugsHoles() throws Exception {
  TableName tn = TableName.valueOf(this.name.getMethodName());
  TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY);
  List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
  int initialSize = services.getAssignmentManager().getRegionStates().getRegionStates().size();
  services.getCatalogJanitor().scan();
  CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
  assertTrue(report.isEmpty());
  int originalCount = ris.size();
  // Remove first, last and middle region. See if hole gets plugged. Table has 26 regions.
  deleteRegion(services, ris.get(ris.size() -1));
  deleteRegion(services, ris.get(3));
  deleteRegion(services, ris.get(0));
  assertEquals(initialSize - 3,
      services.getAssignmentManager().getRegionStates().getRegionStates().size());
  services.getCatalogJanitor().scan();
  report = services.getCatalogJanitor().getLastReport();
  assertEquals(report.toString(), 3, report.getHoles().size());
  MetaFixer fixer = new MetaFixer(services);
  fixer.fixHoles(report);
  services.getCatalogJanitor().scan();
  report = services.getCatalogJanitor().getLastReport();
  assertTrue(report.toString(), report.isEmpty());
  assertEquals(initialSize,
      services.getAssignmentManager().getRegionStates().getRegionStates().size());

  // wait for RITs to settle -- those are the fixed regions being assigned -- or until the
  // watchdog TestRule terminates the test.
  HBaseTestingUtility.await(50,
    () -> isNotEmpty(services.getAssignmentManager().getRegionsInTransition()));

  ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertEquals(originalCount, ris.size());
}
 

private void testOverlapCommon(final TableName tn) throws Exception {
  Table t = TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY);
  TEST_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
  List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertTrue(ris.size() > 5);
  HMaster services = TEST_UTIL.getHBaseCluster().getMaster();
  services.getCatalogJanitor().scan();
  CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
  assertTrue(report.isEmpty());
  // Make a simple overlap spanning second and third region.
  makeOverlap(services, ris.get(1), ris.get(3));
  makeOverlap(services, ris.get(2), ris.get(3));
  makeOverlap(services, ris.get(2), ris.get(4));
}
 

/**
 * Make it so a big overlap spans many Regions, some of which are non-contiguous. Make it so
 * we can fix this condition. HBASE-24247
 */
@Test
public void testOverlapWithMergeOfNonContiguous() throws Exception {
  TableName tn = TableName.valueOf(this.name.getMethodName());
  TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY);
  List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertTrue(ris.size() > 5);
  MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
  services.getCatalogJanitor().scan();
  CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
  assertTrue(report.isEmpty());
  // Make a simple overlap spanning second and third region.
  makeOverlap(services, ris.get(1), ris.get(5));
  // Now Delete a region under the overlap to manufacture non-contiguous sub regions.
  RegionInfo deletedRegion = ris.get(3);
  long pid = services.getAssignmentManager().unassign(deletedRegion);
  while (!services.getMasterProcedureExecutor().isFinished(pid)) {
    Threads.sleep(100);
  }
  GCRegionProcedure procedure =
    new GCRegionProcedure(services.getMasterProcedureExecutor().getEnvironment(), ris.get(3));
  pid = services.getMasterProcedureExecutor().submitProcedure(procedure);
  while (!services.getMasterProcedureExecutor().isFinished(pid)) {
    Threads.sleep(100);
  }
  services.getCatalogJanitor().scan();
  report = services.getCatalogJanitor().getLastReport();
  assertEquals(1, MetaFixer.calculateMerges(10, report.getOverlaps()).size());
  MetaFixer fixer = new MetaFixer(services);
  fixer.fixOverlaps(report);
  HBaseTestingUtility.await(10, () -> {
    try {
      services.getCatalogJanitor().scan();
      final CatalogJanitor.Report postReport = services.getCatalogJanitor().getLastReport();
      return postReport.isEmpty();
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  });
}
 

/** verify region boundaries obtained from MetaScanner */
void verifyRegionsUsingMetaTableAccessor() throws Exception {
  List<RegionInfo> regionList = MetaTableAccessor.getTableRegions(connection, tableName, true);
  verifyTableRegions(regionList.stream()
    .collect(Collectors.toCollection(() -> new TreeSet<>(RegionInfo.COMPARATOR))));
  regionList = MetaTableAccessor.getAllRegions(connection, true);
  verifyTableRegions(regionList.stream()
    .collect(Collectors.toCollection(() -> new TreeSet<>(RegionInfo.COMPARATOR))));
}
 

/**
 * This test covers the case that one of merged parent regions is a merged child region that
 * has not been GCed but there is no reference files anymore. In this case, it will kick off
 * a GC procedure, but no merge will happen.
 */
@Test
public void testMergeWithMergedChildRegion() throws Exception {
  TableName tn = TableName.valueOf(this.name.getMethodName());
  Table t = TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY);
  List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertTrue(ris.size() > 5);
  HMaster services = TEST_UTIL.getHBaseCluster().getMaster();
  CatalogJanitor cj = services.getCatalogJanitor();
  cj.scan();
  CatalogJanitor.Report report = cj.getLastReport();
  assertTrue(report.isEmpty());
  RegionInfo overlapRegion = makeOverlap(services, ris.get(1), ris.get(2));

  cj.scan();
  report = cj.getLastReport();
  assertEquals(2, report.getOverlaps().size());

  // Mark it as a merged child region.
  RegionInfo fakedParentRegion = RegionInfoBuilder.newBuilder(tn).
    setStartKey(overlapRegion.getStartKey()).
    build();

  Table meta = MetaTableAccessor.getMetaHTable(TEST_UTIL.getConnection());
  Put putOfMerged = MetaTableAccessor.makePutFromRegionInfo(overlapRegion,
    HConstants.LATEST_TIMESTAMP);
  String qualifier = String.format(HConstants.MERGE_QUALIFIER_PREFIX_STR + "%04d", 0);
  putOfMerged.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(
    putOfMerged.getRow()).
    setFamily(HConstants.CATALOG_FAMILY).
    setQualifier(Bytes.toBytes(qualifier)).
    setTimestamp(putOfMerged.getTimestamp()).
    setType(Cell.Type.Put).
    setValue(RegionInfo.toByteArray(fakedParentRegion)).
    build());

  meta.put(putOfMerged);

  MetaFixer fixer = new MetaFixer(services);
  fixer.fixOverlaps(report);

  // Wait until all procedures settled down
  HBaseTestingUtility.await(200, () -> {
    return services.getMasterProcedureExecutor().getActiveProcIds().isEmpty();
  });

  // No merge is done, overlap is still there.
  cj.scan();
  report = cj.getLastReport();
  assertEquals(2, report.getOverlaps().size());

  fixer.fixOverlaps(report);

  // Wait until all procedures settled down
  HBaseTestingUtility.await(200, () -> {
    return services.getMasterProcedureExecutor().getActiveProcIds().isEmpty();
  });

  // Merge is done and no more overlaps
  cj.scan();
  report = cj.getLastReport();
  assertEquals(0, report.getOverlaps().size());
}
 

@Test
public void testLocalIndexScanWithMergeSpecialCase() throws Exception {
    String schemaName = generateUniqueName();
    String tableName = schemaName + "." + generateUniqueName();
    String indexName = "IDX_" + generateUniqueName();
    TableName physicalTableName = SchemaUtil.getPhysicalTableName(tableName.getBytes(), false);
    createBaseTable(tableName, "('a','aaaab','def')");
    Connection conn1 = getConnectionForLocalIndexTest();
    try {
        String[] strings =
                { "aa", "aaa", "aaaa", "bb", "cc", "dd", "dff", "g", "h", "i", "j", "k", "l",
                        "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" };
        for (int i = 0; i < 26; i++) {
            conn1.createStatement()
                    .execute("UPSERT INTO " + tableName + " values('" + strings[i] + "'," + i
                            + "," + (i + 1) + "," + (i + 2) + ",'" + strings[25 - i] + "')");
        }
        conn1.commit();
        conn1.createStatement()
                .execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(v1)");
        conn1.createStatement()
        .execute("CREATE LOCAL INDEX " + indexName + "_2 ON " + tableName + "(k3)");

        Admin admin = conn1.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
        List<RegionInfo> regionsOfUserTable =
                MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
                    false);
        admin.mergeRegionsAsync(regionsOfUserTable.get(0).getEncodedNameAsBytes(),
            regionsOfUserTable.get(1).getEncodedNameAsBytes(), false);
        regionsOfUserTable =
                MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
                    false);

        while (regionsOfUserTable.size() != 3) {
            Thread.sleep(100);
            regionsOfUserTable =
                    MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
                        false);
        }
        String query = "SELECT t_id,k1,v1 FROM " + tableName;
        ResultSet rs = conn1.createStatement().executeQuery(query);
        for (int j = 0; j < 26; j++) {
            assertTrue(rs.next());
            assertEquals(strings[25-j], rs.getString("t_id"));
            assertEquals(25-j, rs.getInt("k1"));
            assertEquals(strings[j], rs.getString("V1"));
        }
        query = "SELECT t_id,k1,k3 FROM " + tableName;
        rs = conn1.createStatement().executeQuery(query);
        for (int j = 0; j < 26; j++) {
            assertTrue(rs.next());
            assertEquals(strings[j], rs.getString("t_id"));
            assertEquals(j, rs.getInt("k1"));
            assertEquals(j + 2, rs.getInt("k3"));
        }
    } finally {
        conn1.close();
    }
}
 

@Test
public void testLocalIndexScanAfterRegionsMerge() throws Exception {
    String schemaName = generateUniqueName();
    String tableName = schemaName + "." + generateUniqueName();
    String indexName = "IDX_" + generateUniqueName();
    TableName physicalTableName = SchemaUtil.getPhysicalTableName(tableName.getBytes(), false);
    String indexPhysicalTableName = physicalTableName.getNameAsString();

    createBaseTable(tableName, "('e','j','o')");
    Connection conn1 = getConnectionForLocalIndexTest();
    try {
        String[] strings =
                { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o",
                        "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" };
        for (int i = 0; i < 26; i++) {
            conn1.createStatement()
                    .execute("UPSERT INTO " + tableName + " values('" + strings[i] + "'," + i
                            + "," + (i + 1) + "," + (i + 2) + ",'" + strings[25 - i] + "')");
        }
        conn1.commit();
        conn1.createStatement()
                .execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(v1)");
        conn1.createStatement()
                .execute("CREATE LOCAL INDEX " + indexName + "_2 ON " + tableName + "(k3)");

        ResultSet rs = conn1.createStatement().executeQuery("SELECT * FROM " + tableName);
        assertTrue(rs.next());

        Admin admin = conn1.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
        List<RegionInfo> regionsOfUserTable =
                MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
                    false);
        admin.mergeRegionsAsync(regionsOfUserTable.get(0).getEncodedNameAsBytes(),
            regionsOfUserTable.get(1).getEncodedNameAsBytes(), false);
        regionsOfUserTable =
                MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
                    false);

        while (regionsOfUserTable.size() != 3) {
            Thread.sleep(100);
            regionsOfUserTable =
                    MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
                        false);
        }
        String query = "SELECT t_id,k1,v1 FROM " + tableName;
        rs = conn1.createStatement().executeQuery(query);
        Thread.sleep(1000);
        for (int j = 0; j < 26; j++) {
            assertTrue(rs.next());
            assertEquals(strings[25 - j], rs.getString("t_id"));
            assertEquals(25 - j, rs.getInt("k1"));
            assertEquals(strings[j], rs.getString("V1"));
        }
        rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
        assertEquals(
            "CLIENT PARALLEL " + 3 + "-WAY RANGE SCAN OVER " + indexPhysicalTableName + " [1]\n"
                    + "    SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT",
            QueryUtil.getExplainPlan(rs));

        query = "SELECT t_id,k1,k3 FROM " + tableName;
        rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
        assertEquals(
            "CLIENT PARALLEL " + 3 + "-WAY RANGE SCAN OVER " + indexPhysicalTableName + " [2]\n"
                    + "    SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT",
            QueryUtil.getExplainPlan(rs));

        rs = conn1.createStatement().executeQuery(query);
        Thread.sleep(1000);
        for (int j = 0; j < 26; j++) {
            assertTrue(rs.next());
            assertEquals(strings[j], rs.getString("t_id"));
            assertEquals(j, rs.getInt("k1"));
            assertEquals(j + 2, rs.getInt("k3"));
        }
    } finally {
        conn1.close();
    }
}
 

@Test
public void testLocalIndexScanAfterRegionSplit() throws Exception {
    String schemaName = generateUniqueName();
    String tableName = schemaName + "." + generateUniqueName();
    String indexName = "IDX_" + generateUniqueName();
    TableName physicalTableName = SchemaUtil.getPhysicalTableName(tableName.getBytes(), false);
    String indexPhysicalTableName = physicalTableName.getNameAsString();

    createBaseTable(tableName, "('e','j','o')");
    Connection conn1 = getConnectionForLocalIndexTest();
    try {
        String[] strings =
                { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o",
                        "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" };
        for (int i = 0; i < 26; i++) {
            conn1.createStatement()
                    .execute("UPSERT INTO " + tableName + " values('" + strings[i] + "'," + i
                            + "," + (i + 1) + "," + (i + 2) + ",'" + strings[25 - i] + "')");
        }
        conn1.commit();
        conn1.createStatement()
                .execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(v1)");
        conn1.createStatement()
                .execute("CREATE LOCAL INDEX " + indexName + "_2 ON " + tableName + "(k3)");

        ResultSet rs = conn1.createStatement().executeQuery("SELECT * FROM " + tableName);
        assertTrue(rs.next());

        Admin admin = conn1.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
        for (int i = 1; i < 3; i++) {
            admin.split(physicalTableName, ByteUtil.concat(Bytes.toBytes(strings[3 * i])));
            List<RegionInfo> regionsOfUserTable =
                    MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
                        false);

            while (regionsOfUserTable.size() != (4 + i)) {
                Thread.sleep(100);
                regionsOfUserTable =
                        MetaTableAccessor.getTableRegions(admin.getConnection(),
                            physicalTableName, false);
            }
            assertEquals(4 + i, regionsOfUserTable.size());
            String[] tIdColumnValues = new String[26];
            String[] v1ColumnValues = new String[26];
            int[] k1ColumnValue = new int[26];
            String query = "SELECT t_id,k1,v1 FROM " + tableName;
            rs = conn1.createStatement().executeQuery(query);
            Thread.sleep(1000);
            for (int j = 0; j < 26; j++) {
                assertTrue("No row found at " + j, rs.next());
                tIdColumnValues[j] = rs.getString("t_id");
                k1ColumnValue[j] = rs.getInt("k1");
                v1ColumnValues[j] = rs.getString("V1");
            }
            Arrays.sort(tIdColumnValues);
            Arrays.sort(v1ColumnValues);
            Arrays.sort(k1ColumnValue);
            assertTrue(Arrays.equals(strings, tIdColumnValues));
            assertTrue(Arrays.equals(strings, v1ColumnValues));
            for (int m = 0; m < 26; m++) {
                assertEquals(m, k1ColumnValue[m]);
            }

            rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
            assertEquals("CLIENT PARALLEL " + (4 + i) + "-WAY RANGE SCAN OVER "
                    + indexPhysicalTableName + " [1]\n"
                    + "    SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT",
                QueryUtil.getExplainPlan(rs));

            query = "SELECT t_id,k1,k3 FROM " + tableName;
            rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
            assertEquals(
                "CLIENT PARALLEL "
                        + ((strings[3 * i].compareTo("j") < 0) ? (4 + i) : (4 + i - 1))
                        + "-WAY RANGE SCAN OVER " + indexPhysicalTableName + " [2]\n"
                        + "    SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT",
                QueryUtil.getExplainPlan(rs));
            rs = conn1.createStatement().executeQuery(query);
            Thread.sleep(1000);
            int[] k3ColumnValue = new int[26];
            for (int j = 0; j < 26; j++) {
                assertTrue(rs.next());
                tIdColumnValues[j] = rs.getString("t_id");
                k1ColumnValue[j] = rs.getInt("k1");
                k3ColumnValue[j] = rs.getInt("k3");
            }
            Arrays.sort(tIdColumnValues);
            Arrays.sort(k1ColumnValue);
            Arrays.sort(k3ColumnValue);
            assertTrue(Arrays.equals(strings, tIdColumnValues));
            for (int m = 0; m < 26; m++) {
                assertEquals(m, k1ColumnValue[m]);
                assertEquals(m + 2, k3ColumnValue[m]);
            }
        }
    } finally {
        conn1.close();
    }
}
 

private List<RegionInfo> splitDuringScan(Connection conn1, String tableName, String indexName, String[] strings, Admin admin, boolean isReverse)
        throws SQLException, IOException, InterruptedException {
    ResultSet rs;

    String query = "SELECT t_id,k1,v1 FROM " + tableName;
    rs = conn1.createStatement().executeQuery(query);
    String[] tIdColumnValues = new String[26]; 
    String[] v1ColumnValues = new String[26];
    int[] k1ColumnValue = new int[26];
    for (int j = 0; j < 5; j++) {
        assertTrue(rs.next());
        tIdColumnValues[j] = rs.getString("t_id");
        k1ColumnValue[j] = rs.getInt("k1");
        v1ColumnValues[j] = rs.getString("V1");
    }

    String[] splitKeys = new String[2];
    splitKeys[0] = strings[4];
    splitKeys[1] = strings[12];

    int[] splitInts = new int[2];
    splitInts[0] = 22;
    splitInts[1] = 4;
    List<RegionInfo> regionsOfUserTable = null;
    for(int i = 0; i <=1; i++) {
        boolean split = false;
        for (int j = 0; j < 150 && !split; j++) {
            try {
                if (localIndex) {
                    //With Hbase 2.2 the local index splits trigger longCompactions, and have
                    //to wait for an RS_COMPACTED_FILES_DISCHARGER run before the second split
                    //is successful
                    admin.split(TableName.valueOf(tableName),
                            ByteUtil.concat(Bytes.toBytes(splitKeys[i])));
                } else {
                    admin.split(TableName.valueOf(indexName),
                            ByteUtil.concat(Bytes.toBytes(splitInts[i])));
                }
                split = true;
            } catch (IOException x) {
                // wait up to a minute for the split to succeed
                Thread.sleep(1000);
            }
        }
        assertTrue(split);

        regionsOfUserTable =
                MetaTableAccessor.getTableRegions(admin.getConnection(),
                    TableName.valueOf(localIndex ? tableName : indexName), false);

        while (regionsOfUserTable.size() < (i+2)) {
            Thread.sleep(1000);
            regionsOfUserTable =
                    MetaTableAccessor.getTableRegions(admin.getConnection(),
                        TableName.valueOf(localIndex ? tableName : indexName), false);
        }
        assertTrue(regionsOfUserTable.size() >= (i+2));
    }
    for (int j = 5; j < 26; j++) {
        assertTrue(rs.next());
        tIdColumnValues[j] = rs.getString("t_id");
        k1ColumnValue[j] = rs.getInt("k1");
        v1ColumnValues[j] = rs.getString("V1");
    }
    Arrays.sort(tIdColumnValues);
    Arrays.sort(v1ColumnValues);
    Arrays.sort(k1ColumnValue);
    assertTrue(Arrays.equals(strings, tIdColumnValues));
    assertTrue(Arrays.equals(strings, v1ColumnValues));
    for(int i=0;i<26;i++) {
        assertEquals(i, k1ColumnValue[i]);
    }
    assertFalse(rs.next());
    return regionsOfUserTable;
}
 

@Test
public void testIncompleteMetaTableReplicaInformation() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  final int numRegions = 3;
  final int numReplica = 2;
  try {
    // Create a table and let the meta table be updated with the location of the
    // region locations.
    TableDescriptor desc =
      TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
        .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
    ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
    TEST_UTIL.waitUntilNoRegionsInTransition();
    Set<byte[]> tableRows = new HashSet<>();
    List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
    for (RegionInfo hri : hris) {
      tableRows.add(hri.getRegionName());
    }
    ADMIN.disableTable(tableName);
    // now delete one replica info from all the rows
    // this is to make the meta appear to be only partially updated
    Table metaTable = ADMIN.getConnection().getTable(TableName.META_TABLE_NAME);
    for (byte[] row : tableRows) {
      Delete deleteOneReplicaLocation = new Delete(row);
      deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
        CatalogFamilyFormat.getServerColumn(1));
      deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
        CatalogFamilyFormat.getSeqNumColumn(1));
      deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
        CatalogFamilyFormat.getStartCodeColumn(1));
      metaTable.delete(deleteOneReplicaLocation);
    }
    metaTable.close();
    // even if the meta table is partly updated, when we re-enable the table, we should
    // get back the desired number of replicas for the regions
    ADMIN.enableTable(tableName);
    assertTrue(ADMIN.isTableEnabled(tableName));
    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
    TEST_UTIL.waitUntilNoRegionsInTransition();
    List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
      .getRegionStates().getRegionsOfTable(tableName);
    assertEquals(numRegions * numReplica, regions.size());
  } finally {
    ADMIN.disableTable(tableName);
    ADMIN.deleteTable(tableName);
  }
}
 

@Test
public void testLocalIndexScanAfterRegionSplit() throws Exception {
    createBaseTable(TestUtil.DEFAULT_DATA_TABLE_NAME, null, "('e','j','o')");
    Connection conn1 = DriverManager.getConnection(getUrl());
    try{
        String[] strings = {"a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"};
        for (int i = 0; i < 26; i++) {
            conn1.createStatement().execute(
                "UPSERT INTO " + TestUtil.DEFAULT_DATA_TABLE_NAME + " values('"+strings[i]+"'," + i + ","
                        + (i + 1) + "," + (i + 2) + ",'" + strings[25 - i] + "')");
        }
        conn1.commit();
        conn1.createStatement().execute("CREATE LOCAL INDEX " + TestUtil.DEFAULT_INDEX_TABLE_NAME + " ON " + TestUtil.DEFAULT_DATA_TABLE_NAME + "(v1)");
        conn1.createStatement().execute("CREATE LOCAL INDEX " + TestUtil.DEFAULT_INDEX_TABLE_NAME + "_2 ON " + TestUtil.DEFAULT_DATA_TABLE_NAME + "(k3)");

        ResultSet rs = conn1.createStatement().executeQuery("SELECT * FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME);
        assertTrue(rs.next());
        
        HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin();
        for (int i = 1; i < 5; i++) {
            admin.split(Bytes.toBytes(TestUtil.DEFAULT_DATA_TABLE_NAME), ByteUtil.concat(Bytes.toBytes(strings[3*i])));
            List<HRegionInfo> regionsOfUserTable =
                    MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), admin.getConnection(),
                            TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);

            while (regionsOfUserTable.size() != (4+i)) {
                Thread.sleep(100);
                regionsOfUserTable = MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
                        admin.getConnection(), TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);
            }
            assertEquals(4+i, regionsOfUserTable.size());
            TableName indexTable =
                    TableName.valueOf(MetaDataUtil.getLocalIndexTableName(TestUtil.DEFAULT_DATA_TABLE_NAME));
            List<HRegionInfo> regionsOfIndexTable =
                    MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
                            admin.getConnection(), indexTable, false);

            while (regionsOfIndexTable.size() != (4 + i)) {
                Thread.sleep(100);
                regionsOfIndexTable = MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
                        admin.getConnection(), indexTable, false);
            }
            assertEquals(4 + i, regionsOfIndexTable.size());
            String query = "SELECT t_id,k1,v1 FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME;
            rs = conn1.createStatement().executeQuery(query);
            Thread.sleep(1000);
            for (int j = 0; j < 26; j++) {
                assertTrue(rs.next());
                assertEquals(strings[25-j], rs.getString("t_id"));
                assertEquals(25-j, rs.getInt("k1"));
                assertEquals(strings[j], rs.getString("V1"));
            }
            rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
            assertEquals(
                    "CLIENT PARALLEL " + (4 + i) + "-WAY RANGE SCAN OVER "
                            + MetaDataUtil.getLocalIndexTableName(TestUtil.DEFAULT_DATA_TABLE_NAME) + " [-32768]\n"
                                    + "    SERVER FILTER BY FIRST KEY ONLY\n"
                            + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));
            
            query = "SELECT t_id,k1,k3 FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME;
            rs = conn1.createStatement().executeQuery("EXPLAIN "+query);
            assertEquals(
                "CLIENT PARALLEL "
                        + ((strings[3 * i].compareTo("j") < 0) ? (4 + i) : (4 + i - 1))
                        + "-WAY RANGE SCAN OVER "
                        + MetaDataUtil.getLocalIndexTableName(TestUtil.DEFAULT_DATA_TABLE_NAME) + " [-32767]\n"
                                + "    SERVER FILTER BY FIRST KEY ONLY\n"
                        + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));
            rs = conn1.createStatement().executeQuery(query);
            Thread.sleep(1000);
            for (int j = 0; j < 26; j++) {
                assertTrue(rs.next());
                assertEquals(strings[j], rs.getString("t_id"));
                assertEquals(j, rs.getInt("k1"));
                assertEquals(j+2, rs.getInt("k3"));
            }
        }
   } finally {
        conn1.close();
    }
}
 

private void testMerge(TableName tableName, int mergeCount) throws IOException {
  List<RegionInfo> ris = MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName);
  int originalRegionCount = ris.size();
  assertTrue(originalRegionCount > mergeCount);
  RegionInfo[] regionsToMerge = ris.subList(0, mergeCount).toArray(new RegionInfo [] {});
  int countOfRowsLoaded = 0;
  try (Table table = UTIL.getConnection().getTable(tableName)) {
    countOfRowsLoaded = loadARowPerRegion(table, ris);
  }
  assertEquals(countOfRowsLoaded, UTIL.countRows(tableName));

  // collect AM metrics before test
  collectAssignmentManagerMetrics();
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  MergeTableRegionsProcedure proc =
      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true);
  long procId = procExec.submitProcedure(proc);
  ProcedureTestingUtility.waitProcedure(procExec, procId);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
  MetaTableAccessor.fullScanMetaAndPrint(UTIL.getConnection());
  assertEquals(originalRegionCount - mergeCount + 1,
      MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName).size());

  assertEquals(mergeSubmittedCount + 1, mergeProcMetrics.getSubmittedCounter().getCount());
  assertEquals(mergeFailedCount, mergeProcMetrics.getFailedCounter().getCount());
  assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
  assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
  assertEquals(unassignSubmittedCount + mergeCount,
      unassignProcMetrics.getSubmittedCounter().getCount());
  assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());

  // Need to get the references cleaned out. Close of region will move them
  // to archive so disable and reopen just to get rid of references to later
  // when the catalogjanitor runs, it can do merged region cleanup.
  admin.disableTable(tableName);
  admin.enableTable(tableName);

  // Can I purge the merged regions from hbase:meta? Check that all went
  // well by looking at the merged row up in hbase:meta. It should have no
  // more mention of the merged regions; they are purged as last step in
  // the merged regions cleanup.
  UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(true);
  UTIL.getHBaseCluster().getMaster().getCatalogJanitor().triggerNow();
  byte [] mergedRegion = proc.getMergedRegion().getRegionName();
  while (ris != null && ris.get(0) != null && ris.get(1) != null) {
    ris = MetaTableAccessor.getMergeRegions(UTIL.getConnection(), mergedRegion);
    LOG.info("{} {}", Bytes.toStringBinary(mergedRegion), ris);
    Threads.sleep(1000);
  }
  assertEquals(countOfRowsLoaded, UTIL.countRows(tableName));
}
 

/**
 * Check that all the regions in the snapshot are valid, and accounted for.
 * @param manifest snapshot manifest to inspect
 * @throws IOException if we can't reach hbase:meta or read the files from the FS
 */
private void verifyRegions(final SnapshotManifest manifest) throws IOException {
  List<RegionInfo> regions;
  if (TableName.META_TABLE_NAME.equals(tableName)) {
    regions = MetaTableLocator.getMetaRegions(services.getZooKeeper());
  } else {
    regions = MetaTableAccessor.getTableRegions(services.getConnection(), tableName);
  }
  // Remove the non-default regions
  RegionReplicaUtil.removeNonDefaultRegions(regions);

  Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
  if (regionManifests == null) {
    String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty";
    LOG.error(msg);
    throw new CorruptedSnapshotException(msg);
  }

  String errorMsg = "";
  boolean hasMobStore = false;
  // the mob region is a dummy region, it's not a real region in HBase.
  // the mob region has a special name, it could be found by the region name.
  if (regionManifests.get(MobUtils.getMobRegionInfo(tableName).getEncodedName()) != null) {
    hasMobStore = true;
  }
  int realRegionCount = hasMobStore ? regionManifests.size() - 1 : regionManifests.size();
  if (realRegionCount != regions.size()) {
    errorMsg = "Regions moved during the snapshot '" +
                 ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" +
                 regions.size() + " snapshotted=" + realRegionCount + ".";
    LOG.error(errorMsg);
  }

  // Verify RegionInfo
  for (RegionInfo region : regions) {
    SnapshotRegionManifest regionManifest = regionManifests.get(region.getEncodedName());
    if (regionManifest == null) {
      // could happen due to a move or split race.
      String mesg = " No snapshot region directory found for region:" + region;
      if (errorMsg.isEmpty()) errorMsg = mesg;
      LOG.error(mesg);
      continue;
    }

    verifyRegionInfo(region, regionManifest);
  }

  if (!errorMsg.isEmpty()) {
    throw new CorruptedSnapshotException(errorMsg);
  }

  // Verify Snapshot HFiles
  // Requires the root directory file system as HFiles are stored in the root directory
  SnapshotReferenceUtil.verifySnapshot(services.getConfiguration(),
    CommonFSUtils.getRootDirFileSystem(services.getConfiguration()), manifest);
}
 

@Test
public void testLocalIndexScanAfterRegionsMerge() throws Exception {
    createBaseTable(TestUtil.DEFAULT_DATA_TABLE_NAME, null, "('e','j','o')");
    Connection conn1 = DriverManager.getConnection(getUrl());
    try{
        String[] strings = {"a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"};
        for (int i = 0; i < 26; i++) {
            conn1.createStatement().execute(
                "UPSERT INTO " + TestUtil.DEFAULT_DATA_TABLE_NAME + " values('"+strings[i]+"'," + i + ","
                        + (i + 1) + "," + (i + 2) + ",'" + strings[25 - i] + "')");
        }
        conn1.commit();
        conn1.createStatement().execute("CREATE LOCAL INDEX " + TestUtil.DEFAULT_INDEX_TABLE_NAME + " ON " + TestUtil.DEFAULT_DATA_TABLE_NAME + "(v1)");
        conn1.createStatement().execute("CREATE LOCAL INDEX " + TestUtil.DEFAULT_INDEX_TABLE_NAME + "_2 ON " + TestUtil.DEFAULT_DATA_TABLE_NAME + "(k3)");

        ResultSet rs = conn1.createStatement().executeQuery("SELECT * FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME);
        assertTrue(rs.next());

        HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin();
        List<HRegionInfo> regionsOfUserTable =
                MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), admin.getConnection(),
                    TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);
        admin.mergeRegions(regionsOfUserTable.get(0).getEncodedNameAsBytes(),
            regionsOfUserTable.get(1).getEncodedNameAsBytes(), false);
        regionsOfUserTable =
                MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), admin.getConnection(),
                    TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);

        while (regionsOfUserTable.size() != 3) {
            Thread.sleep(100);
            regionsOfUserTable = MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
                    admin.getConnection(), TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);
        }
        assertEquals(3, regionsOfUserTable.size());
        TableName indexTable =
                TableName.valueOf(MetaDataUtil
                        .getLocalIndexTableName(TestUtil.DEFAULT_DATA_TABLE_NAME));
        List<HRegionInfo> regionsOfIndexTable =
                MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
                        admin.getConnection(), indexTable, false);

        while (regionsOfIndexTable.size() != 3) {
            Thread.sleep(100);
            regionsOfIndexTable = MetaTableAccessor.getTableRegions(
                    getUtility().getZooKeeperWatcher(), admin.getConnection(), indexTable, false);
        }
        assertEquals(3, regionsOfIndexTable.size());
        String query = "SELECT t_id,k1,v1 FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME;
        rs = conn1.createStatement().executeQuery(query);
        Thread.sleep(1000);
        for (int j = 0; j < 26; j++) {
            assertTrue(rs.next());
            assertEquals(strings[25 - j], rs.getString("t_id"));
            assertEquals(25 - j, rs.getInt("k1"));
            assertEquals(strings[j], rs.getString("V1"));
        }
        rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
        assertEquals(
            "CLIENT PARALLEL " + 3 + "-WAY RANGE SCAN OVER "
                    + MetaDataUtil.getLocalIndexTableName(TestUtil.DEFAULT_DATA_TABLE_NAME)
                    + " [-32768]\n" + "    SERVER FILTER BY FIRST KEY ONLY\n"
                    + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));

        query = "SELECT t_id,k1,k3 FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME;
        rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
        assertEquals(
            "CLIENT PARALLEL " + 3 + "-WAY RANGE SCAN OVER "
                    + MetaDataUtil.getLocalIndexTableName(TestUtil.DEFAULT_DATA_TABLE_NAME)
                    + " [-32767]\n" + "    SERVER FILTER BY FIRST KEY ONLY\n"
                    + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));

        rs = conn1.createStatement().executeQuery(query);
        Thread.sleep(1000);
        for (int j = 0; j < 26; j++) {
            assertTrue(rs.next());
            assertEquals(strings[j], rs.getString("t_id"));
            assertEquals(j, rs.getInt("k1"));
            assertEquals(j + 2, rs.getInt("k3"));
        }
   } finally {
        conn1.close();
    }
}