Java Code Examples for org.apache.hadoop.hbase.MetaTableAccessor#getTableRegions()

The following examples show how to use org.apache.hadoop.hbase.MetaTableAccessor#getTableRegions() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: NamespaceStateManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Initialize namespace state cache by scanning meta table.
 */
private void initialize() throws IOException {
  List<NamespaceDescriptor> namespaces = this.master.getClusterSchema().getNamespaces();
  for (NamespaceDescriptor namespace : namespaces) {
    addNamespace(namespace.getName());
    List<TableName> tables = this.master.listTableNamesByNamespace(namespace.getName());
    for (TableName table : tables) {
      if (table.isSystemTable()) {
        continue;
      }
      List<RegionInfo> regions =
          MetaTableAccessor.getTableRegions(this.master.getConnection(), table, true);
      addTable(table, regions.size());
    }
  }
  LOG.info("Finished updating state of " + nsStateCache.size() + " namespaces. ");
  initialized = true;
}
 
Example 2
Source File: TestAdmin1.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testSplitShouldNotHappenIfSplitIsDisabledForTable() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
    .setRegionSplitPolicyClassName(DisabledRegionSplitPolicy.class.getName())
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
  Table table = TEST_UTIL.createTable(htd, null);
  for (int i = 0; i < 10; i++) {
    Put p = new Put(Bytes.toBytes("row" + i));
    byte[] q1 = Bytes.toBytes("q1");
    byte[] v1 = Bytes.toBytes("v1");
    p.addColumn(Bytes.toBytes("f"), q1, v1);
    table.put(p);
  }
  ADMIN.flush(tableName);
  try {
    ADMIN.split(tableName, Bytes.toBytes("row5"));
    Threads.sleep(10000);
  } catch (Exception e) {
    // Nothing to do.
  }
  // Split should not happen.
  List<RegionInfo> allRegions =
    MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName, true);
  assertEquals(1, allRegions.size());
}
 
Example 3
Source File: TestMetaFixer.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Just make sure running fixMeta does right thing for the case
 * of a single-region Table where the region gets dropped.
 * There is nothing much we can do. We can't restore what
 * we don't know about (at least from a read of hbase:meta).
 */
@Test
public void testOneRegionTable() throws IOException {
  TableName tn = TableName.valueOf(this.name.getMethodName());
  TEST_UTIL.createTable(tn, HConstants.CATALOG_FAMILY);
  List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
  services.getCatalogJanitor().scan();
  deleteRegion(services, ris.get(0));
  services.getCatalogJanitor().scan();
  CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
  ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertTrue(ris.isEmpty());
  MetaFixer fixer = new MetaFixer(services);
  fixer.fixHoles(report);
  report = services.getCatalogJanitor().getLastReport();
  assertTrue(report.isEmpty());
  ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertEquals(0, ris.size());
}
 
Example 4
Source File: TestMasterOperationsForRegionReplicas.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateTableWithSingleReplica() throws Exception {
  final int numRegions = 3;
  final int numReplica = 1;
  final TableName tableName = TableName.valueOf(name.getMethodName());
  try {
    TableDescriptor desc =
      TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
        .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
    ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
    TEST_UTIL.waitUntilNoRegionsInTransition();

    validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
    List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
    assertEquals(numRegions * numReplica, hris.size());
  } finally {
    ADMIN.disableTable(tableName);
    ADMIN.deleteTable(tableName);
  }
}
 
Example 5
Source File: TestMetaFixer.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testPlugsHoles() throws Exception {
  TableName tn = TableName.valueOf(this.name.getMethodName());
  TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY);
  List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
  int initialSize = services.getAssignmentManager().getRegionStates().getRegionStates().size();
  services.getCatalogJanitor().scan();
  CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
  assertTrue(report.isEmpty());
  int originalCount = ris.size();
  // Remove first, last and middle region. See if hole gets plugged. Table has 26 regions.
  deleteRegion(services, ris.get(ris.size() -1));
  deleteRegion(services, ris.get(3));
  deleteRegion(services, ris.get(0));
  assertEquals(initialSize - 3,
      services.getAssignmentManager().getRegionStates().getRegionStates().size());
  services.getCatalogJanitor().scan();
  report = services.getCatalogJanitor().getLastReport();
  assertEquals(report.toString(), 3, report.getHoles().size());
  MetaFixer fixer = new MetaFixer(services);
  fixer.fixHoles(report);
  services.getCatalogJanitor().scan();
  report = services.getCatalogJanitor().getLastReport();
  assertTrue(report.toString(), report.isEmpty());
  assertEquals(initialSize,
      services.getAssignmentManager().getRegionStates().getRegionStates().size());

  // wait for RITs to settle -- those are the fixed regions being assigned -- or until the
  // watchdog TestRule terminates the test.
  HBaseTestingUtility.await(50,
    () -> isNotEmpty(services.getAssignmentManager().getRegionsInTransition()));

  ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertEquals(originalCount, ris.size());
}
 
Example 6
Source File: TestMetaFixer.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void testOverlapCommon(final TableName tn) throws Exception {
  Table t = TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY);
  TEST_UTIL.loadTable(t, HConstants.CATALOG_FAMILY);
  List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertTrue(ris.size() > 5);
  HMaster services = TEST_UTIL.getHBaseCluster().getMaster();
  services.getCatalogJanitor().scan();
  CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
  assertTrue(report.isEmpty());
  // Make a simple overlap spanning second and third region.
  makeOverlap(services, ris.get(1), ris.get(3));
  makeOverlap(services, ris.get(2), ris.get(3));
  makeOverlap(services, ris.get(2), ris.get(4));
}
 
Example 7
Source File: TestMetaFixer.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Make it so a big overlap spans many Regions, some of which are non-contiguous. Make it so
 * we can fix this condition. HBASE-24247
 */
@Test
public void testOverlapWithMergeOfNonContiguous() throws Exception {
  TableName tn = TableName.valueOf(this.name.getMethodName());
  TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY);
  List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertTrue(ris.size() > 5);
  MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
  services.getCatalogJanitor().scan();
  CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
  assertTrue(report.isEmpty());
  // Make a simple overlap spanning second and third region.
  makeOverlap(services, ris.get(1), ris.get(5));
  // Now Delete a region under the overlap to manufacture non-contiguous sub regions.
  RegionInfo deletedRegion = ris.get(3);
  long pid = services.getAssignmentManager().unassign(deletedRegion);
  while (!services.getMasterProcedureExecutor().isFinished(pid)) {
    Threads.sleep(100);
  }
  GCRegionProcedure procedure =
    new GCRegionProcedure(services.getMasterProcedureExecutor().getEnvironment(), ris.get(3));
  pid = services.getMasterProcedureExecutor().submitProcedure(procedure);
  while (!services.getMasterProcedureExecutor().isFinished(pid)) {
    Threads.sleep(100);
  }
  services.getCatalogJanitor().scan();
  report = services.getCatalogJanitor().getLastReport();
  assertEquals(1, MetaFixer.calculateMerges(10, report.getOverlaps()).size());
  MetaFixer fixer = new MetaFixer(services);
  fixer.fixOverlaps(report);
  HBaseTestingUtility.await(10, () -> {
    try {
      services.getCatalogJanitor().scan();
      final CatalogJanitor.Report postReport = services.getCatalogJanitor().getLastReport();
      return postReport.isEmpty();
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  });
}
 
Example 8
Source File: TestEndToEndSplitTransaction.java    From hbase with Apache License 2.0 5 votes vote down vote up
/** verify region boundaries obtained from MetaScanner */
void verifyRegionsUsingMetaTableAccessor() throws Exception {
  List<RegionInfo> regionList = MetaTableAccessor.getTableRegions(connection, tableName, true);
  verifyTableRegions(regionList.stream()
    .collect(Collectors.toCollection(() -> new TreeSet<>(RegionInfo.COMPARATOR))));
  regionList = MetaTableAccessor.getAllRegions(connection, true);
  verifyTableRegions(regionList.stream()
    .collect(Collectors.toCollection(() -> new TreeSet<>(RegionInfo.COMPARATOR))));
}
 
Example 9
Source File: TestMetaFixer.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * This test covers the case that one of merged parent regions is a merged child region that
 * has not been GCed but there is no reference files anymore. In this case, it will kick off
 * a GC procedure, but no merge will happen.
 */
@Test
public void testMergeWithMergedChildRegion() throws Exception {
  TableName tn = TableName.valueOf(this.name.getMethodName());
  Table t = TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY);
  List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertTrue(ris.size() > 5);
  HMaster services = TEST_UTIL.getHBaseCluster().getMaster();
  CatalogJanitor cj = services.getCatalogJanitor();
  cj.scan();
  CatalogJanitor.Report report = cj.getLastReport();
  assertTrue(report.isEmpty());
  RegionInfo overlapRegion = makeOverlap(services, ris.get(1), ris.get(2));

  cj.scan();
  report = cj.getLastReport();
  assertEquals(2, report.getOverlaps().size());

  // Mark it as a merged child region.
  RegionInfo fakedParentRegion = RegionInfoBuilder.newBuilder(tn).
    setStartKey(overlapRegion.getStartKey()).
    build();

  Table meta = MetaTableAccessor.getMetaHTable(TEST_UTIL.getConnection());
  Put putOfMerged = MetaTableAccessor.makePutFromRegionInfo(overlapRegion,
    HConstants.LATEST_TIMESTAMP);
  String qualifier = String.format(HConstants.MERGE_QUALIFIER_PREFIX_STR + "%04d", 0);
  putOfMerged.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY).setRow(
    putOfMerged.getRow()).
    setFamily(HConstants.CATALOG_FAMILY).
    setQualifier(Bytes.toBytes(qualifier)).
    setTimestamp(putOfMerged.getTimestamp()).
    setType(Cell.Type.Put).
    setValue(RegionInfo.toByteArray(fakedParentRegion)).
    build());

  meta.put(putOfMerged);

  MetaFixer fixer = new MetaFixer(services);
  fixer.fixOverlaps(report);

  // Wait until all procedures settled down
  HBaseTestingUtility.await(200, () -> {
    return services.getMasterProcedureExecutor().getActiveProcIds().isEmpty();
  });

  // No merge is done, overlap is still there.
  cj.scan();
  report = cj.getLastReport();
  assertEquals(2, report.getOverlaps().size());

  fixer.fixOverlaps(report);

  // Wait until all procedures settled down
  HBaseTestingUtility.await(200, () -> {
    return services.getMasterProcedureExecutor().getActiveProcIds().isEmpty();
  });

  // Merge is done and no more overlaps
  cj.scan();
  report = cj.getLastReport();
  assertEquals(0, report.getOverlaps().size());
}
 
Example 10
Source File: LocalIndexSplitMergeIT.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@Test
public void testLocalIndexScanWithMergeSpecialCase() throws Exception {
    String schemaName = generateUniqueName();
    String tableName = schemaName + "." + generateUniqueName();
    String indexName = "IDX_" + generateUniqueName();
    TableName physicalTableName = SchemaUtil.getPhysicalTableName(tableName.getBytes(), false);
    createBaseTable(tableName, "('a','aaaab','def')");
    Connection conn1 = getConnectionForLocalIndexTest();
    try {
        String[] strings =
                { "aa", "aaa", "aaaa", "bb", "cc", "dd", "dff", "g", "h", "i", "j", "k", "l",
                        "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" };
        for (int i = 0; i < 26; i++) {
            conn1.createStatement()
                    .execute("UPSERT INTO " + tableName + " values('" + strings[i] + "'," + i
                            + "," + (i + 1) + "," + (i + 2) + ",'" + strings[25 - i] + "')");
        }
        conn1.commit();
        conn1.createStatement()
                .execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(v1)");
        conn1.createStatement()
        .execute("CREATE LOCAL INDEX " + indexName + "_2 ON " + tableName + "(k3)");

        Admin admin = conn1.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
        List<RegionInfo> regionsOfUserTable =
                MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
                    false);
        admin.mergeRegionsAsync(regionsOfUserTable.get(0).getEncodedNameAsBytes(),
            regionsOfUserTable.get(1).getEncodedNameAsBytes(), false);
        regionsOfUserTable =
                MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
                    false);

        while (regionsOfUserTable.size() != 3) {
            Thread.sleep(100);
            regionsOfUserTable =
                    MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
                        false);
        }
        String query = "SELECT t_id,k1,v1 FROM " + tableName;
        ResultSet rs = conn1.createStatement().executeQuery(query);
        for (int j = 0; j < 26; j++) {
            assertTrue(rs.next());
            assertEquals(strings[25-j], rs.getString("t_id"));
            assertEquals(25-j, rs.getInt("k1"));
            assertEquals(strings[j], rs.getString("V1"));
        }
        query = "SELECT t_id,k1,k3 FROM " + tableName;
        rs = conn1.createStatement().executeQuery(query);
        for (int j = 0; j < 26; j++) {
            assertTrue(rs.next());
            assertEquals(strings[j], rs.getString("t_id"));
            assertEquals(j, rs.getInt("k1"));
            assertEquals(j + 2, rs.getInt("k3"));
        }
    } finally {
        conn1.close();
    }
}
 
Example 11
Source File: LocalIndexSplitMergeIT.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@Test
public void testLocalIndexScanAfterRegionsMerge() throws Exception {
    String schemaName = generateUniqueName();
    String tableName = schemaName + "." + generateUniqueName();
    String indexName = "IDX_" + generateUniqueName();
    TableName physicalTableName = SchemaUtil.getPhysicalTableName(tableName.getBytes(), false);
    String indexPhysicalTableName = physicalTableName.getNameAsString();

    createBaseTable(tableName, "('e','j','o')");
    Connection conn1 = getConnectionForLocalIndexTest();
    try {
        String[] strings =
                { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o",
                        "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" };
        for (int i = 0; i < 26; i++) {
            conn1.createStatement()
                    .execute("UPSERT INTO " + tableName + " values('" + strings[i] + "'," + i
                            + "," + (i + 1) + "," + (i + 2) + ",'" + strings[25 - i] + "')");
        }
        conn1.commit();
        conn1.createStatement()
                .execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(v1)");
        conn1.createStatement()
                .execute("CREATE LOCAL INDEX " + indexName + "_2 ON " + tableName + "(k3)");

        ResultSet rs = conn1.createStatement().executeQuery("SELECT * FROM " + tableName);
        assertTrue(rs.next());

        Admin admin = conn1.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
        List<RegionInfo> regionsOfUserTable =
                MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
                    false);
        admin.mergeRegionsAsync(regionsOfUserTable.get(0).getEncodedNameAsBytes(),
            regionsOfUserTable.get(1).getEncodedNameAsBytes(), false);
        regionsOfUserTable =
                MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
                    false);

        while (regionsOfUserTable.size() != 3) {
            Thread.sleep(100);
            regionsOfUserTable =
                    MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
                        false);
        }
        String query = "SELECT t_id,k1,v1 FROM " + tableName;
        rs = conn1.createStatement().executeQuery(query);
        Thread.sleep(1000);
        for (int j = 0; j < 26; j++) {
            assertTrue(rs.next());
            assertEquals(strings[25 - j], rs.getString("t_id"));
            assertEquals(25 - j, rs.getInt("k1"));
            assertEquals(strings[j], rs.getString("V1"));
        }
        rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
        assertEquals(
            "CLIENT PARALLEL " + 3 + "-WAY RANGE SCAN OVER " + indexPhysicalTableName + " [1]\n"
                    + "    SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT",
            QueryUtil.getExplainPlan(rs));

        query = "SELECT t_id,k1,k3 FROM " + tableName;
        rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
        assertEquals(
            "CLIENT PARALLEL " + 3 + "-WAY RANGE SCAN OVER " + indexPhysicalTableName + " [2]\n"
                    + "    SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT",
            QueryUtil.getExplainPlan(rs));

        rs = conn1.createStatement().executeQuery(query);
        Thread.sleep(1000);
        for (int j = 0; j < 26; j++) {
            assertTrue(rs.next());
            assertEquals(strings[j], rs.getString("t_id"));
            assertEquals(j, rs.getInt("k1"));
            assertEquals(j + 2, rs.getInt("k3"));
        }
    } finally {
        conn1.close();
    }
}
 
Example 12
Source File: LocalIndexSplitMergeIT.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@Test
public void testLocalIndexScanAfterRegionSplit() throws Exception {
    String schemaName = generateUniqueName();
    String tableName = schemaName + "." + generateUniqueName();
    String indexName = "IDX_" + generateUniqueName();
    TableName physicalTableName = SchemaUtil.getPhysicalTableName(tableName.getBytes(), false);
    String indexPhysicalTableName = physicalTableName.getNameAsString();

    createBaseTable(tableName, "('e','j','o')");
    Connection conn1 = getConnectionForLocalIndexTest();
    try {
        String[] strings =
                { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o",
                        "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" };
        for (int i = 0; i < 26; i++) {
            conn1.createStatement()
                    .execute("UPSERT INTO " + tableName + " values('" + strings[i] + "'," + i
                            + "," + (i + 1) + "," + (i + 2) + ",'" + strings[25 - i] + "')");
        }
        conn1.commit();
        conn1.createStatement()
                .execute("CREATE LOCAL INDEX " + indexName + " ON " + tableName + "(v1)");
        conn1.createStatement()
                .execute("CREATE LOCAL INDEX " + indexName + "_2 ON " + tableName + "(k3)");

        ResultSet rs = conn1.createStatement().executeQuery("SELECT * FROM " + tableName);
        assertTrue(rs.next());

        Admin admin = conn1.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
        for (int i = 1; i < 3; i++) {
            admin.split(physicalTableName, ByteUtil.concat(Bytes.toBytes(strings[3 * i])));
            List<RegionInfo> regionsOfUserTable =
                    MetaTableAccessor.getTableRegions(admin.getConnection(), physicalTableName,
                        false);

            while (regionsOfUserTable.size() != (4 + i)) {
                Thread.sleep(100);
                regionsOfUserTable =
                        MetaTableAccessor.getTableRegions(admin.getConnection(),
                            physicalTableName, false);
            }
            assertEquals(4 + i, regionsOfUserTable.size());
            String[] tIdColumnValues = new String[26];
            String[] v1ColumnValues = new String[26];
            int[] k1ColumnValue = new int[26];
            String query = "SELECT t_id,k1,v1 FROM " + tableName;
            rs = conn1.createStatement().executeQuery(query);
            Thread.sleep(1000);
            for (int j = 0; j < 26; j++) {
                assertTrue("No row found at " + j, rs.next());
                tIdColumnValues[j] = rs.getString("t_id");
                k1ColumnValue[j] = rs.getInt("k1");
                v1ColumnValues[j] = rs.getString("V1");
            }
            Arrays.sort(tIdColumnValues);
            Arrays.sort(v1ColumnValues);
            Arrays.sort(k1ColumnValue);
            assertTrue(Arrays.equals(strings, tIdColumnValues));
            assertTrue(Arrays.equals(strings, v1ColumnValues));
            for (int m = 0; m < 26; m++) {
                assertEquals(m, k1ColumnValue[m]);
            }

            rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
            assertEquals("CLIENT PARALLEL " + (4 + i) + "-WAY RANGE SCAN OVER "
                    + indexPhysicalTableName + " [1]\n"
                    + "    SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT",
                QueryUtil.getExplainPlan(rs));

            query = "SELECT t_id,k1,k3 FROM " + tableName;
            rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
            assertEquals(
                "CLIENT PARALLEL "
                        + ((strings[3 * i].compareTo("j") < 0) ? (4 + i) : (4 + i - 1))
                        + "-WAY RANGE SCAN OVER " + indexPhysicalTableName + " [2]\n"
                        + "    SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT",
                QueryUtil.getExplainPlan(rs));
            rs = conn1.createStatement().executeQuery(query);
            Thread.sleep(1000);
            int[] k3ColumnValue = new int[26];
            for (int j = 0; j < 26; j++) {
                assertTrue(rs.next());
                tIdColumnValues[j] = rs.getString("t_id");
                k1ColumnValue[j] = rs.getInt("k1");
                k3ColumnValue[j] = rs.getInt("k3");
            }
            Arrays.sort(tIdColumnValues);
            Arrays.sort(k1ColumnValue);
            Arrays.sort(k3ColumnValue);
            assertTrue(Arrays.equals(strings, tIdColumnValues));
            for (int m = 0; m < 26; m++) {
                assertEquals(m, k1ColumnValue[m]);
                assertEquals(m + 2, k3ColumnValue[m]);
            }
        }
    } finally {
        conn1.close();
    }
}
 
Example 13
Source File: MutableIndexSplitIT.java    From phoenix with Apache License 2.0 4 votes vote down vote up
private List<RegionInfo> splitDuringScan(Connection conn1, String tableName, String indexName, String[] strings, Admin admin, boolean isReverse)
        throws SQLException, IOException, InterruptedException {
    ResultSet rs;

    String query = "SELECT t_id,k1,v1 FROM " + tableName;
    rs = conn1.createStatement().executeQuery(query);
    String[] tIdColumnValues = new String[26]; 
    String[] v1ColumnValues = new String[26];
    int[] k1ColumnValue = new int[26];
    for (int j = 0; j < 5; j++) {
        assertTrue(rs.next());
        tIdColumnValues[j] = rs.getString("t_id");
        k1ColumnValue[j] = rs.getInt("k1");
        v1ColumnValues[j] = rs.getString("V1");
    }

    String[] splitKeys = new String[2];
    splitKeys[0] = strings[4];
    splitKeys[1] = strings[12];

    int[] splitInts = new int[2];
    splitInts[0] = 22;
    splitInts[1] = 4;
    List<RegionInfo> regionsOfUserTable = null;
    for(int i = 0; i <=1; i++) {
        boolean split = false;
        for (int j = 0; j < 150 && !split; j++) {
            try {
                if (localIndex) {
                    //With Hbase 2.2 the local index splits trigger longCompactions, and have
                    //to wait for an RS_COMPACTED_FILES_DISCHARGER run before the second split
                    //is successful
                    admin.split(TableName.valueOf(tableName),
                            ByteUtil.concat(Bytes.toBytes(splitKeys[i])));
                } else {
                    admin.split(TableName.valueOf(indexName),
                            ByteUtil.concat(Bytes.toBytes(splitInts[i])));
                }
                split = true;
            } catch (IOException x) {
                // wait up to a minute for the split to succeed
                Thread.sleep(1000);
            }
        }
        assertTrue(split);

        regionsOfUserTable =
                MetaTableAccessor.getTableRegions(admin.getConnection(),
                    TableName.valueOf(localIndex ? tableName : indexName), false);

        while (regionsOfUserTable.size() < (i+2)) {
            Thread.sleep(1000);
            regionsOfUserTable =
                    MetaTableAccessor.getTableRegions(admin.getConnection(),
                        TableName.valueOf(localIndex ? tableName : indexName), false);
        }
        assertTrue(regionsOfUserTable.size() >= (i+2));
    }
    for (int j = 5; j < 26; j++) {
        assertTrue(rs.next());
        tIdColumnValues[j] = rs.getString("t_id");
        k1ColumnValue[j] = rs.getInt("k1");
        v1ColumnValues[j] = rs.getString("V1");
    }
    Arrays.sort(tIdColumnValues);
    Arrays.sort(v1ColumnValues);
    Arrays.sort(k1ColumnValue);
    assertTrue(Arrays.equals(strings, tIdColumnValues));
    assertTrue(Arrays.equals(strings, v1ColumnValues));
    for(int i=0;i<26;i++) {
        assertEquals(i, k1ColumnValue[i]);
    }
    assertFalse(rs.next());
    return regionsOfUserTable;
}
 
Example 14
Source File: TestMasterOperationsForRegionReplicas.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Test
public void testIncompleteMetaTableReplicaInformation() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  final int numRegions = 3;
  final int numReplica = 2;
  try {
    // Create a table and let the meta table be updated with the location of the
    // region locations.
    TableDescriptor desc =
      TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
        .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
    ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
    TEST_UTIL.waitUntilNoRegionsInTransition();
    Set<byte[]> tableRows = new HashSet<>();
    List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
    for (RegionInfo hri : hris) {
      tableRows.add(hri.getRegionName());
    }
    ADMIN.disableTable(tableName);
    // now delete one replica info from all the rows
    // this is to make the meta appear to be only partially updated
    Table metaTable = ADMIN.getConnection().getTable(TableName.META_TABLE_NAME);
    for (byte[] row : tableRows) {
      Delete deleteOneReplicaLocation = new Delete(row);
      deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
        CatalogFamilyFormat.getServerColumn(1));
      deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
        CatalogFamilyFormat.getSeqNumColumn(1));
      deleteOneReplicaLocation.addColumns(HConstants.CATALOG_FAMILY,
        CatalogFamilyFormat.getStartCodeColumn(1));
      metaTable.delete(deleteOneReplicaLocation);
    }
    metaTable.close();
    // even if the meta table is partly updated, when we re-enable the table, we should
    // get back the desired number of replicas for the regions
    ADMIN.enableTable(tableName);
    assertTrue(ADMIN.isTableEnabled(tableName));
    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
    TEST_UTIL.waitUntilNoRegionsInTransition();
    List<RegionInfo> regions = TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager()
      .getRegionStates().getRegionsOfTable(tableName);
    assertEquals(numRegions * numReplica, regions.size());
  } finally {
    ADMIN.disableTable(tableName);
    ADMIN.deleteTable(tableName);
  }
}
 
Example 15
Source File: LocalIndexIT.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@Test
public void testLocalIndexScanAfterRegionSplit() throws Exception {
    createBaseTable(TestUtil.DEFAULT_DATA_TABLE_NAME, null, "('e','j','o')");
    Connection conn1 = DriverManager.getConnection(getUrl());
    try{
        String[] strings = {"a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"};
        for (int i = 0; i < 26; i++) {
            conn1.createStatement().execute(
                "UPSERT INTO " + TestUtil.DEFAULT_DATA_TABLE_NAME + " values('"+strings[i]+"'," + i + ","
                        + (i + 1) + "," + (i + 2) + ",'" + strings[25 - i] + "')");
        }
        conn1.commit();
        conn1.createStatement().execute("CREATE LOCAL INDEX " + TestUtil.DEFAULT_INDEX_TABLE_NAME + " ON " + TestUtil.DEFAULT_DATA_TABLE_NAME + "(v1)");
        conn1.createStatement().execute("CREATE LOCAL INDEX " + TestUtil.DEFAULT_INDEX_TABLE_NAME + "_2 ON " + TestUtil.DEFAULT_DATA_TABLE_NAME + "(k3)");

        ResultSet rs = conn1.createStatement().executeQuery("SELECT * FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME);
        assertTrue(rs.next());
        
        HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin();
        for (int i = 1; i < 5; i++) {
            admin.split(Bytes.toBytes(TestUtil.DEFAULT_DATA_TABLE_NAME), ByteUtil.concat(Bytes.toBytes(strings[3*i])));
            List<HRegionInfo> regionsOfUserTable =
                    MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), admin.getConnection(),
                            TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);

            while (regionsOfUserTable.size() != (4+i)) {
                Thread.sleep(100);
                regionsOfUserTable = MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
                        admin.getConnection(), TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);
            }
            assertEquals(4+i, regionsOfUserTable.size());
            TableName indexTable =
                    TableName.valueOf(MetaDataUtil.getLocalIndexTableName(TestUtil.DEFAULT_DATA_TABLE_NAME));
            List<HRegionInfo> regionsOfIndexTable =
                    MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
                            admin.getConnection(), indexTable, false);

            while (regionsOfIndexTable.size() != (4 + i)) {
                Thread.sleep(100);
                regionsOfIndexTable = MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
                        admin.getConnection(), indexTable, false);
            }
            assertEquals(4 + i, regionsOfIndexTable.size());
            String query = "SELECT t_id,k1,v1 FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME;
            rs = conn1.createStatement().executeQuery(query);
            Thread.sleep(1000);
            for (int j = 0; j < 26; j++) {
                assertTrue(rs.next());
                assertEquals(strings[25-j], rs.getString("t_id"));
                assertEquals(25-j, rs.getInt("k1"));
                assertEquals(strings[j], rs.getString("V1"));
            }
            rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
            assertEquals(
                    "CLIENT PARALLEL " + (4 + i) + "-WAY RANGE SCAN OVER "
                            + MetaDataUtil.getLocalIndexTableName(TestUtil.DEFAULT_DATA_TABLE_NAME) + " [-32768]\n"
                                    + "    SERVER FILTER BY FIRST KEY ONLY\n"
                            + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));
            
            query = "SELECT t_id,k1,k3 FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME;
            rs = conn1.createStatement().executeQuery("EXPLAIN "+query);
            assertEquals(
                "CLIENT PARALLEL "
                        + ((strings[3 * i].compareTo("j") < 0) ? (4 + i) : (4 + i - 1))
                        + "-WAY RANGE SCAN OVER "
                        + MetaDataUtil.getLocalIndexTableName(TestUtil.DEFAULT_DATA_TABLE_NAME) + " [-32767]\n"
                                + "    SERVER FILTER BY FIRST KEY ONLY\n"
                        + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));
            rs = conn1.createStatement().executeQuery(query);
            Thread.sleep(1000);
            for (int j = 0; j < 26; j++) {
                assertTrue(rs.next());
                assertEquals(strings[j], rs.getString("t_id"));
                assertEquals(j, rs.getInt("k1"));
                assertEquals(j+2, rs.getInt("k3"));
            }
        }
   } finally {
        conn1.close();
    }
}
 
Example 16
Source File: TestMergeTableRegionsProcedure.java    From hbase with Apache License 2.0 4 votes vote down vote up
private void testMerge(TableName tableName, int mergeCount) throws IOException {
  List<RegionInfo> ris = MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName);
  int originalRegionCount = ris.size();
  assertTrue(originalRegionCount > mergeCount);
  RegionInfo[] regionsToMerge = ris.subList(0, mergeCount).toArray(new RegionInfo [] {});
  int countOfRowsLoaded = 0;
  try (Table table = UTIL.getConnection().getTable(tableName)) {
    countOfRowsLoaded = loadARowPerRegion(table, ris);
  }
  assertEquals(countOfRowsLoaded, UTIL.countRows(tableName));

  // collect AM metrics before test
  collectAssignmentManagerMetrics();
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  MergeTableRegionsProcedure proc =
      new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true);
  long procId = procExec.submitProcedure(proc);
  ProcedureTestingUtility.waitProcedure(procExec, procId);
  ProcedureTestingUtility.assertProcNotFailed(procExec, procId);
  MetaTableAccessor.fullScanMetaAndPrint(UTIL.getConnection());
  assertEquals(originalRegionCount - mergeCount + 1,
      MetaTableAccessor.getTableRegions(UTIL.getConnection(), tableName).size());

  assertEquals(mergeSubmittedCount + 1, mergeProcMetrics.getSubmittedCounter().getCount());
  assertEquals(mergeFailedCount, mergeProcMetrics.getFailedCounter().getCount());
  assertEquals(assignSubmittedCount + 1, assignProcMetrics.getSubmittedCounter().getCount());
  assertEquals(assignFailedCount, assignProcMetrics.getFailedCounter().getCount());
  assertEquals(unassignSubmittedCount + mergeCount,
      unassignProcMetrics.getSubmittedCounter().getCount());
  assertEquals(unassignFailedCount, unassignProcMetrics.getFailedCounter().getCount());

  // Need to get the references cleaned out. Close of region will move them
  // to archive so disable and reopen just to get rid of references to later
  // when the catalogjanitor runs, it can do merged region cleanup.
  admin.disableTable(tableName);
  admin.enableTable(tableName);

  // Can I purge the merged regions from hbase:meta? Check that all went
  // well by looking at the merged row up in hbase:meta. It should have no
  // more mention of the merged regions; they are purged as last step in
  // the merged regions cleanup.
  UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(true);
  UTIL.getHBaseCluster().getMaster().getCatalogJanitor().triggerNow();
  byte [] mergedRegion = proc.getMergedRegion().getRegionName();
  while (ris != null && ris.get(0) != null && ris.get(1) != null) {
    ris = MetaTableAccessor.getMergeRegions(UTIL.getConnection(), mergedRegion);
    LOG.info("{} {}", Bytes.toStringBinary(mergedRegion), ris);
    Threads.sleep(1000);
  }
  assertEquals(countOfRowsLoaded, UTIL.countRows(tableName));
}
 
Example 17
Source File: MasterSnapshotVerifier.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Check that all the regions in the snapshot are valid, and accounted for.
 * @param manifest snapshot manifest to inspect
 * @throws IOException if we can't reach hbase:meta or read the files from the FS
 */
private void verifyRegions(final SnapshotManifest manifest) throws IOException {
  List<RegionInfo> regions;
  if (TableName.META_TABLE_NAME.equals(tableName)) {
    regions = MetaTableLocator.getMetaRegions(services.getZooKeeper());
  } else {
    regions = MetaTableAccessor.getTableRegions(services.getConnection(), tableName);
  }
  // Remove the non-default regions
  RegionReplicaUtil.removeNonDefaultRegions(regions);

  Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
  if (regionManifests == null) {
    String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty";
    LOG.error(msg);
    throw new CorruptedSnapshotException(msg);
  }

  String errorMsg = "";
  boolean hasMobStore = false;
  // the mob region is a dummy region, it's not a real region in HBase.
  // the mob region has a special name, it could be found by the region name.
  if (regionManifests.get(MobUtils.getMobRegionInfo(tableName).getEncodedName()) != null) {
    hasMobStore = true;
  }
  int realRegionCount = hasMobStore ? regionManifests.size() - 1 : regionManifests.size();
  if (realRegionCount != regions.size()) {
    errorMsg = "Regions moved during the snapshot '" +
                 ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" +
                 regions.size() + " snapshotted=" + realRegionCount + ".";
    LOG.error(errorMsg);
  }

  // Verify RegionInfo
  for (RegionInfo region : regions) {
    SnapshotRegionManifest regionManifest = regionManifests.get(region.getEncodedName());
    if (regionManifest == null) {
      // could happen due to a move or split race.
      String mesg = " No snapshot region directory found for region:" + region;
      if (errorMsg.isEmpty()) errorMsg = mesg;
      LOG.error(mesg);
      continue;
    }

    verifyRegionInfo(region, regionManifest);
  }

  if (!errorMsg.isEmpty()) {
    throw new CorruptedSnapshotException(errorMsg);
  }

  // Verify Snapshot HFiles
  // Requires the root directory file system as HFiles are stored in the root directory
  SnapshotReferenceUtil.verifySnapshot(services.getConfiguration(),
    CommonFSUtils.getRootDirFileSystem(services.getConfiguration()), manifest);
}
 
Example 18
Source File: LocalIndexIT.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@Test
public void testLocalIndexScanAfterRegionsMerge() throws Exception {
    createBaseTable(TestUtil.DEFAULT_DATA_TABLE_NAME, null, "('e','j','o')");
    Connection conn1 = DriverManager.getConnection(getUrl());
    try{
        String[] strings = {"a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"};
        for (int i = 0; i < 26; i++) {
            conn1.createStatement().execute(
                "UPSERT INTO " + TestUtil.DEFAULT_DATA_TABLE_NAME + " values('"+strings[i]+"'," + i + ","
                        + (i + 1) + "," + (i + 2) + ",'" + strings[25 - i] + "')");
        }
        conn1.commit();
        conn1.createStatement().execute("CREATE LOCAL INDEX " + TestUtil.DEFAULT_INDEX_TABLE_NAME + " ON " + TestUtil.DEFAULT_DATA_TABLE_NAME + "(v1)");
        conn1.createStatement().execute("CREATE LOCAL INDEX " + TestUtil.DEFAULT_INDEX_TABLE_NAME + "_2 ON " + TestUtil.DEFAULT_DATA_TABLE_NAME + "(k3)");

        ResultSet rs = conn1.createStatement().executeQuery("SELECT * FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME);
        assertTrue(rs.next());

        HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin();
        List<HRegionInfo> regionsOfUserTable =
                MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), admin.getConnection(),
                    TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);
        admin.mergeRegions(regionsOfUserTable.get(0).getEncodedNameAsBytes(),
            regionsOfUserTable.get(1).getEncodedNameAsBytes(), false);
        regionsOfUserTable =
                MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(), admin.getConnection(),
                    TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);

        while (regionsOfUserTable.size() != 3) {
            Thread.sleep(100);
            regionsOfUserTable = MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
                    admin.getConnection(), TableName.valueOf(TestUtil.DEFAULT_DATA_TABLE_NAME), false);
        }
        assertEquals(3, regionsOfUserTable.size());
        TableName indexTable =
                TableName.valueOf(MetaDataUtil
                        .getLocalIndexTableName(TestUtil.DEFAULT_DATA_TABLE_NAME));
        List<HRegionInfo> regionsOfIndexTable =
                MetaTableAccessor.getTableRegions(getUtility().getZooKeeperWatcher(),
                        admin.getConnection(), indexTable, false);

        while (regionsOfIndexTable.size() != 3) {
            Thread.sleep(100);
            regionsOfIndexTable = MetaTableAccessor.getTableRegions(
                    getUtility().getZooKeeperWatcher(), admin.getConnection(), indexTable, false);
        }
        assertEquals(3, regionsOfIndexTable.size());
        String query = "SELECT t_id,k1,v1 FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME;
        rs = conn1.createStatement().executeQuery(query);
        Thread.sleep(1000);
        for (int j = 0; j < 26; j++) {
            assertTrue(rs.next());
            assertEquals(strings[25 - j], rs.getString("t_id"));
            assertEquals(25 - j, rs.getInt("k1"));
            assertEquals(strings[j], rs.getString("V1"));
        }
        rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
        assertEquals(
            "CLIENT PARALLEL " + 3 + "-WAY RANGE SCAN OVER "
                    + MetaDataUtil.getLocalIndexTableName(TestUtil.DEFAULT_DATA_TABLE_NAME)
                    + " [-32768]\n" + "    SERVER FILTER BY FIRST KEY ONLY\n"
                    + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));

        query = "SELECT t_id,k1,k3 FROM " + TestUtil.DEFAULT_DATA_TABLE_NAME;
        rs = conn1.createStatement().executeQuery("EXPLAIN " + query);
        assertEquals(
            "CLIENT PARALLEL " + 3 + "-WAY RANGE SCAN OVER "
                    + MetaDataUtil.getLocalIndexTableName(TestUtil.DEFAULT_DATA_TABLE_NAME)
                    + " [-32767]\n" + "    SERVER FILTER BY FIRST KEY ONLY\n"
                    + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));

        rs = conn1.createStatement().executeQuery(query);
        Thread.sleep(1000);
        for (int j = 0; j < 26; j++) {
            assertTrue(rs.next());
            assertEquals(strings[j], rs.getString("t_id"));
            assertEquals(j, rs.getInt("k1"));
            assertEquals(j + 2, rs.getInt("k3"));
        }
   } finally {
        conn1.close();
    }
}