Java Code Examples for org.apache.hadoop.hbase.util.VersionInfo#getVersion()

The following examples show how to use org.apache.hadoop.hbase.util.VersionInfo#getVersion() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestParalleIndexWriter.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Test
public void testCorrectlyCleansUpResources() throws Exception{
  ExecutorService exec = Executors.newFixedThreadPool(1);
  RegionCoprocessorEnvironment e =Mockito.mock(RegionCoprocessorEnvironment.class);
  Configuration conf =new Configuration();
  Mockito.when(e.getConfiguration()).thenReturn(conf);
  Mockito.when(e.getSharedData()).thenReturn(new ConcurrentHashMap<String,Object>());
  FakeTableFactory factory = new FakeTableFactory(
      Collections.<ImmutableBytesPtr, Table> emptyMap());
  TrackingParallelWriterIndexCommitter writer = new TrackingParallelWriterIndexCommitter(VersionInfo.getVersion());
  Stoppable mockStop = Mockito.mock(Stoppable.class);
  // create a simple writer
  writer.setup(factory, exec, mockStop,e);
  // stop the writer
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
  Mockito.verifyZeroInteractions(mockStop);
}
 
Example 2
Source File: TestParalleWriterIndexCommitter.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Test
public void testCorrectlyCleansUpResources() throws Exception{
  ExecutorService exec = Executors.newFixedThreadPool(1);
  FakeTableFactory factory = new FakeTableFactory(
      Collections.<ImmutableBytesPtr, HTableInterface> emptyMap());
  ParallelWriterIndexCommitter writer = new ParallelWriterIndexCommitter(VersionInfo.getVersion());
  Abortable mockAbort = Mockito.mock(Abortable.class);
  Stoppable mockStop = Mockito.mock(Stoppable.class);
  // create a simple writer
  writer.setup(factory, exec, mockAbort, mockStop, 1);
  // stop the writer
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
  Mockito.verifyZeroInteractions(mockAbort, mockStop);
}
 
Example 3
Source File: BackwardCompatibilityIT.java    From phoenix with Apache License 2.0 6 votes vote down vote up
private static List<String> computeClientVersions() throws Exception {
    String hbaseVersion = VersionInfo.getVersion();
    Pattern p = Pattern.compile("\\d+\\.\\d+");
    Matcher m = p.matcher(hbaseVersion);
    String hbaseProfile = null;
    if (m.find()) {
        hbaseProfile = m.group();
    }
    List<String> clientVersions = Lists.newArrayList();
    ObjectMapper mapper = new ObjectMapper();
    mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
    JsonNode jsonNode = mapper.readTree(new FileReader(COMPATIBLE_CLIENTS_JSON));
    JsonNode HBaseProfile = jsonNode.get(hbaseProfile);
    for (final JsonNode clientVersion : HBaseProfile) {
        clientVersions.add(clientVersion.textValue() + "-HBase-" + hbaseProfile);
    }
    return clientVersions;
}
 
Example 4
Source File: ConnectionlessQueryServicesImpl.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
public ConnectionlessQueryServicesImpl(QueryServices queryServices) {
    super(queryServices);
    metaData = PMetaDataImpl.EMPTY_META_DATA;
    // find the HBase version and use that to determine the KeyValueBuilder that should be used
    String hbaseVersion = VersionInfo.getVersion();
    this.kvBuilder = KeyValueBuilder.get(hbaseVersion);
}
 
Example 5
Source File: TestFailForUnsupportedHBaseVersions.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
/**
 * We don't support WAL Compression for HBase &lt; 0.94.9, so we shouldn't even allow the server
 * to start if both indexing and WAL Compression are enabled for the wrong versions.
 */
@Test
public void testDoesNotSupportCompressedWAL() {
  Configuration conf = HBaseConfiguration.create();
  IndexTestingUtils.setupConfig(conf);
  // get the current version
  String version = VersionInfo.getVersion();
  
  // ensure WAL Compression not enabled
  conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, false);
  
  //we support all versions without WAL Compression
  String supported = Indexer.validateVersion(version, conf);
  assertNull(
    "WAL Compression wasn't enabled, but version "+version+" of HBase wasn't supported! All versions should"
        + " support writing without a compressed WAL. Message: "+supported, supported);

  // enable WAL Compression
  conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);

  // set the version to something we know isn't supported
  version = "0.94.4";
  supported = Indexer.validateVersion(version, conf);
  assertNotNull("WAL Compression was enabled, but incorrectly marked version as supported",
    supported);
  
  //make sure the first version of 0.94 that supports Indexing + WAL Compression works
  version = "0.94.9";
  supported = Indexer.validateVersion(version, conf);
  assertNull(
    "WAL Compression wasn't enabled, but version "+version+" of HBase wasn't supported! Message: "+supported, supported);
  
  //make sure we support snapshot builds too
  version = "0.94.9-SNAPSHOT";
  supported = Indexer.validateVersion(version, conf);
  assertNull(
    "WAL Compression wasn't enabled, but version "+version+" of HBase wasn't supported! Message: "+supported, supported);
}
 
Example 6
Source File: FailForUnsupportedHBaseVersionsIT.java    From phoenix with Apache License 2.0 5 votes vote down vote up
/**
 * We don't support WAL Compression for HBase &lt; 0.94.9, so we shouldn't even allow the server
 * to start if both indexing and WAL Compression are enabled for the wrong versions.
 */
@Test
public void testDoesNotSupportCompressedWAL() {
    Configuration conf = HBaseConfiguration.create();
    IndexTestingUtils.setupConfig(conf);
    // get the current version
    String version = VersionInfo.getVersion();

    // ensure WAL Compression not enabled
    conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, false);

    //we support all versions without WAL Compression
    String supported = Indexer.validateVersion(version, conf);
    assertNull(
            "WAL Compression wasn't enabled, but version "+version+" of HBase wasn't supported! All versions should"
                    + " support writing without a compressed WAL. Message: "+supported, supported);

    // enable WAL Compression
    conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);

    // set the version to something we know isn't supported
    version = "0.94.4";
    supported = Indexer.validateVersion(version, conf);
    assertNotNull("WAL Compression was enabled, but incorrectly marked version as supported",
            supported);

    //make sure the first version of 0.94 that supports Indexing + WAL Compression works
    version = "0.94.9";
    supported = Indexer.validateVersion(version, conf);
    assertNull(
            "WAL Compression wasn't enabled, but version "+version+" of HBase wasn't supported! Message: "+supported, supported);

    //make sure we support snapshot builds too
    version = "0.94.9-SNAPSHOT";
    supported = Indexer.validateVersion(version, conf);
    assertNull(
            "WAL Compression wasn't enabled, but version "+version+" of HBase wasn't supported! Message: "+supported, supported);
}
 
Example 7
Source File: MinVersionTestRunner.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
public void runChild(FrameworkMethod method, RunNotifier notifier) {
    MinVersion methodCondition = method.getAnnotation(MinVersion.class);
    MinVersion classCondition = this.getTestClass().getJavaClass().getAnnotation(MinVersion.class);
    String versionStr = VersionInfo.getVersion();
    int version = VersionUtil.encodeVersion(versionStr);
    if (  (methodCondition == null || version >= VersionUtil.encodeVersion(methodCondition.value()))
       && (classCondition == null || version >= VersionUtil.encodeVersion(classCondition.value()))) {
        super.runChild(method, notifier);
    } else {
        notifier.fireTestIgnored(describeChild(method));
    }
}
 
Example 8
Source File: MetaDataUtilTest.java    From phoenix with Apache License 2.0 5 votes vote down vote up
private Put generateOriginalPut() {
    String version = VersionInfo.getVersion();
    KeyValueBuilder builder = KeyValueBuilder.get(version);
    KeyValue kv = builder.buildPut(wrap(ROW), wrap(TABLE_FAMILY_BYTES), wrap(QUALIFIER),
            wrap(ORIGINAL_VALUE));
    Put put = new Put(ROW);
    KeyValueBuilder.addQuietly(put, kv);
    return put;
}
 
Example 9
Source File: MetaDataUtilTest.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Test
public void testConditionallyAddTagsToPutCells( ) {
    List<Tag> tags = TagUtil.asList(VIEW_MODIFIED_PROPERTY_BYTES, 0, VIEW_MODIFIED_PROPERTY_BYTES.length);
    assertEquals(tags.size(), 1);
    Tag expectedTag = tags.get(0);

    String version = VersionInfo.getVersion();
    KeyValueBuilder builder = KeyValueBuilder.get(version);
    KeyValue kv = builder.buildPut(wrap(ROW), wrap(TABLE_FAMILY_BYTES), wrap(UPDATE_CACHE_FREQUENCY_BYTES), wrap(
            PLong.INSTANCE.toBytes(0)));
    Put put = new Put(ROW);
    KeyValueBuilder.addQuietly(put, kv);

    ExtendedCellBuilder cellBuilder = (ExtendedCellBuilder) RawCellBuilderFactory.create();
    MetaDataUtil.conditionallyAddTagsToPutCells(put, TABLE_FAMILY_BYTES, UPDATE_CACHE_FREQUENCY_BYTES, cellBuilder,
            PInteger.INSTANCE.toBytes(1), VIEW_MODIFIED_PROPERTY_BYTES);

    Cell cell = put.getFamilyCellMap().get(TABLE_FAMILY_BYTES).get(0);

    // To check the cell tag whether view has modified this property
    assertTrue(Bytes.compareTo(expectedTag.getValueArray(), TagUtil.concatTags(EMPTY_BYTE_ARRAY, cell)) == 0);
    assertTrue(Bytes.contains(TagUtil.concatTags(EMPTY_BYTE_ARRAY, cell), expectedTag.getValueArray()));

    // To check tag data can be correctly deserialized
    Iterator<Tag> tagIterator = PrivateCellUtil.tagsIterator(cell);
    assertTrue(tagIterator.hasNext());
    Tag actualTag = tagIterator.next();
    assertTrue(Bytes.compareTo(actualTag.getValueArray(), actualTag.getValueOffset(), actualTag.getValueLength(),
            expectedTag.getValueArray(), expectedTag.getValueOffset(), expectedTag.getValueLength()) == 0);
    assertFalse(tagIterator.hasNext());
}
 
Example 10
Source File: IndexMaintainerTest.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
private void testIndexRowKeyBuilding(String schemaName, String tableName, String dataColumns, String pk, String indexColumns, Object[] values, String includeColumns, String dataProps, String indexProps) throws Exception {
    KeyValueBuilder builder = GenericKeyValueBuilder.INSTANCE;
    testIndexRowKeyBuilding(schemaName, tableName, dataColumns, pk, indexColumns, values, includeColumns, dataProps, indexProps, builder);

    //do the same, but with the client key-value builder, to ensure that works the same
    
    String hbaseVersion = VersionInfo.getVersion();
    if (KeyValueBuilder.get(hbaseVersion) == ClientKeyValueBuilder.INSTANCE) {
        builder = ClientKeyValueBuilder.INSTANCE;
        testIndexRowKeyBuilding(schemaName, tableName, dataColumns, pk, indexColumns, values, includeColumns, dataProps, indexProps, builder);
    }
}
 
Example 11
Source File: TestParalleIndexWriter.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "unchecked", "deprecation" })
@Test
public void testSynchronouslyCompletesAllWrites() throws Exception {
  LOG.info("Starting " + test.getTableNameString());
  LOG.info("Current thread is interrupted: " + Thread.interrupted());
  Abortable abort = new StubAbortable();
  Stoppable stop = Mockito.mock(Stoppable.class);
  ExecutorService exec = Executors.newFixedThreadPool(1);
  Map<ImmutableBytesPtr, HTableInterface> tables =
      new HashMap<ImmutableBytesPtr, HTableInterface>();
  FakeTableFactory factory = new FakeTableFactory(tables);

  ImmutableBytesPtr tableName = new ImmutableBytesPtr(this.test.getTableName());
  Put m = new Put(row);
  m.add(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
  Multimap<HTableInterfaceReference, Mutation> indexUpdates =
      ArrayListMultimap.<HTableInterfaceReference, Mutation> create();
  indexUpdates.put(new HTableInterfaceReference(tableName), m);

  HTableInterface table = Mockito.mock(HTableInterface.class);
  final boolean[] completed = new boolean[] { false };
  Mockito.when(table.batch(Mockito.anyList())).thenAnswer(new Answer<Void>() {

    @Override
    public Void answer(InvocationOnMock invocation) throws Throwable {
      // just keep track that it was called
      completed[0] = true;
      return null;
    }
  });
  Mockito.when(table.getTableName()).thenReturn(test.getTableName());
  // add the table to the set of tables, so its returned to the writer
  tables.put(tableName, table);

  // setup the writer and failure policy
  ParallelWriterIndexCommitter writer = new ParallelWriterIndexCommitter(VersionInfo.getVersion());
  writer.setup(factory, exec, abort, stop, 1);
  writer.write(indexUpdates);
  assertTrue("Writer returned before the table batch completed! Likely a race condition tripped",
    completed[0]);
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
}
 
Example 12
Source File: HBaseCompatLoader.java    From incubator-atlas with Apache License 2.0 4 votes vote down vote up
public synchronized static HBaseCompat getCompat(String classOverride) {

        if (null != cachedCompat) {
            log.debug("Returning cached HBase compatibility layer: {}", cachedCompat);
            return cachedCompat;
        }

        HBaseCompat compat;
        String className = null;
        String classNameSource = null;

        if (null != classOverride) {
            className = classOverride;
            classNameSource = "from explicit configuration";
        } else {
            String hbaseVersion = VersionInfo.getVersion();
            for (String supportedVersion : Arrays.asList("0.94", "0.96", "0.98", "1.0", "1.1")) {
                if (hbaseVersion.startsWith(supportedVersion + ".")) {
                    className = "com.thinkaurelius.titan.diskstorage.hbase.HBaseCompat" + supportedVersion.replaceAll("\\.", "_");
                    classNameSource = "supporting runtime HBase version " + hbaseVersion;
                    break;
                }
            }
            if (null == className) {
                log.info("The HBase version {} is not explicitly supported by Titan.  " +
                         "Loading Titan's compatibility layer for its most recent supported HBase version ({})",
                        hbaseVersion, DEFAULT_HBASE_COMPAT_VERSION);
                className = DEFAULT_HBASE_CLASS_NAME;
                classNameSource = " by default";
            }
        }

        final String errTemplate = " when instantiating HBase compatibility class " + className;

        try {
            compat = (HBaseCompat)Class.forName(className).newInstance();
            log.info("Instantiated HBase compatibility layer {}: {}", classNameSource, compat.getClass().getCanonicalName());
        } catch (IllegalAccessException | ClassNotFoundException | InstantiationException e) {
            throw new RuntimeException(e.getClass().getSimpleName() + errTemplate, e);
        }

        return cachedCompat = compat;
    }
 
Example 13
Source File: TestFailForUnsupportedHBaseVersions.java    From phoenix with BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
/**
 * Test that we correctly abort a RegionServer when we run tests with an unsupported HBase
 * version. The 'completeness' of this test requires that we run the test with both a version of
 * HBase that wouldn't be supported with WAL Compression. Currently, this is the default version
 * (0.94.4) so just running 'mvn test' will run the full test. However, this test will not fail
 * when running against a version of HBase with WALCompression enabled. Therefore, to fully test
 * this functionality, we need to run the test against both a supported and an unsupported version
 * of HBase (as long as we want to support an version of HBase that doesn't support custom WAL
 * Codecs).
 * @throws Exception on failure
 */
@Test(timeout = 300000 /* 5 mins */)
public void testDoesNotStartRegionServerForUnsupportedCompressionAndVersion() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  IndexTestingUtils.setupConfig(conf);
  // enable WAL Compression
  conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);

  // check the version to see if it isn't supported
  String version = VersionInfo.getVersion();
  boolean supported = false;
  if (Indexer.validateVersion(version, conf) == null) {
    supported = true;
  }

  // start the minicluster
  HBaseTestingUtility util = new HBaseTestingUtility(conf);
  util.startMiniCluster();

  // setup the primary table
  HTableDescriptor desc = new HTableDescriptor(
      "testDoesNotStartRegionServerForUnsupportedCompressionAndVersion");
  byte[] family = Bytes.toBytes("f");
  desc.addFamily(new HColumnDescriptor(family));

  // enable indexing to a non-existant index table
  String indexTableName = "INDEX_TABLE";
  ColumnGroup fam1 = new ColumnGroup(indexTableName);
  fam1.add(new CoveredColumn(family, CoveredColumn.ALL_QUALIFIERS));
  CoveredColumnIndexSpecifierBuilder builder = new CoveredColumnIndexSpecifierBuilder();
  builder.addIndexGroup(fam1);
  builder.build(desc);

  // get a reference to the regionserver, so we can ensure it aborts
  HRegionServer server = util.getMiniHBaseCluster().getRegionServer(0);

  // create the primary table
  HBaseAdmin admin = util.getHBaseAdmin();
  if (supported) {
    admin.createTable(desc);
    assertFalse("Hosting regeion server failed, even the HBase version (" + version
        + ") supports WAL Compression.", server.isAborted());
  } else {
    admin.createTableAsync(desc, null);

    // wait for the regionserver to abort - if this doesn't occur in the timeout, assume its
    // broken.
    while (!server.isAborted()) {
      LOG.debug("Waiting on regionserver to abort..");
    }
  }

  // cleanup
  util.shutdownMiniCluster();
}
 
Example 14
Source File: IndexedWALEditCodec.java    From phoenix with Apache License 2.0 4 votes vote down vote up
private static boolean isUseDefaultDecoder() {
    String hbaseVersion = VersionInfo.getVersion();
    return VersionUtil.encodeVersion(hbaseVersion) >= MIN_BINARY_COMPATIBLE_INDEX_CODEC_VERSION;
}
 
Example 15
Source File: TestParalleIndexWriter.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "unchecked", "deprecation" })
@Test
public void testSynchronouslyCompletesAllWrites() throws Exception {
  LOGGER.info("Starting " + test.getTableNameString());
  LOGGER.info("Current thread is interrupted: " + Thread.interrupted());
  Abortable abort = new StubAbortable();
  Stoppable stop = Mockito.mock(Stoppable.class);
  ExecutorService exec = Executors.newFixedThreadPool(1);
  Map<ImmutableBytesPtr, Table> tables =
      new LinkedHashMap<ImmutableBytesPtr, Table>();
  FakeTableFactory factory = new FakeTableFactory(tables);
  RegionCoprocessorEnvironment e =Mockito.mock(RegionCoprocessorEnvironment.class);
  Configuration conf =new Configuration();
  Mockito.when(e.getConfiguration()).thenReturn(conf);
  Mockito.when(e.getSharedData()).thenReturn(new ConcurrentHashMap<String,Object>());
  Region mockRegion = Mockito.mock(Region.class);
  Mockito.when(e.getRegion()).thenReturn(mockRegion);
  TableDescriptor mockTableDesc = Mockito.mock(TableDescriptor.class);
  Mockito.when(mockRegion.getTableDescriptor()).thenReturn(mockTableDesc);
  Mockito.when(mockTableDesc.getTableName()).thenReturn(TableName.valueOf("test"));
  Connection mockConnection = Mockito.mock(Connection.class);
  Mockito.when(e.getConnection()).thenReturn(mockConnection);
  ImmutableBytesPtr tableName = new ImmutableBytesPtr(this.test.getTableName());
  Put m = new Put(row);
  m.addColumn(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
  Multimap<HTableInterfaceReference, Mutation> indexUpdates =
      ArrayListMultimap.<HTableInterfaceReference, Mutation> create();
  indexUpdates.put(new HTableInterfaceReference(tableName), m);

  Table table = Mockito.mock(Table.class);
  final boolean[] completed = new boolean[] { false };
  Mockito.doAnswer(new Answer<Void>() {

    @Override
    public Void answer(InvocationOnMock invocation) throws Throwable {
      // just keep track that it was called
      completed[0] = true;
      return null;
    }
  }).when(table).batch(Mockito.anyList(),Mockito.any());
  Mockito.when(table.getName()).thenReturn(org.apache.hadoop.hbase.TableName.valueOf(test.getTableName()));
  // add the table to the set of tables, so its returned to the writer
  tables.put(tableName, table);

  // setup the writer and failure policy
  TrackingParallelWriterIndexCommitter writer = new TrackingParallelWriterIndexCommitter(VersionInfo.getVersion());
  writer.setup(factory, exec, stop, e);
  writer.write(indexUpdates, true, ScanUtil.UNKNOWN_CLIENT_VERSION);
  assertTrue("Writer returned before the table batch completed! Likely a race condition tripped",
    completed[0]);
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
}
 
Example 16
Source File: TestIndexWriter.java    From phoenix with Apache License 2.0 4 votes vote down vote up
/**
 * With the move to using a pool of threads to write, we need to ensure that we still block until
 * all index writes for a mutation/batch are completed.
 * @throws Exception on failure
 */
@SuppressWarnings({ "unchecked", "deprecation" })
@Test
public void testSynchronouslyCompletesAllWrites() throws Exception {
  LOG.info("Starting " + testName.getTableNameString());
  LOG.info("Current thread is interrupted: " + Thread.interrupted());
  Abortable abort = new StubAbortable();
  Stoppable stop = Mockito.mock(Stoppable.class);
  ExecutorService exec = Executors.newFixedThreadPool(1);
  Map<ImmutableBytesPtr, HTableInterface> tables = new HashMap<ImmutableBytesPtr, HTableInterface>();
  FakeTableFactory factory = new FakeTableFactory(tables);

  byte[] tableName = this.testName.getTableName();
  Put m = new Put(row);
  m.add(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
  Collection<Pair<Mutation, byte[]>> indexUpdates = Arrays.asList(new Pair<Mutation, byte[]>(m,
      tableName));

  HTableInterface table = Mockito.mock(HTableInterface.class);
  final boolean[] completed = new boolean[] { false };
  Mockito.when(table.batch(Mockito.anyList())).thenAnswer(new Answer<Void>() {

    @Override
    public Void answer(InvocationOnMock invocation) throws Throwable {
      // just keep track that it was called
      completed[0] = true;
      return null;
    }
  });
  Mockito.when(table.getTableName()).thenReturn(testName.getTableName());
  // add the table to the set of tables, so its returned to the writer
  tables.put(new ImmutableBytesPtr(tableName), table);

  // setup the writer and failure policy
  ParallelWriterIndexCommitter committer = new ParallelWriterIndexCommitter(VersionInfo.getVersion());
  committer.setup(factory, exec, abort, stop, 2);
  KillServerOnFailurePolicy policy = new KillServerOnFailurePolicy();
  policy.setup(stop, abort);
  IndexWriter writer = new IndexWriter(committer, policy);
  writer.write(indexUpdates);
  assertTrue("Writer returned before the table batch completed! Likely a race condition tripped",
    completed[0]);
  writer.stop(this.testName.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
}
 
Example 17
Source File: FailForUnsupportedHBaseVersionsIT.java    From phoenix with Apache License 2.0 4 votes vote down vote up
/**
 * Test that we correctly abort a RegionServer when we run tests with an unsupported HBase
 * version. The 'completeness' of this test requires that we run the test with both a version of
 * HBase that wouldn't be supported with WAL Compression. Currently, this is the default version
 * (0.94.4) so just running 'mvn test' will run the full test. However, this test will not fail
 * when running against a version of HBase with WALCompression enabled. Therefore, to fully test
 * this functionality, we need to run the test against both a supported and an unsupported version
 * of HBase (as long as we want to support an version of HBase that doesn't support custom WAL
 * Codecs).
 * @throws Exception on failure
 */
@Test(timeout = 300000 /* 5 mins */)
public void testDoesNotStartRegionServerForUnsupportedCompressionAndVersion() throws Exception {
    Configuration conf = HBaseConfiguration.create();
    setUpConfigForMiniCluster(conf);
    IndexTestingUtils.setupConfig(conf);
    // enable WAL Compression
    conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);

    // check the version to see if it isn't supported
    String version = VersionInfo.getVersion();
    boolean supported = false;
    if (Indexer.validateVersion(version, conf) == null) {
        supported = true;
    }

    // start the minicluster
    HBaseTestingUtility util = new HBaseTestingUtility(conf);
    util.startMiniCluster();

    try {
        // setup the primary table
        TableDescriptorBuilder descBuilder = TableDescriptorBuilder.newBuilder(TableName.valueOf(
                "testDoesNotStartRegionServerForUnsupportedCompressionAndVersion"));
        byte[] family = Bytes.toBytes("f");
        
        descBuilder.addColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
        TableDescriptor desc=descBuilder.build();
        // enable indexing to a non-existant index table
        String indexTableName = "INDEX_TABLE";
        ColumnGroup fam1 = new ColumnGroup(indexTableName);
        fam1.add(new CoveredColumn(family, CoveredColumn.ALL_QUALIFIERS));
        CoveredColumnIndexSpecifierBuilder builder = new CoveredColumnIndexSpecifierBuilder();
        builder.addIndexGroup(fam1);
        builder.build(desc);

        // get a reference to the regionserver, so we can ensure it aborts
        HRegionServer server = util.getMiniHBaseCluster().getRegionServer(0);

        // create the primary table
        Admin admin = util.getAdmin();
        if (supported) {
            admin.createTable(desc);
            assertFalse("Hosting regeion server failed, even the HBase version (" + version
                    + ") supports WAL Compression.", server.isAborted());
        } else {
            admin.createTableAsync(desc, null);

            // wait for the regionserver to abort - if this doesn't occur in the timeout, assume its
            // broken.
            while (!server.isAborted()) {
                LOGGER.debug("Waiting on regionserver to abort..");
            }
        }

    } finally {
        // cleanup
        util.shutdownMiniCluster();
    }
}
 
Example 18
Source File: TestIndexWriter.java    From phoenix with Apache License 2.0 4 votes vote down vote up
/**
 * With the move to using a pool of threads to write, we need to ensure that we still block until
 * all index writes for a mutation/batch are completed.
 * @throws Exception on failure
 */
@Test
public void testSynchronouslyCompletesAllWrites() throws Exception {
  LOGGER.info("Starting " + testName.getTableNameString());
  LOGGER.info("Current thread is interrupted: " + Thread.interrupted());
  Abortable abort = new StubAbortable();
  Stoppable stop = Mockito.mock(Stoppable.class);
  RegionCoprocessorEnvironment e =Mockito.mock(RegionCoprocessorEnvironment.class);
  Configuration conf =new Configuration();
  Mockito.when(e.getConfiguration()).thenReturn(conf);
  Mockito.when(e.getSharedData()).thenReturn(new ConcurrentHashMap<String,Object>());
  Region mockRegion = Mockito.mock(Region.class);
  Mockito.when(e.getRegion()).thenReturn(mockRegion);
  TableDescriptor mockTableDesc = Mockito.mock(TableDescriptor.class);
  Mockito.when(mockRegion.getTableDescriptor()).thenReturn(mockTableDesc);
  TableName mockTN = TableName.valueOf("test");
  Mockito.when(mockTableDesc.getTableName()).thenReturn(mockTN);
  Connection mockConnection = Mockito.mock(Connection.class);
  Mockito.when(e.getConnection()).thenReturn(mockConnection);
  ExecutorService exec = Executors.newFixedThreadPool(1);
  Map<ImmutableBytesPtr, Table> tables = new HashMap<ImmutableBytesPtr, Table>();
  FakeTableFactory factory = new FakeTableFactory(tables);

  byte[] tableName = this.testName.getTableName();
  Put m = new Put(row);
  m.addColumn(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
  Collection<Pair<Mutation, byte[]>> indexUpdates = Arrays.asList(new Pair<Mutation, byte[]>(m,
      tableName));

  Table table = Mockito.mock(Table.class);
  final boolean[] completed = new boolean[] { false };
      Mockito.doAnswer(new Answer<Void>() {
          @Override
          public Void answer(InvocationOnMock invocation) throws Throwable {
              // just keep track that it was called
              completed[0] = true;
              return null;
          }
      }).when(table).batch(Mockito.anyList(), Mockito.any());
  Mockito.when(table.getName()).thenReturn(TableName.valueOf(testName.getTableName()));
  // add the table to the set of tables, so its returned to the writer
  tables.put(new ImmutableBytesPtr(tableName), table);

  // setup the writer and failure policy
  TrackingParallelWriterIndexCommitter committer = new TrackingParallelWriterIndexCommitter(VersionInfo.getVersion());
  committer.setup(factory, exec, stop, e);
  KillServerOnFailurePolicy policy = new KillServerOnFailurePolicy();
  policy.setup(stop, e);
  IndexWriter writer = new IndexWriter(committer, policy);
  writer.write(indexUpdates, ScanUtil.UNKNOWN_CLIENT_VERSION);
  assertTrue("Writer returned before the table batch completed! Likely a race condition tripped",
    completed[0]);
  writer.stop(this.testName.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
}
 
Example 19
Source File: FailForUnsupportedHBaseVersionsIT.java    From phoenix with Apache License 2.0 4 votes vote down vote up
/**
 * Test that we correctly abort a RegionServer when we run tests with an unsupported HBase
 * version. The 'completeness' of this test requires that we run the test with both a version of
 * HBase that wouldn't be supported with WAL Compression. Currently, this is the default version
 * (0.94.4) so just running 'mvn test' will run the full test. However, this test will not fail
 * when running against a version of HBase with WALCompression enabled. Therefore, to fully test
 * this functionality, we need to run the test against both a supported and an unsupported version
 * of HBase (as long as we want to support an version of HBase that doesn't support custom WAL
 * Codecs).
 * @throws Exception on failure
 */
@Test(timeout = 300000 /* 5 mins */)
public void testDoesNotStartRegionServerForUnsupportedCompressionAndVersion() throws Exception {
    Configuration conf = HBaseConfiguration.create();
    setUpConfigForMiniCluster(conf);
    IndexTestingUtils.setupConfig(conf);
    // enable WAL Compression
    conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);

    // check the version to see if it isn't supported
    String version = VersionInfo.getVersion();
    boolean supported = false;
    if (Indexer.validateVersion(version, conf) == null) {
        supported = true;
    }

    // start the minicluster
    HBaseTestingUtility util = new HBaseTestingUtility(conf);
    // set replication required parameter
    ConfigUtil.setReplicationConfigIfAbsent(conf);
    try {
        util.startMiniCluster();

        // setup the primary table
        @SuppressWarnings("deprecation")
        HTableDescriptor desc = new HTableDescriptor(
                "testDoesNotStartRegionServerForUnsupportedCompressionAndVersion");
        byte[] family = Bytes.toBytes("f");
        desc.addFamily(new HColumnDescriptor(family));

        // enable indexing to a non-existant index table
        String indexTableName = "INDEX_TABLE";
        ColumnGroup fam1 = new ColumnGroup(indexTableName);
        fam1.add(new CoveredColumn(family, CoveredColumn.ALL_QUALIFIERS));
        CoveredColumnIndexSpecifierBuilder builder = new CoveredColumnIndexSpecifierBuilder();
        builder.addIndexGroup(fam1);
        builder.build(desc);

        // get a reference to the regionserver, so we can ensure it aborts
        HRegionServer server = util.getMiniHBaseCluster().getRegionServer(0);

        // create the primary table
        HBaseAdmin admin = util.getHBaseAdmin();
        if (supported) {
            admin.createTable(desc);
            assertFalse("Hosting regeion server failed, even the HBase version (" + version
                    + ") supports WAL Compression.", server.isAborted());
        } else {
            admin.createTableAsync(desc, null);

            // wait for the regionserver to abort - if this doesn't occur in the timeout, assume its
            // broken.
            while (!server.isAborted()) {
                LOG.debug("Waiting on regionserver to abort..");
            }
        }
    } finally {
        // cleanup
        util.shutdownMiniCluster();
    }
}
 
Example 20
Source File: TestParalleWriterIndexCommitter.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "unchecked"})
@Test
public void testSynchronouslyCompletesAllWrites() throws Exception {
  LOGGER.info("Starting " + test.getTableNameString());
  LOGGER.info("Current thread is interrupted: " + Thread.interrupted());
  RegionCoprocessorEnvironment e =Mockito.mock(RegionCoprocessorEnvironment.class);
  Configuration conf =new Configuration();
  Mockito.when(e.getConfiguration()).thenReturn(conf);
  Mockito.when(e.getSharedData()).thenReturn(new ConcurrentHashMap<String,Object>());
  Region mockRegion = Mockito.mock(Region.class);
  Mockito.when(e.getRegion()).thenReturn(mockRegion);
  TableDescriptor mockTableDesc = Mockito.mock(TableDescriptor.class);
  Mockito.when(mockTableDesc.getTableName()).thenReturn(TableName.valueOf("test"));
  Connection mockConnection = Mockito.mock(Connection.class);
  Mockito.when(e.getConnection()).thenReturn(mockConnection);
  Mockito.when(mockRegion.getTableDescriptor()).thenReturn(mockTableDesc);
  Stoppable stop = Mockito.mock(Stoppable.class);
  ExecutorService exec = Executors.newFixedThreadPool(1);
  Map<ImmutableBytesPtr, Table> tables =
      new LinkedHashMap<ImmutableBytesPtr, Table>();
  FakeTableFactory factory = new FakeTableFactory(tables);

  ImmutableBytesPtr tableName = new ImmutableBytesPtr(this.test.getTableName());
  Put m = new Put(row);
  m.addColumn(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
  Multimap<HTableInterfaceReference, Mutation> indexUpdates =
      ArrayListMultimap.<HTableInterfaceReference, Mutation> create();
  indexUpdates.put(new HTableInterfaceReference(tableName), m);

  Table table = Mockito.mock(Table.class);
  final boolean[] completed = new boolean[] { false };
      Mockito.doAnswer(new Answer<Void>() {
          @Override
          public Void answer(InvocationOnMock invocation) throws Throwable {
              // just keep track that it was called
              completed[0] = true;
              return null;
          }
      }).when(table).batch(Mockito.anyList(), Mockito.any());
      Mockito.when(table.getName()).thenReturn(org.apache.hadoop.hbase.TableName.valueOf(test.getTableName()));
  // add the table to the set of tables, so its returned to the writer
  tables.put(tableName, table);

  // setup the writer and failure policy
  TrackingParallelWriterIndexCommitter writer = new TrackingParallelWriterIndexCommitter(VersionInfo.getVersion());
  writer.setup(factory, exec, stop, e);
  writer.write(indexUpdates, true, ScanUtil.UNKNOWN_CLIENT_VERSION);
  assertTrue("Writer returned before the table batch completed! Likely a race condition tripped",
    completed[0]);
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
}