Java Code Examples for org.apache.hadoop.hbase.util.VersionInfo#getVersion()

@Test
public void testCorrectlyCleansUpResources() throws Exception{
  ExecutorService exec = Executors.newFixedThreadPool(1);
  RegionCoprocessorEnvironment e =Mockito.mock(RegionCoprocessorEnvironment.class);
  Configuration conf =new Configuration();
  Mockito.when(e.getConfiguration()).thenReturn(conf);
  Mockito.when(e.getSharedData()).thenReturn(new ConcurrentHashMap<String,Object>());
  FakeTableFactory factory = new FakeTableFactory(
      Collections.<ImmutableBytesPtr, Table> emptyMap());
  TrackingParallelWriterIndexCommitter writer = new TrackingParallelWriterIndexCommitter(VersionInfo.getVersion());
  Stoppable mockStop = Mockito.mock(Stoppable.class);
  // create a simple writer
  writer.setup(factory, exec, mockStop,e);
  // stop the writer
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
  Mockito.verifyZeroInteractions(mockStop);
}
 

@Test
public void testCorrectlyCleansUpResources() throws Exception{
  ExecutorService exec = Executors.newFixedThreadPool(1);
  FakeTableFactory factory = new FakeTableFactory(
      Collections.<ImmutableBytesPtr, HTableInterface> emptyMap());
  ParallelWriterIndexCommitter writer = new ParallelWriterIndexCommitter(VersionInfo.getVersion());
  Abortable mockAbort = Mockito.mock(Abortable.class);
  Stoppable mockStop = Mockito.mock(Stoppable.class);
  // create a simple writer
  writer.setup(factory, exec, mockAbort, mockStop, 1);
  // stop the writer
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
  Mockito.verifyZeroInteractions(mockAbort, mockStop);
}
 

private static List<String> computeClientVersions() throws Exception {
    String hbaseVersion = VersionInfo.getVersion();
    Pattern p = Pattern.compile("\\d+\\.\\d+");
    Matcher m = p.matcher(hbaseVersion);
    String hbaseProfile = null;
    if (m.find()) {
        hbaseProfile = m.group();
    }
    List<String> clientVersions = Lists.newArrayList();
    ObjectMapper mapper = new ObjectMapper();
    mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
    JsonNode jsonNode = mapper.readTree(new FileReader(COMPATIBLE_CLIENTS_JSON));
    JsonNode HBaseProfile = jsonNode.get(hbaseProfile);
    for (final JsonNode clientVersion : HBaseProfile) {
        clientVersions.add(clientVersion.textValue() + "-HBase-" + hbaseProfile);
    }
    return clientVersions;
}
 

public ConnectionlessQueryServicesImpl(QueryServices queryServices) {
    super(queryServices);
    metaData = PMetaDataImpl.EMPTY_META_DATA;
    // find the HBase version and use that to determine the KeyValueBuilder that should be used
    String hbaseVersion = VersionInfo.getVersion();
    this.kvBuilder = KeyValueBuilder.get(hbaseVersion);
}
 

/**
 * We don't support WAL Compression for HBase < 0.94.9, so we shouldn't even allow the server
 * to start if both indexing and WAL Compression are enabled for the wrong versions.
 */
@Test
public void testDoesNotSupportCompressedWAL() {
  Configuration conf = HBaseConfiguration.create();
  IndexTestingUtils.setupConfig(conf);
  // get the current version
  String version = VersionInfo.getVersion();
  
  // ensure WAL Compression not enabled
  conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, false);
  
  //we support all versions without WAL Compression
  String supported = Indexer.validateVersion(version, conf);
  assertNull(
    "WAL Compression wasn't enabled, but version "+version+" of HBase wasn't supported! All versions should"
        + " support writing without a compressed WAL. Message: "+supported, supported);

  // enable WAL Compression
  conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);

  // set the version to something we know isn't supported
  version = "0.94.4";
  supported = Indexer.validateVersion(version, conf);
  assertNotNull("WAL Compression was enabled, but incorrectly marked version as supported",
    supported);
  
  //make sure the first version of 0.94 that supports Indexing + WAL Compression works
  version = "0.94.9";
  supported = Indexer.validateVersion(version, conf);
  assertNull(
    "WAL Compression wasn't enabled, but version "+version+" of HBase wasn't supported! Message: "+supported, supported);
  
  //make sure we support snapshot builds too
  version = "0.94.9-SNAPSHOT";
  supported = Indexer.validateVersion(version, conf);
  assertNull(
    "WAL Compression wasn't enabled, but version "+version+" of HBase wasn't supported! Message: "+supported, supported);
}
 

/**
 * We don't support WAL Compression for HBase < 0.94.9, so we shouldn't even allow the server
 * to start if both indexing and WAL Compression are enabled for the wrong versions.
 */
@Test
public void testDoesNotSupportCompressedWAL() {
    Configuration conf = HBaseConfiguration.create();
    IndexTestingUtils.setupConfig(conf);
    // get the current version
    String version = VersionInfo.getVersion();

    // ensure WAL Compression not enabled
    conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, false);

    //we support all versions without WAL Compression
    String supported = Indexer.validateVersion(version, conf);
    assertNull(
            "WAL Compression wasn't enabled, but version "+version+" of HBase wasn't supported! All versions should"
                    + " support writing without a compressed WAL. Message: "+supported, supported);

    // enable WAL Compression
    conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);

    // set the version to something we know isn't supported
    version = "0.94.4";
    supported = Indexer.validateVersion(version, conf);
    assertNotNull("WAL Compression was enabled, but incorrectly marked version as supported",
            supported);

    //make sure the first version of 0.94 that supports Indexing + WAL Compression works
    version = "0.94.9";
    supported = Indexer.validateVersion(version, conf);
    assertNull(
            "WAL Compression wasn't enabled, but version "+version+" of HBase wasn't supported! Message: "+supported, supported);

    //make sure we support snapshot builds too
    version = "0.94.9-SNAPSHOT";
    supported = Indexer.validateVersion(version, conf);
    assertNull(
            "WAL Compression wasn't enabled, but version "+version+" of HBase wasn't supported! Message: "+supported, supported);
}
 

@Override
public void runChild(FrameworkMethod method, RunNotifier notifier) {
    MinVersion methodCondition = method.getAnnotation(MinVersion.class);
    MinVersion classCondition = this.getTestClass().getJavaClass().getAnnotation(MinVersion.class);
    String versionStr = VersionInfo.getVersion();
    int version = VersionUtil.encodeVersion(versionStr);
    if (  (methodCondition == null || version >= VersionUtil.encodeVersion(methodCondition.value()))
       && (classCondition == null || version >= VersionUtil.encodeVersion(classCondition.value()))) {
        super.runChild(method, notifier);
    } else {
        notifier.fireTestIgnored(describeChild(method));
    }
}
 

private Put generateOriginalPut() {
    String version = VersionInfo.getVersion();
    KeyValueBuilder builder = KeyValueBuilder.get(version);
    KeyValue kv = builder.buildPut(wrap(ROW), wrap(TABLE_FAMILY_BYTES), wrap(QUALIFIER),
            wrap(ORIGINAL_VALUE));
    Put put = new Put(ROW);
    KeyValueBuilder.addQuietly(put, kv);
    return put;
}
 

@Test
public void testConditionallyAddTagsToPutCells( ) {
    List<Tag> tags = TagUtil.asList(VIEW_MODIFIED_PROPERTY_BYTES, 0, VIEW_MODIFIED_PROPERTY_BYTES.length);
    assertEquals(tags.size(), 1);
    Tag expectedTag = tags.get(0);

    String version = VersionInfo.getVersion();
    KeyValueBuilder builder = KeyValueBuilder.get(version);
    KeyValue kv = builder.buildPut(wrap(ROW), wrap(TABLE_FAMILY_BYTES), wrap(UPDATE_CACHE_FREQUENCY_BYTES), wrap(
            PLong.INSTANCE.toBytes(0)));
    Put put = new Put(ROW);
    KeyValueBuilder.addQuietly(put, kv);

    ExtendedCellBuilder cellBuilder = (ExtendedCellBuilder) RawCellBuilderFactory.create();
    MetaDataUtil.conditionallyAddTagsToPutCells(put, TABLE_FAMILY_BYTES, UPDATE_CACHE_FREQUENCY_BYTES, cellBuilder,
            PInteger.INSTANCE.toBytes(1), VIEW_MODIFIED_PROPERTY_BYTES);

    Cell cell = put.getFamilyCellMap().get(TABLE_FAMILY_BYTES).get(0);

    // To check the cell tag whether view has modified this property
    assertTrue(Bytes.compareTo(expectedTag.getValueArray(), TagUtil.concatTags(EMPTY_BYTE_ARRAY, cell)) == 0);
    assertTrue(Bytes.contains(TagUtil.concatTags(EMPTY_BYTE_ARRAY, cell), expectedTag.getValueArray()));

    // To check tag data can be correctly deserialized
    Iterator<Tag> tagIterator = PrivateCellUtil.tagsIterator(cell);
    assertTrue(tagIterator.hasNext());
    Tag actualTag = tagIterator.next();
    assertTrue(Bytes.compareTo(actualTag.getValueArray(), actualTag.getValueOffset(), actualTag.getValueLength(),
            expectedTag.getValueArray(), expectedTag.getValueOffset(), expectedTag.getValueLength()) == 0);
    assertFalse(tagIterator.hasNext());
}
 

private void testIndexRowKeyBuilding(String schemaName, String tableName, String dataColumns, String pk, String indexColumns, Object[] values, String includeColumns, String dataProps, String indexProps) throws Exception {
    KeyValueBuilder builder = GenericKeyValueBuilder.INSTANCE;
    testIndexRowKeyBuilding(schemaName, tableName, dataColumns, pk, indexColumns, values, includeColumns, dataProps, indexProps, builder);

    //do the same, but with the client key-value builder, to ensure that works the same
    
    String hbaseVersion = VersionInfo.getVersion();
    if (KeyValueBuilder.get(hbaseVersion) == ClientKeyValueBuilder.INSTANCE) {
        builder = ClientKeyValueBuilder.INSTANCE;
        testIndexRowKeyBuilding(schemaName, tableName, dataColumns, pk, indexColumns, values, includeColumns, dataProps, indexProps, builder);
    }
}
 

@SuppressWarnings({ "unchecked", "deprecation" })
@Test
public void testSynchronouslyCompletesAllWrites() throws Exception {
  LOG.info("Starting " + test.getTableNameString());
  LOG.info("Current thread is interrupted: " + Thread.interrupted());
  Abortable abort = new StubAbortable();
  Stoppable stop = Mockito.mock(Stoppable.class);
  ExecutorService exec = Executors.newFixedThreadPool(1);
  Map<ImmutableBytesPtr, HTableInterface> tables =
      new HashMap<ImmutableBytesPtr, HTableInterface>();
  FakeTableFactory factory = new FakeTableFactory(tables);

  ImmutableBytesPtr tableName = new ImmutableBytesPtr(this.test.getTableName());
  Put m = new Put(row);
  m.add(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
  Multimap<HTableInterfaceReference, Mutation> indexUpdates =
      ArrayListMultimap.<HTableInterfaceReference, Mutation> create();
  indexUpdates.put(new HTableInterfaceReference(tableName), m);

  HTableInterface table = Mockito.mock(HTableInterface.class);
  final boolean[] completed = new boolean[] { false };
  Mockito.when(table.batch(Mockito.anyList())).thenAnswer(new Answer<Void>() {

    @Override
    public Void answer(InvocationOnMock invocation) throws Throwable {
      // just keep track that it was called
      completed[0] = true;
      return null;
    }
  });
  Mockito.when(table.getTableName()).thenReturn(test.getTableName());
  // add the table to the set of tables, so its returned to the writer
  tables.put(tableName, table);

  // setup the writer and failure policy
  ParallelWriterIndexCommitter writer = new ParallelWriterIndexCommitter(VersionInfo.getVersion());
  writer.setup(factory, exec, abort, stop, 1);
  writer.write(indexUpdates);
  assertTrue("Writer returned before the table batch completed! Likely a race condition tripped",
    completed[0]);
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
}
 

public synchronized static HBaseCompat getCompat(String classOverride) {

        if (null != cachedCompat) {
            log.debug("Returning cached HBase compatibility layer: {}", cachedCompat);
            return cachedCompat;
        }

        HBaseCompat compat;
        String className = null;
        String classNameSource = null;

        if (null != classOverride) {
            className = classOverride;
            classNameSource = "from explicit configuration";
        } else {
            String hbaseVersion = VersionInfo.getVersion();
            for (String supportedVersion : Arrays.asList("0.94", "0.96", "0.98", "1.0", "1.1")) {
                if (hbaseVersion.startsWith(supportedVersion + ".")) {
                    className = "com.thinkaurelius.titan.diskstorage.hbase.HBaseCompat" + supportedVersion.replaceAll("\\.", "_");
                    classNameSource = "supporting runtime HBase version " + hbaseVersion;
                    break;
                }
            }
            if (null == className) {
                log.info("The HBase version {} is not explicitly supported by Titan.  " +
                         "Loading Titan's compatibility layer for its most recent supported HBase version ({})",
                        hbaseVersion, DEFAULT_HBASE_COMPAT_VERSION);
                className = DEFAULT_HBASE_CLASS_NAME;
                classNameSource = " by default";
            }
        }

        final String errTemplate = " when instantiating HBase compatibility class " + className;

        try {
            compat = (HBaseCompat)Class.forName(className).newInstance();
            log.info("Instantiated HBase compatibility layer {}: {}", classNameSource, compat.getClass().getCanonicalName());
        } catch (IllegalAccessException | ClassNotFoundException | InstantiationException e) {
            throw new RuntimeException(e.getClass().getSimpleName() + errTemplate, e);
        }

        return cachedCompat = compat;
    }
 

/**
 * Test that we correctly abort a RegionServer when we run tests with an unsupported HBase
 * version. The 'completeness' of this test requires that we run the test with both a version of
 * HBase that wouldn't be supported with WAL Compression. Currently, this is the default version
 * (0.94.4) so just running 'mvn test' will run the full test. However, this test will not fail
 * when running against a version of HBase with WALCompression enabled. Therefore, to fully test
 * this functionality, we need to run the test against both a supported and an unsupported version
 * of HBase (as long as we want to support an version of HBase that doesn't support custom WAL
 * Codecs).
 * @throws Exception on failure
 */
@Test(timeout = 300000 /* 5 mins */)
public void testDoesNotStartRegionServerForUnsupportedCompressionAndVersion() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  IndexTestingUtils.setupConfig(conf);
  // enable WAL Compression
  conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);

  // check the version to see if it isn't supported
  String version = VersionInfo.getVersion();
  boolean supported = false;
  if (Indexer.validateVersion(version, conf) == null) {
    supported = true;
  }

  // start the minicluster
  HBaseTestingUtility util = new HBaseTestingUtility(conf);
  util.startMiniCluster();

  // setup the primary table
  HTableDescriptor desc = new HTableDescriptor(
      "testDoesNotStartRegionServerForUnsupportedCompressionAndVersion");
  byte[] family = Bytes.toBytes("f");
  desc.addFamily(new HColumnDescriptor(family));

  // enable indexing to a non-existant index table
  String indexTableName = "INDEX_TABLE";
  ColumnGroup fam1 = new ColumnGroup(indexTableName);
  fam1.add(new CoveredColumn(family, CoveredColumn.ALL_QUALIFIERS));
  CoveredColumnIndexSpecifierBuilder builder = new CoveredColumnIndexSpecifierBuilder();
  builder.addIndexGroup(fam1);
  builder.build(desc);

  // get a reference to the regionserver, so we can ensure it aborts
  HRegionServer server = util.getMiniHBaseCluster().getRegionServer(0);

  // create the primary table
  HBaseAdmin admin = util.getHBaseAdmin();
  if (supported) {
    admin.createTable(desc);
    assertFalse("Hosting regeion server failed, even the HBase version (" + version
        + ") supports WAL Compression.", server.isAborted());
  } else {
    admin.createTableAsync(desc, null);

    // wait for the regionserver to abort - if this doesn't occur in the timeout, assume its
    // broken.
    while (!server.isAborted()) {
      LOG.debug("Waiting on regionserver to abort..");
    }
  }

  // cleanup
  util.shutdownMiniCluster();
}
 

private static boolean isUseDefaultDecoder() {
    String hbaseVersion = VersionInfo.getVersion();
    return VersionUtil.encodeVersion(hbaseVersion) >= MIN_BINARY_COMPATIBLE_INDEX_CODEC_VERSION;
}
 

@SuppressWarnings({ "unchecked", "deprecation" })
@Test
public void testSynchronouslyCompletesAllWrites() throws Exception {
  LOGGER.info("Starting " + test.getTableNameString());
  LOGGER.info("Current thread is interrupted: " + Thread.interrupted());
  Abortable abort = new StubAbortable();
  Stoppable stop = Mockito.mock(Stoppable.class);
  ExecutorService exec = Executors.newFixedThreadPool(1);
  Map<ImmutableBytesPtr, Table> tables =
      new LinkedHashMap<ImmutableBytesPtr, Table>();
  FakeTableFactory factory = new FakeTableFactory(tables);
  RegionCoprocessorEnvironment e =Mockito.mock(RegionCoprocessorEnvironment.class);
  Configuration conf =new Configuration();
  Mockito.when(e.getConfiguration()).thenReturn(conf);
  Mockito.when(e.getSharedData()).thenReturn(new ConcurrentHashMap<String,Object>());
  Region mockRegion = Mockito.mock(Region.class);
  Mockito.when(e.getRegion()).thenReturn(mockRegion);
  TableDescriptor mockTableDesc = Mockito.mock(TableDescriptor.class);
  Mockito.when(mockRegion.getTableDescriptor()).thenReturn(mockTableDesc);
  Mockito.when(mockTableDesc.getTableName()).thenReturn(TableName.valueOf("test"));
  Connection mockConnection = Mockito.mock(Connection.class);
  Mockito.when(e.getConnection()).thenReturn(mockConnection);
  ImmutableBytesPtr tableName = new ImmutableBytesPtr(this.test.getTableName());
  Put m = new Put(row);
  m.addColumn(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
  Multimap<HTableInterfaceReference, Mutation> indexUpdates =
      ArrayListMultimap.<HTableInterfaceReference, Mutation> create();
  indexUpdates.put(new HTableInterfaceReference(tableName), m);

  Table table = Mockito.mock(Table.class);
  final boolean[] completed = new boolean[] { false };
  Mockito.doAnswer(new Answer<Void>() {

    @Override
    public Void answer(InvocationOnMock invocation) throws Throwable {
      // just keep track that it was called
      completed[0] = true;
      return null;
    }
  }).when(table).batch(Mockito.anyList(),Mockito.any());
  Mockito.when(table.getName()).thenReturn(org.apache.hadoop.hbase.TableName.valueOf(test.getTableName()));
  // add the table to the set of tables, so its returned to the writer
  tables.put(tableName, table);

  // setup the writer and failure policy
  TrackingParallelWriterIndexCommitter writer = new TrackingParallelWriterIndexCommitter(VersionInfo.getVersion());
  writer.setup(factory, exec, stop, e);
  writer.write(indexUpdates, true, ScanUtil.UNKNOWN_CLIENT_VERSION);
  assertTrue("Writer returned before the table batch completed! Likely a race condition tripped",
    completed[0]);
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
}
 

/**
 * With the move to using a pool of threads to write, we need to ensure that we still block until
 * all index writes for a mutation/batch are completed.
 * @throws Exception on failure
 */
@SuppressWarnings({ "unchecked", "deprecation" })
@Test
public void testSynchronouslyCompletesAllWrites() throws Exception {
  LOG.info("Starting " + testName.getTableNameString());
  LOG.info("Current thread is interrupted: " + Thread.interrupted());
  Abortable abort = new StubAbortable();
  Stoppable stop = Mockito.mock(Stoppable.class);
  ExecutorService exec = Executors.newFixedThreadPool(1);
  Map<ImmutableBytesPtr, HTableInterface> tables = new HashMap<ImmutableBytesPtr, HTableInterface>();
  FakeTableFactory factory = new FakeTableFactory(tables);

  byte[] tableName = this.testName.getTableName();
  Put m = new Put(row);
  m.add(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
  Collection<Pair<Mutation, byte[]>> indexUpdates = Arrays.asList(new Pair<Mutation, byte[]>(m,
      tableName));

  HTableInterface table = Mockito.mock(HTableInterface.class);
  final boolean[] completed = new boolean[] { false };
  Mockito.when(table.batch(Mockito.anyList())).thenAnswer(new Answer<Void>() {

    @Override
    public Void answer(InvocationOnMock invocation) throws Throwable {
      // just keep track that it was called
      completed[0] = true;
      return null;
    }
  });
  Mockito.when(table.getTableName()).thenReturn(testName.getTableName());
  // add the table to the set of tables, so its returned to the writer
  tables.put(new ImmutableBytesPtr(tableName), table);

  // setup the writer and failure policy
  ParallelWriterIndexCommitter committer = new ParallelWriterIndexCommitter(VersionInfo.getVersion());
  committer.setup(factory, exec, abort, stop, 2);
  KillServerOnFailurePolicy policy = new KillServerOnFailurePolicy();
  policy.setup(stop, abort);
  IndexWriter writer = new IndexWriter(committer, policy);
  writer.write(indexUpdates);
  assertTrue("Writer returned before the table batch completed! Likely a race condition tripped",
    completed[0]);
  writer.stop(this.testName.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
}
 

/**
 * Test that we correctly abort a RegionServer when we run tests with an unsupported HBase
 * version. The 'completeness' of this test requires that we run the test with both a version of
 * HBase that wouldn't be supported with WAL Compression. Currently, this is the default version
 * (0.94.4) so just running 'mvn test' will run the full test. However, this test will not fail
 * when running against a version of HBase with WALCompression enabled. Therefore, to fully test
 * this functionality, we need to run the test against both a supported and an unsupported version
 * of HBase (as long as we want to support an version of HBase that doesn't support custom WAL
 * Codecs).
 * @throws Exception on failure
 */
@Test(timeout = 300000 /* 5 mins */)
public void testDoesNotStartRegionServerForUnsupportedCompressionAndVersion() throws Exception {
    Configuration conf = HBaseConfiguration.create();
    setUpConfigForMiniCluster(conf);
    IndexTestingUtils.setupConfig(conf);
    // enable WAL Compression
    conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);

    // check the version to see if it isn't supported
    String version = VersionInfo.getVersion();
    boolean supported = false;
    if (Indexer.validateVersion(version, conf) == null) {
        supported = true;
    }

    // start the minicluster
    HBaseTestingUtility util = new HBaseTestingUtility(conf);
    util.startMiniCluster();

    try {
        // setup the primary table
        TableDescriptorBuilder descBuilder = TableDescriptorBuilder.newBuilder(TableName.valueOf(
                "testDoesNotStartRegionServerForUnsupportedCompressionAndVersion"));
        byte[] family = Bytes.toBytes("f");
        
        descBuilder.addColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
        TableDescriptor desc=descBuilder.build();
        // enable indexing to a non-existant index table
        String indexTableName = "INDEX_TABLE";
        ColumnGroup fam1 = new ColumnGroup(indexTableName);
        fam1.add(new CoveredColumn(family, CoveredColumn.ALL_QUALIFIERS));
        CoveredColumnIndexSpecifierBuilder builder = new CoveredColumnIndexSpecifierBuilder();
        builder.addIndexGroup(fam1);
        builder.build(desc);

        // get a reference to the regionserver, so we can ensure it aborts
        HRegionServer server = util.getMiniHBaseCluster().getRegionServer(0);

        // create the primary table
        Admin admin = util.getAdmin();
        if (supported) {
            admin.createTable(desc);
            assertFalse("Hosting regeion server failed, even the HBase version (" + version
                    + ") supports WAL Compression.", server.isAborted());
        } else {
            admin.createTableAsync(desc, null);

            // wait for the regionserver to abort - if this doesn't occur in the timeout, assume its
            // broken.
            while (!server.isAborted()) {
                LOGGER.debug("Waiting on regionserver to abort..");
            }
        }

    } finally {
        // cleanup
        util.shutdownMiniCluster();
    }
}
 

/**
 * With the move to using a pool of threads to write, we need to ensure that we still block until
 * all index writes for a mutation/batch are completed.
 * @throws Exception on failure
 */
@Test
public void testSynchronouslyCompletesAllWrites() throws Exception {
  LOGGER.info("Starting " + testName.getTableNameString());
  LOGGER.info("Current thread is interrupted: " + Thread.interrupted());
  Abortable abort = new StubAbortable();
  Stoppable stop = Mockito.mock(Stoppable.class);
  RegionCoprocessorEnvironment e =Mockito.mock(RegionCoprocessorEnvironment.class);
  Configuration conf =new Configuration();
  Mockito.when(e.getConfiguration()).thenReturn(conf);
  Mockito.when(e.getSharedData()).thenReturn(new ConcurrentHashMap<String,Object>());
  Region mockRegion = Mockito.mock(Region.class);
  Mockito.when(e.getRegion()).thenReturn(mockRegion);
  TableDescriptor mockTableDesc = Mockito.mock(TableDescriptor.class);
  Mockito.when(mockRegion.getTableDescriptor()).thenReturn(mockTableDesc);
  TableName mockTN = TableName.valueOf("test");
  Mockito.when(mockTableDesc.getTableName()).thenReturn(mockTN);
  Connection mockConnection = Mockito.mock(Connection.class);
  Mockito.when(e.getConnection()).thenReturn(mockConnection);
  ExecutorService exec = Executors.newFixedThreadPool(1);
  Map<ImmutableBytesPtr, Table> tables = new HashMap<ImmutableBytesPtr, Table>();
  FakeTableFactory factory = new FakeTableFactory(tables);

  byte[] tableName = this.testName.getTableName();
  Put m = new Put(row);
  m.addColumn(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
  Collection<Pair<Mutation, byte[]>> indexUpdates = Arrays.asList(new Pair<Mutation, byte[]>(m,
      tableName));

  Table table = Mockito.mock(Table.class);
  final boolean[] completed = new boolean[] { false };
      Mockito.doAnswer(new Answer<Void>() {
          @Override
          public Void answer(InvocationOnMock invocation) throws Throwable {
              // just keep track that it was called
              completed[0] = true;
              return null;
          }
      }).when(table).batch(Mockito.anyList(), Mockito.any());
  Mockito.when(table.getName()).thenReturn(TableName.valueOf(testName.getTableName()));
  // add the table to the set of tables, so its returned to the writer
  tables.put(new ImmutableBytesPtr(tableName), table);

  // setup the writer and failure policy
  TrackingParallelWriterIndexCommitter committer = new TrackingParallelWriterIndexCommitter(VersionInfo.getVersion());
  committer.setup(factory, exec, stop, e);
  KillServerOnFailurePolicy policy = new KillServerOnFailurePolicy();
  policy.setup(stop, e);
  IndexWriter writer = new IndexWriter(committer, policy);
  writer.write(indexUpdates, ScanUtil.UNKNOWN_CLIENT_VERSION);
  assertTrue("Writer returned before the table batch completed! Likely a race condition tripped",
    completed[0]);
  writer.stop(this.testName.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
}
 

/**
 * Test that we correctly abort a RegionServer when we run tests with an unsupported HBase
 * version. The 'completeness' of this test requires that we run the test with both a version of
 * HBase that wouldn't be supported with WAL Compression. Currently, this is the default version
 * (0.94.4) so just running 'mvn test' will run the full test. However, this test will not fail
 * when running against a version of HBase with WALCompression enabled. Therefore, to fully test
 * this functionality, we need to run the test against both a supported and an unsupported version
 * of HBase (as long as we want to support an version of HBase that doesn't support custom WAL
 * Codecs).
 * @throws Exception on failure
 */
@Test(timeout = 300000 /* 5 mins */)
public void testDoesNotStartRegionServerForUnsupportedCompressionAndVersion() throws Exception {
    Configuration conf = HBaseConfiguration.create();
    setUpConfigForMiniCluster(conf);
    IndexTestingUtils.setupConfig(conf);
    // enable WAL Compression
    conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);

    // check the version to see if it isn't supported
    String version = VersionInfo.getVersion();
    boolean supported = false;
    if (Indexer.validateVersion(version, conf) == null) {
        supported = true;
    }

    // start the minicluster
    HBaseTestingUtility util = new HBaseTestingUtility(conf);
    // set replication required parameter
    ConfigUtil.setReplicationConfigIfAbsent(conf);
    try {
        util.startMiniCluster();

        // setup the primary table
        @SuppressWarnings("deprecation")
        HTableDescriptor desc = new HTableDescriptor(
                "testDoesNotStartRegionServerForUnsupportedCompressionAndVersion");
        byte[] family = Bytes.toBytes("f");
        desc.addFamily(new HColumnDescriptor(family));

        // enable indexing to a non-existant index table
        String indexTableName = "INDEX_TABLE";
        ColumnGroup fam1 = new ColumnGroup(indexTableName);
        fam1.add(new CoveredColumn(family, CoveredColumn.ALL_QUALIFIERS));
        CoveredColumnIndexSpecifierBuilder builder = new CoveredColumnIndexSpecifierBuilder();
        builder.addIndexGroup(fam1);
        builder.build(desc);

        // get a reference to the regionserver, so we can ensure it aborts
        HRegionServer server = util.getMiniHBaseCluster().getRegionServer(0);

        // create the primary table
        HBaseAdmin admin = util.getHBaseAdmin();
        if (supported) {
            admin.createTable(desc);
            assertFalse("Hosting regeion server failed, even the HBase version (" + version
                    + ") supports WAL Compression.", server.isAborted());
        } else {
            admin.createTableAsync(desc, null);

            // wait for the regionserver to abort - if this doesn't occur in the timeout, assume its
            // broken.
            while (!server.isAborted()) {
                LOG.debug("Waiting on regionserver to abort..");
            }
        }
    } finally {
        // cleanup
        util.shutdownMiniCluster();
    }
}
 

@SuppressWarnings({ "unchecked"})
@Test
public void testSynchronouslyCompletesAllWrites() throws Exception {
  LOGGER.info("Starting " + test.getTableNameString());
  LOGGER.info("Current thread is interrupted: " + Thread.interrupted());
  RegionCoprocessorEnvironment e =Mockito.mock(RegionCoprocessorEnvironment.class);
  Configuration conf =new Configuration();
  Mockito.when(e.getConfiguration()).thenReturn(conf);
  Mockito.when(e.getSharedData()).thenReturn(new ConcurrentHashMap<String,Object>());
  Region mockRegion = Mockito.mock(Region.class);
  Mockito.when(e.getRegion()).thenReturn(mockRegion);
  TableDescriptor mockTableDesc = Mockito.mock(TableDescriptor.class);
  Mockito.when(mockTableDesc.getTableName()).thenReturn(TableName.valueOf("test"));
  Connection mockConnection = Mockito.mock(Connection.class);
  Mockito.when(e.getConnection()).thenReturn(mockConnection);
  Mockito.when(mockRegion.getTableDescriptor()).thenReturn(mockTableDesc);
  Stoppable stop = Mockito.mock(Stoppable.class);
  ExecutorService exec = Executors.newFixedThreadPool(1);
  Map<ImmutableBytesPtr, Table> tables =
      new LinkedHashMap<ImmutableBytesPtr, Table>();
  FakeTableFactory factory = new FakeTableFactory(tables);

  ImmutableBytesPtr tableName = new ImmutableBytesPtr(this.test.getTableName());
  Put m = new Put(row);
  m.addColumn(Bytes.toBytes("family"), Bytes.toBytes("qual"), null);
  Multimap<HTableInterfaceReference, Mutation> indexUpdates =
      ArrayListMultimap.<HTableInterfaceReference, Mutation> create();
  indexUpdates.put(new HTableInterfaceReference(tableName), m);

  Table table = Mockito.mock(Table.class);
  final boolean[] completed = new boolean[] { false };
      Mockito.doAnswer(new Answer<Void>() {
          @Override
          public Void answer(InvocationOnMock invocation) throws Throwable {
              // just keep track that it was called
              completed[0] = true;
              return null;
          }
      }).when(table).batch(Mockito.anyList(), Mockito.any());
      Mockito.when(table.getName()).thenReturn(org.apache.hadoop.hbase.TableName.valueOf(test.getTableName()));
  // add the table to the set of tables, so its returned to the writer
  tables.put(tableName, table);

  // setup the writer and failure policy
  TrackingParallelWriterIndexCommitter writer = new TrackingParallelWriterIndexCommitter(VersionInfo.getVersion());
  writer.setup(factory, exec, stop, e);
  writer.write(indexUpdates, true, ScanUtil.UNKNOWN_CLIENT_VERSION);
  assertTrue("Writer returned before the table batch completed! Likely a race condition tripped",
    completed[0]);
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
}