org.apache.hadoop.hbase.regionserver.StoreFile Java Examples

The following examples show how to use org.apache.hadoop.hbase.regionserver.StoreFile. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PreRegionsCreation.java    From hbase-secondary-index with GNU General Public License v3.0 6 votes vote down vote up
public void create(String tableName, String[] columnFamily,
		String startKey, String endKey) throws IOException {
	HTableDescriptor desc = new HTableDescriptor(Bytes.toBytes(tableName));
	for (String cf : columnFamily) {
		HColumnDescriptor cfDesc = new HColumnDescriptor(cf);
		// column family attribute
		cfDesc.setValue(HConstants.VERSIONS,
				String.valueOf(Integer.MAX_VALUE));
		cfDesc.setValue(HColumnDescriptor.BLOOMFILTER,
				StoreFile.BloomType.ROW.toString());
		desc.addFamily(cfDesc);
	}
	if (!admin.tableExists(tableName)) {
		if (null != startKey && null != endKey)
			admin.createTable(desc, Bytes.toBytes(startKey),
					Bytes.toBytes(endKey), NUM_REGIONSERVERS);
	}
}
 
Example #2
Source File: HFileOutputFormat.java    From terrapin with Apache License 2.0 6 votes vote down vote up
public RecordWriter<BytesWritable, BytesWritable> getRecordWriter(
        TaskAttemptContext context) throws IOException {
  // Get the path of the temporary output file
  final Path outputPath = FileOutputFormat.getOutputPath(context);
  final Path outputDir = new FileOutputCommitter(outputPath, context).getWorkPath();
  final Configuration conf = context.getConfiguration();
  final FileSystem fs = outputDir.getFileSystem(conf);

  int blockSize = conf.getInt(Constants.HFILE_BLOCKSIZE, 16384);
  // Default to snappy.
  Compression.Algorithm compressionAlgorithm = getAlgorithm(
      conf.get(Constants.HFILE_COMPRESSION));
  final StoreFile.Writer writer =
      new StoreFile.WriterBuilder(conf, new CacheConfig(conf), fs, blockSize)
          .withFilePath(hfilePath(outputPath, context.getTaskAttemptID().getTaskID().getId()))
          .withCompression(compressionAlgorithm)
          .build();
  return new HFileRecordWriter(writer);
}
 
Example #3
Source File: HFileReaderTest.java    From terrapin with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  int randomNum = (int) (Math.random() * Integer.MAX_VALUE);
  hfilePath = "/tmp/hfile-" + randomNum;
  Configuration conf = new Configuration();
  FileSystem fs = FileSystem.get(conf);
  keyValueMap = Maps.newHashMapWithExpectedSize(10000);
  errorKeys = Sets.newHashSetWithExpectedSize(2000);
  StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, new CacheConfig(conf),
      fs, 4096).
      withFilePath(new Path(hfilePath)).
      withCompression(Compression.Algorithm.NONE).
      build();
  // Add upto 10K values.
  for (int i = 0; i < 10000; i++) {
    byte[] key = String.format("%04d", i).getBytes();
    byte[] value = null;
    // Add a couple of empty values for testing and making sure we return them.
    if (i <= 1) {
      value = "".getBytes();
    } else {
      value = ("v" + (i + 1)).getBytes();
    }
    KeyValue kv = new KeyValue(key,
        Bytes.toBytes("cf"),
        Bytes.toBytes(""),
        value);
    writer.append(kv);
    keyValueMap.put(ByteBuffer.wrap(key), ByteBuffer.wrap(value));
    if (i >= 4000 && i < 6000) {
      errorKeys.add(ByteBuffer.wrap(key));
    }
  }
  writer.close();
  hfileReader = new TestHFileReader(fs,
      hfilePath,
      new CacheConfig(conf),
      new ExecutorServiceFuturePool(Executors.newFixedThreadPool(1)),
      errorKeys);
}
 
Example #4
Source File: TransactionalRegion.java    From hbase-secondary-index with GNU General Public License v3.0 5 votes vote down vote up
@Override
public List<StoreFile> close(final boolean abort) throws IOException {
    prepareToClose();
    if (!commitPendingTransactions.isEmpty()) {
        LOG.warn("Closing transactional region [" + getRegionInfo().getRegionNameAsString() + "], but still have ["
                + commitPendingTransactions.size() + "] transactions  that are pending commit.");
        // TODO resolve from the Global Trx Log.
    }
    return super.close(abort);
}
 
Example #5
Source File: RepairUtil.java    From phoenix with Apache License 2.0 5 votes vote down vote up
public static boolean isLocalIndexStoreFilesConsistent(RegionCoprocessorEnvironment environment, Store store) {
    byte[] startKey = environment.getRegion().getRegionInfo().getStartKey();
    byte[] endKey = environment.getRegion().getRegionInfo().getEndKey();
    byte[] indexKeyEmbedded = startKey.length == 0 ? new byte[endKey.length] : startKey;
    for (StoreFile file : store.getStorefiles()) {
        if (file.getFirstKey().isPresent() && file.getFirstKey().get() != null) {
            byte[] fileFirstRowKey = CellUtil.cloneRow(file.getFirstKey().get());
            if ((fileFirstRowKey != null && Bytes.compareTo(fileFirstRowKey, 0,
                    indexKeyEmbedded.length, indexKeyEmbedded, 0, indexKeyEmbedded.length) != 0)) {
                return false; }
        }
    }
    return true;
}
 
Example #6
Source File: OnlyDeleteExpiredFilesCompactionPolicy.java    From opensoc-streaming with Apache License 2.0 5 votes vote down vote up
@Override
final ArrayList<StoreFile> applyCompactionPolicy(final ArrayList<StoreFile> candidates, final boolean mayUseOffPeak,
    final boolean mayBeStuck) throws IOException {
  LOG.info("Sending empty list for compaction to avoid compaction and do only deletes of files older than TTL");

  return new ArrayList<StoreFile>();
}
 
Example #7
Source File: ExampleRegionObserverWithMetrics.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void postCompactSelection(
    ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends StoreFile> selected, CompactionLifeCycleTracker tracker,
    CompactionRequest request) {
  if (selected != null) {
    filesCompactedCounter.increment(selected.size());
  }
}
 
Example #8
Source File: SimpleRegionObserver.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void postFlush(ObserverContext<RegionCoprocessorEnvironment> c,
    Store store, StoreFile resultFile, FlushLifeCycleTracker tracker) throws IOException {
  ctPostFlush.incrementAndGet();
  if (throwOnPostFlush.get()){
    throw new IOException("throwOnPostFlush is true in postFlush");
  }
}
 
Example #9
Source File: LocalIndexIT.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
        StoreFile resultFile) throws IOException {
    try {
        latch2.await();
    } catch (InterruptedException e1) {
    }
    super.postCompact(e, store, resultFile);
}
 
Example #10
Source File: HFileGenerator.java    From terrapin with Apache License 2.0 5 votes vote down vote up
/**
 * Generate hfiles for testing purpose
 *
 * @param sourceFileSystem source file system
 * @param conf configuration for hfile
 * @param outputFolder output folder for generated hfiles
 * @param partitionerType partitioner type
 * @param numOfPartitions number of partitions
 * @param numOfKeys number of keys
 * @return list of generated hfiles
 * @throws IOException if hfile creation goes wrong
 */
public static List<Path> generateHFiles(FileSystem sourceFileSystem, Configuration conf,
                                        File outputFolder, PartitionerType partitionerType,
                                        int numOfPartitions, int numOfKeys)
    throws IOException {
  StoreFile.Writer[] writers = new StoreFile.Writer[numOfPartitions];
  for (int i = 0; i < numOfPartitions; i++) {
    writers[i] = new StoreFile.WriterBuilder(conf, new CacheConfig(conf), sourceFileSystem, 4096)
        .withFilePath(new Path(String.format("%s/%s", outputFolder.getAbsoluteFile(),
            TerrapinUtil.formatPartitionName(i))))
        .withCompression(Compression.Algorithm.NONE)
        .build();
  }
  Partitioner partitioner = PartitionerFactory.getPartitioner(partitionerType);
  for (int i = 0; i < numOfKeys; i++) {
    byte[] key = String.format("%06d", i).getBytes();
    byte[] value;
    if (i <= 1) {
      value = "".getBytes();
    } else {
      value = ("v" + (i + 1)).getBytes();
    }
    KeyValue kv = new KeyValue(key, Bytes.toBytes("cf"), Bytes.toBytes(""), value);
    int partition = partitioner.getPartition(new BytesWritable(key), new BytesWritable(value),
        numOfPartitions);
    writers[partition].append(kv);
  }
  for (int i = 0; i < numOfPartitions; i++) {
    writers[i].close();
  }
  return Lists.transform(Lists.newArrayList(writers), new Function<StoreFile.Writer, Path>() {
    @Override
    public Path apply(StoreFile.Writer writer) {
      return writer.getPath();
    }
  });
}
 
Example #11
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile,
                        CompactionRequest request) throws IOException {
  // Persist the compaction state after a successful compaction
  if (compactionState != null) {
    compactionState.persist();
  }
}
 
Example #12
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile,
                        CompactionRequest request) throws IOException {
  // Persist the compaction state after a successful compaction
  if (compactionState != null) {
    compactionState.persist();
  }
}
 
Example #13
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile,
                        CompactionRequest request) throws IOException {
  // Persist the compaction state after a successful compaction
  if (compactionState != null) {
    compactionState.persist();
  }
}
 
Example #14
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile,
                        CompactionRequest request) throws IOException {
  // Persist the compaction state after a successful compaction
  if (compactionState != null) {
    compactionState.persist();
  }
}
 
Example #15
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile,
                        CompactionRequest request) throws IOException {
  // Persist the compaction state after a succesful compaction
  if (compactionState != null) {
    compactionState.persist();
  }
}
 
Example #16
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile,
                        CompactionRequest request) throws IOException {
  // Persist the compaction state after a successful compaction
  if (compactionState != null) {
    compactionState.persist();
  }
}
 
Example #17
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void postCompact(
    org.apache.hadoop.hbase.coprocessor.ObserverContext<RegionCoprocessorEnvironment> c,
    Store store, StoreFile resultFile,
    org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  // Persist the compaction state after a successful compaction
  if (compactionState != null) {
    compactionState.persist();
  }
}
 
Example #18
Source File: InvalidListPruneTest.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void postCompact(
        org.apache.hadoop.hbase.coprocessor.ObserverContext<RegionCoprocessorEnvironment> c,
        Store store, StoreFile resultFile,
        org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker tracker,
        CompactionRequest request) throws IOException {
    super.postCompact(c, store, resultFile, tracker, request);
    lastMajorCompactionTime.set(System.currentTimeMillis());
}
 
Example #19
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile,
                        CompactionRequest request) throws IOException {
  // Persist the compaction state after a successful compaction
  if (compactionState != null) {
    compactionState.persist();
  }
}
 
Example #20
Source File: TestCoprocessorInterface.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e,
    Store store, StoreFile resultFile, CompactionLifeCycleTracker tracker,
    CompactionRequest request) {
  postCompactCalled = true;
}
 
Example #21
Source File: InvalidListPruneTest.java    From phoenix-tephra with Apache License 2.0 4 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile,
                        CompactionRequest request) throws IOException {
  super.postCompact(e, store, resultFile, request);
  lastMajorCompactionTime.set(System.currentTimeMillis());
}
 
Example #22
Source File: InvalidListPruneTest.java    From phoenix-tephra with Apache License 2.0 4 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile,
                        CompactionRequest request) throws IOException {
  super.postCompact(e, store, resultFile, request);
  lastMajorCompactionTime.set(System.currentTimeMillis());
}
 
Example #23
Source File: InvalidListPruneTest.java    From phoenix-tephra with Apache License 2.0 4 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile,
                        CompactionRequest request) throws IOException {
  super.postCompact(e, store, resultFile, request);
  lastMajorCompactionTime.set(System.currentTimeMillis());
}
 
Example #24
Source File: DelegateRegionObserver.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@Override
public void postFlush(org.apache.hadoop.hbase.coprocessor.ObserverContext<RegionCoprocessorEnvironment> c,
        Store store, StoreFile resultFile, org.apache.hadoop.hbase.regionserver.FlushLifeCycleTracker tracker)
        throws IOException {
    delegate.postFlush(c, store, resultFile, tracker);
}
 
Example #25
Source File: InvalidListPruneTest.java    From phoenix-tephra with Apache License 2.0 4 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile,
                        CompactionRequest request) throws IOException {
  super.postCompact(e, store, resultFile, request);
  lastMajorCompactionTime.set(System.currentTimeMillis());
}
 
Example #26
Source File: InvalidListPruneTest.java    From phoenix-tephra with Apache License 2.0 4 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile,
                        CompactionRequest request) throws IOException {
  super.postCompact(e, store, resultFile, request);
  lastMajorCompactionTime.set(System.currentTimeMillis());
}
 
Example #27
Source File: ExampleRegionObserverWithMetrics.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public void postFlush(
    ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile,
    FlushLifeCycleTracker tracker) throws IOException {
  flushCounter.increment();
}
 
Example #28
Source File: TestNamespaceAuditor.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
    StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request)
    throws IOException {
  postCompact.countDown();
}
 
Example #29
Source File: SimpleRegionObserver.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    StoreFile resultFile, CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  ctPostCompact.incrementAndGet();
}
 
Example #30
Source File: SimpleRegionObserver.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends StoreFile> selected, CompactionLifeCycleTracker tracker,
    CompactionRequest request) {
  ctPostCompactSelect.incrementAndGet();
}