org.apache.hadoop.hbase.regionserver.StoreFile Java Examples
The following examples show how to use
org.apache.hadoop.hbase.regionserver.StoreFile.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PreRegionsCreation.java From hbase-secondary-index with GNU General Public License v3.0 | 6 votes |
public void create(String tableName, String[] columnFamily, String startKey, String endKey) throws IOException { HTableDescriptor desc = new HTableDescriptor(Bytes.toBytes(tableName)); for (String cf : columnFamily) { HColumnDescriptor cfDesc = new HColumnDescriptor(cf); // column family attribute cfDesc.setValue(HConstants.VERSIONS, String.valueOf(Integer.MAX_VALUE)); cfDesc.setValue(HColumnDescriptor.BLOOMFILTER, StoreFile.BloomType.ROW.toString()); desc.addFamily(cfDesc); } if (!admin.tableExists(tableName)) { if (null != startKey && null != endKey) admin.createTable(desc, Bytes.toBytes(startKey), Bytes.toBytes(endKey), NUM_REGIONSERVERS); } }
Example #2
Source File: HFileOutputFormat.java From terrapin with Apache License 2.0 | 6 votes |
public RecordWriter<BytesWritable, BytesWritable> getRecordWriter( TaskAttemptContext context) throws IOException { // Get the path of the temporary output file final Path outputPath = FileOutputFormat.getOutputPath(context); final Path outputDir = new FileOutputCommitter(outputPath, context).getWorkPath(); final Configuration conf = context.getConfiguration(); final FileSystem fs = outputDir.getFileSystem(conf); int blockSize = conf.getInt(Constants.HFILE_BLOCKSIZE, 16384); // Default to snappy. Compression.Algorithm compressionAlgorithm = getAlgorithm( conf.get(Constants.HFILE_COMPRESSION)); final StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, new CacheConfig(conf), fs, blockSize) .withFilePath(hfilePath(outputPath, context.getTaskAttemptID().getTaskID().getId())) .withCompression(compressionAlgorithm) .build(); return new HFileRecordWriter(writer); }
Example #3
Source File: HFileReaderTest.java From terrapin with Apache License 2.0 | 5 votes |
@BeforeClass public static void setUp() throws Exception { int randomNum = (int) (Math.random() * Integer.MAX_VALUE); hfilePath = "/tmp/hfile-" + randomNum; Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); keyValueMap = Maps.newHashMapWithExpectedSize(10000); errorKeys = Sets.newHashSetWithExpectedSize(2000); StoreFile.Writer writer = new StoreFile.WriterBuilder(conf, new CacheConfig(conf), fs, 4096). withFilePath(new Path(hfilePath)). withCompression(Compression.Algorithm.NONE). build(); // Add upto 10K values. for (int i = 0; i < 10000; i++) { byte[] key = String.format("%04d", i).getBytes(); byte[] value = null; // Add a couple of empty values for testing and making sure we return them. if (i <= 1) { value = "".getBytes(); } else { value = ("v" + (i + 1)).getBytes(); } KeyValue kv = new KeyValue(key, Bytes.toBytes("cf"), Bytes.toBytes(""), value); writer.append(kv); keyValueMap.put(ByteBuffer.wrap(key), ByteBuffer.wrap(value)); if (i >= 4000 && i < 6000) { errorKeys.add(ByteBuffer.wrap(key)); } } writer.close(); hfileReader = new TestHFileReader(fs, hfilePath, new CacheConfig(conf), new ExecutorServiceFuturePool(Executors.newFixedThreadPool(1)), errorKeys); }
Example #4
Source File: TransactionalRegion.java From hbase-secondary-index with GNU General Public License v3.0 | 5 votes |
@Override public List<StoreFile> close(final boolean abort) throws IOException { prepareToClose(); if (!commitPendingTransactions.isEmpty()) { LOG.warn("Closing transactional region [" + getRegionInfo().getRegionNameAsString() + "], but still have [" + commitPendingTransactions.size() + "] transactions that are pending commit."); // TODO resolve from the Global Trx Log. } return super.close(abort); }
Example #5
Source File: RepairUtil.java From phoenix with Apache License 2.0 | 5 votes |
public static boolean isLocalIndexStoreFilesConsistent(RegionCoprocessorEnvironment environment, Store store) { byte[] startKey = environment.getRegion().getRegionInfo().getStartKey(); byte[] endKey = environment.getRegion().getRegionInfo().getEndKey(); byte[] indexKeyEmbedded = startKey.length == 0 ? new byte[endKey.length] : startKey; for (StoreFile file : store.getStorefiles()) { if (file.getFirstKey().isPresent() && file.getFirstKey().get() != null) { byte[] fileFirstRowKey = CellUtil.cloneRow(file.getFirstKey().get()); if ((fileFirstRowKey != null && Bytes.compareTo(fileFirstRowKey, 0, indexKeyEmbedded.length, indexKeyEmbedded, 0, indexKeyEmbedded.length) != 0)) { return false; } } } return true; }
Example #6
Source File: OnlyDeleteExpiredFilesCompactionPolicy.java From opensoc-streaming with Apache License 2.0 | 5 votes |
@Override final ArrayList<StoreFile> applyCompactionPolicy(final ArrayList<StoreFile> candidates, final boolean mayUseOffPeak, final boolean mayBeStuck) throws IOException { LOG.info("Sending empty list for compaction to avoid compaction and do only deletes of files older than TTL"); return new ArrayList<StoreFile>(); }
Example #7
Source File: ExampleRegionObserverWithMetrics.java From hbase with Apache License 2.0 | 5 votes |
@Override public void postCompactSelection( ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends StoreFile> selected, CompactionLifeCycleTracker tracker, CompactionRequest request) { if (selected != null) { filesCompactedCounter.increment(selected.size()); } }
Example #8
Source File: SimpleRegionObserver.java From hbase with Apache License 2.0 | 5 votes |
@Override public void postFlush(ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile, FlushLifeCycleTracker tracker) throws IOException { ctPostFlush.incrementAndGet(); if (throwOnPostFlush.get()){ throw new IOException("throwOnPostFlush is true in postFlush"); } }
Example #9
Source File: LocalIndexIT.java From phoenix with Apache License 2.0 | 5 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile) throws IOException { try { latch2.await(); } catch (InterruptedException e1) { } super.postCompact(e, store, resultFile); }
Example #10
Source File: HFileGenerator.java From terrapin with Apache License 2.0 | 5 votes |
/** * Generate hfiles for testing purpose * * @param sourceFileSystem source file system * @param conf configuration for hfile * @param outputFolder output folder for generated hfiles * @param partitionerType partitioner type * @param numOfPartitions number of partitions * @param numOfKeys number of keys * @return list of generated hfiles * @throws IOException if hfile creation goes wrong */ public static List<Path> generateHFiles(FileSystem sourceFileSystem, Configuration conf, File outputFolder, PartitionerType partitionerType, int numOfPartitions, int numOfKeys) throws IOException { StoreFile.Writer[] writers = new StoreFile.Writer[numOfPartitions]; for (int i = 0; i < numOfPartitions; i++) { writers[i] = new StoreFile.WriterBuilder(conf, new CacheConfig(conf), sourceFileSystem, 4096) .withFilePath(new Path(String.format("%s/%s", outputFolder.getAbsoluteFile(), TerrapinUtil.formatPartitionName(i)))) .withCompression(Compression.Algorithm.NONE) .build(); } Partitioner partitioner = PartitionerFactory.getPartitioner(partitionerType); for (int i = 0; i < numOfKeys; i++) { byte[] key = String.format("%06d", i).getBytes(); byte[] value; if (i <= 1) { value = "".getBytes(); } else { value = ("v" + (i + 1)).getBytes(); } KeyValue kv = new KeyValue(key, Bytes.toBytes("cf"), Bytes.toBytes(""), value); int partition = partitioner.getPartition(new BytesWritable(key), new BytesWritable(value), numOfPartitions); writers[partition].append(kv); } for (int i = 0; i < numOfPartitions; i++) { writers[i].close(); } return Lists.transform(Lists.newArrayList(writers), new Function<StoreFile.Writer, Path>() { @Override public Path apply(StoreFile.Writer writer) { return writer.getPath(); } }); }
Example #11
Source File: TransactionProcessor.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile, CompactionRequest request) throws IOException { // Persist the compaction state after a successful compaction if (compactionState != null) { compactionState.persist(); } }
Example #12
Source File: TransactionProcessor.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile, CompactionRequest request) throws IOException { // Persist the compaction state after a successful compaction if (compactionState != null) { compactionState.persist(); } }
Example #13
Source File: TransactionProcessor.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile, CompactionRequest request) throws IOException { // Persist the compaction state after a successful compaction if (compactionState != null) { compactionState.persist(); } }
Example #14
Source File: TransactionProcessor.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile, CompactionRequest request) throws IOException { // Persist the compaction state after a successful compaction if (compactionState != null) { compactionState.persist(); } }
Example #15
Source File: TransactionProcessor.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile, CompactionRequest request) throws IOException { // Persist the compaction state after a succesful compaction if (compactionState != null) { compactionState.persist(); } }
Example #16
Source File: TransactionProcessor.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile, CompactionRequest request) throws IOException { // Persist the compaction state after a successful compaction if (compactionState != null) { compactionState.persist(); } }
Example #17
Source File: TransactionProcessor.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public void postCompact( org.apache.hadoop.hbase.coprocessor.ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile, org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException { // Persist the compaction state after a successful compaction if (compactionState != null) { compactionState.persist(); } }
Example #18
Source File: InvalidListPruneTest.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public void postCompact( org.apache.hadoop.hbase.coprocessor.ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile, org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException { super.postCompact(c, store, resultFile, tracker, request); lastMajorCompactionTime.set(System.currentTimeMillis()); }
Example #19
Source File: TransactionProcessor.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile, CompactionRequest request) throws IOException { // Persist the compaction state after a successful compaction if (compactionState != null) { compactionState.persist(); } }
Example #20
Source File: TestCoprocessorInterface.java From hbase with Apache License 2.0 | 4 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request) { postCompactCalled = true; }
Example #21
Source File: InvalidListPruneTest.java From phoenix-tephra with Apache License 2.0 | 4 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile, CompactionRequest request) throws IOException { super.postCompact(e, store, resultFile, request); lastMajorCompactionTime.set(System.currentTimeMillis()); }
Example #22
Source File: InvalidListPruneTest.java From phoenix-tephra with Apache License 2.0 | 4 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile, CompactionRequest request) throws IOException { super.postCompact(e, store, resultFile, request); lastMajorCompactionTime.set(System.currentTimeMillis()); }
Example #23
Source File: InvalidListPruneTest.java From phoenix-tephra with Apache License 2.0 | 4 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile, CompactionRequest request) throws IOException { super.postCompact(e, store, resultFile, request); lastMajorCompactionTime.set(System.currentTimeMillis()); }
Example #24
Source File: DelegateRegionObserver.java From phoenix with Apache License 2.0 | 4 votes |
@Override public void postFlush(org.apache.hadoop.hbase.coprocessor.ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile, org.apache.hadoop.hbase.regionserver.FlushLifeCycleTracker tracker) throws IOException { delegate.postFlush(c, store, resultFile, tracker); }
Example #25
Source File: InvalidListPruneTest.java From phoenix-tephra with Apache License 2.0 | 4 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile, CompactionRequest request) throws IOException { super.postCompact(e, store, resultFile, request); lastMajorCompactionTime.set(System.currentTimeMillis()); }
Example #26
Source File: InvalidListPruneTest.java From phoenix-tephra with Apache License 2.0 | 4 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile, CompactionRequest request) throws IOException { super.postCompact(e, store, resultFile, request); lastMajorCompactionTime.set(System.currentTimeMillis()); }
Example #27
Source File: ExampleRegionObserverWithMetrics.java From hbase with Apache License 2.0 | 4 votes |
@Override public void postFlush( ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile, FlushLifeCycleTracker tracker) throws IOException { flushCounter.increment(); }
Example #28
Source File: TestNamespaceAuditor.java From hbase with Apache License 2.0 | 4 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store, StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException { postCompact.countDown(); }
Example #29
Source File: SimpleRegionObserver.java From hbase with Apache License 2.0 | 4 votes |
@Override public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException { ctPostCompact.incrementAndGet(); }
Example #30
Source File: SimpleRegionObserver.java From hbase with Apache License 2.0 | 4 votes |
@Override public void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends StoreFile> selected, CompactionLifeCycleTracker tracker, CompactionRequest request) { ctPostCompactSelect.incrementAndGet(); }