Java Code Examples for org.apache.hadoop.hbase.regionserver.HRegion#closeRegionOperation()
The following examples show how to use
org.apache.hadoop.hbase.regionserver.HRegion#closeRegionOperation() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AggregateRegionObserver.java From Kylin with Apache License 2.0 | 5 votes |
private RegionScanner doPostScannerObserver(final ObserverContext<RegionCoprocessorEnvironment> ctxt, final Scan scan, final RegionScanner innerScanner) throws IOException { byte[] coprocessorEnableBytes = scan.getAttribute(COPROCESSOR_ENABLE); if (coprocessorEnableBytes == null || coprocessorEnableBytes.length == 0 || coprocessorEnableBytes[0] == 0) { return innerScanner; } byte[] typeBytes = scan.getAttribute(TYPE); CoprocessorRowType type = CoprocessorRowType.deserialize(typeBytes); byte[] projectorBytes = scan.getAttribute(PROJECTOR); CoprocessorProjector projector = CoprocessorProjector.deserialize(projectorBytes); byte[] aggregatorBytes = scan.getAttribute(AGGREGATORS); ObserverAggregators aggregators = ObserverAggregators.deserialize(aggregatorBytes); byte[] filterBytes = scan.getAttribute(FILTER); CoprocessorFilter filter = CoprocessorFilter.deserialize(filterBytes); // start/end region operation & sync on scanner is suggested by the // javadoc of RegionScanner.nextRaw() // FIXME: will the lock still work when a iterator is returned? is it safe? Is readonly attribute helping here? by mhb HRegion region = ctxt.getEnvironment().getRegion(); region.startRegionOperation(); try { synchronized (innerScanner) { return new AggregationScanner(type, filter, projector, aggregators, innerScanner); } } finally { region.closeRegionOperation(); } }
Example 2
Source File: PhoenixIndexBuilder.java From phoenix with BSD 3-Clause "New" or "Revised" License | 5 votes |
@Override public void batchStarted(MiniBatchOperationInProgress<Pair<Mutation, Integer>> miniBatchOp) throws IOException { // The entire purpose of this method impl is to get the existing rows for the // table rows being indexed into the block cache, as the index maintenance code // does a point scan per row List<KeyRange> keys = Lists.newArrayListWithExpectedSize(miniBatchOp.size()); List<IndexMaintainer> maintainers = new ArrayList<IndexMaintainer>(); for (int i = 0; i < miniBatchOp.size(); i++) { Mutation m = miniBatchOp.getOperation(i).getFirst(); keys.add(PDataType.VARBINARY.getKeyRange(m.getRow())); maintainers.addAll(getCodec().getIndexMaintainers(m.getAttributesMap())); } Scan scan = IndexManagementUtil.newLocalStateScan(maintainers); ScanRanges scanRanges = ScanRanges.create(Collections.singletonList(keys), SchemaUtil.VAR_BINARY_SCHEMA); scanRanges.setScanStartStopRow(scan); scan.setFilter(scanRanges.getSkipScanFilter()); HRegion region = this.env.getRegion(); RegionScanner scanner = region.getScanner(scan); // Run through the scanner using internal nextRaw method MultiVersionConsistencyControl.setThreadReadPoint(scanner.getMvccReadPoint()); region.startRegionOperation(); try { boolean hasMore; do { List<KeyValue> results = Lists.newArrayList(); // Results are potentially returned even when the return value of s.next is false // since this is an indication of whether or not there are more values after the // ones returned hasMore = scanner.nextRaw(results, null); } while (hasMore); } finally { try { scanner.close(); } finally { region.closeRegionOperation(); } } }
Example 3
Source File: PhoenixIndexBuilder.java From phoenix with Apache License 2.0 | 4 votes |
@Override public void batchStarted(MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException { // The entire purpose of this method impl is to get the existing rows for the // table rows being indexed into the block cache, as the index maintenance code // does a point scan per row List<KeyRange> keys = Lists.newArrayListWithExpectedSize(miniBatchOp.size()); Map<ImmutableBytesWritable, IndexMaintainer> maintainers = new HashMap<ImmutableBytesWritable, IndexMaintainer>(); ImmutableBytesWritable indexTableName = new ImmutableBytesWritable(); for (int i = 0; i < miniBatchOp.size(); i++) { Mutation m = miniBatchOp.getOperation(i); keys.add(PVarbinary.INSTANCE.getKeyRange(m.getRow())); List<IndexMaintainer> indexMaintainers = getCodec().getIndexMaintainers(m.getAttributesMap()); for(IndexMaintainer indexMaintainer: indexMaintainers) { if (indexMaintainer.isImmutableRows() && indexMaintainer.isLocalIndex()) continue; indexTableName.set(indexMaintainer.getIndexTableName()); if (maintainers.get(indexTableName) != null) continue; maintainers.put(indexTableName, indexMaintainer); } } if (maintainers.isEmpty()) return; Scan scan = IndexManagementUtil.newLocalStateScan(new ArrayList<IndexMaintainer>(maintainers.values())); ScanRanges scanRanges = ScanRanges.create(SchemaUtil.VAR_BINARY_SCHEMA, Collections.singletonList(keys), ScanUtil.SINGLE_COLUMN_SLOT_SPAN); scanRanges.initializeScan(scan); scan.setFilter(scanRanges.getSkipScanFilter()); HRegion region = this.env.getRegion(); RegionScanner scanner = region.getScanner(scan); // Run through the scanner using internal nextRaw method region.startRegionOperation(); try { boolean hasMore; do { List<Cell> results = Lists.newArrayList(); // Results are potentially returned even when the return value of s.next is false // since this is an indication of whether or not there are more values after the // ones returned hasMore = scanner.nextRaw(results); } while (hasMore); } finally { try { scanner.close(); } finally { region.closeRegionOperation(); } } }
Example 4
Source File: GroupedAggregateRegionObserver.java From phoenix with Apache License 2.0 | 4 votes |
/** * Used for an aggregate query in which the key order does not necessarily match the group by * key order. In this case, we must collect all distinct groups within a region into a map, * aggregating as we go. * @param limit TODO */ private RegionScanner scanUnordered(ObserverContext<RegionCoprocessorEnvironment> c, Scan scan, final RegionScanner s, final List<Expression> expressions, final ServerAggregators aggregators, long limit) throws IOException { if (logger.isDebugEnabled()) { logger.debug(LogUtil.addCustomAnnotations("Grouped aggregation over unordered rows with scan " + scan + ", group by " + expressions + ", aggregators " + aggregators, ScanUtil.getCustomAnnotations(scan))); } RegionCoprocessorEnvironment env = c.getEnvironment(); Configuration conf = env.getConfiguration(); int estDistVals = conf.getInt(GROUPBY_ESTIMATED_DISTINCT_VALUES_ATTRIB, DEFAULT_GROUPBY_ESTIMATED_DISTINCT_VALUES); byte[] estDistValsBytes = scan.getAttribute(BaseScannerRegionObserver.ESTIMATED_DISTINCT_VALUES); if (estDistValsBytes != null) { // Allocate 1.5x estimation estDistVals = Math.max(MIN_DISTINCT_VALUES, (int) (Bytes.toInt(estDistValsBytes) * 1.5f)); } final boolean spillableEnabled = conf.getBoolean(GROUPBY_SPILLABLE_ATTRIB, DEFAULT_GROUPBY_SPILLABLE); GroupByCache groupByCache = GroupByCacheFactory.INSTANCE.newCache( env, ScanUtil.getTenantId(scan), ScanUtil.getCustomAnnotations(scan), aggregators, estDistVals); boolean success = false; try { boolean hasMore; MultiKeyValueTuple result = new MultiKeyValueTuple(); if (logger.isDebugEnabled()) { logger.debug(LogUtil.addCustomAnnotations("Spillable groupby enabled: " + spillableEnabled, ScanUtil.getCustomAnnotations(scan))); } HRegion region = c.getEnvironment().getRegion(); region.startRegionOperation(); try { do { List<Cell> results = new ArrayList<Cell>(); // Results are potentially returned even when the return // value of s.next is false // since this is an indication of whether or not there are // more values after the // ones returned hasMore = s.nextRaw(results); if (!results.isEmpty()) { result.setKeyValues(results); ImmutableBytesWritable key = TupleUtil.getConcatenatedValue(result, expressions); Aggregator[] rowAggregators = groupByCache.cache(key); // Aggregate values here aggregators.aggregate(rowAggregators, result); } } while (hasMore && groupByCache.size() < limit); } finally { region.closeRegionOperation(); } RegionScanner regionScanner = groupByCache.getScanner(s); // Do not sort here, but sort back on the client instead // The reason is that if the scan ever extends beyond a region // (which can happen if we're basing our parallelization split // points on old metadata), we'll get incorrect query results. success = true; return regionScanner; } finally { if (!success) { Closeables.closeQuietly(groupByCache); } } }
Example 5
Source File: IIEndpoint.java From Kylin with Apache License 2.0 | 4 votes |
@Override public void getRows(RpcController controller, IIProtos.IIRequest request, RpcCallback<IIProtos.IIResponse> done) { CoprocessorRowType type; CoprocessorProjector projector; EndpointAggregators aggregators; CoprocessorFilter filter; type = CoprocessorRowType.deserialize(request.getType().toByteArray()); projector = CoprocessorProjector.deserialize(request.getProjector().toByteArray()); aggregators = EndpointAggregators.deserialize(request.getAggregator().toByteArray()); filter = CoprocessorFilter.deserialize(request.getFilter().toByteArray()); TableRecordInfoDigest tableRecordInfoDigest = aggregators.getTableRecordInfoDigest(); IIProtos.IIResponse response = null; RegionScanner innerScanner = null; HRegion region = null; try { region = env.getRegion(); innerScanner = region.getScanner(buildScan()); region.startRegionOperation(); synchronized (innerScanner) { IIKeyValueCodec codec = new IIKeyValueCodec(tableRecordInfoDigest); //TODO pass projector to codec to skip loading columns Iterable<Slice> slices = codec.decodeKeyValue(new HbaseServerKVIterator(innerScanner)); if (aggregators.isEmpty()) { response = getNonAggregatedResponse(slices, filter, type); } else { response = getAggregatedResponse(slices, filter, type, projector, aggregators); } } } catch (IOException ioe) { System.out.println(ioe.toString()); ResponseConverter.setControllerException(controller, ioe); } finally { IOUtils.closeQuietly(innerScanner); if (region != null) { try { region.closeRegionOperation(); } catch (IOException e) { e.printStackTrace(); throw new RuntimeException(e); } } } done.run(response); }
Example 6
Source File: GroupedAggregateRegionObserver.java From phoenix with BSD 3-Clause "New" or "Revised" License | 4 votes |
/** * Used for an aggregate query in which the key order does not necessarily match the group by * key order. In this case, we must collect all distinct groups within a region into a map, * aggregating as we go. */ private RegionScanner scanUnordered(ObserverContext<RegionCoprocessorEnvironment> c, Scan scan, final RegionScanner s, final List<Expression> expressions, final ServerAggregators aggregators) throws IOException { if (logger.isDebugEnabled()) { logger.debug("Grouped aggregation over unordered rows with scan " + scan + ", group by " + expressions + ", aggregators " + aggregators); } int estDistVals = DEFAULT_ESTIMATED_DISTINCT_VALUES; byte[] estDistValsBytes = scan.getAttribute(ESTIMATED_DISTINCT_VALUES); if (estDistValsBytes != null) { // Allocate 1.5x estimation estDistVals = Math.min(MIN_DISTINCT_VALUES, (int) (Bytes.toInt(estDistValsBytes) * 1.5f)); } RegionCoprocessorEnvironment env = c.getEnvironment(); Configuration conf = env.getConfiguration(); final boolean spillableEnabled = conf.getBoolean(GROUPBY_SPILLABLE_ATTRIB, DEFAULT_GROUPBY_SPILLABLE); GroupByCache groupByCache = GroupByCacheFactory.INSTANCE.newCache( env, ScanUtil.getTenantId(scan), aggregators, estDistVals); boolean success = false; try { boolean hasMore; MultiKeyValueTuple result = new MultiKeyValueTuple(); if (logger.isDebugEnabled()) { logger.debug("Spillable groupby enabled: " + spillableEnabled); } HRegion region = c.getEnvironment().getRegion(); MultiVersionConsistencyControl.setThreadReadPoint(s.getMvccReadPoint()); region.startRegionOperation(); try { do { List<KeyValue> results = new ArrayList<KeyValue>(); // Results are potentially returned even when the return // value of s.next is false // since this is an indication of whether or not there are // more values after the // ones returned hasMore = s.nextRaw(results, null); if (!results.isEmpty()) { result.setKeyValues(results); ImmutableBytesWritable key = TupleUtil.getConcatenatedValue(result, expressions); Aggregator[] rowAggregators = groupByCache.cache(key); // Aggregate values here aggregators.aggregate(rowAggregators, result); } } while (hasMore); } finally { region.closeRegionOperation(); } RegionScanner regionScanner = groupByCache.getScanner(s); // Do not sort here, but sort back on the client instead // The reason is that if the scan ever extends beyond a region // (which can happen if we're basing our parallelization split // points on old metadata), we'll get incorrect query results. success = true; return regionScanner; } finally { if (!success) { Closeables.closeQuietly(groupByCache); } } }
Example 7
Source File: GroupedAggregateRegionObserver.java From phoenix with BSD 3-Clause "New" or "Revised" License | 4 votes |
/** * Used for an aggregate query in which the key order match the group by key order. In this * case, we can do the aggregation as we scan, by detecting when the group by key changes. */ private RegionScanner scanOrdered(final ObserverContext<RegionCoprocessorEnvironment> c, Scan scan, final RegionScanner s, final List<Expression> expressions, final ServerAggregators aggregators) { if (logger.isDebugEnabled()) { logger.debug("Grouped aggregation over ordered rows with scan " + scan + ", group by " + expressions + ", aggregators " + aggregators); } return new BaseRegionScanner() { private ImmutableBytesWritable currentKey = null; @Override public HRegionInfo getRegionInfo() { return s.getRegionInfo(); } @Override public void close() throws IOException { s.close(); } @Override public boolean next(List<KeyValue> results) throws IOException { boolean hasMore; boolean aggBoundary = false; MultiKeyValueTuple result = new MultiKeyValueTuple(); ImmutableBytesWritable key = null; Aggregator[] rowAggregators = aggregators.getAggregators(); HRegion region = c.getEnvironment().getRegion(); MultiVersionConsistencyControl.setThreadReadPoint(s.getMvccReadPoint()); region.startRegionOperation(); try { do { List<KeyValue> kvs = new ArrayList<KeyValue>(); // Results are potentially returned even when the return // value of s.next is false // since this is an indication of whether or not there // are more values after the // ones returned hasMore = s.nextRaw(kvs, null); if (!kvs.isEmpty()) { result.setKeyValues(kvs); key = TupleUtil.getConcatenatedValue(result, expressions); aggBoundary = currentKey != null && currentKey.compareTo(key) != 0; if (!aggBoundary) { aggregators.aggregate(rowAggregators, result); if (logger.isDebugEnabled()) { logger.debug("Row passed filters: " + kvs + ", aggregated values: " + Arrays.asList(rowAggregators)); } currentKey = key; } } } while (hasMore && !aggBoundary); } finally { region.closeRegionOperation(); } if (currentKey != null) { byte[] value = aggregators.toBytes(rowAggregators); KeyValue keyValue = KeyValueUtil.newKeyValue(currentKey.get(), currentKey.getOffset(), currentKey.getLength(), SINGLE_COLUMN_FAMILY, SINGLE_COLUMN, AGG_TIMESTAMP, value, 0, value.length); results.add(keyValue); if (logger.isDebugEnabled()) { logger.debug("Adding new aggregate row: " + keyValue + ",for current key " + Bytes.toStringBinary(currentKey.get(), currentKey.getOffset(), currentKey.getLength()) + ", aggregated values: " + Arrays.asList(rowAggregators)); } // If we're at an aggregation boundary, reset the // aggregators and // aggregate with the current result (which is not a part of // the returned result). if (aggBoundary) { aggregators.reset(rowAggregators); aggregators.aggregate(rowAggregators, result); currentKey = key; } } // Continue if there are more if (hasMore || aggBoundary) { return true; } currentKey = null; return false; } }; }