Java Code Examples for org.apache.hadoop.hbase.client.Scan#getFamilies()
The following examples show how to use
org.apache.hadoop.hbase.client.Scan#getFamilies() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AsyncAggregationClient.java From hbase with Apache License 2.0 | 4 votes |
private static <R, S, P extends Message, Q extends Message, T extends Message> void findMedian( CompletableFuture<R> future, AsyncTable<AdvancedScanResultConsumer> table, ColumnInterpreter<R, S, P, Q, T> ci, Scan scan, NavigableMap<byte[], S> sumByRegion) { double halfSum = ci.divideForAvg(sumByRegion.values().stream().reduce(ci::add).get(), 2L); S movingSum = null; byte[] startRow = null; for (Map.Entry<byte[], S> entry : sumByRegion.entrySet()) { startRow = entry.getKey(); S newMovingSum = ci.add(movingSum, entry.getValue()); if (ci.divideForAvg(newMovingSum, 1L) > halfSum) { break; } movingSum = newMovingSum; } if (startRow != null) { scan.withStartRow(startRow); } // we can not pass movingSum directly to an anonymous class as it is not final. S baseSum = movingSum; byte[] family = scan.getFamilies()[0]; NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(family); byte[] weightQualifier = qualifiers.last(); byte[] valueQualifier = qualifiers.first(); table.scan(scan, new AdvancedScanResultConsumer() { private S sum = baseSum; private R value = null; @Override public void onNext(Result[] results, ScanController controller) { try { for (Result result : results) { Cell weightCell = result.getColumnLatestCell(family, weightQualifier); R weight = ci.getValue(family, weightQualifier, weightCell); sum = ci.add(sum, ci.castToReturnType(weight)); if (ci.divideForAvg(sum, 1L) > halfSum) { if (value != null) { future.complete(value); } else { future.completeExceptionally(new NoSuchElementException()); } controller.terminate(); return; } Cell valueCell = result.getColumnLatestCell(family, valueQualifier); value = ci.getValue(family, valueQualifier, valueCell); } } catch (IOException e) { future.completeExceptionally(e); controller.terminate(); } } @Override public void onError(Throwable error) { future.completeExceptionally(error); } @Override public void onComplete() { if (!future.isDone()) { // we should not reach here as the future should be completed in onNext. future.completeExceptionally(new NoSuchElementException()); } } }); }
Example 2
Source File: AggregateImplementation.java From hbase with Apache License 2.0 | 4 votes |
/** * Gives the maximum for a given combination of column qualifier and column * family, in the given row range as defined in the Scan object. In its * current implementation, it takes one column family and one column qualifier * (if provided). In case of null column qualifier, maximum value for the * entire column family will be returned. */ @Override public void getMax(RpcController controller, AggregateRequest request, RpcCallback<AggregateResponse> done) { InternalScanner scanner = null; AggregateResponse response = null; T max = null; try { ColumnInterpreter<T, S, P, Q, R> ci = constructColumnInterpreterFromRequest(request); T temp; Scan scan = ProtobufUtil.toScan(request.getScan()); scanner = env.getRegion().getScanner(scan); List<Cell> results = new ArrayList<>(); byte[] colFamily = scan.getFamilies()[0]; NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily); byte[] qualifier = null; if (qualifiers != null && !qualifiers.isEmpty()) { qualifier = qualifiers.pollFirst(); } // qualifier can be null. boolean hasMoreRows = false; do { hasMoreRows = scanner.next(results); int listSize = results.size(); for (int i = 0; i < listSize; i++) { temp = ci.getValue(colFamily, qualifier, results.get(i)); max = (max == null || (temp != null && ci.compare(temp, max) > 0)) ? temp : max; } results.clear(); } while (hasMoreRows); if (max != null) { AggregateResponse.Builder builder = AggregateResponse.newBuilder(); builder.addFirstPart(ci.getProtoForCellType(max).toByteString()); response = builder.build(); } } catch (IOException e) { CoprocessorRpcUtils.setControllerException(controller, e); } finally { if (scanner != null) { try { scanner.close(); } catch (IOException ignored) {} } } log.info("Maximum from this region is " + env.getRegion().getRegionInfo().getRegionNameAsString() + ": " + max); done.run(response); }
Example 3
Source File: AggregateImplementation.java From hbase with Apache License 2.0 | 4 votes |
/** * Gives the minimum for a given combination of column qualifier and column * family, in the given row range as defined in the Scan object. In its * current implementation, it takes one column family and one column qualifier * (if provided). In case of null column qualifier, minimum value for the * entire column family will be returned. */ @Override public void getMin(RpcController controller, AggregateRequest request, RpcCallback<AggregateResponse> done) { AggregateResponse response = null; InternalScanner scanner = null; T min = null; try { ColumnInterpreter<T, S, P, Q, R> ci = constructColumnInterpreterFromRequest(request); T temp; Scan scan = ProtobufUtil.toScan(request.getScan()); scanner = env.getRegion().getScanner(scan); List<Cell> results = new ArrayList<>(); byte[] colFamily = scan.getFamilies()[0]; NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily); byte[] qualifier = null; if (qualifiers != null && !qualifiers.isEmpty()) { qualifier = qualifiers.pollFirst(); } boolean hasMoreRows = false; do { hasMoreRows = scanner.next(results); int listSize = results.size(); for (int i = 0; i < listSize; i++) { temp = ci.getValue(colFamily, qualifier, results.get(i)); min = (min == null || (temp != null && ci.compare(temp, min) < 0)) ? temp : min; } results.clear(); } while (hasMoreRows); if (min != null) { response = AggregateResponse.newBuilder().addFirstPart( ci.getProtoForCellType(min).toByteString()).build(); } } catch (IOException e) { CoprocessorRpcUtils.setControllerException(controller, e); } finally { if (scanner != null) { try { scanner.close(); } catch (IOException ignored) {} } } log.info("Minimum from this region is " + env.getRegion().getRegionInfo().getRegionNameAsString() + ": " + min); done.run(response); }
Example 4
Source File: AggregateImplementation.java From hbase with Apache License 2.0 | 4 votes |
/** * Gives the sum for a given combination of column qualifier and column * family, in the given row range as defined in the Scan object. In its * current implementation, it takes one column family and one column qualifier * (if provided). In case of null column qualifier, sum for the entire column * family will be returned. */ @Override public void getSum(RpcController controller, AggregateRequest request, RpcCallback<AggregateResponse> done) { AggregateResponse response = null; InternalScanner scanner = null; long sum = 0L; try { ColumnInterpreter<T, S, P, Q, R> ci = constructColumnInterpreterFromRequest(request); S sumVal = null; T temp; Scan scan = ProtobufUtil.toScan(request.getScan()); scanner = env.getRegion().getScanner(scan); byte[] colFamily = scan.getFamilies()[0]; NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily); byte[] qualifier = null; if (qualifiers != null && !qualifiers.isEmpty()) { qualifier = qualifiers.pollFirst(); } List<Cell> results = new ArrayList<>(); boolean hasMoreRows = false; do { hasMoreRows = scanner.next(results); int listSize = results.size(); for (int i = 0; i < listSize; i++) { temp = ci.getValue(colFamily, qualifier, results.get(i)); if (temp != null) { sumVal = ci.add(sumVal, ci.castToReturnType(temp)); } } results.clear(); } while (hasMoreRows); if (sumVal != null) { response = AggregateResponse.newBuilder().addFirstPart( ci.getProtoForPromotedType(sumVal).toByteString()).build(); } } catch (IOException e) { CoprocessorRpcUtils.setControllerException(controller, e); } finally { if (scanner != null) { try { scanner.close(); } catch (IOException ignored) {} } } log.debug("Sum from this region is " + env.getRegion().getRegionInfo().getRegionNameAsString() + ": " + sum); done.run(response); }
Example 5
Source File: AggregateImplementation.java From hbase with Apache License 2.0 | 4 votes |
/** * Gives the row count for the given column family and column qualifier, in * the given row range as defined in the Scan object. */ @Override public void getRowNum(RpcController controller, AggregateRequest request, RpcCallback<AggregateResponse> done) { AggregateResponse response = null; long counter = 0L; List<Cell> results = new ArrayList<>(); InternalScanner scanner = null; try { Scan scan = ProtobufUtil.toScan(request.getScan()); byte[][] colFamilies = scan.getFamilies(); byte[] colFamily = colFamilies != null ? colFamilies[0] : null; NavigableSet<byte[]> qualifiers = colFamilies != null ? scan.getFamilyMap().get(colFamily) : null; byte[] qualifier = null; if (qualifiers != null && !qualifiers.isEmpty()) { qualifier = qualifiers.pollFirst(); } if (scan.getFilter() == null && qualifier == null) { scan.setFilter(new FirstKeyOnlyFilter()); } scanner = env.getRegion().getScanner(scan); boolean hasMoreRows = false; do { hasMoreRows = scanner.next(results); if (results.size() > 0) { counter++; } results.clear(); } while (hasMoreRows); ByteBuffer bb = ByteBuffer.allocate(8).putLong(counter); bb.rewind(); response = AggregateResponse.newBuilder().addFirstPart( ByteString.copyFrom(bb)).build(); } catch (IOException e) { CoprocessorRpcUtils.setControllerException(controller, e); } finally { if (scanner != null) { try { scanner.close(); } catch (IOException ignored) {} } } log.info("Row counter from this region is " + env.getRegion().getRegionInfo().getRegionNameAsString() + ": " + counter); done.run(response); }
Example 6
Source File: AggregateImplementation.java From hbase with Apache License 2.0 | 4 votes |
/** * Gives a Pair with first object as Sum and second object as row count, * computed for a given combination of column qualifier and column family in * the given row range as defined in the Scan object. In its current * implementation, it takes one column family and one column qualifier (if * provided). In case of null column qualifier, an aggregate sum over all the * entire column family will be returned. * <p> * The average is computed in * AggregationClient#avg(byte[], ColumnInterpreter, Scan) by * processing results from all regions, so its "ok" to pass sum and a Long * type. */ @Override public void getAvg(RpcController controller, AggregateRequest request, RpcCallback<AggregateResponse> done) { AggregateResponse response = null; InternalScanner scanner = null; try { ColumnInterpreter<T, S, P, Q, R> ci = constructColumnInterpreterFromRequest(request); S sumVal = null; Long rowCountVal = 0L; Scan scan = ProtobufUtil.toScan(request.getScan()); scanner = env.getRegion().getScanner(scan); byte[] colFamily = scan.getFamilies()[0]; NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily); byte[] qualifier = null; if (qualifiers != null && !qualifiers.isEmpty()) { qualifier = qualifiers.pollFirst(); } List<Cell> results = new ArrayList<>(); boolean hasMoreRows = false; do { results.clear(); hasMoreRows = scanner.next(results); int listSize = results.size(); for (int i = 0; i < listSize; i++) { sumVal = ci.add(sumVal, ci.castToReturnType(ci.getValue(colFamily, qualifier, results.get(i)))); } rowCountVal++; } while (hasMoreRows); if (sumVal != null) { ByteString first = ci.getProtoForPromotedType(sumVal).toByteString(); AggregateResponse.Builder pair = AggregateResponse.newBuilder(); pair.addFirstPart(first); ByteBuffer bb = ByteBuffer.allocate(8).putLong(rowCountVal); bb.rewind(); pair.setSecondPart(ByteString.copyFrom(bb)); response = pair.build(); } } catch (IOException e) { CoprocessorRpcUtils.setControllerException(controller, e); } finally { if (scanner != null) { try { scanner.close(); } catch (IOException ignored) {} } } done.run(response); }
Example 7
Source File: AggregateImplementation.java From hbase with Apache License 2.0 | 4 votes |
/** * Gives a Pair with first object a List containing Sum and sum of squares, * and the second object as row count. It is computed for a given combination of * column qualifier and column family in the given row range as defined in the * Scan object. In its current implementation, it takes one column family and * one column qualifier (if provided). The idea is get the value of variance first: * the average of the squares less the square of the average a standard * deviation is square root of variance. */ @Override public void getStd(RpcController controller, AggregateRequest request, RpcCallback<AggregateResponse> done) { InternalScanner scanner = null; AggregateResponse response = null; try { ColumnInterpreter<T, S, P, Q, R> ci = constructColumnInterpreterFromRequest(request); S sumVal = null, sumSqVal = null, tempVal = null; long rowCountVal = 0L; Scan scan = ProtobufUtil.toScan(request.getScan()); scanner = env.getRegion().getScanner(scan); byte[] colFamily = scan.getFamilies()[0]; NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily); byte[] qualifier = null; if (qualifiers != null && !qualifiers.isEmpty()) { qualifier = qualifiers.pollFirst(); } List<Cell> results = new ArrayList<>(); boolean hasMoreRows = false; do { tempVal = null; hasMoreRows = scanner.next(results); int listSize = results.size(); for (int i = 0; i < listSize; i++) { tempVal = ci.add(tempVal, ci.castToReturnType(ci.getValue(colFamily, qualifier, results.get(i)))); } results.clear(); sumVal = ci.add(sumVal, tempVal); sumSqVal = ci.add(sumSqVal, ci.multiply(tempVal, tempVal)); rowCountVal++; } while (hasMoreRows); if (sumVal != null) { ByteString first_sumVal = ci.getProtoForPromotedType(sumVal).toByteString(); ByteString first_sumSqVal = ci.getProtoForPromotedType(sumSqVal).toByteString(); AggregateResponse.Builder pair = AggregateResponse.newBuilder(); pair.addFirstPart(first_sumVal); pair.addFirstPart(first_sumSqVal); ByteBuffer bb = ByteBuffer.allocate(8).putLong(rowCountVal); bb.rewind(); pair.setSecondPart(ByteString.copyFrom(bb)); response = pair.build(); } } catch (IOException e) { CoprocessorRpcUtils.setControllerException(controller, e); } finally { if (scanner != null) { try { scanner.close(); } catch (IOException ignored) {} } } done.run(response); }
Example 8
Source File: AggregateImplementation.java From hbase with Apache License 2.0 | 4 votes |
/** * Gives a List containing sum of values and sum of weights. * It is computed for the combination of column * family and column qualifier(s) in the given row range as defined in the * Scan object. In its current implementation, it takes one column family and * two column qualifiers. The first qualifier is for values column and * the second qualifier (optional) is for weight column. */ @Override public void getMedian(RpcController controller, AggregateRequest request, RpcCallback<AggregateResponse> done) { AggregateResponse response = null; InternalScanner scanner = null; try { ColumnInterpreter<T, S, P, Q, R> ci = constructColumnInterpreterFromRequest(request); S sumVal = null, sumWeights = null, tempVal = null, tempWeight = null; Scan scan = ProtobufUtil.toScan(request.getScan()); scanner = env.getRegion().getScanner(scan); byte[] colFamily = scan.getFamilies()[0]; NavigableSet<byte[]> qualifiers = scan.getFamilyMap().get(colFamily); byte[] valQualifier = null, weightQualifier = null; if (qualifiers != null && !qualifiers.isEmpty()) { valQualifier = qualifiers.pollFirst(); // if weighted median is requested, get qualifier for the weight column weightQualifier = qualifiers.pollLast(); } List<Cell> results = new ArrayList<>(); boolean hasMoreRows = false; do { tempVal = null; tempWeight = null; hasMoreRows = scanner.next(results); int listSize = results.size(); for (int i = 0; i < listSize; i++) { Cell kv = results.get(i); tempVal = ci.add(tempVal, ci.castToReturnType(ci.getValue(colFamily, valQualifier, kv))); if (weightQualifier != null) { tempWeight = ci.add(tempWeight, ci.castToReturnType(ci.getValue(colFamily, weightQualifier, kv))); } } results.clear(); sumVal = ci.add(sumVal, tempVal); sumWeights = ci.add(sumWeights, tempWeight); } while (hasMoreRows); ByteString first_sumVal = ci.getProtoForPromotedType(sumVal).toByteString(); S s = sumWeights == null ? ci.castToReturnType(ci.getMinValue()) : sumWeights; ByteString first_sumWeights = ci.getProtoForPromotedType(s).toByteString(); AggregateResponse.Builder pair = AggregateResponse.newBuilder(); pair.addFirstPart(first_sumVal); pair.addFirstPart(first_sumWeights); response = pair.build(); } catch (IOException e) { CoprocessorRpcUtils.setControllerException(controller, e); } finally { if (scanner != null) { try { scanner.close(); } catch (IOException ignored) {} } } done.run(response); }