Java Code Examples for org.apache.hadoop.hbase.client.Scan#setStopRow()
The following examples show how to use
org.apache.hadoop.hbase.client.Scan#setStopRow() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HbaseHostApplicationMapDao.java From pinpoint with Apache License 2.0 | 6 votes |
private Scan createScan(Application parentApplication, Range range) { Objects.requireNonNull(parentApplication, "parentApplication"); if (logger.isDebugEnabled()) { logger.debug("scan parentApplication:{}, range:{}", parentApplication, range); } // TODO need common logic for creating scanner final long startTime = TimeUtils.reverseTimeMillis(timeSlot.getTimeSlot(range.getFrom())); final long endTime = TimeUtils.reverseTimeMillis(timeSlot.getTimeSlot(range.getTo()) + 1); // start key is replaced by end key because timestamp has been reversed final byte[] startKey = createKey(parentApplication, endTime); final byte[] endKey = createKey(parentApplication, startTime); Scan scan = new Scan(); scan.setCaching(this.scanCacheSize); scan.setStartRow(startKey); scan.setStopRow(endKey); scan.setId("HostApplicationScan_Ver2"); return scan; }
Example 2
Source File: TransformerBaseRunner.java From BigDataArchitect with Apache License 2.0 | 6 votes |
/** * 初始化scan集合 * * @param job * @return */ protected List<Scan> initScans(Job job) { Configuration conf = job.getConfiguration(); // 获取运行时间: yyyy-MM-dd String date = conf.get(GlobalConstants.RUNNING_DATE_PARAMES); long startDate = TimeUtil.parseString2Long(date); long endDate = startDate + GlobalConstants.DAY_OF_MILLISECONDS; Scan scan = new Scan(); // 定义hbase扫描的开始rowkey和结束rowkey scan.setStartRow(Bytes.toBytes("" + startDate)); scan.setStopRow(Bytes.toBytes("" + endDate)); scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(EventLogConstants.HBASE_NAME_EVENT_LOGS)); Filter filter = this.fetchHbaseFilter(); if (filter != null) { scan.setFilter(filter); } // 优化设置cache scan.setBatch(500); scan.setCacheBlocks(true); // 启动cache blocks scan.setCaching(1000); // 设置每次返回的行数,默认值100,设置较大的值可以提高速度(减少rpc操作),但是较大的值可能会导致内存异常。 return Lists.newArrayList(scan); }
Example 3
Source File: ScanRanges.java From phoenix with BSD 3-Clause "New" or "Revised" License | 6 votes |
public void setScanStartStopRow(Scan scan) { if (isEverything()) { return; } if (isDegenerate()) { scan.setStartRow(KeyRange.EMPTY_RANGE.getLowerRange()); scan.setStopRow(KeyRange.EMPTY_RANGE.getUpperRange()); return; } byte[] expectedKey; expectedKey = ScanUtil.getMinKey(schema, ranges); if (expectedKey != null) { scan.setStartRow(expectedKey); } expectedKey = ScanUtil.getMaxKey(schema, ranges); if (expectedKey != null) { scan.setStopRow(expectedKey); } }
Example 4
Source File: HBase_1_1_2_ClientService.java From localization_nifi with Apache License 2.0 | 6 votes |
protected ResultScanner getResults(final Table table, final byte[] startRow, final byte[] endRow, final Collection<Column> columns) throws IOException { final Scan scan = new Scan(); scan.setStartRow(startRow); scan.setStopRow(endRow); if (columns != null) { for (Column col : columns) { if (col.getQualifier() == null) { scan.addFamily(col.getFamily()); } else { scan.addColumn(col.getFamily(), col.getQualifier()); } } } return table.getScanner(scan); }
Example 5
Source File: HbaseAgentInfoDao.java From pinpoint with Apache License 2.0 | 6 votes |
private Scan createScan(String agentId, long currentTime) { Scan scan = new Scan(); byte[] agentIdBytes = Bytes.toBytes(agentId); long startTime = TimeUtils.reverseTimeMillis(currentTime); byte[] startKeyBytes = RowKeyUtils.concatFixedByteAndLong(agentIdBytes, HbaseTableConstatns.AGENT_NAME_MAX_LEN, startTime); byte[] endKeyBytes = RowKeyUtils.concatFixedByteAndLong(agentIdBytes, HbaseTableConstatns.AGENT_NAME_MAX_LEN, Long.MAX_VALUE); scan.setStartRow(startKeyBytes); scan.setStopRow(endKeyBytes); scan.addFamily(descriptor.getColumnFamilyName()); scan.setMaxVersions(1); scan.setCaching(SCANNER_CACHING); return scan; }
Example 6
Source File: HbaseMapStatisticsCalleeDao.java From pinpoint with Apache License 2.0 | 6 votes |
private Scan createScan(Application application, Range range, byte[] family) { range = rangeFactory.createStatisticsRange(range); if (logger.isDebugEnabled()) { logger.debug("scan time:{} ", range.prettyToString()); } // start key is replaced by end key because timestamp has been reversed byte[] startKey = ApplicationMapStatisticsUtils.makeRowKey(application.getName(), application.getServiceTypeCode(), range.getTo()); byte[] endKey = ApplicationMapStatisticsUtils.makeRowKey(application.getName(), application.getServiceTypeCode(), range.getFrom()); Scan scan = new Scan(); scan.setCaching(SCAN_CACHE_SIZE); scan.setStartRow(startKey); scan.setStopRow(endKey); scan.addFamily(family); scan.setId("ApplicationStatisticsScan"); return scan; }
Example 7
Source File: MetaDataEndpointImpl.java From phoenix with BSD 3-Clause "New" or "Revised" License | 5 votes |
/** * @param tableName parent table's name * @return true if there exist a table that use this table as their base table. * TODO: should we pass a timestamp here? */ private boolean hasViews(HRegion region, byte[] tenantId, PTable table) throws IOException { byte[] schemaName = table.getSchemaName().getBytes(); byte[] tableName = table.getTableName().getBytes(); Scan scan = new Scan(); // If the table is multi-tenant, we need to check across all tenant_ids, // so we can't constrain the row key. Otherwise, any views would have // the same tenantId. if (!table.isMultiTenant()) { byte[] startRow = ByteUtil.concat(tenantId, QueryConstants.SEPARATOR_BYTE_ARRAY); byte[] stopRow = ByteUtil.nextKey(startRow); scan.setStartRow(startRow); scan.setStopRow(stopRow); } SingleColumnValueFilter filter1 = new SingleColumnValueFilter(TABLE_FAMILY_BYTES, BASE_SCHEMA_NAME_BYTES, EQUAL, schemaName); filter1.setFilterIfMissing(schemaName.length > 0); SingleColumnValueFilter filter2 = new SingleColumnValueFilter(TABLE_FAMILY_BYTES, BASE_TABLE_NAME_BYTES, EQUAL, tableName); filter2.setFilterIfMissing(true); BinaryComparator comparator = new BinaryComparator(ByteUtil.concat(tenantId, QueryConstants.SEPARATOR_BYTE_ARRAY, schemaName, QueryConstants.SEPARATOR_BYTE_ARRAY, tableName)); RowFilter filter3 = new RowFilter(CompareOp.NOT_EQUAL,comparator); Filter filter = new FilterList(filter1,filter2,filter3); scan.setFilter(filter); RegionScanner scanner = region.getScanner(scan); try { List<KeyValue> results = newArrayList(); scanner.next(results); return results.size() > 0; } finally { scanner.close(); } }
Example 8
Source File: PcapGetterHBaseImpl.java From opensoc-streaming with Apache License 2.0 | 5 votes |
/** * Creates the scan request. * * @param pcapsResponse * the pcaps response * @param keysMap * the keys map * @param startTime * the start time * @param endTime * the end time * @param maxResultSize * the max result size * @return the scan * @throws IOException * Signals that an I/O exception has occurred. */ @VisibleForTesting Scan createScanRequest(PcapsResponse pcapsResponse, Map<String, String> keysMap, long startTime, long endTime, long maxResultSize) throws IOException { Scan scan = new Scan(); // set column family, qualifier scan.addColumn(ConfigurationUtil.getColumnFamily(), ConfigurationUtil.getColumnQualifier()); // set start and stop keys scan.setStartRow(keysMap.get(HBaseConfigConstants.START_KEY).getBytes()); scan.setStopRow(keysMap.get(HBaseConfigConstants.END_KEY).getBytes()); // set max results size : remaining size = max results size - ( current // pcaps response size + possible maximum row size) long remainingSize = maxResultSize - (pcapsResponse.getResponseSize() + ConfigurationUtil.getMaxRowSize()); if (remainingSize > 0) { scan.setMaxResultSize(remainingSize); } // set max versions scan.setMaxVersions(ConfigurationUtil.getConfiguration().getInt( "hbase.table.column.maxVersions")); // set time range setTimeRangeOnScan(scan, startTime, endTime); return scan; }
Example 9
Source File: HBaseLogReader.java From eagle with Apache License 2.0 | 5 votes |
/** * TODO If the required field is null for a row, then this row will not be fetched. That could be a * problem for counting Need another version of read to strictly get the number of rows which will return * all the columns for a column family */ @Override public void open() throws IOException { if (isOpen) { return; // silently return } try { tbl = EagleConfigFactory.load().getHTable(schema.getTable()); } catch (RuntimeException ex) { throw new IOException(ex); } String rowkeyRegex = buildRegex2(searchTags); RegexStringComparator regexStringComparator = new RegexStringComparator(rowkeyRegex); regexStringComparator.setCharset(Charset.forName("ISO-8859-1")); RowFilter filter = new RowFilter(CompareOp.EQUAL, regexStringComparator); FilterList filterList = new FilterList(); filterList.addFilter(filter); Scan s1 = new Scan(); // reverse timestamp, startRow is stopKey, and stopRow is startKey s1.setStartRow(stopKey); s1.setStopRow(startKey); s1.setFilter(filterList); // TODO the # of cached rows should be minimum of (pagesize and 100) s1.setCaching(100); // TODO not optimized for all applications s1.setCacheBlocks(true); // scan specified columnfamily and qualifiers for (byte[] qualifier : qualifiers) { s1.addColumn(schema.getColumnFamily().getBytes(), qualifier); } rs = tbl.getScanner(s1); isOpen = true; }
Example 10
Source File: DataJanitorState.java From phoenix-tephra with Apache License 2.0 | 5 votes |
/** * Delete empty region records saved on or before the given time. * * @param time time in milliseconds */ public void deleteEmptyRegionsOnOrBeforeTime(long time) throws IOException { try (Table stateTable = stateTableSupplier.get()) { Scan scan = new Scan(); scan.setStopRow(makeEmptyRegionTimeKey(Bytes.toBytes(time + 1), EMPTY_BYTE_ARRAY)); scan.addColumn(FAMILY, EMPTY_REGION_TIME_COL); deleteFromScan(stateTable, scan); } }
Example 11
Source File: DataJanitorState.java From phoenix-tephra with Apache License 2.0 | 5 votes |
/** * Delete empty region records saved on or before the given time. * * @param time time in milliseconds */ public void deleteEmptyRegionsOnOrBeforeTime(long time) throws IOException { try (Table stateTable = stateTableSupplier.get()) { Scan scan = new Scan(); scan.setStopRow(makeEmptyRegionTimeKey(Bytes.toBytes(time + 1), EMPTY_BYTE_ARRAY)); scan.addColumn(FAMILY, EMPTY_REGION_TIME_COL); deleteFromScan(stateTable, scan); } }
Example 12
Source File: DataJanitorState.java From phoenix-tephra with Apache License 2.0 | 5 votes |
/** * Delete empty region records saved on or before the given time. * * @param time time in milliseconds */ public void deleteEmptyRegionsOnOrBeforeTime(long time) throws IOException { try (HTableInterface stateTable = stateTableSupplier.get()) { Scan scan = new Scan(); scan.setStopRow(makeEmptyRegionTimeKey(Bytes.toBytes(time + 1), EMPTY_BYTE_ARRAY)); scan.addColumn(FAMILY, EMPTY_REGION_TIME_COL); deleteFromScan(stateTable, scan); } }
Example 13
Source File: UngroupedAggregateRegionObserver.java From phoenix with Apache License 2.0 | 5 votes |
@Override public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s) throws IOException { s = super.preScannerOpen(e, scan, s); if (ScanUtil.isAnalyzeTable(scan)) { // We are setting the start row and stop row such that it covers the entire region. As part // of Phonenix-1263 we are storing the guideposts against the physical table rather than // individual tenant specific tables. scan.setStartRow(HConstants.EMPTY_START_ROW); scan.setStopRow(HConstants.EMPTY_END_ROW); scan.setFilter(null); } return s; }
Example 14
Source File: DataJanitorState.java From phoenix-tephra with Apache License 2.0 | 5 votes |
/** * Delete empty region records saved on or before the given time. * * @param time time in milliseconds */ public void deleteEmptyRegionsOnOrBeforeTime(long time) throws IOException { try (Table stateTable = stateTableSupplier.get()) { Scan scan = new Scan(); scan.setStopRow(makeEmptyRegionTimeKey(Bytes.toBytes(time + 1), EMPTY_BYTE_ARRAY)); scan.addColumn(FAMILY, EMPTY_REGION_TIME_COL); deleteFromScan(stateTable, scan); } }
Example 15
Source File: SlicedRowFilterGTSDecoderIterator.java From warp10-platform with Apache License 2.0 | 4 votes |
public SlicedRowFilterGTSDecoderIterator(long now, long timespan, List<Metadata> metadatas, Connection conn, TableName tableName, byte[] colfam, boolean writeTimestamp, KeyStore keystore, boolean useBlockCache) { this.keystore = keystore; this.now = now; this.timespan = timespan; this.hbaseAESKey = keystore.getKey(KeyStore.AES_HBASE_DATA); this.writeTimestamp = writeTimestamp; // // Check that if 'timespan' is < 0 then 'now' is either Long.MAX_VALUE or congruent to 0 modulo DEFAULT_MODULUS // if (timespan < 0) { if (Long.MAX_VALUE != now && 0 != (now % Constants.DEFAULT_MODULUS)) { throw new RuntimeException("Incompatible 'timespan' (" + timespan + ") and 'now' (" + now + ")"); } } // // Create a SlicedRowFilter for the prefix, class id, labels id and ts // We include the prefix so we exit the filter early when the last // matching row has been reached // // 128BITS int[] bounds = { 0, 24 }; // // Create singleton for each classId/labelsId combo // // TODO(hbs): we should really create multiple scanner, one per class Id for example, // List<Pair<byte[], byte[]>> ranges = new ArrayList<Pair<byte[], byte[]>>(); for (Metadata metadata: metadatas) { byte[][] keys = getKeys(metadata, now, timespan); byte[] lower = keys[0]; byte[] upper = keys[1]; this.metadatas.put(new String(Arrays.copyOfRange(lower, prefix.length, prefix.length + 16), StandardCharsets.ISO_8859_1), metadata); Pair<byte[],byte[]> range = new Pair<byte[],byte[]>(lower, upper); ranges.add(range); } SlicedRowFilter filter = new SlicedRowFilter(bounds, ranges, timespan < 0 ? -timespan : Long.MAX_VALUE); // // Create scanner. The start key is the lower bound of the first range // Scan scan = new Scan(); scan.addFamily(colfam); // (HBaseStore.GTS_COLFAM, Longs.toByteArray(Long.MAX_VALUE - modulus)); scan.setStartRow(filter.getStartKey()); byte[] filterStopKey = filter.getStopKey(); // Add one byte at the end (we can do that because we know the slice is the whole key) byte[] stopRow = Arrays.copyOf(filterStopKey, filterStopKey.length + 1); scan.setStopRow(stopRow); scan.setFilter(filter); scan.setMaxResultSize(1000000L); scan.setBatch(50000); scan.setCaching(50000); scan.setCacheBlocks(useBlockCache); Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_HBASE_CLIENT_FILTERED_SCANNERS, Sensision.EMPTY_LABELS, 1); Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_HBASE_CLIENT_FILTERED_SCANNERS_RANGES, Sensision.EMPTY_LABELS, ranges.size()); try { this.htable = conn.getTable(tableName); this.scanner = this.htable.getScanner(scan); iter = scanner.iterator(); } catch (IOException ioe) { LOG.error("",ioe); this.iter = null; } }
Example 16
Source File: GridTableHBaseBenchmark.java From kylin with Apache License 2.0 | 4 votes |
private static void jumpScan(Connection conn, boolean[] hits, Stats stats) throws IOException { final int jumpThreshold = 6; // compensate for Scan() overhead, totally by experience Table table = conn.getTable(TableName.valueOf(TEST_TABLE)); try { stats.markStart(); int i = 0; while (i < N_ROWS) { // find the first hit int start = i; while (start + 1 < N_ROWS && !hits[start]) start++; // find the last hit within jumpThreshold int end = start + 1; int jump = end + 1; while (jump < N_ROWS && (end + jumpThreshold > jump)) { if (hits[jump]) { end = jump; } jump++; } if (start < N_ROWS) { Scan scan = new Scan(); scan.setStartRow(Bytes.toBytes(start)); scan.setStopRow(Bytes.toBytes(end)); scan.addFamily(CF); ResultScanner scanner = table.getScanner(scan); i = start; for (Result r : scanner) { stats.consume(r); dot(i, N_ROWS); i++; } } i = end; } stats.markEnd(); } finally { IOUtils.closeQuietly(table); } }
Example 17
Source File: QueryEndpoint.java From yuzhouwan with Apache License 2.0 | 4 votes |
@Override public void queryByStartRowAndEndRow(RpcController controller, DataProtos.DataQueryRequest request, RpcCallback<DataQueryResponse> done) { DataProtos.DataQueryResponse response = null; InternalScanner scanner = null; try { String startRow = request.getStartRow(); String endRow = request.getEndRow(); String regionStartKey = Bytes.toString(this.env.getRegion().getRegionInfo().getStartKey()); String regionEndKey = Bytes.toString(this.env.getRegion().getRegionInfo().getEndKey()); if (request.getIsSalting()) { // 如果加盐过则在key前添加盐值 String startSalt = null; String endSalt = null; if (StrUtils.isNotEmpty(regionStartKey)) { startSalt = regionStartKey.split("_")[0]; // 加盐的方式为盐值+"_",所以取_前面的 } if (StrUtils.isNotEmpty(regionEndKey)) { endSalt = regionStartKey.split("_")[0]; //加盐的方式为盐值+"_",所以取_前面的 } if (startSalt != null) { if (null != startRow) { startRow = startSalt + "_" + startRow; endRow = endSalt + "_" + endRow; } } } Scan scan = new Scan(); if (null != startRow) { scan.setStartRow(Bytes.toBytes(startRow)); } if (null != endRow) { if (request.getIncluedEnd()) { Filter filter = new InclusiveStopFilter(Bytes.toBytes(endRow)); scan.setFilter(filter); } else { scan.setStopRow(Bytes.toBytes(endRow)); } } scanner = this.env.getRegion().getScanner(scan); List<Cell> results = new ArrayList<>(); boolean hasMore; DataProtos.DataQueryResponse.Builder responseBuilder = DataProtos.DataQueryResponse.newBuilder(); do { hasMore = scanner.next(results); DataProtos.DataQueryResponse.Row.Builder rowBuilder = DataProtos.DataQueryResponse.Row.newBuilder(); if (results.size() > 0) { rowBuilder.setRowKey(ByteString.copyFrom(results.get(0).getRow())); for (Cell kv : results) { queryBuilder(rowBuilder, ByteString.copyFrom(kv.getFamily()), ByteString.copyFrom(kv.getQualifier()), ByteString.copyFrom(kv.getRow()), ByteString.copyFrom(kv.getValue())); } } responseBuilder.addRowList(rowBuilder); results.clear(); } while (hasMore); response = responseBuilder.build(); } catch (IOException ignored) { ResponseConverter.setControllerException(controller, ignored); } finally { if (scanner != null) { try { scanner.close(); } catch (IOException e) { _log.error(ExceptionUtils.errorInfo(e)); } } } done.run(response); }
Example 18
Source File: DownloadData.java From Transwarp-Sample-Code with MIT License | 4 votes |
/** * 获取文件 * @param fileName 文件名 */ public int getFile(String fileName) { try { HTable hTable = new HTable(configuration, constant.HBASE_TABLE_NAME); String fileName2 = fileName; if (System.getProperty("os.name").toLowerCase().contains("windows")) { fileName2 = constant.UPLOAD_DIR + "\\" + fileName; System.out.println(fileName2); } else { fileName2 = constant.UPLOAD_DIR + "/" + fileName; } String fileNameMD5 = md5crypt(fileName2); String result_filename; byte[] result_filedata; int result_fileCount=0; byte [] startRowKey = Bytes.toBytes(fileNameMD5); byte [] stopRowKey = Bytes.toBytes(fileNameMD5+"9"); Scan scan = new Scan(); scan.setStartRow(startRowKey); scan.setStopRow(stopRowKey); ResultScanner scanner = hTable.getScanner(scan); for (Result r : scanner) { result_filename = Bytes.toString(r.getValue(Bytes.toBytes("file"), Bytes.toBytes("filename"))); System.out.println(result_filename); if (fileName2.compareTo(result_filename) == 0){ result_fileCount++; result_filedata = r.getValue(Bytes.toBytes("data"), Bytes.toBytes("bytes")); FileUtil.byte2File(result_filedata, constant.DOWNLOAD_DIR, fileName); } } if (result_fileCount != 0) System.out.println("Download "+ result_fileCount + " files"); else System.out.println("Not found:"+fileName); hTable.close(); return result_fileCount; } catch (Exception e) { e.printStackTrace(); } return 0; }
Example 19
Source File: IndexedTable.java From hbase-secondary-index with GNU General Public License v3.0 | 4 votes |
/** * Open up an indexed scanner. Results will come back in the indexed order, but will contain RowResults from the * original table. * * @param indexId the id of the index to use * @param indexStartRow (created from the IndexKeyGenerator) * @param indexStopRow (created from the IndexKeyGenerator) * @param indexColumns in the index table * @param indexFilter filter to run on the index'ed table. This can only use columns that have been added to the * index. * @param baseColumns from the original table * @return scanner * @throws IOException * @throws IndexNotFoundException */ public ResultScanner getIndexedScanner(final String indexId, final byte[] indexStartRow, final byte[] indexStopRow, final byte[][] indexColumns, final Filter indexFilter, final byte[][] baseColumns) throws IOException, IndexNotFoundException { IndexSpecification indexSpec = this.indexedTableDescriptor.getIndex(indexId); if (indexSpec == null) { updateIndexedTableDescriptor(); indexSpec = this.indexedTableDescriptor.getIndex(indexId); if (indexSpec == null) { throw new IndexNotFoundException("Index " + indexId + " not defined in table " + super.getTableDescriptor().getNameAsString()); } } verifyIndexColumns(indexColumns, indexSpec); // TODO, verify/remove index columns from baseColumns HTable indexTable = indexIdToTable.get(indexId); byte[][] allIndexColumns; if (indexColumns != null) { allIndexColumns = new byte[indexColumns.length + 1][]; System.arraycopy(indexColumns, 0, allIndexColumns, 0, indexColumns.length); allIndexColumns[indexColumns.length] = INDEX_BASE_ROW_COLUMN; } else { byte[][] allColumns = indexSpec.getAllColumns(); allIndexColumns = new byte[allColumns.length + 1][]; System.arraycopy(allColumns, 0, allIndexColumns, 0, allColumns.length); allIndexColumns[allColumns.length] = INDEX_BASE_ROW_COLUMN; } Scan indexScan = new Scan(); indexScan.setFilter(indexFilter); // indexScan.addColumns(allIndexColumns); ScanUtil.addColumns(indexScan,allIndexColumns); if (indexStartRow != null) { indexScan.setStartRow(indexStartRow); } if (indexStopRow != null) { indexScan.setStopRow(indexStopRow); } ResultScanner indexScanner = indexTable.getScanner(indexScan); return new ScannerWrapper(indexScanner, baseColumns); }
Example 20
Source File: ScanRanges.java From phoenix with Apache License 2.0 | 4 votes |
public void initializeScan(Scan scan) { scan.setStartRow(scanRange.getLowerRange()); scan.setStopRow(scanRange.getUpperRange()); }