Java Code Examples for org.apache.hadoop.hbase.client.ResultScanner#iterator()
The following examples show how to use
org.apache.hadoop.hbase.client.ResultScanner#iterator() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestThriftConnection.java From hbase with Apache License 2.0 | 6 votes |
private void testScanWithFilters(Connection connection, String tableName) throws IOException { createTable(thriftAdmin, tableName); try (Table table = connection.getTable(TableName.valueOf(tableName))){ FilterList filterList = new FilterList(); PrefixFilter prefixFilter = new PrefixFilter(Bytes.toBytes("testrow")); ColumnValueFilter columnValueFilter = new ColumnValueFilter(FAMILYA, QUALIFIER_1, CompareOperator.EQUAL, VALUE_1); filterList.addFilter(prefixFilter); filterList.addFilter(columnValueFilter); Scan scan = new Scan(); scan.readVersions(2); scan.setFilter(filterList); ResultScanner scanner = table.getScanner(scan); Iterator<Result> iterator = scanner.iterator(); assertTrue(iterator.hasNext()); int counter = 0; while (iterator.hasNext()) { Result result = iterator.next(); counter += result.size(); } assertEquals(2, counter); } }
Example 2
Source File: TestSCVFWithMiniCluster.java From hbase with Apache License 2.0 | 6 votes |
private void verify(Scan scan) throws IOException { ResultScanner scanner = htable.getScanner(scan); Iterator<Result> it = scanner.iterator(); /* Then */ int count = 0; try { while (it.hasNext()) { it.next(); count++; } } finally { scanner.close(); } assertEquals(expected, count); }
Example 3
Source File: HBaseLookupTable.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
public HBaseScanBasedIterator(Table table) { try { Scan scan = new Scan(); scan.setCaching(1000); ResultScanner scanner = table.getScanner(HBaseLookupRowEncoder.CF); scannerIterator = scanner.iterator(); } catch (IOException e) { logger.error("error when scan HBase", e); } }
Example 4
Source File: HBaseLookupTable.java From kylin with Apache License 2.0 | 5 votes |
public HBaseScanBasedIterator(Table table) { try { Scan scan = new Scan(); scan.setCaching(1000); ResultScanner scanner = table.getScanner(HBaseLookupRowEncoder.CF); scannerIterator = scanner.iterator(); } catch (IOException e) { logger.error("error when scan HBase", e); } }
Example 5
Source File: TestHBaseStorage.java From spork with Apache License 2.0 | 5 votes |
/** * load from hbase 'TESTTABLE_1' using HBaseBinary format, and store it into * 'TESTTABLE_2' using HBaseBinaryFormat * * @throws IOException */ @Test public void testStoreToHBase_1() throws IOException { prepareTable(TESTTABLE_1, true, DataFormat.HBaseBinary); prepareTable(TESTTABLE_2, false, DataFormat.HBaseBinary); pig.getPigContext().getProperties() .setProperty(MRConfiguration.FILEOUTPUTCOMMITTER_MARKSUCCESSFULJOBS, "true"); scanTable1(pig, DataFormat.HBaseBinary); pig.store("a", "hbase://" + TESTTABLE_2, "org.apache.pig.backend.hadoop.hbase.HBaseStorage('" + TESTCOLUMN_A + " " + TESTCOLUMN_B + " " + TESTCOLUMN_C + "','-caster HBaseBinaryConverter')"); HTable table = new HTable(conf, TESTTABLE_2); ResultScanner scanner = table.getScanner(new Scan()); Iterator<Result> iter = scanner.iterator(); int i = 0; for (i = 0; iter.hasNext(); ++i) { Result result = iter.next(); String v = i + ""; String rowKey = Bytes.toString(result.getRow()); int col_a = Bytes.toInt(getColValue(result, TESTCOLUMN_A)); double col_b = Bytes.toDouble(getColValue(result, TESTCOLUMN_B)); String col_c = Bytes.toString(getColValue(result, TESTCOLUMN_C)); Assert.assertEquals("00".substring(v.length()) + v, rowKey); Assert.assertEquals(i, col_a); Assert.assertEquals(i + 0.0, col_b, 1e-6); Assert.assertEquals("Text_" + i, col_c); } Assert.assertEquals(100, i); pig.getPigContext().getProperties() .setProperty(MRConfiguration.FILEOUTPUTCOMMITTER_MARKSUCCESSFULJOBS, "false"); table.close(); }
Example 6
Source File: TestHBaseStorage.java From spork with Apache License 2.0 | 5 votes |
/** * load from hbase 'TESTTABLE_1' using HBaseBinary format, and store it into * 'TESTTABLE_2' using HBaseBinaryFormat projecting out column c * * @throws IOException */ @Test public void testStoreToHBase_1_with_projection() throws IOException { System.getProperties().setProperty("pig.usenewlogicalplan", "false"); prepareTable(TESTTABLE_1, true, DataFormat.HBaseBinary); prepareTable(TESTTABLE_2, false, DataFormat.HBaseBinary); scanTable1(pig, DataFormat.HBaseBinary); pig.registerQuery("b = FOREACH a GENERATE rowKey, col_a, col_b;"); pig.store("b", TESTTABLE_2, "org.apache.pig.backend.hadoop.hbase.HBaseStorage('" + TESTCOLUMN_A + " " + TESTCOLUMN_B + "','-caster HBaseBinaryConverter')"); HTable table = new HTable(conf, TESTTABLE_2); ResultScanner scanner = table.getScanner(new Scan()); Iterator<Result> iter = scanner.iterator(); int i = 0; for (i = 0; iter.hasNext(); ++i) { Result result = iter.next(); String v = String.valueOf(i); String rowKey = Bytes.toString(result.getRow()); int col_a = Bytes.toInt(getColValue(result, TESTCOLUMN_A)); double col_b = Bytes.toDouble(getColValue(result, TESTCOLUMN_B)); Assert.assertEquals("00".substring(v.length()) + v, rowKey); Assert.assertEquals(i, col_a); Assert.assertEquals(i + 0.0, col_b, 1e-6); } Assert.assertEquals(100, i); table.close(); }
Example 7
Source File: TestHBaseStorage.java From spork with Apache License 2.0 | 5 votes |
/** * load from hbase 'TESTTABLE_1' using HBaseBinary format, and store it into * 'TESTTABLE_2' using UTF-8 Plain Text format * * @throws IOException */ @Test public void testStoreToHBase_2() throws IOException { prepareTable(TESTTABLE_1, true, DataFormat.HBaseBinary); prepareTable(TESTTABLE_2, false, DataFormat.HBaseBinary); scanTable1(pig, DataFormat.HBaseBinary); pig.store("a", TESTTABLE_2, "org.apache.pig.backend.hadoop.hbase.HBaseStorage('" + TESTCOLUMN_A + " " + TESTCOLUMN_B + " " + TESTCOLUMN_C + "')"); HTable table = new HTable(conf, TESTTABLE_2); ResultScanner scanner = table.getScanner(new Scan()); Iterator<Result> iter = scanner.iterator(); int i = 0; for (i = 0; iter.hasNext(); ++i) { Result result = iter.next(); String v = i + ""; String rowKey = new String(result.getRow()); int col_a = Integer.parseInt(new String(getColValue(result, TESTCOLUMN_A))); double col_b = Double.parseDouble(new String(getColValue(result, TESTCOLUMN_B))); String col_c = new String(getColValue(result, TESTCOLUMN_C)); Assert.assertEquals("00".substring(v.length()) + v, rowKey); Assert.assertEquals(i, col_a); Assert.assertEquals(i + 0.0, col_b, 1e-6); Assert.assertEquals("Text_" + i, col_c); } Assert.assertEquals(100, i); table.close(); }
Example 8
Source File: TestHBaseStorage.java From spork with Apache License 2.0 | 5 votes |
/** * load from hbase 'TESTTABLE_1' using HBaseBinary format, and store it into * 'TESTTABLE_2' using UTF-8 Plain Text format projecting column c * * @throws IOException */ @Test public void testStoreToHBase_2_with_projection() throws IOException { prepareTable(TESTTABLE_1, true, DataFormat.HBaseBinary); prepareTable(TESTTABLE_2, false, DataFormat.UTF8PlainText); scanTable1(pig, DataFormat.HBaseBinary); pig.registerQuery("b = FOREACH a GENERATE rowKey, col_a, col_b;"); pig.store("b", TESTTABLE_2, "org.apache.pig.backend.hadoop.hbase.HBaseStorage('" + TESTCOLUMN_A + " " + TESTCOLUMN_B + "')"); HTable table = new HTable(conf, TESTTABLE_2); ResultScanner scanner = table.getScanner(new Scan()); Iterator<Result> iter = scanner.iterator(); int i = 0; for (i = 0; iter.hasNext(); ++i) { Result result = iter.next(); String v = i + ""; String rowKey = new String(result.getRow()); int col_a = Integer.parseInt(new String(getColValue(result, TESTCOLUMN_A))); double col_b = Double.parseDouble(new String(getColValue(result, TESTCOLUMN_B))); Assert.assertEquals("00".substring(v.length()) + v, rowKey); Assert.assertEquals(i, col_a); Assert.assertEquals(i + 0.0, col_b, 1e-6); } Assert.assertEquals(100, i); table.close(); }
Example 9
Source File: TestHBaseStorage.java From spork with Apache License 2.0 | 5 votes |
/** * load from hbase 'TESTTABLE_1' using UTF-8 Plain Text format, and store it * into 'TESTTABLE_2' using UTF-8 Plain Text format projecting column c * * @throws IOException */ @Test public void testStoreToHBase_3_with_projection_no_caster() throws IOException { prepareTable(TESTTABLE_1, true, DataFormat.UTF8PlainText); prepareTable(TESTTABLE_2, false, DataFormat.UTF8PlainText); scanTable1(pig, DataFormat.UTF8PlainText); pig.registerQuery("b = FOREACH a GENERATE rowKey, col_a, col_b;"); pig.store("b", TESTTABLE_2, "org.apache.pig.backend.hadoop.hbase.HBaseStorage('" + TESTCOLUMN_A + " " + TESTCOLUMN_B + "')"); HTable table = new HTable(conf, TESTTABLE_2); ResultScanner scanner = table.getScanner(new Scan()); Iterator<Result> iter = scanner.iterator(); int i = 0; for (i = 0; iter.hasNext(); ++i) { Result result = iter.next(); String v = i + ""; String rowKey = new String(result.getRow()); String col_a = new String(getColValue(result, TESTCOLUMN_A)); String col_b = new String(getColValue(result, TESTCOLUMN_B)); Assert.assertEquals("00".substring(v.length()) + v, rowKey); Assert.assertEquals(i + "", col_a); Assert.assertEquals(i + 0.0 + "", col_b); } Assert.assertEquals(100, i); table.close(); }
Example 10
Source File: IndexVerificationOutputRepository.java From phoenix with Apache License 2.0 | 5 votes |
public Iterator<IndexVerificationOutputRow> getOutputRowIterator(long ts, byte[] indexName) throws IOException { Scan scan = new Scan(); byte[] partialKey = generatePartialOutputTableRowKey(ts, indexName); scan.withStartRow(partialKey); scan.withStopRow(ByteUtil.calculateTheClosestNextRowKeyForPrefix(partialKey)); ResultScanner scanner = outputTable.getScanner(scan); return new IndexVerificationOutputRowIterator(scanner.iterator()); }
Example 11
Source File: HBaseMetricQueryClient.java From jstorm with Apache License 2.0 | 5 votes |
@Override public List<Object> getMetricData(String metricId, MetricType metricType, int win, long start, long end, int size) { long metricIdLong = Long.parseLong(metricId); byte[] startKey = MetricBaseData.makeKey(metricIdLong, win, start); byte[] endKey = MetricBaseData.makeKey(metricIdLong, win, end); HTableInterface table = getHTableInterface(TABLE_METRIC_DATA); Scan scan = new Scan(startKey, endKey); //scan.setBatch(10); scan.setCaching(CACHE_SIZE); ResultScanner scanner = null; try { scanner = getScanner(table, scan); Iterator<Result> rows = scanner.iterator(); if (rows != null) { List<Object> ret = new ArrayList<>(size); while (rows.hasNext()) { Result row = rows.next(); Object obj = parseMetricDataRow(row, metricType); if (obj != null) { ret.add(obj); } } return ret; } } catch (Exception ex) { logger.error("Scan error, metric id:{}, metric type:{}", metricId, metricType, ex); } finally { if (scanner != null) { scanner.close(); } closeTable(table); } return new ArrayList<>(0); }
Example 12
Source File: HbaseSessions.java From hugegraph with Apache License 2.0 | 4 votes |
public RowIterator(ResultScanner resultScanner) { this.resultScanner = resultScanner; this.results = resultScanner.iterator(); }
Example 13
Source File: SyncTable.java From hbase with Apache License 2.0 | 4 votes |
/** * Rescan the given range directly from the source and target tables. * Count and log differences, and if this is not a dry run, output Puts and Deletes * to make the target table match the source table for this range */ private void syncRange(Context context, ImmutableBytesWritable startRow, ImmutableBytesWritable stopRow) throws IOException, InterruptedException { Scan scan = sourceTableHash.initScan(); scan.withStartRow(startRow.copyBytes()); scan.withStopRow(stopRow.copyBytes()); ResultScanner sourceScanner = sourceTable.getScanner(scan); CellScanner sourceCells = new CellScanner(sourceScanner.iterator()); ResultScanner targetScanner = targetTable.getScanner(new Scan(scan)); CellScanner targetCells = new CellScanner(targetScanner.iterator()); boolean rangeMatched = true; byte[] nextSourceRow = sourceCells.nextRow(); byte[] nextTargetRow = targetCells.nextRow(); while(nextSourceRow != null || nextTargetRow != null) { boolean rowMatched; int rowComparison = compareRowKeys(nextSourceRow, nextTargetRow); if (rowComparison < 0) { if (LOG.isDebugEnabled()) { LOG.debug("Target missing row: " + Bytes.toString(nextSourceRow)); } context.getCounter(Counter.TARGETMISSINGROWS).increment(1); rowMatched = syncRowCells(context, nextSourceRow, sourceCells, EMPTY_CELL_SCANNER); nextSourceRow = sourceCells.nextRow(); // advance only source to next row } else if (rowComparison > 0) { if (LOG.isDebugEnabled()) { LOG.debug("Source missing row: " + Bytes.toString(nextTargetRow)); } context.getCounter(Counter.SOURCEMISSINGROWS).increment(1); rowMatched = syncRowCells(context, nextTargetRow, EMPTY_CELL_SCANNER, targetCells); nextTargetRow = targetCells.nextRow(); // advance only target to next row } else { // current row is the same on both sides, compare cell by cell rowMatched = syncRowCells(context, nextSourceRow, sourceCells, targetCells); nextSourceRow = sourceCells.nextRow(); nextTargetRow = targetCells.nextRow(); } if (!rowMatched) { rangeMatched = false; } } sourceScanner.close(); targetScanner.close(); context.getCounter(rangeMatched ? Counter.RANGESMATCHED : Counter.RANGESNOTMATCHED) .increment(1); }
Example 14
Source File: TestMultithreadedTableMapper.java From hbase with Apache License 2.0 | 4 votes |
/** * Looks at every value of the mapreduce output and verifies that indeed * the values have been reversed. * * @param table Table to scan. * @throws IOException * @throws NullPointerException if we failed to find a cell value */ private void verifyAttempt(final Table table) throws IOException, NullPointerException { Scan scan = new Scan(); scan.addFamily(INPUT_FAMILY); scan.addFamily(OUTPUT_FAMILY); ResultScanner scanner = table.getScanner(scan); try { Iterator<Result> itr = scanner.iterator(); assertTrue(itr.hasNext()); while(itr.hasNext()) { Result r = itr.next(); if (LOG.isDebugEnabled()) { if (r.size() > 2 ) { throw new IOException("Too many results, expected 2 got " + r.size()); } } byte[] firstValue = null; byte[] secondValue = null; int count = 0; for(Cell kv : r.listCells()) { if (count == 0) { firstValue = CellUtil.cloneValue(kv); }else if (count == 1) { secondValue = CellUtil.cloneValue(kv); }else if (count == 2) { break; } count++; } String first = ""; if (firstValue == null) { throw new NullPointerException(Bytes.toString(r.getRow()) + ": first value is null"); } first = Bytes.toString(firstValue); String second = ""; if (secondValue == null) { throw new NullPointerException(Bytes.toString(r.getRow()) + ": second value is null"); } byte[] secondReversed = new byte[secondValue.length]; for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) { secondReversed[i] = secondValue[j]; } second = Bytes.toString(secondReversed); if (first.compareTo(second) != 0) { if (LOG.isDebugEnabled()) { LOG.debug("second key is not the reverse of first. row=" + Bytes.toStringBinary(r.getRow()) + ", first value=" + first + ", second value=" + second); } fail(); } } } finally { scanner.close(); } }
Example 15
Source File: TestTableMapReduceBase.java From hbase with Apache License 2.0 | 4 votes |
/** * Looks at every value of the mapreduce output and verifies that indeed * the values have been reversed. * @param table Table to scan. * @throws IOException * @throws NullPointerException if we failed to find a cell value */ private void verifyAttempt(final Table table) throws IOException, NullPointerException { Scan scan = new Scan(); TableInputFormat.addColumns(scan, columns); ResultScanner scanner = table.getScanner(scan); try { Iterator<Result> itr = scanner.iterator(); assertTrue(itr.hasNext()); while(itr.hasNext()) { Result r = itr.next(); if (getLog().isDebugEnabled()) { if (r.size() > 2 ) { throw new IOException("Too many results, expected 2 got " + r.size()); } } byte[] firstValue = null; byte[] secondValue = null; int count = 0; for(Cell kv : r.listCells()) { if (count == 0) { firstValue = CellUtil.cloneValue(kv); } if (count == 1) { secondValue = CellUtil.cloneValue(kv); } count++; if (count == 2) { break; } } if (firstValue == null) { throw new NullPointerException(Bytes.toString(r.getRow()) + ": first value is null"); } String first = Bytes.toString(firstValue); if (secondValue == null) { throw new NullPointerException(Bytes.toString(r.getRow()) + ": second value is null"); } byte[] secondReversed = new byte[secondValue.length]; for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) { secondReversed[i] = secondValue[j]; } String second = Bytes.toString(secondReversed); if (first.compareTo(second) != 0) { if (getLog().isDebugEnabled()) { getLog().debug("second key is not the reverse of first. row=" + Bytes.toStringBinary(r.getRow()) + ", first value=" + first + ", second value=" + second); } fail(); } } } finally { scanner.close(); } }