org.apache.hadoop.hbase.client.Scan Java Examples

The following examples show how to use org.apache.hadoop.hbase.client.Scan. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: phoenix   Author: cloudera-labs   File: WhereCompilerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleVariableFullPkSalted() throws SQLException {
    PhoenixConnection pconn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class);
    pconn.createStatement().execute("CREATE TABLE t (k varchar primary key, v varchar) SALT_BUCKETS=20");
    String query = "select * from t where k='a'";
    PhoenixPreparedStatement pstmt = newPreparedStatement(pconn, query);
    QueryPlan plan = pstmt.optimizeQuery();
    Scan scan = plan.getContext().getScan();
    Filter filter = scan.getFilter();
    assertNull(filter);
    byte[] key = new byte[2];
    PVarchar.INSTANCE.toBytes("a", key, 1);
    key[0] = SaltingUtil.getSaltingByte(key, 1, 1, 20);
    byte[] expectedStartKey = key;
    byte[] expectedEndKey = ByteUtil.nextKey(ByteUtil.concat(key, QueryConstants.SEPARATOR_BYTE_ARRAY));
    byte[] startKey = scan.getStartRow();
    byte[] stopKey = scan.getStopRow();
    assertTrue(Bytes.compareTo(expectedStartKey, startKey) == 0);
    assertTrue(Bytes.compareTo(expectedEndKey, stopKey) == 0);
}
 
Example #2
Source Project: hbase   Author: apache   File: QuotaTableUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns a list of {@code Delete} to remove all entries returned by the passed scanner.
 * @param connection connection to re-use
 * @param scan the scanner to use to generate the list of deletes
 */
static List<Delete> createDeletesForExistingSnapshotsFromScan(Connection connection, Scan scan)
    throws IOException {
  List<Delete> deletes = new ArrayList<>();
  try (Table quotaTable = connection.getTable(QUOTA_TABLE_NAME);
      ResultScanner rs = quotaTable.getScanner(scan)) {
    for (Result r : rs) {
      CellScanner cs = r.cellScanner();
      while (cs.advance()) {
        Cell c = cs.current();
        byte[] family = Bytes.copy(c.getFamilyArray(), c.getFamilyOffset(), c.getFamilyLength());
        byte[] qual =
            Bytes.copy(c.getQualifierArray(), c.getQualifierOffset(), c.getQualifierLength());
        Delete d = new Delete(r.getRow());
        d.addColumns(family, qual);
        deletes.add(d);
      }
    }
    return deletes;
  }
}
 
Example #3
Source Project: phoenix   Author: forcedotcom   File: ScanUtil.java    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
public static void andFilterAtEnd(Scan scan, Filter andWithFilter) {
    if (andWithFilter == null) {
        return;
    }
    Filter filter = scan.getFilter();
    if (filter == null) {
        scan.setFilter(andWithFilter); 
    } else if (filter instanceof FilterList && ((FilterList)filter).getOperator() == FilterList.Operator.MUST_PASS_ALL) {
        FilterList filterList = (FilterList)filter;
        List<Filter> allFilters = new ArrayList<Filter>(filterList.getFilters().size() + 1);
        allFilters.addAll(filterList.getFilters());
        allFilters.add(andWithFilter);
        scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,allFilters));
    } else {
        scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL,Arrays.asList(filter, andWithFilter)));
    }
}
 
Example #4
Source Project: hbase   Author: apache   File: TestSpaceQuotaViolationPolicyRefresherChore.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMissingAllColumns() throws IOException {
  when(chore.fetchSnapshotsFromQuotaTable()).thenCallRealMethod();
  ResultScanner scanner = mock(ResultScanner.class);
  Table quotaTable = mock(Table.class);
  when(conn.getTable(QuotaUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable);
  when(quotaTable.getScanner(any(Scan.class))).thenReturn(scanner);

  List<Result> results = new ArrayList<>();
  results.add(Result.create(Collections.emptyList()));
  when(scanner.iterator()).thenReturn(results.iterator());
  try {
    chore.fetchSnapshotsFromQuotaTable();
    fail("Expected an IOException, but did not receive one.");
  } catch (IOException e) {
    // Expected an error because we had no cells in the row.
    // This should only happen due to programmer error.
  }
}
 
Example #5
Source Project: phoenix   Author: cloudera-labs   File: WhereCompilerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPartialRangeFilter() throws SQLException {
    // I know these id's are ridiculous, but users can write queries that look like this
    String tenantId1 = "001";
    String tenantId2 = "02";
    String query = String.format("select * from %s where organization_id > '%s' AND organization_id < '%s'",
            ATABLE_NAME, tenantId1, tenantId2);
    PhoenixConnection pconn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class);
    PhoenixPreparedStatement pstmt = newPreparedStatement(pconn, query);
    QueryPlan plan = pstmt.optimizeQuery();
    Scan scan = plan.getContext().getScan();

    assertNull(scan.getFilter());
    byte[] wideLower = ByteUtil.nextKey(StringUtil.padChar(Bytes.toBytes(tenantId1), 15));
    byte[] wideUpper = StringUtil.padChar(Bytes.toBytes(tenantId2), 15);
    assertArrayEquals(wideLower, scan.getStartRow());
    assertArrayEquals(wideUpper, scan.getStopRow());
}
 
Example #6
Source Project: hbase   Author: apache   File: TestFiltersWithBinaryComponentComparator.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRowFilterWithBinaryComponentComparator() throws IOException {
  //SELECT * from table where a=1 and b > 10 and b < 20 and c > 90 and c < 100 and d=1
  tableName = TableName.valueOf(name.getMethodName());
  Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE);
  generateRows(ht, family, qf);
  FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
  setRowFilters(filterList);
  Scan scan = createScan(filterList);
  List<Cell> result = getResults(ht,scan);
  for(Cell cell: result){
    byte[] key = CellUtil.cloneRow(cell);
    int a = Bytes.readAsInt(key,aOffset,4);
    int b = Bytes.readAsInt(key,bOffset,4);
    int c = Bytes.readAsInt(key,cOffset,4);
    int d = Bytes.readAsInt(key,dOffset,4);
    assertTrue(a == 1 &&
               b > 10 &&
               b < 20 &&
               c > 90 &&
               c < 100 &&
               d == 1);
  }
  ht.close();
}
 
Example #7
Source Project: phoenix-tephra   Author: apache   File: TransactionAwareHTableTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCheckpointRollback() throws Exception {
  // start a transaction, using checkpoints between writes
  transactionContext.start();
  transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
  transactionContext.checkpoint();
  transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
  transactionContext.checkpoint();
  transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));

  transactionContext.abort();

  transactionContext.start();
  verifyRow(transactionAwareHTable, TestBytes.row, null);
  verifyRow(transactionAwareHTable, TestBytes.row2, null);
  verifyRow(transactionAwareHTable, TestBytes.row3, null);

  Scan scan = new Scan();
  ResultScanner scanner = transactionAwareHTable.getScanner(scan);
  assertNull(scanner.next());
  scanner.close();
  transactionContext.finish();
}
 
Example #8
Source Project: phoenix   Author: cloudera-labs   File: WhereCompilerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBetweenFilter() throws SQLException {
    String tenantId = "000000000000001";
    String query = "select * from atable where organization_id='" + tenantId + "' and a_integer between 0 and 10";
    PhoenixConnection pconn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class);
    PhoenixPreparedStatement pstmt = newPreparedStatement(pconn, query);
    QueryPlan plan = pstmt.optimizeQuery();
    Scan scan = plan.getContext().getScan();
    Filter filter = scan.getFilter();
    assertEquals(
            singleKVFilter(and(
                constantComparison(
                    CompareOp.GREATER_OR_EQUAL,
                    A_INTEGER,
                    0),
                constantComparison(
                    CompareOp.LESS_OR_EQUAL,
                    A_INTEGER,
                    10))),
            filter);
}
 
Example #9
Source Project: phoenix   Author: apache   File: WhereOptimizerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRVCExpressionThroughOr() throws SQLException {
    String tenantId =  "000000000000001";
    String entityId =  "002333333333331";
    String entityId1 = "002333333333330";
    String entityId2 = "002333333333332";
    String query = "select * from atable where (organization_id,entity_id) >= (?,?) and organization_id = ? and  (entity_id = ? or entity_id = ?)";
    List<Object> binds = Arrays.<Object>asList(tenantId, entityId, tenantId, entityId1, entityId2);
    StatementContext context = compileStatement(query, binds);
    Scan scan = context.getScan();
    byte[] expectedStartRow = ByteUtil.concat(PVarchar.INSTANCE.toBytes(tenantId), PVarchar.INSTANCE.toBytes(entityId1));
    byte[] expectedStopRow = ByteUtil.concat(PVarchar.INSTANCE.toBytes(tenantId), PVarchar.INSTANCE.toBytes(entityId2), QueryConstants.SEPARATOR_BYTE_ARRAY);
    assertArrayEquals(expectedStartRow, scan.getStartRow());
    assertArrayEquals(expectedStopRow, scan.getStopRow());
    Filter filter = scan.getFilter();
    assertTrue(filter instanceof SkipScanFilter);
    SkipScanFilter skipScanFilter = (SkipScanFilter)filter;
    List<List<KeyRange>> skipScanRanges = Arrays.asList(
            Arrays.asList(KeyRange.getKeyRange(ByteUtil.concat(PVarchar.INSTANCE.toBytes(tenantId), PVarchar.INSTANCE.toBytes(entityId1))),
                          KeyRange.getKeyRange(ByteUtil.concat(PVarchar.INSTANCE.toBytes(tenantId), PVarchar.INSTANCE.toBytes(entityId2)))));
    assertEquals(skipScanRanges, skipScanFilter.getSlots());
}
 
Example #10
Source Project: phoenix   Author: forcedotcom   File: WhereClauseOptimizerTest.java    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test
public void testLikeExtractKeyExpression2() throws SQLException {
    String tenantId = "000000000000001";
    String keyPrefix = "002";
    // TODO: verify that _ at end of like doesn't go to equals
    String query = "select * from atable where organization_id = ? and entity_id  LIKE '" + keyPrefix + "_'";
    Scan scan = new Scan();
    List<Object> binds = Arrays.<Object>asList(tenantId);
    compileStatement(query, scan, binds);

    assertNotNull(scan.getFilter());

    byte[] startRow = ByteUtil.concat(PDataType.VARCHAR.toBytes(tenantId),ByteUtil.fillKey(PDataType.VARCHAR.toBytes(keyPrefix),15));
    assertArrayEquals(startRow, scan.getStartRow());
    byte[] stopRow = ByteUtil.concat(PDataType.VARCHAR.toBytes(tenantId),ByteUtil.fillKey(ByteUtil.nextKey(PDataType.VARCHAR.toBytes(keyPrefix)),15));
    assertArrayEquals(stopRow, scan.getStopRow());
}
 
Example #11
Source Project: phoenix   Author: apache   File: WhereOptimizerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMultiKeyBindExpression() throws SQLException {
    String tenantId = "000000000000001";
    String keyPrefix = "002";
    String query = "select * from atable where organization_id=? and substr(entity_id,1,3)=?";
    List<Object> binds = Arrays.<Object>asList(tenantId,keyPrefix);
    Scan scan = compileStatement(query, binds).getScan();

    assertNull(scan.getFilter());
    byte[] startRow = ByteUtil.concat(
        PVarchar.INSTANCE.toBytes(tenantId),StringUtil.padChar(PVarchar.INSTANCE.toBytes(keyPrefix),15));
    assertArrayEquals(startRow, scan.getStartRow());
    byte[] stopRow = ByteUtil.concat(
        PVarchar.INSTANCE.toBytes(tenantId),StringUtil.padChar(ByteUtil.nextKey(PVarchar.INSTANCE.toBytes(keyPrefix)),15));
    assertArrayEquals(stopRow, scan.getStopRow());
}
 
Example #12
Source Project: phoenix-tephra   Author: apache   File: DataJanitorState.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Delete prune upper bounds for the regions that are not in the given exclude set, and the
 * prune upper bound is less than the given value.
 * After the invalid list is pruned up to deletionPruneUpperBound, we do not need entries for regions that have
 * prune upper bound less than deletionPruneUpperBound. We however limit the deletion to only regions that are
 * no longer in existence (due to deletion, etc.), to avoid update/delete race conditions.
 *
 * @param deletionPruneUpperBound prune upper bound below which regions will be deleted
 * @param excludeRegions set of regions that should not be deleted
 * @throws IOException when not able to delete data in HBase
 */
public void deletePruneUpperBounds(long deletionPruneUpperBound, SortedSet<byte[]> excludeRegions)
  throws IOException {
  try (Table stateTable = stateTableSupplier.get()) {
    byte[] startRow = makeRegionKey(EMPTY_BYTE_ARRAY);
    Scan scan = new Scan(startRow, REGION_KEY_PREFIX_STOP);
    scan.addColumn(FAMILY, PRUNE_UPPER_BOUND_COL);

    try (ResultScanner scanner = stateTable.getScanner(scan)) {
      Result next;
      while ((next = scanner.next()) != null) {
        byte[] region = getRegionFromKey(next.getRow());
        if (!excludeRegions.contains(region)) {
          byte[] timeBytes = next.getValue(FAMILY, PRUNE_UPPER_BOUND_COL);
          if (timeBytes != null) {
            long pruneUpperBoundRegion = Bytes.toLong(timeBytes);
            if (pruneUpperBoundRegion < deletionPruneUpperBound) {
              stateTable.delete(new Delete(next.getRow()));
            }
          }
        }
      }
    }
  }
}
 
Example #13
Source Project: phoenix   Author: apache   File: WhereCompilerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAndFilter() throws SQLException {
    String tenantId = "000000000000001";
    String query = "select * from atable where organization_id=? and a_integer=0 and a_string='foo'";
    List<Object> binds = Arrays.<Object>asList(tenantId);

    PhoenixConnection pconn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class);
    PhoenixPreparedStatement pstmt = newPreparedStatement(pconn, query);
    bindParams(pstmt, binds);
    QueryPlan plan = pstmt.optimizeQuery();
    Scan scan = plan.getContext().getScan();
    Filter filter = scan.getFilter();

    assertEquals(
        multiEncodedKVFilter(and(
            constantComparison(
                CompareOp.EQUAL,
                A_INTEGER,
                0),
            constantComparison(
                CompareOp.EQUAL,
                A_STRING,
                "foo")), TWO_BYTE_QUALIFIERS),
        filter);
}
 
Example #14
Source Project: hbase   Author: apache   File: TestServerSideScanMetricsFromClientSide.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @return The approximate heap size of a cell in the test table. All cells should have
 *         approximately the same heap size, so the value is cached to avoid repeating the
 *         calculation
 * @throws Exception on unexpected failure
 */
private long getCellHeapSize() throws Exception {
  if (CELL_HEAP_SIZE == -1) {
    // Do a partial scan that will return a single result with a single cell
    Scan scan = new Scan();
    scan.setMaxResultSize(1);
    scan.setAllowPartialResults(true);
    ResultScanner scanner = TABLE.getScanner(scan);

    Result result = scanner.next();

    assertTrue(result != null);
    assertTrue(result.rawCells() != null);
    assertTrue(result.rawCells().length == 1);

    CELL_HEAP_SIZE = result.rawCells()[0].heapSize();
    scanner.close();
  }

  return CELL_HEAP_SIZE;
}
 
Example #15
Source Project: phoenix   Author: cloudera-labs   File: WhereOptimizerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNullAtStartOfRVC() throws SQLException {
    String tenantId = null;
    String parentId = "000000000000002";
    Date createdDate = new Date(System.currentTimeMillis());
    
    String query = "select * from entity_history where (organization_id, parent_id, created_date) >= (?,?,?)";
    List<Object> binds = Arrays.<Object>asList(tenantId, parentId, createdDate);
    StatementContext context = compileStatement(query, binds);
    Scan scan = context.getScan();
    Filter filter = scan.getFilter();
    assertNull(filter);
    byte[] expectedStartRow = ByteUtil.concat(new byte[15], ByteUtil.previousKey(PChar.INSTANCE.toBytes(parentId)), PDate.INSTANCE.toBytes(createdDate));
    assertArrayEquals(expectedStartRow, scan.getStartRow());
    assertArrayEquals(HConstants.EMPTY_END_ROW, scan.getStopRow());
}
 
Example #16
Source Project: hbase   Author: apache   File: TestReplicationSink.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Insert a mix of puts and deletes
 * @throws Exception
 */
@Test
public void testMixedPutDelete() throws Exception {
  List<WALEntry> entries = new ArrayList<>(BATCH_SIZE/2);
  List<Cell> cells = new ArrayList<>();
  for(int i = 0; i < BATCH_SIZE/2; i++) {
    entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells));
  }
  SINK.replicateEntries(entries, CellUtil.createCellScanner(cells), replicationClusterId,
    baseNamespaceDir, hfileArchiveDir);

  entries = new ArrayList<>(BATCH_SIZE);
  cells = new ArrayList<>();
  for(int i = 0; i < BATCH_SIZE; i++) {
    entries.add(createEntry(TABLE_NAME1, i,
        i % 2 != 0 ? KeyValue.Type.Put: KeyValue.Type.DeleteColumn, cells));
  }

  SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()),
    replicationClusterId, baseNamespaceDir, hfileArchiveDir);
  Scan scan = new Scan();
  ResultScanner scanRes = table1.getScanner(scan);
  assertEquals(BATCH_SIZE/2, scanRes.next(BATCH_SIZE).length);
}
 
Example #17
Source Project: phoenix   Author: apache   File: StatementContext.java    License: Apache License 2.0 5 votes vote down vote up
public StatementContext(PhoenixStatement statement, ColumnResolver resolver, BindManager binds, Scan scan, SequenceManager seqManager, boolean isRequestMetricsEnabled) {
    this.statement = statement;
    this.resolver = resolver;
    this.scan = scan;
    this.sequences = seqManager;
    this.binds = binds;
    this.aggregates = new AggregationManager();
    this.expressions = new ExpressionManager();
    PhoenixConnection connection = statement.getConnection();
    ReadOnlyProps props = connection.getQueryServices().getProps();
    String timeZoneID = props.get(QueryServices.DATE_FORMAT_TIMEZONE_ATTRIB,
            DateUtil.DEFAULT_TIME_ZONE_ID);
    this.dateFormat = props.get(QueryServices.DATE_FORMAT_ATTRIB, DateUtil.DEFAULT_DATE_FORMAT);
    this.dateFormatter = DateUtil.getDateFormatter(dateFormat, timeZoneID);
    this.timeFormat = props.get(QueryServices.TIME_FORMAT_ATTRIB, DateUtil.DEFAULT_TIME_FORMAT);
    this.timeFormatter = DateUtil.getTimeFormatter(timeFormat, timeZoneID);
    this.timestampFormat = props.get(QueryServices.TIMESTAMP_FORMAT_ATTRIB, DateUtil.DEFAULT_TIMESTAMP_FORMAT);
    this.timestampFormatter = DateUtil.getTimestampFormatter(timestampFormat, timeZoneID);
    this.dateFormatTimeZone = DateUtil.getTimeZone(timeZoneID);
    this.numberFormat = props.get(QueryServices.NUMBER_FORMAT_ATTRIB, NumberUtil.DEFAULT_NUMBER_FORMAT);
    this.tempPtr = new ImmutableBytesWritable();
    this.currentTable = resolver != null && !resolver.getTables().isEmpty() ? resolver.getTables().get(0) : null;
    this.whereConditionColumns = new ArrayList<Pair<byte[], byte[]>>();
    this.dataColumns = this.currentTable == null ? Collections.<PColumn, Integer> emptyMap() : Maps
            .<PColumn, Integer> newLinkedHashMap();
    this.subqueryResults = Maps.<SelectStatement, Object> newHashMap();
    this.readMetricsQueue = new ReadMetricQueue(isRequestMetricsEnabled,connection.getLogLevel());
    this.overAllQueryMetrics = new OverAllQueryMetrics(isRequestMetricsEnabled,connection.getLogLevel());
    this.retryingPersistentCache = Maps.<Long, Boolean> newHashMap();
}
 
Example #18
Source Project: phoenix-tephra   Author: apache   File: TransactionAwareHTableTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCheckpointInvalidate() throws Exception {
  // start a transaction, using checkpoints between writes
  transactionContext.start();
  Transaction origTx = transactionContext.getCurrentTransaction();
  transactionAwareHTable.put(new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
  transactionContext.checkpoint();
  Transaction checkpointTx1 = transactionContext.getCurrentTransaction();
  transactionAwareHTable.put(new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
  transactionContext.checkpoint();
  Transaction checkpointTx2 = transactionContext.getCurrentTransaction();
  transactionAwareHTable.put(new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));

  TransactionSystemClient txClient = new InMemoryTxSystemClient(txManager);
  txClient.invalidate(transactionContext.getCurrentTransaction().getTransactionId());

  // check that writes are not visible
  TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
  TransactionContext txContext2 = new TransactionContext(txClient, txTable2);
  txContext2.start();
  Transaction newTx = txContext2.getCurrentTransaction();

  // all 3 writes pointers from the previous transaction should now be excluded
  assertTrue(newTx.isExcluded(origTx.getWritePointer()));
  assertTrue(newTx.isExcluded(checkpointTx1.getWritePointer()));
  assertTrue(newTx.isExcluded(checkpointTx2.getWritePointer()));

  verifyRow(txTable2, TestBytes.row, null);
  verifyRow(txTable2, TestBytes.row2, null);
  verifyRow(txTable2, TestBytes.row3, null);

  Scan scan = new Scan();
  ResultScanner scanner = txTable2.getScanner(scan);
  assertNull(scanner.next());
  scanner.close();
  txContext2.finish();
}
 
Example #19
Source Project: nifi   Author: apache   File: HBase_2_ClientService.java    License: Apache License 2.0 5 votes vote down vote up
protected ResultScanner getResults(final Table table, final Collection<Column> columns, final Filter filter, final long minTime, List<String> authorizations) throws IOException {
    // Create a new scan. We will set the min timerange as the latest timestamp that
    // we have seen so far. The minimum timestamp is inclusive, so we will get duplicates.
    // We will record any cells that have the latest timestamp, so that when we scan again,
    // we know to throw away those duplicates.
    final Scan scan = new Scan();
    scan.setTimeRange(minTime, Long.MAX_VALUE);

    if (authorizations != null && authorizations.size() > 0) {
        scan.setAuthorizations(new Authorizations(authorizations));
    }

    if (filter != null) {
        scan.setFilter(filter);
    }

    if (columns != null) {
        for (Column col : columns) {
            if (col.getQualifier() == null) {
                scan.addFamily(col.getFamily());
            } else {
                scan.addColumn(col.getFamily(), col.getQualifier());
            }
        }
    }

    return table.getScanner(scan);
}
 
Example #20
Source Project: phoenix   Author: apache   File: BaseResultIterators.java    License: Apache License 2.0 5 votes vote down vote up
public ScanLocator(Scan scan, int outerListIndex, int innerListIndex, boolean isFirstScan, boolean isLastScan) {
	this.outerListIndex = outerListIndex;
	this.innerListIndex = innerListIndex;
	this.scan = scan;
	this.isFirstScan = isFirstScan;
	this.isLastScan = isLastScan;
}
 
Example #21
Source Project: hbase   Author: apache   File: TestStoreScanner.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWontNextToNext() throws IOException {
  // build the scan file:
  KeyValue [] kvs = new KeyValue[] {
      create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "a", 1, KeyValue.Type.Put, "dont-care")
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);

  Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1"));
  // this only uses maxVersions (default=1) and TimeRange (default=all)
  try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, getCols("a"), scanners)) {
    List<Cell> results = new ArrayList<>();
    scan.next(results);
    assertEquals(1, results.size());
    assertEquals(kvs[0], results.get(0));
    // should be ok...
    // now scan _next_ again.
    results.clear();
    scan.next(results);
    assertEquals(1, results.size());
    assertEquals(kvs[2], results.get(0));

    results.clear();
    scan.next(results);
    assertEquals(0, results.size());
  }
}
 
Example #22
Source Project: wifi   Author: gglinux   File: HBaseTable.java    License: Apache License 2.0 5 votes vote down vote up
public static void scan() throws Exception {
	HTable table = new HTable(cfg, tableName);
	Scan s = new Scan();
	ResultScanner rs = table.getScanner(s);
	for (Result r : rs) {
		System.out.println("Scan: " + r);
	}
}
 
Example #23
Source Project: phoenix   Author: cloudera-labs   File: WhereOptimizerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testLiteralConcatExpression() throws SQLException {
    String query = "select * from atable where null||'foo'||'bar' = 'foobar'";
    Scan scan = new Scan();
    List<Object> binds = Collections.emptyList();
    compileStatement(query, binds);

    assertNull(scan.getFilter());
    assertEquals(0, scan.getStartRow().length);
    assertEquals(0, scan.getStopRow().length);
}
 
Example #24
Source Project: phoenix   Author: forcedotcom   File: DefaultParallelIteratorRegionSplitter.java    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
protected List<HRegionLocation> getAllRegions() throws SQLException {
    Scan scan = context.getScan();
    PTable table = tableRef.getTable();
    List<HRegionLocation> allTableRegions = context.getConnection().getQueryServices().getAllTableRegions(table.getPhysicalName().getBytes());
    // If we're not salting, then we've already intersected the minMaxRange with the scan range
    // so there's nothing to do here.
    return filterRegions(allTableRegions, scan.getStartRow(), scan.getStopRow());
}
 
Example #25
Source Project: hbase   Author: apache   File: TestMasterTransitions.java    License: Apache License 2.0 5 votes vote down vote up
private static int addToEachStartKey(final int expected) throws IOException {
  Table t = TEST_UTIL.getConnection().getTable(TABLENAME);
  Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
  int rows = 0;
  Scan scan = new Scan();
  scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
  ResultScanner s = meta.getScanner(scan);
  for (Result r = null; (r = s.next()) != null;) {
    RegionInfo hri = CatalogFamilyFormat.getRegionInfo(r);
    if (hri == null) break;
    if (!hri.getTable().equals(TABLENAME)) {
      continue;
    }

    // If start key, add 'aaa'.
    if(!hri.getTable().equals(TABLENAME)) {
      continue;
    }
    byte [] row = getStartKey(hri);
    Put p = new Put(row);
    p.setDurability(Durability.SKIP_WAL);
    p.addColumn(getTestFamily(), getTestQualifier(), row);
    t.put(p);
    rows++;
  }
  s.close();
  Assert.assertEquals(expected, rows);
  t.close();
  meta.close();
  return rows;
}
 
Example #26
Source Project: phoenix   Author: apache   File: LimitCompilerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testLimit() throws SQLException {
    String tenantId = "000000000000001";
    String query = "select * from atable where organization_id='" + tenantId + "' limit 5";
    List<Object> binds = Collections.emptyList();
    QueryPlan plan = compileStatement(query, binds);
    Scan scan = plan.getContext().getScan();
    
    assertNull(scan.getFilter());
    assertArrayEquals(PVarchar.INSTANCE.toBytes(tenantId), scan.getStartRow());
    assertArrayEquals(ByteUtil.nextKey(PVarchar.INSTANCE.toBytes(tenantId)), scan.getStopRow());
    assertEquals(plan.getLimit(),Integer.valueOf(5));
}
 
Example #27
Source Project: SpyGlass   Author: ParallelAI   File: GenerateTestingHTables.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Method to print-out an HTable
 */
private static void printHTable(TestingTable testingTable)
		throws IOException {

	HTable table = new HTable(config, testingTable.name());

	Scan s = new Scan();
	// Let scanner know which columns we are interested in
	ResultScanner scanner = table.getScanner(s);

	LOG.info("Printing HTable: " + Bytes.toString(table.getTableName()));

	try {
		// Iterate results
		for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
			String key = Bytes.toString(rr.getRow());
			Iterator<KeyValue> iter = rr.list().iterator();

			String header = "Key:\t";
			String data = key + "\t";

			while (iter.hasNext()) {
				KeyValue kv = iter.next();
				header += Bytes.toString(kv.getFamily()) + ":"
						+ Bytes.toString(kv.getQualifier()) + "\t";
				data += Bytes.toString(kv.getValue()) + "\t";
			}

			LOG.info(header);
			LOG.info(data);
		}
		System.out.println();
	} finally {
		// Make sure you close your scanners when you are done!
		// Thats why we have it inside a try/finally clause
		scanner.close();
		table.close();
	}
}
 
Example #28
Source Project: phoenix   Author: apache   File: ParallelIteratorsSplitTest.java    License: Apache License 2.0 5 votes vote down vote up
private static Collection<?> foreach(KeyRange[][] ranges, int[] widths, KeyRange[] expectedSplits) {
    RowKeySchema schema = buildSchema(widths);
    List<List<KeyRange>> slots = Lists.transform(Lists.newArrayList(ranges), ARRAY_TO_LIST);
    SkipScanFilter filter = new SkipScanFilter(slots, schema);
    // Always set start and stop key to max to verify we are using the information in skipscan
    // filter over the scan's KMIN and KMAX.
    Scan scan = new Scan().setFilter(filter);
    ScanRanges scanRanges = ScanRanges.createSingleSpan(schema, slots);
    List<Object> ret = Lists.newArrayList();
    ret.add(new Object[] {scan, scanRanges, Arrays.<KeyRange>asList(expectedSplits)});
    return ret;
}
 
Example #29
Source Project: phoenix   Author: forcedotcom   File: WhereClauseCompileTest.java    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Test
public void testSecondPkColInListFilter() throws SQLException {
    String tenantId = "000000000000001";
    String entityId1 = "00000000000000X";
    String entityId2 = "00000000000000Y";
    String query = String.format("select * from %s where organization_id='%s' AND entity_id IN ('%s','%s')",
            ATABLE_NAME, tenantId, entityId1, entityId2);
    PhoenixConnection pconn = DriverManager.getConnection(getUrl(), TEST_PROPERTIES).unwrap(PhoenixConnection.class);
    PhoenixPreparedStatement pstmt = new PhoenixPreparedStatement(pconn, query);
    QueryPlan plan = pstmt.optimizeQuery();
    Scan scan = plan.getContext().getScan();
    byte[] startRow = PDataType.VARCHAR.toBytes(tenantId + entityId1);
    assertArrayEquals(startRow, scan.getStartRow());
    byte[] stopRow = PDataType.VARCHAR.toBytes(tenantId + entityId2);
    assertArrayEquals(ByteUtil.nextKey(stopRow), scan.getStopRow());

    Filter filter = scan.getFilter();

    assertEquals(
        new SkipScanFilter(
            ImmutableList.of(
                Arrays.asList(pointRange(tenantId)),
                Arrays.asList(
                    pointRange(entityId1),
                    pointRange(entityId2))),
            plan.getContext().getResolver().getTables().get(0).getTable().getRowKeySchema()),
        filter);
}
 
Example #30
Source Project: phoenix   Author: forcedotcom   File: GroupedAggregateRegionObserver.java    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
/**
 * Replaces the RegionScanner s with a RegionScanner that groups by the key formed by the list
 * of expressions from the scan and returns the aggregated rows of each group. For example,
 * given the following original rows in the RegionScanner: KEY COL1 row1 a row2 b row3 a row4 a
 * the following rows will be returned for COUNT(*): KEY COUNT a 3 b 1 The client is required to
 * do a sort and a final aggregation, since multiple rows with the same key may be returned from
 * different regions.
 */
@Override
protected RegionScanner doPostScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
        Scan scan, RegionScanner s) throws IOException {
    boolean keyOrdered = false;
    byte[] expressionBytes = scan.getAttribute(UNORDERED_GROUP_BY_EXPRESSIONS);

    if (expressionBytes == null) {
        expressionBytes = scan.getAttribute(KEY_ORDERED_GROUP_BY_EXPRESSIONS);
        if (expressionBytes == null) {
            return s;
        }
        keyOrdered = true;
    }
    List<Expression> expressions = deserializeGroupByExpressions(expressionBytes);

    ServerAggregators aggregators =
            ServerAggregators.deserialize(scan
                    .getAttribute(GroupedAggregateRegionObserver.AGGREGATORS), c
                    .getEnvironment().getConfiguration());

    final ScanProjector p = ScanProjector.deserializeProjectorFromScan(scan);
    final HashJoinInfo j = HashJoinInfo.deserializeHashJoinFromScan(scan);
    RegionScanner innerScanner = s;
    if (p != null || j != null) {
        innerScanner =
                new HashJoinRegionScanner(s, p, j, ScanUtil.getTenantId(scan),
                        c.getEnvironment());
    }

    if (keyOrdered) { // Optimize by taking advantage that the rows are
                      // already in the required group by key order
        return scanOrdered(c, scan, innerScanner, expressions, aggregators);
    } else { // Otherwse, collect them all up in an in memory map
        return scanUnordered(c, scan, innerScanner, expressions, aggregators);
    }
}