Java Code Examples for org.apache.hadoop.hbase.client.Scan#setAttribute()

/**
 * 初始化scan集合
 * 
 * @param job
 * @return
 */
private List<Scan> initScans(Job job) {
    Configuration conf = job.getConfiguration();
    // 获取运行时间: yyyy-MM-dd
    String date = conf.get(GlobalConstants.RUNNING_DATE_PARAMES);
    long startDate = TimeUtil.parseString2Long(date);
    long endDate = startDate + GlobalConstants.DAY_OF_MILLISECONDS;

    Scan scan = new Scan();
    // 定义hbase扫描的开始rowkey和结束rowkey
    scan.setStartRow(Bytes.toBytes("" + startDate));
    scan.setStopRow(Bytes.toBytes("" + endDate));

    FilterList filterList = new FilterList();
    // 定义mapper中需要获取的列名
    String[] columns = new String[] { 
    		EventLogConstants.LOG_COLUMN_NAME_UUID, // 用户id
            EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, // 服务器时间
            EventLogConstants.LOG_COLUMN_NAME_PLATFORM, // 平台名称
            EventLogConstants.LOG_COLUMN_NAME_BROWSER_NAME, // 浏览器名称
            EventLogConstants.LOG_COLUMN_NAME_BROWSER_VERSION // 浏览器版本号
    };
    filterList.addFilter(this.getColumnFilter(columns));

    scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(EventLogConstants.HBASE_NAME_EVENT_LOGS));
    scan.setFilter(filterList);
    return Lists.newArrayList(scan);
}
 

private PeekingResultIterator nextIterator() throws SQLException {
    if (index >= scans.size()) {
        return EMPTY_ITERATOR;
    }
    ReadMetricQueue readMetrics = context.getReadMetricsQueue();
    while (index < scans.size()) {
        Scan currentScan = scans.get(index++);
        if (remainingOffset != null) {
            currentScan.setAttribute(BaseScannerRegionObserver.SCAN_OFFSET, PInteger.INSTANCE.toBytes(remainingOffset));
        }
        ScanMetricsHolder scanMetricsHolder =
                ScanMetricsHolder.getInstance(readMetrics, tableName, currentScan,
                    context.getConnection().getLogLevel());
        TableResultIterator itr =
                new TableResultIterator(mutationState, currentScan, scanMetricsHolder,
                        renewLeaseThreshold, plan, scanGrouper, caches);
        PeekingResultIterator peekingItr = iteratorFactory.newIterator(context, itr, currentScan, tableName, plan);
        Tuple tuple;
        if ((tuple = peekingItr.peek()) == null) {
            peekingItr.close();
            continue;
        } else if ((remainingOffset = QueryUtil.getRemainingOffset(tuple)) != null) {
            peekingItr.next();
            peekingItr.close();
            continue;
        }
        context.getConnection().addIteratorForLeaseRenewal(itr);
        return peekingItr;
    }
    return EMPTY_ITERATOR;
}
 

/**
 * Set the scan attribute
 *
 * @param reversed if true, scan will be backward order
 * @param mobScanRaw if true, scan will get the mob reference
 */
public void setScan(Scan scan, boolean reversed, boolean mobScanRaw) {
  scan.setReversed(reversed);
  scan.readVersions(4);
  if(mobScanRaw) {
    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
  }
}
 

public static void serializeProjectorIntoScan(Scan scan, TupleProjector projector,
        boolean projectDynColsInWildcardQueries) {
    scan.setAttribute(SCAN_PROJECTOR, serializeProjectorIntoBytes(projector));
    if (projectDynColsInWildcardQueries) {
        scan.setAttribute(WILDCARD_SCAN_INCLUDES_DYNAMIC_COLUMNS, TRUE_BYTES);
    }
}
 

/**
 * Tests a MR scan using specific start and stop rows.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
private void testScan(String start, String stop, String last)
    throws IOException, InterruptedException, ClassNotFoundException {
  String jobName =
      "Scan" + (start != null ? start.toUpperCase(Locale.ROOT) : "Empty") + "To" +
          (stop != null ? stop.toUpperCase(Locale.ROOT) : "Empty");
  LOG.info("Before map/reduce startup - job " + jobName);
  Configuration c = new Configuration(TEST_UTIL.getConfiguration());

  c.set(KEY_STARTROW, start != null ? start : "");
  c.set(KEY_LASTROW, last != null ? last : "");

  List<Scan> scans = new ArrayList<>();

  for (String tableName : TABLES) {
    Scan scan = new Scan();

    scan.addFamily(INPUT_FAMILY);
    scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(tableName));

    if (start != null) {
      scan.withStartRow(Bytes.toBytes(start));
    }
    if (stop != null) {
      scan.withStopRow(Bytes.toBytes(stop));
    }

    scans.add(scan);

    LOG.info("scan before: " + scan);
  }

  runJob(jobName, c, scans);
}
 

@Test
@Ignore
public void validateAccurateRecordsWithStoreFileAndMemstore() throws SQLException, IOException, InterruptedException{
    int i=0;
    TableName tableName=TableName.valueOf(sqlUtil.getConglomID(SCHEMA_NAME+".A"));
    try(Admin admin = connection.getAdmin()) {
        Table table = connection.getTable(tableName);
        Scan scan=new Scan();
        scan.setCaching(50);
        scan.setBatch(50);
        scan.setMaxVersions();
        scan.setAttribute(MRConstants.SPLICE_SCAN_MEMSTORE_ONLY,HConstants.EMPTY_BYTE_ARRAY);
        try(SkeletonClientSideRegionScanner clientSideRegionScanner=
                    new HBaseClientSideRegionScanner(table,
                          table.getConfiguration(), FSUtils.getCurrentFileSystem(table.getConfiguration()),
                          FSUtils.getRootDir(table.getConfiguration()),
                          table.getTableDescriptor(),
                          connection.getRegionLocator(tableName).getRegionLocation(scan.getStartRow()).getRegionInfo(),
                          scan,
                          connection.getRegionLocator(tableName).getRegionLocation(scan.getStartRow()).getHostnamePort())){
            List results=new ArrayList();
            while(clientSideRegionScanner.nextRaw(results)){
                i++;
                results.clear();
            }
        }
        Assert.assertEquals("Results Returned Are Not Accurate",500,i);
    }
}
 

@Override
public void preScannerOpen(final Scan scan) {
  final int maxVersions = scan.getMaxVersions();
  if ((maxVersions > 0) && (maxVersions < Integer.MAX_VALUE)) {
    scan.setAttribute(OLD_MAX_VERSIONS_KEY, ByteBuffer.allocate(4).putInt(maxVersions).array());
  }
  scan.setMaxVersions();
}
 

private static Scan mockScan(byte[] startKey, byte[] endKey) {
    Scan scan = new StubInternalScanner();
    scan.setAttribute(MRConstants.SPLICE_SCAN_MEMSTORE_ONLY, SIConstants.TRUE_BYTES);
    scan.setAttribute(ClientRegionConstants.SPLICE_SCAN_MEMSTORE_PARTITION_BEGIN_KEY, startKey);
    scan.setAttribute(ClientRegionConstants.SPLICE_SCAN_MEMSTORE_PARTITION_END_KEY, endKey);
    return scan;
}
 

/**
 * Test getSplits only puts up one Connection.
 * In past it has put up many Connections. Each Connection setup comes with a fresh new cache
 * so we have to do fresh hit on hbase:meta. Should only do one Connection when doing getSplits
 * even if a MultiTableInputFormat.
 * @throws IOException
 */
@Test
public void testMRSplitsConnectionCount() throws IOException {
  // Make instance of MTIFB.
  MultiTableInputFormatBase mtif = new MultiTableInputFormatBase() {
    @Override
    public RecordReader<ImmutableBytesWritable, Result> createRecordReader(InputSplit split,
        TaskAttemptContext context)
    throws IOException, InterruptedException {
      return super.createRecordReader(split, context);
    }
  };
  // Pass it a mocked JobContext. Make the JC return our Configuration.
  // Load the Configuration so it returns our special Connection so we can interpolate
  // canned responses.
  JobContext mockedJobContext = Mockito.mock(JobContext.class);
  Configuration c = HBaseConfiguration.create();
  c.set(ConnectionUtils.HBASE_CLIENT_CONNECTION_IMPL, MRSplitsConnection.class.getName());
  Mockito.when(mockedJobContext.getConfiguration()).thenReturn(c);
  // Invent a bunch of scans. Have each Scan go against a different table so a good spread.
  List<Scan> scans = new ArrayList<>();
  for (int i = 0; i < 10; i++) {
    Scan scan = new Scan();
    String tableName = this.name.getMethodName() + i;
    scan.setAttribute(SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(tableName));
    scans.add(scan);
  }
  mtif.setScans(scans);
  // Get splits. Assert that that more than one.
  List<InputSplit> splits = mtif.getSplits(mockedJobContext);
  Assert.assertTrue(splits.size() > 0);
  // Assert only one Connection was made (see the static counter we have in the mocked
  // Connection MRSplitsConnection Constructor.
  Assert.assertEquals(1, MRSplitsConnection.creations.get());
}
 

/**
     * 初始化scan集合
     * 
     * @param job
     * @return
     */
    private List<Scan> initScans(Job job) {
        // 时间戳+....
        Configuration conf = job.getConfiguration();
        // 获取运行时间: yyyy-MM-dd
        String date = conf.get(GlobalConstants.RUNNING_DATE_PARAMES);
        long startDate = TimeUtil.parseString2Long(date);
        long endDate = startDate + GlobalConstants.DAY_OF_MILLISECONDS;

        Scan scan = new Scan();
        // 定义hbase扫描的开始rowkey和结束rowkey
        scan.setStartRow(Bytes.toBytes("" + startDate));
        scan.setStopRow(Bytes.toBytes("" + endDate));
        
        FilterList filterList = new FilterList();
        // 过滤数据，只分析launch事件
        filterList.addFilter(new SingleColumnValueFilter(Bytes.toBytes(EventLogConstants.EVENT_LOGS_FAMILY_NAME), Bytes.toBytes(EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME), CompareOp.EQUAL, Bytes.toBytes(EventEnum.LAUNCH.alias)));
        // 定义mapper中需要获取的列名
        String[] columns = new String[] {
        			EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME, 
        			EventLogConstants.LOG_COLUMN_NAME_UUID, 
        			EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, 
        			EventLogConstants.LOG_COLUMN_NAME_PLATFORM, 
        			EventLogConstants.LOG_COLUMN_NAME_BROWSER_NAME, 
        			EventLogConstants.LOG_COLUMN_NAME_BROWSER_VERSION };
//        scan.addColumn(family, qualifier)
        filterList.addFilter(this.getColumnFilter(columns));

        scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(EventLogConstants.HBASE_NAME_EVENT_LOGS));
        scan.setFilter(filterList);
        return Lists.newArrayList(scan);
    }
 

public static void serializeIntoScan(Scan scan) {
    scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE);
}
 

public static void setTenantId(Scan scan, byte[] tenantId) {
    scan.setAttribute(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
}
 

public static void setLocalIndex(Scan scan) {
    scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX, PDataType.TRUE_BYTES);
}
 

/**
 * Getting the reference data from files
 * @throws IOException
 */
@Test
public void testGetReferencesFromFiles() throws IOException {
  final Configuration conf = HBaseConfiguration.create();
  init(name.getMethodName(), conf, false);

  //Put data in memstore
  this.store.add(new KeyValue(row, family, qf1, 1, value), null);
  this.store.add(new KeyValue(row, family, qf2, 1, value), null);
  //flush
  flush(1);

  //Add more data
  this.store.add(new KeyValue(row, family, qf3, 1, value), null);
  this.store.add(new KeyValue(row, family, qf4, 1, value), null);
  //flush
  flush(2);

  //Add more data
  this.store.add(new KeyValue(row, family, qf5, 1, value), null);
  this.store.add(new KeyValue(row, family, qf6, 1, value), null);
  //flush
  flush(3);

  Scan scan = new Scan(get);
  scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
  InternalScanner scanner = (InternalScanner) store.getScanner(scan,
    scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
    0);

  List<Cell> results = new ArrayList<>();
  scanner.next(results);
  Collections.sort(results, CellComparatorImpl.COMPARATOR);
  scanner.close();

  //Compare
  Assert.assertEquals(expected.size(), results.size());
  for(int i=0; i<results.size(); i++) {
    Cell cell = results.get(i);
    Assert.assertTrue(MobUtils.isMobReferenceCell(cell));
  }
}
 

private long updateStatisticsInternal(PName physicalName, PTable logicalTable) throws SQLException {
    ReadOnlyProps props = connection.getQueryServices().getProps();
    final long msMinBetweenUpdates = props
            .getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB,
                    props.getLong(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB,
                            QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS) / 2);
    byte[] tenantIdBytes = ByteUtil.EMPTY_BYTE_ARRAY;
    Long scn = connection.getSCN();
    // Always invalidate the cache
    long clientTimeStamp = connection.getSCN() == null ? HConstants.LATEST_TIMESTAMP : scn;
    String query = "SELECT CURRENT_DATE()," + LAST_STATS_UPDATE_TIME + " FROM " + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME
            + " WHERE " + PHYSICAL_NAME + "='" + physicalName.getString() + "' AND " + COLUMN_FAMILY
            + " IS NULL AND " + REGION_NAME + " IS NULL AND " + LAST_STATS_UPDATE_TIME + " IS NOT NULL";
    ResultSet rs = connection.createStatement().executeQuery(query);
    long msSinceLastUpdate = Long.MAX_VALUE;
    if (rs.next()) {
        msSinceLastUpdate = rs.getLong(1) - rs.getLong(2);
    }
    long rowCount = 0;
    if (msSinceLastUpdate >= msMinBetweenUpdates) {
        /*
         * Execute a COUNT(*) through PostDDLCompiler as we need to use the logicalTable passed through,
         * since it may not represent a "real" table in the case of the view indexes of a base table.
         */
        PostDDLCompiler compiler = new PostDDLCompiler(connection);
        TableRef tableRef = new TableRef(null, logicalTable, clientTimeStamp, false);
        MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, null, null, clientTimeStamp);
        Scan scan = plan.getContext().getScan();
        scan.setCacheBlocks(false);
        scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES);
        MutationState mutationState = plan.execute();
        rowCount = mutationState.getUpdateCount();
    }

    /*
     *  Update the stats table so that client will pull the new one with the updated stats.
     *  Even if we don't run the command due to the last update time, invalidate the cache.
     *  This supports scenarios in which a major compaction was manually initiated and the
     *  client wants the modified stats to be reflected immediately.
     */
    connection.getQueryServices().clearTableFromCache(tenantIdBytes,
            Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())),
            Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())), clientTimeStamp);
    return rowCount;
}
 

public static void setReversed(Scan scan) {
    // TODO: set attribute dynamically here to prevent dependency on newer HBase release
    scan.setAttribute(REVERSED_ATTR, PDataType.TRUE_BYTES);
}
 

public static void serializeIntoScan(Scan scan) {
    scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE);
}
 

public static void setScanAttributesForIndexReadRepair(Scan scan, PTable table, PhoenixConnection phoenixConnection) throws SQLException {
    if (table.isTransactional() || table.getType() != PTableType.INDEX) {
        return;
    }
    PTable indexTable = table;
    if (indexTable.getIndexType() != PTable.IndexType.GLOBAL) {
        return;
    }
    String schemaName = indexTable.getParentSchemaName().getString();
    String tableName = indexTable.getParentTableName().getString();
    PTable dataTable;
    try {
        dataTable = PhoenixRuntime.getTable(phoenixConnection, SchemaUtil.getTableName(schemaName, tableName));
    } catch (TableNotFoundException e) {
        // This index table must be being deleted. No need to set the scan attributes
        return;
    }
    // MetaDataClient modifies the index table name for view indexes if the parent view of an index has a child
    // view. This, we need to recreate a PTable object with the correct table name for the rest of this code to work
    if (indexTable.getViewIndexId() != null && indexTable.getName().getString().contains(QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR)) {
        int lastIndexOf = indexTable.getName().getString().lastIndexOf(QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR);
        String indexName = indexTable.getName().getString().substring(lastIndexOf + 1);
        indexTable = PhoenixRuntime.getTable(phoenixConnection, indexName);
    }
    if (!dataTable.getIndexes().contains(indexTable)) {
        return;
    }
    if (scan.getAttribute(PhoenixIndexCodec.INDEX_PROTO_MD) == null) {
        ImmutableBytesWritable ptr = new ImmutableBytesWritable();
        IndexMaintainer.serialize(dataTable, ptr, Collections.singletonList(indexTable), phoenixConnection);
        scan.setAttribute(PhoenixIndexCodec.INDEX_PROTO_MD, ByteUtil.copyKeyBytesIfNecessary(ptr));
    }
    scan.setAttribute(BaseScannerRegionObserver.CHECK_VERIFY_COLUMN, TRUE_BYTES);
    scan.setAttribute(BaseScannerRegionObserver.PHYSICAL_DATA_TABLE_NAME, dataTable.getPhysicalName().getBytes());
    IndexMaintainer indexMaintainer = indexTable.getIndexMaintainer(dataTable, phoenixConnection);
    byte[] emptyCF = indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary();
    byte[] emptyCQ = indexMaintainer.getEmptyKeyValueQualifier();
    scan.setAttribute(BaseScannerRegionObserver.EMPTY_COLUMN_FAMILY_NAME, emptyCF);
    scan.setAttribute(BaseScannerRegionObserver.EMPTY_COLUMN_QUALIFIER_NAME, emptyCQ);
    if (scan.getAttribute(BaseScannerRegionObserver.VIEW_CONSTANTS) == null) {
        BaseQueryPlan.serializeViewConstantsIntoScan(scan, dataTable);
    }
    addEmptyColumnToScan(scan, emptyCF, emptyCQ);
}
 

public static void setTenantId(Scan scan, byte[] tenantId) {
    scan.setAttribute(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
}
 

public static void unsetReversed(Scan scan) {
    scan.setAttribute(BaseScannerRegionObserver.REVERSE_SCAN, PDataType.FALSE_BYTES);
    scan.setLoadColumnFamiliesOnDemand(true);
}