Java Code Examples for org.apache.hadoop.hbase.client.Scan#setAttribute()

The following examples show how to use org.apache.hadoop.hbase.client.Scan#setAttribute() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * 初始化scan集合
 * 
 * @param job
 * @return
 */
private List<Scan> initScans(Job job) {
    Configuration conf = job.getConfiguration();
    // 获取运行时间: yyyy-MM-dd
    String date = conf.get(GlobalConstants.RUNNING_DATE_PARAMES);
    long startDate = TimeUtil.parseString2Long(date);
    long endDate = startDate + GlobalConstants.DAY_OF_MILLISECONDS;

    Scan scan = new Scan();
    // 定义hbase扫描的开始rowkey和结束rowkey
    scan.setStartRow(Bytes.toBytes("" + startDate));
    scan.setStopRow(Bytes.toBytes("" + endDate));

    FilterList filterList = new FilterList();
    // 定义mapper中需要获取的列名
    String[] columns = new String[] { 
    		EventLogConstants.LOG_COLUMN_NAME_UUID, // 用户id
            EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, // 服务器时间
            EventLogConstants.LOG_COLUMN_NAME_PLATFORM, // 平台名称
            EventLogConstants.LOG_COLUMN_NAME_BROWSER_NAME, // 浏览器名称
            EventLogConstants.LOG_COLUMN_NAME_BROWSER_VERSION // 浏览器版本号
    };
    filterList.addFilter(this.getColumnFilter(columns));

    scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(EventLogConstants.HBASE_NAME_EVENT_LOGS));
    scan.setFilter(filterList);
    return Lists.newArrayList(scan);
}
 
Example 2
/**
     * 初始化scan集合
     * 
     * @param job
     * @return
     */
    private List<Scan> initScans(Job job) {
        // 时间戳+....
        Configuration conf = job.getConfiguration();
        // 获取运行时间: yyyy-MM-dd
        String date = conf.get(GlobalConstants.RUNNING_DATE_PARAMES);
        long startDate = TimeUtil.parseString2Long(date);
        long endDate = startDate + GlobalConstants.DAY_OF_MILLISECONDS;

        Scan scan = new Scan();
        // 定义hbase扫描的开始rowkey和结束rowkey
        scan.setStartRow(Bytes.toBytes("" + startDate));
        scan.setStopRow(Bytes.toBytes("" + endDate));
        
        FilterList filterList = new FilterList();
        // 过滤数据,只分析launch事件
        filterList.addFilter(new SingleColumnValueFilter(Bytes.toBytes(EventLogConstants.EVENT_LOGS_FAMILY_NAME), Bytes.toBytes(EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME), CompareOp.EQUAL, Bytes.toBytes(EventEnum.LAUNCH.alias)));
        // 定义mapper中需要获取的列名
        String[] columns = new String[] {
        			EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME, 
        			EventLogConstants.LOG_COLUMN_NAME_UUID, 
        			EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, 
        			EventLogConstants.LOG_COLUMN_NAME_PLATFORM, 
        			EventLogConstants.LOG_COLUMN_NAME_BROWSER_NAME, 
        			EventLogConstants.LOG_COLUMN_NAME_BROWSER_VERSION };
//        scan.addColumn(family, qualifier)
        filterList.addFilter(this.getColumnFilter(columns));

        scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(EventLogConstants.HBASE_NAME_EVENT_LOGS));
        scan.setFilter(filterList);
        return Lists.newArrayList(scan);
    }
 
Example 3
/**
 * Test getSplits only puts up one Connection.
 * In past it has put up many Connections. Each Connection setup comes with a fresh new cache
 * so we have to do fresh hit on hbase:meta. Should only do one Connection when doing getSplits
 * even if a MultiTableInputFormat.
 * @throws IOException
 */
@Test
public void testMRSplitsConnectionCount() throws IOException {
  // Make instance of MTIFB.
  MultiTableInputFormatBase mtif = new MultiTableInputFormatBase() {
    @Override
    public RecordReader<ImmutableBytesWritable, Result> createRecordReader(InputSplit split,
        TaskAttemptContext context)
    throws IOException, InterruptedException {
      return super.createRecordReader(split, context);
    }
  };
  // Pass it a mocked JobContext. Make the JC return our Configuration.
  // Load the Configuration so it returns our special Connection so we can interpolate
  // canned responses.
  JobContext mockedJobContext = Mockito.mock(JobContext.class);
  Configuration c = HBaseConfiguration.create();
  c.set(ConnectionUtils.HBASE_CLIENT_CONNECTION_IMPL, MRSplitsConnection.class.getName());
  Mockito.when(mockedJobContext.getConfiguration()).thenReturn(c);
  // Invent a bunch of scans. Have each Scan go against a different table so a good spread.
  List<Scan> scans = new ArrayList<>();
  for (int i = 0; i < 10; i++) {
    Scan scan = new Scan();
    String tableName = this.name.getMethodName() + i;
    scan.setAttribute(SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(tableName));
    scans.add(scan);
  }
  mtif.setScans(scans);
  // Get splits. Assert that that more than one.
  List<InputSplit> splits = mtif.getSplits(mockedJobContext);
  Assert.assertTrue(splits.size() > 0);
  // Assert only one Connection was made (see the static counter we have in the mocked
  // Connection MRSplitsConnection Constructor.
  Assert.assertEquals(1, MRSplitsConnection.creations.get());
}
 
Example 4
private static Scan mockScan(byte[] startKey, byte[] endKey) {
    Scan scan = new StubInternalScanner();
    scan.setAttribute(MRConstants.SPLICE_SCAN_MEMSTORE_ONLY, SIConstants.TRUE_BYTES);
    scan.setAttribute(ClientRegionConstants.SPLICE_SCAN_MEMSTORE_PARTITION_BEGIN_KEY, startKey);
    scan.setAttribute(ClientRegionConstants.SPLICE_SCAN_MEMSTORE_PARTITION_END_KEY, endKey);
    return scan;
}
 
Example 5
Source Project: geowave   File: MergingServerOp.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preScannerOpen(final Scan scan) {
  final int maxVersions = scan.getMaxVersions();
  if ((maxVersions > 0) && (maxVersions < Integer.MAX_VALUE)) {
    scan.setAttribute(OLD_MAX_VERSIONS_KEY, ByteBuffer.allocate(4).putInt(maxVersions).array());
  }
  scan.setMaxVersions();
}
 
Example 6
@Test
@Ignore
public void validateAccurateRecordsWithStoreFileAndMemstore() throws SQLException, IOException, InterruptedException{
    int i=0;
    TableName tableName=TableName.valueOf(sqlUtil.getConglomID(SCHEMA_NAME+".A"));
    try(Admin admin = connection.getAdmin()) {
        Table table = connection.getTable(tableName);
        Scan scan=new Scan();
        scan.setCaching(50);
        scan.setBatch(50);
        scan.setMaxVersions();
        scan.setAttribute(MRConstants.SPLICE_SCAN_MEMSTORE_ONLY,HConstants.EMPTY_BYTE_ARRAY);
        try(SkeletonClientSideRegionScanner clientSideRegionScanner=
                    new HBaseClientSideRegionScanner(table,
                          table.getConfiguration(), FSUtils.getCurrentFileSystem(table.getConfiguration()),
                          FSUtils.getRootDir(table.getConfiguration()),
                          table.getTableDescriptor(),
                          connection.getRegionLocator(tableName).getRegionLocation(scan.getStartRow()).getRegionInfo(),
                          scan,
                          connection.getRegionLocator(tableName).getRegionLocation(scan.getStartRow()).getHostnamePort())){
            List results=new ArrayList();
            while(clientSideRegionScanner.nextRaw(results)){
                i++;
                results.clear();
            }
        }
        Assert.assertEquals("Results Returned Are Not Accurate",500,i);
    }
}
 
Example 7
/**
 * Tests a MR scan using specific start and stop rows.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
private void testScan(String start, String stop, String last)
    throws IOException, InterruptedException, ClassNotFoundException {
  String jobName =
      "Scan" + (start != null ? start.toUpperCase(Locale.ROOT) : "Empty") + "To" +
          (stop != null ? stop.toUpperCase(Locale.ROOT) : "Empty");
  LOG.info("Before map/reduce startup - job " + jobName);
  Configuration c = new Configuration(TEST_UTIL.getConfiguration());

  c.set(KEY_STARTROW, start != null ? start : "");
  c.set(KEY_LASTROW, last != null ? last : "");

  List<Scan> scans = new ArrayList<>();

  for (String tableName : TABLES) {
    Scan scan = new Scan();

    scan.addFamily(INPUT_FAMILY);
    scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(tableName));

    if (start != null) {
      scan.withStartRow(Bytes.toBytes(start));
    }
    if (stop != null) {
      scan.withStopRow(Bytes.toBytes(stop));
    }

    scans.add(scan);

    LOG.info("scan before: " + scan);
  }

  runJob(jobName, c, scans);
}
 
Example 8
Source Project: phoenix   File: TupleProjector.java    License: Apache License 2.0 5 votes vote down vote up
public static void serializeProjectorIntoScan(Scan scan, TupleProjector projector,
        boolean projectDynColsInWildcardQueries) {
    scan.setAttribute(SCAN_PROJECTOR, serializeProjectorIntoBytes(projector));
    if (projectDynColsInWildcardQueries) {
        scan.setAttribute(WILDCARD_SCAN_INCLUDES_DYNAMIC_COLUMNS, TRUE_BYTES);
    }
}
 
Example 9
Source Project: hbase   File: TestMobStoreScanner.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Set the scan attribute
 *
 * @param reversed if true, scan will be backward order
 * @param mobScanRaw if true, scan will get the mob reference
 */
public void setScan(Scan scan, boolean reversed, boolean mobScanRaw) {
  scan.setReversed(reversed);
  scan.readVersions(4);
  if(mobScanRaw) {
    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
  }
}
 
Example 10
Source Project: phoenix   File: SerialIterators.java    License: Apache License 2.0 5 votes vote down vote up
private PeekingResultIterator nextIterator() throws SQLException {
    if (index >= scans.size()) {
        return EMPTY_ITERATOR;
    }
    ReadMetricQueue readMetrics = context.getReadMetricsQueue();
    while (index < scans.size()) {
        Scan currentScan = scans.get(index++);
        if (remainingOffset != null) {
            currentScan.setAttribute(BaseScannerRegionObserver.SCAN_OFFSET, PInteger.INSTANCE.toBytes(remainingOffset));
        }
        ScanMetricsHolder scanMetricsHolder =
                ScanMetricsHolder.getInstance(readMetrics, tableName, currentScan,
                    context.getConnection().getLogLevel());
        TableResultIterator itr =
                new TableResultIterator(mutationState, currentScan, scanMetricsHolder,
                        renewLeaseThreshold, plan, scanGrouper, caches);
        PeekingResultIterator peekingItr = iteratorFactory.newIterator(context, itr, currentScan, tableName, plan);
        Tuple tuple;
        if ((tuple = peekingItr.peek()) == null) {
            peekingItr.close();
            continue;
        } else if ((remainingOffset = QueryUtil.getRemainingOffset(tuple)) != null) {
            peekingItr.next();
            peekingItr.close();
            continue;
        }
        context.getConnection().addIteratorForLeaseRenewal(itr);
        return peekingItr;
    }
    return EMPTY_ITERATOR;
}
 
Example 11
Source Project: phoenix   File: ScanUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static void unsetReversed(Scan scan) {
    scan.setAttribute(BaseScannerRegionObserver.REVERSE_SCAN, PDataType.FALSE_BYTES);
    scan.setLoadColumnFamiliesOnDemand(true);
}
 
Example 12
public static void serializeIntoScan(Scan scan) {
    scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE);
}
 
Example 13
public static void setReversed(Scan scan) {
    // TODO: set attribute dynamically here to prevent dependency on newer HBase release
    scan.setAttribute(REVERSED_ATTR, PDataType.TRUE_BYTES);
}
 
Example 14
public static void serializeIntoScan(Scan scan) {
    scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE);
}
 
Example 15
Source Project: phoenix   File: ScanUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static void setTenantId(Scan scan, byte[] tenantId) {
    scan.setAttribute(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
}
 
Example 16
Source Project: phoenix   File: ScanUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static void setLocalIndex(Scan scan) {
    scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX, PDataType.TRUE_BYTES);
}
 
Example 17
Source Project: hbase   File: TestHMobStore.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Getting the reference data from files
 * @throws IOException
 */
@Test
public void testGetReferencesFromFiles() throws IOException {
  final Configuration conf = HBaseConfiguration.create();
  init(name.getMethodName(), conf, false);

  //Put data in memstore
  this.store.add(new KeyValue(row, family, qf1, 1, value), null);
  this.store.add(new KeyValue(row, family, qf2, 1, value), null);
  //flush
  flush(1);

  //Add more data
  this.store.add(new KeyValue(row, family, qf3, 1, value), null);
  this.store.add(new KeyValue(row, family, qf4, 1, value), null);
  //flush
  flush(2);

  //Add more data
  this.store.add(new KeyValue(row, family, qf5, 1, value), null);
  this.store.add(new KeyValue(row, family, qf6, 1, value), null);
  //flush
  flush(3);

  Scan scan = new Scan(get);
  scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
  InternalScanner scanner = (InternalScanner) store.getScanner(scan,
    scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
    0);

  List<Cell> results = new ArrayList<>();
  scanner.next(results);
  Collections.sort(results, CellComparatorImpl.COMPARATOR);
  scanner.close();

  //Compare
  Assert.assertEquals(expected.size(), results.size());
  for(int i=0; i<results.size(); i++) {
    Cell cell = results.get(i);
    Assert.assertTrue(MobUtils.isMobReferenceCell(cell));
  }
}
 
Example 18
Source Project: phoenix   File: MetaDataClient.java    License: Apache License 2.0 4 votes vote down vote up
private long updateStatisticsInternal(PName physicalName, PTable logicalTable) throws SQLException {
    ReadOnlyProps props = connection.getQueryServices().getProps();
    final long msMinBetweenUpdates = props
            .getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB,
                    props.getLong(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB,
                            QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS) / 2);
    byte[] tenantIdBytes = ByteUtil.EMPTY_BYTE_ARRAY;
    Long scn = connection.getSCN();
    // Always invalidate the cache
    long clientTimeStamp = connection.getSCN() == null ? HConstants.LATEST_TIMESTAMP : scn;
    String query = "SELECT CURRENT_DATE()," + LAST_STATS_UPDATE_TIME + " FROM " + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME
            + " WHERE " + PHYSICAL_NAME + "='" + physicalName.getString() + "' AND " + COLUMN_FAMILY
            + " IS NULL AND " + REGION_NAME + " IS NULL AND " + LAST_STATS_UPDATE_TIME + " IS NOT NULL";
    ResultSet rs = connection.createStatement().executeQuery(query);
    long msSinceLastUpdate = Long.MAX_VALUE;
    if (rs.next()) {
        msSinceLastUpdate = rs.getLong(1) - rs.getLong(2);
    }
    long rowCount = 0;
    if (msSinceLastUpdate >= msMinBetweenUpdates) {
        /*
         * Execute a COUNT(*) through PostDDLCompiler as we need to use the logicalTable passed through,
         * since it may not represent a "real" table in the case of the view indexes of a base table.
         */
        PostDDLCompiler compiler = new PostDDLCompiler(connection);
        TableRef tableRef = new TableRef(null, logicalTable, clientTimeStamp, false);
        MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, null, null, clientTimeStamp);
        Scan scan = plan.getContext().getScan();
        scan.setCacheBlocks(false);
        scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES);
        MutationState mutationState = plan.execute();
        rowCount = mutationState.getUpdateCount();
    }

    /*
     *  Update the stats table so that client will pull the new one with the updated stats.
     *  Even if we don't run the command due to the last update time, invalidate the cache.
     *  This supports scenarios in which a major compaction was manually initiated and the
     *  client wants the modified stats to be reflected immediately.
     */
    connection.getQueryServices().clearTableFromCache(tenantIdBytes,
            Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())),
            Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())), clientTimeStamp);
    return rowCount;
}
 
Example 19
Source Project: phoenix   File: IndexUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static void setScanAttributesForIndexReadRepair(Scan scan, PTable table, PhoenixConnection phoenixConnection) throws SQLException {
    if (table.isTransactional() || table.getType() != PTableType.INDEX) {
        return;
    }
    PTable indexTable = table;
    if (indexTable.getIndexType() != PTable.IndexType.GLOBAL) {
        return;
    }
    String schemaName = indexTable.getParentSchemaName().getString();
    String tableName = indexTable.getParentTableName().getString();
    PTable dataTable;
    try {
        dataTable = PhoenixRuntime.getTable(phoenixConnection, SchemaUtil.getTableName(schemaName, tableName));
    } catch (TableNotFoundException e) {
        // This index table must be being deleted. No need to set the scan attributes
        return;
    }
    // MetaDataClient modifies the index table name for view indexes if the parent view of an index has a child
    // view. This, we need to recreate a PTable object with the correct table name for the rest of this code to work
    if (indexTable.getViewIndexId() != null && indexTable.getName().getString().contains(QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR)) {
        int lastIndexOf = indexTable.getName().getString().lastIndexOf(QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR);
        String indexName = indexTable.getName().getString().substring(lastIndexOf + 1);
        indexTable = PhoenixRuntime.getTable(phoenixConnection, indexName);
    }
    if (!dataTable.getIndexes().contains(indexTable)) {
        return;
    }
    if (scan.getAttribute(PhoenixIndexCodec.INDEX_PROTO_MD) == null) {
        ImmutableBytesWritable ptr = new ImmutableBytesWritable();
        IndexMaintainer.serialize(dataTable, ptr, Collections.singletonList(indexTable), phoenixConnection);
        scan.setAttribute(PhoenixIndexCodec.INDEX_PROTO_MD, ByteUtil.copyKeyBytesIfNecessary(ptr));
    }
    scan.setAttribute(BaseScannerRegionObserver.CHECK_VERIFY_COLUMN, TRUE_BYTES);
    scan.setAttribute(BaseScannerRegionObserver.PHYSICAL_DATA_TABLE_NAME, dataTable.getPhysicalName().getBytes());
    IndexMaintainer indexMaintainer = indexTable.getIndexMaintainer(dataTable, phoenixConnection);
    byte[] emptyCF = indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary();
    byte[] emptyCQ = indexMaintainer.getEmptyKeyValueQualifier();
    scan.setAttribute(BaseScannerRegionObserver.EMPTY_COLUMN_FAMILY_NAME, emptyCF);
    scan.setAttribute(BaseScannerRegionObserver.EMPTY_COLUMN_QUALIFIER_NAME, emptyCQ);
    if (scan.getAttribute(BaseScannerRegionObserver.VIEW_CONSTANTS) == null) {
        BaseQueryPlan.serializeViewConstantsIntoScan(scan, dataTable);
    }
    addEmptyColumnToScan(scan, emptyCF, emptyCQ);
}
 
Example 20
Source Project: phoenix   File: ScanUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static void setTenantId(Scan scan, byte[] tenantId) {
    scan.setAttribute(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
}