Java Code Examples for org.apache.hadoop.hbase.client.Scan#setAttribute()
The following examples show how to use
org.apache.hadoop.hbase.client.Scan#setAttribute() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ActiveUserRunner.java From BigDataArchitect with Apache License 2.0 | 5 votes |
/** * 初始化scan集合 * * @param job * @return */ private List<Scan> initScans(Job job) { Configuration conf = job.getConfiguration(); // 获取运行时间: yyyy-MM-dd String date = conf.get(GlobalConstants.RUNNING_DATE_PARAMES); long startDate = TimeUtil.parseString2Long(date); long endDate = startDate + GlobalConstants.DAY_OF_MILLISECONDS; Scan scan = new Scan(); // 定义hbase扫描的开始rowkey和结束rowkey scan.setStartRow(Bytes.toBytes("" + startDate)); scan.setStopRow(Bytes.toBytes("" + endDate)); FilterList filterList = new FilterList(); // 定义mapper中需要获取的列名 String[] columns = new String[] { EventLogConstants.LOG_COLUMN_NAME_UUID, // 用户id EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, // 服务器时间 EventLogConstants.LOG_COLUMN_NAME_PLATFORM, // 平台名称 EventLogConstants.LOG_COLUMN_NAME_BROWSER_NAME, // 浏览器名称 EventLogConstants.LOG_COLUMN_NAME_BROWSER_VERSION // 浏览器版本号 }; filterList.addFilter(this.getColumnFilter(columns)); scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(EventLogConstants.HBASE_NAME_EVENT_LOGS)); scan.setFilter(filterList); return Lists.newArrayList(scan); }
Example 2
Source File: SerialIterators.java From phoenix with Apache License 2.0 | 5 votes |
private PeekingResultIterator nextIterator() throws SQLException { if (index >= scans.size()) { return EMPTY_ITERATOR; } ReadMetricQueue readMetrics = context.getReadMetricsQueue(); while (index < scans.size()) { Scan currentScan = scans.get(index++); if (remainingOffset != null) { currentScan.setAttribute(BaseScannerRegionObserver.SCAN_OFFSET, PInteger.INSTANCE.toBytes(remainingOffset)); } ScanMetricsHolder scanMetricsHolder = ScanMetricsHolder.getInstance(readMetrics, tableName, currentScan, context.getConnection().getLogLevel()); TableResultIterator itr = new TableResultIterator(mutationState, currentScan, scanMetricsHolder, renewLeaseThreshold, plan, scanGrouper, caches); PeekingResultIterator peekingItr = iteratorFactory.newIterator(context, itr, currentScan, tableName, plan); Tuple tuple; if ((tuple = peekingItr.peek()) == null) { peekingItr.close(); continue; } else if ((remainingOffset = QueryUtil.getRemainingOffset(tuple)) != null) { peekingItr.next(); peekingItr.close(); continue; } context.getConnection().addIteratorForLeaseRenewal(itr); return peekingItr; } return EMPTY_ITERATOR; }
Example 3
Source File: TestMobStoreScanner.java From hbase with Apache License 2.0 | 5 votes |
/** * Set the scan attribute * * @param reversed if true, scan will be backward order * @param mobScanRaw if true, scan will get the mob reference */ public void setScan(Scan scan, boolean reversed, boolean mobScanRaw) { scan.setReversed(reversed); scan.readVersions(4); if(mobScanRaw) { scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); } }
Example 4
Source File: TupleProjector.java From phoenix with Apache License 2.0 | 5 votes |
public static void serializeProjectorIntoScan(Scan scan, TupleProjector projector, boolean projectDynColsInWildcardQueries) { scan.setAttribute(SCAN_PROJECTOR, serializeProjectorIntoBytes(projector)); if (projectDynColsInWildcardQueries) { scan.setAttribute(WILDCARD_SCAN_INCLUDES_DYNAMIC_COLUMNS, TRUE_BYTES); } }
Example 5
Source File: MultiTableInputFormatTestBase.java From hbase with Apache License 2.0 | 5 votes |
/** * Tests a MR scan using specific start and stop rows. * * @throws IOException * @throws ClassNotFoundException * @throws InterruptedException */ private void testScan(String start, String stop, String last) throws IOException, InterruptedException, ClassNotFoundException { String jobName = "Scan" + (start != null ? start.toUpperCase(Locale.ROOT) : "Empty") + "To" + (stop != null ? stop.toUpperCase(Locale.ROOT) : "Empty"); LOG.info("Before map/reduce startup - job " + jobName); Configuration c = new Configuration(TEST_UTIL.getConfiguration()); c.set(KEY_STARTROW, start != null ? start : ""); c.set(KEY_LASTROW, last != null ? last : ""); List<Scan> scans = new ArrayList<>(); for (String tableName : TABLES) { Scan scan = new Scan(); scan.addFamily(INPUT_FAMILY); scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(tableName)); if (start != null) { scan.withStartRow(Bytes.toBytes(start)); } if (stop != null) { scan.withStopRow(Bytes.toBytes(stop)); } scans.add(scan); LOG.info("scan before: " + scan); } runJob(jobName, c, scans); }
Example 6
Source File: ClientSideRegionScannerIT.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
@Test @Ignore public void validateAccurateRecordsWithStoreFileAndMemstore() throws SQLException, IOException, InterruptedException{ int i=0; TableName tableName=TableName.valueOf(sqlUtil.getConglomID(SCHEMA_NAME+".A")); try(Admin admin = connection.getAdmin()) { Table table = connection.getTable(tableName); Scan scan=new Scan(); scan.setCaching(50); scan.setBatch(50); scan.setMaxVersions(); scan.setAttribute(MRConstants.SPLICE_SCAN_MEMSTORE_ONLY,HConstants.EMPTY_BYTE_ARRAY); try(SkeletonClientSideRegionScanner clientSideRegionScanner= new HBaseClientSideRegionScanner(table, table.getConfiguration(), FSUtils.getCurrentFileSystem(table.getConfiguration()), FSUtils.getRootDir(table.getConfiguration()), table.getTableDescriptor(), connection.getRegionLocator(tableName).getRegionLocation(scan.getStartRow()).getRegionInfo(), scan, connection.getRegionLocator(tableName).getRegionLocation(scan.getStartRow()).getHostnamePort())){ List results=new ArrayList(); while(clientSideRegionScanner.nextRaw(results)){ i++; results.clear(); } } Assert.assertEquals("Results Returned Are Not Accurate",500,i); } }
Example 7
Source File: MergingServerOp.java From geowave with Apache License 2.0 | 5 votes |
@Override public void preScannerOpen(final Scan scan) { final int maxVersions = scan.getMaxVersions(); if ((maxVersions > 0) && (maxVersions < Integer.MAX_VALUE)) { scan.setAttribute(OLD_MAX_VERSIONS_KEY, ByteBuffer.allocate(4).putInt(maxVersions).array()); } scan.setMaxVersions(); }
Example 8
Source File: MemstoreAwareObserverTest.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
private static Scan mockScan(byte[] startKey, byte[] endKey) { Scan scan = new StubInternalScanner(); scan.setAttribute(MRConstants.SPLICE_SCAN_MEMSTORE_ONLY, SIConstants.TRUE_BYTES); scan.setAttribute(ClientRegionConstants.SPLICE_SCAN_MEMSTORE_PARTITION_BEGIN_KEY, startKey); scan.setAttribute(ClientRegionConstants.SPLICE_SCAN_MEMSTORE_PARTITION_END_KEY, endKey); return scan; }
Example 9
Source File: TestMultiTableInputFormatBase.java From hbase with Apache License 2.0 | 5 votes |
/** * Test getSplits only puts up one Connection. * In past it has put up many Connections. Each Connection setup comes with a fresh new cache * so we have to do fresh hit on hbase:meta. Should only do one Connection when doing getSplits * even if a MultiTableInputFormat. * @throws IOException */ @Test public void testMRSplitsConnectionCount() throws IOException { // Make instance of MTIFB. MultiTableInputFormatBase mtif = new MultiTableInputFormatBase() { @Override public RecordReader<ImmutableBytesWritable, Result> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { return super.createRecordReader(split, context); } }; // Pass it a mocked JobContext. Make the JC return our Configuration. // Load the Configuration so it returns our special Connection so we can interpolate // canned responses. JobContext mockedJobContext = Mockito.mock(JobContext.class); Configuration c = HBaseConfiguration.create(); c.set(ConnectionUtils.HBASE_CLIENT_CONNECTION_IMPL, MRSplitsConnection.class.getName()); Mockito.when(mockedJobContext.getConfiguration()).thenReturn(c); // Invent a bunch of scans. Have each Scan go against a different table so a good spread. List<Scan> scans = new ArrayList<>(); for (int i = 0; i < 10; i++) { Scan scan = new Scan(); String tableName = this.name.getMethodName() + i; scan.setAttribute(SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(tableName)); scans.add(scan); } mtif.setScans(scans); // Get splits. Assert that that more than one. List<InputSplit> splits = mtif.getSplits(mockedJobContext); Assert.assertTrue(splits.size() > 0); // Assert only one Connection was made (see the static counter we have in the mocked // Connection MRSplitsConnection Constructor. Assert.assertEquals(1, MRSplitsConnection.creations.get()); }
Example 10
Source File: NewInstallUserRunner.java From BigDataArchitect with Apache License 2.0 | 5 votes |
/** * 初始化scan集合 * * @param job * @return */ private List<Scan> initScans(Job job) { // 时间戳+.... Configuration conf = job.getConfiguration(); // 获取运行时间: yyyy-MM-dd String date = conf.get(GlobalConstants.RUNNING_DATE_PARAMES); long startDate = TimeUtil.parseString2Long(date); long endDate = startDate + GlobalConstants.DAY_OF_MILLISECONDS; Scan scan = new Scan(); // 定义hbase扫描的开始rowkey和结束rowkey scan.setStartRow(Bytes.toBytes("" + startDate)); scan.setStopRow(Bytes.toBytes("" + endDate)); FilterList filterList = new FilterList(); // 过滤数据,只分析launch事件 filterList.addFilter(new SingleColumnValueFilter(Bytes.toBytes(EventLogConstants.EVENT_LOGS_FAMILY_NAME), Bytes.toBytes(EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME), CompareOp.EQUAL, Bytes.toBytes(EventEnum.LAUNCH.alias))); // 定义mapper中需要获取的列名 String[] columns = new String[] { EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME, EventLogConstants.LOG_COLUMN_NAME_UUID, EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, EventLogConstants.LOG_COLUMN_NAME_PLATFORM, EventLogConstants.LOG_COLUMN_NAME_BROWSER_NAME, EventLogConstants.LOG_COLUMN_NAME_BROWSER_VERSION }; // scan.addColumn(family, qualifier) filterList.addFilter(this.getColumnFilter(columns)); scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(EventLogConstants.HBASE_NAME_EVENT_LOGS)); scan.setFilter(filterList); return Lists.newArrayList(scan); }
Example 11
Source File: UngroupedAggregateRegionObserver.java From phoenix with Apache License 2.0 | 4 votes |
public static void serializeIntoScan(Scan scan) { scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE); }
Example 12
Source File: ScanUtil.java From phoenix with Apache License 2.0 | 4 votes |
public static void setTenantId(Scan scan, byte[] tenantId) { scan.setAttribute(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId); }
Example 13
Source File: ScanUtil.java From phoenix with Apache License 2.0 | 4 votes |
public static void setLocalIndex(Scan scan) { scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX, PDataType.TRUE_BYTES); }
Example 14
Source File: TestHMobStore.java From hbase with Apache License 2.0 | 4 votes |
/** * Getting the reference data from files * @throws IOException */ @Test public void testGetReferencesFromFiles() throws IOException { final Configuration conf = HBaseConfiguration.create(); init(name.getMethodName(), conf, false); //Put data in memstore this.store.add(new KeyValue(row, family, qf1, 1, value), null); this.store.add(new KeyValue(row, family, qf2, 1, value), null); //flush flush(1); //Add more data this.store.add(new KeyValue(row, family, qf3, 1, value), null); this.store.add(new KeyValue(row, family, qf4, 1, value), null); //flush flush(2); //Add more data this.store.add(new KeyValue(row, family, qf5, 1, value), null); this.store.add(new KeyValue(row, family, qf6, 1, value), null); //flush flush(3); Scan scan = new Scan(get); scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); InternalScanner scanner = (InternalScanner) store.getScanner(scan, scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()), 0); List<Cell> results = new ArrayList<>(); scanner.next(results); Collections.sort(results, CellComparatorImpl.COMPARATOR); scanner.close(); //Compare Assert.assertEquals(expected.size(), results.size()); for(int i=0; i<results.size(); i++) { Cell cell = results.get(i); Assert.assertTrue(MobUtils.isMobReferenceCell(cell)); } }
Example 15
Source File: MetaDataClient.java From phoenix with Apache License 2.0 | 4 votes |
private long updateStatisticsInternal(PName physicalName, PTable logicalTable) throws SQLException { ReadOnlyProps props = connection.getQueryServices().getProps(); final long msMinBetweenUpdates = props .getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, props.getLong(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, QueryServicesOptions.DEFAULT_STATS_UPDATE_FREQ_MS) / 2); byte[] tenantIdBytes = ByteUtil.EMPTY_BYTE_ARRAY; Long scn = connection.getSCN(); // Always invalidate the cache long clientTimeStamp = connection.getSCN() == null ? HConstants.LATEST_TIMESTAMP : scn; String query = "SELECT CURRENT_DATE()," + LAST_STATS_UPDATE_TIME + " FROM " + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " WHERE " + PHYSICAL_NAME + "='" + physicalName.getString() + "' AND " + COLUMN_FAMILY + " IS NULL AND " + REGION_NAME + " IS NULL AND " + LAST_STATS_UPDATE_TIME + " IS NOT NULL"; ResultSet rs = connection.createStatement().executeQuery(query); long msSinceLastUpdate = Long.MAX_VALUE; if (rs.next()) { msSinceLastUpdate = rs.getLong(1) - rs.getLong(2); } long rowCount = 0; if (msSinceLastUpdate >= msMinBetweenUpdates) { /* * Execute a COUNT(*) through PostDDLCompiler as we need to use the logicalTable passed through, * since it may not represent a "real" table in the case of the view indexes of a base table. */ PostDDLCompiler compiler = new PostDDLCompiler(connection); TableRef tableRef = new TableRef(null, logicalTable, clientTimeStamp, false); MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, null, null, clientTimeStamp); Scan scan = plan.getContext().getScan(); scan.setCacheBlocks(false); scan.setAttribute(BaseScannerRegionObserver.ANALYZE_TABLE, PDataType.TRUE_BYTES); MutationState mutationState = plan.execute(); rowCount = mutationState.getUpdateCount(); } /* * Update the stats table so that client will pull the new one with the updated stats. * Even if we don't run the command due to the last update time, invalidate the cache. * This supports scenarios in which a major compaction was manually initiated and the * client wants the modified stats to be reflected immediately. */ connection.getQueryServices().clearTableFromCache(tenantIdBytes, Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())), Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())), clientTimeStamp); return rowCount; }
Example 16
Source File: ScanUtil.java From phoenix with BSD 3-Clause "New" or "Revised" License | 4 votes |
public static void setReversed(Scan scan) { // TODO: set attribute dynamically here to prevent dependency on newer HBase release scan.setAttribute(REVERSED_ATTR, PDataType.TRUE_BYTES); }
Example 17
Source File: UngroupedAggregateRegionObserver.java From phoenix with Apache License 2.0 | 4 votes |
public static void serializeIntoScan(Scan scan) { scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE); }
Example 18
Source File: IndexUtil.java From phoenix with Apache License 2.0 | 4 votes |
public static void setScanAttributesForIndexReadRepair(Scan scan, PTable table, PhoenixConnection phoenixConnection) throws SQLException { if (table.isTransactional() || table.getType() != PTableType.INDEX) { return; } PTable indexTable = table; if (indexTable.getIndexType() != PTable.IndexType.GLOBAL) { return; } String schemaName = indexTable.getParentSchemaName().getString(); String tableName = indexTable.getParentTableName().getString(); PTable dataTable; try { dataTable = PhoenixRuntime.getTable(phoenixConnection, SchemaUtil.getTableName(schemaName, tableName)); } catch (TableNotFoundException e) { // This index table must be being deleted. No need to set the scan attributes return; } // MetaDataClient modifies the index table name for view indexes if the parent view of an index has a child // view. This, we need to recreate a PTable object with the correct table name for the rest of this code to work if (indexTable.getViewIndexId() != null && indexTable.getName().getString().contains(QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR)) { int lastIndexOf = indexTable.getName().getString().lastIndexOf(QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR); String indexName = indexTable.getName().getString().substring(lastIndexOf + 1); indexTable = PhoenixRuntime.getTable(phoenixConnection, indexName); } if (!dataTable.getIndexes().contains(indexTable)) { return; } if (scan.getAttribute(PhoenixIndexCodec.INDEX_PROTO_MD) == null) { ImmutableBytesWritable ptr = new ImmutableBytesWritable(); IndexMaintainer.serialize(dataTable, ptr, Collections.singletonList(indexTable), phoenixConnection); scan.setAttribute(PhoenixIndexCodec.INDEX_PROTO_MD, ByteUtil.copyKeyBytesIfNecessary(ptr)); } scan.setAttribute(BaseScannerRegionObserver.CHECK_VERIFY_COLUMN, TRUE_BYTES); scan.setAttribute(BaseScannerRegionObserver.PHYSICAL_DATA_TABLE_NAME, dataTable.getPhysicalName().getBytes()); IndexMaintainer indexMaintainer = indexTable.getIndexMaintainer(dataTable, phoenixConnection); byte[] emptyCF = indexMaintainer.getEmptyKeyValueFamily().copyBytesIfNecessary(); byte[] emptyCQ = indexMaintainer.getEmptyKeyValueQualifier(); scan.setAttribute(BaseScannerRegionObserver.EMPTY_COLUMN_FAMILY_NAME, emptyCF); scan.setAttribute(BaseScannerRegionObserver.EMPTY_COLUMN_QUALIFIER_NAME, emptyCQ); if (scan.getAttribute(BaseScannerRegionObserver.VIEW_CONSTANTS) == null) { BaseQueryPlan.serializeViewConstantsIntoScan(scan, dataTable); } addEmptyColumnToScan(scan, emptyCF, emptyCQ); }
Example 19
Source File: ScanUtil.java From phoenix with Apache License 2.0 | 4 votes |
public static void setTenantId(Scan scan, byte[] tenantId) { scan.setAttribute(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId); }
Example 20
Source File: ScanUtil.java From phoenix with Apache License 2.0 | 4 votes |
public static void unsetReversed(Scan scan) { scan.setAttribute(BaseScannerRegionObserver.REVERSE_SCAN, PDataType.FALSE_BYTES); scan.setLoadColumnFamiliesOnDemand(true); }