org.apache.kylin.metadata.model.MeasureDesc Java Examples
The following examples show how to use
org.apache.kylin.metadata.model.MeasureDesc.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RawMeasureType.java From kylin with Apache License 2.0 | 6 votes |
public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, final MeasureDesc measureDesc) { //is raw query if (!digest.isRawQuery) return null; TblColRef rawColumn = getRawColumn(measureDesc.getFunction()); if (!digest.allColumns.isEmpty() && !digest.allColumns.contains(rawColumn)) { return null; } unmatchedAggregations.remove(measureDesc.getFunction()); //contain one raw measure : cost * 0.9 return new CapabilityResult.CapabilityInfluence() { @Override public double suggestCostMultiplier() { return 0.9; } @Override public MeasureDesc getInvolvedMeasure() { return measureDesc; } }; }
Example #2
Source File: CubeDesc.java From kylin with Apache License 2.0 | 6 votes |
@SuppressWarnings("deprecation") private void initMeasureColumns() { if (measures == null || measures.isEmpty()) { return; } for (MeasureDesc m : measures) { m.setName(m.getName().toUpperCase(Locale.ROOT)); if (m.getDependentMeasureRef() != null) { m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase(Locale.ROOT)); } FunctionDesc func = m.getFunction(); func.init(model); allColumns.addAll(func.getParameter().getColRefs()); if (ExtendedColumnMeasureType.FUNC_EXTENDED_COLUMN.equalsIgnoreCase(m.getFunction().getExpression())) { FunctionDesc functionDesc = m.getFunction(); List<TblColRef> hosts = ExtendedColumnMeasureType.getExtendedColumnHosts(functionDesc); TblColRef extendedColumn = ExtendedColumnMeasureType.getExtendedColumn(functionDesc); initExtendedColumnMap(hosts.toArray(new TblColRef[hosts.size()]), extendedColumn); } } }
Example #3
Source File: OLAPAggregateRel.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
FunctionDesc findInMeasures(FunctionDesc aggFunc, List<MeasureDesc> measures) { for (MeasureDesc m : measures) { if (aggFunc.equals(m.getFunction())) { return m.getFunction(); } } // no count(col) measure found, use count(1) to replace it. if (aggFunc.isCount()) { FunctionDesc func = findCountConstantFunc(measures); if (func != null) return func; } return aggFunc; }
Example #4
Source File: MeasureAggregators.java From kylin with Apache License 2.0 | 6 votes |
public MeasureAggregators(MeasureDesc... measureDescs) { descLength = measureDescs.length; aggs = new MeasureAggregator[descLength]; Map<String, Integer> measureIndexMap = new HashMap<String, Integer>(); for (int i = 0; i < descLength; i++) { FunctionDesc func = measureDescs[i].getFunction(); aggs[i] = func.getMeasureType().newAggregator(); measureIndexMap.put(measureDescs[i].getName(), i); } // fill back dependent aggregator for (int i = 0; i < descLength; i++) { String depMsrRef = measureDescs[i].getDependentMeasureRef(); if (depMsrRef != null) { int index = measureIndexMap.get(depMsrRef); aggs[i].setDependentAggregator(aggs[index]); } } }
Example #5
Source File: CubeStorageEngine.java From Kylin with Apache License 2.0 | 6 votes |
private void setThreshold(Collection<TblColRef> dimensions, List<RowValueDecoder> valueDecoders, StorageContext context) { if (RowValueDecoder.hasMemHungryCountDistinct(valueDecoders) == false) { return; } int rowSizeEst = dimensions.size() * 3; for (RowValueDecoder decoder : valueDecoders) { MeasureDesc[] measures = decoder.getMeasures(); BitSet projectionIndex = decoder.getProjectionIndex(); for (int i = projectionIndex.nextSetBit(0); i >= 0; i = projectionIndex.nextSetBit(i + 1)) { FunctionDesc func = measures[i].getFunction(); rowSizeEst += func.getReturnDataType().getSpaceEstimate(); } } long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst; context.setThreshold((int) rowEst); }
Example #6
Source File: ITStorageTest.java From kylin with Apache License 2.0 | 6 votes |
private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) { int count = 0; ITupleIterator iterator = null; try { SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", /*allCol*/ Collections.<TblColRef> emptySet(), /*join*/ null, // groups, /*subqueryJoinParticipants*/ Sets.<TblColRef> newHashSet(), // /*dynamicGroupByColumns*/ Collections.<TblColRef, TupleExpression> emptyMap(), // /*groupByExpression*/ false, // /*metricCol*/ Collections.<TblColRef> emptySet(), aggregations, /*aggrSqlCalls*/ Collections.<SQLCall> emptyList(), // /*dynamicAggregations*/ Collections.<DynamicFunctionDesc> emptyList(), // /*runtimeDimensionColumns*/ Collections.<TblColRef> emptySet(), // /*runtimeMetricColumns*/ Collections.<TblColRef> emptySet(), // /*filter col*/ Collections.<TblColRef> emptySet(), filter, null, // /*sortCol*/ new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(), false, false, false, new HashSet<MeasureDesc>()); iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations)); while (iterator.hasNext()) { ITuple tuple = iterator.next(); System.out.println("Tuple = " + tuple); count++; } } finally { if (iterator != null) iterator.close(); } return count; }
Example #7
Source File: CubeCapabilityChecker.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
private static void tryCustomMeasureTypes(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube, CapabilityResult result) { CubeDesc cubeDesc = cube.getDescriptor(); List<String> influencingMeasures = Lists.newArrayList(); for (MeasureDesc measure : cubeDesc.getMeasures()) { // if (unmatchedDimensions.isEmpty() && unmatchedAggregations.isEmpty()) // break; MeasureType<?> measureType = measure.getFunction().getMeasureType(); if (measureType instanceof BasicMeasureType) continue; CapabilityInfluence inf = measureType.influenceCapabilityCheck(unmatchedDimensions, unmatchedAggregations, digest, measure); if (inf != null) { result.influences.add(inf); influencingMeasures.add(measure.getName() + "@" + measureType.getClass()); } } if (influencingMeasures.size() != 0) logger.info("Cube {} CapabilityInfluences: {}", cube.getCanonicalName(), StringUtils.join(influencingMeasures, ",")); }
Example #8
Source File: OLAPSortRel.java From Kylin with Apache License 2.0 | 6 votes |
@Override public void implementRewrite(RewriteImplementor implementor) { implementor.visitChild(this, getChild()); for (RelFieldCollation fieldCollation : this.collation.getFieldCollations()) { int index = fieldCollation.getFieldIndex(); StorageContext.OrderEnum order = getOrderEnum(fieldCollation.getDirection()); OLAPRel olapChild = (OLAPRel) this.getChild(); TblColRef orderCol = olapChild.getColumnRowType().getAllColumns().get(index); MeasureDesc measure = findMeasure(orderCol); if (measure != null) { this.context.storageContext.addSort(measure, order); } this.context.storageContext.markSort(); } this.rowType = this.deriveRowType(); this.columnRowType = buildColumnRowType(); }
Example #9
Source File: TopNMeasureTypeTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
@Test public void test() { CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); MeasureDesc topSellerMeasure = null; for (MeasureDesc measureDesc : desc.getMeasures()) { if (measureDesc.getName().equals("TOP_SELLER")) { topSellerMeasure = measureDesc; break; } } TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create(topSellerMeasure.getFunction().getExpression(), topSellerMeasure.getFunction().getReturnDataType()); topSellerMeasure.getFunction().getConfiguration().clear(); List<TblColRef> colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction()); assertTrue(colsNeedDict != null && colsNeedDict.size() == 1); TblColRef sellerColRef = topSellerMeasure.getFunction().getParameter().getColRefs().get(1); topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getIdentity(), "int:6"); colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction()); assertTrue(colsNeedDict.size() == 0); }
Example #10
Source File: CubeDesc.java From Kylin with Apache License 2.0 | 6 votes |
private void initMeasureReferenceToColumnFamily() { if (measures == null || measures.size() == 0) return; Map<String, MeasureDesc> measureCache = new HashMap<String, MeasureDesc>(); for (MeasureDesc m : measures) measureCache.put(m.getName(), m); for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { MeasureDesc[] measureDescs = new MeasureDesc[c.getMeasureRefs().length]; for (int i = 0; i < c.getMeasureRefs().length; i++) { measureDescs[i] = measureCache.get(c.getMeasureRefs()[i]); } c.setMeasures(measureDescs); c.setColumnFamilyName(cf.getName()); } } }
Example #11
Source File: CubeDesc.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
@SuppressWarnings("deprecation") private void initMeasureColumns() { if (measures == null || measures.isEmpty()) { return; } for (MeasureDesc m : measures) { m.setName(m.getName().toUpperCase(Locale.ROOT)); if (m.getDependentMeasureRef() != null) { m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase(Locale.ROOT)); } FunctionDesc func = m.getFunction(); func.init(model); allColumns.addAll(func.getParameter().getColRefs()); if (ExtendedColumnMeasureType.FUNC_EXTENDED_COLUMN.equalsIgnoreCase(m.getFunction().getExpression())) { FunctionDesc functionDesc = m.getFunction(); List<TblColRef> hosts = ExtendedColumnMeasureType.getExtendedColumnHosts(functionDesc); TblColRef extendedColumn = ExtendedColumnMeasureType.getExtendedColumn(functionDesc); initExtendedColumnMap(hosts.toArray(new TblColRef[hosts.size()]), extendedColumn); } } }
Example #12
Source File: CubeDesc.java From Kylin with Apache License 2.0 | 6 votes |
private void initMeasureReferenceToColumnFamily() { if (measures == null || measures.size() == 0) return; Map<String, MeasureDesc> measureCache = new HashMap<String, MeasureDesc>(); for (MeasureDesc m : measures) measureCache.put(m.getName(), m); for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { MeasureDesc[] measureDescs = new MeasureDesc[c.getMeasureRefs().length]; for (int i = 0; i < c.getMeasureRefs().length; i++) { measureDescs[i] = measureCache.get(c.getMeasureRefs()[i]); } c.setMeasures(measureDescs); c.setColumnFamilyName(cf.getName()); } } }
Example #13
Source File: KeyValueCreator.java From kylin with Apache License 2.0 | 6 votes |
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) { cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName()); qBytes = Bytes.toBytes(colDesc.getQualifier()); timestamp = 0; // use 0 for timestamp refIndex = colDesc.getMeasureIndex(); refMeasures = colDesc.getMeasures(); codec = new BufferedMeasureCodec(refMeasures); colValues = new Object[refMeasures.length]; isFullCopy = true; List<MeasureDesc> measures = cubeDesc.getMeasures(); for (int i = 0; i < measures.size(); i++) { if (refIndex.length <= i || refIndex[i] != i) isFullCopy = false; } }
Example #14
Source File: OLAPAggregateRel.java From kylin with Apache License 2.0 | 6 votes |
FunctionDesc findInMeasures(FunctionDesc aggFunc, List<MeasureDesc> measures) { for (MeasureDesc m : measures) { if (aggFunc.equals(m.getFunction())) { return m.getFunction(); } } // no count(col) measure found, use count(1) to replace it. if (aggFunc.isCount()) { FunctionDesc func = findCountConstantFunc(measures); if (func != null) return func; } return aggFunc; }
Example #15
Source File: BaseCuboidBuilder.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
private void checkMrDictClolumn(){ Set<String> mrDictColumnSet = new HashSet<>(); if (kylinConfig.getMrHiveDictColumns() != null) { Collections.addAll(mrDictColumnSet, kylinConfig.getMrHiveDictColumns()); } for (MeasureDesc measure : measureDescList) { if (measure.getFunction().getExpression().equalsIgnoreCase(FunctionDesc.FUNC_COUNT_DISTINCT)) { FunctionDesc functionDesc = measure.getFunction(); TblColRef colRef = functionDesc.getParameter().getColRefs().get(0); if (mrDictColumnSet.contains(JoinedFlatTable.colName(colRef, true))) { functionDesc.setMrDict(true); logger.info("setMrDict for {}", colRef); measure.setFunction(functionDesc); } } } }
Example #16
Source File: FragmentCuboidReaderPerfTest.java From kylin with Apache License 2.0 | 6 votes |
public Pair<List<TblColRef>, List<MeasureDesc>> getReadDimensionsAndMetrics() { Set<TblColRef> dimensionsSet = testHelper.simulateDimensions("STREAMING_V2_TABLE.MINUTE_START"); List<TblColRef> dimensions = new ArrayList<>(dimensionsSet); FunctionDesc metricFunc = testHelper.simulateMetric("STREAMING_V2_TABLE.GMV", "SUM", "decimal(19,6)"); List<MeasureDesc> allMetrics = cubeInstance.getDescriptor().getMeasures(); List<MeasureDesc> targetMetrics = Lists.newArrayList(); for (MeasureDesc metric : allMetrics) { if (metric.getFunction().equals(metricFunc)) { targetMetrics.add(metric); } if (metric.getName().equalsIgnoreCase("_COUNT_")) { // targetMetrics.add(metric); } } return new Pair<>(dimensions, targetMetrics); }
Example #17
Source File: MeasureAggregators.java From Kylin with Apache License 2.0 | 6 votes |
public MeasureAggregators(MeasureDesc... measureDescs) { descs = measureDescs; aggs = new MeasureAggregator[descs.length]; Map<String, Integer> measureIndexMap = new HashMap<String, Integer>(); for (int i = 0; i < descs.length; i++) { FunctionDesc func = descs[i].getFunction(); aggs[i] = MeasureAggregator.create(func.getExpression(), func.getReturnType()); measureIndexMap.put(descs[i].getName(), i); } // fill back dependent aggregator for (int i = 0; i < descs.length; i++) { String depMsrRef = descs[i].getDependentMeasureRef(); if (depMsrRef != null) { int index = measureIndexMap.get(depMsrRef); aggs[i].setDependentAggregator(aggs[index]); } } }
Example #18
Source File: CubeDesc.java From Kylin with Apache License 2.0 | 5 votes |
private void initMeasureColumns(Map<String, TableDesc> tables) { if (measures == null || measures.isEmpty()) { return; } TableDesc factTable = tables.get(getFactTable()); for (MeasureDesc m : measures) { m.setName(m.getName().toUpperCase()); if (m.getDependentMeasureRef() != null) { m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase()); } FunctionDesc f = m.getFunction(); f.setExpression(f.getExpression().toUpperCase()); f.setReturnDataType(DataType.getInstance(f.getReturnType())); ParameterDesc p = f.getParameter(); p.normalizeColumnValue(); if (p.isColumnType()) { ArrayList<TblColRef> colRefs = Lists.newArrayList(); for (String cName : p.getValue().split("\\s*,\\s*")) { ColumnDesc sourceColumn = factTable.findColumnByName(cName); TblColRef colRef = new TblColRef(sourceColumn); colRefs.add(colRef); allColumns.add(colRef); } if (colRefs.isEmpty() == false) p.setColRefs(colRefs); } // verify holistic count distinct as a dependent measure if (m.isHolisticCountDistinct() && StringUtils.isBlank(m.getDependentMeasureRef())) { throw new IllegalStateException(m + " is a holistic count distinct but it has no DependentMeasureRef defined!"); } } }
Example #19
Source File: ExtendedColumnMeasureType.java From kylin with Apache License 2.0 | 5 votes |
@Override public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, final MeasureDesc measureDesc) { TblColRef extendedCol = getExtendedColumn(measureDesc.getFunction()); if (!unmatchedDimensions.contains(extendedCol)) { return null; } if (digest.filterColumns.contains(extendedCol)) { return null; } unmatchedDimensions.remove(extendedCol); return new CapabilityResult.CapabilityInfluence() { @Override public double suggestCostMultiplier() { return 0.9; } @Override public MeasureDesc getInvolvedMeasure() { return measureDesc; } }; }
Example #20
Source File: RowValueDecoderTest.java From Kylin with Apache License 2.0 | 5 votes |
@Test public void testDecode() throws Exception { CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor(); HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0]; MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures()); BigDecimal sum = new BigDecimal("333.1234567"); BigDecimal min = new BigDecimal("333.1111111"); BigDecimal max = new BigDecimal("333.1999999"); LongWritable count = new LongWritable(2); ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE); codec.encode(new Object[] { sum, min, max, count }, buf); buf.flip(); byte[] valueBytes = new byte[buf.limit()]; System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit()); RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol); for (MeasureDesc measure : cubeDesc.getMeasures()) { FunctionDesc aggrFunc = measure.getFunction(); int index = hbaseCol.findMeasureIndex(aggrFunc); rowValueDecoder.setIndex(index); } rowValueDecoder.decode(valueBytes); List<String> measureNames = rowValueDecoder.getNames(); Object[] measureValues = rowValueDecoder.getValues(); assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString()); assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues)); }
Example #21
Source File: FragmentCuboidReaderPerfTest.java From kylin with Apache License 2.0 | 5 votes |
public void scanPerformance() throws Exception { Pair<List<TblColRef>, List<MeasureDesc>> readDimAndMetrics = getReadDimensionsAndMetrics(); for (int i = 1; i < 5; i++) { scan(i, readDimAndMetrics.getFirst(), readDimAndMetrics.getSecond().toArray(new MeasureDesc[0])); } }
Example #22
Source File: CubeDesc.java From kylin with Apache License 2.0 | 5 votes |
public boolean hasMemoryHungryMeasures() { for (MeasureDesc measure : measures) { if (measure.getFunction().getMeasureType().isMemoryHungry()) { return true; } } return false; }
Example #23
Source File: LongIngester.java From kylin with Apache License 2.0 | 5 votes |
@Override public Long valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { if (values.length > 1) throw new IllegalArgumentException(); if (values[0] == null || values[0].length() == 0) return new Long(0L); else return Long.valueOf(values[0]); }
Example #24
Source File: GTCubeStorageQueryBase.java From kylin with Apache License 2.0 | 5 votes |
private TupleFilter checkHavingCanPushDown(TupleFilter havingFilter, Set<TblColRef> groupsD, List<FunctionDesc> aggregations, Set<FunctionDesc> metrics) { // must have only one segment Segments<CubeSegment> readySegs = cubeInstance.getSegments(SegmentStatusEnum.READY); if (readySegs.size() != 1) { logger.info("Can not push down having filter, must have only one segment"); return null; } // sharded-by column must on group by CubeDesc desc = cubeInstance.getDescriptor(); Set<TblColRef> shardBy = desc.getShardByColumns(); if (groupsD == null || shardBy.isEmpty() || !groupsD.containsAll(shardBy)) return null; // OK, push down logger.info("Push down having filter {}", havingFilter); // convert columns in the filter Set<TblColRef> aggrOutCols = new HashSet<>(); TupleFilter.collectColumns(havingFilter, aggrOutCols); for (TblColRef aggrOutCol : aggrOutCols) { int aggrIdxOnSql = aggrOutCol.getColumnDesc().getZeroBasedIndex(); // aggr index marked in OLAPAggregateRel FunctionDesc aggrFunc = aggregations.get(aggrIdxOnSql); // calculate the index of this aggr among all the metrics that is sending to storage int aggrIdxAmongMetrics = 0; for (MeasureDesc m : cubeDesc.getMeasures()) { if (aggrFunc.equals(m.getFunction())) break; if (metrics.contains(m.getFunction())) aggrIdxAmongMetrics++; } aggrOutCol.getColumnDesc().setId("" + (aggrIdxAmongMetrics + 1)); } return havingFilter; }
Example #25
Source File: OLAPTable.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
private List<ColumnDesc> listSourceColumns() { ProjectManager mgr = ProjectManager.getInstance(olapSchema.getConfig()); List<ColumnDesc> tableColumns = mgr.listExposedColumns(olapSchema.getProjectName(), sourceTable, exposeMore); List<ColumnDesc> metricColumns = Lists.newArrayList(); List<MeasureDesc> countMeasures = mgr.listEffectiveRewriteMeasures(olapSchema.getProjectName(), sourceTable.getIdentity()); HashSet<String> metFields = new HashSet<String>(); for (MeasureDesc m : countMeasures) { FunctionDesc func = m.getFunction(); String fieldName = func.getRewriteFieldName(); if (!metFields.contains(fieldName)) { metFields.add(fieldName); ColumnDesc fakeCountCol = func.newFakeRewriteColumn(sourceTable); metricColumns.add(fakeCountCol); } } Collections.sort(tableColumns, new Comparator<ColumnDesc>() { @Override public int compare(ColumnDesc o1, ColumnDesc o2) { return o1.getZeroBasedIndex() - o2.getZeroBasedIndex(); } }); return Lists.newArrayList(Iterables.concat(tableColumns, metricColumns)); }
Example #26
Source File: CubeDescTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Test public void testBadInit15() throws Exception { thrown.expect(IllegalStateException.class); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3); Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT"); thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") duplicates"); HBaseColumnDesc colDesc = new HBaseColumnDesc(); colDesc.setQualifier("M"); colDesc.setMeasureRefs( new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "TRANS_CNT", "ITEM_COUNT_SUM" }); cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc; cubeDesc.initMeasureReferenceToColumnFamily(); }
Example #27
Source File: FragmentCuboidReaderPerfTest.java From kylin with Apache License 2.0 | 5 votes |
private void readRow(int time, List<TblColRef> dimensions, MeasureDesc[] metrics) throws IOException { System.out.println("start " + time + " read, " + dimensions.size() + " dimensions," + metrics.length + " measures"); TblColRef[] dimArray = dimensions.toArray(new TblColRef[dimensions.size()]); Random rand = new Random(); int randReadNum = 100; int[][] readRows = new int[fragments.length][randReadNum]; List<FragmentCuboidReader> fragmentCuboidReaders = Lists.newArrayList(); for (int i = 0; i < fragments.length; i++) { FragmentMetaInfo fragmentMetaInfo = fragments[i].getMetaInfo(); FragmentData fragmentData = new FragmentData(fragmentMetaInfo, fragments[i].getDataFile()); Map<TblColRef, Dictionary<String>> dictionaryMap = fragmentData .getDimensionDictionaries(parsedStreamingCubeInfo.dimensionsUseDictEncoding); DimensionEncoding[] dimensionEncodings = ParsedStreamingCubeInfo.getDimensionEncodings( parsedStreamingCubeInfo.cubeDesc, dimArray, dictionaryMap); FragmentCuboidReader fragmentCuboidReader = new FragmentCuboidReader(parsedStreamingCubeInfo.cubeDesc, fragmentData, fragmentMetaInfo.getBasicCuboidMetaInfo(), dimArray, metrics, dimensionEncodings); fragmentCuboidReaders.add(fragmentCuboidReader); for (int j = 0; j < randReadNum; j++) { readRows[i][j] = rand.nextInt((int) fragmentMetaInfo.getNumberOfRows()); } } Stopwatch sw = Stopwatch.createUnstarted(); sw.start(); int rowNum = 0; for (int i = 0; i < fragments.length; i++) { for (int j = 0; j < readRows.length; j++) { fragmentCuboidReaders.get(i).read(readRows[i][j]); } } sw.stop(); long takeTime = sw.elapsed(MILLISECONDS); System.out.println(time + " scan finished, total rows:" + rowNum); System.out.println(time + " scan took:" + takeTime + ",rowsPerSec:" + (rowNum / takeTime) * 1000); }
Example #28
Source File: LongIngester.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Override public Long valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { if (values.length > 1) throw new IllegalArgumentException(); if (values[0] == null || values[0].length() == 0) return new Long(0L); else return Long.valueOf(values[0]); }
Example #29
Source File: RowValueDecoder.java From Kylin with Apache License 2.0 | 5 votes |
public RowValueDecoder(HBaseColumnDesc hbaseColumn) { this.hbaseColumn = hbaseColumn; this.projectionIndex = new BitSet(); this.names = new ArrayList<String>(); this.measures = hbaseColumn.getMeasures(); for (MeasureDesc measure : measures) { this.names.add(measure.getFunction().getRewriteFieldName()); } this.codec = new MeasureCodec(measures); this.values = new Object[measures.length]; }
Example #30
Source File: CubeDesc.java From Kylin with Apache License 2.0 | 5 votes |
public boolean hasHolisticCountDistinctMeasures() { for (MeasureDesc measure : measures) { if (measure.getFunction().isHolisticCountDistinct()) { return true; } } return false; }