Java Code Examples for org.apache.kylin.cube.model.RowKeyColDesc

The following examples show how to use org.apache.kylin.cube.model.RowKeyColDesc. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: kylin-on-parquet-v2   Source File: CubeDescCreator.java    License: Apache License 2.0 6 votes vote down vote up
public static int getTimeRowKeyColDesc(String tableName, RowKeyColDesc[] rowKeyColDescs) {
    int idx = 0;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.DAY_DATE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.WEEK_BEGIN_DATE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.MONTH.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.YEAR.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.TIME_HOUR.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.TIME_MINUTE.toString(), idx + 1);
    idx++;
    return idx;
}
 
Example 2
Source Project: kylin-on-parquet-v2   Source File: RowRecordReader.java    License: Apache License 2.0 6 votes vote down vote up
private Map<String, DimensionEncoding> getDimensionEncodings(FragmentMetaInfo fragmentMetaInfo,
        List<DimensionMetaInfo> allDimensions, FSDataInputStream dictInputStream) throws IOException {
    Map<String, Dictionary> dictionaryMap = readAllDimensionsDictionary(fragmentMetaInfo, dictInputStream);

    Map<String, DimensionEncoding> result = Maps.newHashMap();
    for (DimensionMetaInfo dimension : allDimensions) {
        TblColRef col = cubeDesc.getModel().findColumn(dimension.getName());
        RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col);
        if (colDesc.isUsingDictionary()) {
            @SuppressWarnings({ "unchecked" })
            Dictionary<String> dict = dictionaryMap.get(dimension.getName());
            if (dict == null) {
                logger.error("No dictionary found for dict-encoding column " + col);
                throw new RuntimeException("No dictionary found for dict-encoding column " + col);
            } else {
                result.put(dimension.getName(), new DictionaryDimEnc(dict));
            }
        } else {
            result.put(
                    dimension.getName(),
                    DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(),
                            colDesc.getEncodingVersion()));
        }
    }
    return result;
}
 
Example 3
Source Project: kylin-on-parquet-v2   Source File: RowKeyAttrRule.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    RowKeyDesc row = cube.getRowkey();
    if (row == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
        return;
    }

    RowKeyColDesc[] rcd = row.getRowKeyColumns();
    if (rcd == null || rcd.length == 0) {
        context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
        return;
    }

    for (int i = 0; i < rcd.length; i++) {
        RowKeyColDesc rd = rcd[i];
        if (rd.getColumn() == null || rd.getColumn().length() == 0) {
            context.addResult(ResultLevel.ERROR, "Rowkey column empty");
        }

    }

}
 
Example 4
Source Project: kylin-on-parquet-v2   Source File: CubeDimEncMap.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public DimensionEncoding get(TblColRef col) {
    DimensionEncoding result = encMap.get(col);
    if (result == null) {
        RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col);
        if (colDesc.isUsingDictionary()) {
            // special dictionary encoding
            Dictionary<String> dict = getDictionary(col);
            if (dict == null) {
                logger.warn("No dictionary found for dict-encoding column " + col + ", segment " + seg);
                result = new FixedLenDimEnc(0);
            } else {
                result = new DictionaryDimEnc(dict);
            }
        } else {
            // normal case
            result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(), colDesc.getEncodingVersion());
        }
        encMap.put(col, result);
    }
    return result;
}
 
Example 5
public static DimensionEncoding[] getDimensionEncodings(CubeDesc cubeDesc, TblColRef[] dimensions,
        Map<TblColRef, Dictionary<String>> dimDictMap) {
    DimensionEncoding[] result = new DimensionEncoding[dimensions.length];
    for (int i = 0; i < dimensions.length; i++) {
        TblColRef dimension = dimensions[i];
        RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(dimension);
        if (colDesc.isUsingDictionary()) {
            @SuppressWarnings({ "unchecked" })
            Dictionary<String> dict = dimDictMap.get(dimension);
            if (dict == null) {
                throw new RuntimeException("No dictionary found for dict-encoding column " + dimension);
            } else {
                result[i] = new DictionaryDimEnc(dict);
            }
        } else {
            result[i] = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(),
                    colDesc.getEncodingVersion());
        }
    }
    return result;
}
 
Example 6
public void init(CubeDesc cubeDesc, CubeJoinedFlatTableEnrich intermediateTableDesc) {
    dimensions = Lists.newArrayList();
    columnsIndex = new int[Long.bitCount(cuboidID)];

    int colIdx = 0;
    RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
    for (int i = 0; i < allColumns.length; i++) {
        // NOTE: the order of column in list!!!
        long bitmask = 1L << allColumns[i].getBitIndex();
        if ((cuboidID & bitmask) != 0) {
            TblColRef colRef = allColumns[i].getColRef();
            dimensions.add(colRef);
            columnsIndex[colIdx] = intermediateTableDesc.getColumnIndex(colRef);
            colIdx++;
        }
    }
}
 
Example 7
Source Project: kylin   Source File: CubeDescCreator.java    License: Apache License 2.0 6 votes vote down vote up
public static int getTimeRowKeyColDesc(String tableName, RowKeyColDesc[] rowKeyColDescs) {
    int idx = 0;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.DAY_DATE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.WEEK_BEGIN_DATE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.MONTH.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.YEAR.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.TIME_HOUR.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, TimePropertyEnum.TIME_MINUTE.toString(), idx + 1);
    idx++;
    return idx;
}
 
Example 8
Source Project: kylin   Source File: RowRecordReader.java    License: Apache License 2.0 6 votes vote down vote up
private Map<String, DimensionEncoding> getDimensionEncodings(FragmentMetaInfo fragmentMetaInfo,
        List<DimensionMetaInfo> allDimensions, FSDataInputStream dictInputStream) throws IOException {
    Map<String, Dictionary> dictionaryMap = readAllDimensionsDictionary(fragmentMetaInfo, dictInputStream);

    Map<String, DimensionEncoding> result = Maps.newHashMap();
    for (DimensionMetaInfo dimension : allDimensions) {
        TblColRef col = cubeDesc.getModel().findColumn(dimension.getName());
        RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col);
        if (colDesc.isUsingDictionary()) {
            @SuppressWarnings({ "unchecked" })
            Dictionary<String> dict = dictionaryMap.get(dimension.getName());
            if (dict == null) {
                logger.error("No dictionary found for dict-encoding column " + col);
                throw new RuntimeException("No dictionary found for dict-encoding column " + col);
            } else {
                result.put(dimension.getName(), new DictionaryDimEnc(dict));
            }
        } else {
            result.put(
                    dimension.getName(),
                    DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(),
                            colDesc.getEncodingVersion()));
        }
    }
    return result;
}
 
Example 9
Source Project: kylin   Source File: RowKeyAttrRule.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    RowKeyDesc row = cube.getRowkey();
    if (row == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
        return;
    }

    RowKeyColDesc[] rcd = row.getRowKeyColumns();
    if (rcd == null || rcd.length == 0) {
        context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
        return;
    }

    for (int i = 0; i < rcd.length; i++) {
        RowKeyColDesc rd = rcd[i];
        if (rd.getColumn() == null || rd.getColumn().length() == 0) {
            context.addResult(ResultLevel.ERROR, "Rowkey column empty");
        }

    }

}
 
Example 10
Source Project: kylin   Source File: CubeDimEncMap.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public DimensionEncoding get(TblColRef col) {
    DimensionEncoding result = encMap.get(col);
    if (result == null) {
        RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col);
        if (colDesc.isUsingDictionary()) {
            // special dictionary encoding
            Dictionary<String> dict = getDictionary(col);
            if (dict == null) {
                logger.warn("No dictionary found for dict-encoding column " + col + ", segment " + seg);
                result = new FixedLenDimEnc(0);
            } else {
                result = new DictionaryDimEnc(dict);
            }
        } else {
            // normal case
            result = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(), colDesc.getEncodingVersion());
        }
        encMap.put(col, result);
    }
    return result;
}
 
Example 11
Source Project: kylin   Source File: ParsedStreamingCubeInfo.java    License: Apache License 2.0 6 votes vote down vote up
public static DimensionEncoding[] getDimensionEncodings(CubeDesc cubeDesc, TblColRef[] dimensions,
        Map<TblColRef, Dictionary<String>> dimDictMap) {
    DimensionEncoding[] result = new DimensionEncoding[dimensions.length];
    for (int i = 0; i < dimensions.length; i++) {
        TblColRef dimension = dimensions[i];
        RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(dimension);
        if (colDesc.isUsingDictionary()) {
            @SuppressWarnings({ "unchecked" })
            Dictionary<String> dict = dimDictMap.get(dimension);
            if (dict == null) {
                throw new RuntimeException("No dictionary found for dict-encoding column " + dimension);
            } else {
                result[i] = new DictionaryDimEnc(dict);
            }
        } else {
            result[i] = DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs(),
                    colDesc.getEncodingVersion());
        }
    }
    return result;
}
 
Example 12
Source Project: kylin   Source File: ParsedStreamingCubeInfo.java    License: Apache License 2.0 6 votes vote down vote up
public void init(CubeDesc cubeDesc, CubeJoinedFlatTableEnrich intermediateTableDesc) {
    dimensions = Lists.newArrayList();
    columnsIndex = new int[Long.bitCount(cuboidID)];

    int colIdx = 0;
    RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
    for (int i = 0; i < allColumns.length; i++) {
        // NOTE: the order of column in list!!!
        long bitmask = 1L << allColumns[i].getBitIndex();
        if ((cuboidID & bitmask) != 0) {
            TblColRef colRef = allColumns[i].getColRef();
            dimensions.add(colRef);
            columnsIndex[colIdx] = intermediateTableDesc.getColumnIndex(colRef);
            colIdx++;
        }
    }
}
 
Example 13
Source Project: kylin-on-parquet-v2   Source File: CubeDescCreator.java    License: Apache License 2.0 5 votes vote down vote up
public static RowKeyColDesc getRowKeyColDesc(String tableName, String column, int id) {
    RowKeyColDesc rowKeyColDesc = new RowKeyColDesc();
    rowKeyColDesc.setIndex(Integer.toString(id));
    rowKeyColDesc.setColumn(tableName.substring(tableName.lastIndexOf(".") + 1) + "." + column);
    rowKeyColDesc.setEncoding(DictionaryDimEnc.ENCODING_NAME);
    rowKeyColDesc.setShardBy(false);
    return rowKeyColDesc;
}
 
Example 14
private long getQueryFilterMask(Set<TblColRef> filterColumnD) {
    long filterMask = 0;

    logger.info("Filter column set for query: {}", filterColumnD);
    if (filterColumnD.isEmpty() == false) {
        RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
        for (int i = 0; i < allColumns.length; i++) {
            if (filterColumnD.contains(allColumns[i].getColRef())) {
                filterMask |= 1L << allColumns[i].getBitIndex();
            }
        }
    }
    logger.info("Filter mask is: {}", filterMask);
    return filterMask;
}
 
Example 15
Source Project: kylin-on-parquet-v2   Source File: JoinedFlatTable.java    License: Apache License 2.0 5 votes vote down vote up
public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) {
    final String tableName = flatDesc.getTableName();
    StringBuilder sql = new StringBuilder();
    sql.append("INSERT OVERWRITE TABLE " + quoteIdentifier(tableName, null) + " SELECT * FROM " + quoteIdentifier(tableName, null));

    if (flatDesc.getClusterBy() != null) {
        appendClusterStatement(sql, flatDesc.getClusterBy());
    } else if (flatDesc.getDistributedBy() != null) {
        appendDistributeStatement(sql, Lists.newArrayList(flatDesc.getDistributedBy()));
    } else {
        int redistColumnCount = cubeDesc.getConfig().getHiveRedistributeColumnCount();

        RowKeyColDesc[] rowKeyColDescs = cubeDesc.getRowkey().getRowKeyColumns();

        if (rowKeyColDescs.length < redistColumnCount)
            redistColumnCount = rowKeyColDescs.length;

        List<TblColRef> redistColumns = Lists.newArrayListWithCapacity(redistColumnCount);

        for (int i = 0; i < redistColumnCount; i++) {
            redistColumns.add(rowKeyColDescs[i].getColRef());
        }

        appendDistributeStatement(sql, redistColumns);
    }

    return sql.toString();
}
 
Example 16
Source Project: kylin-on-parquet-v2   Source File: Cuboid.java    License: Apache License 2.0 5 votes vote down vote up
private List<TblColRef> translateIdToColumns(long cuboidID) {
    List<TblColRef> dimesnions = new ArrayList<TblColRef>();
    RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
    for (int i = 0; i < allColumns.length; i++) {
        // NOTE: the order of column in list!!!
        long bitmask = 1L << allColumns[i].getBitIndex();
        if ((cuboidID & bitmask) != 0) {
            TblColRef colRef = allColumns[i].getColRef();
            dimesnions.add(colRef);
        }
    }
    return dimesnions;
}
 
Example 17
Source Project: kylin-on-parquet-v2   Source File: RowKeySplitter.java    License: Apache License 2.0 5 votes vote down vote up
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
    this.enableSharding = cubeSeg.isEnableSharding();
    this.cubeDesc = cubeSeg.getCubeDesc();
    IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg);

    for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
        dimEncoding.get(rowKeyColDesc.getColRef());
    }

    this.colIO = new RowKeyColumnIO(dimEncoding);

    this.splitBuffers = new ByteArray[splitLen];
    this.splitOffsets = new int[splitLen];
    this.bufferSize = 0;
}
 
Example 18
Source Project: kylin-on-parquet-v2   Source File: ResponseResultSchema.java    License: Apache License 2.0 5 votes vote down vote up
private void init(Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics) {
    this.dimensions = new TblColRef[selectedDimensions.size()];
    this.metrics = new FunctionDesc[selectedMetrics.size()];
    this.measures = new MeasureDesc[selectedMetrics.size()];
    this.dimDataTypes = new DataType[dimensions.length];
    this.metricsDataTypes = new DataType[metrics.length];
    // sort dimensions according to the rowKey definition
    dimColIdxMap = Maps.newHashMap();
    RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();
    int colIdx = 0;
    for (RowKeyColDesc rowKeyColDesc : rowKeyDesc.getRowKeyColumns()) {
        TblColRef dimension = rowKeyColDesc.getColRef();
        if (selectedDimensions.contains(dimension)) {
            dimensions[colIdx] = dimension;
            dimDataTypes[colIdx] = dimension.getType();
            dimColIdxMap.put(dimension, colIdx);
            colIdx++;
        }
    }

    nDimensions = colIdx;

    colIdx = 0;
    // metrics
    metricsColIdxMap = Maps.newHashMap();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc func = measure.getFunction();
        if (selectedMetrics.contains(func)) {
            metrics[colIdx] = func;
            measures[colIdx] = measure;
            metricsColIdxMap.put(func.getParameter().getColRef(), colIdx);
            metricsDataTypes[colIdx] = func.getReturnDataType();
            colIdx++;
        }
    }

    nMetrics = colIdx;
}
 
Example 19
Source Project: kylin-on-parquet-v2   Source File: CubeController.java    License: Apache License 2.0 5 votes vote down vote up
private void validateColumnFamily(CubeDesc cubeDesc) {
    Set<String> columnFamilyMetricsSet = Sets.newHashSet();
    for (HBaseColumnFamilyDesc hBaseColumnFamilyDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc hBaseColumnDesc : hBaseColumnFamilyDesc.getColumns()) {
            for (String columnName : hBaseColumnDesc.getMeasureRefs()) {
                columnFamilyMetricsSet.add(columnName);
            }
        }
    }
    for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
        if (!columnFamilyMetricsSet.contains(measureDesc.getName())) {
            throw new BadRequestException("column family lack measure:" + measureDesc.getName());
        }
    }
    if (cubeDesc.getMeasures().size() != columnFamilyMetricsSet.size()) {
        throw new BadRequestException(
                "the number of input measure and the number of measure defined in cubedesc are not consistent");
    }

    for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
        Object[] encodingConf = DimensionEncoding.parseEncodingConf(rowKeyColDesc.getEncoding());
        String encodingName = (String) encodingConf[0];
        String[] encodingArgs = (String[]) encodingConf[1];

        if (!DimensionEncodingFactory.isValidEncoding(encodingName, encodingArgs,
                rowKeyColDesc.getEncodingVersion())) {
            throw new BadRequestException("Illegal row key column desc: " + rowKeyColDesc);
        }
    }
}
 
Example 20
Source Project: kylin   Source File: CubeDescCreator.java    License: Apache License 2.0 5 votes vote down vote up
public static RowKeyColDesc getRowKeyColDesc(String tableName, String column, int id) {
    RowKeyColDesc rowKeyColDesc = new RowKeyColDesc();
    rowKeyColDesc.setIndex(Integer.toString(id));
    rowKeyColDesc.setColumn(tableName.substring(tableName.lastIndexOf(".") + 1) + "." + column);
    rowKeyColDesc.setEncoding(DictionaryDimEnc.ENCODING_NAME);
    rowKeyColDesc.setShardBy(false);
    return rowKeyColDesc;
}
 
Example 21
Source Project: kylin   Source File: GTCubeStorageQueryBase.java    License: Apache License 2.0 5 votes vote down vote up
private long getQueryFilterMask(Set<TblColRef> filterColumnD) {
    long filterMask = 0;

    logger.info("Filter column set for query: {}", filterColumnD);
    if (filterColumnD.isEmpty() == false) {
        RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
        for (int i = 0; i < allColumns.length; i++) {
            if (filterColumnD.contains(allColumns[i].getColRef())) {
                filterMask |= 1L << allColumns[i].getBitIndex();
            }
        }
    }
    logger.info("Filter mask is: {}", filterMask);
    return filterMask;
}
 
Example 22
Source Project: kylin   Source File: JoinedFlatTable.java    License: Apache License 2.0 5 votes vote down vote up
public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) {
    final String tableName = flatDesc.getTableName();
    StringBuilder sql = new StringBuilder();
    sql.append("INSERT OVERWRITE TABLE " + quoteIdentifier(tableName, null) + " SELECT * FROM " + quoteIdentifier(tableName, null));

    if (flatDesc.getClusterBy() != null) {
        appendClusterStatement(sql, flatDesc.getClusterBy());
    } else if (flatDesc.getDistributedBy() != null) {
        appendDistributeStatement(sql, Lists.newArrayList(flatDesc.getDistributedBy()));
    } else {
        int redistColumnCount = cubeDesc.getConfig().getHiveRedistributeColumnCount();

        RowKeyColDesc[] rowKeyColDescs = cubeDesc.getRowkey().getRowKeyColumns();

        if (rowKeyColDescs.length < redistColumnCount)
            redistColumnCount = rowKeyColDescs.length;

        List<TblColRef> redistColumns = Lists.newArrayListWithCapacity(redistColumnCount);

        for (int i = 0; i < redistColumnCount; i++) {
            redistColumns.add(rowKeyColDescs[i].getColRef());
        }

        appendDistributeStatement(sql, redistColumns);
    }

    return sql.toString();
}
 
Example 23
Source Project: kylin   Source File: Cuboid.java    License: Apache License 2.0 5 votes vote down vote up
private List<TblColRef> translateIdToColumns(long cuboidID) {
    List<TblColRef> dimesnions = new ArrayList<TblColRef>();
    RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns();
    for (int i = 0; i < allColumns.length; i++) {
        // NOTE: the order of column in list!!!
        long bitmask = 1L << allColumns[i].getBitIndex();
        if ((cuboidID & bitmask) != 0) {
            TblColRef colRef = allColumns[i].getColRef();
            dimesnions.add(colRef);
        }
    }
    return dimesnions;
}
 
Example 24
Source Project: kylin   Source File: RowKeySplitter.java    License: Apache License 2.0 5 votes vote down vote up
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
    this.enableSharding = cubeSeg.isEnableSharding();
    this.cubeDesc = cubeSeg.getCubeDesc();
    IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg);

    for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
        dimEncoding.get(rowKeyColDesc.getColRef());
    }

    this.colIO = new RowKeyColumnIO(dimEncoding);

    this.splitBuffers = new ByteArray[splitLen];
    this.splitOffsets = new int[splitLen];
    this.bufferSize = 0;
}
 
Example 25
Source Project: kylin   Source File: ResponseResultSchema.java    License: Apache License 2.0 5 votes vote down vote up
private void init(Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics) {
    this.dimensions = new TblColRef[selectedDimensions.size()];
    this.metrics = new FunctionDesc[selectedMetrics.size()];
    this.measures = new MeasureDesc[selectedMetrics.size()];
    this.dimDataTypes = new DataType[dimensions.length];
    this.metricsDataTypes = new DataType[metrics.length];
    // sort dimensions according to the rowKey definition
    dimColIdxMap = Maps.newHashMap();
    RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();
    int colIdx = 0;
    for (RowKeyColDesc rowKeyColDesc : rowKeyDesc.getRowKeyColumns()) {
        TblColRef dimension = rowKeyColDesc.getColRef();
        if (selectedDimensions.contains(dimension)) {
            dimensions[colIdx] = dimension;
            dimDataTypes[colIdx] = dimension.getType();
            dimColIdxMap.put(dimension, colIdx);
            colIdx++;
        }
    }

    nDimensions = colIdx;

    colIdx = 0;
    // metrics
    metricsColIdxMap = Maps.newHashMap();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc func = measure.getFunction();
        if (selectedMetrics.contains(func)) {
            metrics[colIdx] = func;
            measures[colIdx] = measure;
            metricsColIdxMap.put(func.getParameter().getColRef(), colIdx);
            metricsDataTypes[colIdx] = func.getReturnDataType();
            colIdx++;
        }
    }

    nMetrics = colIdx;
}
 
Example 26
Source Project: kylin   Source File: CubeController.java    License: Apache License 2.0 5 votes vote down vote up
private void validateColumnFamily(CubeDesc cubeDesc) {
    Set<String> columnFamilyMetricsSet = Sets.newHashSet();
    for (HBaseColumnFamilyDesc hBaseColumnFamilyDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc hBaseColumnDesc : hBaseColumnFamilyDesc.getColumns()) {
            for (String columnName : hBaseColumnDesc.getMeasureRefs()) {
                columnFamilyMetricsSet.add(columnName);
            }
        }
    }
    for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
        if (!columnFamilyMetricsSet.contains(measureDesc.getName())) {
            throw new BadRequestException("column family lack measure:" + measureDesc.getName());
        }
    }
    if (cubeDesc.getMeasures().size() != columnFamilyMetricsSet.size()) {
        throw new BadRequestException(
                "the number of input measure and the number of measure defined in cubedesc are not consistent");
    }

    for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
        Object[] encodingConf = DimensionEncoding.parseEncodingConf(rowKeyColDesc.getEncoding());
        String encodingName = (String) encodingConf[0];
        String[] encodingArgs = (String[]) encodingConf[1];

        if (!DimensionEncodingFactory.isValidEncoding(encodingName, encodingArgs,
                rowKeyColDesc.getEncodingVersion())) {
            throw new BadRequestException("Illegal row key column desc: " + rowKeyColDesc);
        }
    }
}
 
Example 27
Source Project: Kylin   Source File: RowKeyAttrRule.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    RowKeyDesc row = cube.getRowkey();
    if (row == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
        return;
    }

    RowKeyColDesc[] rcd = row.getRowKeyColumns();
    if (rcd == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
        return;
    }
    if(rcd.length == 0){
        context.addResult(ResultLevel.ERROR, "Rowkey columns is empty");
        return;       	
    }

    for (int i = 0; i < rcd.length; i++) {
        RowKeyColDesc rd = rcd[i];
        if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary())&&!rd.getDictionary().equals("false"))) {
            context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute");
        }
        if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary())||rd.getDictionary().equals("false"))) {
            context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty");
        }
    }

}
 
Example 28
Source Project: Kylin   Source File: MandatoryColumnRule.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    Set<String> mands = new HashSet<String>();
    RowKeyColDesc[] cols = cube.getRowkey().getRowKeyColumns();
    if (cols == null || cols.length == 0) {
        return;
    }
    for (int i = 0; i < cols.length; i++) {
        RowKeyColDesc rowKeyColDesc = cols[i];
        if (rowKeyColDesc.isMandatory()) {
            mands.add(rowKeyColDesc.getColumn());
        }
    }
    if (mands.isEmpty()) {
        return;
    }
    String[][] groups = cube.getRowkey().getAggregationGroups();
    for (int i = 0; i < groups.length; i++) {
        String[] group = groups[i];
        for (int j = 0; j < group.length; j++) {
            String col = group[j];
            if (mands.contains(col)) {
                context.addResult(ResultLevel.ERROR, "mandatory column " + col + " must not be in aggregation group [" + ArrayUtils.toString(group) + "]");
            }
        }
    }

}
 
Example 29
Source Project: Kylin   Source File: Cuboid.java    License: Apache License 2.0 5 votes vote down vote up
private List<TblColRef> translateIdToColumns(long cuboidID) {
    List<TblColRef> dimesnions = new ArrayList<TblColRef>();
    RowKeyColDesc[] allColumns = cube.getRowkey().getRowKeyColumns();
    for (int i = 0; i < allColumns.length; i++) {
        // NOTE: the order of column in list!!!
        long bitmask = 1L << allColumns[i].getBitIndex();
        if ((cuboidID & bitmask) != 0) {
            TblColRef colRef = allColumns[i].getColRef();
            dimesnions.add(colRef);
        }
    }
    return dimesnions;
}
 
Example 30
Source Project: Kylin   Source File: CubeSizeEstimationCLI.java    License: Apache License 2.0 5 votes vote down vote up
private static int[] estimateRowKeyColSpace(RowKeyDesc rowKeyDesc, long[] cardinality) {
    RowKeyColDesc[] rowKeyColDescs = rowKeyDesc.getRowKeyColumns();
    int[] ret = new int[rowKeyColDescs.length];
    for (int i = 0; i < rowKeyColDescs.length; ++i) {
        RowKeyColDesc rowKeyColDesc = rowKeyColDescs[rowKeyColDescs.length - 1 - i];
        if (rowKeyColDesc.getDictionary() == null) {
            if (rowKeyColDesc.getLength() == 0)
                throw new IllegalStateException("The non-dictionary col " + rowKeyColDesc.getColumn() + " has length of 0");
            ret[i] = rowKeyColDesc.getLength();
        } else {
            ret[i] = estimateDictionaryColSpace(cardinality[i]);
        }
    }
    return ret;
}