/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.kylin.cube.model; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.net.util.Base64; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.common.persistence.ResourceStore; import org.apache.kylin.common.persistence.RootPersistentEntity; import org.apache.kylin.common.util.Array; import org.apache.kylin.common.util.CaseInsensitiveStringMap; import org.apache.kylin.common.util.JsonUtil; import org.apache.kylin.metadata.MetadataConstants; import org.apache.kylin.metadata.MetadataManager; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.DataType; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.JoinDesc; import org.apache.kylin.metadata.model.MeasureDesc; import org.apache.kylin.metadata.model.ParameterDesc; import org.apache.kylin.metadata.model.TableDesc; import org.apache.kylin.metadata.model.TblColRef; /** * Created with IntelliJ IDEA. User: lukhan Date: 9/24/13 Time: 10:40 AM To * change this template use File | Settings | File Templates. */ @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE) public class CubeDesc extends RootPersistentEntity { public static enum DeriveType { LOOKUP, PK_FK } public static class DeriveInfo { public DeriveType type; public DimensionDesc dimension; public TblColRef[] columns; public boolean isOneToOne; // only used when ref from derived to host DeriveInfo(DeriveType type, DimensionDesc dimension, TblColRef[] columns, boolean isOneToOne) { this.type = type; this.dimension = dimension; this.columns = columns; this.isOneToOne = isOneToOne; } @Override public String toString() { return "DeriveInfo [type=" + type + ", dimension=" + dimension + ", columns=" + Arrays.toString(columns) + ", isOneToOne=" + isOneToOne + "]"; } } private KylinConfig config; private DataModelDesc model; @JsonProperty("name") private String name; @JsonProperty("model_name") private String modelName; @JsonProperty("description") private String description; @JsonProperty("null_string") private String[] nullStrings; @JsonProperty("dimensions") private List<DimensionDesc> dimensions; @JsonProperty("measures") private List<MeasureDesc> measures; @JsonProperty("rowkey") private RowKeyDesc rowkey; @JsonProperty("hbase_mapping") private HBaseMappingDesc hbaseMapping; @JsonProperty("signature") private String signature; @JsonProperty("notify_list") private List<String> notifyList; private Map<String, Map<String, TblColRef>> columnMap = new HashMap<String, Map<String, TblColRef>>(); private LinkedHashSet<TblColRef> allColumns = new LinkedHashSet<TblColRef>(); private LinkedHashSet<TblColRef> dimensionColumns = new LinkedHashSet<TblColRef>(); private Map<TblColRef, DeriveInfo> derivedToHostMap = Maps.newHashMap(); private Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedMap = Maps.newHashMap(); /** * Error messages during resolving json metadata */ private List<String> errors = new ArrayList<String>(); /** * @return all columns this cube can support, including derived */ public Set<TblColRef> listAllColumns() { return allColumns; } /** * @return dimension columns including derived, BUT NOT measures */ public Set<TblColRef> listDimensionColumnsIncludingDerived() { return dimensionColumns; } /** * @return dimension columns excluding derived and measures */ public List<TblColRef> listDimensionColumnsExcludingDerived() { List<TblColRef> result = new ArrayList<TblColRef>(); for (TblColRef col : dimensionColumns) { if (isDerived(col) == false) result.add(col); } return result; } /** * Find FunctionDesc by Full Expression. * * @return */ public FunctionDesc findFunctionOnCube(FunctionDesc manualFunc) { for (MeasureDesc m : measures) { if (m.getFunction().equals(manualFunc)) return m.getFunction(); } return null; } public TblColRef findColumnRef(String table, String column) { Map<String, TblColRef> cols = columnMap.get(table); if (cols == null) return null; else return cols.get(column); } public DimensionDesc findDimensionByColumn(TblColRef col) { for (DimensionDesc dim : dimensions) { if (ArrayUtils.contains(dim.getColumnRefs(), col)) return dim; } return null; } public DimensionDesc findDimensionByTable(String lookupTableName) { lookupTableName = lookupTableName.toUpperCase(); for (DimensionDesc dim : dimensions) if (dim.getTable() != null && dim.getTable().equals(lookupTableName)) return dim; return null; } public DimensionDesc findDimensionByName(String dimName) { dimName = dimName.toUpperCase(); for (DimensionDesc dim : dimensions) { if (dimName.equals(dim.getName())) return dim; } return null; } /** * Get all functions from each measure. * * @return */ public List<FunctionDesc> listAllFunctions() { List<FunctionDesc> functions = new ArrayList<FunctionDesc>(); for (MeasureDesc m : measures) { functions.add(m.getFunction()); } return functions; } /** * @deprecated use getModel().getAllTables() instead * @return */ public List<TableDesc> listTables() { MetadataManager metaMgr = MetadataManager.getInstance(config); HashSet<String> tableNames = new HashSet<String>(); List<TableDesc> result = new ArrayList<TableDesc>(); tableNames.add(this.getFactTable().toUpperCase()); for (DimensionDesc dim : dimensions) { String table = dim.getTable(); if (table != null) tableNames.add(table.toUpperCase()); } for (String tableName : tableNames) { result.add(metaMgr.getTableDesc(tableName)); } return result; } public boolean isDerived(TblColRef col) { return derivedToHostMap.containsKey(col); } public DeriveInfo getHostInfo(TblColRef derived) { return derivedToHostMap.get(derived); } public Map<Array<TblColRef>, List<DeriveInfo>> getHostToDerivedInfo(List<TblColRef> rowCols, Collection<TblColRef> wantedCols) { Map<Array<TblColRef>, List<DeriveInfo>> result = new HashMap<Array<TblColRef>, List<DeriveInfo>>(); for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedMap.entrySet()) { Array<TblColRef> hostCols = entry.getKey(); boolean hostOnRow = rowCols.containsAll(Arrays.asList(hostCols.data)); if (!hostOnRow) continue; List<DeriveInfo> wantedInfo = new ArrayList<DeriveInfo>(); for (DeriveInfo info : entry.getValue()) { if (wantedCols == null || Collections.disjoint(wantedCols, Arrays.asList(info.columns)) == false) // has // any // wanted // columns? wantedInfo.add(info); } if (wantedInfo.size() > 0) result.put(hostCols, wantedInfo); } return result; } public String getResourcePath() { return getCubeDescResourcePath(name); } public static String getCubeDescResourcePath(String descName) { return ResourceStore.CUBE_DESC_RESOURCE_ROOT + "/" + descName + MetadataConstants.FILE_SURFIX; } // ============================================================================ public HBaseMappingDesc getHBaseMapping() { return hbaseMapping; } public void setHBaseMapping(HBaseMappingDesc hbaseMapping) { this.hbaseMapping = hbaseMapping; } public KylinConfig getConfig() { return config; } public void setConfig(KylinConfig config) { this.config = config; } public String getName() { return name; } public void setName(String name) { this.name = name; } public String getModelName() { return modelName; } public void setModelName(String modelName) { this.modelName = modelName; } public DataModelDesc getModel() { return model; } public void setModel(DataModelDesc model) { this.model = model; } public String getDescription() { return description; } public void setDescription(String description) { this.description = description; } public String getFactTable() { return model.getFactTable().toUpperCase(); } public String[] getNullStrings() { return nullStrings; } public List<DimensionDesc> getDimensions() { return dimensions; } public void setDimensions(List<DimensionDesc> dimensions) { this.dimensions = dimensions; } public List<MeasureDesc> getMeasures() { return measures; } public void setMeasures(List<MeasureDesc> measures) { this.measures = measures; } public RowKeyDesc getRowkey() { return rowkey; } public void setRowkey(RowKeyDesc rowkey) { this.rowkey = rowkey; } public String getSignature() { return signature; } public void setSignature(String signature) { this.signature = signature; } public List<String> getNotifyList() { return notifyList; } public void setNotifyList(List<String> notifyList) { this.notifyList = notifyList; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; CubeDesc cubeDesc = (CubeDesc) o; if (!name.equals(cubeDesc.name)) return false; if (!getFactTable().equals(cubeDesc.getFactTable())) return false; return true; } @Override public int hashCode() { int result = 0; result = 31 * result + name.hashCode(); result = 31 * result + getFactTable().hashCode(); return result; } @Override public String toString() { return "CubeDesc [name=" + name + "]"; } public String calculateSignature() { MessageDigest md = null; try { md = MessageDigest.getInstance("MD5"); StringBuilder sigString = new StringBuilder(); sigString.append(this.name).append("|").append(this.getFactTable()).append("|").append(JsonUtil.writeValueAsString(this.model.getPartitionDesc())).append("|").append(JsonUtil.writeValueAsString(this.dimensions)).append("|").append(JsonUtil.writeValueAsString(this.measures)).append("|").append(JsonUtil.writeValueAsString(this.rowkey)).append("|").append(JsonUtil.writeValueAsString(this.hbaseMapping)); byte[] signature = md.digest(sigString.toString().getBytes()); return new String(Base64.encodeBase64(signature)); } catch (NoSuchAlgorithmException | JsonProcessingException e) { throw new RuntimeException("Failed to calculate signature"); } } public Map<String, TblColRef> buildColumnNameAbbreviation() { Map<String, TblColRef> r = new CaseInsensitiveStringMap<TblColRef>(); for (TblColRef col : listDimensionColumnsExcludingDerived()) { r.put(col.getName(), col); } return r; } public void init(KylinConfig config, Map<String, TableDesc> tables) { this.errors.clear(); this.config = config; if (this.modelName == null || this.modelName.length() == 0) { this.addError("The cubeDesc '" + this.getName() + "' doesn't have data model specified."); } this.model = MetadataManager.getInstance(config).getDataModelDesc(this.modelName); if (this.model == null) { this.addError("No data model found with name '" + modelName + "'."); } for (DimensionDesc dim : dimensions) { dim.init(this, tables); } sortDimAndMeasure(); initDimensionColumns(tables); initMeasureColumns(tables); rowkey.init(this); if (hbaseMapping != null) { hbaseMapping.init(this); } initMeasureReferenceToColumnFamily(); // check all dimension columns are presented on rowkey List<TblColRef> dimCols = listDimensionColumnsExcludingDerived(); if (rowkey.getRowKeyColumns().length != dimCols.size()) { addError("RowKey columns count (" + rowkey.getRowKeyColumns().length + ") does not match dimension columns count (" + dimCols.size() + "). "); } } private void initDimensionColumns(Map<String, TableDesc> tables) { for (DimensionDesc dim : dimensions) { JoinDesc join = dim.getJoin(); // init dimension columns ArrayList<TblColRef> dimCols = Lists.newArrayList(); String[] colStrs = dim.getColumn(); // when column is omitted, special case if (colStrs == null && dim.isDerived() || ArrayUtils.contains(colStrs, "{FK}")) { for (TblColRef col : join.getForeignKeyColumns()) { dimCols.add(initDimensionColRef(col)); } } // normal case else { if (colStrs == null || colStrs.length == 0) throw new IllegalStateException("Dimension column must not be blank " + dim); for (String colStr : colStrs) { dimCols.add(initDimensionColRef(dim, colStr)); } // fill back column ref in hierarchy if (dim.isHierarchy()) { for (int i = 0; i < dimCols.size(); i++) dim.getHierarchy()[i].setColumnRef(dimCols.get(i)); } } TblColRef[] dimColArray = (TblColRef[]) dimCols.toArray(new TblColRef[dimCols.size()]); dim.setColumnRefs(dimColArray); // init derived columns TblColRef[] hostCols = dimColArray; if (dim.isDerived()) { String[] derived = dim.getDerived(); String[][] split = splitDerivedColumnAndExtra(derived); String[] derivedNames = split[0]; String[] derivedExtra = split[1]; TblColRef[] derivedCols = new TblColRef[derivedNames.length]; for (int i = 0; i < derivedNames.length; i++) { derivedCols[i] = initDimensionColRef(dim, derivedNames[i]); } initDerivedMap(hostCols, DeriveType.LOOKUP, dim, derivedCols, derivedExtra); } // PK-FK derive the other side if (join != null) { TblColRef[] fk = join.getForeignKeyColumns(); TblColRef[] pk = join.getPrimaryKeyColumns(); for (int i = 0; i < fk.length; i++) { int find = ArrayUtils.indexOf(hostCols, fk[i]); if (find >= 0) { TblColRef derivedCol = initDimensionColRef(pk[i]); initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol); } } for (int i = 0; i < pk.length; i++) { int find = ArrayUtils.indexOf(hostCols, pk[i]); if (find >= 0) { TblColRef derivedCol = initDimensionColRef(fk[i]); initDerivedMap(hostCols[find], DeriveType.PK_FK, dim, derivedCol); } } } } } private String[][] splitDerivedColumnAndExtra(String[] derived) { String[] cols = new String[derived.length]; String[] extra = new String[derived.length]; for (int i = 0; i < derived.length; i++) { String str = derived[i]; int cut = str.indexOf(":"); if (cut >= 0) { cols[i] = str.substring(0, cut); extra[i] = str.substring(cut + 1).trim(); } else { cols[i] = str; extra[i] = ""; } } return new String[][] { cols, extra }; } private void initDerivedMap(TblColRef hostCol, DeriveType type, DimensionDesc dimension, TblColRef derivedCol) { initDerivedMap(new TblColRef[] { hostCol }, type, dimension, new TblColRef[] { derivedCol }, null); } private void initDerivedMap(TblColRef[] hostCols, DeriveType type, DimensionDesc dimension, TblColRef[] derivedCols, String[] extra) { if (hostCols.length == 0 || derivedCols.length == 0) throw new IllegalStateException("host/derived columns must not be empty"); Array<TblColRef> hostColArray = new Array<TblColRef>(hostCols); List<DeriveInfo> infoList = hostToDerivedMap.get(hostColArray); if (infoList == null) { hostToDerivedMap.put(hostColArray, infoList = new ArrayList<DeriveInfo>()); } infoList.add(new DeriveInfo(type, dimension, derivedCols, false)); for (int i = 0; i < derivedCols.length; i++) { TblColRef derivedCol = derivedCols[i]; boolean isOneToOne = type == DeriveType.PK_FK || ArrayUtils.contains(hostCols, derivedCol) || (extra != null && extra[i].contains("1-1")); derivedToHostMap.put(derivedCol, new DeriveInfo(type, dimension, hostCols, isOneToOne)); } } private TblColRef initDimensionColRef(DimensionDesc dim, String colName) { TableDesc table = dim.getTableDesc(); ColumnDesc col = table.findColumnByName(colName); if (col == null) throw new IllegalArgumentException("No column '" + colName + "' found in table " + table); TblColRef ref = new TblColRef(col); return initDimensionColRef(ref); } private TblColRef initDimensionColRef(TblColRef ref) { TblColRef existing = findColumnRef(ref.getTable(), ref.getName()); if (existing != null) { return existing; } allColumns.add(ref); dimensionColumns.add(ref); Map<String, TblColRef> cols = columnMap.get(ref.getTable()); if (cols == null) { columnMap.put(ref.getTable(), cols = new HashMap<String, TblColRef>()); } cols.put(ref.getName(), ref); return ref; } private void initMeasureColumns(Map<String, TableDesc> tables) { if (measures == null || measures.isEmpty()) { return; } TableDesc factTable = tables.get(getFactTable()); for (MeasureDesc m : measures) { m.setName(m.getName().toUpperCase()); if (m.getDependentMeasureRef() != null) { m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase()); } FunctionDesc f = m.getFunction(); f.setExpression(f.getExpression().toUpperCase()); f.setReturnDataType(DataType.getInstance(f.getReturnType())); ParameterDesc p = f.getParameter(); p.normalizeColumnValue(); if (p.isColumnType()) { ArrayList<TblColRef> colRefs = Lists.newArrayList(); for (String cName : p.getValue().split("\\s*,\\s*")) { ColumnDesc sourceColumn = factTable.findColumnByName(cName); TblColRef colRef = new TblColRef(sourceColumn); colRefs.add(colRef); allColumns.add(colRef); } if (colRefs.isEmpty() == false) p.setColRefs(colRefs); } // verify holistic count distinct as a dependent measure if (m.isHolisticCountDistinct() && StringUtils.isBlank(m.getDependentMeasureRef())) { throw new IllegalStateException(m + " is a holistic count distinct but it has no DependentMeasureRef defined!"); } } } private void initMeasureReferenceToColumnFamily() { if (measures == null || measures.size() == 0) return; Map<String, MeasureDesc> measureCache = new HashMap<String, MeasureDesc>(); for (MeasureDesc m : measures) measureCache.put(m.getName(), m); for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) { for (HBaseColumnDesc c : cf.getColumns()) { MeasureDesc[] measureDescs = new MeasureDesc[c.getMeasureRefs().length]; for (int i = 0; i < c.getMeasureRefs().length; i++) { measureDescs[i] = measureCache.get(c.getMeasureRefs()[i]); } c.setMeasures(measureDescs); c.setColumnFamilyName(cf.getName()); } } } private void sortDimAndMeasure() { sortDimensionsByID(); sortMeasuresByID(); for (DimensionDesc dim : dimensions) { sortHierarchiesByLevel(dim.getHierarchy()); } } private void sortDimensionsByID() { Collections.sort(dimensions, new Comparator<DimensionDesc>() { @Override public int compare(DimensionDesc d1, DimensionDesc d2) { Integer id1 = d1.getId(); Integer id2 = d2.getId(); return id1.compareTo(id2); } }); } private void sortMeasuresByID() { if (measures == null) { measures = Lists.newArrayList(); } Collections.sort(measures, new Comparator<MeasureDesc>() { @Override public int compare(MeasureDesc m1, MeasureDesc m2) { Integer id1 = m1.getId(); Integer id2 = m2.getId(); return id1.compareTo(id2); } }); } private void sortHierarchiesByLevel(HierarchyDesc[] hierarchies) { if (hierarchies != null) { Arrays.sort(hierarchies, new Comparator<HierarchyDesc>() { @Override public int compare(HierarchyDesc h1, HierarchyDesc h2) { Integer level1 = Integer.parseInt(h1.getLevel()); Integer level2 = Integer.parseInt(h2.getLevel()); return level1.compareTo(level2); } }); } } public boolean hasHolisticCountDistinctMeasures() { for (MeasureDesc measure : measures) { if (measure.getFunction().isHolisticCountDistinct()) { return true; } } return false; } /** * Add error info and thrown exception out * * @param message */ public void addError(String message) { addError(message, false); } /** * @param message * error message * @param silent * if throw exception */ public void addError(String message, boolean silent) { if (!silent) { throw new IllegalStateException(message); } else { this.errors.add(message); } } public List<String> getError() { return this.errors; } public HBaseMappingDesc getHbaseMapping() { return hbaseMapping; } public void setHbaseMapping(HBaseMappingDesc hbaseMapping) { this.hbaseMapping = hbaseMapping; } public void setNullStrings(String[] nullStrings) { this.nullStrings = nullStrings; } }