org.apache.kylin.metadata.realization.IRealization Java Examples

public static IRealization selectRealization(OLAPContext olapContext) throws NoRealizationFoundException {

        ProjectManager prjMgr = ProjectManager.getInstance(olapContext.olapSchema.getConfig());
        String factTableName = olapContext.firstTableScan.getTableName();
        String projectName = olapContext.olapSchema.getProjectName();
        List<IRealization> realizations = Lists.newArrayList(prjMgr.getRealizationsByTable(projectName, factTableName));
        logger.info("Find candidates by table " + factTableName + " and project=" + projectName + " : " + StringUtils.join(realizations, ","));

        //rule based realization selection, rules might reorder realizations or remove specific realization
        RoutingRule.applyRules(realizations, olapContext);

        if (realizations.size() == 0) {
            throw new NoRealizationFoundException("Can't find any realization. Please confirm with providers. SQL digest: " + olapContext.getSQLDigest().toString());
        }

        logger.info("The realizations remaining: ");
        logger.info(RoutingRule.getPrintableText(realizations));
        logger.info("The realization being chosen: " + realizations.get(0).getName());

        return realizations.get(0);
    }
 

public static List<Dictionary<String>> getAllDictionaries(TblColRef col, IRealization realization) {
    Set<Dictionary<String>> result = Sets.newHashSet();
    if (realization instanceof CubeInstance) {
        final CubeInstance cube = (CubeInstance) realization;
        for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) {
            result.add(segment.getDictionary(col));
        }
    } else if (realization instanceof HybridInstance) {
        final HybridInstance hybridInstance = (HybridInstance) realization;
        for (IRealization entry : hybridInstance.getRealizations()) {
            result.addAll(getAllDictionaries(col, entry));
        }
    } else {
        throw new IllegalStateException("All leaf realizations should be CubeInstance");
    }
    return Lists.newArrayList(result);
}
 

private String getRootFactTableForRealization(IRealization realization) {
    if (realization == null) {
        logger.warn("Cannot find realization %s", realization);
        return null;
    }
    DataModelDesc model = realization.getModel();
    if (model == null) {
        logger.warn("The model for realization %s is null", realization.getName());
        return null;
    }
    TableRef rootFactTable = model.getRootFactTable();
    if (rootFactTable == null) {
        logger.warn("The root table for model %s is null", model.getName());
        return null;
    }
    return rootFactTable.getTableIdentity();
}
 

private void wireProjectAndRealizations(Collection<ProjectInstance> projectInstances) {
    if (projectInstances.isEmpty())
        return;

    RealizationRegistry registry = RealizationRegistry.getInstance(config);
    for (ProjectInstance projectInstance : projectInstances) {
        for (RealizationEntry realization : projectInstance.getRealizationEntries()) {
            IRealization rel = registry.getRealization(realization.getType(), realization.getRealization());
            if (rel != null) {
                rel.setProjectName(projectInstance.getName());
            } else {
                logger.warn("Realization '" + realization + "' defined under project '" + projectInstance + "' is not found");
            }
        }
    }
}
 

static HybridSignature getHybridSignature(KylinConfig config, String realizationName) {
    HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(realizationName);
    if (hybridInstance == null) {
        return null;
    }
    IRealization[] realizations = hybridInstance.getRealizations();
    Set<RealizationSignature> realizationSignatureSet = Sets.newHashSetWithExpectedSize(realizations.length);
    for (IRealization realization : realizations) {
        RealizationSignature realizationSignature = null;
        if (realization.getType() == RealizationType.CUBE) {
            realizationSignature = CubeSignature.getCubeSignature(config, realization.getName());
        } else if (realization.getType() == RealizationType.HYBRID) {
            realizationSignature = getHybridSignature(config, realization.getName());
        }
        if (realizationSignature != null) {
            realizationSignatureSet.add(realizationSignature);
        }
    }
    return new HybridSignature(realizationName, realizationSignatureSet);
}
 

@Override
public CapabilityResult isCapable(SQLDigest digest) {
    CapabilityResult result = new CapabilityResult();
    result.cost = Integer.MAX_VALUE;

    for (IRealization realization : getRealizations()) {
        CapabilityResult child = realization.isCapable(digest);
        if (child.capable) {
            result.capable = true;
            result.cost = Math.min(result.cost, child.cost);
            result.influences.addAll(child.influences);
        } else {
            result.incapableCause = child.incapableCause;
        }
    }

    if (result.cost > 0)
        result.cost--; // let hybrid win its children

    return result;
}
 

@Override
public void apply(List<IRealization> realizations, OLAPContext olapContext) {
    if (realizations.size() > 0) {
        IRealization first = realizations.get(0);

        if (first instanceof CubeInstance) {
            CubeInstance cube = (CubeInstance) first;
            adjustOLAPContextIfNecessary(cube, olapContext);
        }

        if (first instanceof IIInstance) {
            IIInstance ii = (IIInstance) first;
            adjustOLAPContextIfNecessary(ii, olapContext);
        }
    }
}
 

@Override
public CapabilityResult isCapable(SQLDigest digest) {
    CapabilityResult result = new CapabilityResult();
    result.cost = Integer.MAX_VALUE;

    for (IRealization realization : getRealizations()) {
        CapabilityResult child = realization.isCapable(digest);
        if (child.capable) {
            result.capable = true;
            result.cost = Math.min(result.cost, child.cost);
            result.influences.addAll(child.influences);
        } else {
            result.incapableCause = child.incapableCause;
        }
    }

    if (result.cost > 0)
        result.cost--; // let hybrid win its children

    return result;
}
 

private void resetRealizationInContext(OLAPContext olapContext) {
    IRealization realization = olapContext.realization;
    if (realization == null) {
        return;
    }
    KylinConfig config = getConfig();
    HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(realization.getName());
    if (hybridInstance != null) {
        olapContext.realization = hybridInstance;
        return;
    }
    CubeInstance cubeInstance = CubeManager.getInstance(config).getCube(realization.getName());
    if (cubeInstance != null) {
        olapContext.realization = cubeInstance;
    }
}
 

public List<MeasureDesc> listEffectiveRewriteMeasures(String project, String table, boolean onlyRewriteMeasure) {
    Set<IRealization> realizations = getRealizationsByTable(project, table);
    List<MeasureDesc> result = Lists.newArrayList();
    for (IRealization r : realizations) {
        if (!r.isReady())
            continue;

        for (MeasureDesc m : r.getMeasures()) {
            FunctionDesc func = m.getFunction();
            if (belongToTable(func, table, r.getModel())) {
                if (!onlyRewriteMeasure || func.needRewrite()) {
                    result.add(m);
                }
            }
        }
    }
    return result;
}
 

public static String getPrintableText(List<IRealization> realizations) {
    StringBuffer sb = new StringBuffer();
    sb.append("[");
    for (IRealization r : realizations) {
        sb.append(r.getName());
        sb.append(",");
    }
    if (sb.charAt(sb.length() - 1) != '[')
        sb.deleteCharAt(sb.length() - 1);
    sb.append("]");
    return sb.toString();
}
 

private void markExposedTablesAndColumns(ProjectCache prjCache, IRealization realization) {
    if (!realization.isReady()) {
        return;
    }

    for (TblColRef col : realization.getAllColumns()) {
        TableCache tableCache = prjCache.tables.get(col.getTable());
        prjCache.exposedTables.add(tableCache.tableDesc);
        tableCache.exposed = true;
        tableCache.exposedColumns.add(col.getColumnDesc());
    }
}
 

protected List<Integer> findRealizationsOf(List<IRealization> realizations, RealizationType type) {
    List<Integer> itemIndexes = Lists.newArrayList();
    for (int i = 0; i < realizations.size(); ++i) {
        if (realizations.get(i).getType() == type) {
            itemIndexes.add(i);
        }
    }
    return itemIndexes;
}
 

public static String getPrintableText(List<Candidate> candidates) {
    StringBuffer sb = new StringBuffer();
    sb.append("[");
    for (Candidate candidate : candidates) {
        IRealization r = candidate.realization;
        sb.append(r.getCanonicalName());
        sb.append(",");
    }
    if (sb.charAt(sb.length() - 1) != '[')
        sb.deleteCharAt(sb.length() - 1);
    sb.append("]");
    return sb.toString();
}
 

@Override
public int getCost() {
    int c = Integer.MAX_VALUE;
    for (IRealization realization : getRealizations()) {
        c = Math.min(realization.getCost(), c);
    }
    return c;
}
 

public static String getPrintableText(List<Candidate> candidates) {
    StringBuffer sb = new StringBuffer();
    sb.append("[");
    for (Candidate candidate : candidates) {
        IRealization r = candidate.realization;
        sb.append(r.getCanonicalName());
        sb.append(",");
    }
    if (sb.charAt(sb.length() - 1) != '[')
        sb.deleteCharAt(sb.length() - 1);
    sb.append("]");
    return sb.toString();
}
 

public List<MeasureDesc> listEffectiveRewriteMeasures(String project, String factTable) {
    Set<IRealization> realizations = getRealizationsByTable(project, factTable);
    List<MeasureDesc> result = Lists.newArrayList();
    for (IRealization r : realizations) {
        if (r.getFactTable().equalsIgnoreCase(factTable) && r.isReady()) {
            for (MeasureDesc m : r.getMeasures()) {
                FunctionDesc func = m.getFunction();
                if (func.needRewrite())
                    result.add(m);
            }
        }
    }
    return result;
}
 

private void retrieveResourcePath(IRealization realization) throws IOException {
    if (realization == null) {
        return;
    }
    logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType());
    if (realization instanceof CubeInstance) {
        CubeInstance cube = (CubeInstance) realization;
        CubeDesc cubeDesc = cubeDescManager.getCubeDesc(cube.getDescName());
        DataModelDesc modelDesc = metadataManager.getDataModelDesc(cubeDesc.getModelName());
        // add tables
        addTables(modelDesc);
        // add streaming stuff
        addStreamingConfig(cube);
        // add streamingV2
        addStreamingV2Config(cube);
        // add cube
        addRequired(CubeDesc.concatResourcePath(cubeDesc.getName()));
        // add project
        addRequired(ProjectInstance.concatResourcePath(cube.getProject()));
        //add Segments and Jobs
        addSegAndJob(cube);

    } else if (realization instanceof HybridInstance) {
        HybridInstance hybridInstance = (HybridInstance) realization;
        addRequired(HybridInstance.concatResourcePath(hybridInstance.getName()));
        for (IRealization iRealization : hybridInstance.getRealizations()) {
            if (iRealization.getType() != RealizationType.CUBE) {
                throw new RuntimeException("Hybrid " + iRealization.getName() + " contains non cube child "
                        + iRealization.getName() + " with type " + iRealization.getType());
            }
            retrieveResourcePath(iRealization);
        }
    } else {
        logger.warn("Unknown realization type: " + realization.getType());
    }
}
 

private void addRealization(IRealization realization) {
    if (realization instanceof HybridInstance) {
        addHybrid((HybridInstance) realization);
    } else if (realization instanceof CubeInstance) {
        cubes.add((CubeInstance) realization);
    } else {
        logger.warn("Realization {} is neither hybrid nor cube", realization);
    }
}
 

@Override
public void apply(List<IRealization> realizations, OLAPContext olapContext) {
    for (Iterator<IRealization> iterator = realizations.iterator(); iterator.hasNext();) {
        IRealization realization = iterator.next();
        if (!realization.isCapable(olapContext.getSQLDigest())) {
            iterator.remove();
        }
    }
}
 

@Test
public void testBasics() throws Exception {
    HybridInstance hybridInstance = getHybridManager().getHybridInstance("test_kylin_hybrid_ready");
    System.out.println(JsonUtil.writeValueAsIndentString(hybridInstance));

    IRealization[] realizations = hybridInstance.getRealizations();
    Assert.assertEquals(realizations.length, 2);

    IRealization lastReal = hybridInstance.getLatestRealization();
    Assert.assertTrue(lastReal instanceof CubeInstance);
    Assert.assertEquals(lastReal.getName(), "test_kylin_cube_with_slr_ready_2_segments");

}
 

private IRealization getRealization(KylinConfig config, String name) {
    HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(name);
    if (hybridInstance != null) {
        return hybridInstance;
    }
    return CubeManager.getInstance(config).getCube(name);
}
 

private IRealization getRealization(KylinConfig config, String name) {
    HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(name);
    if (hybridInstance != null) {
        return hybridInstance;
    }
    return CubeManager.getInstance(config).getCube(name);
}
 

@Override
public IStorageQuery createQuery(IRealization realization) {

    if (realization.getType() == RealizationType.CUBE) {

        CubeInstance cubeInstance = (CubeInstance) realization;
        String cubeStorageQuery;
        if (cubeInstance.getStorageType() == IStorageAware.ID_HBASE) {//v2 query engine cannot go with v1 storage now
            throw new IllegalStateException(
                    "Storage Engine (id=" + IStorageAware.ID_HBASE + ") is not supported any more");
        } else {
            cubeStorageQuery = v2CubeStorageQuery;//by default use v2
        }

        IStorageQuery ret;
        try {
            ret = (IStorageQuery) Class.forName(cubeStorageQuery).getConstructor(CubeInstance.class)
                    .newInstance((CubeInstance) realization);
        } catch (Exception e) {
            throw new RuntimeException("Failed to initialize storage query for " + cubeStorageQuery, e);
        }

        return ret;
    } else {
        throw new IllegalArgumentException("Unknown realization type " + realization.getType());
    }
}
 

private static TblColRef getPartitionCol(IRealization realization) {
    String modelName = realization.getModel().getName();
    DataModelDesc dataModelDesc = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getDataModelDesc(modelName);
    PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
    Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
    TblColRef partitionColRef = partitionDesc.getPartitionDateColumnRef();
    Preconditions.checkArgument(partitionColRef != null,
            "getPartitionDateColumnRef for " + realization + " is null");
    return partitionColRef;
}
 

@Override
public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException {
    try (AutoLock l = lock.lockForWrite()) {
        for (IRealization real : ProjectManager.getInstance(config).listAllRealizations(project)) {
            if (real instanceof HybridInstance) {
                crud.reloadQuietly(real.getName());
            }
        }
    }
}
 

@Override
public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException {
    ProjectManager projectManager = ProjectManager.getInstance(config);
    for (IRealization real : projectManager.listAllRealizations(project)) {
        if (real instanceof CubeInstance) {
            reloadCubeQuietly(real.getName());
        }
    }
    projectManager.reloadProjectL2Cache(project);
}
 

/**
 * To keep "select * from LOOKUP_TABLE" has consistent and latest result, we manually choose
 * CubeInstance here to answer such query.
 */
public CubeInstance findLatestSnapshot(List<RealizationEntry> realizationEntries, String lookupTableName,
        CubeInstance cubeInstance) {
    CubeInstance cube = null;
    try {
        if (!realizationEntries.isEmpty()) {
            long maxBuildTime = Long.MIN_VALUE;
            RealizationRegistry registry = RealizationRegistry.getInstance(config);
            for (RealizationEntry entry : realizationEntries) {
                IRealization realization = registry.getRealization(entry.getType(), entry.getRealization());
                if (realization != null && realization.isReady() && realization instanceof CubeInstance) {
                    CubeInstance current = (CubeInstance) realization;
                    if (checkMeetSnapshotTable(current, lookupTableName)) {
                        CubeSegment segment = current.getLatestReadySegment();
                        if (segment != null) {
                            long latestBuildTime = segment.getLastBuildTime();
                            if (latestBuildTime > maxBuildTime) {
                                maxBuildTime = latestBuildTime;
                                cube = current;
                            }
                        }
                    }
                }
            }
        }
    } catch (Exception e) {
        logger.info("Unexpected error.", e);
    }
    if (!cubeInstance.equals(cube)) {
        logger.debug("Picked cube {} over {} as it provides a more recent snapshot of the lookup table {}", cube,
                cubeInstance, lookupTableName);
    }
    return cube == null ? cubeInstance : cube;
}
 

@Override
public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException {
    for (IRealization real : ProjectManager.getInstance(config).listAllRealizations(project)) {
        if (real instanceof CubeInstance) {
            String descName = ((CubeInstance) real).getDescName();
            reloadCubeDescQuietly(descName);
        }
    }
}
 

public static void applyRules(List<IRealization> realizations, OLAPContext olapContext) {
    for (RoutingRule rule : rules) {
        logger.info("Initial realizations order:");
        logger.info(getPrintableText(realizations));
        logger.info("Applying rule " + rule);

        rule.apply(realizations, olapContext);

        logger.info(getPrintableText(realizations));
        logger.info("===================================================");
    }
}