Java Code Examples for org.apache.kylin.metadata.realization.RealizationType#CUBE

The following examples show how to use org.apache.kylin.metadata.realization.RealizationType#CUBE . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: RealizationSignature.java From kylin with Apache License 2.0

6 votes

static HybridSignature getHybridSignature(KylinConfig config, String realizationName) {
    HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(realizationName);
    if (hybridInstance == null) {
        return null;
    }
    IRealization[] realizations = hybridInstance.getRealizations();
    Set<RealizationSignature> realizationSignatureSet = Sets.newHashSetWithExpectedSize(realizations.length);
    for (IRealization realization : realizations) {
        RealizationSignature realizationSignature = null;
        if (realization.getType() == RealizationType.CUBE) {
            realizationSignature = CubeSignature.getCubeSignature(config, realization.getName());
        } else if (realization.getType() == RealizationType.HYBRID) {
            realizationSignature = getHybridSignature(config, realization.getName());
        }
        if (realizationSignature != null) {
            realizationSignatureSet.add(realizationSignature);
        }
    }
    return new HybridSignature(realizationName, realizationSignatureSet);
}

Example 2

Source File: CubeService.java From kylin with Apache License 2.0

6 votes

protected boolean isCubeInProject(String projectName, CubeInstance target) {
    ProjectManager projectManager = getProjectManager();
    ProjectInstance project = projectManager.getProject(projectName);
    if (project == null) {
        return false;
    }
    for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
        if (projectDataModel.getType() == RealizationType.CUBE) {
            CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
            if (cube == null) {
                logger.error("Project " + projectName + " contains realization " + projectDataModel.getRealization()
                        + " which is not found by CubeManager");
                continue;
            }
            if (cube.equals(target)) {
                return true;
            }
        }
    }
    return false;
}

Example 3

Source File: CubeService.java From kylin with Apache License 2.0

6 votes

public List<CubeInstance> listAllCubes(String projectName) {
    ProjectManager projectManager = getProjectManager();
    ProjectInstance project = projectManager.getProject(projectName);
    if (project == null) {
        return Collections.emptyList();
    }
    ArrayList<CubeInstance> result = new ArrayList<CubeInstance>();
    for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
        if (projectDataModel.getType() == RealizationType.CUBE) {
            CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
            if (cube != null)
                result.add(cube);
            else
                logger.error("Cube instance " + projectDataModel.getRealization() + " is failed to load");
        }
    }
    return result;
}

Example 4

Source File: HybridCubeCLI.java From kylin with Apache License 2.0

6 votes

private void checkSegmentOffset(List<RealizationEntry> realizationEntries) {
    List<SegmentRange> segmentRanges = Lists.newArrayList();

    for (RealizationEntry entry : realizationEntries) {
        if (entry.getType() != RealizationType.CUBE) {
            throw new IllegalArgumentException("Wrong realization type: " + entry.getType() + ", only cube supported. ");
        }

        CubeInstance cubeInstance = cubeManager.getCube(entry.getRealization());
        Segments<CubeSegment> segments = cubeInstance.getSegments();

        for (CubeSegment segment : segments) {
            segmentRanges.add(segment.getSegRange());
        }
    }

    if (segmentRanges.size() >= 2) {
        Collections.sort(segmentRanges);

        for (int i = 0; i < segmentRanges.size() - 1; i++) {
            if (segmentRanges.get(i).overlaps(segmentRanges.get(i + 1))) {
                throw new IllegalArgumentException("Segments has overlap, could not hybrid. First Segment Range: [" + segmentRanges.get(i).start.v + "," + segmentRanges.get(i).end.v + "], Second Segment Range: [" + segmentRanges.get(i + 1).start.v + "," + segmentRanges.get(i + 1).end.v + "]");
            }
        }
    }
}

Example 5

Source File: HybridCubeCLI.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private void checkSegmentOffset(List<RealizationEntry> realizationEntries) {
    List<SegmentRange> segmentRanges = Lists.newArrayList();

    for (RealizationEntry entry : realizationEntries) {
        if (entry.getType() != RealizationType.CUBE) {
            throw new IllegalArgumentException("Wrong realization type: " + entry.getType() + ", only cube supported. ");
        }

        CubeInstance cubeInstance = cubeManager.getCube(entry.getRealization());
        Segments<CubeSegment> segments = cubeInstance.getSegments();

        for (CubeSegment segment : segments) {
            segmentRanges.add(segment.getSegRange());
        }
    }

    if (segmentRanges.size() >= 2) {
        Collections.sort(segmentRanges);

        for (int i = 0; i < segmentRanges.size() - 1; i++) {
            if (segmentRanges.get(i).overlaps(segmentRanges.get(i + 1))) {
                throw new IllegalArgumentException("Segments has overlap, could not hybrid. First Segment Range: [" + segmentRanges.get(i).start.v + "," + segmentRanges.get(i).end.v + "], Second Segment Range: [" + segmentRanges.get(i + 1).start.v + "," + segmentRanges.get(i + 1).end.v + "]");
            }
        }
    }
}

Example 6

Source File: RealizationSignature.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

static HybridSignature getHybridSignature(KylinConfig config, String realizationName) {
    HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(realizationName);
    if (hybridInstance == null) {
        return null;
    }
    IRealization[] realizations = hybridInstance.getRealizations();
    Set<RealizationSignature> realizationSignatureSet = Sets.newHashSetWithExpectedSize(realizations.length);
    for (IRealization realization : realizations) {
        RealizationSignature realizationSignature = null;
        if (realization.getType() == RealizationType.CUBE) {
            realizationSignature = CubeSignature.getCubeSignature(config, realization.getName());
        } else if (realization.getType() == RealizationType.HYBRID) {
            realizationSignature = getHybridSignature(config, realization.getName());
        }
        if (realizationSignature != null) {
            realizationSignatureSet.add(realizationSignature);
        }
    }
    return new HybridSignature(realizationName, realizationSignatureSet);
}

Example 7

Source File: CubeMetaExtractor.java From kylin with Apache License 2.0

5 votes

private void retrieveResourcePath(IRealization realization) throws IOException {
    if (realization == null) {
        return;
    }
    logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType());
    if (realization instanceof CubeInstance) {
        CubeInstance cube = (CubeInstance) realization;
        CubeDesc cubeDesc = cubeDescManager.getCubeDesc(cube.getDescName());
        DataModelDesc modelDesc = metadataManager.getDataModelDesc(cubeDesc.getModelName());
        // add tables
        addTables(modelDesc);
        // add streaming stuff
        addStreamingConfig(cube);
        // add streamingV2
        addStreamingV2Config(cube);
        // add cube
        addRequired(CubeDesc.concatResourcePath(cubeDesc.getName()));
        // add project
        addRequired(ProjectInstance.concatResourcePath(cube.getProject()));
        //add Segments and Jobs
        addSegAndJob(cube);

    } else if (realization instanceof HybridInstance) {
        HybridInstance hybridInstance = (HybridInstance) realization;
        addRequired(HybridInstance.concatResourcePath(hybridInstance.getName()));
        for (IRealization iRealization : hybridInstance.getRealizations()) {
            if (iRealization.getType() != RealizationType.CUBE) {
                throw new RuntimeException("Hybrid " + iRealization.getName() + " contains non cube child "
                        + iRealization.getName() + " with type " + iRealization.getType());
            }
            retrieveResourcePath(iRealization);
        }
    } else {
        logger.warn("Unknown realization type: " + realization.getType());
    }
}

Example 8

Source File: StreamStorage.java From kylin with Apache License 2.0

5 votes

@Override
public IStorageQuery createQuery(IRealization realization) {
    if (realization.getType() == RealizationType.CUBE) {
        CubeInstance cubeInstance = (CubeInstance) realization;
        return new StreamStorageQuery(cubeInstance, getStreamingDataSearchClient());
    } else {
        throw new IllegalArgumentException("Unknown realization type " + realization.getType());
    }
}

Example 9

Source File: HBaseStorage.java From kylin with Apache License 2.0

5 votes

@Override
public IStorageQuery createQuery(IRealization realization) {

    if (realization.getType() == RealizationType.CUBE) {

        CubeInstance cubeInstance = (CubeInstance) realization;
        String cubeStorageQuery;
        if (cubeInstance.getStorageType() == IStorageAware.ID_HBASE) {//v2 query engine cannot go with v1 storage now
            throw new IllegalStateException(
                    "Storage Engine (id=" + IStorageAware.ID_HBASE + ") is not supported any more");
        } else {
            cubeStorageQuery = v2CubeStorageQuery;//by default use v2
        }

        IStorageQuery ret;
        try {
            ret = (IStorageQuery) Class.forName(cubeStorageQuery).getConstructor(CubeInstance.class)
                    .newInstance((CubeInstance) realization);
        } catch (Exception e) {
            throw new RuntimeException("Failed to initialize storage query for " + cubeStorageQuery, e);
        }

        return ret;
    } else {
        throw new IllegalArgumentException("Unknown realization type " + realization.getType());
    }
}

Example 10

Source File: CubeMetaExtractor.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private void retrieveResourcePath(IRealization realization) throws IOException {
    if (realization == null) {
        return;
    }
    logger.info("Deal with realization {} of type {}", realization.getName(), realization.getType());
    if (realization instanceof CubeInstance) {
        CubeInstance cube = (CubeInstance) realization;
        CubeDesc cubeDesc = cubeDescManager.getCubeDesc(cube.getDescName());
        DataModelDesc modelDesc = metadataManager.getDataModelDesc(cubeDesc.getModelName());
        // add tables
        addTables(modelDesc);
        // add streaming stuff
        addStreamingConfig(cube);
        // add streamingV2
        addStreamingV2Config(cube);
        // add cube
        addRequired(CubeDesc.concatResourcePath(cubeDesc.getName()));
        // add project
        addRequired(ProjectInstance.concatResourcePath(cube.getProject()));
        //add Segments and Jobs
        addSegAndJob(cube);

    } else if (realization instanceof HybridInstance) {
        HybridInstance hybridInstance = (HybridInstance) realization;
        addRequired(HybridInstance.concatResourcePath(hybridInstance.getName()));
        for (IRealization iRealization : hybridInstance.getRealizations()) {
            if (iRealization.getType() != RealizationType.CUBE) {
                throw new RuntimeException("Hybrid " + iRealization.getName() + " contains non cube child "
                        + iRealization.getName() + " with type " + iRealization.getType());
            }
            retrieveResourcePath(iRealization);
        }
    } else {
        logger.warn("Unknown realization type: " + realization.getType());
    }
}

Example 11

Source File: DashboardService.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private List<CubeInstance> getCubeByHybrid(HybridInstance hybridInstance) {
    List<CubeInstance> cubeInstances = Lists.newArrayList();
    List<RealizationEntry> realizationEntries = hybridInstance.getRealizationEntries();
    for (RealizationEntry realizationEntry : realizationEntries) {
        String reName = realizationEntry.getRealization();
        if (RealizationType.CUBE == realizationEntry.getType()) {
            CubeInstance cubeInstance = getCubeManager().getCube(reName);
            cubeInstances.add(cubeInstance);
        } else if (RealizationType.HYBRID == realizationEntry.getType()) {
            HybridInstance innerHybridInstance = getHybridManager().getHybridInstance(reName);
            cubeInstances.addAll(getCubeByHybrid(innerHybridInstance));
        }
    }
    return cubeInstances;
}

Example 12

Source File: SimpleQueryMoreColumnsCubeFirstRule.java From Kylin with Apache License 2.0

5 votes

@Override
public void apply(List<IRealization> realizations, OLAPContext olapContext) {
    List<Integer> itemIndexes = super.findRealizationsOf(realizations, RealizationType.CUBE);

    if (olapContext.isSimpleQuery()) {
        PartialSorter.partialSort(realizations, itemIndexes, new Comparator<IRealization>() {
            @Override
            public int compare(IRealization o1, IRealization o2) {
                CubeInstance c1 = (CubeInstance) o1;
                CubeInstance c2 = (CubeInstance) o2;
                return c1.getDescriptor().listDimensionColumnsIncludingDerived().size() - c2.getDescriptor().listDimensionColumnsIncludingDerived().size();
            }
        });
    }
}

Example 13

Source File: CubesSortRule.java From Kylin with Apache License 2.0

5 votes

@Override
public void apply(List<IRealization> realizations, OLAPContext olapContext) {

    // sort cube candidates, 0) the cost indicator, 1) the lesser header
    // columns the better, 2) the lesser body columns the better
    List<Integer> items = super.findRealizationsOf(realizations, RealizationType.CUBE);
    PartialSorter.partialSort(realizations, items, new Comparator<IRealization>() {
        @Override
        public int compare(IRealization o1, IRealization o2) {
            CubeInstance c1 = (CubeInstance) o1;
            CubeInstance c2 = (CubeInstance) o2;
            int comp = 0;
            comp = c1.getCost() - c2.getCost();
            if (comp != 0) {
                return comp;
            }

            CubeDesc schema1 = c1.getDescriptor();
            CubeDesc schema2 = c2.getDescriptor();

            comp = schema1.listDimensionColumnsIncludingDerived().size() - schema2.listDimensionColumnsIncludingDerived().size();
            if (comp != 0)
                return comp;

            comp = schema1.getMeasures().size() - schema2.getMeasures().size();
            return comp;
        }
    });

}

Example 14

Source File: StreamStorage.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Override
public IStorageQuery createQuery(IRealization realization) {
    if (realization.getType() == RealizationType.CUBE) {
        CubeInstance cubeInstance = (CubeInstance) realization;
        return new StreamStorageQuery(cubeInstance, getStreamingDataSearchClient());
    } else {
        throw new IllegalArgumentException("Unknown realization type " + realization.getType());
    }
}

Example 15

Source File: HBaseStorage.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Override
public IStorageQuery createQuery(IRealization realization) {

    if (realization.getType() == RealizationType.CUBE) {

        CubeInstance cubeInstance = (CubeInstance) realization;
        String cubeStorageQuery;
        if (cubeInstance.getStorageType() == IStorageAware.ID_HBASE) {//v2 query engine cannot go with v1 storage now
            throw new IllegalStateException(
                    "Storage Engine (id=" + IStorageAware.ID_HBASE + ") is not supported any more");
        } else {
            cubeStorageQuery = v2CubeStorageQuery;//by default use v2
        }

        IStorageQuery ret;
        try {
            ret = (IStorageQuery) Class.forName(cubeStorageQuery).getConstructor(CubeInstance.class)
                    .newInstance((CubeInstance) realization);
        } catch (Exception e) {
            throw new RuntimeException("Failed to initialize storage query for " + cubeStorageQuery, e);
        }

        return ret;
    } else {
        throw new IllegalArgumentException("Unknown realization type " + realization.getType());
    }
}

Example 16

Source File: CubeInstance.java From kylin with Apache License 2.0

4 votes

@Override
public RealizationType getType() {
    return RealizationType.CUBE;
}

Example 17

Source File: CubeManager.java From kylin with Apache License 2.0

4 votes

@Override
public RealizationType getRealizationType() {
    return RealizationType.CUBE;
}

Example 18

Source File: CubeManager.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Override
public RealizationType getRealizationType() {
    return RealizationType.CUBE;
}

Example 19

Source File: CubeInstance.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Override
public RealizationType getType() {
    return RealizationType.CUBE;
}

Example 20

Source File: CubeInstance.java From Kylin with Apache License 2.0

4 votes

@Override
public RealizationType getType() {
    return RealizationType.CUBE;
}