org.apache.kylin.metadata.model.SegmentStatusEnum Java Examples

private CubeSegment newSegment(CubeInstance cube, TSRange tsRange, SegmentRange segRange) {
    DataModelDesc modelDesc = cube.getModel();

    CubeSegment segment = new CubeSegment();
    segment.setUuid(RandomUtil.randomUUID().toString());
    segment.setName(CubeSegment.makeSegmentName(tsRange, segRange, modelDesc));
    segment.setCreateTimeUTC(System.currentTimeMillis());
    segment.setCubeInstance(cube);

    // let full build range be backward compatible
    if (tsRange == null && segRange == null)
        tsRange = new TSRange(0L, Long.MAX_VALUE);

    segment.setTSRange(tsRange);
    segment.setSegRange(segRange);
    segment.setStatus(SegmentStatusEnum.NEW);
    segment.setStorageLocationIdentifier(generateStorageLocation(cube.getEngineType()));
    Map<String, String> additionalInfo = segment.getAdditionalInfo();
    additionalInfo.put("storageType", "" + cube.getStorageType());
    segment.setAdditionalInfo(additionalInfo);
    segment.setCubeInstance(cube);

    segment.validate();
    return segment;
}
 

@Override
public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
    if (!(split instanceof FileSplit)) {
        throw new IllegalArgumentException("Only compatible with FileSplits.");
    } else {
        logger.debug("CFG_Cube_Name: " + BatchConstants.CFG_CUBE_NAME);
        cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase(Locale.ROOT);
        segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME).toUpperCase(Locale.ROOT);

        KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
        CubeManager cubeManager = CubeManager.getInstance(config);
        cube = cubeManager.getCube(cubeName);
        cubeDesc = cube.getDescriptor();
        cubeSegment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
    }
}
 

public static boolean isAbleToOptimizeCubingPlan(CubeSegment segment) {
    CubeInstance cube = segment.getCubeInstance();
    if (!cube.getConfig().isCubePlannerEnabled())
        return false;

    if (cube.getSegments(SegmentStatusEnum.READY_PENDING).size() > 0) {
        logger.info("Has read pending segments and will not enable cube planner.");
        return false;
    }
    List<CubeSegment> readySegments = cube.getSegments(SegmentStatusEnum.READY);
    List<CubeSegment> newSegments = cube.getSegments(SegmentStatusEnum.NEW);
    if (newSegments.size() <= 1 && //
            (readySegments.size() == 0 || //
                    (cube.getConfig().isCubePlannerEnabledForExistingCube() && readySegments.size() == 1
                            && readySegments.get(0).getSegRange().equals(segment.getSegRange())))) {
        return true;
    } else {
        return false;
    }
}
 

private boolean isInOptimize(CubeInstance cube) {
    Segments<CubeSegment> readyPendingSegments = cube.getSegments(SegmentStatusEnum.READY_PENDING);
    if (readyPendingSegments.size() > 0) {
        logger.info("The cube {} has READY_PENDING segments {}. It's not allowed for building",
            cube.getName(), readyPendingSegments);
        return true;
    }
    Segments<CubeSegment> newSegments = cube.getSegments(SegmentStatusEnum.NEW);
    for (CubeSegment newSegment : newSegments) {
        String jobId = newSegment.getLastBuildJobID();
        if (jobId == null) {
            continue;
        }
        AbstractExecutable job = coordinator.getExecutableManager().getJob(jobId);
        if (job != null && job instanceof CubingJob) {
            CubingJob cubingJob = (CubingJob) job;
            if (CubingJob.CubingJobTypeEnum.OPTIMIZE.toString().equals(cubingJob.getJobType())) {
                logger.info("The cube {} is in optimization. It's not allowed to build new segments during optimization.", cube.getName());
                return true;
            }
        }
    }
    return false;
}
 

void prepareTestCheckSegmentBuildJobFromMetadata() {
    CubeSegment cubeSegment = stubCubSegment(SegmentStatusEnum.NEW, 100L, 200L);
    CubeInstance cubeInstance = stubCubeInstance(cubeSegment);
    config = stubKylinConfig();
    when(cubeInstance.getConfig()).thenReturn(config);

    cubeManager = stubCubeManager(cubeInstance, false);

    Map<String, CubingJob> cubingJobMap = new HashMap<>();
    cubingJobMap.put(mockBuildJob1, stubCubingJob(ExecutableState.SUCCEED));
    cubingJobMap.put(mockBuildJob2, stubCubingJob(ExecutableState.DISCARDED));
    cubingJobMap.put(mockBuildJob3, stubCubingJob(ExecutableState.DISCARDED));
    cubingJobMap.put(mockBuildJob4, stubCubingJob(ExecutableState.ERROR));

    executableManager = stubExecutableManager(cubingJobMap);
    streamingCoordinator = stubStreamingCoordinator(config, cubeManager, executableManager);
    clusterManager = stubReceiverClusterManager(streamingCoordinator);
    when(streamingCoordinator.getClusterManager()).thenReturn(clusterManager);
}
 

private boolean isInOptimize(CubeInstance cube) {
    Segments<CubeSegment> readyPendingSegments = cube.getSegments(SegmentStatusEnum.READY_PENDING);
    if (readyPendingSegments.size() > 0) {
        logger.info("The cube {} has READY_PENDING segments {}. It's not allowed for building", cube.getName(),
                readyPendingSegments);
        return true;
    }
    Segments<CubeSegment> newSegments = cube.getSegments(SegmentStatusEnum.NEW);
    for (CubeSegment newSegment : newSegments) {
        String jobId = newSegment.getLastBuildJobID();
        if (jobId == null) {
            continue;
        }
        AbstractExecutable job = getExecutableManager().getJob(jobId);
        if (job != null && job instanceof CubingJob) {
            CubingJob cubingJob = (CubingJob) job;
            if (CubingJob.CubingJobTypeEnum.OPTIMIZE.toString().equals(cubingJob.getJobType())) {
                logger.info(
                        "The cube {} is in optimization. It's not allowed to build new segments during optimization.",
                        cube.getName());
                return true;
            }
        }
    }
    return false;
}
 

List<CubeSegment> getToRemoveSegs(CubeInstance cube, CubeSegment mergedSegment) {
    Segments tobe = cube.calculateToBeSegments(mergedSegment);

    if (!tobe.contains(mergedSegment))
        throw new IllegalStateException(
                "For Cube " + cube + ", segment " + mergedSegment + " is expected but not in the tobe " + tobe);

    if (mergedSegment.getStatus() == SegmentStatusEnum.NEW)
        mergedSegment.setStatus(SegmentStatusEnum.READY);

    List<CubeSegment> toRemoveSegs = Lists.newArrayList();
    for (CubeSegment s : cube.getSegments()) {
        if (!tobe.contains(s))
            toRemoveSegs.add(s);
    }

    return toRemoveSegs;
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
    final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));

    Set<Long> recommendCuboids = cube.getCuboidsRecommend();
    try {
        List<CubeSegment> newSegments = cube.getSegments(SegmentStatusEnum.READY_PENDING);
        Map<Long, Long> recommendCuboidsWithStats = CuboidStatsReaderUtil
                .readCuboidStatsFromSegments(recommendCuboids, newSegments);
        if (recommendCuboidsWithStats == null) {
            throw new RuntimeException("Fail to get statistics info for recommended cuboids after optimization!!!");
        }
        cubeManager.promoteCheckpointOptimizeSegments(cube, recommendCuboidsWithStats,
                newSegments.toArray(new CubeSegment[newSegments.size()]));
        return new ExecuteResult();
    } catch (Exception e) {
        logger.error("fail to update cube after build", e);
        return ExecuteResult.createError(e);
    }
}
 

private void cancelCheckpointJobInner(CheckpointExecutable checkpointExecutable) throws IOException {
    List<String> segmentIdList = Lists.newLinkedList();
    List<String> jobIdList = Lists.newLinkedList();
    jobIdList.add(checkpointExecutable.getId());
    setRelatedIdList(checkpointExecutable, segmentIdList, jobIdList);

    CubeInstance cubeInstance = getCubeManager()
            .getCube(CubingExecutableUtil.getCubeName(checkpointExecutable.getParams()));
    if (!segmentIdList.isEmpty()) {
        List<CubeSegment> toRemoveSegments = Lists.newLinkedList();
        for (String segmentId : segmentIdList) {
            final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
            if (segment != null && segment.getStatus() != SegmentStatusEnum.READY) {
                toRemoveSegments.add(segment);
            }
        }

        getCubeManager().dropOptmizingSegments(cubeInstance, toRemoveSegments.toArray(new CubeSegment[] {}));
    }

    for (String jobId : jobIdList) {
        getExecutableManager().discardJob(jobId);
    }
}
 

@Override
protected void doSetup(Context context) throws IOException, InterruptedException {
    super.bindCurrentConfiguration(context.getConfiguration());
    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase(Locale.ROOT);
    segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME);
    cube = CubeManager.getInstance(config).getCube(cubeName);
    segment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
    if (segment == null) {
        logger.debug("segment {} is null during setup!", segmentName);
        throw new IllegalArgumentException("Segment is null, job quit!");
    }
    colNeedDictMap = Maps.newHashMap();
    Set<TblColRef> columnsNeedDict = cube.getDescriptor().getAllColumnsNeedDictionaryBuilt();
    for (TblColRef column : columnsNeedDict) {
        colNeedDictMap.put(column.getName(), column);
    }
}
 

/**
 * @param IIInstance
 * @param startDate  (pass 0 if full build)
 * @param endDate    (pass 0 if full build)
 * @return
 */
public IISegment buildSegment(IIInstance IIInstance, long startDate, long endDate) {
    IISegment segment = new IISegment();
    String incrementalSegName = IISegment.getSegmentName(startDate, endDate);
    segment.setUuid(UUID.randomUUID().toString());
    segment.setName(incrementalSegName);
    segment.setCreateTimeUTC(System.currentTimeMillis());
    segment.setDateRangeStart(startDate);
    segment.setDateRangeEnd(endDate);
    segment.setStatus(SegmentStatusEnum.NEW);
    segment.setStorageLocationIdentifier(generateStorageLocation());

    segment.setIIInstance(IIInstance);

    return segment;
}
 

@Test
// For JIRA: KYLIN-4229
public void testDateFormatWithFullBuildAndFilter() {
    this.createTestMetadata();
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("fifty_dim_full_build_cube");
    CubeSegment mockSeg = new CubeSegment();

    mockSeg.setCubeInstance(cube);
    mockSeg.setName("FULL_BUILD");
    mockSeg.setUuid(RandomUtil.randomUUID().toString());
    mockSeg.setStorageLocationIdentifier(RandomUtil.randomUUID().toString());
    mockSeg.setStatus(SegmentStatusEnum.READY);
    mockSeg.setTSRange(new SegmentRange.TSRange(0L, Long.MAX_VALUE));
    IJoinedFlatTableDesc flatTableDesc = new CubeJoinedFlatTableDesc(mockSeg);
    DataModelDesc model = flatTableDesc.getDataModel();
    model.setFilterCondition("A > 1");

    JoinedFlatTable.generateSelectDataStatement(flatTableDesc);
}
 

public void checkEnableCubeCondition(CubeInstance cube) {
    aclEvaluate.checkProjectWritePermission(cube);
    Message msg = MsgPicker.getMsg();
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();

    if (!cube.getStatus().equals(RealizationStatusEnum.DISABLED)) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getENABLE_NOT_DISABLED_CUBE(), cubeName, ostatus));
    }

    if (cube.getSegments(SegmentStatusEnum.READY).size() == 0 && !cube.getDescriptor().isStreamingCube()) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getNO_READY_SEGMENT(), cubeName));
    }

    if (!cube.getDescriptor().checkSignature()) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getINCONSISTENT_CUBE_DESC_SIGNATURE(), cube.getDescriptor()));
    }
}
 

@Test
public void testPruneSegWithFilterIN() {
    // legacy cube segments does not have DimensionRangeInfo, but with TSRange can do some pruning
    CubeInstance cube = CubeManager.getInstance(getTestConfig())
            .getCube("test_kylin_cube_without_slr_left_join_ready_2_segments");
    TblColRef col = cube.getModel().findColumn("TEST_KYLIN_FACT.CAL_DT");
    CubeSegment seg = cube.getSegments(SegmentStatusEnum.READY).get(0);
    TSRange tsRange = seg.getTSRange();
    String start = DateFormat.formatToTimeStr(tsRange.start.v, "yyyy-MM-dd");
    CubeSegment seg2 = cube.getSegments(SegmentStatusEnum.READY).get(1);
    TSRange tsRange2 = seg2.getTSRange();
    try (SetAndUnsetSystemProp sns = new SetAndUnsetSystemProp("kylin.query.skip-empty-segments", "false")) {

        {
            TupleFilter inFilter = new ConstantTupleFilter(Sets.newHashSet(start,
                    DateFormat.formatToTimeStr(tsRange2.end.v + 1000 * 60 * 60 * 24L, "yyyy-MM-dd")));
            TupleFilter filter = compare(col, FilterOperatorEnum.IN, inFilter);
            SegmentPruner segmentPruner = new SegmentPruner(filter);
            Assert.assertTrue(segmentPruner.check(seg));
            Assert.assertFalse(segmentPruner.check(seg2));

        }
    }
}
 

@Override
protected void doSetup(Context context) throws IOException, InterruptedException {
    super.bindCurrentConfiguration(context.getConfiguration());
    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase(Locale.ROOT);
    segmentName = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_NAME);
    cube = CubeManager.getInstance(config).getCube(cubeName);
    segment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
    if (segment == null) {
        logger.debug("segment {} is null during setup!", segmentName);
        throw new IllegalArgumentException("Segment is null, job quit!");
    }
    colNeedDictMap = Maps.newHashMap();
    Set<TblColRef> columnsNeedDict = cube.getDescriptor().getAllColumnsNeedDictionaryBuilt();
    for (TblColRef column : columnsNeedDict) {
        colNeedDictMap.put(column.getName(), column);
    }
}
 

public static boolean isAbleToOptimizeCubingPlan(CubeSegment segment) {
    CubeInstance cube = segment.getCubeInstance();
    if (!cube.getConfig().isCubePlannerEnabled())
        return false;

    if (cube.getSegments(SegmentStatusEnum.READY_PENDING).size() > 0) {
        logger.info("Has read pending segments and will not enable cube planner.");
        return false;
    }
    List<CubeSegment> readySegments = cube.getSegments(SegmentStatusEnum.READY);
    List<CubeSegment> newSegments = cube.getSegments(SegmentStatusEnum.NEW);
    if (newSegments.size() <= 1 && //
            (readySegments.size() == 0 || //
                    (cube.getConfig().isCubePlannerEnabledForExistingCube() && readySegments.size() == 1
                            && readySegments.get(0).getSegRange().equals(segment.getSegRange())))) {
        return true;
    } else {
        return false;
    }
}
 

@Override
protected void setup(Context context) throws IOException {
    super.publishConfiguration(context.getConfiguration());

    Configuration conf = context.getConfiguration();

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
    IIManager mgr = IIManager.getInstance(config);
    IIInstance ii = mgr.getII(conf.get(BatchConstants.CFG_II_NAME));
    IISegment seg = ii.getSegment(conf.get(BatchConstants.CFG_II_SEGMENT_NAME), SegmentStatusEnum.NEW);
    this.info = new TableRecordInfo(seg);
    this.rec = this.info.createTableRecord();

    outputKey = new LongWritable();
    outputValue = new ImmutableBytesWritable(rec.getBytes());

    schema = HCatInputFormat.getTableSchema(context.getConfiguration());
    
    fields = schema.getFields();
}
 

public static List<Dictionary<String>> getAllDictionaries(TblColRef col, IRealization realization) {
    Set<Dictionary<String>> result = Sets.newHashSet();
    if (realization instanceof CubeInstance) {
        final CubeInstance cube = (CubeInstance) realization;
        for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) {
            result.add(segment.getDictionary(col));
        }
    } else if (realization instanceof HybridInstance) {
        final HybridInstance hybridInstance = (HybridInstance) realization;
        for (IRealization entry : hybridInstance.getRealizations()) {
            result.addAll(getAllDictionaries(col, entry));
        }
    } else {
        throw new IllegalStateException("All leaf realizations should be CubeInstance");
    }
    return Lists.newArrayList(result);
}
 

protected void checkCubeState(CubeInstance cube) {
    if (cube.getStatus() != RealizationStatusEnum.READY)
        throw new IllegalStateException("Cannot migrate cube that is not in READY state.");

    for (CubeSegment segment : cube.getSegments()) {
        if (segment.getStatus() != SegmentStatusEnum.READY) {
            throw new IllegalStateException("At least one segment is not in READY state");
        }
    }
}
 

@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN
        + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public void migrateCube(CubeInstance cube, String projectName) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    if (!config.isAllowAutoMigrateCube()) {
        throw new InternalErrorException("One click migration is disabled, please contact your ADMIN");
    }

    for (CubeSegment segment : cube.getSegments()) {
        if (segment.getStatus() != SegmentStatusEnum.READY) {
            throw new InternalErrorException(
                    "At least one segment is not in READY state. Please check whether there are Running or Error jobs.");
        }
    }

    String srcCfgUri = config.getAutoMigrateCubeSrcConfig();
    String dstCfgUri = config.getAutoMigrateCubeDestConfig();

    Preconditions.checkArgument(StringUtils.isNotEmpty(srcCfgUri), "Source configuration should not be empty.");
    Preconditions.checkArgument(StringUtils.isNotEmpty(dstCfgUri),
            "Destination configuration should not be empty.");

    String stringBuilder = ("%s/bin/kylin.sh org.apache.kylin.tool.CubeMigrationCLI %s %s %s %s %s %s true true");
    String cmd = String.format(Locale.ROOT, stringBuilder, KylinConfig.getKylinHome(), srcCfgUri, dstCfgUri,
            cube.getName(), projectName, config.isAutoMigrateCubeCopyAcl(), config.isAutoMigrateCubePurge());

    logger.info("One click migration cmd: " + cmd);

    CliCommandExecutor exec = new CliCommandExecutor();
    PatternedLogger patternedLogger = new PatternedLogger(logger);

    try {
        exec.execute(cmd, patternedLogger, null);
    } catch (IOException e) {
        throw new InternalErrorException("Failed to perform one-click migrating", e);
    }
}
 

/**
 * Update a cube status from disable to ready.
 *
 * @return
 * @throws CubeIntegrityException
 * @throws IOException
 * @throws JobException
 */
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION')  or hasPermission(#cube, 'MANAGEMENT')")
public CubeInstance enableCube(CubeInstance cube) throws IOException, JobException {
    String cubeName = cube.getName();

    RealizationStatusEnum ostatus = cube.getStatus();
    if (!cube.getStatus().equals(RealizationStatusEnum.DISABLED)) {
        throw new InternalErrorException("Only disabled cube can be enabled, status of " + cubeName + " is " + ostatus);
    }

    if (cube.getSegments(SegmentStatusEnum.READY).size() == 0) {
        throw new InternalErrorException("Cube " + cubeName + " dosen't contain any READY segment");
    }

    final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
    if (!cubingJobs.isEmpty()) {
        throw new JobException("Enable is not allowed with a running job.");
    }
    if (!cube.getDescriptor().calculateSignature().equals(cube.getDescriptor().getSignature())) {
        this.releaseAllSegments(cube);
    }

    cube.setStatus(RealizationStatusEnum.READY);
    try {
        return getCubeManager().updateCube(cube);
    } catch (IOException e) {
        cube.setStatus(ostatus);
        throw e;
    }
}
 

public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {

        srcConfig = srcCfg;
        srcStore = ResourceStore.getStore(srcConfig);
        dstConfig = dstCfg;
        dstStore = ResourceStore.getStore(dstConfig);

        CubeManager cubeManager = CubeManager.getInstance(srcConfig);
        CubeInstance cube = cubeManager.getCube(cubeName);
        logger.info("cube to be moved is : " + cubeName);

        if (cube.getStatus() != RealizationStatusEnum.READY)
            throw new IllegalStateException("Cannot migrate cube that is not in READY state.");

        for (CubeSegment segment : cube.getSegments()) {
            if (segment.getStatus() != SegmentStatusEnum.READY) {
                throw new IllegalStateException("At least one segment is not in READY state");
            }
        }

        checkAndGetHbaseUrl();

        Configuration conf = HBaseConfiguration.create();
        hbaseAdmin = new HBaseAdmin(conf);

        hdfsFS = FileSystem.get(new Configuration());

        operations = new ArrayList<Opt>();

        copyFilesInMetaStore(cube, overwriteIfExists);
        renameFoldersInHdfs(cube);
        changeHtableHost(cube);
        addCubeIntoProject(cubeName, projectName);

        if (realExecute.equalsIgnoreCase("true")) {
            doOpts();
        } else {
            showOpts();
        }
    }
 

@Test
public void testAutoMergeNormal() throws Exception {
    CubeManager mgr = CubeManager.getInstance(getTestConfig());
    CubeInstance cube = mgr.getCube("test_kylin_cube_with_slr_empty").latestCopyForWrite();

    cube.getDescriptor().setAutoMergeTimeRanges(new long[] { 2000, 6000 });

    mgr.updateCube(new CubeUpdate(cube).setStatus(RealizationStatusEnum.READY));
    assertEquals(RealizationStatusEnum.READY, cube.getStatus());
    assertTrue(cube.needAutoMerge());

    // no segment at first
    assertEquals(0, cube.getSegments().size());

    // append first
    CubeSegment seg1 = mgr.appendSegment(cube, new TSRange(0L, 1000L), null, null, null);
    mgr.updateCubeSegStatus(seg1, SegmentStatusEnum.READY);

    CubeSegment seg2 = mgr.appendSegment(cube, new TSRange(1000L, 2000L), null, null, null);
    mgr.updateCubeSegStatus(seg2, SegmentStatusEnum.READY);

    cube = mgr.getCube(cube.getName());
    assertEquals(2, cube.getSegments().size());

    SegmentRange mergedSeg = cube.autoMergeCubeSegments();

    assertTrue(mergedSeg != null);
}
 

private void checkAllowBuilding(CubeInstance cube) {
    if (cube.getConfig().isCubePlannerEnabled()) {
        Segments<CubeSegment> readyPendingSegments = cube.getSegments(SegmentStatusEnum.READY_PENDING);
        if (readyPendingSegments.size() > 0) {
            throw new BadRequestException("The cube " + cube.getName() + " has READY_PENDING segments "
                    + readyPendingSegments + ". It's not allowed for building");
        }
    }
}
 

@Test
public void testAppendNonPartitioned2() throws IOException {
    CubeManager mgr = mgr();
    CubeInstance cube = mgr.getCube("test_kylin_cube_without_slr_ready");

    // override partition desc
    cube.getModel().setPartitionDesc(new PartitionDesc());

    // assert one ready segment
    assertEquals(1, cube.getSegments().size());
    CubeSegment seg = cube.getSegments(SegmentStatusEnum.READY).get(0);
    assertEquals(SegmentStatusEnum.READY, seg.getStatus());

    // append again, for non-partitioned cube, it becomes a full refresh
    CubeSegment seg2 = mgr.appendSegment(cube);
    assertEquals(new TSRange(0L, Long.MAX_VALUE), seg2.getTSRange());
    assertEquals(new TSRange(0L, Long.MAX_VALUE), seg2.getSegRange());
    
    assertEquals(1, cube.getSegments().size()); // older cube not changed
    cube = mgr.getCube(cube.getName());
    assertEquals(2, cube.getSegments().size()); // the updated cube

    // non-partitioned cannot merge, throw exception
    try {
        mgr.mergeSegments(cube, null, new SegmentRange(0L, Long.MAX_VALUE), false);
        fail();
    } catch (IllegalStateException ex) {
        // good
    }
}
 

@Test
public void copyCubeSegmentTest() {
    int origSegCount = cubeInstance.getSegments().size();
    CubeInstance newCubeInstance = CubeInstance.getCopyOf(cubeInstance);

    CubeSegment mockSeg = new CubeSegment();
    mockSeg.setUuid(RandomUtil.randomUUID().toString());
    mockSeg.setStorageLocationIdentifier(RandomUtil.randomUUID().toString());
    mockSeg.setStatus(SegmentStatusEnum.READY);
    newCubeInstance.getSegments().add(mockSeg);

    Assert.assertEquals(origSegCount, cubeInstance.getSegments().size());
    Assert.assertEquals(origSegCount + 1, newCubeInstance.getSegments().size());
}
 

public void promoteNewlyOptimizeSegments(CubeInstance cube, CubeSegment... optimizedSegments)
        throws IOException {
    CubeInstance cubeCopy = cube.latestCopyForWrite();
    CubeSegment[] segCopy = cube.regetSegments(optimizedSegments);

    for (CubeSegment seg : segCopy) {
        seg.setStatus(SegmentStatusEnum.READY_PENDING);
    }

    CubeUpdate update = new CubeUpdate(cubeCopy);
    update.setToUpdateSegs(segCopy);
    updateCube(update);
}
 

public List<CubeSegment> getOptimizeSegments() {
    CubeInstance cube = (CubeInstance) seg.getRealization();
    List<CubeSegment> newSegments = Lists.newArrayList(cube.getSegments(SegmentStatusEnum.READY_PENDING));
    List<CubeSegment> oldSegments = Lists.newArrayListWithExpectedSize(newSegments.size());
    for (CubeSegment segment : newSegments) {
        oldSegments.add(cube.getOriginalSegmentToOptimize(segment));
    }
    return oldSegments;
}
 

static CubeSignature getCubeSignature(KylinConfig config, String realizationName) {
    CubeInstance cubeInstance = CubeManager.getInstance(config).getCube(realizationName);
    if (cubeInstance == null) {
        return null;
    }
    if (!cubeInstance.isReady()) {
        return new CubeSignature(realizationName, RealizationStatusEnum.DISABLED, null);
    }
    List<CubeSegment> readySegments = cubeInstance.getSegments(SegmentStatusEnum.READY);
    Set<SegmentSignature> segmentSignatureSet = Sets.newHashSetWithExpectedSize(readySegments.size());
    for (CubeSegment cubeSeg : readySegments) {
        segmentSignatureSet.add(new SegmentSignature(cubeSeg.getName(), cubeSeg.getLastBuildTime()));
    }
    return new CubeSignature(realizationName, RealizationStatusEnum.READY, segmentSignatureSet);
}
 

private List<HBaseKeyRange> buildScanRanges(TupleFilter flatFilter, Collection<TblColRef> dimensionColumns) {

        List<HBaseKeyRange> result = Lists.newArrayList();

        // build row key range for each cube segment
        for (CubeSegment cubeSeg : cubeInstance.getSegments(SegmentStatusEnum.READY)) {

            // consider derived (lookup snapshot), filter on dimension may
            // differ per segment
            List<Collection<ColumnValueRange>> orAndDimRanges = translateToOrAndDimRanges(flatFilter, cubeSeg);
            if (orAndDimRanges == null) { // has conflict
                continue;
            }

            List<HBaseKeyRange> scanRanges = Lists.newArrayListWithCapacity(orAndDimRanges.size());
            for (Collection<ColumnValueRange> andDimRanges : orAndDimRanges) {
                HBaseKeyRange rowKeyRange = new HBaseKeyRange(dimensionColumns, andDimRanges, cubeSeg, cubeDesc);
                scanRanges.add(rowKeyRange);
            }

            List<HBaseKeyRange> mergedRanges = mergeOverlapRanges(scanRanges);
            mergedRanges = mergeTooManyRanges(mergedRanges);
            result.addAll(mergedRanges);
        }

        dropUnhitSegments(result);

        return result;
    }