org.apache.kylin.metadata.model.PartitionDesc Java Examples

public static DimensionRangeInfo tryDeduceRangeFromPartitionCol(CubeSegment seg, TblColRef col) {
    DataModelDesc model = seg.getModel();
    PartitionDesc part = model.getPartitionDesc();

    if (!part.isPartitioned())
        return null;
    if (!col.equals(part.getPartitionDateColumnRef()))
        return null;

    // deduce the dim range from TSRange
    TSRange tsRange = seg.getTSRange();
    if (tsRange.start.isMin || tsRange.end.isMax)
        return null; // DimensionRangeInfo cannot express infinite

    String min = tsRangeToStr(tsRange.start.v, part);
    String max = tsRangeToStr(tsRange.end.v - 1, part); // note the -1, end side is exclusive
    return new DimensionRangeInfo(min, max);
}
 

public static DataModelDesc generateKylinModel(String owner, String tableName, List<String> dimensions,
        List<String> measures, PartitionDesc partitionDesc) {
    ModelDimensionDesc modelDimensionDesc = new ModelDimensionDesc();
    modelDimensionDesc.setTable(tableName);
    modelDimensionDesc.setColumns(dimensions.toArray(new String[dimensions.size()]));

    DataModelDesc kylinModel = new DataModelDesc();
    kylinModel.setName(tableName.replace('.', '_'));
    kylinModel.setOwner(owner);
    kylinModel.setDescription("");
    kylinModel.setLastModified(0L);
    kylinModel.setRootFactTableName(tableName);
    kylinModel.setJoinTables(new JoinTableDesc[0]);
    kylinModel.setDimensions(Lists.newArrayList(modelDimensionDesc));
    kylinModel.setMetrics(measures.toArray(new String[measures.size()]));
    kylinModel.setFilterCondition("");
    kylinModel.setPartitionDesc(partitionDesc);
    kylinModel.setCapacity(DataModelDesc.RealizationCapacity.SMALL);
    kylinModel.updateRandomUuid();

    return kylinModel;
}
 

public DataModelDescResponse(DataModelDesc dataModelDesc) {
    setUuid(dataModelDesc.getUuid());
    setLastModified(dataModelDesc.getLastModified());
    setVersion(dataModelDesc.getVersion());
    setName(dataModelDesc.getName());
    setOwner(dataModelDesc.getOwner());
    setDraft(dataModelDesc.isDraft());
    setDescription(dataModelDesc.getDescription());
    setRootFactTableName(dataModelDesc.getRootFactTableName());
    setJoinTables(dataModelDesc.getJoinTables());
    setDimensions(dataModelDesc.getDimensions());
    setMetrics(dataModelDesc.getMetrics());
    setFilterCondition(dataModelDesc.getFilterCondition());
    if (dataModelDesc.getPartitionDesc() != null)
        setPartitionDesc(PartitionDesc.getCopyOf(dataModelDesc.getPartitionDesc()));
    setCapacity(dataModelDesc.getCapacity());
}
 

@Test
public void testAppendNonPartitioned() throws IOException {
    CubeManager mgr = mgr();
    CubeInstance cube = mgr.getCube("test_kylin_cube_without_slr_empty");

    // override partition desc
    cube.getModel().setPartitionDesc(new PartitionDesc());

    // first append, creates a new & single segment
    CubeSegment seg = mgr.appendSegment(cube);
    assertEquals(new TSRange(0L, Long.MAX_VALUE), seg.getTSRange());
    assertEquals(new TSRange(0L, Long.MAX_VALUE), seg.getSegRange());
    
    assertEquals(0, cube.getSegments().size()); // older cube not changed
    cube = mgr.getCube(cube.getName());
    assertEquals(1, cube.getSegments().size()); // the updated cube

    // second append, throw IllegalStateException because the first segment is not built
    try {
        mgr.appendSegment(cube);
        fail();
    } catch (IllegalStateException ex) {
        // good
    }
}
 

public DataModelDescResponse(DataModelDesc dataModelDesc) {
    setUuid(dataModelDesc.getUuid());
    setLastModified(dataModelDesc.getLastModified());
    setVersion(dataModelDesc.getVersion());
    setName(dataModelDesc.getName());
    setOwner(dataModelDesc.getOwner());
    setDraft(dataModelDesc.isDraft());
    setDescription(dataModelDesc.getDescription());
    setRootFactTableName(dataModelDesc.getRootFactTableName());
    setJoinTables(dataModelDesc.getJoinTables());
    setDimensions(dataModelDesc.getDimensions());
    setMetrics(dataModelDesc.getMetrics());
    setFilterCondition(dataModelDesc.getFilterCondition());
    if (dataModelDesc.getPartitionDesc() != null)
        setPartitionDesc(PartitionDesc.getCopyOf(dataModelDesc.getPartitionDesc()));
    setCapacity(dataModelDesc.getCapacity());
}
 

@Test
public void testAppendNonPartitioned() throws IOException {
    CubeManager mgr = mgr();
    CubeInstance cube = mgr.getCube("test_kylin_cube_without_slr_empty");

    // override partition desc
    cube.getModel().setPartitionDesc(new PartitionDesc());

    // first append, creates a new & single segment
    CubeSegment seg = mgr.appendSegment(cube);
    assertEquals(new TSRange(0L, Long.MAX_VALUE), seg.getTSRange());
    assertEquals(new TSRange(0L, Long.MAX_VALUE), seg.getSegRange());
    
    assertEquals(0, cube.getSegments().size()); // older cube not changed
    cube = mgr.getCube(cube.getName());
    assertEquals(1, cube.getSegments().size()); // the updated cube

    // second append, throw IllegalStateException because the first segment is not built
    try {
        mgr.appendSegment(cube);
        fail();
    } catch (IllegalStateException ex) {
        // good
    }
}
 

private static String tsRangeToStr(long ts, PartitionDesc part) {
    String value;
    DataType partitionColType = part.getPartitionDateColumnRef().getType();
    if (partitionColType.isDate()) {
        value = DateFormat.formatToDateStr(ts);
    } else if (partitionColType.isTimeFamily()) {
        value = DateFormat.formatToTimeWithoutMilliStr(ts);
    } else if (partitionColType.isStringFamily() || partitionColType.isIntegerFamily()) {//integer like 20160101
        String partitionDateFormat = part.getPartitionDateFormat();
        if (StringUtils.isEmpty(partitionDateFormat)) {
            value = "" + ts;
        } else {
            value = DateFormat.formatToDateStr(ts, partitionDateFormat);
        }
    } else {
        throw new RuntimeException("Type " + partitionColType + " is not valid partition column type");
    }
    return value;
}
 

private void setDateEnv(IJoinedFlatTableDesc flatDesc) {
    DataModelDesc model = flatDesc.getDataModel();
    PartitionDesc partDesc = model.getPartitionDesc();
    SegmentRange segRange = flatDesc.getSegRange();
    long startInclusive = (Long) segRange.start.v;
    long endExclusive = (Long) segRange.end.v;
    //
    String startDate = "";
    String endDate = "";
    String partitionColumnDateFormat = partDesc.getPartitionDateFormat();
    if (partDesc.getPartitionTimeColumn() == null && partDesc.getPartitionDateColumn() == null) {
        startDate = String.valueOf(startInclusive);
        endDate = String.valueOf(endExclusive);
    } else {
        startDate = DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat);
        endDate = DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat);
    }
    setKeyValue(START_DATE, startDate);
    setKeyValue(END_DATE, endDate);
}
 

private void setDateEnv(IJoinedFlatTableDesc flatDesc) {
    DataModelDesc model = flatDesc.getDataModel();
    PartitionDesc partDesc = model.getPartitionDesc();
    SegmentRange segRange = flatDesc.getSegRange();
    long startInclusive = (Long) segRange.start.v;
    long endExclusive = (Long) segRange.end.v;
    //
    String startDate = "";
    String endDate = "";
    String partitionColumnDateFormat = partDesc.getPartitionDateFormat();
    if (partDesc.getPartitionTimeColumn() == null && partDesc.getPartitionDateColumn() == null) {
        startDate = String.valueOf(startInclusive);
        endDate = String.valueOf(endExclusive);
    } else {
        startDate = DateFormat.formatToDateStr(startInclusive, partitionColumnDateFormat);
        endDate = DateFormat.formatToDateStr(endExclusive, partitionColumnDateFormat);
    }
    setKeyValue(START_DATE, startDate);
    setKeyValue(END_DATE, endDate);
}
 

/**
 * Use the segment start time as the map key, the time unit depends on the partition columns
 * If the partition_time_column is null, the unit is day;
 *                            otherwise, the unit is second
 */
private String getSegmentStartTime(CubeSegment segment) {
    long startTime = segment.getTSRange().start.v;
    DataModelDesc model = segment.getModel();
    PartitionDesc partitionDesc = model.getPartitionDesc();
    if (partitionDesc == null || !partitionDesc.isPartitioned()) {
        return "0";
    } else if (partitionDesc.partitionColumnIsTimeMillis()) {
        return "" + startTime;
    } else if (partitionDesc.getPartitionTimeColumnRef() != null) {
        return "" + startTime / 1000L;
    } else if (partitionDesc.getPartitionDateColumnRef() != null) {
        return "" + startTime / 86400000L;
    }
    return "0";
}
 

public static DimensionRangeInfo tryDeduceRangeFromPartitionCol(CubeSegment seg, TblColRef col) {
    DataModelDesc model = seg.getModel();
    PartitionDesc part = model.getPartitionDesc();

    if (!part.isPartitioned())
        return null;
    if (!col.equals(part.getPartitionDateColumnRef()))
        return null;

    // deduce the dim range from TSRange
    TSRange tsRange = seg.getTSRange();
    if (tsRange.start.isMin || tsRange.end.isMax)
        return null; // DimensionRangeInfo cannot express infinite

    String min = tsRangeToStr(tsRange.start.v, part);
    String max = tsRangeToStr(tsRange.end.v - 1, part); // note the -1, end side is exclusive
    return new DimensionRangeInfo(min, max);
}
 

private static String tsRangeToStr(long ts, PartitionDesc part) {
    String value;
    DataType partitionColType = part.getPartitionDateColumnRef().getType();
    if (partitionColType.isDate()) {
        value = DateFormat.formatToDateStr(ts);
    } else if (partitionColType.isTimeFamily()) {
        value = DateFormat.formatToTimeWithoutMilliStr(ts);
    } else if (partitionColType.isStringFamily() || partitionColType.isIntegerFamily()) {//integer like 20160101
        String partitionDateFormat = part.getPartitionDateFormat();
        if (StringUtils.isEmpty(partitionDateFormat)) {
            value = "" + ts;
        } else {
            value = DateFormat.formatToDateStr(ts, partitionDateFormat);
        }
    } else {
        throw new RuntimeException("Type " + partitionColType + " is not valid partition column type");
    }
    return value;
}
 

public static DataModelDesc generateKylinModel(String owner, String tableName, List<String> dimensions,
        List<String> measures, PartitionDesc partitionDesc) {
    ModelDimensionDesc modelDimensionDesc = new ModelDimensionDesc();
    modelDimensionDesc.setTable(tableName);
    modelDimensionDesc.setColumns(dimensions.toArray(new String[dimensions.size()]));

    DataModelDesc kylinModel = new DataModelDesc();
    kylinModel.setName(tableName.replace('.', '_'));
    kylinModel.setOwner(owner);
    kylinModel.setDescription("");
    kylinModel.setLastModified(0L);
    kylinModel.setRootFactTableName(tableName);
    kylinModel.setJoinTables(new JoinTableDesc[0]);
    kylinModel.setDimensions(Lists.newArrayList(modelDimensionDesc));
    kylinModel.setMetrics(measures.toArray(new String[measures.size()]));
    kylinModel.setFilterCondition("");
    kylinModel.setPartitionDesc(partitionDesc);
    kylinModel.setCapacity(DataModelDesc.RealizationCapacity.SMALL);
    kylinModel.updateRandomUuid();

    return kylinModel;
}
 

private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine, SqlDialect sqlDialect) {
    final String sep = singleLine ? " " : "\n";

    StringBuilder whereBuilder = new StringBuilder();
    whereBuilder.append("WHERE 1=1");

    DataModelDesc model = flatDesc.getDataModel();
    if (StringUtils.isNotEmpty(model.getFilterCondition())) {
        String filterCondition = model.getFilterCondition();
        if (flatDesc.getSegment() != null) {
            JoinedFormatter formatter = new JoinedFormatter(flatDesc);
            filterCondition = formatter.formatSentence(model.getFilterCondition());
        }
        String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc, filterCondition, null);
        whereBuilder.append(" AND (").append(quotedFilterCondition).append(") "); // -> filter condition contains special character may cause bug
    }
    if (flatDesc.getSegment() != null) {
        PartitionDesc partDesc = model.getPartitionDesc();
        if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
            SegmentRange segRange = flatDesc.getSegRange();

            if (segRange != null && !segRange.isInfinite()) {
                whereBuilder.append(" AND (");
                String quotedPartitionCond = quoteIdentifierInSqlExpr(flatDesc,
                        partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, flatDesc.getSegment(), segRange, null), sqlDialect);
                whereBuilder.append(quotedPartitionCond);
                whereBuilder.append(")" + sep);
            }
        }
    }

    sql.append(whereBuilder.toString());
}
 

CubeSegment appendSegment(CubeInstance cube, TSRange tsRange, SegmentRange segRange,
        Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd)
        throws IOException {
    CubeInstance cubeCopy = cube.latestCopyForWrite(); // get a latest copy

    checkInputRanges(tsRange, segRange);

    // fix start/end a bit
    PartitionDesc partitionDesc = cubeCopy.getModel().getPartitionDesc();
    if (partitionDesc != null && partitionDesc.isPartitioned()) {
        // if missing start, set it to where last time ends
        if (tsRange != null && tsRange.start.v == 0) {
            CubeDesc cubeDesc = cubeCopy.getDescriptor();
            CubeSegment last = cubeCopy.getLastSegment();
            if (last == null)
                tsRange = new TSRange(cubeDesc.getPartitionDateStart(), tsRange.end.v);
            else if (!last.isOffsetCube())
                tsRange = new TSRange(last.getTSRange().end.v, tsRange.end.v);
        }
    } else {
        // full build
        tsRange = null;
        segRange = null;
    }

    CubeSegment newSegment = newSegment(cubeCopy, tsRange, segRange);
    newSegment.setSourcePartitionOffsetStart(sourcePartitionOffsetStart);
    newSegment.setSourcePartitionOffsetEnd(sourcePartitionOffsetEnd);
    validateNewSegments(cubeCopy, newSegment);

    CubeUpdate update = new CubeUpdate(cubeCopy);
    update.setToAddSegs(newSegment);
    updateCube(update);
    return newSegment;
}
 

private static TblColRef getPartitionCol(IRealization realization) {
    String modelName = realization.getModel().getName();
    DataModelDesc dataModelDesc = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getDataModelDesc(modelName);
    PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
    Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
    TblColRef partitionColRef = partitionDesc.getPartitionDateColumnRef();
    Preconditions.checkArgument(partitionColRef != null,
            "getPartitionDateColumnRef for " + realization + " is null");
    return partitionColRef;
}
 

private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine,
        IJdbcMetadata metadata, Map<String, String> metaMap) {
    final String sep = singleLine ? " " : "\n";

    StringBuilder whereBuilder = new StringBuilder();
    whereBuilder.append("WHERE 1=1");

    DataModelDesc model = flatDesc.getDataModel();
    if (StringUtils.isNotEmpty(model.getFilterCondition())) {
        whereBuilder.append(" AND (").append(model.getFilterCondition()).append(") ");
    }

    if (flatDesc.getSegment() != null) {
        PartitionDesc partDesc = model.getPartitionDesc();
        if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
            SegmentRange segRange = flatDesc.getSegRange();

            if (segRange != null && !segRange.isInfinite()) {
                whereBuilder.append(" AND (");
                whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc,
                        flatDesc.getSegment(), segRange,
                        col -> getTableColumnIdentityQuoted(col, metadata, metaMap, true)));
                whereBuilder.append(")");
                whereBuilder.append(sep);
            }
        }
    }
    sql.append(whereBuilder.toString());
}
 

@Test
public void testAppendNonPartitioned2() throws IOException {
    CubeManager mgr = mgr();
    CubeInstance cube = mgr.getCube("test_kylin_cube_without_slr_ready");

    // override partition desc
    cube.getModel().setPartitionDesc(new PartitionDesc());

    // assert one ready segment
    assertEquals(1, cube.getSegments().size());
    CubeSegment seg = cube.getSegments(SegmentStatusEnum.READY).get(0);
    assertEquals(SegmentStatusEnum.READY, seg.getStatus());

    // append again, for non-partitioned cube, it becomes a full refresh
    CubeSegment seg2 = mgr.appendSegment(cube);
    assertEquals(new TSRange(0L, Long.MAX_VALUE), seg2.getTSRange());
    assertEquals(new TSRange(0L, Long.MAX_VALUE), seg2.getSegRange());
    
    assertEquals(1, cube.getSegments().size()); // older cube not changed
    cube = mgr.getCube(cube.getName());
    assertEquals(2, cube.getSegments().size()); // the updated cube

    // non-partitioned cannot merge, throw exception
    try {
        mgr.mergeSegments(cube, null, new SegmentRange(0L, Long.MAX_VALUE), false);
        fail();
    } catch (IllegalStateException ex) {
        // good
    }
}
 

public static PartitionDesc getPartitionDesc(String tableName) {
    PartitionDesc partitionDesc = new PartitionDesc();

    partitionDesc.setPartitionDateColumn(tableName + "." + TimePropertyEnum.DAY_DATE.toString());
    partitionDesc.setPartitionTimeColumn(tableName + "." + TimePropertyEnum.DAY_TIME.toString());
    return partitionDesc;
}
 

@Test
public void testAppendNonPartitioned2() throws IOException {
    CubeManager mgr = mgr();
    CubeInstance cube = mgr.getCube("test_kylin_cube_without_slr_ready");

    // override partition desc
    cube.getModel().setPartitionDesc(new PartitionDesc());

    // assert one ready segment
    assertEquals(1, cube.getSegments().size());
    CubeSegment seg = cube.getSegments(SegmentStatusEnum.READY).get(0);
    assertEquals(SegmentStatusEnum.READY, seg.getStatus());

    // append again, for non-partitioned cube, it becomes a full refresh
    CubeSegment seg2 = mgr.appendSegment(cube);
    assertEquals(new TSRange(0L, Long.MAX_VALUE), seg2.getTSRange());
    assertEquals(new TSRange(0L, Long.MAX_VALUE), seg2.getSegRange());
    
    assertEquals(1, cube.getSegments().size()); // older cube not changed
    cube = mgr.getCube(cube.getName());
    assertEquals(2, cube.getSegments().size()); // the updated cube

    // non-partitioned cannot merge, throw exception
    try {
        mgr.mergeSegments(cube, null, new SegmentRange(0L, Long.MAX_VALUE), false);
        fail();
    } catch (IllegalStateException ex) {
        // good
    }
}
 

CubeSegment appendSegment(CubeInstance cube, TSRange tsRange, SegmentRange segRange,
        Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd)
        throws IOException {
    CubeInstance cubeCopy = cube.latestCopyForWrite(); // get a latest copy

    checkInputRanges(tsRange, segRange);

    // fix start/end a bit
    PartitionDesc partitionDesc = cubeCopy.getModel().getPartitionDesc();
    if (partitionDesc != null && partitionDesc.isPartitioned()) {
        // if missing start, set it to where last time ends
        if (tsRange != null && tsRange.start.v == 0) {
            CubeDesc cubeDesc = cubeCopy.getDescriptor();
            CubeSegment last = cubeCopy.getLastSegment();
            if (last == null)
                tsRange = new TSRange(cubeDesc.getPartitionDateStart(), tsRange.end.v);
            else if (!last.isOffsetCube())
                tsRange = new TSRange(last.getTSRange().end.v, tsRange.end.v);
        }
    } else {
        // full build
        tsRange = null;
        segRange = null;
    }

    CubeSegment newSegment = newSegment(cubeCopy, tsRange, segRange);
    newSegment.setSourcePartitionOffsetStart(sourcePartitionOffsetStart);
    newSegment.setSourcePartitionOffsetEnd(sourcePartitionOffsetEnd);
    validateNewSegments(cubeCopy, newSegment);

    CubeUpdate update = new CubeUpdate(cubeCopy);
    update.setToAddSegs(newSegment);
    updateCube(update);
    return newSegment;
}
 

private static TblColRef getPartitionCol(IRealization realization) {
    String modelName = realization.getModel().getName();
    DataModelDesc dataModelDesc = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getDataModelDesc(modelName);
    PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
    Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
    TblColRef partitionColRef = partitionDesc.getPartitionDateColumnRef();
    Preconditions.checkArgument(partitionColRef != null,
            "getPartitionDateColumnRef for " + realization + " is null");
    return partitionColRef;
}
 

private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine, SqlDialect sqlDialect) {
    final String sep = singleLine ? " " : "\n";

    StringBuilder whereBuilder = new StringBuilder();
    whereBuilder.append("WHERE 1=1");

    DataModelDesc model = flatDesc.getDataModel();
    if (StringUtils.isNotEmpty(model.getFilterCondition())) {
        String filterCondition = model.getFilterCondition();
        if (flatDesc.getSegment() != null) {
            JoinedFormatter formatter = new JoinedFormatter(flatDesc);
            filterCondition = formatter.formatSentence(model.getFilterCondition());
        }
        String quotedFilterCondition = quoteIdentifierInSqlExpr(flatDesc, filterCondition, null);
        whereBuilder.append(" AND (").append(quotedFilterCondition).append(") "); // -> filter condition contains special character may cause bug
    }
    if (flatDesc.getSegment() != null) {
        PartitionDesc partDesc = model.getPartitionDesc();
        if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
            SegmentRange segRange = flatDesc.getSegRange();

            if (segRange != null && !segRange.isInfinite()) {
                whereBuilder.append(" AND (");
                String quotedPartitionCond = quoteIdentifierInSqlExpr(flatDesc,
                        partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc, flatDesc.getSegment(), segRange, null), sqlDialect);
                whereBuilder.append(quotedPartitionCond);
                whereBuilder.append(")" + sep);
            }
        }
    }

    sql.append(whereBuilder.toString());
}
 

private void updatePartitionDesc(CubeDesc oldModel, DataModelDesc dm) {

        CubePartitionDesc partition = oldModel.getCubePartitionDesc();
        PartitionDesc newPartition = new PartitionDesc();

        if (partition.getPartitionDateColumn() != null) {
            String partitionCol = partition.getPartitionDateColumn();

            String[] tablecolumn = partitionCol.split("\\.");
            if (tablecolumn != null && tablecolumn.length == 2) {
                // pattern is <tablename>.<colname>
                String tableFullName = getMetadataManager().appendDBName(tablecolumn[0]);
                newPartition.setPartitionDateColumn(tableFullName + "." + tablecolumn[1]);
            } else {

                if (partitionCol.indexOf(".") < 0) {
                    // pattern is <colname>
                    partitionCol = dm.getFactTable() + "." + partitionCol;
                }

                newPartition.setPartitionDateColumn(partitionCol);
            }
        }

        // only append is supported
        newPartition.setCubePartitionType(PartitionDesc.PartitionType.APPEND);

        newPartition.setPartitionDateStart(partition.getPartitionDateStart());

        dm.setPartitionDesc(newPartition);
    }
 

private static void appendWhereStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine,
        IJdbcMetadata metadata, Map<String, String> metaMap) {
    final String sep = singleLine ? " " : "\n";

    StringBuilder whereBuilder = new StringBuilder();
    whereBuilder.append("WHERE 1=1");

    DataModelDesc model = flatDesc.getDataModel();
    if (StringUtils.isNotEmpty(model.getFilterCondition())) {
        whereBuilder.append(" AND (").append(model.getFilterCondition()).append(") ");
    }

    if (flatDesc.getSegment() != null) {
        PartitionDesc partDesc = model.getPartitionDesc();
        if (partDesc != null && partDesc.getPartitionDateColumn() != null) {
            SegmentRange segRange = flatDesc.getSegRange();

            if (segRange != null && !segRange.isInfinite()) {
                whereBuilder.append(" AND (");
                whereBuilder.append(partDesc.getPartitionConditionBuilder().buildDateRangeCondition(partDesc,
                        flatDesc.getSegment(), segRange,
                        col -> getTableColumnIdentityQuoted(col, metadata, metaMap, true)));
                whereBuilder.append(")");
                whereBuilder.append(sep);
            }
        }
    }
    sql.append(whereBuilder.toString());
}
 

public static PartitionDesc getPartitionDesc(String tableName) {
    PartitionDesc partitionDesc = new PartitionDesc();

    partitionDesc.setPartitionDateColumn(tableName + "." + TimePropertyEnum.DAY_DATE.toString());
    partitionDesc.setPartitionTimeColumn(tableName + "." + TimePropertyEnum.DAY_TIME.toString());
    return partitionDesc;
}
 

public CubeSegment refreshSegment(CubeInstance cube, TSRange tsRange, SegmentRange segRange)
        throws IOException {
    CubeInstance cubeCopy = cube.latestCopyForWrite(); // get a latest copy

    checkInputRanges(tsRange, segRange);
    PartitionDesc partitionDesc = cubeCopy.getModel().getPartitionDesc();
    if (partitionDesc == null || partitionDesc.isPartitioned() == false) {
        // full build
        tsRange = null;
        segRange = null;
    }

    CubeSegment newSegment = newSegment(cubeCopy, tsRange, segRange);

    Pair<Boolean, Boolean> pair = cubeCopy.getSegments().fitInSegments(newSegment);
    if (pair.getFirst() == false || pair.getSecond() == false)
        throw new IllegalArgumentException("The new refreshing segment " + newSegment
                + " does not match any existing segment in cube " + cubeCopy);

    if (segRange != null) {
        CubeSegment toRefreshSeg = null;
        for (CubeSegment cubeSegment : cubeCopy.getSegments()) {
            if (cubeSegment.getSegRange().equals(segRange)) {
                toRefreshSeg = cubeSegment;
                break;
            }
        }

        if (toRefreshSeg == null) {
            throw new IllegalArgumentException(
                    "For streaming cube, only one segment can be refreshed at one time");
        }

        newSegment.setSourcePartitionOffsetStart(toRefreshSeg.getSourcePartitionOffsetStart());
        newSegment.setSourcePartitionOffsetEnd(toRefreshSeg.getSourcePartitionOffsetEnd());
    }

    CubeUpdate update = new CubeUpdate(cubeCopy);
    update.setToAddSegs(newSegment);
    updateCube(update);

    return newSegment;
}
 

public static DataModelDesc dealWithMappingForModel(DataModelDesc other,
        Map<String, TableSchemaUpdateMapping> mappings) {
    // For filter condition, not support
    if (!Strings.isNullOrEmpty(other.getFilterCondition())) {
        throw new UnsupportedOperationException("Cannot deal with filter condition " + other.getFilterCondition());
    }

    DataModelDesc copy = DataModelDesc.getCopyOf(other);
    copy.setLastModified(other.getLastModified());

    // mapping for root fact table identity
    TableSchemaUpdateMapping rootMapping = getTableSchemaUpdateMapping(mappings, other.getRootFactTableName());
    if (rootMapping != null) {
        TableDesc rootFactTable = other.getRootFactTable().getTableDesc();
        copy.setRootFactTableName(
                rootMapping.getTableIdentity(rootFactTable.getDatabase(), rootFactTable.getName()));
    }

    // mapping for joins
    JoinTableDesc[] joinTables = other.getJoinTables();
    JoinTableDesc[] joinTablesCopy = new JoinTableDesc[joinTables.length];
    for (int i = 0; i < joinTables.length; i++) {
        JoinTableDesc joinTable = joinTables[i];
        joinTablesCopy[i] = JoinTableDesc.getCopyOf(joinTable);
        String tableIdentity = joinTable.getTable();
        TableSchemaUpdateMapping mapping = getTableSchemaUpdateMapping(mappings, tableIdentity);
        if (mapping != null && mapping.isTableIdentityChanged()) {
            joinTablesCopy[i].setTable(mapping.getTableIdentity(tableIdentity));
        }
    }
    copy.setJoinTables(joinTablesCopy);

    // mapping for partition columns
    PartitionDesc partDesc = other.getPartitionDesc();
    PartitionDesc partCopy = PartitionDesc.getCopyOf(partDesc);
    if (partDesc.getPartitionDateColumnRef() != null) {
        partCopy.setPartitionDateColumn(
                replacePartitionCol(partDesc.getPartitionDateColumnRef().getCanonicalName(), mappings));
    }
    if (partDesc.getPartitionTimeColumnRef() != null) {
        partCopy.setPartitionTimeColumn(
                replacePartitionCol(partDesc.getPartitionTimeColumnRef().getCanonicalName(), mappings));
    }
    copy.setPartitionDesc(partCopy);

    return copy;
}
 

/**
 * Smartly figure out the TOBE segments once all new segments are built.
 * - Ensures no gap, no overlap
 * - Favors new segments over the old
 * - Favors big segments over the small
 */
private List<CubeSegment> calculateToBeSegments(CubeInstance cube, CubeSegment... newSegments) {
    CubeDesc cubeDesc = cube.getDescriptor();
    PartitionDesc partDesc = cubeDesc.getModel().getPartitionDesc();

    List<CubeSegment> tobe = Lists.newArrayList(cube.getSegments());
    if (newSegments != null)
        tobe.addAll(Arrays.asList(newSegments));
    if (tobe.size() == 0)
        return tobe;

    // sort by start time, then end time
    Collections.sort(tobe);

    // check first segment start time
    CubeSegment firstSeg = tobe.get(0);
    if (firstSeg.getDateRangeStart() != partDesc.getPartitionDateStart()) {
        throw new IllegalStateException("For " + cube + ", the first segment, " + firstSeg + ", must start at " + partDesc.getPartitionDateStart());
    }
    firstSeg.validate();

    for (int i = 0, j = 1; j < tobe.size(); ) {
        CubeSegment is = tobe.get(i);
        CubeSegment js = tobe.get(j);
        js.validate();

        // check i is either ready or new
        if (!isNew(is) && !isReady(is)) {
            tobe.remove(i);
            continue;
        }

        // check j is either ready or new
        if (!isNew(js) && !isReady(js)) {
            tobe.remove(j);
            continue;
        }

        // if i, j competes
        if (is.getDateRangeStart() == js.getDateRangeStart()) {
            // if both new or ready, favor the bigger segment
            if (isReady(is) && isReady(js) || isNew(is) && isNew(js)) {
                if (is.getDateRangeEnd() <= js.getDateRangeEnd()) {
                    tobe.remove(i);
                } else {
                    tobe.remove(j);
                }
            }
            // otherwise, favor the new segment
            else if (isNew(is)) {
                tobe.remove(j);
            } else {
                tobe.remove(i);
            }
            continue;
        }

        // if i, j in sequence
        if (is.getDateRangeEnd() == js.getDateRangeStart()) {
            i++;
            j++;
            continue;
        }

        // seems j not fitting
        tobe.remove(j);
    }

    return tobe;
}
 

private boolean isExactAggregation(StorageContext context, Cuboid cuboid, Collection<TblColRef> groups,
        Set<TblColRef> othersD, Set<TblColRef> singleValuesD, Set<TblColRef> derivedPostAggregation,
        Collection<FunctionDesc> functionDescs, List<SQLDigest.SQLCall> aggrSQLCalls, boolean groupByExpression) {
    if (context.isNeedStorageAggregation()) {
        logger.info("exactAggregation is false because need storage aggregation");
        return false;
    }

    if (cuboid.requirePostAggregation()) {
        logger.info("exactAggregation is false because cuboid {}=>{}", cuboid.getInputID(), cuboid.getId());
        return false;
    }

    // derived aggregation is bad, unless expanded columns are already in group by
    if (!groups.containsAll(derivedPostAggregation)) {
        logger.info("exactAggregation is false because derived column require post aggregation: {}",
                derivedPostAggregation);
        return false;
    }

    // other columns (from filter) is bad, unless they are ensured to have single value
    if (!singleValuesD.containsAll(othersD)) {
        logger.info("exactAggregation is false because some column not on group by: {} (single value column: {})",
                othersD, singleValuesD);
        return false;
    }

    //for DimensionAsMetric like max(cal_dt), the dimension column maybe not in real group by
    for (FunctionDesc functionDesc : functionDescs) {
        if (functionDesc.isDimensionAsMetric()) {
            logger.info("exactAggregation is false because has DimensionAsMetric");
            return false;
        }
    }
    for (SQLDigest.SQLCall aggrSQLCall : aggrSQLCalls) {
        if (aggrSQLCall.function.equals(BitmapMeasureType.FUNC_INTERSECT_COUNT_DISTINCT)
        || aggrSQLCall.function.equals(BitmapMeasureType.FUNC_INTERSECT_VALUE)) {
            logger.info("exactAggregation is false because has INTERSECT_COUNT OR INTERSECT_VALUE");
            return false;
        }
    }

    // for partitioned cube, the partition column must belong to group by or has single value
    PartitionDesc partDesc = cuboid.getCubeDesc().getModel().getPartitionDesc();
    if (partDesc.isPartitioned()) {
        TblColRef col = partDesc.getPartitionDateColumnRef();
        if (!groups.contains(col) && !singleValuesD.contains(col)) {
            logger.info("exactAggregation is false because cube is partitioned and {} is not on group by", col);
            return false;
        }
    }

    // for group by expression like: group by seller_id/100. seller_id_1(200) get 2, seller_id_2(201) also get 2, so can't aggregate exactly
    if (groupByExpression) {
        logger.info("exactAggregation is false because group by expression");
        return false;
    }

    logger.info("exactAggregation is true, cuboid id is {}", cuboid.getId());
    return true;
}