Java Code Examples for org.apache.kylin.cube.CubeInstance#setStatus()
The following examples show how to use
org.apache.kylin.cube.CubeInstance#setStatus() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: CubeService.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
|
/**
* Update a cube status from disable to ready.
*
* @return
* @throws IOException
*/
public CubeInstance enableCube(CubeInstance cube) throws IOException {
boolean cubeStatusUpdated = false;
RealizationStatusEnum ostatus = cube.getStatus();
try {
CubeInstance cubeInstance = getCubeManager().updateCubeStatus(cube, RealizationStatusEnum.READY);
cubeStatusUpdated = true;
// for streaming cube.
if (cube.getDescriptor().isStreamingCube()) {
getStreamingCoordinator().assignCube(cube.getName());
}
return cubeInstance;
} catch (Exception e) {
cube.setStatus(ostatus);
// roll back if cube status updated
if (cubeStatusUpdated) {
logger.info("roll back cube status to:{}", ostatus);
getCubeManager().updateCubeStatus(cube, ostatus);
}
throw e;
}
}
Example 2
| Source File: CubeService.java From kylin with Apache License 2.0 | 6 votes |
|
/**
* Update a cube status from disable to ready.
*
* @return
* @throws IOException
*/
public CubeInstance enableCube(CubeInstance cube) throws IOException {
boolean cubeStatusUpdated = false;
RealizationStatusEnum ostatus = cube.getStatus();
try {
CubeInstance cubeInstance = getCubeManager().updateCubeStatus(cube, RealizationStatusEnum.READY);
cubeStatusUpdated = true;
// for streaming cube.
if (cube.getDescriptor().isStreamingCube()) {
getStreamingCoordinator().assignCube(cube.getName());
}
return cubeInstance;
} catch (Exception e) {
cube.setStatus(ostatus);
// roll back if cube status updated
if (cubeStatusUpdated) {
logger.info("roll back cube status to:{}", ostatus);
getCubeManager().updateCubeStatus(cube, ostatus);
}
throw e;
}
}
Example 3
| Source File: CubeService.java From Kylin with Apache License 2.0 | 6 votes |
|
/**
* Update a cube status from ready to disabled.
*
* @return
* @throws CubeIntegrityException
* @throws IOException
* @throws JobException
*/
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
@Caching(evict = {@CacheEvict(value = QueryController.SUCCESS_QUERY_CACHE, allEntries = true), @CacheEvict(value = QueryController.EXCEPTION_QUERY_CACHE, allEntries = true)})
public CubeInstance disableCube(CubeInstance cube) throws IOException, JobException {
String cubeName = cube.getName();
RealizationStatusEnum ostatus = cube.getStatus();
if (null != ostatus && !RealizationStatusEnum.READY.equals(ostatus)) {
throw new InternalErrorException("Only ready cube can be disabled, status of " + cubeName + " is " + ostatus);
}
cube.setStatus(RealizationStatusEnum.DISABLED);
try {
return getCubeManager().updateCube(cube);
} catch (IOException e) {
cube.setStatus(ostatus);
throw e;
}
}
Example 4
| Source File: CubeInstanceCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
public static CubeInstance generateKylinCubeInstance(String owner, String tableName) {
CubeInstance cubeInstance = new CubeInstance();
cubeInstance.setName(tableName.replace('.', '_'));
cubeInstance.setSegments(new Segments<CubeSegment>());
cubeInstance.setDescName(tableName.replace('.', '_'));
cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
cubeInstance.setOwner(owner);
cubeInstance.setCreateTimeUTC(0L);
cubeInstance.updateRandomUuid();
return cubeInstance;
}
Example 5
| Source File: CubeControllerTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
@Test
public void tesDeleteDescBrokenCube() throws Exception {
final String cubeName = "ci_left_join_cube";
CubeInstance cubeInstance = cubeService.getCubeManager().getCube(cubeName);
CubeDesc cubeDesc = cubeInstance.getDescriptor();
cubeDesc.setModel(null);
cubeInstance.setStatus(RealizationStatusEnum.DESCBROKEN);
cubeController.deleteCube(cubeName);
Assert.assertNull(cubeService.getCubeManager().getCube(cubeName));
}
Example 6
| Source File: CubeInstanceCreator.java From kylin with Apache License 2.0 | 5 votes |
|
public static CubeInstance generateKylinCubeInstance(String owner, String tableName) {
CubeInstance cubeInstance = new CubeInstance();
cubeInstance.setName(tableName.replace('.', '_'));
cubeInstance.setSegments(new Segments<CubeSegment>());
cubeInstance.setDescName(tableName.replace('.', '_'));
cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
cubeInstance.setOwner(owner);
cubeInstance.setCreateTimeUTC(0L);
cubeInstance.updateRandomUuid();
return cubeInstance;
}
Example 7
| Source File: CubeControllerTest.java From kylin with Apache License 2.0 | 5 votes |
|
@Test
public void tesDeleteDescBrokenCube() throws Exception {
final String cubeName = "ci_left_join_cube";
CubeInstance cubeInstance = cubeService.getCubeManager().getCube(cubeName);
CubeDesc cubeDesc = cubeInstance.getDescriptor();
cubeDesc.setModel(null);
cubeInstance.setStatus(RealizationStatusEnum.DESCBROKEN);
cubeController.deleteCube(cubeName);
Assert.assertNull(cubeService.getCubeManager().getCube(cubeName));
}
Example 8
| Source File: CubeService.java From Kylin with Apache License 2.0 | 5 votes |
|
/**
* Update a cube status from disable to ready.
*
* @return
* @throws CubeIntegrityException
* @throws IOException
* @throws JobException
*/
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'OPERATION') or hasPermission(#cube, 'MANAGEMENT')")
public CubeInstance enableCube(CubeInstance cube) throws IOException, JobException {
String cubeName = cube.getName();
RealizationStatusEnum ostatus = cube.getStatus();
if (!cube.getStatus().equals(RealizationStatusEnum.DISABLED)) {
throw new InternalErrorException("Only disabled cube can be enabled, status of " + cubeName + " is " + ostatus);
}
if (cube.getSegments(SegmentStatusEnum.READY).size() == 0) {
throw new InternalErrorException("Cube " + cubeName + " dosen't contain any READY segment");
}
final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
if (!cubingJobs.isEmpty()) {
throw new JobException("Enable is not allowed with a running job.");
}
if (!cube.getDescriptor().calculateSignature().equals(cube.getDescriptor().getSignature())) {
this.releaseAllSegments(cube);
}
cube.setStatus(RealizationStatusEnum.READY);
try {
return getCubeManager().updateCube(cube);
} catch (IOException e) {
cube.setStatus(ostatus);
throw e;
}
}
Example 9
| Source File: CubeMetaExtractor.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
private void addSegAndJob(CubeInstance cube) {
if (includeSegments) {
addRequired(CubeInstance.concatResourcePath(cube.getName()));
for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) {
addRequired(CubeSegment.getStatisticsResourcePath(cube.getName(), segment.getUuid()));
if (includeSegmentDetails) {
for (String dictPat : segment.getDictionaryPaths()) {
addRequired(dictPat);
}
for (String snapshotPath : segment.getSnapshotPaths()) {
addRequired(snapshotPath);
}
}
if (includeJobs) {
String lastJobId = segment.getLastBuildJobID();
if (StringUtils.isEmpty(lastJobId)) {
throw new RuntimeException("No job exist for segment :" + segment);
} else {
try {
if (onlyJobOutput) {
addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + lastJobId);
} else {
ExecutablePO executablePO = executableDao.getJob(lastJobId);
addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + lastJobId);
addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + lastJobId);
for (ExecutablePO task : executablePO.getTasks()) {
addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + task.getUuid());
addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
}
}
} catch (PersistentException e) {
throw new RuntimeException("PersistentException", e);
}
}
}
}
} else {
if (includeJobs) {
logger.warn("It's useless to set includeJobs to true when includeSegments is set to false");
}
cube.setStatus(RealizationStatusEnum.DISABLED);
cubesToTrimAndSave.add(cube);
}
}
Example 10
| Source File: ExtendCubeToHybridCLI.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
public void createFromCube(String projectName, String cubeName, String partitionDateStr) throws Exception {
logger.info("Create hybrid for cube[" + cubeName + "], project[" + projectName + "], partition_date[" + partitionDateStr + "].");
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
if (!validateCubeInstance(cubeInstance)) {
return;
}
CubeDesc cubeDesc = cubeDescManager.getCubeDesc(cubeInstance.getDescName());
DataModelDesc dataModelDesc = metadataManager.getDataModelDesc(cubeDesc.getModelName());
if (StringUtils.isEmpty(dataModelDesc.getPartitionDesc().getPartitionDateColumn())) {
logger.error("No incremental cube, no need to extend.");
return;
}
String owner = cubeInstance.getOwner();
long partitionDate = partitionDateStr != null ? DateFormat.stringToMillis(partitionDateStr) : 0;
// get new name for old cube and cube_desc
String newCubeDescName = renameCube(cubeDesc.getName());
String newCubeInstanceName = renameCube(cubeInstance.getName());
while (cubeDescManager.getCubeDesc(newCubeDescName) != null)
newCubeDescName = renameCube(newCubeDescName);
while (cubeManager.getCube(newCubeInstanceName) != null)
newCubeInstanceName = renameCube(newCubeInstanceName);
// create new cube_instance for old segments
CubeInstance newCubeInstance = CubeInstance.getCopyOf(cubeInstance);
newCubeInstance.setName(newCubeInstanceName);
newCubeInstance.setDescName(newCubeDescName);
newCubeInstance.updateRandomUuid();
Iterator<CubeSegment> segmentIterator = newCubeInstance.getSegments().iterator();
CubeSegment currentSeg = null;
while (segmentIterator.hasNext()) {
currentSeg = segmentIterator.next();
if (partitionDateStr != null && (currentSeg.getTSRange().start.v >= partitionDate || currentSeg.getTSRange().end.v > partitionDate)) {
segmentIterator.remove();
logger.info("CubeSegment[" + currentSeg + "] was removed.");
}
}
if (currentSeg != null && partitionDateStr != null && partitionDate != currentSeg.getTSRange().end.v) {
logger.error("PartitionDate must be end date of one segment.");
return;
}
if (currentSeg != null && partitionDateStr == null)
partitionDate = currentSeg.getTSRange().end.v;
cubeManager.createCube(newCubeInstance, projectName, owner);
logger.info("CubeInstance was saved at: " + newCubeInstance.getResourcePath());
// create new cube for old segments
CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
newCubeDesc.setName(newCubeDescName);
newCubeDesc.updateRandomUuid();
newCubeDesc.init(kylinConfig);
newCubeDesc.setPartitionDateEnd(partitionDate);
newCubeDesc.calculateSignature();
cubeDescManager.createCubeDesc(newCubeDesc);
logger.info("CubeDesc was saved at: " + newCubeDesc.getResourcePath());
// update old cube_desc to new-version metadata
cubeDesc.setPartitionDateStart(partitionDate);
cubeDesc.setEngineType(IEngineAware.ID_MR_V2);
cubeDesc.setStorageType(IStorageAware.ID_SHARDED_HBASE);
cubeDesc.calculateSignature();
cubeDescManager.updateCubeDesc(cubeDesc);
logger.info("CubeDesc was saved at: " + cubeDesc.getResourcePath());
// clear segments for old cube
cubeInstance.setSegments(new Segments());
cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
store.checkAndPutResource(cubeInstance.getResourcePath(), cubeInstance, CubeManager.CUBE_SERIALIZER);
logger.info("CubeInstance was saved at: " + cubeInstance.getResourcePath());
// create hybrid model for these two cubes
List<RealizationEntry> realizationEntries = Lists.newArrayListWithCapacity(2);
realizationEntries.add(RealizationEntry.create(RealizationType.CUBE, cubeInstance.getName()));
realizationEntries.add(RealizationEntry.create(RealizationType.CUBE, newCubeInstance.getName()));
HybridInstance hybridInstance = HybridInstance.create(kylinConfig, renameHybrid(cubeInstance.getName()), realizationEntries);
store.checkAndPutResource(hybridInstance.getResourcePath(), hybridInstance, HybridManager.HYBRID_SERIALIZER);
ProjectManager.getInstance(kylinConfig).moveRealizationToProject(RealizationType.HYBRID, hybridInstance.getName(), projectName, owner);
logger.info("HybridInstance was saved at: " + hybridInstance.getResourcePath());
// copy Acl from old cube to new cube
copyAcl(cubeInstance.getId(), newCubeInstance.getId(), projectName);
logger.info("Acl copied from [" + cubeName + "] to [" + newCubeInstanceName + "].");
}
Example 11
| Source File: ExtendCubeToHybridCLI.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
public void createFromCube(String projectName, String cubeName, String partitionDateStr) throws Exception {
logger.info("Create hybrid for cube[" + cubeName + "], project[" + projectName + "], partition_date[" + partitionDateStr + "].");
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
if (!validateCubeInstance(cubeInstance)) {
return;
}
CubeDesc cubeDesc = cubeDescManager.getCubeDesc(cubeInstance.getDescName());
DataModelDesc dataModelDesc = metadataManager.getDataModelDesc(cubeDesc.getModelName());
if (StringUtils.isEmpty(dataModelDesc.getPartitionDesc().getPartitionDateColumn())) {
logger.error("No incremental cube, no need to extend.");
return;
}
String owner = cubeInstance.getOwner();
long partitionDate = partitionDateStr != null ? DateFormat.stringToMillis(partitionDateStr) : 0;
// get new name for old cube and cube_desc
String newCubeDescName = renameCube(cubeDesc.getName());
String newCubeInstanceName = renameCube(cubeInstance.getName());
while (cubeDescManager.getCubeDesc(newCubeDescName) != null)
newCubeDescName = renameCube(newCubeDescName);
while (cubeManager.getCube(newCubeInstanceName) != null)
newCubeInstanceName = renameCube(newCubeInstanceName);
// create new cube_instance for old segments
CubeInstance newCubeInstance = CubeInstance.getCopyOf(cubeInstance);
newCubeInstance.setName(newCubeInstanceName);
newCubeInstance.setDescName(newCubeDescName);
newCubeInstance.updateRandomUuid();
Iterator<CubeSegment> segmentIterator = newCubeInstance.getSegments().iterator();
CubeSegment currentSeg = null;
while (segmentIterator.hasNext()) {
currentSeg = segmentIterator.next();
if (partitionDateStr != null && (currentSeg.getTSRange().start.v >= partitionDate || currentSeg.getTSRange().end.v > partitionDate)) {
segmentIterator.remove();
logger.info("CubeSegment[" + currentSeg + "] was removed.");
}
}
if (partitionDateStr != null && partitionDate != currentSeg.getTSRange().end.v) {
logger.error("PartitionDate must be end date of one segment.");
return;
}
if (currentSeg != null && partitionDateStr == null)
partitionDate = currentSeg.getTSRange().end.v;
cubeManager.createCube(newCubeInstance, projectName, owner);
logger.info("CubeInstance was saved at: " + newCubeInstance.getResourcePath());
// create new cube for old segments
CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
newCubeDesc.setName(newCubeDescName);
newCubeDesc.updateRandomUuid();
newCubeDesc.init(kylinConfig);
newCubeDesc.setPartitionDateEnd(partitionDate);
newCubeDesc.calculateSignature();
cubeDescManager.createCubeDesc(newCubeDesc);
logger.info("CubeDesc was saved at: " + newCubeDesc.getResourcePath());
// update old cube_desc to new-version metadata
cubeDesc.setPartitionDateStart(partitionDate);
cubeDesc.setEngineType(IEngineAware.ID_MR_V2);
cubeDesc.setStorageType(IStorageAware.ID_SHARDED_HBASE);
cubeDesc.calculateSignature();
cubeDescManager.updateCubeDesc(cubeDesc);
logger.info("CubeDesc was saved at: " + cubeDesc.getResourcePath());
// clear segments for old cube
cubeInstance.setSegments(new Segments<CubeSegment>());
cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
store.checkAndPutResource(cubeInstance.getResourcePath(), cubeInstance, CubeManager.CUBE_SERIALIZER);
logger.info("CubeInstance was saved at: " + cubeInstance.getResourcePath());
// create hybrid model for these two cubes
List<RealizationEntry> realizationEntries = Lists.newArrayListWithCapacity(2);
realizationEntries.add(RealizationEntry.create(RealizationType.CUBE, cubeInstance.getName()));
realizationEntries.add(RealizationEntry.create(RealizationType.CUBE, newCubeInstance.getName()));
HybridInstance hybridInstance = HybridInstance.create(kylinConfig, renameHybrid(cubeInstance.getName()), realizationEntries);
store.checkAndPutResource(hybridInstance.getResourcePath(), hybridInstance, HybridManager.HYBRID_SERIALIZER);
ProjectManager.getInstance(kylinConfig).moveRealizationToProject(RealizationType.HYBRID, hybridInstance.getName(), projectName, owner);
logger.info("HybridInstance was saved at: " + hybridInstance.getResourcePath());
// copy Acl from old cube to new cube
copyAcl(cubeInstance.getId(), newCubeInstance.getId(), projectName);
logger.info("Acl copied from [" + cubeName + "] to [" + newCubeInstanceName + "].");
}
Example 12
| Source File: CubeService.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
/**
* Update a cube status from ready to disabled.
*
* @return
* @throws IOException
* @throws JobException
*/
public CubeInstance disableCube(CubeInstance cube) throws IOException {
aclEvaluate.checkProjectWritePermission(cube);
Message msg = MsgPicker.getMsg();
String cubeName = cube.getName();
RealizationStatusEnum ostatus = cube.getStatus();
if (null != ostatus && !RealizationStatusEnum.READY.equals(ostatus)) {
throw new BadRequestException(
String.format(Locale.ROOT, msg.getDISABLE_NOT_READY_CUBE(), cubeName, ostatus));
}
boolean isStreamingCube = cube.getDescriptor().isStreamingCube();
boolean cubeStatusUpdated = false;
try {
CubeInstance cubeInstance = getCubeManager().updateCubeStatus(cube, RealizationStatusEnum.DISABLED);
cubeStatusUpdated = true;
// for streaming cube.
if (isStreamingCube) {
//drop not ready segments
CubeSegment[] buildingSegments = new CubeSegment[cubeInstance.getBuildingSegments().size()];
Segments segments = cubeInstance.getBuildingSegments();
if (!CollectionUtils.isEmpty(segments)) {
for (int i = 0; i < segments.size(); i++) {
buildingSegments[i] = (CubeSegment) segments.get(i);
}
getCubeManager().dropOptmizingSegments(cubeInstance, buildingSegments);
}
//unAssign cube
getStreamingCoordinator().unAssignCube(cubeName);
//discard jobs
releaseAllJobs(cubeInstance);
}
return cubeInstance;
} catch (Exception e) {
cube.setStatus(ostatus);
// roll back if cube status updated
if (cubeStatusUpdated) {
logger.info("roll back cube status to:{}", ostatus);
getCubeManager().updateCubeStatus(cube, ostatus);
}
throw e;
}
}
Example 13
| Source File: CubeMetaExtractor.java From kylin with Apache License 2.0 | 4 votes |
|
private void addSegAndJob(CubeInstance cube) {
if (includeSegments) {
addRequired(CubeInstance.concatResourcePath(cube.getName()));
for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) {
addRequired(CubeSegment.getStatisticsResourcePath(cube.getName(), segment.getUuid()));
if (includeSegmentDetails) {
for (String dictPat : segment.getDictionaryPaths()) {
addRequired(dictPat);
}
for (String snapshotPath : segment.getSnapshotPaths()) {
addRequired(snapshotPath);
}
}
if (includeJobs) {
String lastJobId = segment.getLastBuildJobID();
if (StringUtils.isEmpty(lastJobId)) {
throw new RuntimeException("No job exist for segment :" + segment);
} else {
try {
if (onlyJobOutput) {
addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + lastJobId);
} else {
ExecutablePO executablePO = executableDao.getJob(lastJobId);
addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + lastJobId);
addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + lastJobId);
for (ExecutablePO task : executablePO.getTasks()) {
addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + task.getUuid());
addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
}
}
} catch (PersistentException e) {
throw new RuntimeException("PersistentException", e);
}
}
}
}
} else {
if (includeJobs) {
logger.warn("It's useless to set includeJobs to true when includeSegments is set to false");
}
cube.setStatus(RealizationStatusEnum.DISABLED);
cubesToTrimAndSave.add(cube);
}
}
Example 14
| Source File: ExtendCubeToHybridCLI.java From kylin with Apache License 2.0 | 4 votes |
|
public void createFromCube(String projectName, String cubeName, String partitionDateStr) throws Exception {
logger.info("Create hybrid for cube[" + cubeName + "], project[" + projectName + "], partition_date[" + partitionDateStr + "].");
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
if (!validateCubeInstance(cubeInstance)) {
return;
}
CubeDesc cubeDesc = cubeDescManager.getCubeDesc(cubeInstance.getDescName());
DataModelDesc dataModelDesc = metadataManager.getDataModelDesc(cubeDesc.getModelName());
if (StringUtils.isEmpty(dataModelDesc.getPartitionDesc().getPartitionDateColumn())) {
logger.error("No incremental cube, no need to extend.");
return;
}
String owner = cubeInstance.getOwner();
long partitionDate = partitionDateStr != null ? DateFormat.stringToMillis(partitionDateStr) : 0;
// get new name for old cube and cube_desc
String newCubeDescName = renameCube(cubeDesc.getName());
String newCubeInstanceName = renameCube(cubeInstance.getName());
while (cubeDescManager.getCubeDesc(newCubeDescName) != null)
newCubeDescName = renameCube(newCubeDescName);
while (cubeManager.getCube(newCubeInstanceName) != null)
newCubeInstanceName = renameCube(newCubeInstanceName);
// create new cube_instance for old segments
CubeInstance newCubeInstance = CubeInstance.getCopyOf(cubeInstance);
newCubeInstance.setName(newCubeInstanceName);
newCubeInstance.setDescName(newCubeDescName);
newCubeInstance.updateRandomUuid();
Iterator<CubeSegment> segmentIterator = newCubeInstance.getSegments().iterator();
CubeSegment currentSeg = null;
while (segmentIterator.hasNext()) {
currentSeg = segmentIterator.next();
if (partitionDateStr != null && (currentSeg.getTSRange().start.v >= partitionDate || currentSeg.getTSRange().end.v > partitionDate)) {
segmentIterator.remove();
logger.info("CubeSegment[" + currentSeg + "] was removed.");
}
}
if (currentSeg != null && partitionDateStr != null && partitionDate != currentSeg.getTSRange().end.v) {
logger.error("PartitionDate must be end date of one segment.");
return;
}
if (currentSeg != null && partitionDateStr == null)
partitionDate = currentSeg.getTSRange().end.v;
cubeManager.createCube(newCubeInstance, projectName, owner);
logger.info("CubeInstance was saved at: " + newCubeInstance.getResourcePath());
// create new cube for old segments
CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
newCubeDesc.setName(newCubeDescName);
newCubeDesc.updateRandomUuid();
newCubeDesc.init(kylinConfig);
newCubeDesc.setPartitionDateEnd(partitionDate);
newCubeDesc.calculateSignature();
cubeDescManager.createCubeDesc(newCubeDesc);
logger.info("CubeDesc was saved at: " + newCubeDesc.getResourcePath());
// update old cube_desc to new-version metadata
cubeDesc.setPartitionDateStart(partitionDate);
cubeDesc.setEngineType(IEngineAware.ID_MR_V2);
cubeDesc.setStorageType(IStorageAware.ID_SHARDED_HBASE);
cubeDesc.calculateSignature();
cubeDescManager.updateCubeDesc(cubeDesc);
logger.info("CubeDesc was saved at: " + cubeDesc.getResourcePath());
// clear segments for old cube
cubeInstance.setSegments(new Segments());
cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
store.checkAndPutResource(cubeInstance.getResourcePath(), cubeInstance, CubeManager.CUBE_SERIALIZER);
logger.info("CubeInstance was saved at: " + cubeInstance.getResourcePath());
// create hybrid model for these two cubes
List<RealizationEntry> realizationEntries = Lists.newArrayListWithCapacity(2);
realizationEntries.add(RealizationEntry.create(RealizationType.CUBE, cubeInstance.getName()));
realizationEntries.add(RealizationEntry.create(RealizationType.CUBE, newCubeInstance.getName()));
HybridInstance hybridInstance = HybridInstance.create(kylinConfig, renameHybrid(cubeInstance.getName()), realizationEntries);
store.checkAndPutResource(hybridInstance.getResourcePath(), hybridInstance, HybridManager.HYBRID_SERIALIZER);
ProjectManager.getInstance(kylinConfig).moveRealizationToProject(RealizationType.HYBRID, hybridInstance.getName(), projectName, owner);
logger.info("HybridInstance was saved at: " + hybridInstance.getResourcePath());
// copy Acl from old cube to new cube
copyAcl(cubeInstance.getId(), newCubeInstance.getId(), projectName);
logger.info("Acl copied from [" + cubeName + "] to [" + newCubeInstanceName + "].");
}
Example 15
| Source File: ExtendCubeToHybridCLI.java From kylin with Apache License 2.0 | 4 votes |
|
public void createFromCube(String projectName, String cubeName, String partitionDateStr) throws Exception {
logger.info("Create hybrid for cube[" + cubeName + "], project[" + projectName + "], partition_date[" + partitionDateStr + "].");
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
if (!validateCubeInstance(cubeInstance)) {
return;
}
CubeDesc cubeDesc = cubeDescManager.getCubeDesc(cubeInstance.getDescName());
DataModelDesc dataModelDesc = metadataManager.getDataModelDesc(cubeDesc.getModelName());
if (StringUtils.isEmpty(dataModelDesc.getPartitionDesc().getPartitionDateColumn())) {
logger.error("No incremental cube, no need to extend.");
return;
}
String owner = cubeInstance.getOwner();
long partitionDate = partitionDateStr != null ? DateFormat.stringToMillis(partitionDateStr) : 0;
// get new name for old cube and cube_desc
String newCubeDescName = renameCube(cubeDesc.getName());
String newCubeInstanceName = renameCube(cubeInstance.getName());
while (cubeDescManager.getCubeDesc(newCubeDescName) != null)
newCubeDescName = renameCube(newCubeDescName);
while (cubeManager.getCube(newCubeInstanceName) != null)
newCubeInstanceName = renameCube(newCubeInstanceName);
// create new cube_instance for old segments
CubeInstance newCubeInstance = CubeInstance.getCopyOf(cubeInstance);
newCubeInstance.setName(newCubeInstanceName);
newCubeInstance.setDescName(newCubeDescName);
newCubeInstance.updateRandomUuid();
Iterator<CubeSegment> segmentIterator = newCubeInstance.getSegments().iterator();
CubeSegment currentSeg = null;
while (segmentIterator.hasNext()) {
currentSeg = segmentIterator.next();
if (partitionDateStr != null && (currentSeg.getTSRange().start.v >= partitionDate || currentSeg.getTSRange().end.v > partitionDate)) {
segmentIterator.remove();
logger.info("CubeSegment[" + currentSeg + "] was removed.");
}
}
if (partitionDateStr != null && partitionDate != currentSeg.getTSRange().end.v) {
logger.error("PartitionDate must be end date of one segment.");
return;
}
if (currentSeg != null && partitionDateStr == null)
partitionDate = currentSeg.getTSRange().end.v;
cubeManager.createCube(newCubeInstance, projectName, owner);
logger.info("CubeInstance was saved at: " + newCubeInstance.getResourcePath());
// create new cube for old segments
CubeDesc newCubeDesc = CubeDesc.getCopyOf(cubeDesc);
newCubeDesc.setName(newCubeDescName);
newCubeDesc.updateRandomUuid();
newCubeDesc.init(kylinConfig);
newCubeDesc.setPartitionDateEnd(partitionDate);
newCubeDesc.calculateSignature();
cubeDescManager.createCubeDesc(newCubeDesc);
logger.info("CubeDesc was saved at: " + newCubeDesc.getResourcePath());
// update old cube_desc to new-version metadata
cubeDesc.setPartitionDateStart(partitionDate);
cubeDesc.setEngineType(IEngineAware.ID_MR_V2);
cubeDesc.setStorageType(IStorageAware.ID_SHARDED_HBASE);
cubeDesc.calculateSignature();
cubeDescManager.updateCubeDesc(cubeDesc);
logger.info("CubeDesc was saved at: " + cubeDesc.getResourcePath());
// clear segments for old cube
cubeInstance.setSegments(new Segments<CubeSegment>());
cubeInstance.setStatus(RealizationStatusEnum.DISABLED);
store.checkAndPutResource(cubeInstance.getResourcePath(), cubeInstance, CubeManager.CUBE_SERIALIZER);
logger.info("CubeInstance was saved at: " + cubeInstance.getResourcePath());
// create hybrid model for these two cubes
List<RealizationEntry> realizationEntries = Lists.newArrayListWithCapacity(2);
realizationEntries.add(RealizationEntry.create(RealizationType.CUBE, cubeInstance.getName()));
realizationEntries.add(RealizationEntry.create(RealizationType.CUBE, newCubeInstance.getName()));
HybridInstance hybridInstance = HybridInstance.create(kylinConfig, renameHybrid(cubeInstance.getName()), realizationEntries);
store.checkAndPutResource(hybridInstance.getResourcePath(), hybridInstance, HybridManager.HYBRID_SERIALIZER);
ProjectManager.getInstance(kylinConfig).moveRealizationToProject(RealizationType.HYBRID, hybridInstance.getName(), projectName, owner);
logger.info("HybridInstance was saved at: " + hybridInstance.getResourcePath());
// copy Acl from old cube to new cube
copyAcl(cubeInstance.getId(), newCubeInstance.getId(), projectName);
logger.info("Acl copied from [" + cubeName + "] to [" + newCubeInstanceName + "].");
}
Example 16
| Source File: CubeService.java From kylin with Apache License 2.0 | 4 votes |
|
/**
* Update a cube status from ready to disabled.
*
* @return
* @throws IOException
* @throws JobException
*/
public CubeInstance disableCube(CubeInstance cube) throws IOException {
aclEvaluate.checkProjectWritePermission(cube);
Message msg = MsgPicker.getMsg();
String cubeName = cube.getName();
RealizationStatusEnum ostatus = cube.getStatus();
if (null != ostatus && !RealizationStatusEnum.READY.equals(ostatus)) {
throw new BadRequestException(
String.format(Locale.ROOT, msg.getDISABLE_NOT_READY_CUBE(), cubeName, ostatus));
}
boolean isStreamingCube = cube.getDescriptor().isStreamingCube();
boolean cubeStatusUpdated = false;
try {
CubeInstance cubeInstance = getCubeManager().updateCubeStatus(cube, RealizationStatusEnum.DISABLED);
cubeStatusUpdated = true;
// for streaming cube.
if (isStreamingCube) {
//drop not ready segments
CubeSegment[] buildingSegments = new CubeSegment[cubeInstance.getBuildingSegments().size()];
Segments segments = cubeInstance.getBuildingSegments();
if (!CollectionUtils.isEmpty(segments)) {
for (int i = 0; i < segments.size(); i++) {
buildingSegments[i] = (CubeSegment) segments.get(i);
}
getCubeManager().dropOptmizingSegments(cubeInstance, buildingSegments);
}
//unAssign cube
getStreamingCoordinator().unAssignCube(cubeName);
//discard jobs
releaseAllJobs(cubeInstance);
}
return cubeInstance;
} catch (Exception e) {
cube.setStatus(ostatus);
// roll back if cube status updated
if (cubeStatusUpdated) {
logger.info("roll back cube status to:{}", ostatus);
getCubeManager().updateCubeStatus(cube, ostatus);
}
throw e;
}
}
