Java Code Examples for org.apache.kylin.cube.CubeSegment#parseSegmentName()
The following examples show how to use
org.apache.kylin.cube.CubeSegment#parseSegmentName() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: Coordinator.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
private boolean triggerSegmentBuild(String cubeName, String segmentName) {
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
try {
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
logger.info("submit streaming segment build, cube:{} segment:{}", cubeName, segmentName);
CubeSegment newSeg = getCubeManager().appendSegment(cubeInstance,
new TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
DefaultChainedExecutable executable = new StreamingCubingEngine().createStreamingCubingJob(newSeg,
"SYSTEM");
getExecutableManager().addJob(executable);
CubingJob cubingJob = (CubingJob) executable;
newSeg.setLastBuildJobID(cubingJob.getId());
SegmentJobBuildInfo segmentJobBuildInfo = new SegmentJobBuildInfo(cubeName, segmentName, cubingJob.getId());
jobStatusChecker.addSegmentBuildJob(segmentJobBuildInfo);
SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
state.setBuildStartTime(System.currentTimeMillis());
state.setState(SegmentBuildState.BuildState.State.BUILDING);
state.setJobId(cubingJob.getId());
streamMetadataStore.updateSegmentBuildState(cubeName, segmentName, state);
return true;
} catch (Exception e) {
logger.error("streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
return false;
}
}
Example 2
| Source File: StreamingCubeSegment.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
public static StreamingCubeSegment parseSegment(CubeInstance cubeInstance, File segmentFolder,
IStreamingSegmentStore segmentStore) {
Pair<Long, Long> segmentStartEnd = CubeSegment.parseSegmentName(segmentFolder.getName());
StreamingCubeSegment segment = new StreamingCubeSegment(cubeInstance, segmentStore, segmentStartEnd.getFirst(),
segmentStartEnd.getSecond());
State state = segmentStore.getSegmentState();
segment.saveState(state);
return segment;
}
Example 3
| Source File: StreamingSegmentManager.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
public void purgeSegment(String segmentName) {
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
StreamingCubeSegment segment = activeSegments.remove(segmentRange.getFirst());
if (segment == null) {
segment = immutableSegments.remove(segmentRange.getFirst());
}
segmentSourceStartPositions.remove(segmentRange.getFirst());
if (segment != null) {
segment.purge();
}
}
Example 4
| Source File: JobService.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
|
public void resubmitJob(JobInstance job) throws IOException {
aclEvaluate.checkProjectOperationPermission(job);
Coordinator coordinator = Coordinator.getInstance();
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
String cubeName = job.getRelatedCube();
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
String segmentName = job.getRelatedSegmentName();
try {
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
logger.info("submit streaming segment build, cube:{} segment:{}", cubeName, segmentName);
CubeSegment newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
DefaultChainedExecutable executable = new StreamingCubingEngine().createStreamingCubingJob(newSeg, aclEvaluate.getCurrentUserName());
coordinator.getExecutableManager().addJob(executable);
CubingJob cubingJob = (CubingJob) executable;
newSeg.setLastBuildJobID(cubingJob.getId());
SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
state.setBuildStartTime(System.currentTimeMillis());
state.setState(SegmentBuildState.BuildState.State.BUILDING);
state.setJobId(cubingJob.getId());
coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
} catch (Exception e) {
logger.error("streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
throw e;
}
}
Example 5
| Source File: Coordinator.java From kylin with Apache License 2.0 | 5 votes |
|
private boolean triggerSegmentBuild(String cubeName, String segmentName) {
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
try {
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
logger.info("submit streaming segment build, cube:{} segment:{}", cubeName, segmentName);
CubeSegment newSeg = getCubeManager().appendSegment(cubeInstance,
new TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
DefaultChainedExecutable executable = new StreamingCubingEngine().createStreamingCubingJob(newSeg,
"SYSTEM");
getExecutableManager().addJob(executable);
CubingJob cubingJob = (CubingJob) executable;
newSeg.setLastBuildJobID(cubingJob.getId());
SegmentJobBuildInfo segmentJobBuildInfo = new SegmentJobBuildInfo(cubeName, segmentName, cubingJob.getId());
jobStatusChecker.addSegmentBuildJob(segmentJobBuildInfo);
SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
state.setBuildStartTime(System.currentTimeMillis());
state.setState(SegmentBuildState.BuildState.State.BUILDING);
state.setJobId(cubingJob.getId());
streamMetadataStore.updateSegmentBuildState(cubeName, segmentName, state);
return true;
} catch (Exception e) {
logger.error("streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
return false;
}
}
Example 6
| Source File: StreamingCubeSegment.java From kylin with Apache License 2.0 | 5 votes |
|
public static StreamingCubeSegment parseSegment(CubeInstance cubeInstance, File segmentFolder,
IStreamingSegmentStore segmentStore) {
Pair<Long, Long> segmentStartEnd = CubeSegment.parseSegmentName(segmentFolder.getName());
StreamingCubeSegment segment = new StreamingCubeSegment(cubeInstance, segmentStore, segmentStartEnd.getFirst(),
segmentStartEnd.getSecond());
State state = segmentStore.getSegmentState();
segment.saveState(state);
return segment;
}
Example 7
| Source File: StreamingSegmentManager.java From kylin with Apache License 2.0 | 5 votes |
|
public void purgeSegment(String segmentName) {
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
StreamingCubeSegment segment = activeSegments.remove(segmentRange.getFirst());
if (segment == null) {
segment = immutableSegments.remove(segmentRange.getFirst());
}
segmentSourceStartPositions.remove(segmentRange.getFirst());
if (segment != null) {
segment.purge();
}
}
Example 8
| Source File: JobService.java From kylin with Apache License 2.0 | 5 votes |
|
public void resubmitJob(JobInstance job) throws IOException {
aclEvaluate.checkProjectOperationPermission(job);
Coordinator coordinator = Coordinator.getInstance();
CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
String cubeName = job.getRelatedCube();
CubeInstance cubeInstance = cubeManager.getCube(cubeName);
String segmentName = job.getRelatedSegmentName();
try {
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
logger.info("submit streaming segment build, cube:{} segment:{}", cubeName, segmentName);
CubeSegment newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
DefaultChainedExecutable executable = new StreamingCubingEngine().createStreamingCubingJob(newSeg, aclEvaluate.getCurrentUserName());
coordinator.getExecutableManager().addJob(executable);
CubingJob cubingJob = (CubingJob) executable;
newSeg.setLastBuildJobID(cubingJob.getId());
SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
state.setBuildStartTime(System.currentTimeMillis());
state.setState(SegmentBuildState.BuildState.State.BUILDING);
state.setJobId(cubingJob.getId());
coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
} catch (Exception e) {
logger.error("streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
throw e;
}
}
Example 9
| Source File: Coordinator.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
private List<String> findSegmentsCanBuild(String cubeName) {
List<String> result = Lists.newArrayList();
CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
// in optimization
if (isInOptimize(cubeInstance)) {
return result;
}
int allowMaxBuildingSegments = cubeInstance.getConfig().getMaxBuildingSegments();
CubeSegment latestHistoryReadySegment = cubeInstance.getLatestReadySegment();
long minSegmentStart = -1;
if (latestHistoryReadySegment != null) {
minSegmentStart = latestHistoryReadySegment.getTSRange().end.v;
} else {
// there is no ready segment, to make cube planner work, only 1 segment can build
logger.info("there is no ready segments for cube:{}, so only allow 1 segment build concurrently", cubeName);
allowMaxBuildingSegments = 1;
}
CubeAssignment assignments = streamMetadataStore.getAssignmentsByCube(cubeName);
Set<Integer> cubeAssignedReplicaSets = assignments.getReplicaSetIDs();
List<SegmentBuildState> segmentStates = streamMetadataStore.getSegmentBuildStates(cubeName);
Collections.sort(segmentStates);
// TODO need to check whether it is in optimization
int inBuildingSegments = cubeInstance.getBuildingSegments().size();
int leftQuota = allowMaxBuildingSegments - inBuildingSegments;
for (int i = 0; i < segmentStates.size(); i++) {
SegmentBuildState segmentState = segmentStates.get(i);
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentState.getSegmentName());
if (segmentRange.getFirst() < minSegmentStart) {
logger.warn("the cube segment state is not clear correctly, cube:{} segment:{}, clear it", cubeName,
segmentState.getSegmentName());
streamMetadataStore.removeSegmentBuildState(cubeName, segmentState.getSegmentName());
continue;
}
if (segmentState.isInBuilding()) {
inBuildingSegments++;
String jobId = segmentState.getState().getJobId();
logger.info("there is segment in building, cube:{} segment:{} jobId:{}", cubeName,
segmentState.getSegmentName(), jobId);
long buildStartTime = segmentState.getState().getBuildStartTime();
if (buildStartTime != 0 && jobId != null) {
long buildDuration = System.currentTimeMillis() - buildStartTime;
if (buildDuration < 40 * 60 * 1000) { // if build time larger than 40 minutes, check the job status
continue;
}
CubingJob cubingJob = (CubingJob) getExecutableManager().getJob(jobId);
ExecutableState jobState = cubingJob.getStatus();
if (ExecutableState.SUCCEED.equals(jobState)) { // job is already succeed, remove the build state
CubeSegment cubeSegment = cubeInstance.getSegment(segmentState.getSegmentName(), null);
if (cubeSegment != null && SegmentStatusEnum.READY == cubeSegment.getStatus()) {
logger.info(
"job:{} is already succeed, and segment:{} is ready, remove segment build state",
jobId, segmentState.getSegmentName());
streamMetadataStore.removeSegmentBuildState(cubeName, segmentState.getSegmentName());
}
continue;
} else if (ExecutableState.ERROR.equals(jobState)) {
logger.info("job:{} is error, resume the job", jobId);
getExecutableManager().resumeJob(jobId);
continue;
} else if (ExecutableState.DISCARDED.equals(jobState)) {
// if the job has been discard manually, just think that the segment is not in building
logger.info("job:{} is discard, reset the job state in metaStore", jobId);
SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
state.setBuildStartTime(0);
state.setState(SegmentBuildState.BuildState.State.WAIT);
state.setJobId(cubingJob.getId());
streamMetadataStore.updateSegmentBuildState(cubeName, segmentState.getSegmentName(), state);
segmentState.setState(state);
logger.info("segment:{} is discard", segmentState.getSegmentName());
continue;
} else {
logger.info("job:{} is in running, job state: {}", jobId, jobState);
continue;
}
}
}
if (leftQuota <= 0) {
logger.info("No left quota to build segments for cube:{}", cubeName);
return result;
}
if (!checkSegmentIsReadyToBuild(segmentStates, i, cubeAssignedReplicaSets)) {
break;
}
result.add(segmentState.getSegmentName());
leftQuota--;
}
return result;
}
Example 10
| Source File: BuildJobSubmitter.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
/**
* @return list of segment which could be submitted a segment build job
*/
@NonSideEffect
List<String> checkSegmentBuildJobFromMetadata(String cubeName) {
List<String> result = Lists.newArrayList();
CubeInstance cubeInstance = coordinator.getCubeManager().getCube(cubeName);
// in optimization
if (isInOptimize(cubeInstance)) {
return result;
}
int allowMaxBuildingSegments = cubeInstance.getConfig().getMaxBuildingSegments();
CubeSegment latestHistoryReadySegment = cubeInstance.getLatestReadySegment();
long minSegmentStart = -1;
if (latestHistoryReadySegment != null) {
minSegmentStart = latestHistoryReadySegment.getTSRange().end.v;
} else {
// there is no ready segment, to make cube planner work, only 1 segment can build
logger.info("there is no ready segments for cube:{}, so only allow 1 segment build concurrently", cubeName);
allowMaxBuildingSegments = 1;
}
CubeAssignment assignments = coordinator.getStreamMetadataStore().getAssignmentsByCube(cubeName);
Set<Integer> cubeAssignedReplicaSets = assignments.getReplicaSetIDs();
List<SegmentBuildState> segmentStates = coordinator.getStreamMetadataStore().getSegmentBuildStates(cubeName);
int inBuildingSegments = cubeInstance.getBuildingSegments().size();
int leftQuota = allowMaxBuildingSegments - inBuildingSegments;
boolean stillQuotaForNewSegment = true;
// Sort it so we can iterate segments from eariler one to newer one
Collections.sort(segmentStates);
for (int i = 0; i < segmentStates.size(); i++) {
boolean needRebuild = false;
if (leftQuota <= 0) {
logger.info("No left quota to build segments for cube:{} at {}", cubeName, leftQuota);
stillQuotaForNewSegment = false;
}
SegmentBuildState segmentState = segmentStates.get(i);
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentState.getSegmentName());
// If we have a exist historcial segment, we should not let new realtime segment overwrite it, it is so dangrous,
// we just delete the entry to ignore the segment which should not exist
if (segmentRange.getFirst() < minSegmentStart) {
logger.warn(
"The cube segment state is not correct because it belongs to historcial part, cube:{} segment:{}, clear it.",
cubeName, segmentState.getSegmentName());
coordinator.getStreamMetadataStore().removeSegmentBuildState(cubeName, segmentState.getSegmentName());
continue;
}
// We already have a building job for current segment
if (segmentState.isInBuilding()) {
needRebuild = checkSegmentBuildingJob(segmentState, cubeName, cubeInstance);
if (!needRebuild)
continue;
} else if (segmentState.isInWaiting()) {
// The data maybe uploaded to remote completely, or job is discard
// These two case should be submit a building job, just let go through it
}
boolean readyToBuild = checkSegmentIsReadyToBuild(segmentStates, i, cubeAssignedReplicaSets);
if (!readyToBuild) {
logger.debug("Segment {} {} is not ready to submit a building job.", cubeName, segmentState);
} else if (stillQuotaForNewSegment || needRebuild) {
result.add(segmentState.getSegmentName());
leftQuota--;
}
}
if (logger.isDebugEnabled() && !result.isEmpty()) {
logger.debug("{} Candidate segment list to be built : {}.", cubeName, String.join(", ", result));
}
return result;
}
Example 11
| Source File: BuildJobSubmitter.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
/**
* Submit a build job for streaming segment
*
* @return true if submit succeed ; else false
*/
@NotAtomicIdempotent
boolean submitSegmentBuildJob(String cubeName, String segmentName) {
logger.info("Try submit streaming segment build job, cube:{} segment:{}", cubeName, segmentName);
CubeInstance cubeInstance = coordinator.getCubeManager().getCube(cubeName);
try {
// Step 1. create a new segment if not exists
CubeSegment newSeg = null;
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
boolean segmentExists = false;
for (CubeSegment segment : cubeInstance.getSegments()) {
SegmentRange.TSRange tsRange = segment.getTSRange();
if (tsRange.start.v.equals(segmentRange.getFirst()) && segmentRange.getSecond().equals(tsRange.end.v)) {
segmentExists = true;
newSeg = segment;
}
}
if (segmentExists) {
logger.warn("Segment {} exists, it will be forced deleted.", segmentName);
coordinator.getCubeManager().updateCubeDropSegments(cubeInstance, newSeg);
}
logger.debug("Create segment for {} {} .", cubeName, segmentName);
newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
// Step 2. create and submit new build job
DefaultChainedExecutable executable = getStreamingCubingJob(newSeg);
coordinator.getExecutableManager().addJob(executable);
String jobId = executable.getId();
newSeg.setLastBuildJobID(jobId);
// Step 3. add it to job trigger list
SegmentJobBuildInfo segmentJobBuildInfo = new SegmentJobBuildInfo(cubeName, segmentName, jobId);
addToJobTrackList(segmentJobBuildInfo);
// Step 4. add job to stream metadata in case of current node dead
SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
state.setBuildStartTime(System.currentTimeMillis());
state.setState(SegmentBuildState.BuildState.State.BUILDING);
state.setJobId(jobId);
logger.debug("Commit building job {} for {} {} .", jobId, cubeName, segmentName);
coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
return true;
} catch (Exception e) {
logger.error("Streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
return false;
}
}
Example 12
| Source File: Coordinator.java From kylin with Apache License 2.0 | 4 votes |
|
private List<String> findSegmentsCanBuild(String cubeName) {
List<String> result = Lists.newArrayList();
CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
// in optimization
if (isInOptimize(cubeInstance)) {
return result;
}
int allowMaxBuildingSegments = cubeInstance.getConfig().getMaxBuildingSegments();
CubeSegment latestHistoryReadySegment = cubeInstance.getLatestReadySegment();
long minSegmentStart = -1;
if (latestHistoryReadySegment != null) {
minSegmentStart = latestHistoryReadySegment.getTSRange().end.v;
} else {
// there is no ready segment, to make cube planner work, only 1 segment can build
logger.info("there is no ready segments for cube:{}, so only allow 1 segment build concurrently", cubeName);
allowMaxBuildingSegments = 1;
}
CubeAssignment assignments = streamMetadataStore.getAssignmentsByCube(cubeName);
Set<Integer> cubeAssignedReplicaSets = assignments.getReplicaSetIDs();
List<SegmentBuildState> segmentStates = streamMetadataStore.getSegmentBuildStates(cubeName);
Collections.sort(segmentStates);
// TODO need to check whether it is in optimization
int inBuildingSegments = cubeInstance.getBuildingSegments().size();
int leftQuota = allowMaxBuildingSegments - inBuildingSegments;
for (int i = 0; i < segmentStates.size(); i++) {
SegmentBuildState segmentState = segmentStates.get(i);
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentState.getSegmentName());
if (segmentRange.getFirst() < minSegmentStart) {
logger.warn("the cube segment state is not clear correctly, cube:{} segment:{}, clear it", cubeName,
segmentState.getSegmentName());
streamMetadataStore.removeSegmentBuildState(cubeName, segmentState.getSegmentName());
continue;
}
if (segmentState.isInBuilding()) {
inBuildingSegments++;
String jobId = segmentState.getState().getJobId();
logger.info("there is segment in building, cube:{} segment:{} jobId:{}", cubeName,
segmentState.getSegmentName(), jobId);
long buildStartTime = segmentState.getState().getBuildStartTime();
if (buildStartTime != 0 && jobId != null) {
long buildDuration = System.currentTimeMillis() - buildStartTime;
if (buildDuration < 40 * 60 * 1000) { // if build time larger than 40 minutes, check the job status
continue;
}
CubingJob cubingJob = (CubingJob) getExecutableManager().getJob(jobId);
ExecutableState jobState = cubingJob.getStatus();
if (ExecutableState.SUCCEED.equals(jobState)) { // job is already succeed, remove the build state
CubeSegment cubeSegment = cubeInstance.getSegment(segmentState.getSegmentName(), null);
if (cubeSegment != null && SegmentStatusEnum.READY == cubeSegment.getStatus()) {
logger.info(
"job:{} is already succeed, and segment:{} is ready, remove segment build state",
jobId, segmentState.getSegmentName());
streamMetadataStore.removeSegmentBuildState(cubeName, segmentState.getSegmentName());
}
continue;
} else if (ExecutableState.ERROR.equals(jobState)) {
logger.info("job:{} is error, resume the job", jobId);
getExecutableManager().resumeJob(jobId);
continue;
} else if (ExecutableState.DISCARDED.equals(jobState)) {
// if the job has been discard manually, just think that the segment is not in building
logger.info("job:{} is discard, reset the job state in metaStore", jobId);
SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
state.setBuildStartTime(0);
state.setState(SegmentBuildState.BuildState.State.WAIT);
state.setJobId(cubingJob.getId());
streamMetadataStore.updateSegmentBuildState(cubeName, segmentState.getSegmentName(), state);
segmentState.setState(state);
logger.info("segment:{} is discard", segmentState.getSegmentName());
continue;
} else {
logger.info("job:{} is in running, job state: {}", jobId, jobState);
continue;
}
}
}
if (leftQuota <= 0) {
logger.info("No left quota to build segments for cube:{}", cubeName);
return result;
}
if (!checkSegmentIsReadyToBuild(segmentStates, i, cubeAssignedReplicaSets)) {
break;
}
result.add(segmentState.getSegmentName());
leftQuota--;
}
return result;
}
Example 13
| Source File: BuildJobSubmitter.java From kylin with Apache License 2.0 | 4 votes |
|
/**
* @return list of segment which could be submitted a segment build job
*/
@NonSideEffect
List<String> checkSegmentBuildJobFromMetadata(String cubeName) {
List<String> result = Lists.newArrayList();
CubeInstance cubeInstance = coordinator.getCubeManager().getCube(cubeName);
// in optimization
if (isInOptimize(cubeInstance)) {
return result;
}
int allowMaxBuildingSegments = cubeInstance.getConfig().getMaxBuildingSegments();
CubeSegment latestHistoryReadySegment = cubeInstance.getLatestReadySegment();
long minSegmentStart = -1;
if (latestHistoryReadySegment != null) {
minSegmentStart = latestHistoryReadySegment.getTSRange().end.v;
} else {
// there is no ready segment, to make cube planner work, only 1 segment can build
logger.info("there is no ready segments for cube:{}, so only allow 1 segment build concurrently", cubeName);
allowMaxBuildingSegments = 1;
}
CubeAssignment assignments = coordinator.getStreamMetadataStore().getAssignmentsByCube(cubeName);
Set<Integer> cubeAssignedReplicaSets = assignments.getReplicaSetIDs();
List<SegmentBuildState> segmentStates = coordinator.getStreamMetadataStore().getSegmentBuildStates(cubeName);
int inBuildingSegments = cubeInstance.getBuildingSegments().size();
int leftQuota = allowMaxBuildingSegments - inBuildingSegments;
boolean stillQuotaForNewSegment = true;
// Sort it so we can iterate segments from eariler one to newer one
Collections.sort(segmentStates);
for (int i = 0; i < segmentStates.size(); i++) {
boolean needRebuild = false;
if (leftQuota <= 0) {
logger.info("No left quota to build segments for cube:{} at {}", cubeName, leftQuota);
stillQuotaForNewSegment = false;
}
SegmentBuildState segmentState = segmentStates.get(i);
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentState.getSegmentName());
// If we have a exist historcial segment, we should not let new realtime segment overwrite it, it is so dangrous,
// we just delete the entry to ignore the segment which should not exist
if (segmentRange.getFirst() < minSegmentStart) {
logger.warn(
"The cube segment state is not correct because it belongs to historcial part, cube:{} segment:{}, clear it.",
cubeName, segmentState.getSegmentName());
coordinator.getStreamMetadataStore().removeSegmentBuildState(cubeName, segmentState.getSegmentName());
continue;
}
// We already have a building job for current segment
if (segmentState.isInBuilding()) {
needRebuild = checkSegmentBuildingJob(segmentState, cubeName, cubeInstance);
if (!needRebuild)
continue;
} else if (segmentState.isInWaiting()) {
// The data maybe uploaded to remote completely, or job is discard
// These two case should be submit a building job, just let go through it
}
boolean readyToBuild = checkSegmentIsReadyToBuild(segmentStates, i, cubeAssignedReplicaSets);
if (!readyToBuild) {
logger.debug("Segment {} {} is not ready to submit a building job.", cubeName, segmentState);
} else if (stillQuotaForNewSegment || needRebuild) {
result.add(segmentState.getSegmentName());
leftQuota--;
}
}
if (logger.isDebugEnabled() && !result.isEmpty()) {
logger.debug("{} Candidate segment list to be built : {}.", cubeName, String.join(", ", result));
}
return result;
}
Example 14
| Source File: BuildJobSubmitter.java From kylin with Apache License 2.0 | 4 votes |
|
/**
* Submit a build job for streaming segment
*
* @return true if submit succeed ; else false
*/
@NotAtomicIdempotent
boolean submitSegmentBuildJob(String cubeName, String segmentName) {
logger.info("Try submit streaming segment build job, cube:{} segment:{}", cubeName, segmentName);
CubeInstance cubeInstance = coordinator.getCubeManager().getCube(cubeName);
try {
// Step 1. create a new segment if not exists
CubeSegment newSeg = null;
Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
boolean segmentExists = false;
for (CubeSegment segment : cubeInstance.getSegments()) {
SegmentRange.TSRange tsRange = segment.getTSRange();
if (tsRange.start.v.equals(segmentRange.getFirst()) && segmentRange.getSecond().equals(tsRange.end.v)) {
segmentExists = true;
newSeg = segment;
}
}
if (segmentExists) {
logger.warn("Segment {} exists, it will be forced deleted.", segmentName);
coordinator.getCubeManager().updateCubeDropSegments(cubeInstance, newSeg);
}
logger.debug("Create segment for {} {} .", cubeName, segmentName);
newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
// Step 2. create and submit new build job
DefaultChainedExecutable executable = getStreamingCubingJob(newSeg);
coordinator.getExecutableManager().addJob(executable);
String jobId = executable.getId();
newSeg.setLastBuildJobID(jobId);
// Step 3. add it to job trigger list
SegmentJobBuildInfo segmentJobBuildInfo = new SegmentJobBuildInfo(cubeName, segmentName, jobId);
addToJobTrackList(segmentJobBuildInfo);
// Step 4. add job to stream metadata in case of current node dead
SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
state.setBuildStartTime(System.currentTimeMillis());
state.setState(SegmentBuildState.BuildState.State.BUILDING);
state.setJobId(jobId);
logger.debug("Commit building job {} for {} {} .", jobId, cubeName, segmentName);
coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
return true;
} catch (Exception e) {
logger.error("Streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
return false;
}
}
