Java Code Examples for org.apache.kylin.rest.response.SQLResponse#getCubeSegmentStatisticsList()
The following examples show how to use
org.apache.kylin.rest.response.SQLResponse#getCubeSegmentStatisticsList() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: QueryMetricsFacade.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
|
/**
* report query related metrics
*/
private static void updateMetricsToReservoir(SQLRequest sqlRequest, SQLResponse sqlResponse) {
if (!KylinConfig.getInstanceFromEnv().isKylinMetricsReporterForQueryEnabled()) {
return;
}
String user = SecurityContextHolder.getContext().getAuthentication().getName();
if (user == null) {
user = "unknown";
}
for (QueryContext.RPCStatistics entry : QueryContextFacade.current().getRpcStatisticsList()) {
RecordEvent rpcMetricsEvent = new TimedRecordEvent(
KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQueryRpcCall());
setRPCWrapper(rpcMetricsEvent, //
norm(sqlRequest.getProject()), entry.getRealizationName(), entry.getRpcServer(),
entry.getException());
setRPCStats(rpcMetricsEvent, //
entry.getCallTimeMs(), entry.getSkippedRows(), entry.getScannedRows(), entry.getReturnedRows(),
entry.getAggregatedRows());
//For update rpc level related metrics
MetricsManager.getInstance().update(rpcMetricsEvent);
}
for (QueryContext.CubeSegmentStatisticsResult contextEntry : sqlResponse.getCubeSegmentStatisticsList()) {
RecordEvent queryMetricsEvent = new TimedRecordEvent(
KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQuery());
setQueryWrapper(queryMetricsEvent, //
user, sqlRequest.getSql(), sqlResponse.isStorageCacheUsed() ? "CACHE" : contextEntry.getQueryType(),
norm(sqlRequest.getProject()), contextEntry.getRealization(), contextEntry.getRealizationType(),
sqlResponse.getThrowable());
long totalStorageReturnCount = 0L;
if (contextEntry.getQueryType().equalsIgnoreCase(OLAPQuery.EnumeratorTypeEnum.OLAP.name())) {
for (Map<String, QueryContext.CubeSegmentStatistics> cubeEntry : contextEntry.getCubeSegmentStatisticsMap()
.values()) {
for (QueryContext.CubeSegmentStatistics segmentEntry : cubeEntry.values()) {
RecordEvent cubeSegmentMetricsEvent = new TimedRecordEvent(
KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQueryCube());
setCubeWrapper(cubeSegmentMetricsEvent, //
norm(sqlRequest.getProject()), segmentEntry.getCubeName(), segmentEntry.getSegmentName(),
segmentEntry.getSourceCuboidId(), segmentEntry.getTargetCuboidId(),
segmentEntry.getFilterMask());
setCubeStats(cubeSegmentMetricsEvent, //
segmentEntry.getCallCount(), segmentEntry.getCallTimeSum(), segmentEntry.getCallTimeMax(),
segmentEntry.getStorageSkippedRows(), segmentEntry.getStorageScannedRows(),
segmentEntry.getStorageReturnedRows(), segmentEntry.getStorageAggregatedRows(),
segmentEntry.isIfSuccess(), 1.0 / cubeEntry.size());
totalStorageReturnCount += segmentEntry.getStorageReturnedRows();
//For update cube segment level related query metrics
MetricsManager.getInstance().update(cubeSegmentMetricsEvent);
}
}
} else {
if (!sqlResponse.getIsException()) {
totalStorageReturnCount = sqlResponse.getResults().size();
}
}
setQueryStats(queryMetricsEvent, //
sqlResponse.getDuration(), sqlResponse.getResults() == null ? 0 : sqlResponse.getResults().size(),
totalStorageReturnCount);
//For update query level metrics
MetricsManager.getInstance().update(queryMetricsEvent);
}
}
Example 2
| Source File: QueryMetricsFacade.java From kylin with Apache License 2.0 | 4 votes |
|
/**
* report query related metrics
*/
private static void updateMetricsToReservoir(SQLRequest sqlRequest, SQLResponse sqlResponse) {
if (!KylinConfig.getInstanceFromEnv().isKylinMetricsReporterForQueryEnabled()) {
return;
}
String user = SecurityContextHolder.getContext().getAuthentication().getName();
if (user == null) {
user = "unknown";
}
for (QueryContext.RPCStatistics entry : QueryContextFacade.current().getRpcStatisticsList()) {
RecordEvent rpcMetricsEvent = new TimedRecordEvent(
KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQueryRpcCall());
setRPCWrapper(rpcMetricsEvent, //
norm(sqlRequest.getProject()), entry.getRealizationName(), entry.getRpcServer(),
entry.getException());
setRPCStats(rpcMetricsEvent, //
entry.getCallTimeMs(), entry.getSkippedRows(), entry.getScannedRows(), entry.getReturnedRows(),
entry.getAggregatedRows());
//For update rpc level related metrics
MetricsManager.getInstance().update(rpcMetricsEvent);
}
for (QueryContext.CubeSegmentStatisticsResult contextEntry : sqlResponse.getCubeSegmentStatisticsList()) {
RecordEvent queryMetricsEvent = new TimedRecordEvent(
KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQuery());
setQueryWrapper(queryMetricsEvent, //
user, sqlRequest.getSql(), sqlResponse.isStorageCacheUsed() ? "CACHE" : contextEntry.getQueryType(),
norm(sqlRequest.getProject()), contextEntry.getRealization(), contextEntry.getRealizationType(),
sqlResponse.getThrowable());
long totalStorageReturnCount = 0L;
if (contextEntry.getQueryType().equalsIgnoreCase(OLAPQuery.EnumeratorTypeEnum.OLAP.name())) {
for (Map<String, QueryContext.CubeSegmentStatistics> cubeEntry : contextEntry.getCubeSegmentStatisticsMap()
.values()) {
for (QueryContext.CubeSegmentStatistics segmentEntry : cubeEntry.values()) {
RecordEvent cubeSegmentMetricsEvent = new TimedRecordEvent(
KylinConfig.getInstanceFromEnv().getKylinMetricsSubjectQueryCube());
setCubeWrapper(cubeSegmentMetricsEvent, //
norm(sqlRequest.getProject()), segmentEntry.getCubeName(), segmentEntry.getSegmentName(),
segmentEntry.getSourceCuboidId(), segmentEntry.getTargetCuboidId(),
segmentEntry.getFilterMask());
setCubeStats(cubeSegmentMetricsEvent, //
segmentEntry.getCallCount(), segmentEntry.getCallTimeSum(), segmentEntry.getCallTimeMax(),
segmentEntry.getStorageSkippedRows(), segmentEntry.getStorageScannedRows(),
segmentEntry.getStorageReturnedRows(), segmentEntry.getStorageAggregatedRows(),
segmentEntry.isIfSuccess(), 1.0 / cubeEntry.size());
totalStorageReturnCount += segmentEntry.getStorageReturnedRows();
//For update cube segment level related query metrics
MetricsManager.getInstance().update(cubeSegmentMetricsEvent);
}
}
} else {
if (!sqlResponse.getIsException()) {
totalStorageReturnCount = sqlResponse.getResults().size();
}
}
setQueryStats(queryMetricsEvent, //
sqlResponse.getDuration(), sqlResponse.getResults() == null ? 0 : sqlResponse.getResults().size(),
totalStorageReturnCount);
//For update query level metrics
MetricsManager.getInstance().update(queryMetricsEvent);
}
}
Example 3
| Source File: QueryService.java From kylin with Apache License 2.0 | 4 votes |
|
public void logQuery(final String queryId, final SQLRequest request, final SQLResponse response) {
final String user = aclEvaluate.getCurrentUserName();
final List<String> realizationNames = new LinkedList<>();
final Set<Long> cuboidIds = new HashSet<Long>();
float duration = response.getDuration() / (float) 1000;
boolean storageCacheUsed = response.isStorageCacheUsed();
boolean isPushDown = response.isPushDown();
if (!response.isHitExceptionCache() && null != OLAPContext.getThreadLocalContexts()) {
for (OLAPContext ctx : OLAPContext.getThreadLocalContexts()) {
Cuboid cuboid = ctx.storageContext.getCuboid();
if (cuboid != null) {
//Some queries do not involve cuboid, e.g. lookup table query
cuboidIds.add(cuboid.getId());
}
if (ctx.realization != null) {
realizationNames.add(ctx.realization.getCanonicalName());
}
}
}
// if Realization Names is empty, get value from SQLResponse.
if (realizationNames.isEmpty()) {
if (!Strings.isNullOrEmpty(response.getCube())) {
realizationNames.addAll(Lists.newArrayList(StringUtil.splitByComma(response.getCube())));
}
}
// if Cuboid Ids is empty, get value from SQLResponse.
if (cuboidIds.isEmpty()) {
List<QueryContext.CubeSegmentStatisticsResult> cubeSegmentStatisticsList =
response.getCubeSegmentStatisticsList();
if (CollectionUtils.isNotEmpty(cubeSegmentStatisticsList)) {
cubeSegmentStatisticsList.forEach(cubeSegmentStatResult -> {
if (MapUtils.isNotEmpty(cubeSegmentStatResult.getCubeSegmentStatisticsMap())) {
cubeSegmentStatResult.getCubeSegmentStatisticsMap().values().forEach(cubeSegmentStatMap -> {
cubeSegmentStatMap.values().forEach(cubeSegmentStat -> {
cuboidIds.add(cubeSegmentStat.getTargetCuboidId());
});
});
}
});
}
}
int resultRowCount = 0;
if (!response.getIsException() && response.getResults() != null) {
resultRowCount = response.getResults().size();
}
String newLine = System.getProperty("line.separator");
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(newLine);
stringBuilder.append("==========================[QUERY]===============================").append(newLine);
stringBuilder.append("Query Id: ").append(queryId).append(newLine);
stringBuilder.append("SQL: ").append(request.getSql()).append(newLine);
stringBuilder.append("User: ").append(user).append(newLine);
stringBuilder.append("Success: ").append((null == response.getExceptionMessage())).append(newLine);
stringBuilder.append("Duration: ").append(duration).append(newLine);
stringBuilder.append("Project: ").append(request.getProject()).append(newLine);
stringBuilder.append("Realization Names: ").append(realizationNames).append(newLine);
stringBuilder.append("Cuboid Ids: ").append(cuboidIds).append(newLine);
stringBuilder.append("Total scan count: ").append(response.getTotalScanCount()).append(newLine);
stringBuilder.append("Total scan bytes: ").append(response.getTotalScanBytes()).append(newLine);
stringBuilder.append("Result row count: ").append(resultRowCount).append(newLine);
stringBuilder.append("Accept Partial: ").append(request.isAcceptPartial()).append(newLine);
stringBuilder.append("Is Partial Result: ").append(response.isPartial()).append(newLine);
stringBuilder.append("Hit Exception Cache: ").append(response.isHitExceptionCache()).append(newLine);
stringBuilder.append("Storage cache used: ").append(storageCacheUsed).append(newLine);
stringBuilder.append("Is Query Push-Down: ").append(isPushDown).append(newLine);
stringBuilder.append("Is Prepare: ").append(BackdoorToggles.getPrepareOnly()).append(newLine);
stringBuilder.append("Trace URL: ").append(response.getTraceUrl()).append(newLine);
stringBuilder.append("Message: ").append(response.getExceptionMessage()).append(newLine);
stringBuilder.append("==========================[QUERY]===============================").append(newLine);
logger.info(stringBuilder.toString());
}
