Java Code Examples for org.apache.kylin.common.util.Pair#newPair()

public Pair<T, T> mapRange(String col, T beginValue, T endValue, String returnCol) {
    int colIdx = tableDesc.findColumnByName(col).getZeroBasedIndex();
    int returnIdx = tableDesc.findColumnByName(returnCol).getZeroBasedIndex();
    Comparator<T> colComp = getComparator(colIdx);
    Comparator<T> returnComp = getComparator(returnIdx);

    T returnBegin = null;
    T returnEnd = null;
    for (T[] row : data.values()) {
        if (between(beginValue, row[colIdx], endValue, colComp)) {
            T returnValue = row[returnIdx];
            if (returnBegin == null || returnComp.compare(returnValue, returnBegin) < 0) {
                returnBegin = returnValue;
            }
            if (returnEnd == null || returnComp.compare(returnValue, returnEnd) > 0) {
                returnEnd = returnValue;
            }
        }
    }
    if (returnBegin == null && returnEnd == null)
        return null;
    else
        return Pair.newPair(returnBegin, returnEnd);
}
 

public static final Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(final long baseCuboidId, final long childCuboidId, final int measureCount) {
    final Pair<ImmutableBitSet, ImmutableBitSet> parentDimensionAndMetricColumnBitSet = getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount);
    ImmutableBitSet parentDimensions = parentDimensionAndMetricColumnBitSet.getFirst();
    ImmutableBitSet measureColumns = parentDimensionAndMetricColumnBitSet.getSecond();
    ImmutableBitSet childDimensions = parentDimensions;
    long mask = Long.highestOneBit(baseCuboidId);
    long parentCuboidIdActualLength = (long)Long.SIZE - Long.numberOfLeadingZeros(baseCuboidId);
    int index = 0;
    for (int i = 0; i < parentCuboidIdActualLength; i++) {
        if ((mask & baseCuboidId) > 0) {
            if ((mask & childCuboidId) == 0) {
                // this dim will be aggregated
                childDimensions = childDimensions.set(index, false);
            }
            index++;
        }
        mask = mask >> 1;
    }
    return Pair.newPair(childDimensions, measureColumns);
}
 

/**
 * Re-encode with measures in Object[] format.
 * @param key
 * @param value
 * @return
 * @throws IOException
 */
public Pair<Text, Object[]> reEncode2(Text key, Text value) throws IOException {
    if (initialized == false) {
        throw new IllegalStateException("Not initialized");
    }

    Object[] measureObjs = new Object[measureDescs.size()];
    codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), measureObjs);
    // re-encode measures if dictionary is used
    if (dictMeasures.size() > 0) {
        for (Pair<Integer, MeasureIngester> pair : dictMeasures) {
            int i = pair.getFirst();
            MeasureIngester ingester = pair.getSecond();
            measureObjs[i] = ingester.reEncodeDictionary(measureObjs[i], measureDescs.get(i), oldDicts, newDicts);
        }

    }
    return Pair.newPair(processKey(key), measureObjs);
}
 

public static Pair<Long, Long> getSizeAndLastModified(String path) throws IOException {
    FileSystem fs = HadoopUtil.getFileSystem(path);

    // get all contained files if path is directory
    ArrayList<FileStatus> allFiles = new ArrayList<>();
    FileStatus status = fs.getFileStatus(new Path(path));
    if (status.isFile()) {
        allFiles.add(status);
    } else {
        FileStatus[] listStatus = fs.listStatus(new Path(path));
        allFiles.addAll(Arrays.asList(listStatus));
    }

    long size = 0;
    long lastModified = 0;
    for (FileStatus file : allFiles) {
        size += file.getLen();
        lastModified = Math.max(lastModified, file.getModificationTime());
    }

    return Pair.newPair(size, lastModified);
}
 

/**
 * Re-encode with measures in Object[] format.
 * @param key
 * @param value
 * @return
 * @throws IOException
 */
public Pair<Text, Object[]> reEncode2(Text key, Text value) throws IOException {
    if (initialized == false) {
        throw new IllegalStateException("Not initialized");
    }

    Object[] measureObjs = new Object[measureDescs.size()];
    codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), measureObjs);
    // re-encode measures if dictionary is used
    if (dictMeasures.size() > 0) {
        for (Pair<Integer, MeasureIngester> pair : dictMeasures) {
            int i = pair.getFirst();
            MeasureIngester ingester = pair.getSecond();
            measureObjs[i] = ingester.reEncodeDictionary(measureObjs[i], measureDescs.get(i), oldDicts, newDicts);
        }

    }
    return Pair.newPair(processKey(key), measureObjs);
}
 

/**
 * Re-encode with both dimension and measure in encoded (Text) format.
 * @param key
 * @param value
 * @return
 * @throws IOException
 */
public Pair<Text, Text> reEncode(Text key, Text value) throws IOException {
    if (initialized == false) {
        throw new IllegalStateException("Not initialized");
    }
    Object[] measureObjs = new Object[measureDescs.size()];
    // re-encode measures if dictionary is used
    if (dictMeasures.size() > 0) {
        codec.decode(ByteBuffer.wrap(value.getBytes(), 0, value.getLength()), measureObjs);
        for (Pair<Integer, MeasureIngester> pair : dictMeasures) {
            int i = pair.getFirst();
            MeasureIngester ingester = pair.getSecond();
            measureObjs[i] = ingester.reEncodeDictionary(measureObjs[i], measureDescs.get(i), oldDicts, newDicts);
        }

        ByteBuffer valueBuf = codec.encode(measureObjs);
        textValue.set(valueBuf.array(), 0, valueBuf.position());
        return Pair.newPair(processKey(key), textValue);
    } else {
        return Pair.newPair(processKey(key), value);
    }
}
 

public static final Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(final long baseCuboidId, final long childCuboidId, final int measureCount) {
    final Pair<ImmutableBitSet, ImmutableBitSet> parentDimensionAndMetricColumnBitSet = getDimensionAndMetricColumnBitSet(baseCuboidId, measureCount);
    ImmutableBitSet parentDimensions = parentDimensionAndMetricColumnBitSet.getFirst();
    ImmutableBitSet measureColumns = parentDimensionAndMetricColumnBitSet.getSecond();
    ImmutableBitSet childDimensions = parentDimensions;
    long mask = Long.highestOneBit(baseCuboidId);
    long parentCuboidIdActualLength = (long)Long.SIZE - Long.numberOfLeadingZeros(baseCuboidId);
    int index = 0;
    for (int i = 0; i < parentCuboidIdActualLength; i++) {
        if ((mask & baseCuboidId) > 0) {
            if ((mask & childCuboidId) == 0) {
                // this dim will be aggregated
                childDimensions = childDimensions.set(index, false);
            }
            index++;
        }
        mask = mask >> 1;
    }
    return Pair.newPair(childDimensions, measureColumns);
}
 

public static void execLimitAndValidateNew(List<Pair<String, String>> queries, String prj, String joinType,
                                           Map<String, CompareEntity> recAndQueryResult) {

    int appendLimitQueries = 0;
    for (Pair<String, String> query : queries) {
        logger.info("execLimitAndValidate on query: " + query.getFirst());
        String sql = changeJoinType(query.getSecond(), joinType);

        Pair<String, String> sqlAndAddedLimitSql = Pair.newPair(sql, sql);
        if (!sql.toLowerCase(Locale.ROOT).contains("limit ")) {
            sqlAndAddedLimitSql.setSecond(sql + " limit 5");
            appendLimitQueries++;
        }

        Dataset<Row> kapResult = (recAndQueryResult == null) ? queryWithKap(prj, joinType, sqlAndAddedLimitSql)
                : queryWithKap(prj, joinType, sqlAndAddedLimitSql, recAndQueryResult);
        addQueryPath(recAndQueryResult, query, sql);
        Dataset<Row> sparkResult = queryWithSpark(prj, sql, query.getFirst());
        List<Row> kapRows = SparkQueryTest.castDataType(kapResult, sparkResult).toJavaRDD().collect();
        List<Row> sparkRows = sparkResult.toJavaRDD().collect();
        if (!compareResults(normRows(sparkRows), normRows(kapRows), CompareLevel.SUBSET)) {
            throw new IllegalArgumentException("Result not match");
        }
    }
    logger.info("Queries appended with limit: " + appendLimitQueries);
}
 

public static Pair<Integer, Integer> getReplacePos(SqlNode node, String inputSql) {
    if (inputSql == null) {
        return Pair.newPair(0, 0);
    }
    String[] lines = inputSql.split("\n");
    SqlParserPos pos = node.getParserPosition();
    int lineStart = pos.getLineNum();
    int lineEnd = pos.getEndLineNum();
    int columnStart = pos.getColumnNum() - 1;
    int columnEnd = pos.getEndColumnNum();
    //for the case that sql is multi lines
    for (int i = 0; i < lineStart - 1; i++) {
        columnStart += lines[i].length() + 1;
    }
    for (int i = 0; i < lineEnd - 1; i++) {
        columnEnd += lines[i].length() + 1;
    }
    //for calcite's bug CALCITE-1875
    Pair<Integer, Integer> startEndPos = getPosWithBracketsCompletion(inputSql, columnStart, columnEnd);
    return startEndPos;
}
 

public Pair<Boolean, Boolean> fitInSegments(ISegment newOne) {
    if (this.isEmpty())
        return null;

    ISegment first = this.get(0);
    ISegment last = this.get(this.size() - 1);
    Endpoint start = newOne.getSegRange().start;
    Endpoint end = newOne.getSegRange().end;
    boolean startFit = false;
    boolean endFit = false;
    for (ISegment sss : this) {
        if (sss == newOne)
            continue;
        startFit = startFit || (start.equals(sss.getSegRange().start) || start.equals(sss.getSegRange().end));
        endFit = endFit || (end.equals(sss.getSegRange().start) || end.equals(sss.getSegRange().end));
    }
    if (!startFit && endFit && newOne == first)
        startFit = true;
    if (!endFit && startFit && newOne == last)
        endFit = true;

    return Pair.newPair(startFit, endFit);
}
 

public Pair<T, T> findMergeOffsetsByDateRange(TSRange tsRange, long skipSegDateRangeCap) {
    // must be offset cube
    Segments result = new Segments();
    for (ISegment seg : this) {

        // include if date range overlaps
        if (tsRange.overlaps(seg.getTSRange())) {

            // reject too big segment
            if (seg.getTSRange().duration() > skipSegDateRangeCap)
                break;

            // reject holes
            if (result.size() > 0 && !result.getLast().getSegRange().connects(seg.getSegRange()))
                break;

            result.add(seg);
        }
    }

    if (result.size() <= 1)
        return null;
    else
        return (Pair<T, T>) Pair.newPair(result.getFirst(), result.getLast());
}
 

public Pair<String, String> getJobServerWithState() throws IOException {
    String url = baseUrl + "/service_discovery/state/is_active_job_node";
    HttpGet get = new HttpGet(url);
    HttpResponse response = null;
    try {
        response = client.execute(get);
        String msg = EntityUtils.toString(response.getEntity());

        if (response.getStatusLine().getStatusCode() != 200) {
            throw new IOException(INVALID_RESPONSE + response.getStatusLine().getStatusCode()
                    + " with getting job server state  " + url + "\n" + msg);
        }
        return Pair.newPair(host + ":" + port, msg);
    } finally {
        cleanup(get, response);
    }
}
 

public Pair<Boolean, Boolean> fitInSegments(ISegment newOne) {
    if (this.isEmpty())
        return null;

    ISegment first = this.get(0);
    ISegment last = this.get(this.size() - 1);
    Endpoint start = newOne.getSegRange().start;
    Endpoint end = newOne.getSegRange().end;
    boolean startFit = false;
    boolean endFit = false;
    for (ISegment sss : this) {
        if (sss == newOne)
            continue;
        startFit = startFit || (start.equals(sss.getSegRange().start) || start.equals(sss.getSegRange().end));
        endFit = endFit || (end.equals(sss.getSegRange().start) || end.equals(sss.getSegRange().end));
    }
    if (!startFit && endFit && newOne == first)
        startFit = true;
    if (!endFit && startFit && newOne == last)
        endFit = true;

    return Pair.newPair(startFit, endFit);
}
 

public static final Pair<ImmutableBitSet, ImmutableBitSet> getDimensionAndMetricColumnBitSet(final long cuboidId, final int measureCount) {
    int cardinality = Long.bitCount(cuboidId);
    BitSet dimension = new BitSet();
    dimension.set(0, cardinality);
    BitSet metrics = new BitSet();
    metrics.set(cardinality, cardinality + measureCount);
    return Pair.newPair(new ImmutableBitSet(dimension), new ImmutableBitSet(metrics));
}
 

private static Pair<Integer, Integer> getPosWithBracketsCompletion(String inputSql, int left, int right) {
    int leftBracketNum = 0;
    int rightBracketNum = 0;
    String substring = inputSql.substring(left, right);
    for (int i = 0; i < substring.length(); i++) {
        char temp = substring.charAt(i);
        if (temp == '(') {
            leftBracketNum++;
        }
        if (temp == ')') {
            rightBracketNum++;
            if (leftBracketNum < rightBracketNum) {
                while ('(' != inputSql.charAt(left - 1)) {
                    left--;
                }
                left--;
                leftBracketNum++;
            }
        }
    }
    while (rightBracketNum < leftBracketNum) {
        while (')' != inputSql.charAt(right)) {
            right++;
        }
        right++;
        rightBracketNum++;
    }
    return Pair.newPair(left, right);
}
 

@Override
public Pair<TableDesc, TableExtDesc> loadTableMetadata(String database, String table, String prj)
        throws SQLException {
    TableDesc tableDesc = new TableDesc();
    tableDesc.setDatabase(database.toUpperCase(Locale.ROOT));
    tableDesc.setName(table.toUpperCase(Locale.ROOT));
    tableDesc.setUuid(RandomUtil.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setSourceType(ISourceAware.ID_JDBC);

    Connection con = SqlUtil.getConnection(dbconf);
    DatabaseMetaData dbmd = con.getMetaData();

    try (ResultSet rs = jdbcMetadataDialect.getTable(dbmd, database, table)) {
        String tableType = null;
        while (rs.next()) {
            tableType = rs.getString("TABLE_TYPE");
        }
        if (tableType != null) {
            tableDesc.setTableType(tableType);
        } else {
            throw new RuntimeException(
                    String.format(Locale.ROOT, "table %s not found in schema:%s", table, database));
        }
    }

    try (ResultSet rs = jdbcMetadataDialect.listColumns(dbmd, database, table)) {
        tableDesc.setColumns(extractColumnFromMeta(rs));
    } finally {
        DBUtils.closeQuietly(con);
    }

    TableExtDesc tableExtDesc = new TableExtDesc();
    tableExtDesc.setIdentity(tableDesc.getIdentity());
    tableExtDesc.setUuid(RandomUtil.randomUUID().toString());
    tableExtDesc.setLastModified(0);
    tableExtDesc.init(prj);

    return Pair.newPair(tableDesc, tableExtDesc);
}
 

private Pair<Integer, Integer> countDerivedInfo(Map<Array<TblColRef>, List<DeriveInfo>> hostToDerivedInfo) {
    int pkfkCount = 0;
    int lookupCount = 0;
    for (Entry<Array<TblColRef>, List<DeriveInfo>> entry : hostToDerivedInfo.entrySet()) {
        for (DeriveInfo deriveInfo : entry.getValue()) {
            if (deriveInfo.type == DeriveType.PK_FK)
                pkfkCount++;
            if (deriveInfo.type == DeriveType.LOOKUP)
                lookupCount++;
        }
    }
    return Pair.newPair(pkfkCount, lookupCount);
}
 

@Override
protected Pair<String, String> formatNotifications(ExecutableContext context, ExecutableState state) {
    CubeInstance cubeInstance = CubeManager.getInstance(context.getConfig())
            .getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final Output output = getManager().getOutput(getId());
    if (state != ExecutableState.ERROR
            && !cubeInstance.getDescriptor().getStatusNeedNotify().contains(state.toString())) {
        logger.info("state:" + state + " no need to notify users");
        return null;
    }

    if (!MailNotificationUtil.hasMailNotification(state)) {
        logger.info("Cannot find email template for job state: " + state);
        return null;
    }

    Map<String, Object> dataMap = Maps.newHashMap();
    dataMap.put("job_name", getName());
    dataMap.put("env_name", getDeployEnvName());
    dataMap.put("submitter", StringUtil.noBlank(getSubmitter(), "missing submitter"));
    dataMap.put("job_engine", MailNotificationUtil.getLocalHostName());
    dataMap.put("project_name", getProjectName());
    dataMap.put("cube_name", cubeInstance.getName());
    dataMap.put("source_records_count", String.valueOf(findSourceRecordCount()));
    dataMap.put("start_time", new Date(getStartTime()).toString());
    dataMap.put("duration", getDuration() / 60000 + "mins");
    dataMap.put("mr_waiting", getMapReduceWaitTime() / 60000 + "mins");
    dataMap.put("last_update_time", new Date(getLastModified()).toString());

    if (state == ExecutableState.ERROR) {
        AbstractExecutable errorTask = null;
        Output errorOutput = null;
        for (AbstractExecutable task : getTasks()) {
            errorOutput = getManager().getOutput(task.getId());
            if (errorOutput.getState() == ExecutableState.ERROR) {
                errorTask = task;
                break;
            }
        }
        Preconditions.checkNotNull(errorTask,
                "None of the sub tasks of cubing job " + getId() + " is error and this job should become success.");

        dataMap.put("error_step", errorTask.getName());
        if (errorTask instanceof MapReduceExecutable) {
            final String mrJobId = errorOutput.getExtra().get(ExecutableConstants.MR_JOB_ID);
            dataMap.put("mr_job_id", StringUtil.noBlank(mrJobId, "Not initialized"));
        } else {
            dataMap.put("mr_job_id", MailNotificationUtil.NA);
        }
        dataMap.put("error_log",
                Matcher.quoteReplacement(StringUtil.noBlank(output.getVerboseMsg(), "no error message")));
    }

    String content = MailNotificationUtil.getMailContent(state, dataMap);
    String title = MailNotificationUtil.getMailTitle("JOB", state.toString(), getDeployEnvName(), getProjectName(),
            cubeInstance.getName());
    return Pair.newPair(title, content);
}
 

@Override
public Pair<TableDesc, TableExtDesc> loadTableMetadata(String database, String table, String prj)
        throws SQLException {
    TableDesc tableDesc = new TableDesc();
    tableDesc.setDatabase(database.toUpperCase(Locale.ROOT));
    tableDesc.setName(table.toUpperCase(Locale.ROOT));
    tableDesc.setUuid(UUID.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setProject(prj);
    tableDesc.setSourceType(JdbcSource.SOURCE_ID);

    try (CachedRowSet tables = dataSource.getTable(database, table)) {
        String tableType = null;
        while (tables.next()) {
            tableType = tables.getString("TABLE_TYPE");
        }
        if (tableType != null) {
            tableDesc.setTableType(tableType);
        } else {
            throw new RuntimeException(String.format(Locale.ROOT, "table %s not found in schema:%s", table, database));
        }
    }

    try (CachedRowSet columns = dataSource.listColumns(database, table)) {
        List<ColumnDesc> columnDescs = new ArrayList<>();

        while (columns.next()) {
            String cname = columns.getString("COLUMN_NAME");
            int type = columns.getInt("DATA_TYPE");
            int csize = columns.getInt("COLUMN_SIZE");
            int digits = columns.getInt("DECIMAL_DIGITS");
            int pos = columns.getInt("ORDINAL_POSITION");
            String remarks = columns.getString("REMARKS");

            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(cname.toUpperCase(Locale.ROOT));

            String kylinType = dataSource.toKylinTypeName(type);
            if ("any".equals(kylinType)) {
                String typeName = columns.getString("TYPE_NAME");
                int kylinTypeId = dataSource.toKylinTypeId(typeName, type);
                kylinType = dataSource.toKylinTypeName(kylinTypeId);
            }
            int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1;
            precision = Math.min(precision, KylinConfig.getInstanceFromEnv().getDefaultVarcharPrecision());
            int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1;

            cdesc.setDatatype(new DataType(kylinType, precision, scale).toString());
            cdesc.setId(String.valueOf(pos));
            cdesc.setComment(remarks);
            columnDescs.add(cdesc);
        }

        tableDesc.setColumns(columnDescs.toArray(new ColumnDesc[columnDescs.size()]));

        TableExtDesc tableExtDesc = new TableExtDesc();
        tableExtDesc.setIdentity(tableDesc.getIdentity());
        tableExtDesc.setUuid(UUID.randomUUID().toString());
        tableExtDesc.setLastModified(0);
        tableExtDesc.init(prj);

        return Pair.newPair(tableDesc, tableExtDesc);
    }
}
 

protected Pair<Short, Short> getShardNumAndBaseShard() {
    return Pair.newPair(cubeSeg.getCuboidShardNum(cuboid.getId()), cubeSeg.getCuboidBaseShard(cuboid.getId()));
}