Java Code Examples for org.apache.kylin.common.util.ClassUtil#newInstance()

The following examples show how to use org.apache.kylin.common.util.ClassUtil#newInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: PartitionDesc.java From kylin with Apache License 2.0

6 votes

public void init(DataModelDesc model) {
    if (StringUtils.isEmpty(partitionDateColumn))
        return;

    //Support CustomYearMonthDayFieldPartitionConditionBuilder, partitionDateColumn split by ","
    partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
    if (partitionConditionBuilder instanceof CustomYearMonthDayFieldPartitionConditionBuilder) {
        ((CustomYearMonthDayFieldPartitionConditionBuilder)partitionConditionBuilder).init(this, model);
    } else {
        partitionDateColumnRef = model.findColumn(partitionDateColumn);
        partitionDateColumn = partitionDateColumnRef.getIdentity();
        if (StringUtils.isBlank(partitionTimeColumn) == false) {
            partitionTimeColumnRef = model.findColumn(partitionTimeColumn);
            partitionTimeColumn = partitionTimeColumnRef.getIdentity();
        }
    }

}

Example 2

Source File: PartitionDesc.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public void init(DataModelDesc model) {
    if (StringUtils.isEmpty(partitionDateColumn))
        return;

    //Support CustomYearMonthDayFieldPartitionConditionBuilder, partitionDateColumn split by ","
    partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
    if (partitionConditionBuilder instanceof CustomYearMonthDayFieldPartitionConditionBuilder) {
        ((CustomYearMonthDayFieldPartitionConditionBuilder)partitionConditionBuilder).init(this, model);
    } else {
        partitionDateColumnRef = model.findColumn(partitionDateColumn);
        partitionDateColumn = partitionDateColumnRef.getIdentity();
        if (StringUtils.isBlank(partitionTimeColumn) == false) {
            partitionTimeColumnRef = model.findColumn(partitionTimeColumn);
            partitionTimeColumn = partitionTimeColumnRef.getIdentity();
        }
    }

}

Example 3

Source File: DictionaryManager.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private Dictionary<String> buildDictFromReadableTable(IReadableTable inpTable, DictionaryInfo dictInfo,
        String builderClass, TblColRef col) throws IOException {
    Dictionary<String> dictionary;
    IDictionaryValueEnumerator columnValueEnumerator = null;
    try {
        columnValueEnumerator = new TableColumnValueEnumerator(inpTable.getReader(),
                dictInfo.getSourceColumnIndex());
        if (builderClass == null) {
            dictionary = DictionaryGenerator.buildDictionary(DataType.getType(dictInfo.getDataType()),
                    columnValueEnumerator);
        } else {
            IDictionaryBuilder builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass);
            dictionary = DictionaryGenerator.buildDictionary(builder, dictInfo, columnValueEnumerator);
        }
    } catch (Exception ex) {
        throw new RuntimeException("Failed to create dictionary on " + col, ex);
    } finally {
        if (columnValueEnumerator != null)
            columnValueEnumerator.close();
    }
    return dictionary;
}

Example 4

Source File: DictionaryManager.java From kylin with Apache License 2.0

6 votes

private Dictionary<String> buildDictFromReadableTable(IReadableTable inpTable, DictionaryInfo dictInfo,
        String builderClass, TblColRef col) throws IOException {
    Dictionary<String> dictionary;
    IDictionaryValueEnumerator columnValueEnumerator = null;
    try {
        columnValueEnumerator = new TableColumnValueEnumerator(inpTable.getReader(),
                dictInfo.getSourceColumnIndex());
        if (builderClass == null) {
            dictionary = DictionaryGenerator.buildDictionary(DataType.getType(dictInfo.getDataType()),
                    columnValueEnumerator);
        } else {
            IDictionaryBuilder builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass);
            dictionary = DictionaryGenerator.buildDictionary(builder, dictInfo, columnValueEnumerator);
        }
    } catch (Exception ex) {
        throw new RuntimeException("Failed to create dictionary on " + col, ex);
    } finally {
        if (columnValueEnumerator != null)
            columnValueEnumerator.close();
    }
    return dictionary;
}

Example 5

Source File: QueryUtil.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

static void initPushDownConvertersIfNeeded(KylinConfig kylinConfig) {
    String[] currentConverters = pushDownConverters.stream().map(Object::getClass).map(Class::getCanonicalName)
            .toArray(String[]::new);
    String[] configConverters = kylinConfig.getPushDownConverterClassNames();
    boolean skipInit = Objects.deepEquals(currentConverters, configConverters);

    if (skipInit) {
        return;
    }

    List<IPushDownConverter> converters = Lists.newArrayList();
    for (String clz : configConverters) {
        try {
            IPushDownConverter converter = (IPushDownConverter) ClassUtil.newInstance(clz);
            converters.add(converter);
        } catch (Exception e) {
            throw new IllegalStateException("Failed to init pushdown converter", e);
        }
    }
    pushDownConverters = Collections.unmodifiableList(converters);
}

Example 6

Source File: ExternalAclProvider.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public static ExternalAclProvider getInstance() {
    if (singleton == null) {
        synchronized (ExternalAclProvider.class) {
            if (singleton == null) {
                String cls = KylinConfig.getInstanceFromEnv().getExternalAclProvider();
                if (!StringUtils.isBlank(cls)) {
                    singleton = (ExternalAclProvider) ClassUtil.newInstance(cls);
                    singleton.init();
                }
            }
        }
    }
    return singleton;
}

Example 7

Source File: AbstractPushdownRunner.java From kylin with Apache License 2.0

5 votes

@Override
public String convertSql(KylinConfig kylinConfig, String sql, String project, String defaultSchema,
                         boolean isPrepare) {
    String converted = sql;
    for (String converterName : kylinConfig.getPushDownConverterClassNames()) {
        IPushDownConverter converter = (IPushDownConverter) ClassUtil.newInstance(converterName);
        String tmp = converter.convert(converted, project, defaultSchema, isPrepare);
        if (!converted.equals(tmp)) {
            logger.info("the query is converted to {} after applying converter {}", tmp, converterName);
        }
        converted = tmp;
    }
    return converted;
}

Example 8

Source File: Broadcaster.java From kylin with Apache License 2.0

5 votes

private SyncErrorHandler getSyncErrorHandler(KylinConfig config) {
    String clzName = config.getCacheSyncErrorHandler();
    if (StringUtils.isEmpty(clzName)) {
        clzName = DefaultSyncErrorHandler.class.getName();
    }
    return (SyncErrorHandler) ClassUtil.newInstance(clzName);
}

Example 9

Source File: QueryUtil.java From kylin with Apache License 2.0

5 votes

private static void initQueryTransformers() {
    List<IQueryTransformer> transformers = Lists.newArrayList();

    String[] classes = KylinConfig.getInstanceFromEnv().getQueryTransformers();
    for (String clz : classes) {
        try {
            IQueryTransformer t = (IQueryTransformer) ClassUtil.newInstance(clz);
            transformers.add(t);
        } catch (Exception e) {
            throw new IllegalStateException("Failed to init query transformer", e);
        }
    }

    queryTransformers = transformers;
}

Example 10

Source File: PushDownExecutor.java From kylin with Apache License 2.0

5 votes

public Pair<List<List<String>>, List<SelectedColumnMeta>> pushDownQuery(String project,
        String sql, String defaultSchema, SQLException sqlException, boolean isSelect,
        boolean isPrepare) throws Exception {

    if (!kylinConfig.isPushDownEnabled()) {
        return null;
    }

    if (isSelect) {
        logger.info("Query failed to utilize pre-calculation, routing to other engines", sqlException);
        if (!isExpectedCause(sqlException)) {
            logger.info("quit doPushDownQuery because prior exception thrown is unexpected");
            return null;
        }
    } else {
        Preconditions.checkState(sqlException == null);
        logger.info("Kylin cannot support non-select queries, routing to other engines");
    }

    List<String> ids = kylinConfig.getPushDownRunnerIds();

    if (ids.isEmpty() && StringUtils.isNotEmpty(kylinConfig.getPushDownRunnerClassName())) {
        IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(
                kylinConfig.getPushDownRunnerClassName()
        );
        runner.init(kylinConfig);
        return queryBySingleRunner(runner, project, sql, defaultSchema, sqlException,
                isSelect, isPrepare);
    } else {
        return queryByMultiJdbcRunners(ids, project, sql, defaultSchema, sqlException,
                isSelect, isPrepare);
    }
}

Example 11

Source File: OLAPToEnumerableConverter.java From kylin with Apache License 2.0

5 votes

protected void doAccessControl(List<OLAPContext> contexts) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String controllerCls = config.getQueryAccessController();
    if (null != controllerCls && !controllerCls.isEmpty()) {
        OLAPContext.IAccessController accessController = (OLAPContext.IAccessController) ClassUtil.newInstance(controllerCls);
        accessController.check(contexts, config);
    }
}

Example 12

Source File: SparkUHCDictionary.java From kylin with Apache License 2.0

5 votes

@Override
public Tuple2<String, Tuple3<Writable, Writable, String>> call(Tuple2<Integer, List<String>> columnValues) throws Exception {
    if (initialized == false) {
        synchronized (SparkFactDistinct.class) {
            if (initialized == false) {
                init();
            }
        }
    }

    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config);
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         DataOutputStream outputStream = new DataOutputStream(baos)) {
        TblColRef col = uhcColumns.get(columnValues._1);
        logger.info("Processing column " + col.getName());
        if (cube.getDescriptor().getShardByColumns().contains(col)) {
            //for ShardByColumns
            builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
            builder.init(null, 0, null);
        } else {
            //for GlobalDictionaryColumns
            DictionaryInfo dictionaryInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype());
            String builderClass = cubeDesc.getDictionaryBuilderClass(col);
            builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass);
            builder.init(dictionaryInfo, 0, hdfsDir);
        }
        Iterator<String> values = columnValues._2.iterator();
        while (values.hasNext()) {
            builder.addValue(values.next());
        }
        Dictionary<String> dict = builder.build();
        String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX;
        logger.info("Dictionary file name is " + dictFileName);

        outputStream.writeUTF(dict.getClass().getName());
        dict.write(outputStream);
        Tuple3 tuple3 = new Tuple3(NullWritable.get(), new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName);
        return new Tuple2<>(BatchConstants.CFG_OUTPUT_DICT, tuple3);
    }
}

Example 13

Source File: SparkUHCDictionary.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Override
public Tuple2<String, Tuple3<Writable, Writable, String>> call(Tuple2<Integer, List<String>> columnValues) throws Exception {
    if (initialized == false) {
        synchronized (SparkFactDistinct.class) {
            if (initialized == false) {
                init();
            }
        }
    }

    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config);
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         DataOutputStream outputStream = new DataOutputStream(baos)) {
        TblColRef col = uhcColumns.get(columnValues._1);
        logger.info("Processing column " + col.getName());
        if (cube.getDescriptor().getShardByColumns().contains(col)) {
            //for ShardByColumns
            builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
            builder.init(null, 0, null);
        } else {
            //for GlobalDictionaryColumns
            DictionaryInfo dictionaryInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype());
            String builderClass = cubeDesc.getDictionaryBuilderClass(col);
            builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass);
            builder.init(dictionaryInfo, 0, hdfsDir);
        }
        Iterator<String> values = columnValues._2.iterator();
        while (values.hasNext()) {
            builder.addValue(values.next());
        }
        Dictionary<String> dict = builder.build();
        String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX;
        logger.info("Dictionary file name is " + dictFileName);

        outputStream.writeUTF(dict.getClass().getName());
        dict.write(outputStream);
        Tuple3 tuple3 = new Tuple3(NullWritable.get(), new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName);
        return new Tuple2<>(BatchConstants.CFG_OUTPUT_DICT, tuple3);
    }
}

Example 14

Source File: UHCDictionaryReducer.java From kylin with Apache License 2.0

5 votes

@Override
protected void doSetup(Context context) throws IOException {
    super.bindCurrentConfiguration(context.getConfiguration());
    Configuration conf = context.getConfiguration();
    mos = new MultipleOutputs(context);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
    String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
    CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
    CubeDesc cubeDesc = cube.getDescriptor();
    List<TblColRef> uhcColumns = cubeDesc.getAllUHCColumns();

    int taskId = context.getTaskAttemptID().getTaskID().getId();
    col = uhcColumns.get(taskId);
    logger.info("column name: " + col.getIdentity());

    if (cube.getDescriptor().getShardByColumns().contains(col)) {
        //for ShardByColumns
        builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
        builder.init(null, 0, null);
    } else {
        //for GlobalDictionaryColumns
        String hdfsDir = conf.get(BatchConstants.CFG_GLOBAL_DICT_BASE_DIR);
        DictionaryInfo dictionaryInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype());
        String builderClass = cubeDesc.getDictionaryBuilderClass(col);
        builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass);
        builder.init(dictionaryInfo, 0, hdfsDir);
    }
}

Example 15

Source File: AbstractPushdownRunner.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Override
public String convertSql(KylinConfig kylinConfig, String sql, String project, String defaultSchema,
                         boolean isPrepare) {
    String converted = sql;
    for (String converterName : kylinConfig.getPushDownConverterClassNames()) {
        IPushDownConverter converter = (IPushDownConverter) ClassUtil.newInstance(converterName);
        String tmp = converter.convert(converted, project, defaultSchema, isPrepare);
        if (!converted.equals(tmp)) {
            logger.info("the query is converted to {} after applying converter {}", tmp, converterName);
        }
        converted = tmp;
    }
    return converted;
}

Example 16

Source File: Broadcaster.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private SyncErrorHandler getSyncErrorHandler(KylinConfig config) {
    String clzName = config.getCacheSyncErrorHandler();
    if (StringUtils.isEmpty(clzName)) {
        clzName = DefaultSyncErrorHandler.class.getName();
    }
    return (SyncErrorHandler) ClassUtil.newInstance(clzName);
}

Example 17

Source File: QueryUtil.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private static void initQueryTransformers() {
    List<IQueryTransformer> transformers = Lists.newArrayList();

    String[] classes = KylinConfig.getInstanceFromEnv().getQueryTransformers();
    for (String clz : classes) {
        try {
            IQueryTransformer t = (IQueryTransformer) ClassUtil.newInstance(clz);
            transformers.add(t);
        } catch (Exception e) {
            throw new IllegalStateException("Failed to init query transformer", e);
        }
    }

    queryTransformers = transformers;
}

Example 18

Source File: PushDownExecutor.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public Pair<List<List<String>>, List<SelectedColumnMeta>> pushDownQuery(String project, String sql,
        String defaultSchema, SQLException sqlException, boolean isSelect, boolean isPrepare) throws Exception {

    if (!kylinConfig.isPushDownEnabled()) {
        return null;
    }

    if (isSelect) {
        logger.info("Query failed to utilize pre-calculation, routing to other engines", sqlException);
        if (!isExpectedCause(sqlException)) {
            logger.info("quit doPushDownQuery because prior exception thrown is unexpected");
            return null;
        }
    } else {
        Preconditions.checkState(sqlException == null);
        logger.info("Kylin cannot support non-select queries, routing to other engines");
    }

    List<String> ids = kylinConfig.getPushDownRunnerIds();

    if (ids.isEmpty() && StringUtils.isNotEmpty(kylinConfig.getPushDownRunnerClassName())) {
        IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(kylinConfig.getPushDownRunnerClassName());
        runner.init(kylinConfig);
        return queryBySingleRunner(runner, project, sql, defaultSchema, sqlException, isSelect, isPrepare);
    } else {
        return queryByMultiJdbcRunners(ids, project, sql, defaultSchema, sqlException, isSelect, isPrepare);
    }
}

Example 19

Source File: OLAPToEnumerableConverter.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

protected void doAccessControl(List<OLAPContext> contexts) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String controllerCls = config.getQueryAccessController();
    if (null != controllerCls && !controllerCls.isEmpty()) {
        OLAPContext.IAccessController accessController = (OLAPContext.IAccessController) ClassUtil.newInstance(controllerCls);
        accessController.check(contexts, config);
    }
}

Example 20

Source File: KylinConfigBase.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

public DistributedLockFactory getDistributedLockFactory() {
    String clsName = getOptional("kylin.metadata.distributed-lock-impl",
            "org.apache.kylin.job.lock.zookeeper.ZookeeperDistributedLock$Factory");
    return (DistributedLockFactory) ClassUtil.newInstance(clsName);
}