Java Code Examples for org.apache.kylin.common.util.ClassUtil#newInstance()

The following examples show how to use org.apache.kylin.common.util.ClassUtil#newInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: PartitionDesc.java    From kylin with Apache License 2.0 6 votes vote down vote up
public void init(DataModelDesc model) {
    if (StringUtils.isEmpty(partitionDateColumn))
        return;

    //Support CustomYearMonthDayFieldPartitionConditionBuilder, partitionDateColumn split by ","
    partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
    if (partitionConditionBuilder instanceof CustomYearMonthDayFieldPartitionConditionBuilder) {
        ((CustomYearMonthDayFieldPartitionConditionBuilder)partitionConditionBuilder).init(this, model);
    } else {
        partitionDateColumnRef = model.findColumn(partitionDateColumn);
        partitionDateColumn = partitionDateColumnRef.getIdentity();
        if (StringUtils.isBlank(partitionTimeColumn) == false) {
            partitionTimeColumnRef = model.findColumn(partitionTimeColumn);
            partitionTimeColumn = partitionTimeColumnRef.getIdentity();
        }
    }

}
 
Example 2
Source File: PartitionDesc.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public void init(DataModelDesc model) {
    if (StringUtils.isEmpty(partitionDateColumn))
        return;

    //Support CustomYearMonthDayFieldPartitionConditionBuilder, partitionDateColumn split by ","
    partitionConditionBuilder = (IPartitionConditionBuilder) ClassUtil.newInstance(partitionConditionBuilderClz);
    if (partitionConditionBuilder instanceof CustomYearMonthDayFieldPartitionConditionBuilder) {
        ((CustomYearMonthDayFieldPartitionConditionBuilder)partitionConditionBuilder).init(this, model);
    } else {
        partitionDateColumnRef = model.findColumn(partitionDateColumn);
        partitionDateColumn = partitionDateColumnRef.getIdentity();
        if (StringUtils.isBlank(partitionTimeColumn) == false) {
            partitionTimeColumnRef = model.findColumn(partitionTimeColumn);
            partitionTimeColumn = partitionTimeColumnRef.getIdentity();
        }
    }

}
 
Example 3
Source File: DictionaryManager.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private Dictionary<String> buildDictFromReadableTable(IReadableTable inpTable, DictionaryInfo dictInfo,
        String builderClass, TblColRef col) throws IOException {
    Dictionary<String> dictionary;
    IDictionaryValueEnumerator columnValueEnumerator = null;
    try {
        columnValueEnumerator = new TableColumnValueEnumerator(inpTable.getReader(),
                dictInfo.getSourceColumnIndex());
        if (builderClass == null) {
            dictionary = DictionaryGenerator.buildDictionary(DataType.getType(dictInfo.getDataType()),
                    columnValueEnumerator);
        } else {
            IDictionaryBuilder builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass);
            dictionary = DictionaryGenerator.buildDictionary(builder, dictInfo, columnValueEnumerator);
        }
    } catch (Exception ex) {
        throw new RuntimeException("Failed to create dictionary on " + col, ex);
    } finally {
        if (columnValueEnumerator != null)
            columnValueEnumerator.close();
    }
    return dictionary;
}
 
Example 4
Source File: DictionaryManager.java    From kylin with Apache License 2.0 6 votes vote down vote up
private Dictionary<String> buildDictFromReadableTable(IReadableTable inpTable, DictionaryInfo dictInfo,
        String builderClass, TblColRef col) throws IOException {
    Dictionary<String> dictionary;
    IDictionaryValueEnumerator columnValueEnumerator = null;
    try {
        columnValueEnumerator = new TableColumnValueEnumerator(inpTable.getReader(),
                dictInfo.getSourceColumnIndex());
        if (builderClass == null) {
            dictionary = DictionaryGenerator.buildDictionary(DataType.getType(dictInfo.getDataType()),
                    columnValueEnumerator);
        } else {
            IDictionaryBuilder builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass);
            dictionary = DictionaryGenerator.buildDictionary(builder, dictInfo, columnValueEnumerator);
        }
    } catch (Exception ex) {
        throw new RuntimeException("Failed to create dictionary on " + col, ex);
    } finally {
        if (columnValueEnumerator != null)
            columnValueEnumerator.close();
    }
    return dictionary;
}
 
Example 5
Source File: QueryUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
static void initPushDownConvertersIfNeeded(KylinConfig kylinConfig) {
    String[] currentConverters = pushDownConverters.stream().map(Object::getClass).map(Class::getCanonicalName)
            .toArray(String[]::new);
    String[] configConverters = kylinConfig.getPushDownConverterClassNames();
    boolean skipInit = Objects.deepEquals(currentConverters, configConverters);

    if (skipInit) {
        return;
    }

    List<IPushDownConverter> converters = Lists.newArrayList();
    for (String clz : configConverters) {
        try {
            IPushDownConverter converter = (IPushDownConverter) ClassUtil.newInstance(clz);
            converters.add(converter);
        } catch (Exception e) {
            throw new IllegalStateException("Failed to init pushdown converter", e);
        }
    }
    pushDownConverters = Collections.unmodifiableList(converters);
}
 
Example 6
Source File: ExternalAclProvider.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public static ExternalAclProvider getInstance() {
    if (singleton == null) {
        synchronized (ExternalAclProvider.class) {
            if (singleton == null) {
                String cls = KylinConfig.getInstanceFromEnv().getExternalAclProvider();
                if (!StringUtils.isBlank(cls)) {
                    singleton = (ExternalAclProvider) ClassUtil.newInstance(cls);
                    singleton.init();
                }
            }
        }
    }
    return singleton;
}
 
Example 7
Source File: AbstractPushdownRunner.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Override
public String convertSql(KylinConfig kylinConfig, String sql, String project, String defaultSchema,
                         boolean isPrepare) {
    String converted = sql;
    for (String converterName : kylinConfig.getPushDownConverterClassNames()) {
        IPushDownConverter converter = (IPushDownConverter) ClassUtil.newInstance(converterName);
        String tmp = converter.convert(converted, project, defaultSchema, isPrepare);
        if (!converted.equals(tmp)) {
            logger.info("the query is converted to {} after applying converter {}", tmp, converterName);
        }
        converted = tmp;
    }
    return converted;
}
 
Example 8
Source File: Broadcaster.java    From kylin with Apache License 2.0 5 votes vote down vote up
private SyncErrorHandler getSyncErrorHandler(KylinConfig config) {
    String clzName = config.getCacheSyncErrorHandler();
    if (StringUtils.isEmpty(clzName)) {
        clzName = DefaultSyncErrorHandler.class.getName();
    }
    return (SyncErrorHandler) ClassUtil.newInstance(clzName);
}
 
Example 9
Source File: QueryUtil.java    From kylin with Apache License 2.0 5 votes vote down vote up
private static void initQueryTransformers() {
    List<IQueryTransformer> transformers = Lists.newArrayList();

    String[] classes = KylinConfig.getInstanceFromEnv().getQueryTransformers();
    for (String clz : classes) {
        try {
            IQueryTransformer t = (IQueryTransformer) ClassUtil.newInstance(clz);
            transformers.add(t);
        } catch (Exception e) {
            throw new IllegalStateException("Failed to init query transformer", e);
        }
    }

    queryTransformers = transformers;
}
 
Example 10
Source File: PushDownExecutor.java    From kylin with Apache License 2.0 5 votes vote down vote up
public Pair<List<List<String>>, List<SelectedColumnMeta>> pushDownQuery(String project,
        String sql, String defaultSchema, SQLException sqlException, boolean isSelect,
        boolean isPrepare) throws Exception {

    if (!kylinConfig.isPushDownEnabled()) {
        return null;
    }

    if (isSelect) {
        logger.info("Query failed to utilize pre-calculation, routing to other engines", sqlException);
        if (!isExpectedCause(sqlException)) {
            logger.info("quit doPushDownQuery because prior exception thrown is unexpected");
            return null;
        }
    } else {
        Preconditions.checkState(sqlException == null);
        logger.info("Kylin cannot support non-select queries, routing to other engines");
    }

    List<String> ids = kylinConfig.getPushDownRunnerIds();

    if (ids.isEmpty() && StringUtils.isNotEmpty(kylinConfig.getPushDownRunnerClassName())) {
        IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(
                kylinConfig.getPushDownRunnerClassName()
        );
        runner.init(kylinConfig);
        return queryBySingleRunner(runner, project, sql, defaultSchema, sqlException,
                isSelect, isPrepare);
    } else {
        return queryByMultiJdbcRunners(ids, project, sql, defaultSchema, sqlException,
                isSelect, isPrepare);
    }
}
 
Example 11
Source File: OLAPToEnumerableConverter.java    From kylin with Apache License 2.0 5 votes vote down vote up
protected void doAccessControl(List<OLAPContext> contexts) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String controllerCls = config.getQueryAccessController();
    if (null != controllerCls && !controllerCls.isEmpty()) {
        OLAPContext.IAccessController accessController = (OLAPContext.IAccessController) ClassUtil.newInstance(controllerCls);
        accessController.check(contexts, config);
    }
}
 
Example 12
Source File: SparkUHCDictionary.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Override
public Tuple2<String, Tuple3<Writable, Writable, String>> call(Tuple2<Integer, List<String>> columnValues) throws Exception {
    if (initialized == false) {
        synchronized (SparkFactDistinct.class) {
            if (initialized == false) {
                init();
            }
        }
    }

    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config);
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         DataOutputStream outputStream = new DataOutputStream(baos)) {
        TblColRef col = uhcColumns.get(columnValues._1);
        logger.info("Processing column " + col.getName());
        if (cube.getDescriptor().getShardByColumns().contains(col)) {
            //for ShardByColumns
            builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
            builder.init(null, 0, null);
        } else {
            //for GlobalDictionaryColumns
            DictionaryInfo dictionaryInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype());
            String builderClass = cubeDesc.getDictionaryBuilderClass(col);
            builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass);
            builder.init(dictionaryInfo, 0, hdfsDir);
        }
        Iterator<String> values = columnValues._2.iterator();
        while (values.hasNext()) {
            builder.addValue(values.next());
        }
        Dictionary<String> dict = builder.build();
        String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX;
        logger.info("Dictionary file name is " + dictFileName);

        outputStream.writeUTF(dict.getClass().getName());
        dict.write(outputStream);
        Tuple3 tuple3 = new Tuple3(NullWritable.get(), new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName);
        return new Tuple2<>(BatchConstants.CFG_OUTPUT_DICT, tuple3);
    }
}
 
Example 13
Source File: SparkUHCDictionary.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Override
public Tuple2<String, Tuple3<Writable, Writable, String>> call(Tuple2<Integer, List<String>> columnValues) throws Exception {
    if (initialized == false) {
        synchronized (SparkFactDistinct.class) {
            if (initialized == false) {
                init();
            }
        }
    }

    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(config);
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         DataOutputStream outputStream = new DataOutputStream(baos)) {
        TblColRef col = uhcColumns.get(columnValues._1);
        logger.info("Processing column " + col.getName());
        if (cube.getDescriptor().getShardByColumns().contains(col)) {
            //for ShardByColumns
            builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
            builder.init(null, 0, null);
        } else {
            //for GlobalDictionaryColumns
            DictionaryInfo dictionaryInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype());
            String builderClass = cubeDesc.getDictionaryBuilderClass(col);
            builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass);
            builder.init(dictionaryInfo, 0, hdfsDir);
        }
        Iterator<String> values = columnValues._2.iterator();
        while (values.hasNext()) {
            builder.addValue(values.next());
        }
        Dictionary<String> dict = builder.build();
        String dictFileName = col.getIdentity() + "/" + col.getName() + DICT_FILE_POSTFIX;
        logger.info("Dictionary file name is " + dictFileName);

        outputStream.writeUTF(dict.getClass().getName());
        dict.write(outputStream);
        Tuple3 tuple3 = new Tuple3(NullWritable.get(), new ArrayPrimitiveWritable(baos.toByteArray()), dictFileName);
        return new Tuple2<>(BatchConstants.CFG_OUTPUT_DICT, tuple3);
    }
}
 
Example 14
Source File: UHCDictionaryReducer.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Override
protected void doSetup(Context context) throws IOException {
    super.bindCurrentConfiguration(context.getConfiguration());
    Configuration conf = context.getConfiguration();
    mos = new MultipleOutputs(context);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
    String cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
    CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
    CubeDesc cubeDesc = cube.getDescriptor();
    List<TblColRef> uhcColumns = cubeDesc.getAllUHCColumns();

    int taskId = context.getTaskAttemptID().getTaskID().getId();
    col = uhcColumns.get(taskId);
    logger.info("column name: " + col.getIdentity());

    if (cube.getDescriptor().getShardByColumns().contains(col)) {
        //for ShardByColumns
        builder = DictionaryGenerator.newDictionaryBuilder(col.getType());
        builder.init(null, 0, null);
    } else {
        //for GlobalDictionaryColumns
        String hdfsDir = conf.get(BatchConstants.CFG_GLOBAL_DICT_BASE_DIR);
        DictionaryInfo dictionaryInfo = new DictionaryInfo(col.getColumnDesc(), col.getDatatype());
        String builderClass = cubeDesc.getDictionaryBuilderClass(col);
        builder = (IDictionaryBuilder) ClassUtil.newInstance(builderClass);
        builder.init(dictionaryInfo, 0, hdfsDir);
    }
}
 
Example 15
Source File: AbstractPushdownRunner.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Override
public String convertSql(KylinConfig kylinConfig, String sql, String project, String defaultSchema,
                         boolean isPrepare) {
    String converted = sql;
    for (String converterName : kylinConfig.getPushDownConverterClassNames()) {
        IPushDownConverter converter = (IPushDownConverter) ClassUtil.newInstance(converterName);
        String tmp = converter.convert(converted, project, defaultSchema, isPrepare);
        if (!converted.equals(tmp)) {
            logger.info("the query is converted to {} after applying converter {}", tmp, converterName);
        }
        converted = tmp;
    }
    return converted;
}
 
Example 16
Source File: Broadcaster.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private SyncErrorHandler getSyncErrorHandler(KylinConfig config) {
    String clzName = config.getCacheSyncErrorHandler();
    if (StringUtils.isEmpty(clzName)) {
        clzName = DefaultSyncErrorHandler.class.getName();
    }
    return (SyncErrorHandler) ClassUtil.newInstance(clzName);
}
 
Example 17
Source File: QueryUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private static void initQueryTransformers() {
    List<IQueryTransformer> transformers = Lists.newArrayList();

    String[] classes = KylinConfig.getInstanceFromEnv().getQueryTransformers();
    for (String clz : classes) {
        try {
            IQueryTransformer t = (IQueryTransformer) ClassUtil.newInstance(clz);
            transformers.add(t);
        } catch (Exception e) {
            throw new IllegalStateException("Failed to init query transformer", e);
        }
    }

    queryTransformers = transformers;
}
 
Example 18
Source File: PushDownExecutor.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public Pair<List<List<String>>, List<SelectedColumnMeta>> pushDownQuery(String project, String sql,
        String defaultSchema, SQLException sqlException, boolean isSelect, boolean isPrepare) throws Exception {

    if (!kylinConfig.isPushDownEnabled()) {
        return null;
    }

    if (isSelect) {
        logger.info("Query failed to utilize pre-calculation, routing to other engines", sqlException);
        if (!isExpectedCause(sqlException)) {
            logger.info("quit doPushDownQuery because prior exception thrown is unexpected");
            return null;
        }
    } else {
        Preconditions.checkState(sqlException == null);
        logger.info("Kylin cannot support non-select queries, routing to other engines");
    }

    List<String> ids = kylinConfig.getPushDownRunnerIds();

    if (ids.isEmpty() && StringUtils.isNotEmpty(kylinConfig.getPushDownRunnerClassName())) {
        IPushDownRunner runner = (IPushDownRunner) ClassUtil.newInstance(kylinConfig.getPushDownRunnerClassName());
        runner.init(kylinConfig);
        return queryBySingleRunner(runner, project, sql, defaultSchema, sqlException, isSelect, isPrepare);
    } else {
        return queryByMultiJdbcRunners(ids, project, sql, defaultSchema, sqlException, isSelect, isPrepare);
    }
}
 
Example 19
Source File: OLAPToEnumerableConverter.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
protected void doAccessControl(List<OLAPContext> contexts) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String controllerCls = config.getQueryAccessController();
    if (null != controllerCls && !controllerCls.isEmpty()) {
        OLAPContext.IAccessController accessController = (OLAPContext.IAccessController) ClassUtil.newInstance(controllerCls);
        accessController.check(contexts, config);
    }
}
 
Example 20
Source File: KylinConfigBase.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up
public DistributedLockFactory getDistributedLockFactory() {
    String clsName = getOptional("kylin.metadata.distributed-lock-impl",
            "org.apache.kylin.job.lock.zookeeper.ZookeeperDistributedLock$Factory");
    return (DistributedLockFactory) ClassUtil.newInstance(clsName);
}