Java Code Examples for org.apache.kylin.common.KylinConfig#getMetadataUrl()

The following examples show how to use org.apache.kylin.common.KylinConfig#getMetadataUrl() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ResourceStore.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private static ResourceStore createResourceStore(KylinConfig kylinConfig) {
    StorageURL metadataUrl = kylinConfig.getMetadataUrl();
    logger.info("Using metadata url {} for resource store", metadataUrl);
    String clsName = kylinConfig.getResourceStoreImpls().get(metadataUrl.getScheme());
    try {
        Class<? extends ResourceStore> cls = ClassUtil.forName(clsName, ResourceStore.class);
        ResourceStore store = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
        if (!store.exists(METASTORE_UUID_TAG)) {
            store.checkAndPutResource(METASTORE_UUID_TAG, new StringEntity(store.createMetaStoreUUID()), 0,
                    StringEntity.serializer);
        }
        return store;
    } catch (Throwable e) {
        throw new IllegalArgumentException("Failed to find metadata store by url: " + metadataUrl, e);
    }
}

Example 2

Source File: HBaseResourceStore.java From Kylin with Apache License 2.0

6 votes

public HBaseResourceStore(KylinConfig kylinConfig) throws IOException {
    super(kylinConfig);

    String metadataUrl = kylinConfig.getMetadataUrl();
    // split TABLE@HBASE_URL
    int cut = metadataUrl.indexOf('@');
    tableNameBase = cut < 0 ? DEFAULT_TABLE_NAME : metadataUrl.substring(0, cut);
    hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1);

    createHTableIfNeeded(getAllInOneTableName());

    //        tableNameMap = new LinkedHashMap<String, String>();
    //        for (Entry<String, String> entry : TABLE_SUFFIX_MAP.entrySet()) {
    //            String pathPrefix = entry.getKey();
    //            String tableName = tableNameBase + entry.getValue();
    //            tableNameMap.put(pathPrefix, tableName);
    //            createHTableIfNeeded(tableName);
    //        }

}

Example 3

Source File: ResourceStore.java From kylin with Apache License 2.0

6 votes

private static ResourceStore createResourceStore(KylinConfig kylinConfig) {
    StorageURL metadataUrl = kylinConfig.getMetadataUrl();
    logger.info("Using metadata url {} for resource store", metadataUrl);
    String clsName = kylinConfig.getResourceStoreImpls().get(metadataUrl.getScheme());
    try {
        Class<? extends ResourceStore> cls = ClassUtil.forName(clsName, ResourceStore.class);
        ResourceStore store = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
        if (!store.exists(METASTORE_UUID_TAG)) {
            store.checkAndPutResource(METASTORE_UUID_TAG, new StringEntity(store.createMetaStoreUUID()), 0,
                    StringEntity.serializer);
        }
        return store;
    } catch (Throwable e) {
        throw new IllegalArgumentException("Failed to find metadata store by url: " + metadataUrl, e);
    }
}

Example 4

Source File: ResourceStore.java From Kylin with Apache License 2.0

5 votes

public static ResourceStore getStore(KylinConfig kylinConfig) {
    ResourceStore r = CACHE.get(kylinConfig);
    List<Throwable> es = new ArrayList<Throwable>();
    if (r == null) {
        logger.info("Using metadata url " + kylinConfig.getMetadataUrl() + " for resource store");
        for (Class<? extends ResourceStore> cls : knownImpl) {

            try {
                r = cls.getConstructor(KylinConfig.class).newInstance(kylinConfig);
            } catch (Exception e) {
                es.add(e);
            } catch (NoClassDefFoundError er) {
                // may throw NoClassDefFoundError
                es.add(er);
            }
            if (r != null) {
                break;
            }
        }
        if (r == null) {
            for (Throwable exceptionOrError : es) {
                logger.error("Create new store instance failed ", exceptionOrError);
            }
            throw new IllegalArgumentException("Failed to find metadata store by url: " + kylinConfig.getMetadataUrl());
        }

        CACHE.put(kylinConfig, r);
    }
    return r;
}

Example 5

Source File: FileResourceStore.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public FileResourceStore(KylinConfig kylinConfig) {
    super(kylinConfig);
    root = new File(getPath(kylinConfig)).getAbsoluteFile();
    if (!root.exists())
        throw new IllegalArgumentException(
                "File not exist by '" + kylinConfig.getMetadataUrl() + "': " + root.getAbsolutePath());
}

Example 6

Source File: JDBCConnectionManager.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private Map<String, String> initDbcpProps(KylinConfig config) {
    // metadataUrl is like "kylin_default_instance@jdbc,url=jdbc:mysql://localhost:3306/kylin,username=root,password=xxx"
    StorageURL metadataUrl = config.getMetadataUrl();
    JDBCResourceStore.checkScheme(metadataUrl);

    LinkedHashMap<String, String> ret = new LinkedHashMap<>(metadataUrl.getAllParameters());
    List<String> mandatoryItems = Arrays.asList("url", "username", PASSWORD);

    for (String item : mandatoryItems) {
        Preconditions.checkNotNull(ret.get(item),
                "Setting item \"" + item + "\" is mandatory for Jdbc connections.");
    }

    // Check whether password encrypted
    if ("true".equals(ret.get("passwordEncrypted"))) {
        String password = ret.get("password");
        ret.put("password", EncryptUtil.decrypt(password));
        ret.remove("passwordEncrypted");
    }

    logger.info("Connecting to Jdbc with url:{} by user {}", ret.get("url"), ret.get("username"));

    putIfMissing(ret, "driverClassName", "com.mysql.jdbc.Driver");
    putIfMissing(ret, "maxActive", "5");
    putIfMissing(ret, "maxIdle", "5");
    putIfMissing(ret, "maxWait", "1000");
    putIfMissing(ret, "removeAbandoned", "true");
    putIfMissing(ret, "removeAbandonedTimeout", "180");
    putIfMissing(ret, "testOnBorrow", "true");
    putIfMissing(ret, "testWhileIdle", "true");
    putIfMissing(ret, "validationQuery", "select 1");
    return ret;
}

Example 7

Source File: HBaseResourceStore.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private StorageURL buildMetadataUrl(KylinConfig kylinConfig) throws IOException {
    StorageURL url = kylinConfig.getMetadataUrl();
    if (!url.getScheme().equals("hbase"))
        throw new IOException("Cannot create HBaseResourceStore. Url not match. Url: " + url);

    // control timeout for prompt error report
    Map<String, String> newParams = new LinkedHashMap<>();
    newParams.put("hbase.client.scanner.timeout.period", kylinConfig.getHbaseClientScannerTimeoutPeriod());
    newParams.put("hbase.rpc.timeout", kylinConfig.getHbaseRpcTimeout());
    newParams.put("hbase.client.retries.number", kylinConfig.getHbaseClientRetriesNumber());
    newParams.putAll(url.getAllParameters());

    return url.copy(newParams);
}

Example 8

Source File: JDBCResourceStore.java From kylin with Apache License 2.0

5 votes

public JDBCResourceStore(KylinConfig kylinConfig) throws SQLException, IOException {
    super(kylinConfig);
    StorageURL metadataUrl = kylinConfig.getMetadataUrl();
    checkScheme(metadataUrl);
    this.metadataIdentifier = metadataUrl.getIdentifier();
    this.tableNames[0] = metadataIdentifier;
    this.tableNames[1] = metadataIdentifier + "_log";
    this.connectionManager = JDBCConnectionManager.getConnectionManager();
    for (int i = 0; i < tableNames.length; i++) {
        createTableIfNeeded(tableNames[i]);
    }
}

Example 9

Source File: JDBCResourceStore.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public JDBCResourceStore(KylinConfig kylinConfig) throws SQLException, IOException {
    super(kylinConfig);
    StorageURL metadataUrl = kylinConfig.getMetadataUrl();
    checkScheme(metadataUrl);
    this.metadataIdentifier = metadataUrl.getIdentifier();
    this.tableNames[0] = metadataIdentifier;
    this.tableNames[1] = metadataIdentifier + "_log";
    this.connectionManager = JDBCConnectionManager.getConnectionManager();
    for (int i = 0; i < tableNames.length; i++) {
        createTableIfNeeded(tableNames[i]);
    }
}

Example 10

Source File: FileResourceStore.java From kylin with Apache License 2.0

5 votes

public FileResourceStore(KylinConfig kylinConfig) {
    super(kylinConfig);
    root = new File(getPath(kylinConfig)).getAbsoluteFile();
    if (!root.exists())
        throw new IllegalArgumentException(
                "File not exist by '" + kylinConfig.getMetadataUrl() + "': " + root.getAbsolutePath());
}

Example 11

Source File: JDBCConnectionManager.java From kylin with Apache License 2.0

5 votes

private Map<String, String> initDbcpProps(KylinConfig config) {
    // metadataUrl is like "kylin_default_instance@jdbc,url=jdbc:mysql://localhost:3306/kylin,username=root,password=xxx"
    StorageURL metadataUrl = config.getMetadataUrl();
    JDBCResourceStore.checkScheme(metadataUrl);

    LinkedHashMap<String, String> ret = new LinkedHashMap<>(metadataUrl.getAllParameters());
    List<String> mandatoryItems = Arrays.asList("url", "username", PASSWORD);

    for (String item : mandatoryItems) {
        Preconditions.checkNotNull(ret.get(item),
                "Setting item \"" + item + "\" is mandatory for Jdbc connections.");
    }

    // Check whether password encrypted
    if ("true".equals(ret.get("passwordEncrypted"))) {
        String password = ret.get("password");
        ret.put("password", EncryptUtil.decrypt(password));
        ret.remove("passwordEncrypted");
    }

    logger.info("Connecting to Jdbc with url:{} by user {}", ret.get("url"), ret.get("username"));

    putIfMissing(ret, "driverClassName", "com.mysql.jdbc.Driver");
    putIfMissing(ret, "maxActive", "5");
    putIfMissing(ret, "maxIdle", "5");
    putIfMissing(ret, "maxWait", "1000");
    putIfMissing(ret, "removeAbandoned", "true");
    putIfMissing(ret, "removeAbandonedTimeout", "180");
    putIfMissing(ret, "testOnBorrow", "true");
    putIfMissing(ret, "testWhileIdle", "true");
    putIfMissing(ret, "validationQuery", "select 1");
    return ret;
}

Example 12

Source File: HBaseResourceStore.java From kylin with Apache License 2.0

5 votes

private StorageURL buildMetadataUrl(KylinConfig kylinConfig) throws IOException {
    StorageURL url = kylinConfig.getMetadataUrl();
    if (!url.getScheme().equals("hbase"))
        throw new IOException("Cannot create HBaseResourceStore. Url not match. Url: " + url);

    // control timeout for prompt error report
    Map<String, String> newParams = new LinkedHashMap<>();
    newParams.put("hbase.client.scanner.timeout.period", kylinConfig.getHbaseClientScannerTimeoutPeriod());
    newParams.put("hbase.rpc.timeout", kylinConfig.getHbaseRpcTimeout());
    newParams.put("hbase.client.retries.number", kylinConfig.getHbaseClientRetriesNumber());
    newParams.putAll(url.getAllParameters());

    return url.copy(newParams);
}

Example 13

Source File: HDFSResourceStore.java From kylin with Apache License 2.0

4 votes

public HDFSResourceStore(KylinConfig kylinConfig) throws Exception {
    this(kylinConfig, kylinConfig.getMetadataUrl());
}

Example 14

Source File: FileResourceStore.java From Kylin with Apache License 2.0

4 votes

public FileResourceStore(KylinConfig kylinConfig) {
    super(kylinConfig);
    root = new File(kylinConfig.getMetadataUrl()).getAbsoluteFile();
    if (root.exists() == false)
        throw new IllegalArgumentException("File not exist by '" + kylinConfig.getMetadataUrl() + "': " + root.getAbsolutePath());
}

Example 15

Source File: IdentifierFileResourceStore.java From kylin with Apache License 2.0

4 votes

protected String getPath(KylinConfig kylinConfig) {
    StorageURL metadataUrl = kylinConfig.getMetadataUrl();
    Preconditions.checkState(IFILE_SCHEME.equals(metadataUrl.getScheme()));
    return metadataUrl.getParameter("path");
}

Example 16

Source File: IdentifierFileResourceStore.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

protected String getPath(KylinConfig kylinConfig) {
    StorageURL metadataUrl = kylinConfig.getMetadataUrl();
    Preconditions.checkState(IFILE_SCHEME.equals(metadataUrl.getScheme()));
    return metadataUrl.getParameter("path");
}

Example 17

Source File: HDFSResourceStore.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

public HDFSResourceStore(KylinConfig kylinConfig) throws Exception {
    this(kylinConfig, kylinConfig.getMetadataUrl());
}