Java Code Examples for org.apache.kylin.common.util.StringSplitter

The following examples show how to use org.apache.kylin.common.util.StringSplitter. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: kylin   Source File: TableDesc.java    License: Apache License 2.0 6 votes vote down vote up
public void setName(String name) {
    if (name != null) {
        String[] splits = StringSplitter.split(name, ".");
        if (splits.length == 2) {
            this.setDatabase(splits[0]);
            this.name = splits[1];
        } else if (splits.length == 1) {
            this.name = splits[0];
        }
    } else {
        this.name = null;
    }
    if (identity != null) {
        setIdentity();
    }
}
 
Example 2
Source Project: Kylin   Source File: AbstractHadoopJob.java    License: Apache License 2.0 6 votes vote down vote up
public void addInputDirs(String input, Job job) throws IOException {
    for (String inp : StringSplitter.split(input, ",")) {
        inp = inp.trim();
        if (inp.endsWith("/*")) {
            inp = inp.substring(0, inp.length() - 2);
            FileSystem fs = FileSystem.get(job.getConfiguration());
            Path path = new Path(inp);
            FileStatus[] fileStatuses = fs.listStatus(path);
            boolean hasDir = false;
            for (FileStatus stat : fileStatuses) {
                if (stat.isDirectory() && !stat.getPath().getName().startsWith("_")) {
                    hasDir = true;
                    addInputDirs(stat.getPath().toString(), job);
                }
            }
            if (fileStatuses.length > 0 && !hasDir) {
                addInputDirs(path.toString(), job);
            }
        } else {
            logger.debug("Add input " + inp);
            FileInputFormat.addInputPath(job, new Path(inp));
        }
    }
}
 
Example 3
Source Project: Kylin   Source File: PartitionDesc.java    License: Apache License 2.0 6 votes vote down vote up
public void init(Map<String, TableDesc> tables) {
    if (StringUtils.isNotEmpty(partitionDateColumn)) {
        partitionDateColumn = partitionDateColumn.toUpperCase();

        String[] columns = StringSplitter.split(partitionDateColumn, ".");

        if (null != columns && columns.length == 3) {
            String tableName = columns[0].toUpperCase() + "." + columns[1].toUpperCase();

            TableDesc table = tables.get(tableName);
            ColumnDesc col = table.findColumnByName(columns[2]);
            if (col != null) {
                partitionDateColumnRef = new TblColRef(col);
            } else {
                throw new IllegalStateException("The column '" + partitionDateColumn + "' provided in 'partition_date_column' doesn't exist.");
            }
        } else {
            throw new IllegalStateException("The 'partition_date_column' format is invalid: " + partitionDateColumn + ", it should be {db}.{table}.{column}.");
        }
    }
}
 
Example 4
Source Project: kylin-on-parquet-v2   Source File: TableDesc.java    License: Apache License 2.0 5 votes vote down vote up
public void setName(String name) {
    if (name != null) {
        String[] splits = StringSplitter.split(name, ".");
        if (splits.length == 2) {
            this.setDatabase(splits[0]);
            this.name = splits[1];
        } else if (splits.length == 1) {
            this.name = splits[0];
        }
    } else {
        this.name = null;
    }
}
 
Example 5
Source Project: kylin-on-parquet-v2   Source File: DFSFileTableReader.java    License: Apache License 2.0 5 votes vote down vote up
private String[] split(String line, String delim) {
    // FIXME CVS line should be parsed considering escapes
    String[] str = StringSplitter.split(line, delim);

    // un-escape CSV
    if (DFSFileTable.DELIM_COMMA.equals(delim)) {
        for (int i = 0; i < str.length; i++) {
            str[i] = unescapeCsv(str[i]);
        }
    }

    return str;
}
 
Example 6
Source Project: kylin-on-parquet-v2   Source File: DFSFileTableReader.java    License: Apache License 2.0 5 votes vote down vote up
private String autoDetectDelim(String line) {
    if (expectedColumnNumber > 0) {
        for (String delim : DETECT_DELIMS) {
            if (StringSplitter.split(line, delim).length == expectedColumnNumber) {
                logger.info("Auto detect delim to be '" + delim + "', split line to " + expectedColumnNumber + " columns -- " + line);
                return delim;
            }
        }
    }

    logger.info("Auto detect delim to be null, will take THE-WHOLE-LINE as a single value, for " + filePath);
    return null;
}
 
Example 7
Source Project: kylin   Source File: DFSFileTableReader.java    License: Apache License 2.0 5 votes vote down vote up
private String[] split(String line, String delim) {
    // FIXME CVS line should be parsed considering escapes
    String[] str = StringSplitter.split(line, delim);

    // un-escape CSV
    if (DFSFileTable.DELIM_COMMA.equals(delim)) {
        for (int i = 0; i < str.length; i++) {
            str[i] = unescapeCsv(str[i]);
        }
    }

    return str;
}
 
Example 8
Source Project: kylin   Source File: DFSFileTableReader.java    License: Apache License 2.0 5 votes vote down vote up
private String autoDetectDelim(String line) {
    if (expectedColumnNumber > 0) {
        for (String delim : DETECT_DELIMS) {
            if (StringSplitter.split(line, delim).length == expectedColumnNumber) {
                logger.info("Auto detect delim to be '" + delim + "', split line to " + expectedColumnNumber + " columns -- " + line);
                return delim;
            }
        }
    }

    logger.info("Auto detect delim to be null, will take THE-WHOLE-LINE as a single value, for " + filePath);
    return null;
}
 
Example 9
Source Project: Kylin   Source File: FileTableReader.java    License: Apache License 2.0 5 votes vote down vote up
private String[] split(String line, String delim) {
    // FIXME CVS line should be parsed considering escapes
    String str[] = StringSplitter.split(line, delim);

    // un-escape CSV
    if (ReadableTable.DELIM_COMMA.equals(delim)) {
        for (int i = 0; i < str.length; i++) {
            str[i] = unescapeCsv(str[i]);
        }
    }

    return str;
}
 
Example 10
Source Project: Kylin   Source File: FileTableReader.java    License: Apache License 2.0 5 votes vote down vote up
private String autoDetectDelim(String line) {
    if (expectedColumnNumber > 0) {
        for (String delim : DETECT_DELIMS) {
            if (StringSplitter.split(line, delim).length == expectedColumnNumber) {
                logger.info("Auto detect delim to be '" + delim + "', split line to " + expectedColumnNumber + " columns -- " + line);
                return delim;
            }
        }
    }

    logger.info("Auto detect delim to be null, will take THE-WHOLE-LINE as a single value, for " + filePath);
    return null;
}
 
Example 11
Source Project: Kylin   Source File: TableDesc.java    License: Apache License 2.0 5 votes vote down vote up
public void setName(String name) {
    if (name != null) {
        String[] splits = StringSplitter.split(name, ".");
        if (splits.length == 2) {
            this.setDatabase(splits[0]);
            this.name = splits[1];
        } else if (splits.length == 1) {
            this.name = splits[0];
        }
    } else {
        this.name = name;
    }
}
 
Example 12
Source Project: kylin-on-parquet-v2   Source File: AbstractHadoopJob.java    License: Apache License 2.0 4 votes vote down vote up
public static int addInputDirs(String input, Job job) throws IOException {
    int folderNum = addInputDirs(StringSplitter.split(input, ","), job);
    logger.info("Number of added folders:" + folderNum);
    return folderNum;
}
 
Example 13
Source Project: kylin   Source File: AbstractHadoopJob.java    License: Apache License 2.0 4 votes vote down vote up
public static int addInputDirs(String input, Job job) throws IOException {
    int folderNum = addInputDirs(StringSplitter.split(input, ","), job);
    logger.info("Number of added folders:" + folderNum);
    return folderNum;
}