Java Code Examples for org.apache.kylin.common.persistence.ResourceStore#listResources()

The following examples show how to use org.apache.kylin.common.persistence.ResourceStore#listResources() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DraftManager.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public List<Draft> list(String project) throws IOException {
    List<Draft> result = new ArrayList<>();
    ResourceStore store = getStore();
    NavigableSet<String> listPath = store.listResources(ResourceStore.DRAFT_RESOURCE_ROOT);
    if (listPath == null)
        return result;
    
    for (String path : listPath) {
        Draft draft = store.getResource(path, DRAFT_SERIALIZER);
        
        if (draft == null)
            continue;
        
        if (project == null || project.equals(draft.getProject()))
            result.add(draft);
    }
    return result;
}
Example 2
Source File: SnapshotManager.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private String checkDupByInfo(SnapshotTable snapshot) throws IOException {
    ResourceStore store = MetadataManager.getInstance(this.config).getStore();
    String resourceDir = snapshot.getResourceDir();
    ArrayList<String> existings = store.listResources(resourceDir);
    if (existings == null)
        return null;

    TableSignature sig = snapshot.getSignature();
    for (String existing : existings) {
        SnapshotTable existingTable = load(existing, false); // skip cache,
        // direct
        // load from
        // store
        if (existingTable != null && sig.equals(existingTable.getSignature()))
            return existing;
    }

    return null;
}
Example 3
Source File: ExtTableSnapshotInfoManager.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private ExtTableSnapshotInfo checkDupByInfo(ExtTableSnapshotInfo snapshot) throws IOException {
    ResourceStore store = TableMetadataManager.getInstance(this.config).getStore();
    String resourceDir = snapshot.getResourceDir();
    NavigableSet<String> existings = store.listResources(resourceDir);
    if (existings == null)
        return null;

    TableSignature sig = snapshot.getSignature();
    for (String existing : existings) {
        ExtTableSnapshotInfo existingSnapshot = load(existing);
        // direct load from store
        if (existingSnapshot != null && sig.equals(existingSnapshot.getSignature()))
            return existingSnapshot;
    }
    return null;
}
Example 4
Source File: DictionaryManager.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private DictionaryInfo checkDupByContent(DictionaryInfo dictInfo, Dictionary<String> dict) throws IOException {
    ResourceStore store = getStore();
    NavigableSet<String> existings = store.listResources(dictInfo.getResourceDir());
    if (existings == null)
        return null;

    logger.info("{} existing dictionaries of the same column", existings.size());
    if (existings.size() > 100) {
        logger.warn("Too many dictionaries under {}, dict count: {}", dictInfo.getResourceDir(), existings.size());
    }

    for (String existing : existings) {
        try {
            if (existing.endsWith(".dict")) {
                DictionaryInfo existingInfo = getDictionaryInfo(existing);
                if (existingInfo != null && dict.equals(existingInfo.getDictionaryObject())) {
                    return existingInfo;
                }
            }
        } catch (Exception ex) {
            logger.error("Tolerate exception checking dup dictionary " + existing, ex);
        }
    }

    return null;
}
Example 5
Source File: DictionaryManager.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private String checkDupByInfo(DictionaryInfo dictInfo) throws IOException {
    ResourceStore store = MetadataManager.getInstance(config).getStore();
    ArrayList<String> existings = store.listResources(dictInfo.getResourceDir());
    if (existings == null)
        return null;

    TableSignature input = dictInfo.getInput();
    for (String existing : existings) {
        DictionaryInfo existingInfo = load(existing, false); // skip cache,
        // direct
        // load from
        // store
        if (input.equals(existingInfo.getInput()))
            return existing;
    }

    return null;
}
Example 6
Source File: DraftManager.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public List<Draft> list(String project) throws IOException {
    List<Draft> result = new ArrayList<>();
    ResourceStore store = getStore();
    NavigableSet<String> listPath = store.listResources(ResourceStore.DRAFT_RESOURCE_ROOT);
    if (listPath == null)
        return result;
    
    for (String path : listPath) {
        Draft draft = store.getResource(path, DRAFT_SERIALIZER);
        
        if (draft == null)
            continue;
        
        if (project == null || project.equals(draft.getProject()))
            result.add(draft);
    }
    return result;
}
Example 7
Source File: SnapshotManager.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private String checkDupByInfo(SnapshotTable snapshot) throws IOException {
    ResourceStore store = getStore();
    String resourceDir = snapshot.getResourceDir();
    NavigableSet<String> existings = store.listResources(resourceDir);
    if (existings == null)
        return null;

    TableSignature sig = snapshot.getSignature();
    for (String existing : existings) {
        SnapshotTable existingTable = load(existing, false); // skip cache,
        // direct load from store
        if (existingTable != null && sig.equals(existingTable.getSignature()))
            return existing;
    }

    return null;
}
Example 8
Source File: DictionaryManager.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private String checkDupByContent(DictionaryInfo dictInfo, Dictionary<?> dict) throws IOException {
    ResourceStore store = MetadataManager.getInstance(config).getStore();
    ArrayList<String> existings = store.listResources(dictInfo.getResourceDir());
    if (existings == null)
        return null;

    for (String existing : existings) {
        logger.info("Checking dup dict :" + existing);
        DictionaryInfo existingInfo = load(existing, true); // skip cache,
        // direct load
        // from store
        if (existingInfo == null)
            logger.info("existingInfo is null");

        if (existingInfo != null && dict.equals(existingInfo.getDictionaryObject()))
            return existing;
    }

    return null;
}
Example 9
Source File: ExtTableSnapshotInfoManager.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private ExtTableSnapshotInfo checkDupByInfo(ExtTableSnapshotInfo snapshot) throws IOException {
    ResourceStore store = TableMetadataManager.getInstance(this.config).getStore();
    String resourceDir = snapshot.getResourceDir();
    NavigableSet<String> existings = store.listResources(resourceDir);
    if (existings == null)
        return null;

    TableSignature sig = snapshot.getSignature();
    for (String existing : existings) {
        ExtTableSnapshotInfo existingSnapshot = load(existing);
        // direct load from store
        if (existingSnapshot != null && sig.equals(existingSnapshot.getSignature()))
            return existingSnapshot;
    }
    return null;
}
Example 10
Source File: SnapshotManager.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private String checkDupByContent(SnapshotTable snapshot) throws IOException {
    ResourceStore store = MetadataManager.getInstance(this.config).getStore();
    String resourceDir = snapshot.getResourceDir();
    ArrayList<String> existings = store.listResources(resourceDir);
    if (existings == null)
        return null;

    for (String existing : existings) {
        SnapshotTable existingTable = load(existing, true); // skip cache, direct load from store
        if (existingTable != null && existingTable.equals(snapshot))
            return existing;
    }

    return null;
}
Example 11
Source File: CubeManagerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGetAllCubes() throws Exception {
    final ResourceStore store = ResourceStore.getStore(getTestConfig());
    final NavigableSet<String> cubePath = store.listResources(ResourceStore.CUBE_RESOURCE_ROOT);
    assertTrue(cubePath.size() > 1);

    final List<CubeInstance> cubes = store.getAllResources(ResourceStore.CUBE_RESOURCE_ROOT,
            CubeManager.CUBE_SERIALIZER);
    assertEquals(cubePath.size(), cubes.size());
}
Example 12
Source File: MetadataVersionRefresher.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public static void collectFiles(ResourceStore src, String path, List<String> ret) throws IOException {
    NavigableSet<String> children = src.listResources(path);

    if (children == null) {
        // case of resource (not a folder)

        ret.add(path);
    } else {
        // case of folder

        for (String child : children) {
            collectFiles(src, child, ret);
        }
    }
}
Example 13
Source File: DictionaryManager.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public void removeDictionaries(String srcTable, String srcCol) throws IOException {
    DictionaryInfo info = new DictionaryInfo();
    info.setSourceTable(srcTable);
    info.setSourceColumn(srcCol);

    ResourceStore store = getStore();
    NavigableSet<String> existings = store.listResources(info.getResourceDir());
    if (existings == null)
        return;

    for (String existing : existings)
        removeDictionary(existing);
}
Example 14
Source File: ExtTableSnapshotInfoManager.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public Set<String> getAllExtSnapshotResPaths() throws IOException {
    Set<String> result = Sets.newHashSet();
    ResourceStore store = TableMetadataManager.getInstance(this.config).getStore();
    Set<String> snapshotTablePaths = store.listResources(ResourceStore.EXT_SNAPSHOT_RESOURCE_ROOT);
    if (snapshotTablePaths == null) {
        return result;
    }
    for (String snapshotTablePath : snapshotTablePaths) {
        Set<String> snapshotPaths = store.listResources(snapshotTablePath);
        if (snapshotPaths != null) {
            result.addAll(snapshotPaths);
        }
    }
    return result;
}
Example 15
Source File: SnapshotManager.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private String checkDupByContent(SnapshotTable snapshot) throws IOException {
    ResourceStore store = getStore();
    String resourceDir = snapshot.getResourceDir();
    NavigableSet<String> existings = store.listResources(resourceDir);
    if (existings == null)
        return null;

    for (String existing : existings) {
        SnapshotTable existingTable = load(existing, true); // skip cache, direct load from store
        if (existingTable != null && existingTable.equals(snapshot))
            return existing;
    }

    return null;
}
Example 16
Source File: CubeManagerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testGetAllCubes() throws Exception {
    final ResourceStore store = ResourceStore.getStore(getTestConfig());
    final NavigableSet<String> cubePath = store.listResources(ResourceStore.CUBE_RESOURCE_ROOT);
    assertTrue(cubePath.size() > 1);

    final List<CubeInstance> cubes = store.getAllResources(ResourceStore.CUBE_RESOURCE_ROOT,
            CubeManager.CUBE_SERIALIZER);
    assertEquals(cubePath.size(), cubes.size());
}
Example 17
Source File: MetadataVersionRefresher.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public static void collectFiles(ResourceStore src, String path, List<String> ret) throws IOException {
    NavigableSet<String> children = src.listResources(path);

    if (children == null) {
        // case of resource (not a folder)

        ret.add(path);
    } else {
        // case of folder

        for (String child : children) {
            collectFiles(src, child, ret);
        }
    }
}
Example 18
Source File: ExtTableSnapshotInfoManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public Set<String> getAllExtSnapshotResPaths() throws IOException {
    Set<String> result = Sets.newHashSet();
    ResourceStore store = TableMetadataManager.getInstance(this.config).getStore();
    Set<String> snapshotTablePaths = store.listResources(ResourceStore.EXT_SNAPSHOT_RESOURCE_ROOT);
    if (snapshotTablePaths == null) {
        return result;
    }
    for (String snapshotTablePath : snapshotTablePaths) {
        Set<String> snapshotPaths = store.listResources(snapshotTablePath);
        if (snapshotPaths != null) {
            result.addAll(snapshotPaths);
        }
    }
    return result;
}
Example 19
Source File: SnapshotManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private String checkDupByContent(SnapshotTable snapshot) throws IOException {
    ResourceStore store = getStore();
    String resourceDir = snapshot.getResourceDir();
    NavigableSet<String> existings = store.listResources(resourceDir);
    if (existings == null)
        return null;

    for (String existing : existings) {
        SnapshotTable existingTable = load(existing, true); // skip cache, direct load from store
        if (existingTable != null && existingTable.equals(snapshot))
            return existing;
    }

    return null;
}
Example 20
Source File: DictionaryManager.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
public void removeDictionaries(String srcTable, String srcCol) throws IOException {
    DictionaryInfo info = new DictionaryInfo();
    info.setSourceTable(srcTable);
    info.setSourceColumn(srcCol);

    ResourceStore store = MetadataManager.getInstance(config).getStore();
    ArrayList<String> existings = store.listResources(info.getResourceDir());
    if (existings == null)
        return;

    for (String existing : existings)
        removeDictionary(existing);
}