org.apache.kylin.common.persistence.Serializer Java Examples

The following examples show how to use org.apache.kylin.common.persistence.Serializer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: ModelServiceTest.java From kylin with Apache License 2.0

6 votes

@Test
public void testSuccessModelUpdate() throws IOException, JobException {
    Serializer<DataModelDesc> serializer = modelService.getDataModelManager().getDataModelSerializer();
    
    List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_inner_join_model", "default", true);
    Assert.assertTrue(dataModelDescs.size() == 1);

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    serializer.serialize(dataModelDescs.get(0), new DataOutputStream(baos));
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    DataModelDesc deserialize = serializer.deserialize(new DataInputStream(bais));

    deserialize.setOwner("somebody");
    DataModelDesc dataModelDesc = modelService.updateModelAndDesc("default", deserialize);
    Assert.assertTrue(dataModelDesc.getOwner().equals("somebody"));
}

Example #2

Source File: ModelServiceTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void testSuccessModelUpdate() throws IOException, JobException {
    Serializer<DataModelDesc> serializer = modelService.getDataModelManager().getDataModelSerializer();
    
    List<DataModelDesc> dataModelDescs = modelService.listAllModels("ci_inner_join_model", "default", true);
    Assert.assertTrue(dataModelDescs.size() == 1);

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    serializer.serialize(dataModelDescs.get(0), new DataOutputStream(baos));
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    DataModelDesc deserialize = serializer.deserialize(new DataInputStream(bais));

    deserialize.setOwner("somebody");
    DataModelDesc dataModelDesc = modelService.updateModelAndDesc("default", deserialize);
    Assert.assertTrue(dataModelDesc.getOwner().equals("somebody"));
}

Example #3

Source File: DstClusterUtil.java From kylin with Apache License 2.0

5 votes

public <T extends RootPersistentEntity> void putMetaResource(String resPath, T obj, Serializer<T> serializer,
        boolean withoutCheck) throws IOException {
    if (ifExecute) {
        if (withoutCheck) {
            resourceStore.putResource(resPath, obj, System.currentTimeMillis(), serializer);
        } else {
            resourceStore.checkAndPutResource(resPath, obj, System.currentTimeMillis(), serializer);
        }
    }
    logger.info("saved resource {}", resPath);
}

Example #4

Source File: TableSchemaUpdaterTest.java From kylin with Apache License 2.0

5 votes

private <T extends RootPersistentEntity> T reinit(T obj, Serializer<T> serializer) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    serializer.serialize(obj, dos);
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    DataInputStream dis = new DataInputStream(bais);
    return serializer.deserialize(dis);
}

Example #5

Source File: ExtendCubeToHybridCLI.java From kylin with Apache License 2.0

5 votes

private void copyAcl(String origCubeId, String newCubeId, String projectName) throws Exception {
    String projectResPath = ProjectInstance.concatResourcePath(projectName);
    Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
    ProjectInstance project = store.getResource(projectResPath, projectSerializer);
    String projUUID = project.getUuid();
    Table aclHtable = null;
    try {
        aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));

        // cube acl
        Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
        if (result.listCells() != null) {
            for (Cell cell : result.listCells()) {
                byte[] family = CellUtil.cloneFamily(cell);
                byte[] column = CellUtil.cloneQualifier(cell);
                byte[] value = CellUtil.cloneValue(cell);

                // use the target project uuid as the parent
                if (Bytes.toString(family).equals(ACL_INFO_FAMILY) && Bytes.toString(column).equals(ACL_INFO_FAMILY_PARENT_COLUMN)) {
                    String valueString = "{\"id\":\"" + projUUID + "\",\"type\":\"org.apache.kylin.metadata.project.ProjectInstance\"}";
                    value = Bytes.toBytes(valueString);
                }
                Put put = new Put(Bytes.toBytes(newCubeId));
                put.add(family, column, value);
                aclHtable.put(put);
            }
        }
    } finally {
        IOUtils.closeQuietly(aclHtable);
    }
}

Example #6

Source File: CacheControllerTest.java From kylin with Apache License 2.0

5 votes

@Test
public void testClearCacheForCubeMigration() throws IOException {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String CUBENAME = "test_kylin_cube_without_slr_desc";

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
    CubeDesc cubeDesc = cubeDescManager.getCubeDesc(CUBENAME);
    DataModelDesc modelDesc = cubeDesc.getModel();
    Map<String, String> tableToProjects = new HashMap<>();
    for (TableRef tableRef : modelDesc.getAllTables()) {
        tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject());
    }

    String uuid = cubeDesc.getUuid();
    String signature = cubeDesc.getSignature();

    assertEquals(cubeDesc.getRetentionRange(), 0);

    //update cubeDesc
    cubeDesc.setRetentionRange(2018);
    cubeDesc.updateRandomUuid();

    //directly update metadata in store to simulate cube migration
    Serializer<CubeDesc> cubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class);
    getStore().checkAndPutResource(cubeDesc.getResourcePath(), cubeDesc, cubeDescSerializer);

    CubeMigrationRequest request = new CubeMigrationRequest();
    request.setCube(cubeDesc.getName());
    request.setModel(modelDesc.getName());
    request.setProject(modelDesc.getProject());
    request.setTableToProjects(tableToProjects);

    cacheController.clearCacheForCubeMigration(request);

    assertEquals(2018, cubeDescManager.getCubeDesc(CUBENAME).getRetentionRange());
    assertEquals(signature, cubeDescManager.getCubeDesc(CUBENAME).getSignature());
    assertNotEquals(uuid, cubeDescManager.getCubeDesc(CUBENAME).getUuid());
}

Example #7

Source File: ExtendCubeToHybridCLI.java From kylin with Apache License 2.0

5 votes

private void copyAcl(String origCubeId, String newCubeId, String projectName) throws Exception {
    String projectResPath = ProjectInstance.concatResourcePath(projectName);
    Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
    ProjectInstance project = store.getResource(projectResPath, projectSerializer);
    String projUUID = project.getUuid();
    Table aclHtable = null;
    try {
        aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));

        // cube acl
        Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
        if (result.listCells() != null) {
            for (Cell cell : result.listCells()) {
                byte[] family = CellUtil.cloneFamily(cell);
                byte[] column = CellUtil.cloneQualifier(cell);
                byte[] value = CellUtil.cloneValue(cell);

                // use the target project uuid as the parent
                if (Bytes.toString(family).equals(ACL_INFO_FAMILY) && Bytes.toString(column).equals(ACL_INFO_FAMILY_PARENT_COLUMN)) {
                    String valueString = "{\"id\":\"" + projUUID + "\",\"type\":\"org.apache.kylin.metadata.project.ProjectInstance\"}";
                    value = Bytes.toBytes(valueString);
                }
                Put put = new Put(Bytes.toBytes(newCubeId));
                put.add(family, column, value);
                aclHtable.put(put);
            }
        }
    } finally {
        IOUtils.closeQuietly(aclHtable);
    }
}

Example #8

Source File: SCCreator.java From kylin with Apache License 2.0

5 votes

private <T extends RootPersistentEntity> void saveSystemCubeMetadataToFile(String fileName, T metadata,
                                                                           Serializer serializer) throws IOException {
    ByteArrayOutputStream buf = new ByteArrayOutputStream();
    DataOutputStream dout = new DataOutputStream(buf);
    serializer.serialize(metadata, dout);
    dout.close();
    buf.close();

    saveToFile(fileName, buf.toString("UTF-8"));
}

Example #9

Source File: SCCreator.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private <T extends RootPersistentEntity> void saveSystemCubeMetadataToFile(String fileName, T metadata,
                                                                           Serializer serializer) throws IOException {
    ByteArrayOutputStream buf = new ByteArrayOutputStream();
    DataOutputStream dout = new DataOutputStream(buf);
    serializer.serialize(metadata, dout);
    dout.close();
    buf.close();

    saveToFile(fileName, buf.toString("UTF-8"));
}

Example #10

Source File: ExtendCubeToHybridCLI.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private void copyAcl(String origCubeId, String newCubeId, String projectName) throws Exception {
    String projectResPath = ProjectInstance.concatResourcePath(projectName);
    Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
    ProjectInstance project = store.getResource(projectResPath, projectSerializer);
    String projUUID = project.getUuid();
    Table aclHtable = null;
    try {
        aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));

        // cube acl
        Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
        if (result.listCells() != null) {
            for (Cell cell : result.listCells()) {
                byte[] family = CellUtil.cloneFamily(cell);
                byte[] column = CellUtil.cloneQualifier(cell);
                byte[] value = CellUtil.cloneValue(cell);

                // use the target project uuid as the parent
                if (Bytes.toString(family).equals(ACL_INFO_FAMILY) && Bytes.toString(column).equals(ACL_INFO_FAMILY_PARENT_COLUMN)) {
                    String valueString = "{\"id\":\"" + projUUID + "\",\"type\":\"org.apache.kylin.metadata.project.ProjectInstance\"}";
                    value = Bytes.toBytes(valueString);
                }
                Put put = new Put(Bytes.toBytes(newCubeId));
                put.add(family, column, value);
                aclHtable.put(put);
            }
        }
    } finally {
        IOUtils.closeQuietly(aclHtable);
    }
}

Example #11

Source File: CacheControllerTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Test
public void testClearCacheForCubeMigration() throws IOException {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String CUBENAME = "test_kylin_cube_without_slr_desc";

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
    CubeDesc cubeDesc = cubeDescManager.getCubeDesc(CUBENAME);
    DataModelDesc modelDesc = cubeDesc.getModel();
    Map<String, String> tableToProjects = new HashMap<>();
    for (TableRef tableRef : modelDesc.getAllTables()) {
        tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject());
    }

    String uuid = cubeDesc.getUuid();
    String signature = cubeDesc.getSignature();

    assertEquals(cubeDesc.getRetentionRange(), 0);

    //update cubeDesc
    cubeDesc.setRetentionRange(2018);
    cubeDesc.updateRandomUuid();

    //directly update metadata in store to simulate cube migration
    Serializer<CubeDesc> cubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class);
    getStore().checkAndPutResource(cubeDesc.getResourcePath(), cubeDesc, cubeDescSerializer);

    CubeMigrationRequest request = new CubeMigrationRequest();
    request.setCube(cubeDesc.getName());
    request.setModel(modelDesc.getName());
    request.setProject(modelDesc.getProject());
    request.setTableToProjects(tableToProjects);

    cacheController.clearCacheForCubeMigration(request);

    assertEquals(2018, cubeDescManager.getCubeDesc(CUBENAME).getRetentionRange());
    assertEquals(signature, cubeDescManager.getCubeDesc(CUBENAME).getSignature());
    assertNotEquals(uuid, cubeDescManager.getCubeDesc(CUBENAME).getUuid());
}

Example #12

Source File: ExtendCubeToHybridCLI.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private void copyAcl(String origCubeId, String newCubeId, String projectName) throws Exception {
    String projectResPath = ProjectInstance.concatResourcePath(projectName);
    Serializer<ProjectInstance> projectSerializer = new JsonSerializer<ProjectInstance>(ProjectInstance.class);
    ProjectInstance project = store.getResource(projectResPath, projectSerializer);
    String projUUID = project.getUuid();
    Table aclHtable = null;
    try {
        aclHtable = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(kylinConfig.getMetadataUrlPrefix() + "_acl"));

        // cube acl
        Result result = aclHtable.get(new Get(Bytes.toBytes(origCubeId)));
        if (result.listCells() != null) {
            for (Cell cell : result.listCells()) {
                byte[] family = CellUtil.cloneFamily(cell);
                byte[] column = CellUtil.cloneQualifier(cell);
                byte[] value = CellUtil.cloneValue(cell);

                // use the target project uuid as the parent
                if (Bytes.toString(family).equals(ACL_INFO_FAMILY) && Bytes.toString(column).equals(ACL_INFO_FAMILY_PARENT_COLUMN)) {
                    String valueString = "{\"id\":\"" + projUUID + "\",\"type\":\"org.apache.kylin.metadata.project.ProjectInstance\"}";
                    value = Bytes.toBytes(valueString);
                }
                Put put = new Put(Bytes.toBytes(newCubeId));
                put.add(family, column, value);
                aclHtable.put(put);
            }
        }
    } finally {
        IOUtils.closeQuietly(aclHtable);
    }
}

Example #13

Source File: DstClusterUtil.java From kylin with Apache License 2.0

4 votes

private <T extends RootPersistentEntity> void putMetaResource(String resPath, T obj, Serializer<T> serializer)
        throws IOException {
    putMetaResource(resPath, obj, serializer, true);
}

Example #14

Source File: DataModelManager.java From kylin with Apache License 2.0

4 votes

public Serializer<DataModelDesc> getDataModelSerializer() {
    return crud.getSerializer();
}

Example #15

Source File: CachedCrudAssist.java From kylin with Apache License 2.0

4 votes

public Serializer<DataModelDesc> getSerializer() {
    return (Serializer<DataModelDesc>) serializer;
}

Example #16

Source File: ITJDBCResourceStoreTest.java From kylin with Apache License 2.0

4 votes

public static Serializer<ByteEntity> getSerializer() {
    return serializer;
}

Example #17

Source File: ITJDBCResourceStoreTest.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

public static Serializer<ByteEntity> getSerializer() {
    return serializer;
}

Example #18

Source File: CachedCrudAssist.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

public Serializer<DataModelDesc> getSerializer() {
    return (Serializer<DataModelDesc>) serializer;
}

Example #19

Source File: DataModelManager.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

public Serializer<DataModelDesc> getDataModelSerializer() {
    return crud.getSerializer();
}