Java Code Examples for org.apache.kylin.cube.CubeInstance#getFirstSegment()

The following examples show how to use org.apache.kylin.cube.CubeInstance#getFirstSegment() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: RowKeyDecoderTest.java From kylin with Apache License 2.0

5 votes

@Ignore
@Test
public void testDecodeWithSlr() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");

    RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());

    byte[] key = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, -104, -106, -128, 11, 54, -105, 50, 48, 50, 49, 51, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };

    rowKeyDecoder.decode(key);
    List<String> values = rowKeyDecoder.getValues();
    assertEquals("[10000000, " + millis("2012-01-02") + ", 20213, Collectibles, Postcards, US StateCities & Towns, ABIN, 0, -99]", values.toString());
}

Example 2

Source File: BaseCuboidMapperTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testMapperWithNull() throws Exception {
    String cubeName = "test_kylin_cube_with_slr_1_new_segment";
    String segmentName = "20130331080000_20131212080000";
    mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
    mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
    // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
    // metadata);
    mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrances\\NAuction15123456789\\N"));
    List<Pair<Text, Text>> result = mapDriver.run();

    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeInstance cube = cubeMgr.getCube(cubeName);

    assertEquals(1, result.size());
    Text rowkey = result.get(0).getFirst();
    byte[] key = rowkey.getBytes();
    byte[] header = Bytes.head(key, 26);
    byte[] sellerId = Bytes.tail(header, 18);
    byte[] cuboidId = Bytes.head(header, 8);
    byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);

    RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
    decoder.decode(key);
    assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, null, Auction, 0, 15]", decoder.getValues().toString());

    assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
    assertEquals(511, Bytes.toLong(cuboidId));
    assertEquals(22, restKey.length);

    verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "0", "0", "0");
}

Example 3

Source File: BaseCuboidMapperTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testMapperWithHeader() throws Exception {
    String cubeName = "test_kylin_cube_with_slr_1_new_segment";
    String segmentName = "20130331080000_20131212080000";
    mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, cubeName);
    mapDriver.getConfiguration().set(BatchConstants.CFG_CUBE_SEGMENT_NAME, segmentName);
    // mapDriver.getConfiguration().set(BatchConstants.CFG_METADATA_URL,
    // metadata);
    mapDriver.withInput(new Text("key"), new Text("2012-12-15118480Health & BeautyFragrancesWomenAuction15123456789132.33"));
    List<Pair<Text, Text>> result = mapDriver.run();

    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeInstance cube = cubeMgr.getCube(cubeName);

    assertEquals(1, result.size());
    Text rowkey = result.get(0).getFirst();
    byte[] key = rowkey.getBytes();
    byte[] header = Bytes.head(key, 26);
    byte[] sellerId = Bytes.tail(header, 18);
    byte[] cuboidId = Bytes.head(header, 8);
    byte[] restKey = Bytes.tail(key, rowkey.getLength() - 26);

    RowKeyDecoder decoder = new RowKeyDecoder(cube.getFirstSegment());
    decoder.decode(key);
    assertEquals("[123456789, 2012-12-15, 11848, Health & Beauty, Fragrances, Women, Auction, 0, 15]", decoder.getValues().toString());

    assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
    assertEquals(511, Bytes.toLong(cuboidId));
    assertEquals(22, restKey.length);

    verifyMeasures(cube.getDescriptor().getMeasures(), result.get(0).getSecond(), "132.33", "132.33", "132.33");
}

Example 4

Source File: GTScanReqSerDerTest.java From kylin with Apache License 2.0

5 votes

@Test
public void testGTInfo() {
    CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("test_kylin_cube_with_slr_ready");
    CubeSegment segment = cube.getFirstSegment();

    Cuboid baseCuboid = Cuboid.getBaseCuboid(cube.getDescriptor());
    GTInfo info = CubeGridTable.newGTInfo(baseCuboid, new CubeDimEncMap(segment));
    GTInfo.serializer.serialize(info, buffer);
    buffer.flip();

    GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
    this.compareTwoGTInfo(info, sInfo);
}

Example 5

Source File: RowKeySplitterTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testWithSlr() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");

    RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 10, 20);
    // base cuboid rowkey
    byte[] input = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
    rowKeySplitter.split(input, input.length);

    assertEquals(10, rowKeySplitter.getBufferSize());
}

Example 6

Source File: RowKeyEncoderTest.java From kylin with Apache License 2.0

5 votes

@Ignore
@Test
public void testEncodeWithSlr2() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
    // CubeSegment seg = cube.getTheOnlySegment();
    CubeDesc cubeDesc = cube.getDescriptor();
    // String data =
    // "1234567892013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
    String[] data = new String[9];
    data[0] = "123456789";
    data[1] = null;
    data[2] = null;
    data[3] = null;
    data[4] = null;
    data[5] = null;
    data[6] = null;
    data[7] = null;
    data[8] = null;

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
    RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(43 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    byte[] shard = Arrays.copyOfRange(encodedKey, 0, RowConstants.ROWKEY_SHARDID_LEN);
    byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength());
    @SuppressWarnings("unused")
    byte[] sellerId = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), 18 + rowKeyEncoder.getHeaderLength());
    byte[] rest = Arrays.copyOfRange(encodedKey, 4 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    assertEquals(0, Bytes.toShort(shard));
    //assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
    assertEquals(511, Bytes.toLong(cuboidId));
    assertArrayEquals(new byte[] { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, rest);
}

Example 7

Source File: RowKeyEncoderTest.java From kylin with Apache License 2.0

5 votes

@Ignore
@Test
public void testEncodeWithSlr() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
    // CubeSegment seg = cube.getTheOnlySegment();
    CubeDesc cubeDesc = cube.getDescriptor();
    // String data =
    // "1234567892013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
    String[] data = new String[9];
    data[0] = "123456789";
    data[1] = "2012-12-15";
    data[2] = "11848";
    data[3] = "Health & Beauty";
    data[4] = "Fragrances";
    data[5] = "Women";
    data[6] = "FP-GTC";
    data[7] = "0";
    data[8] = "15";

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
    RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(43 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    byte[] shard = Arrays.copyOfRange(encodedKey, 0, RowConstants.ROWKEY_SHARDID_LEN);
    @SuppressWarnings("unused")
    byte[] sellerId = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), 4 + rowKeyEncoder.getHeaderLength());
    byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength());
    byte[] rest = Arrays.copyOfRange(encodedKey, 4 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    assertEquals(0, Bytes.toShort(shard));
    //        assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
    assertEquals(511, Bytes.toLong(cuboidId));
    assertArrayEquals(new byte[] { 11, 55, -13, 49, 49, 56, 52, 56, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest);
}

Example 8

Source File: RowKeyEncoderTest.java From kylin with Apache License 2.0

5 votes

@Test
public void testEncodeWithoutSlr() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
    // CubeSegment seg = cube.getTheOnlySegment();
    CubeDesc cubeDesc = cube.getDescriptor();
    // String data =
    // "2013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
    String[] data = new String[8];
    data[0] = "2012-12-15";
    data[1] = "11848";
    data[2] = "Health & Beauty";
    data[3] = "Fragrances";
    data[4] = "Women";
    data[5] = "FP-GTC";
    data[6] = "0";
    data[7] = "15";

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
    RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength());
    byte[] rest = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), encodedKey.length);
    assertEquals(255, Bytes.toLong(cuboidId));
    assertArrayEquals(new byte[] { 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest);
}

Example 9

Source File: RowKeyDecoderTest.java From kylin with Apache License 2.0

5 votes

@Test
public void testEncodeAndDecodeWithUtf8() throws IOException {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
    CubeDesc cubeDesc = cube.getDescriptor();

    String[] data = new String[8];
    data[0] = "2012-12-15";
    data[1] = "11848";
    data[2] = "Health & Beauty";
    data[3] = "Fragrances";
    data[4] = "Women";
    data[5] = "刊登格式测试";// UTF-8
    data[6] = "0";
    data[7] = "15";

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
    RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length);

    RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
    rowKeyDecoder.decode(encodedKey);
    List<String> values = rowKeyDecoder.getValues();
    assertEquals("[" + millis("2012-12-15") + ", 11848, Health & Beauty, Fragrances, Women, 刊登格式, 0, 15]", values.toString());
}

Example 10

Source File: RowKeyDecoderTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Test
public void testDecodeWithoutSlr() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");

    RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());

    byte[] key = { 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };

    rowKeyDecoder.decode(key);
    List<String> values = rowKeyDecoder.getValues();
    assertEquals("[" + millis("2012-12-15") + ", 11848, Health & Beauty, Fragrances, Women, FP-GTC, 0, 15]", values.toString());
}

Example 11

Source File: RowKeySplitterTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testWithoutSlr() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");

    RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 10, 20);
    // base cuboid rowkey
    byte[] input = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
    rowKeySplitter.split(input, input.length);

    assertEquals(9, rowKeySplitter.getBufferSize());
}

Example 12

Source File: GTScanReqSerDerTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Test
public void testGTInfo() {
    CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("test_kylin_cube_with_slr_ready");
    CubeSegment segment = cube.getFirstSegment();

    Cuboid baseCuboid = Cuboid.getBaseCuboid(cube.getDescriptor());
    GTInfo info = CubeGridTable.newGTInfo(baseCuboid, new CubeDimEncMap(segment));
    GTInfo.serializer.serialize(info, buffer);
    buffer.flip();

    GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
    this.compareTwoGTInfo(info, sInfo);
}

Example 13

Source File: RowKeyDecoderTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testEncodeAndDecodeWithUtf8() throws IOException {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
    CubeDesc cubeDesc = cube.getDescriptor();

    byte[][] data = new byte[8][];
    data[0] = Bytes.toBytes("2012-12-15");
    data[1] = Bytes.toBytes("11848");
    data[2] = Bytes.toBytes("Health & Beauty");
    data[3] = Bytes.toBytes("Fragrances");
    data[4] = Bytes.toBytes("Women");
    data[5] = Bytes.toBytes("刊登格式测试");// UTF-8
    data[6] = Bytes.toBytes("0");
    data[7] = Bytes.toBytes("15");

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
    AbstractRowKeyEncoder rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(30, encodedKey.length);

    RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
    rowKeyDecoder.decode(encodedKey);
    List<String> names = rowKeyDecoder.getNames(null);
    List<String> values = rowKeyDecoder.getValues();
    assertEquals("[CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString());
    assertEquals("[2012-12-15, 11848, Health & Beauty, Fragrances, Women, 刊登格式, 0, 15]", values.toString());
}

Example 14

Source File: RowKeySplitterTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Test
public void testWithSlr() throws Exception {
    //has shard
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");

    RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 11, 20);
    // base cuboid rowkey
    byte[] input = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, -104, -106, -128, 11, 54, -105, 55, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };
    rowKeySplitter.split(input);

    assertEquals(11, rowKeySplitter.getBufferSize());
}

Example 15

Source File: RowKeyEncoderTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Ignore
@Test
public void testEncodeWithSlr2() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
    // CubeSegment seg = cube.getTheOnlySegment();
    CubeDesc cubeDesc = cube.getDescriptor();
    // String data =
    // "1234567892013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
    String[] data = new String[9];
    data[0] = "123456789";
    data[1] = null;
    data[2] = null;
    data[3] = null;
    data[4] = null;
    data[5] = null;
    data[6] = null;
    data[7] = null;
    data[8] = null;

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
    RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(43 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    byte[] shard = Arrays.copyOfRange(encodedKey, 0, RowConstants.ROWKEY_SHARDID_LEN);
    byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength());
    @SuppressWarnings("unused")
    byte[] sellerId = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), 18 + rowKeyEncoder.getHeaderLength());
    byte[] rest = Arrays.copyOfRange(encodedKey, 4 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    assertEquals(0, Bytes.toShort(shard));
    //assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
    assertEquals(511, Bytes.toLong(cuboidId));
    assertArrayEquals(new byte[] { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, rest);
}

Example 16

Source File: RowKeyEncoderTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Ignore
@Test
public void testEncodeWithSlr() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
    // CubeSegment seg = cube.getTheOnlySegment();
    CubeDesc cubeDesc = cube.getDescriptor();
    // String data =
    // "1234567892013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
    String[] data = new String[9];
    data[0] = "123456789";
    data[1] = "2012-12-15";
    data[2] = "11848";
    data[3] = "Health & Beauty";
    data[4] = "Fragrances";
    data[5] = "Women";
    data[6] = "FP-GTC";
    data[7] = "0";
    data[8] = "15";

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
    RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(43 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    byte[] shard = Arrays.copyOfRange(encodedKey, 0, RowConstants.ROWKEY_SHARDID_LEN);
    @SuppressWarnings("unused")
    byte[] sellerId = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), 4 + rowKeyEncoder.getHeaderLength());
    byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength());
    byte[] rest = Arrays.copyOfRange(encodedKey, 4 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    assertEquals(0, Bytes.toShort(shard));
    //        assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
    assertEquals(511, Bytes.toLong(cuboidId));
    assertArrayEquals(new byte[] { 11, 55, -13, 49, 49, 56, 52, 56, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest);
}

Example 17

Source File: RowKeyEncoderTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Test
public void testEncodeWithoutSlr() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
    // CubeSegment seg = cube.getTheOnlySegment();
    CubeDesc cubeDesc = cube.getDescriptor();
    // String data =
    // "2013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
    String[] data = new String[8];
    data[0] = "2012-12-15";
    data[1] = "11848";
    data[2] = "Health & Beauty";
    data[3] = "Fragrances";
    data[4] = "Women";
    data[5] = "FP-GTC";
    data[6] = "0";
    data[7] = "15";

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
    RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length);
    byte[] cuboidId = Arrays.copyOfRange(encodedKey, RowConstants.ROWKEY_SHARDID_LEN, rowKeyEncoder.getHeaderLength());
    byte[] rest = Arrays.copyOfRange(encodedKey, rowKeyEncoder.getHeaderLength(), encodedKey.length);
    assertEquals(255, Bytes.toLong(cuboidId));
    assertArrayEquals(new byte[] { 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 }, rest);
}

Example 18

Source File: RowKeyDecoderTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Test
public void testEncodeAndDecodeWithUtf8() throws IOException {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");
    CubeDesc cubeDesc = cube.getDescriptor();

    String[] data = new String[8];
    data[0] = "2012-12-15";
    data[1] = "11848";
    data[2] = "Health & Beauty";
    data[3] = "Fragrances";
    data[4] = "Women";
    data[5] = "刊登格式测试";// UTF-8
    data[6] = "0";
    data[7] = "15";

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
    RowKeyEncoder rowKeyEncoder = new RowKeyEncoder(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(22 + rowKeyEncoder.getHeaderLength(), encodedKey.length);

    RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());
    rowKeyDecoder.decode(encodedKey);
    List<String> values = rowKeyDecoder.getValues();
    assertEquals("[" + millis("2012-12-15") + ", 11848, Health & Beauty, Fragrances, Women, 刊登格式, 0, 15]", values.toString());
}

Example 19

Source File: RowKeyDecoderTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Ignore
@Test
public void testDecodeWithSlr() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");

    RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());

    byte[] key = { 0, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, -104, -106, -128, 11, 54, -105, 50, 48, 50, 49, 51, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };

    rowKeyDecoder.decode(key);
    List<String> values = rowKeyDecoder.getValues();
    assertEquals("[10000000, " + millis("2012-01-02") + ", 20213, Collectibles, Postcards, US StateCities & Towns, ABIN, 0, -99]", values.toString());
}

Example 20

Source File: RowKeyDecoderTest.java From Kylin with Apache License 2.0

4 votes

@Test
public void testDecodeWithSlr() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");

    RowKeyDecoder rowKeyDecoder = new RowKeyDecoder(cube.getFirstSegment());

    byte[] key = { 0, 0, 0, 0, 0, 0, 1, -1, 49, 48, 48, 48, 48, 48, 48, 48, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 11, 54, -105, 55, 13, 71, 114, 65, 66, 73, 78, 9, 9, 9, 9, 9, 9, 9, 9, 0, 10, 0 };

    rowKeyDecoder.decode(key);
    List<String> names = rowKeyDecoder.getNames(null);
    List<String> values = rowKeyDecoder.getValues();

    assertEquals("[SELLER_ID, CAL_DT, LEAF_CATEG_ID, META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME, LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD]", names.toString());
    assertEquals("[10000000, 2012-01-02, 20213, Collectibles, Postcards, US StateCities & Towns, ABIN, 0, -99]", values.toString());

}