com.esotericsoftware.kryo.Kryo Java Examples

public void write(Kryo kryo, Output output, ArrayMetaData<short[]> object) {
    try {
        short[] arrData = arrayMetaData.getArrData();
        arrayMetaData.send(output);
        int arrSegNum = arrayMetaData.getSegNum();
        for (int i = 0; i < arrSegNum; i++) {
            int from = arrayMetaData.getFrom(i);
            int to = arrayMetaData.getTo(i);
            for (int j = from; j < to; j++) {
                output.writeShort(arrData[j]);
            }
        }
    } catch (IOException e) {
        LOG.error("double array write exception", e);
        System.exit(1);
    }
}
 

/**
 * Reads a read-metadata from a file or resource.
 * @param whereFrom the file or resource containing the read-metadata.
 * @throws IllegalArgumentException if {@code whereFrom} is {@code null}.
 * @throws UserException if any problem occurred where deserializing.
 * @return never {@code null}.
 */
public static ReadMetadata readStandalone(final String whereFrom) {
    try (final InputStream inputStream = BucketUtils.openFile(whereFrom);
         final Input input = new Input(inputStream)) {
        final Kryo kryo = new Kryo();
        final Serializer serializer = new Serializer();
        final String magicString = input.readString();
        if (!Objects.equals(MAGIC_STRING, magicString)) {
            throw new UserException.BadInput("Bad file format in " + whereFrom +
                    "; it does not seem to be a valid read-metadata serialization");
        }
        final String versionString = input.readString();
        if (!Objects.equals(VERSION_STRING, versionString)) {
            throw new UserException.BadInput("Bad file format in " + whereFrom +
                    "; it contains an incompatible version " + versionString + "(expected: " + VERSION_STRING + ")");
        }
        final ReadMetadata result = serializer.read(kryo, input, ReadMetadata.class);
        if (result == null) {
            throw new UserException.BadInput("Missing read-metadata in " + whereFrom);
        }
        return result;

    } catch (final Exception ex) {
        throw new UserException.CouldNotCreateOutputFile(whereFrom, ex);
    }
}
 

@Override
public byte[] serialize(Object entry) {
    if (entry == null) {
        return ArrayUtils.EMPTY_BYTE_ARRAY;
    }

    Kryo kryo = kryoPool.borrow();
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream(BUFFER_SIZE);
    Output output = new Output(outputStream);

    if (type == null) {
        kryo.writeClassAndObject(output, entry);
    } else {
        kryo.writeObject(output, entry);
    }
    kryoPool.release(kryo);
    output.flush();
    byte[] result = outputStream.toByteArray();
    output.close();
    return result;
}
 

@Test
public void testSerialize() throws Exception {
    final PSTree tree = new PSTree(1);
    tree.addNode(2, "n2", 1, 0, "none");
    tree.addNode(3, "n3", 1, 0, "none");
    tree.addNode(4, "n4", 2, 0, "none");

    try {
        final File tempFile = createTempFile("test", ".dat");
        final Kryo kryo = new Kryo();
        kryo.setReferences(false);
        Output output = new Output(new FileOutputStream(tempFile));
        kryo.writeObject(output, tree);
        output.close();

        final Input input = new Input(new FileInputStream(tempFile));
        final PSTree treeIn = kryo.readObject(input, PSTree.class);
        Assert.assertEquals(treeIn, tree);
    } catch (FileNotFoundException e) {
        throw new IOException("Error with Kryo IO", e);
    }
}
 

/**
 * Creates a Kryo serializer and writes the default registrations out to a
 * comma separated file with one entry per line:
 *
 * <pre>
 * id,class
 * </pre>
 *
 * <p>The produced file is used to check that the registered IDs don't change
 * in future Flink versions.
 *
 * <p>This method is not used in the tests, but documents how the test file
 * has been created and can be used to re-create it if needed.
 *
 * @param filePath File path to write registrations to
 */
private void writeDefaultKryoRegistrations(String filePath) throws IOException {
	final File file = new File(filePath);
	if (file.exists()) {
		assertTrue(file.delete());
	}

	final Kryo kryo = new KryoSerializer<>(Integer.class, new ExecutionConfig()).getKryo();
	final int nextId = kryo.getNextRegistrationId();

	try (BufferedWriter writer = new BufferedWriter(new FileWriter(file))) {
		for (int i = 0; i < nextId; i++) {
			Registration registration = kryo.getRegistration(i);
			String str = registration.getId() + "," + registration.getType().getName();
			writer.write(str, 0, str.length());
			writer.newLine();
		}

		System.out.println("Created file with registrations at " + file.getAbsolutePath());
	}
}
 

/**
 * Ensure that the ProfilePeriod can undergo Kryo serialization which
 * occurs when the Profiler is running in Storm.
 */
@Test
public void testKryoSerialization() {
  ProfilePeriod expected = ProfilePeriod.fromTimestamp(AUG2016, 1, TimeUnit.HOURS);
  Kryo kryo = new Kryo();

  // serialize
  ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
  Output output = new Output(byteStream);
  kryo.writeObject(output, expected);

  // validate serialization
  byte[] bits = output.toBytes();
  assertNotNull(bits);

  // deserialize
  Input input = new Input(new ByteArrayInputStream(bits));
  ProfilePeriod actual = kryo.readObject(input, ProfilePeriod.class);

  // validate deserialization
  assertNotNull(actual);
  assertEquals(expected, actual);
}
 

/**
 * 序列化
 *
 * @param obj 需要序更列化的对象
 * @return 序列化后的byte 数组
 * @throws TransactionException
 */
@Override
public byte[] serialize(Object obj) throws TransactionException {
    byte[] bytes;
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    try {
        //获取kryo对象
        Kryo kryo = new Kryo();
        Output output = new Output(outputStream);
        kryo.writeObject(output, obj);
        bytes = output.toBytes();
        output.flush();
    } catch (Exception ex) {
        throw new TransactionException("kryo serialize error" + ex.getMessage());
    } finally {
        try {
            outputStream.flush();
            outputStream.close();
        } catch (IOException e) {

        }
    }
    return bytes;
}
 

@SuppressWarnings("unchecked")
@Override
public void readFields(final DataInput input) throws IOException {
    try {
        Kryo kryo = new Kryo();
        kryo.register(HBaseEdge.class, new HBaseEdgeSerializer());
        final ByteArrayInputStream bais = new ByteArrayInputStream(WritableUtils.readCompressedByteArray(input));
        this.edge = kryo.readObject(new Input(bais), HBaseEdge.class);
        Class<? extends Writable> cls = Class.forName(Text.readString(input)).asSubclass(Writable.class);
        Writable writable;
        if (cls.equals(NullWritable.class)) {
            writable = NullWritable.get();
        } else {
            writable = cls.newInstance();
        }
        writable.readFields(input);
        this.value = writable != NullWritable.get() ? (V) writable : null;
    } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) {
        throw new IOException("Failed writable init", e);
    }
}
 

/**
 * This method checkpoints the given operator.
 * @param oper The operator to checkpoint.
 * @param bos The ByteArrayOutputStream which saves the checkpoint data temporarily.
 * @return new operator.
 */
public static <T> T checkpoint(T oper, ByteArrayOutputStream bos) throws Exception
{
  Kryo kryo = new Kryo();

  Output loutput = new Output(bos);
  kryo.writeObject(loutput, oper);
  loutput.close();

  Input lInput = new Input(bos.toByteArray());
  @SuppressWarnings("unchecked")
  T checkPointedOper = kryo.readObject(lInput, (Class<T>)oper.getClass());
  lInput.close();

  return checkPointedOper;
}
 

@Override
public MapMetaData<Byte> read(Kryo kryo, Input input, Class<MapMetaData<Byte>> type) {
    try {
        thatMapMetaData = mapMetaData.recv(input);
        int thatMapSegNum = thatMapMetaData.getSegNum();
        List<Map<String, Byte>> mapDataList = new ArrayList<>(thatMapSegNum);
        thatMapMetaData.setMapDataList(mapDataList);

        for (int i = 0; i < thatMapSegNum; i++) {
            int dataNum = thatMapMetaData.getDataNum(i);
            Map<String, Byte> mapData = new HashMap<>(dataNum);
            mapDataList.add(mapData);
            for (int j = 0; j < dataNum; j++) {
                String key = input.readString();
                Byte val = input.readByte();
                mapData.put(key, val);
            }
        }
    } catch (IOException e) {
        LOG.error("double array read exception", e);
        System.exit(1);
    }
    return thatMapMetaData;
}
 

public static void main(String[] args) {
    Kryo kryo = new Kryo();

    // 序列化
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    Output output = new Output(os);
    TestUser user = new TestUser();
    kryo.writeObject(output, user);
    output.close();
    byte[] bytes = os.toByteArray();

    // 反序列化
    Input input = new Input(new ByteArrayInputStream(bytes));
    user = kryo.readObject(input, TestUser.class);
    input.close();

}
 

/**
 * 根据所注册的Class，初始化Kryo实例
 * 
 * @return
 */
protected Kryo initKryo() {
	Kryo kryo = new Kryo();
	kryo.setReferences(true); 
	kryo.setRegistrationRequired(true); 
	kryo.setInstantiatorStrategy(new StdInstantiatorStrategy());
	Set<Class<?>> keys = propertyMap.keySet();
	for(Class<?> key : keys){
		if(propertyMap.get(key) != null){
			kryo.register(key, propertyMap.get(key));
		} else {
			kryo.register(key);
		}
	}
	
	return kryo;
}
 

@Override
public BulkWriteResult read(Kryo kryo, Input input, Class<BulkWriteResult> type) {
		WriteResult globalStatus = kryo.readObject(input,WriteResult.class);
		int notRunSize = input.readInt();
		IntHashSet notRunRows = new IntHashSet(notRunSize);
		for(int i=0;i<notRunSize;i++){
				notRunRows.add(input.readInt());
		}
		int failedSize = input.readInt();
		IntObjectHashMap<WriteResult> failedRows = new IntObjectHashMap<>(failedSize,0.9f);
		for(int i=0;i<failedSize;i++){
				int k = input.readInt();
				WriteResult result = kryo.readObject(input,WriteResult.class);
				failedRows.put(k,result);
		}
		return new BulkWriteResult(globalStatus,notRunRows,failedRows);
}
 

@Test
public void testKryoInChildClasspath() throws Exception {
	final Class<?> avroClass = AvroKryoSerializerUtils.class;

	final URL avroLocation = avroClass.getProtectionDomain().getCodeSource().getLocation();
	final URL kryoLocation = Kryo.class.getProtectionDomain().getCodeSource().getLocation();

	final ClassLoader parentClassLoader = new FilteredClassLoader(
			avroClass.getClassLoader(), AvroKryoSerializerUtils.class.getName());

	final ClassLoader userAppClassLoader = FlinkUserCodeClassLoaders.childFirst(
			new URL[] { avroLocation, kryoLocation },
			parentClassLoader,
			CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS.defaultValue().split(";"));

	final Class<?> userLoadedAvroClass = Class.forName(avroClass.getName(), false, userAppClassLoader);
	assertNotEquals(avroClass, userLoadedAvroClass);

	// call the 'addAvroGenericDataArrayRegistration(...)' method
	final Method m = userLoadedAvroClass.getMethod("addAvroGenericDataArrayRegistration", LinkedHashMap.class);

	final LinkedHashMap<String, ?> map = new LinkedHashMap<>();
	m.invoke(userLoadedAvroClass.newInstance(), map);

	assertEquals(1, map.size());
}
 

KryoHolder(Kryo kryo) {
            this.kryo = kryo;

            //   register
            this.kryo.register(Arrays.asList("").getClass(), new ArraysAsListSerializer());
            this.kryo.register(Collections.EMPTY_LIST.getClass(), new CollectionsEmptyListSerializer());
            this.kryo.register(Collections.EMPTY_MAP.getClass(), new CollectionsEmptyMapSerializer());
            this.kryo.register(Collections.EMPTY_SET.getClass(), new CollectionsEmptySetSerializer());
            this.kryo.register(Collections.singletonList("").getClass(), new CollectionsSingletonListSerializer());
            this.kryo.register(Collections.singleton("").getClass(), new CollectionsSingletonSetSerializer());
            this.kryo.register(Collections.singletonMap("", "").getClass(), new CollectionsSingletonMapSerializer());
            this.kryo.register(GregorianCalendar.class, new GregorianCalendarSerializer());
            this.kryo.register(InvocationHandler.class, new JdkProxySerializer());
            // register CGLibProxySerializer, works in combination with the appropriate action in handleUnregisteredClass (see below)
            this.kryo.register(CGLibProxySerializer.CGLibProxyMarker.class, new CGLibProxySerializer());
            UnmodifiableCollectionsSerializer.registerSerializers(this.kryo);
            SynchronizedCollectionsSerializer.registerSerializers(this.kryo);

            //其他第三方
//        // joda datetime
//        kryo.register(DateTime.class, new JodaDateTimeSerializer());
//        // wicket
//        kryo.register(MiniMap.class, new MiniMapSerializer());
//        // guava ImmutableList
//        ImmutableListSerializer.registerSerializers(kryo);
        }
 

@Override
public Summary read(Kryo kryo, Input input, Class<Summary> type) {
    Summary summary = new Summary();
    int len = input.readInt();
    summary.value = new float[len];
    for (int i = 0; i < len; i++) {
        summary.value[i] = input.readFloat();
    }

    len = input.readInt();
    summary.rmin = new double[len];
    for (int i = 0; i < len; i++) {
        summary.rmin[i] = input.readDouble();
    }

    len = input.readInt();
    summary.rmax = new double[len];
    for (int i = 0; i < len; i++) {
        summary.rmax[i] = input.readDouble();
    }

    len = input.readInt();
    summary.w = new float[len];
    for (int i = 0; i < len; i++) {
        summary.w[i] = input.readFloat();
    }

    summary.capacity = input.readInt();
    summary.cursor = input.readInt();
    summary.B = input.readDouble();
    summary.exact = input.readBoolean();
    summary.eps = input.readDouble();

    summary.poolMap = new TreeMap<>();
    PoolNode poolNode = summary.getPoolNode(summary.capacity);

    return summary;
}
 

@Override
public void write(final Kryo kryo, final Output output, final ImmutableList<Object> object) {
  output.writeInt(object.size(), true);
  final UnmodifiableIterator iterator = object.iterator();

  while (iterator.hasNext()) {
    final Object value = iterator.next();
    kryo.writeClassAndObject(output, value);
  }
}
 

@Override
public Interpolation.BounceIn copy(Kryo kryo, Interpolation.BounceIn original) {
	try {
		float[] widths = (float[]) fieldW.get(original);
		return new Interpolation.BounceIn(widths.length);
	} catch (IllegalAccessException e) {
		throw new RuntimeException(e);
	}
}
 

private byte[] kryoEncode(Object o) {
    Kryo kryo = createKryo();

    Output output = new Output(bufferSize, getMaxSize());
    kryo.writeClassAndObject(output, o);
    return output.toBytes();
}
 

@Override
public <T> T deSerialize(final byte[] param, final Class<T> clazz) throws HmilyException {
    T object;
    try (ByteArrayInputStream inputStream = new ByteArrayInputStream(param)) {
        Kryo kryo = new Kryo();
        Input input = new Input(inputStream);
        object = kryo.readObject(input, clazz);
        input.close();
    } catch (IOException e) {
        throw new HmilyException("kryo deSerialize error" + e.getMessage());
    }
    return object;
}
 

@Override
public RelOptNamespaceTable read(final Kryo kryo, final Input input, final Class<RelOptNamespaceTable> type) {
  final List<String> path = kryo.readObject(input, ArrayList.class);
  final DremioPrepareTable relOptTable = catalog.getTable(path);
  if(relOptTable == null){
    throw new IllegalStateException("Unable to retrieve table: " + new NamespaceKey(path));
  }
  NamespaceTable namespace = relOptTable.unwrap(NamespaceTable.class);
  return new RelOptNamespaceTable(namespace, cluster);
}
 

private static Kryo newKryo() {
	Kryo kryo = new Kryo();
    kryo.register(RpcBody.class);
    kryo.register(RpcMethod.class);
    kryo.setDefaultSerializer(CompatibleFieldSerializer.class);
    return kryo;
}
 

@Override
public void read(Kryo kryo, Input in) {
  this.offset = Platform.BYTE_ARRAY_OFFSET;
  this.numBytes = in.readInt();
  this.base = new byte[numBytes];
  in.read((byte[]) base);
}
 

/**
 * Restores the checkpointed operator.
 * @param checkPointOper The checkpointed operator.
 * @param bos The ByteArrayOutputStream which saves the checkpoint data temporarily.
 */
@SuppressWarnings({"unchecked", "rawtypes"})
public static <T> T restoreCheckPoint(T checkPointOper, ByteArrayOutputStream bos) throws Exception
{
  Kryo kryo = new Kryo();

  Input lInput = new Input(bos.toByteArray());
  T oper = kryo.readObject(lInput, (Class<T>)checkPointOper.getClass());
  lInput.close();

  return oper;
}
 

public RocksDbCacheOperator(TopologyContext context, String cacheDir) {
    this.stormConf = context.getStormConf();

    this.maxFlushSize = ConfigExtension.getTransactionCacheBatchFlushSize(stormConf);

    Options rocksDbOpt = new Options();
    rocksDbOpt.setCreateMissingColumnFamilies(true).setCreateIfMissing(true);
    long bufferSize =
            ConfigExtension.getTransactionCacheBlockSize(stormConf) != null ? ConfigExtension.getTransactionCacheBlockSize(stormConf) : (1 * SizeUnit.GB);
    rocksDbOpt.setWriteBufferSize(bufferSize);
    int maxBufferNum = ConfigExtension.getTransactionMaxCacheBlockNum(stormConf) != null ? ConfigExtension.getTransactionMaxCacheBlockNum(stormConf) : 3;
    rocksDbOpt.setMaxWriteBufferNumber(maxBufferNum);

    // Config for log of RocksDb
    rocksDbOpt.setMaxLogFileSize(1073741824); // 1G
    rocksDbOpt.setKeepLogFileNum(1);
    rocksDbOpt.setInfoLogLevel(InfoLogLevel.WARN_LEVEL);
    
    try {
        Map<Object, Object> conf = new HashMap<Object, Object>();
        conf.put(ROCKSDB_ROOT_DIR, cacheDir);
        conf.put(ROCKSDB_RESET, true);
        initDir(conf);
        initDb(null, rocksDbOpt);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    kryo = new Kryo();
    output = new Output(200, 2000000000);
    input = new Input(1);

    LOG.info("Finished rocksDb cache init: maxFlushSize={}, bufferSize={}, maxBufferNum={}", maxFlushSize, bufferSize, maxBufferNum);
}
 

@Override
public void write(ObjectDataOutput objectDataOutput, Object data) throws IOException {
    Kryo kryo = new Kryo();

    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    Output output = new Output(bos);
    kryo.writeClassAndObject(output, data);
    objectDataOutput.writeByteArray(output.getBuffer());
}
 

@Test(groups = "sv", dataProvider = "forSerialization")
public void testSerialization(final CpxVariantInducingAssemblyContig cpxVariantInducingAssemblyContig) {
    final ByteArrayOutputStream bos = new ByteArrayOutputStream();
    final Output out = new Output(bos);
    final Kryo kryo = new Kryo();
    kryo.writeClassAndObject(out, cpxVariantInducingAssemblyContig);
    out.flush();

    final ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    final Input in = new Input(bis);
    @SuppressWarnings("unchecked")
    final CpxVariantInducingAssemblyContig roundTrip = (CpxVariantInducingAssemblyContig) kryo.readClassAndObject(in);
    Assert.assertEquals(cpxVariantInducingAssemblyContig, roundTrip);
    Assert.assertEquals(cpxVariantInducingAssemblyContig.hashCode(), roundTrip.hashCode());
}
 

@Override
public void write(Kryo kryo, Output output, AttributeContentEvent event) {		
	output.writeLong(event.learningNodeId, true);
	output.writeInt(event.obsIndex, true);
	output.writeDouble(event.attrVal, PRECISION, true);
	output.writeInt(event.classVal, true);
	output.writeDouble(event.weight, PRECISION, true);
	output.writeBoolean(event.isNominal);
}
 

private void checkKryoInitialized() {
	if (this.kryo == null) {
		this.kryo = new Kryo();

		Kryo.DefaultInstantiatorStrategy instantiatorStrategy = new Kryo.DefaultInstantiatorStrategy();
		instantiatorStrategy.setFallbackInstantiatorStrategy(new StdInstantiatorStrategy());
		kryo.setInstantiatorStrategy(instantiatorStrategy);

		this.kryo.setAsmEnabled(true);
		this.kryo.register(type);
	}
}
 

@SuppressFBWarnings(value = "RV_RETURN_VALUE_IGNORED_BAD_PRACTICE",justification = "Intentional")
public void returnInstance(Kryo kryo){
    /*
     * If the pool is full, then we will allow kryo to run out of scope,
     * which will allow the GC to collect it. Thus, we can suppress
     * the findbugs warning
     */
    if(instances.size()< this.poolSize){
        instances.offer(kryo);
    }

}