Java Code Examples for org.apache.flink.runtime.state.filesystem.FileStateHandle

The following examples show how to use org.apache.flink.runtime.state.filesystem.FileStateHandle. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: SavepointV2Serializer.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static void serializeStreamStateHandle(
		StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);

	} else if (stateHandle instanceof FileStateHandle) {
		dos.writeByte(FILE_STREAM_STATE_HANDLE);
		FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
		dos.writeLong(stateHandle.getStateSize());
		dos.writeUTF(fileStateHandle.getFilePath().toString());

	} else if (stateHandle instanceof ByteStreamStateHandle) {
		dos.writeByte(BYTE_STREAM_STATE_HANDLE);
		ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
		dos.writeUTF(byteStreamStateHandle.getHandleName());
		byte[] internalData = byteStreamStateHandle.getData();
		dos.writeInt(internalData.length);
		dos.write(byteStreamStateHandle.getData());
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
	}

	dos.flush();
}
 
Example 2
Source Project: Flink-CEPplus   Source File: SavepointV2Serializer.java    License: Apache License 2.0 6 votes vote down vote up
public static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
	final int type = dis.read();
	if (NULL_HANDLE == type) {
		return null;
	} else if (FILE_STREAM_STATE_HANDLE == type) {
		long size = dis.readLong();
		String pathString = dis.readUTF();
		return new FileStateHandle(new Path(pathString), size);
	} else if (BYTE_STREAM_STATE_HANDLE == type) {
		String handleName = dis.readUTF();
		int numBytes = dis.readInt();
		byte[] data = new byte[numBytes];
		dis.readFully(data);
		return new ByteStreamStateHandle(handleName, data);
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
	}
}
 
Example 3
Source Project: Flink-CEPplus   Source File: SavepointV1Serializer.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
	int type = dis.read();
	if (NULL_HANDLE == type) {
		return null;
	} else if (FILE_STREAM_STATE_HANDLE == type) {
		long size = dis.readLong();
		String pathString = dis.readUTF();
		return new FileStateHandle(new Path(pathString), size);
	} else if (BYTE_STREAM_STATE_HANDLE == type) {
		String handleName = dis.readUTF();
		int numBytes = dis.readInt();
		byte[] data = new byte[numBytes];
		dis.readFully(data);
		return new ByteStreamStateHandle(handleName, data);
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
	}
}
 
Example 4
Source Project: flink-learning   Source File: MetadataSerializer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 解析 operatorSubtaskState 的 ManagedKeyedState
 *
 * @param operatorSubtaskState operatorSubtaskState
 */
private static void parseManagedKeyedState(OperatorSubtaskState operatorSubtaskState) {
    // 遍历当前 subtask 的 KeyedState
    for (KeyedStateHandle keyedStateHandle : operatorSubtaskState.getManagedKeyedState()) {
        // 本案例针对 Flink RocksDB 的增量 Checkpoint 引发的问题,
        // 因此仅处理 IncrementalRemoteKeyedStateHandle
        if (keyedStateHandle instanceof IncrementalRemoteKeyedStateHandle) {
            // 获取 RocksDB 的 sharedState
            Map<StateHandleID, StreamStateHandle> sharedState =
                    ((IncrementalRemoteKeyedStateHandle) keyedStateHandle).getSharedState();
            // 遍历 sharedState 中所有的 sst 文件,key 为 sst 文件名,value 为对应的 hdfs 文件 Handle
            for (Map.Entry<StateHandleID, StreamStateHandle> entry : sharedState.entrySet()) {
                // 打印 sst 文件名
                System.out.println("sstable 文件名:" + entry.getKey());
                if (entry.getValue() instanceof FileStateHandle) {
                    Path filePath = ((FileStateHandle) entry.getValue()).getFilePath();
                    // 打印 sst 文件对应的 hdfs 文件位置
                    System.out.println("sstable 文件对应的 hdfs 位置:" + filePath.getPath());
                }
            }
        }
    }
}
 
Example 5
Source Project: flink   Source File: SavepointV2Serializer.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static void serializeStreamStateHandle(
		StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);

	} else if (stateHandle instanceof FileStateHandle) {
		dos.writeByte(FILE_STREAM_STATE_HANDLE);
		FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
		dos.writeLong(stateHandle.getStateSize());
		dos.writeUTF(fileStateHandle.getFilePath().toString());

	} else if (stateHandle instanceof ByteStreamStateHandle) {
		dos.writeByte(BYTE_STREAM_STATE_HANDLE);
		ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
		dos.writeUTF(byteStreamStateHandle.getHandleName());
		byte[] internalData = byteStreamStateHandle.getData();
		dos.writeInt(internalData.length);
		dos.write(byteStreamStateHandle.getData());
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
	}

	dos.flush();
}
 
Example 6
Source Project: flink   Source File: SavepointV2Serializer.java    License: Apache License 2.0 6 votes vote down vote up
public static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
	final int type = dis.read();
	if (NULL_HANDLE == type) {
		return null;
	} else if (FILE_STREAM_STATE_HANDLE == type) {
		long size = dis.readLong();
		String pathString = dis.readUTF();
		return new FileStateHandle(new Path(pathString), size);
	} else if (BYTE_STREAM_STATE_HANDLE == type) {
		String handleName = dis.readUTF();
		int numBytes = dis.readInt();
		byte[] data = new byte[numBytes];
		dis.readFully(data);
		return new ByteStreamStateHandle(handleName, data);
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
	}
}
 
Example 7
Source Project: flink   Source File: SavepointV1Serializer.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
	int type = dis.read();
	if (NULL_HANDLE == type) {
		return null;
	} else if (FILE_STREAM_STATE_HANDLE == type) {
		long size = dis.readLong();
		String pathString = dis.readUTF();
		return new FileStateHandle(new Path(pathString), size);
	} else if (BYTE_STREAM_STATE_HANDLE == type) {
		String handleName = dis.readUTF();
		int numBytes = dis.readInt();
		byte[] data = new byte[numBytes];
		dis.readFully(data);
		return new ByteStreamStateHandle(handleName, data);
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
	}
}
 
Example 8
Source Project: flink-learning   Source File: MetadataSerializer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 解析 operatorSubtaskState 的 ManagedKeyedState
 *
 * @param operatorSubtaskState operatorSubtaskState
 */
private static void parseManagedKeyedState(OperatorSubtaskState operatorSubtaskState) {
    // 遍历当前 subtask 的 KeyedState
    for (KeyedStateHandle keyedStateHandle : operatorSubtaskState.getManagedKeyedState()) {
        // 本案例针对 Flink RocksDB 的增量 Checkpoint 引发的问题,
        // 因此仅处理 IncrementalRemoteKeyedStateHandle
        if (keyedStateHandle instanceof IncrementalRemoteKeyedStateHandle) {
            // 获取 RocksDB 的 sharedState
            Map<StateHandleID, StreamStateHandle> sharedState =
                    ((IncrementalRemoteKeyedStateHandle) keyedStateHandle).getSharedState();
            // 遍历 sharedState 中所有的 sst 文件,key 为 sst 文件名,value 为对应的 hdfs 文件 Handle
            for (Map.Entry<StateHandleID, StreamStateHandle> entry : sharedState.entrySet()) {
                // 打印 sst 文件名
                System.out.println("sstable 文件名:" + entry.getKey());
                if (entry.getValue() instanceof FileStateHandle) {
                    Path filePath = ((FileStateHandle) entry.getValue()).getFilePath();
                    // 打印 sst 文件对应的 hdfs 文件位置
                    System.out.println("sstable 文件对应的 hdfs 位置:" + filePath.getPath());
                }
            }
        }
    }
}
 
Example 9
Source Project: Flink-CEPplus   Source File: SavepointV1Serializer.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
public static void serializeStreamStateHandle(
		StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);

	} else if (stateHandle instanceof FileStateHandle) {
		dos.writeByte(FILE_STREAM_STATE_HANDLE);
		FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
		dos.writeLong(stateHandle.getStateSize());
		dos.writeUTF(fileStateHandle.getFilePath().toString());

	} else if (stateHandle instanceof ByteStreamStateHandle) {
		dos.writeByte(BYTE_STREAM_STATE_HANDLE);
		ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
		dos.writeUTF(byteStreamStateHandle.getHandleName());
		byte[] internalData = byteStreamStateHandle.getData();
		dos.writeInt(internalData.length);
		dos.write(byteStreamStateHandle.getData());

	} else {
		throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
	}

	dos.flush();
}
 
Example 10
Source Project: flink-learning   Source File: MetadataSerializer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 解析 operatorSubtaskState 的 ManagedOperatorState
 *
 * @param operatorSubtaskState operatorSubtaskState
 */
private static void parseManagedOperatorState(OperatorSubtaskState operatorSubtaskState) {
    // 遍历当前 subtask 的 OperatorState
    for (OperatorStateHandle operatorStateHandle : operatorSubtaskState.getManagedOperatorState()) {
        StreamStateHandle delegateStateHandle = operatorStateHandle.getDelegateStateHandle();
        if (delegateStateHandle instanceof FileStateHandle) {
            Path filePath = ((FileStateHandle) delegateStateHandle).getFilePath();
            System.out.println(filePath.getPath());
        }
    }
}
 
Example 11
Source Project: flink   Source File: SavepointV1Serializer.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
public static void serializeStreamStateHandle(
		StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);

	} else if (stateHandle instanceof FileStateHandle) {
		dos.writeByte(FILE_STREAM_STATE_HANDLE);
		FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
		dos.writeLong(stateHandle.getStateSize());
		dos.writeUTF(fileStateHandle.getFilePath().toString());

	} else if (stateHandle instanceof ByteStreamStateHandle) {
		dos.writeByte(BYTE_STREAM_STATE_HANDLE);
		ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
		dos.writeUTF(byteStreamStateHandle.getHandleName());
		byte[] internalData = byteStreamStateHandle.getData();
		dos.writeInt(internalData.length);
		dos.write(byteStreamStateHandle.getData());

	} else {
		throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
	}

	dos.flush();
}
 
Example 12
Source Project: flink-learning   Source File: MetadataSerializer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 解析 operatorSubtaskState 的 ManagedOperatorState
 *
 * @param operatorSubtaskState operatorSubtaskState
 */
private static void parseManagedOperatorState(OperatorSubtaskState operatorSubtaskState) {
    // 遍历当前 subtask 的 OperatorState
    for (OperatorStateHandle operatorStateHandle : operatorSubtaskState.getManagedOperatorState()) {
        StreamStateHandle delegateStateHandle = operatorStateHandle.getDelegateStateHandle();
        if (delegateStateHandle instanceof FileStateHandle) {
            Path filePath = ((FileStateHandle) delegateStateHandle).getFilePath();
            System.out.println(filePath.getPath());
        }
    }
}
 
Example 13
Source Project: flink   Source File: MetadataV2V3SerializerBase.java    License: Apache License 2.0 5 votes vote down vote up
static void serializeStreamStateHandle(StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {
	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);

	} else if (stateHandle instanceof RelativeFileStateHandle) {
		dos.writeByte(RELATIVE_STREAM_STATE_HANDLE);
		RelativeFileStateHandle relativeFileStateHandle = (RelativeFileStateHandle) stateHandle;
		dos.writeUTF(relativeFileStateHandle.getRelativePath());
		dos.writeLong(relativeFileStateHandle.getStateSize());
	} else if (stateHandle instanceof FileStateHandle) {
		dos.writeByte(FILE_STREAM_STATE_HANDLE);
		FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
		dos.writeLong(stateHandle.getStateSize());
		dos.writeUTF(fileStateHandle.getFilePath().toString());

	} else if (stateHandle instanceof ByteStreamStateHandle) {
		dos.writeByte(BYTE_STREAM_STATE_HANDLE);
		ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
		dos.writeUTF(byteStreamStateHandle.getHandleName());
		byte[] internalData = byteStreamStateHandle.getData();
		dos.writeInt(internalData.length);
		dos.write(byteStreamStateHandle.getData());
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
	}

	dos.flush();
}
 
Example 14
public RetrievableStreamStateHandle(Path filePath, long stateSize) {
	Preconditions.checkNotNull(filePath);
	this.wrappedStreamStateHandle = new FileStateHandle(filePath, stateSize);
}
 
Example 15
Source Project: flink   Source File: RetrievableStreamStateHandle.java    License: Apache License 2.0 4 votes vote down vote up
public RetrievableStreamStateHandle(Path filePath, long stateSize) {
	Preconditions.checkNotNull(filePath);
	this.wrappedStreamStateHandle = new FileStateHandle(filePath, stateSize);
}
 
Example 16
Source Project: flink   Source File: RetrievableStreamStateHandle.java    License: Apache License 2.0 4 votes vote down vote up
public RetrievableStreamStateHandle(Path filePath, long stateSize) {
	Preconditions.checkNotNull(filePath);
	this.wrappedStreamStateHandle = new FileStateHandle(filePath, stateSize);
}