org.apache.flink.streaming.connectors.kinesis.proxy.GetShardListResult Java Examples

public static KinesisProxyInterface noShardsFoundForRequestedStreamsBehaviour() {

		return new KinesisProxyInterface() {
			@Override
			public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
				return new GetShardListResult(); // not setting any retrieved shards for result
			}

			@Override
			public String getShardIterator(StreamShardHandle shard, String shardIteratorType, Object startingMarker) {
				return null;
			}

			@Override
			public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
				return null;
			}
		};

	}
 

public List<String> readAllMessages(String streamName) throws Exception {
	KinesisProxyInterface kinesisProxy = KinesisProxy.create(properties);
	Map<String, String> streamNamesWithLastSeenShardIds = new HashMap<>();
	streamNamesWithLastSeenShardIds.put(streamName, null);

	GetShardListResult shardListResult = kinesisProxy.getShardList(streamNamesWithLastSeenShardIds);
	int maxRecordsToFetch = 10;

	List<String> messages = new ArrayList<>();
	// retrieve records from all shards
	for (StreamShardHandle ssh : shardListResult.getRetrievedShardListOfStream(streamName)) {
		String shardIterator = kinesisProxy.getShardIterator(ssh, "TRIM_HORIZON", null);
		GetRecordsResult getRecordsResult = kinesisProxy.getRecords(shardIterator, maxRecordsToFetch);
		List<Record> aggregatedRecords = getRecordsResult.getRecords();
		for (Record record : aggregatedRecords) {
			messages.add(new String(record.getData().array()));
		}
	}
	return messages;
}
 

@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
	GetShardListResult result = new GetShardListResult();
	for (Map.Entry<String, List<StreamShardHandle>> streamsWithShards : streamsWithListOfShards.entrySet()) {
		String streamName = streamsWithShards.getKey();
		for (StreamShardHandle shard : streamsWithShards.getValue()) {
			if (streamNamesWithLastSeenShardIds.get(streamName) == null) {
				result.addRetrievedShardToStream(streamName, shard);
			} else {
				if (StreamShardHandle.compareShardIds(
					shard.getShard().getShardId(), streamNamesWithLastSeenShardIds.get(streamName)) > 0) {
					result.addRetrievedShardToStream(streamName, shard);
				}
			}
		}
	}
	return result;
}
 

@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
	GetShardListResult result = new GetShardListResult();
	for (Map.Entry<String, List<StreamShardHandle>> streamsWithShards : streamsWithListOfShards.entrySet()) {
		String streamName = streamsWithShards.getKey();
		for (StreamShardHandle shard : streamsWithShards.getValue()) {
			if (streamNamesWithLastSeenShardIds.get(streamName) == null) {
				result.addRetrievedShardToStream(streamName, shard);
			} else {
				if (compareShardIds(
					shard.getShard().getShardId(), streamNamesWithLastSeenShardIds.get(streamName)) > 0) {
					result.addRetrievedShardToStream(streamName, shard);
				}
			}
		}
	}
	return result;
}
 

public static KinesisProxyInterface noShardsFoundForRequestedStreamsBehaviour() {

		return new KinesisProxyInterface() {
			@Override
			public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
				return new GetShardListResult(); // not setting any retrieved shards for result
			}

			@Override
			public String getShardIterator(StreamShardHandle shard, String shardIteratorType, Object startingMarker) {
				return null;
			}

			@Override
			public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
				return null;
			}
		};

	}
 

/**
 * A utility function that does the following:
 *
 * <p>1. Find new shards for each stream that we haven't seen before
 * 2. For each new shard, determine whether this consumer subtask should subscribe to them;
 * 	  if yes, it is added to the returned list of shards
 * 3. Update the subscribedStreamsToLastDiscoveredShardIds state so that we won't get shards
 *    that we have already seen before the next time this function is called
 */
public List<StreamShardHandle> discoverNewShardsToSubscribe() throws InterruptedException {

	List<StreamShardHandle> newShardsToSubscribe = new LinkedList<>();

	GetShardListResult shardListResult = kinesis.getShardList(subscribedStreamsToLastDiscoveredShardIds);
	if (shardListResult.hasRetrievedShards()) {
		Set<String> streamsWithNewShards = shardListResult.getStreamsWithRetrievedShards();

		for (String stream : streamsWithNewShards) {
			List<StreamShardHandle> newShardsOfStream = shardListResult.getRetrievedShardListOfStream(stream);
			for (StreamShardHandle newShard : newShardsOfStream) {
				int hashCode = shardAssigner.assign(newShard, totalNumberOfConsumerSubtasks);
				if (isThisSubtaskShouldSubscribeTo(hashCode, totalNumberOfConsumerSubtasks, indexOfThisConsumerSubtask)) {
					newShardsToSubscribe.add(newShard);
				}
			}

			advanceLastDiscoveredShardOfStream(
				stream, shardListResult.getLastSeenShardOfStream(stream).getShard().getShardId());
		}
	}

	return newShardsToSubscribe;
}
 

@Override
public List<String> readAllMessages(String streamName) throws Exception {
	KinesisProxyInterface kinesisProxy = KinesisProxy.create(properties);
	Map<String, String> streamNamesWithLastSeenShardIds = new HashMap<>();
	streamNamesWithLastSeenShardIds.put(streamName, null);

	GetShardListResult shardListResult = kinesisProxy.getShardList(streamNamesWithLastSeenShardIds);
	int maxRecordsToFetch = 10;

	List<String> messages = new ArrayList<>();
	// retrieve records from all shards
	for (StreamShardHandle ssh : shardListResult.getRetrievedShardListOfStream(streamName)) {
		String shardIterator = kinesisProxy.getShardIterator(ssh, "TRIM_HORIZON", null);
		GetRecordsResult getRecordsResult = kinesisProxy.getRecords(shardIterator, maxRecordsToFetch);
		List<Record> aggregatedRecords = getRecordsResult.getRecords();
		for (Record record : aggregatedRecords) {
			messages.add(new String(record.getData().array()));
		}
	}
	return messages;
}
 

@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
	GetShardListResult result = new GetShardListResult();
	for (Map.Entry<String, List<StreamShardHandle>> streamsWithShards : streamsWithListOfShards.entrySet()) {
		String streamName = streamsWithShards.getKey();
		for (StreamShardHandle shard : streamsWithShards.getValue()) {
			if (streamNamesWithLastSeenShardIds.get(streamName) == null) {
				result.addRetrievedShardToStream(streamName, shard);
			} else {
				if (StreamShardHandle.compareShardIds(
					shard.getShard().getShardId(), streamNamesWithLastSeenShardIds.get(streamName)) > 0) {
					result.addRetrievedShardToStream(streamName, shard);
				}
			}
		}
	}
	return result;
}
 

@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
	GetShardListResult result = new GetShardListResult();
	for (Map.Entry<String, List<StreamShardHandle>> streamsWithShards : streamsWithListOfShards.entrySet()) {
		String streamName = streamsWithShards.getKey();
		for (StreamShardHandle shard : streamsWithShards.getValue()) {
			if (streamNamesWithLastSeenShardIds.get(streamName) == null) {
				result.addRetrievedShardToStream(streamName, shard);
			} else {
				if (compareShardIds(
					shard.getShard().getShardId(), streamNamesWithLastSeenShardIds.get(streamName)) > 0) {
					result.addRetrievedShardToStream(streamName, shard);
				}
			}
		}
	}
	return result;
}
 

/**
 * A utility function that does the following:
 *
 * <p>1. Find new shards for each stream that we haven't seen before
 * 2. For each new shard, determine whether this consumer subtask should subscribe to them;
 * 	  if yes, it is added to the returned list of shards
 * 3. Update the subscribedStreamsToLastDiscoveredShardIds state so that we won't get shards
 *    that we have already seen before the next time this function is called
 */
public List<StreamShardHandle> discoverNewShardsToSubscribe() throws InterruptedException {

	List<StreamShardHandle> newShardsToSubscribe = new LinkedList<>();

	GetShardListResult shardListResult = kinesis.getShardList(subscribedStreamsToLastDiscoveredShardIds);
	if (shardListResult.hasRetrievedShards()) {
		Set<String> streamsWithNewShards = shardListResult.getStreamsWithRetrievedShards();

		for (String stream : streamsWithNewShards) {
			List<StreamShardHandle> newShardsOfStream = shardListResult.getRetrievedShardListOfStream(stream);
			for (StreamShardHandle newShard : newShardsOfStream) {
				int hashCode = shardAssigner.assign(newShard, totalNumberOfConsumerSubtasks);
				if (isThisSubtaskShouldSubscribeTo(hashCode, totalNumberOfConsumerSubtasks, indexOfThisConsumerSubtask)) {
					newShardsToSubscribe.add(newShard);
				}
			}

			advanceLastDiscoveredShardOfStream(
				stream, shardListResult.getLastSeenShardOfStream(stream).getShard().getShardId());
		}
	}

	return newShardsToSubscribe;
}
 

@Override
public List<String> readAllMessages(String streamName) throws Exception {
	KinesisProxyInterface kinesisProxy = KinesisProxy.create(properties);
	Map<String, String> streamNamesWithLastSeenShardIds = new HashMap<>();
	streamNamesWithLastSeenShardIds.put(streamName, null);

	GetShardListResult shardListResult = kinesisProxy.getShardList(streamNamesWithLastSeenShardIds);
	int maxRecordsToFetch = 10;

	List<String> messages = new ArrayList<>();
	// retrieve records from all shards
	for (StreamShardHandle ssh : shardListResult.getRetrievedShardListOfStream(streamName)) {
		String shardIterator = kinesisProxy.getShardIterator(ssh, "TRIM_HORIZON", null);
		GetRecordsResult getRecordsResult = kinesisProxy.getRecords(shardIterator, maxRecordsToFetch);
		List<Record> aggregatedRecords = getRecordsResult.getRecords();
		for (Record record : aggregatedRecords) {
			messages.add(new String(record.getData().array()));
		}
	}
	return messages;
}
 

@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
	GetShardListResult result = new GetShardListResult();
	for (Map.Entry<String, List<StreamShardHandle>> streamsWithShards : streamsWithListOfShards.entrySet()) {
		String streamName = streamsWithShards.getKey();
		for (StreamShardHandle shard : streamsWithShards.getValue()) {
			if (streamNamesWithLastSeenShardIds.get(streamName) == null) {
				result.addRetrievedShardToStream(streamName, shard);
			} else {
				if (StreamShardHandle.compareShardIds(
					shard.getShard().getShardId(), streamNamesWithLastSeenShardIds.get(streamName)) > 0) {
					result.addRetrievedShardToStream(streamName, shard);
				}
			}
		}
	}
	return result;
}
 

@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
	GetShardListResult result = new GetShardListResult();
	for (Map.Entry<String, List<StreamShardHandle>> streamsWithShards : streamsWithListOfShards.entrySet()) {
		String streamName = streamsWithShards.getKey();
		for (StreamShardHandle shard : streamsWithShards.getValue()) {
			if (streamNamesWithLastSeenShardIds.get(streamName) == null) {
				result.addRetrievedShardToStream(streamName, shard);
			} else {
				if (compareShardIds(
					shard.getShard().getShardId(), streamNamesWithLastSeenShardIds.get(streamName)) > 0) {
					result.addRetrievedShardToStream(streamName, shard);
				}
			}
		}
	}
	return result;
}
 

/**
 * A utility function that does the following:
 *
 * <p>1. Find new shards for each stream that we haven't seen before
 * 2. For each new shard, determine whether this consumer subtask should subscribe to them;
 * 	  if yes, it is added to the returned list of shards
 * 3. Update the subscribedStreamsToLastDiscoveredShardIds state so that we won't get shards
 *    that we have already seen before the next time this function is called
 */
public List<StreamShardHandle> discoverNewShardsToSubscribe() throws InterruptedException {

	List<StreamShardHandle> newShardsToSubscribe = new LinkedList<>();

	GetShardListResult shardListResult = kinesis.getShardList(subscribedStreamsToLastDiscoveredShardIds);
	if (shardListResult.hasRetrievedShards()) {
		Set<String> streamsWithNewShards = shardListResult.getStreamsWithRetrievedShards();

		for (String stream : streamsWithNewShards) {
			List<StreamShardHandle> newShardsOfStream = shardListResult.getRetrievedShardListOfStream(stream);
			for (StreamShardHandle newShard : newShardsOfStream) {
				int hashCode = shardAssigner.assign(newShard, totalNumberOfConsumerSubtasks);
				if (isThisSubtaskShouldSubscribeTo(hashCode, totalNumberOfConsumerSubtasks, indexOfThisConsumerSubtask)) {
					newShardsToSubscribe.add(newShard);
				}
			}

			advanceLastDiscoveredShardOfStream(
				stream, shardListResult.getLastSeenShardOfStream(stream).getShard().getShardId());
		}
	}

	return newShardsToSubscribe;
}
 

public static KinesisProxyInterface noShardsFoundForRequestedStreamsBehaviour() {

		return new KinesisProxyInterface() {
			@Override
			public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
				return new GetShardListResult(); // not setting any retrieved shards for result
			}

			@Override
			public String getShardIterator(StreamShardHandle shard, String shardIteratorType, Object startingMarker) {
				return null;
			}

			@Override
			public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) {
				return null;
			}
		};

	}
 

@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
	return null;
}
 

@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
	return null;
}
 

@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
	return null;
}
 

@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
	return null;
}
 

@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
	return null;
}
 

@Override
public GetShardListResult getShardList(Map<String, String> streamNamesWithLastSeenShardIds) {
	return null;
}