/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.alibaba.flink.connectors.sls.datastream.source;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.core.io.InputSplit;
import com.alibaba.flink.connectors.common.exception.ErrorUtils;
import com.alibaba.flink.connectors.common.reader.AbstractPartitionNumsListener;
import com.alibaba.flink.connectors.common.reader.Interruptible;
import com.alibaba.flink.connectors.common.reader.RecordReader;
import com.alibaba.flink.connectors.common.util.RetryUtils;
import com.aliyun.openservices.log.common.Consts;
import com.aliyun.openservices.log.common.LogGroupData;
import com.aliyun.openservices.log.common.Shard;
import com.aliyun.openservices.log.exception.LogException;
import com.aliyun.openservices.log.response.BatchGetLogResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* Sls record reader.
*/
public class SlsRecordReader extends AbstractPartitionNumsListener implements RecordReader<List<LogGroupData>, String>,
Interruptible {
private static final Logger LOG = LoggerFactory.getLogger(SlsRecordReader.class);
protected String endPoint = null;
protected String accessKeyId = null;
protected String accessKeySecret = null;
protected String project = null;
protected String logStore = null;
protected int startInSec;
protected int stopInSec;
protected int maxRetryTime = 3;
private int batchGetSize = 10;
private int shardId = -1;
private String lastSuccessfulCursor;
private long lastMonotonyIncreaseProgress = -1;
private String nextBeginCursor;
private String stopCursor;
private String consumerGroup = null;
SlsClientProvider slsClientProvider = null;
private long currentWatermark;
private int lastSuccessMessageTimestamp;
private volatile boolean interrupted = false;
private transient List<LogGroupData> rawLogGroupDatas;
private Configuration properties;
private long lastLogPrintTime = 0L;
private long dataFetchedDelay = 0;
boolean genFetchTask = true;
long mLastFetchRawSize = Long.MAX_VALUE;
long mLastFetchCount = Long.MAX_VALUE;
long mLastFetchTime = Long.MIN_VALUE;
private boolean directMode = false;
private transient Shard shard;
private boolean isReadOnlyShard = false;
private String endCursor;
private long lastPartitionChangedTime = 0L;
public SlsRecordReader(
String endPoint,
String accessKeyId,
String accessKeySecret,
String project,
String logStore,
int startInSec,
int stopInSec,
int maxRetryTime,
int batchGetSize,
List<Shard> initShardList,
Configuration properties,
String consumerGroup) {
this.endPoint = endPoint;
this.accessKeyId = accessKeyId;
this.accessKeySecret = accessKeySecret;
this.project = project;
this.logStore = logStore;
this.startInSec = startInSec;
this.stopInSec = stopInSec;
this.maxRetryTime = maxRetryTime;
this.batchGetSize = batchGetSize;
setInitPartitionCount(null == initShardList ? 0 : initShardList.size());
this.properties = properties;
this.consumerGroup = consumerGroup;
}
public SlsRecordReader(
String endPoint,
Configuration properties,
String project,
String logStore,
int startInSec,
int stopInSec,
int maxRetryTime,
int batchGetSize,
List<Shard> initShardList,
String consumerGroup) {
this.endPoint = endPoint;
this.project = project;
this.logStore = logStore;
this.startInSec = startInSec;
this.stopInSec = stopInSec;
this.maxRetryTime = maxRetryTime;
this.batchGetSize = batchGetSize;
setInitPartitionCount(null == initShardList ? 0 : initShardList.size());
this.properties = properties;
this.consumerGroup = consumerGroup;
}
public List<Shard> getShardsList() throws LogException {
slsClientProvider = getSlsClientProvider();
List<Shard> shards = slsClientProvider.getClient().ListShard(project, logStore).GetShards();
return shards;
}
@Override
public int getPartitionsNums() {
try {
return getShardsList().size();
} catch (LogException e) {
throw new RuntimeException(e);
}
}
@Override
public String getReaderName() {
return "SlsRecordReader-" + String.valueOf(project) + String.valueOf(logStore) + " endPoint:" +
String.valueOf(endPoint);
}
SlsClientProvider getSlsClientProvider(){
if (null == slsClientProvider) {
if (null != accessKeyId && null != accessKeySecret && !accessKeyId.isEmpty() &&
!accessKeySecret.isEmpty()) {
slsClientProvider = new SlsClientProvider(
endPoint,
accessKeyId,
accessKeySecret,
consumerGroup,
directMode);
} else {
slsClientProvider = new SlsClientProvider(
endPoint,
properties,
consumerGroup,
directMode);
}
}
return slsClientProvider;
}
@Override
public void open(
InputSplit split, RuntimeContext context) throws IOException {
SlsInputSplit slsInputSplit = (SlsInputSplit) split;
LOG.info(String.format("open project[%s] logStore[%s],consumer[%s]-%s startTime[%d)", project, logStore,
accessKeyId,
slsInputSplit.toString(), startInSec));
int curRetry = 0;
while (curRetry++ < maxRetryTime) {
try {
List<Shard> shardsList = getShardsList();
if (initPartitionCount != shardsList.size()){
ErrorUtils.throwException(
String.format("Source {%s} partitions number has changed from {%s} to {%s} \n " +
"Wait the failover finish, blink is trying to recovery from " +
"source partition change", getReaderName(),
String.valueOf(initPartitionCount), String.valueOf(getPartitionsNums())));
}
this.shardId = split.getSplitNumber();
for (Shard shard: shardsList) {
if (shard.GetShardId() == this.shardId){
this.shard = shard;
break;
}
}
if (shard.getStatus().equalsIgnoreCase("readonly")) {
LOG.info("ShardId " + shard.GetShardId() + " status:readOnly");
isReadOnlyShard = true;
this.endCursor = getSlsClientProvider().getClient().GetCursor(project, logStore, shardId, Consts
.CursorMode.END).GetCursor();
} else {
LOG.info("ShardId " + shard.GetShardId() + " status:readwrite");
isReadOnlyShard = false;
}
this.nextBeginCursor = getSlsClientProvider().getClient()
.GetCursor(project, logStore, shardId, startInSec)
.GetCursor();
if (stopInSec == Integer.MAX_VALUE) {
this.stopCursor = null;
} else {
this.stopCursor = getSlsClientProvider().getClient()
.GetCursor(project, logStore, shardId, stopInSec).GetCursor();
}
if (consumerGroup == null) {
LOG.info(String.format(
"Open method get init cursor, " +
"project[%s]-logStore[%s]-shardId[%d]-startInSec[%d]-Cursor[%s]",
project,
logStore,
shardId,
startInSec,
nextBeginCursor));
} else {
LOG.info(String.format(
"Open method get init cursor, " +
"project[%s]-logStore[%s]-shardId[%d]-startInSec[%d]-Cursor[%s]-ConsumerGroup[%s]",
project,
logStore,
shardId,
startInSec,
nextBeginCursor,
consumerGroup));
}
break;
} catch (LogException e) {
LOG.error("Error in get shard list", e);
// refresh sts account
getSlsClientProvider().getClient(true, true);
if (curRetry == maxRetryTime) {
ErrorUtils.throwException(
e.getMessage());
}
try {
Thread.sleep(curRetry * 500);
} catch (Exception e1) {
}
}
}
initPartitionNumsListener();
}
@Override
public boolean next() throws IOException, InterruptedException {
List<LogGroupData> datas = new ArrayList<>();
int currRetryTime = 0;
while (datas.size() == 0 && (stopCursor == null || !stopCursor.equals(nextBeginCursor))) {
if (isPartitionChanged()){
ErrorUtils.throwException(
String.format("Source {%s} partitions number has changed from {%s} to {%s} \n " +
"Wait the failover finish, blink is trying to recovery from " +
"source partition change", getReaderName(),
String.valueOf(initPartitionCount), String.valueOf(getPartitionsNums())));
}
if (interrupted) {
return false;
}
if (isReadOnlyShard && nextBeginCursor.equals(endCursor)){
LOG.info(String.format("CurrentRecordRead reached end, " +
"project[%s]-logStore[%s]-shardId[%d]-progress[%d]-delay[%d]-Cursor[%s]-" +
"nextCursor[%s]-EndCursor[%s]", project,
logStore, shardId, lastSuccessMessageTimestamp,
dataFetchedDelay,
lastSuccessfulCursor,
nextBeginCursor,
endCursor));
return false;
}
// 退火算法减轻对服务端的压力
genFetchTask = true;
if (mLastFetchRawSize < 1024 * 1024 && mLastFetchCount < batchGetSize) {
genFetchTask = (System.currentTimeMillis() - mLastFetchTime > 500);
} else if (mLastFetchRawSize < 2 * 1024 * 1024 && mLastFetchCount < batchGetSize) {
genFetchTask = (System.currentTimeMillis() - mLastFetchTime > 200);
} else if (mLastFetchRawSize < 4 * 1024 * 1024 && mLastFetchCount < batchGetSize) {
genFetchTask = (System.currentTimeMillis() - mLastFetchTime > 50);
}
if (genFetchTask) {
try {
BatchGetLogResponse batchGetLogResponse = null;
batchGetLogResponse = getSlsClientProvider().getClient().BatchGetLog(
project,
logStore,
shardId,
batchGetSize,
nextBeginCursor,
stopCursor);
mLastFetchRawSize = batchGetLogResponse.GetRawSize();
mLastFetchCount = batchGetLogResponse.GetCount();
mLastFetchTime = System.currentTimeMillis();
if (batchGetLogResponse.GetCount() > 0) {
lastMonotonyIncreaseProgress = lastSuccessMessageTimestamp;
lastSuccessfulCursor = nextBeginCursor;
lastSuccessMessageTimestamp = getSlsClientProvider().getClient()
.GetCursorTime(project, logStore,
shardId,
batchGetLogResponse.GetNextCursor())
.GetCursorTime();
currentWatermark = lastSuccessMessageTimestamp * 1000L;
dataFetchedDelay = System.currentTimeMillis() - currentWatermark;
datas.addAll(batchGetLogResponse.GetLogGroups());
}
nextBeginCursor = batchGetLogResponse.GetNextCursor();
} catch (LogException e) {
// refresh sts account
getSlsClientProvider().getClient(true, true);
currRetryTime++;
if (currRetryTime <= maxRetryTime) {
Thread.sleep(currRetryTime * 1000);
} else {
throw new RuntimeException("ERROR in next method :"
+ "ErrorCode " + e.GetErrorCode()
+ "ErrorMessage " + e.GetErrorMessage()
+ "RequestId " + e.GetRequestId());
}
}
} else {
try {
Thread.sleep(50);
} catch (Exception e){
//ignore
}
}
}
// reach end.
if (datas.isEmpty()) {
return false;
}
// 每分钟打印一条delay日志
if (System.currentTimeMillis() - lastLogPrintTime >= 60000) {
if (null != consumerGroup) {
// 更新服务端的消费进度
try {
getSlsClientProvider().getClient().UpdateCheckPoint(project, logStore, consumerGroup, shardId,
lastSuccessfulCursor);
} catch (Exception e){
LOG.error("Update CheckPoint Error and Ignore it ", e);
//ignore this exception
}
}
lastLogPrintTime = System.currentTimeMillis();
LOG.info(String.format("Next method get current cursor, " +
"project[%s]-logStore[%s]-shardId[%d]-progress[%d]-delay[%d]-Cursor[%s]-" +
"nextCursor[%s]-EndCursor[%s]", project,
logStore, shardId, lastSuccessMessageTimestamp,
System.currentTimeMillis() - lastSuccessMessageTimestamp,
lastSuccessfulCursor,
nextBeginCursor,
stopCursor));
}
rawLogGroupDatas = datas;
return true;
}
@Override
protected void partitionNumsChangeListener(int newPartitionsCount, int initPartitionCount) {
if (newPartitionsCount > initPartitionCount) {
triggerPartitionNumFailOver();
} else if (newPartitionsCount < initPartitionCount){
if (lastPartitionChangedTime == 0L){
lastPartitionChangedTime = System.currentTimeMillis();
} else if (System.currentTimeMillis() - lastPartitionChangedTime > 5 * 60 * 1000){
triggerPartitionNumFailOver();
}
} else {
lastPartitionChangedTime = 0L;
}
}
@Override
public List<LogGroupData> getMessage() {
return rawLogGroupDatas;
}
@Override
public void close() throws IOException {
destroyPartitionNumsListener();
}
@Override
public void seek(String s) throws IOException {
// 0 means the start of
if (s.equalsIgnoreCase(SlsSourceFunction.NEW_SLS_START_FLAG)) {
try {
this.nextBeginCursor = RetryUtils.executeWithRetry(
() -> getSlsClientProvider().getClient().GetCursor(project, logStore, shardId, Consts.CursorMode.BEGIN).GetCursor(),
maxRetryTime, 10000, false);
} catch (Exception e){
throw ErrorUtils.getException(e.getMessage());
}
} else {
this.nextBeginCursor = s;
}
}
@Override
public String getProgress() throws IOException {
return nextBeginCursor;
}
@Override
public long getDelay() {
return currentWatermark;
}
@Override
public long getFetchedDelay() {
return dataFetchedDelay;
}
@Override
public boolean isHeartBeat() {
return false;
}
@Override
public long getWatermark() {
return currentWatermark;
}
@Override
public void interrupt() {
interrupted = true;
}
public SlsRecordReader setDirectMode(boolean directMode) {
this.directMode = directMode;
return this;
}
}
