/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.dtstack.jlogstash.inputs;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;

import org.codehaus.plexus.util.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;

import com.dtstack.jlogstash.annotation.Required;
import com.dtstack.jlogstash.decoder.IDecode;
import com.dtstack.jlogstash.decoder.JsonDecoder;
import com.dtstack.jlogstash.decoder.MultilineDecoder;
import com.dtstack.jlogstash.decoder.PlainDecoder;
import com.google.common.collect.Lists;

/**
 * Reason: jlogstash 文件类型的读入插件
 * Date: 2016年11月19日
 * Company: www.dtstack.com
 * @author xuchao
 *
 */
public class File extends BaseInput{
	
	private static final long serialVersionUID = -1822028651072758886L;

	private static final Logger logger = LoggerFactory.getLogger(File.class);
	
	private static String encoding = "UTF-8";
	
	private static Map<String, Object> pathcodecMap = null;

	/**指定文件的行的聚合规则key:文件名称, val:eg:multiline,json,plain*/
	private Map<String, IDecode> codecMap = new ConcurrentHashMap<String, IDecode>();
	
	@Required(required=true)
	private  static  List<String> path;
	
	private  static List<String> exclude;
	
	private  static int maxOpenFiles = 0;//0表示没有上限
		
	private  static String startPosition = "end";//one of ["beginning", "end"]
	
	/**key:文件夹路径, 匹配信息列表,  10s检测一次*/
	private Map<String, List<String>> moniDic = new ConcurrentHashMap<String,  List<String>>();
	
	/**文件当前读取位置点*/
	private ConcurrentHashMap<String, Long> fileCurrPos = new ConcurrentHashMap<String, Long>();
	
	private static String sinceDbPath = "./sincedb.yaml";
		
	private static int sinceDbWriteInterval = 15; //sincedb.yaml 更新频率(时间s)
	
	/**当读取设置行之后更新当前文件读取位置*/
	private static int readLineNum4UpdateMap = 1000;
	
	private static byte delimiter = '\n';
	
	private List<String> realPaths = Lists.newArrayList();
	
	/**默认值:cpu线程数*/
	public static int readFileThreadNum = -1;
		
	private Map<Integer, BlockingQueue<String>> threadReadFileMap = new ConcurrentHashMap<>();
		
	private ConcurrentHashMap<String, Long> monitorMap = new ConcurrentHashMap<String, Long>();
	
	private boolean runFlag = false;
	
	private ExecutorService executor;
	
	private ScheduledExecutorService scheduleExecutor;
	
	private ReentrantLock writeFileLock = new ReentrantLock();

	public File(Map config) {
		super(config);
	}
	
	public void init(){
		
		if(pathcodecMap == null || pathcodecMap.size() == 0){
			return;
		}
		
		if(path == null){
			path = Lists.newArrayList();
		}
		
		//设置codec
		for(Entry<String, Object> entry : pathcodecMap.entrySet()){
			String filePatternName = entry.getKey();
			Object codecInfo = entry.getValue();
			path.add(filePatternName);
			
			if(codecInfo instanceof String){
				if("json".equals(codecInfo)){
		            codecMap.put(filePatternName, new  JsonDecoder());
		        }else if("plain".equals(codecInfo)){
		        	codecMap.put(filePatternName, new PlainDecoder());
		        }else{
		        	logger.error("invalid codec type:{}. please check config!", codecInfo);
		        	System.exit(-1);
		        }
			}else if( codecInfo instanceof Map){
				IDecode multilineDecoder = createMultiLineDecoder((Map)codecInfo);
				codecMap.put(filePatternName, multilineDecoder);
			}else{
				logger.error("invalid codec type:{}, please check param of 'pathcodecMap'.", codecInfo.getClass());
				System.exit(-1);
			}
		}
	}

	@Override
	public void prepare() {
		
		init();
		
		if(path == null || path.size() == 0){
			logger.error("don't set any input file. [path, pathcodecMap] must not be empty at the same time.");
			System.exit(-1);
		}
		
		if(realPaths.size() == 0){
			List<String> ps = generateRealPath();
			realPaths.addAll(ps);
			
			if(maxOpenFiles > 0 && realPaths.size() > maxOpenFiles){
				logger.error("file numbers is exceed, maxOpenFiles is {}", maxOpenFiles);
				System.exit(-1);
			}
		}
		
		if(readFileThreadNum <= 0){
			readFileThreadNum = Runtime.getRuntime().availableProcessors();
		}
		
		checkoutSinceDb();
		filterFinishFile();
		ReadLineUtil.setDelimiter(delimiter);
		
		for(String fileStr : realPaths){
			addFile(fileStr);
		}
		
		runFlag = true;
	}
	
	/**
	 * 获取文件路径。支持精准路径、目录或者模糊匹配的文件。会过滤排除的文件。
	 * @return
	 */
	private List<String> generateRealPath(){
		
		List<String> ps = Lists.newArrayList();
		for(String p : path){
			
			if(p.contains("*") || p.contains("?")){//模糊匹配
				ps.addAll(getPatternFile(p));
				continue;
			}
			
			java.io.File file = new java.io.File(p);
			if(!file.exists()){
				logger.error("file:{} is not exists.", p);
				System.exit(-1);
			}
			
			if(file.isDirectory()){
				for(java.io.File tmpFile : file.listFiles()){
					ps.add(tmpFile.getPath());
				}
				
				addMonitorDic(p, null);
			}else{
				ps.add(file.getPath());
			}
		}
		
		Iterator<String> it = ps.iterator();
		for( ;it.hasNext();){
			String name = it.next();
			if(isExcludeFile(name)){
				it.remove();
			}
		}
		
		return ps;
	}
	
	/**
	 * 返回匹配正则的file列表。
	 * @param patternName
	 * @return
	 */
	private List<String> getPatternFile(String patternName){
		
		List<String> fileList = Lists.newArrayList(); 
		String dir = patternName.substring(0, patternName.lastIndexOf("/"));
		String filePattern = patternName.substring(patternName.lastIndexOf("/") + 1);
		java.io.File dirFile = new java.io.File(dir);
		if(!dirFile.isDirectory()){
			logger.info("don't exists dir in pattern:{}", patternName);
			return fileList;
		}
		
		addMonitorDic(dir, filePattern);
		
		for(java.io.File tmpFile : dirFile.listFiles()){
			
			if(tmpFile.isDirectory()){
				//FIXME 暂时不对指定文件夹下的子文件做模糊匹配,如果有需求在该处修改
				continue;
			}
			
			if(filePatternMatcher(filePattern, tmpFile.getName())){
				fileList.add(tmpFile.getPath());
			}
		}
		
		return fileList;
	}
	
	/**
	 * 把patternName的list存入moniDic,其key为dir。
	 * @param dir
	 * @param patternName
	 */
	public void addMonitorDic(String dir, String patternName){
		List<String> patternList = moniDic.get(dir);
		if(patternList == null){
			patternList = Lists.newArrayList();
			moniDic.put(dir, patternList);
		}
		
		if(patternName != null){
			patternList.add(patternName);
		}
		
	}
	
	/**
	 * 文件名正则匹配。
	 * @param pattern
	 * @param str
	 * @return
	 */
	private boolean filePatternMatcher(String pattern, String str) {
		
		pattern = pattern.replace("\\", "").replace("/", "");
		str = str.replace("\\", "").replace("/", "");
		
        int patternLength = pattern.length();
        int strLength = str.length();
        int strIndex = 0;
        char ch;
        for (int patternIndex = 0; patternIndex < patternLength; patternIndex++) {
            ch = pattern.charAt(patternIndex);
            if (ch == '*') {
                //通配符星号*表示可以匹配任意多个字符
                while (strIndex < strLength) {
                    if (filePatternMatcher(pattern.substring(patternIndex + 1),
                            str.substring(strIndex))) {
                        return true;
                    }
                    strIndex++;
                }
            } else if (ch == '?') {
                //通配符问号?表示匹配任意一个字符
                strIndex++;
                if (strIndex > strLength) {
                    //表示str中已经没有字符匹配?了。
                    return false;
                }
            } else {
                if ((strIndex >= strLength) || (ch != str.charAt(strIndex))) {
                    return false;
                }
                strIndex++;
            }
        }
        return (strIndex == strLength);
    }
		
	/**
	 * 过滤排除文件
	 * @param fileName
	 * @return
	 */
	public boolean isExcludeFile(String fileName){
		
		if(exclude == null){
			return false;
		}
		
		for(String patternName : exclude){
			if(filePatternMatcher(patternName, fileName)){
				return true;
			}
		}
		
		return false;
	}
	
	/**
	 * 把sincedb.yaml的内容存入fileCurrPos。
	 */
	private void checkoutSinceDb(){
		Yaml yaml = new Yaml();
		java.io.File sinceFile = new java.io.File(sinceDbPath);
		if(!sinceFile.exists()){
			return;
		}
		
		InputStream io = null;
		try {
			io = new FileInputStream(sinceFile);
			Map<String, Object> fileMap = yaml.loadAs(io, Map.class);
			if(fileMap == null){
				return;
			}

			for(Entry<String, Object> tmp : fileMap.entrySet()){
				if(tmp.getValue() == null){
					continue;
				}

				fileCurrPos.put(tmp.getKey(), Long.valueOf(tmp.getValue() + ""));
			}
		} catch (FileNotFoundException e) {
			logger.error("open file:{} err:{}!", sinceDbPath, e.getCause());
			System.exit(1);
		}finally{
			if(io != null){
				try {
					io.close();
				} catch (IOException e) {
					logger.error("", e);
				}
			}
		}
	}
	
	/**
	 * 过滤掉已经读取完成的文件
	 */
	private void filterFinishFile(){
		for(Entry<String, Long> entry : fileCurrPos.entrySet()){
			if(entry.getValue().longValue() == -1l){//表示该文件已经读取完成
				realPaths.remove(entry.getKey());
			}
		}
	}
	
	/**
	 * 使用替换的方式防止出现写不全的情况
	 */
	private void dumpSinceDb(){
		
		FileWriter fw = null;
		boolean isSuccess = false;
		String tmpSinceDbName = sinceDbPath + ".tmp";
		
		try{
			writeFileLock.lock();
			Yaml tmpYaml = new Yaml();
			fw = new FileWriter(tmpSinceDbName);
			tmpYaml.dump(fileCurrPos, fw);
			isSuccess = true;
		}catch(Exception e){
			logger.error("", e);
			logger.info("curr file pos:{}", fileCurrPos);
		}finally{
			try {
				fw.close();
			} catch (IOException e) {
				logger.error("", e);
			}
			
			writeFileLock.unlock();
		}
		
		if(!isSuccess){
			return;
		}
		
		java.io.File srcFile = new java.io.File(tmpSinceDbName);
		java.io.File dstFile = new java.io.File(sinceDbPath);
		try {
			FileUtils.rename(srcFile, dstFile);
		} catch (IOException e) {
			logger.error("", e);
		}
		
	}
	
	public void addFile(String fileName){
		int hashCode = Math.abs(fileName.hashCode());
		int index = hashCode % readFileThreadNum;
		BlockingQueue<String> readQueue = threadReadFileMap.get(index);
		
		if(readQueue == null){
			readQueue = new LinkedBlockingQueue<>();
			threadReadFileMap.put(index, readQueue);
		}
		
		readQueue.offer(fileName);
	}

	@Override
	public void emit() {
		executor = Executors.newFixedThreadPool(readFileThreadNum + 2);
		scheduleExecutor = Executors.newScheduledThreadPool(1);
		
		executor.submit(new MonitorChangeRunnable());
		executor.submit(new MonitorNewFileRunnable());
		for(int i=0; i<readFileThreadNum; i++){
			executor.submit(new FileRunnable(i));
		}
		
		scheduleExecutor.scheduleWithFixedDelay(new DumpSinceDbRunnable(), 
				sinceDbWriteInterval, sinceDbWriteInterval, TimeUnit.SECONDS);
	}

	@Override
	public void release() {
		//在正常关闭的时候记录offset
		executor.shutdownNow();
		scheduleExecutor.shutdownNow();
		dumpSinceDb();
	}
	
	public IDecode getDecoder(String fileName){
		for(Entry<String, IDecode> entry : codecMap.entrySet()){
			if(filePatternMatcher(entry.getKey(), fileName)){
				return entry.getValue();
			}
		}
		
		logger.info("can't find decoder from config. return default decoder.");
		return this.getDecoder();
	}
		
		
	class FileRunnable implements Runnable{
								
		private final int index;
		
		public FileRunnable(int index) {
			this.index = index;
		}

		public void run() {
			
			while(runFlag){
				
				BlockingQueue<String> needReadList = threadReadFileMap.get(index);
				if(needReadList == null){
					logger.warn("invalid FileRunnable thread, threadReadFileMap don't init needReadList of this index:{}.", index);
					return;
				}
				
				String readFileName = null;
				try {
					readFileName = needReadList.poll(10, TimeUnit.SECONDS);
					if(readFileName == null){
						continue;
					}
				} catch (InterruptedException e) {
					logger.error("", e);
					continue;
				}
					
				long lastModTime = 0l;	
				IReader reader = null;
				try {
					java.io.File readFile = new java.io.File(readFileName);
					if(!readFile.exists()){
						logger.error("file:{} is not exists!", readFileName);
						continue;
					}
					
					lastModTime = readFile.lastModified();
					reader = ReadFactory.createReader(readFile, encoding, fileCurrPos, startPosition);
					if(reader == null){
						continue;
					}
					
					String line = null;
					int readLineNum = 0;
					IDecode fileDecoder = getDecoder(readFileName);
					boolean isMultiLine = false;
					if(fileDecoder instanceof MultilineDecoder){
						isMultiLine = true;
					}
					
					while( (line = reader.readLine()) != null){
						readLineNum++;
						
						if(!"".equals(line.trim())){
							Map<String, Object> event = null;
							
							if(isMultiLine){
								event = fileDecoder.decode(line, readFileName);
							}else{
								event = fileDecoder.decode(line);
							}
							
							if (event != null && event.size() > 0){
								event.put("path", readFileName);
								event.put("offset", reader.getCurrBufPos());
								process(event);
							}
						}
						
						if(readLineNum%readLineNum4UpdateMap == 0){
							fileCurrPos.put(reader.getFileName(), reader.getCurrBufPos());
						}
					}
					
					fileCurrPos.put(readFileName, reader.getCurrBufPos());
				} catch (Exception e) {
					logger.error("", e);
				}finally{
					if(reader != null && reader.needMonitorChg()){
						monitorMap.put(readFileName, lastModTime);//确保文件回到监控列表
					}
				}
			}
		}
	}
	
	/**
	 * 监控文件变化,将有变化的文件插入到needReadList里
	 * 2s查看一次
	 * FIXME 需要对已经删除的文件做清理
	 * @author xuchao
	 *
	 */
	class MonitorChangeRunnable implements Runnable{

		public void run() {
			
			while(runFlag){
				try {
					Thread.sleep(500);
				} catch (InterruptedException e) {
					logger.error("", e);
				}
				
				Iterator<Entry<String, Long>> iterator = monitorMap.entrySet().iterator();
				for( ;iterator.hasNext(); ){
					Entry<String, Long> entry = iterator.next();
					java.io.File monitorFile = new java.io.File(entry.getKey());
					if(!monitorFile.exists()){
						logger.info("file:{} not exists,may be delete!", entry.getKey());
						continue;
					}
										
//					if(monitorFile.lastModified() > entry.getValue()){
//						iterator.remove();
//						addFile(entry.getKey());
//					}
					if(fileCurrPos.get(entry.getKey())<monitorFile.length()){
						iterator.remove();
						addFile(entry.getKey());
					}	
				}
				
			}
		}
		
	}
	
	/** 
	 * 监控新出现的符合条件的文件,并加入到文件读取列表里
	 * @author xuchao
	 *
	 */
	class MonitorNewFileRunnable implements Runnable{

		public void run() {
			
			while(runFlag){
				
				try {
					Thread.sleep(10000);
				} catch (InterruptedException e) {
					logger.error("", e);
				}
				
				for(Entry<String, List<String>> dirTmp : moniDic.entrySet()){
					
					String dicName = dirTmp.getKey();
					List<String> patternList = dirTmp.getValue();
					
					java.io.File file = new java.io.File(dicName);					
					
					if(!file.exists() || !file.isDirectory()){
						continue;
					}
					
					for(java.io.File tmpFile : file.listFiles()){
						
						if(tmpFile.isDirectory()){//FIXME 不监控新出现的子文件夹
							continue;
						}
						
						if(isExcludeFile(tmpFile.getPath())){
							continue;
						}
						
						if(fileCurrPos.get(tmpFile.getPath()) != null && fileCurrPos.get(tmpFile.getPath()).longValue() == -1l){
							continue;//已经完结的数据
						}
						
						if(patternList.size() == 0 && !realPaths.contains(tmpFile.getPath())){
							realPaths.add(tmpFile.getPath());
							addFile(tmpFile.getPath());
							continue;
						}
						
						for(String patternName : patternList){
							if(filePatternMatcher(patternName, tmpFile.getName()) && !realPaths.contains(tmpFile.getPath())){
								realPaths.add(tmpFile.getPath());
								addFile(tmpFile.getPath());
								break;
							}
						}
					}
				}
			}
		}
		
	}
	
	class DumpSinceDbRunnable implements Runnable{

		public void run() {
			dumpSinceDb();
		}
		
	}
}