package com.gs.spider.dao; import com.google.gson.Gson; import com.gs.spider.model.commons.Webpage; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; import us.codecraft.webmagic.ResultItems; import us.codecraft.webmagic.Task; import us.codecraft.webmagic.pipeline.Pipeline; import java.io.File; import java.io.IOException; /** * Created by gsh199449 on 2016/12/8. */ @Component public class JsonFilePipeline implements Pipeline { private final static Logger LOG = LogManager.getLogger(JsonFilePipeline.class); private final static Gson gson = new Gson(); @Override public void process(ResultItems resultItems, Task task) { Webpage webpage = CommonWebpagePipeline.convertResultItems2Webpage(resultItems); try { FileUtils.writeStringToFile( new File("gather_platform_data/" + webpage.getSpiderUUID() + ".json"), gson.toJson(webpage) + "\n", true); } catch (IOException e) { LOG.error("序列化网页信息出错,{}", e.getLocalizedMessage()); } } }