com.safframework.tony.common.utils.Preconditions Java Examples

The following examples show how to use com.safframework.tony.common.utils.Preconditions. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SpitalConvAdapter.java    From cv4j with Apache License 2.0 6 votes vote down vote up
@Override
    public void onBindViewHolder(final SpitalConvAdapter.ViewHolder holder, int position) {

        if (position == 0) {
            holder.image.setImageBitmap(mBitmap);
        } else {
            String filterName = mList.get(position);
            if (Preconditions.isNotBlank(filterName)) {
                CommonFilter filter = (CommonFilter)getFilter(filterName);
                RxImageData.bitmap(mBitmap)
//                        .placeHolder(R.drawable.test_spital_conv)
                        .addFilter(filter)
                        .into(holder.image);
            }
        }

        holder.text.setText(map.get(position));
    }
 
Example #2
Source File: Spider.java    From NetDiscovery with Apache License 2.0 6 votes vote down vote up
private Spider() {
    String queueType = SpiderConfig.getInstance().getQueueType();
    if (Preconditions.isNotBlank(queueType)) {
        switch (queueType) {
            case Constant.QUEUE_TYPE_DEFAULT:
                this.queue = new DefaultQueue();
                break;
            case Constant.QUEUE_TYPE_DISRUPTOR:
                this.queue = new DisruptorQueue();
                break;
            default:
                break;
        }
    }

    if (this.queue == null) {
        this.queue = new DefaultQueue();
    }

    initSpiderConfig();
}
 
Example #3
Source File: UrlParser.java    From PicCrawler with Apache License 2.0 6 votes vote down vote up
@Override
public List<String> parse(Document doc) {

    List<String> urls = new ArrayList<>();

    Elements links = doc.select("a[href]");

    if (Preconditions.isNotBlank(links)) {

        for (Element src : links) {
            if (Preconditions.isNotBlank(src.attr("abs:href"))) {

                String href = src.attr("abs:href");
                log.info(href);
                urls.add(href);
            }
        }
    }

    return urls;
}
 
Example #4
Source File: CookiesPool.java    From NetDiscovery with Apache License 2.0 6 votes vote down vote up
/**
 * 保存单个cookie字符串
 * @param request
 * @param cookie
 */
public void saveCookie(Request request, String cookie) {

    if (Preconditions.isNotBlank(cookie)) {

        CookiesGroup cookiesGroup = CookiesPool.getInsatance().getCookieGroup(request.getUrlParser().getHost());
        List<HttpCookie> httpCookieList = new ArrayList<>();

        if (cookiesGroup ==null) {
            cookiesGroup = new CookiesGroup(request.getUrlParser().getHost());
            httpCookieList.addAll(HttpCookie.parse(cookie));
            cookiesGroup.putAllCookies(httpCookieList);
            CookiesPool.getInsatance().addCookieGroup(cookiesGroup);
        } else {
            httpCookieList.addAll(HttpCookie.parse(cookie));
            cookiesGroup.putAllCookies(httpCookieList);
        }
    }
}
 
Example #5
Source File: PicCrawlerClient.java    From PicCrawler with Apache License 2.0 6 votes vote down vote up
/**
 * 下载整个网页的全部图片
 * @param url
 */
public void downloadWebPageImages(String url) {

    if (Preconditions.isNotBlank(url)) {

        isWebPage = true;

        pageParser = new PicParser();

        Flowable.just(url)
                .map(s->httpManager.createHttpWithGet(s))
                .map(response->parseHtmlToImages(response,(PicParser)pageParser))
                .subscribe(urls -> downloadPics(urls),
                        throwable-> System.out.println(throwable.getMessage()));
    }
}
 
Example #6
Source File: PicCrawlerClient.java    From PicCrawler with Apache License 2.0 6 votes vote down vote up
/**
 * 下载多个网页的全部图片
 * @param urls
 */
public void downloadWebPageImages(List<String> urls) {

    if (Preconditions.isNotBlank(urls)) {

        isWebPage = true;

        pageParser = new PicParser();

        Flowable.fromIterable(urls)
                .parallel()
                .map(url->httpManager.createHttpWithGet(url))
                .map(response->parseHtmlToImages(response,(PicParser)pageParser))
                .sequential()
                .subscribe(list -> downloadPics(list),
                        throwable-> System.out.println(throwable.getMessage()));
    }
}
 
Example #7
Source File: Spider.java    From NetDiscovery with Apache License 2.0 6 votes vote down vote up
public Spider request(Request... requests) {
    checkIfRunning();

    if (Preconditions.isNotBlank(requests)) {

        Arrays.asList(requests)
                .stream()
                .forEach(request -> {
                    queue.push(request.spiderName(name));
                });

        signalNewRequest();
    }

    return this;
}
 
Example #8
Source File: CSVPipeline.java    From NetDiscovery with Apache License 2.0 6 votes vote down vote up
@Override
public void process(ResultItems resultItems) {
    StringBuilder sb = new StringBuilder();

    for (Map.Entry<String, Object> entry : resultItems.getAll().entrySet()) {
        sb.append(entry.getValue()).append(",");
    }

    String[] ss = sb.toString().split(",");

    if (Preconditions.isNotBlank(ss)) {

        List<String> dataList = Arrays.asList(ss);
        SpiderUtils.exportCsv(csvFile,dataList, Charset.forName("GBK"));
    }
}
 
Example #9
Source File: SpiderUtils.java    From NetDiscovery with Apache License 2.0 6 votes vote down vote up
/**
 * 导出csv
 *
 * @param file
 * @param dataList
 * @param charset
 * @return
 */
public static boolean exportCsv(File file, List<String> dataList, Charset charset) {

    boolean isSucess = false;

    FileOutputStream out = null;
    OutputStreamWriter osw = null;
    BufferedWriter bw = null;
    try {
        out = new FileOutputStream(file);
        osw = new OutputStreamWriter(out, charset);
        bw = new BufferedWriter(osw);
        if (Preconditions.isNotBlank(dataList)) {
            for (String data : dataList) {
                bw.append(data).append("\r");
            }
        }
        isSucess = true;
    } catch (Exception e) {
        isSucess = false;
    } finally {
        IOUtils.closeQuietly(bw,osw,out);
    }

    return isSucess;
}
 
Example #10
Source File: Pipeline.java    From NetDiscovery with Apache License 2.0 6 votes vote down vote up
/**
 * 方便在 pipeline 中往队列中发起爬取任务(进行深度爬取)
 * @param spider
 * @param originalRequest 原始的request,新的request可以继承原始request的header信息
 * @param url
 */
public void push(Spider spider, Request originalRequest, String url) {

    if (spider==null || originalRequest==null || Preconditions.isBlank(url)) {
        return;
    }

    Request request = new Request(url,spider.getName());     // 根据spider的名称来创建request
    Map<String,String> header = originalRequest.getHeader(); // 从原始request中获取header
    if (Preconditions.isNotBlank(header)) {                  // 将原始request的header复制到新的request

        header.forEach((key,value)->{

            request.header(key,value);
        });
    }

    spider.getQueue().push(request);
}
 
Example #11
Source File: ProxyResourceDaoImpl.java    From ProxyPool with Apache License 2.0 6 votes vote down vote up
@Override
public boolean saveResourcePlan(ResourcePlan resourcePlan) {
    boolean result = false;
    if(resourcePlan.getAddTime() == 0) { //insert
        resourcePlan.setAddTime(new Date().getTime());
        resourcePlan.setModTime(new Date().getTime());

        mongoTemplate.save(resourcePlan, Constant.COL_NAME_RESOURCE_PLAN);
        result = Preconditions.isNotBlank(resourcePlan.getId());

    } else {                            //update
        Query query = new Query().addCriteria(Criteria.where("_id").is(resourcePlan.getId()));
        Update update = new Update();
        update.set("startPageNum", resourcePlan.getStartPageNum());
        update.set("endPageNum", resourcePlan.getEndPageNum());
        update.set("modTime", new Date().getTime());

        WriteResult writeResult = mongoTemplate.updateFirst(query, update, Constant.COL_NAME_RESOURCE_PLAN);
        result = writeResult!=null && writeResult.getN() > 0;
    }

    return result;
}
 
Example #12
Source File: GridViewFilterAdapter.java    From cv4j with Apache License 2.0 6 votes vote down vote up
@Override
public void onBindViewHolder(final GridViewFilterAdapter.ViewHolder holder, int position) {

    String filterName = mList.get(position);

    if (position == 0) {
        holder.image.setImageBitmap(mBitmap);
    } else {

        if (Preconditions.isNotBlank(filterName)) {
            CommonFilter filter = (CommonFilter)getFilter(filterName);
            RxImageData.bitmap(mBitmap)
                    .addFilter(filter)
                    .into(holder.image);
        }

    }

    holder.text.setText(filterName);
}
 
Example #13
Source File: TraceAspect.java    From SAF-AOP with Apache License 2.0 6 votes vote down vote up
@Around("methodAnnotatedWithTrace() || constructorAnnotatedTrace()")
public Object traceMethod(final ProceedingJoinPoint joinPoint) throws Throwable {
    MethodSignature methodSignature = (MethodSignature) joinPoint.getSignature();

    Trace trace = methodSignature.getMethod().getAnnotation(Trace.class);
    if (trace!=null && !trace.enable()) {
        return joinPoint.proceed();
    }

    String className = methodSignature.getDeclaringType().getSimpleName();
    String methodName = methodSignature.getName();
    final StopWatch stopWatch = new StopWatch();
    stopWatch.start();
    Object result = joinPoint.proceed();
    stopWatch.stop();

    if (Preconditions.isBlank(className)) {
        className = "Anonymous class";
    }

    L.i(className, buildLogMessage(methodName, stopWatch.getElapsedTime()));

    return result;
}
 
Example #14
Source File: RPCServiceImpl.java    From NetDiscovery with Apache License 2.0 6 votes vote down vote up
@Override
public SpiderBean detail(String spiderName) {
    if (Preconditions.isNotBlank(spiderName) && spiders.get(spiderName) != null) {
        Spider spider = spiders.get(spiderName);

        SpiderBean entity = new SpiderBean();
        entity.setSpiderName(spiderName);
        entity.setSpiderStatus(spider.getSpiderStatus());
        entity.setLeftRequestSize(spider.getQueue().getLeftRequests(spiderName));
        entity.setTotalRequestSize(spider.getQueue().getTotalRequests(spiderName));
        entity.setConsumedRequestSize(entity.getTotalRequestSize() - entity.getLeftRequestSize());
        entity.setQueueType(spider.getQueue().getClass().getSimpleName());
        entity.setDownloaderType(spider.getDownloader().getClass().getSimpleName());

        return entity;
    }

    return null;
}
 
Example #15
Source File: SpiderJob.java    From NetDiscovery with Apache License 2.0 6 votes vote down vote up
@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
    log.info("定时任务开始");

    log.info("jobName="+context.getJobDetail().getKey().getName());

    JobDataMap dataMap = context.getJobDetail().getJobDataMap();
    Spider spider = (Spider) dataMap.get("spider");
    Request[] requests = (Request[]) dataMap.get("requests");

    if (spider!=null && Preconditions.isNotBlank(requests)) {

        log.info("spiderName="+spider.getName());

        if (spider.getSpiderStatus() == Spider.SPIDER_STATUS_INIT
                || spider.getSpiderStatus() == Spider.SPIDER_STATUS_STOPPED) {

            spider.run();
        } else if (spider.getSpiderStatus() == Spider.SPIDER_STATUS_PAUSE) {

            spider.resume();
        }

        Stream.of(requests).forEach(request -> spider.getQueue().pushToRunninSpider(request,spider));
    }
}
 
Example #16
Source File: SeleniumDownloader.java    From NetDiscovery with Apache License 2.0 5 votes vote down vote up
@Override
public Maybe<Response> download(Request request) {

    return Maybe.create(new MaybeOnSubscribe<String>(){

        @Override
        public void subscribe(MaybeEmitter emitter) throws Exception {

            if (webDriver!=null) {
                webDriver.get(request.getUrl());

                if (Preconditions.isNotBlank(actions)) {

                    actions.forEach(
                            action-> action.perform(webDriver)
                    );
                }

                emitter.onSuccess(webDriver.getPageSource());
            }
        }
    })
    .compose(new DownloaderDelayTransformer(request))
    .map(new Function<String, Response>() {

        @Override
        public Response apply(String html) throws Exception {

            Response response = new Response();
            response.setContent(html.getBytes());
            response.setStatusCode(Constant.OK_STATUS_CODE);
            response.setContentType(getContentType(webDriver));
            return response;
        }
    });
}
 
Example #17
Source File: PicCrawlerClient.java    From PicCrawler with Apache License 2.0 5 votes vote down vote up
/**
 *
 * @param proxyList 代理的host的列表
 * @return
 */
public PicCrawlerClient addProxyPool(List<Proxy> proxyList) {

    if (Preconditions.isNotBlank(proxyList)) {
        httpParamBuilder.addProxyPool(proxyList);
    }
    return this;
}
 
Example #18
Source File: ProxyDaoImpl.java    From ProxyPool with Apache License 2.0 5 votes vote down vote up
@Override
public List<ProxyData> findProxyByCond(QueryProxyDTO queryProxyDTO, boolean isGetAll) {
    Query query = new Query();
    if(Preconditions.isNotBlank(queryProxyDTO.getType()) && !"all".equals(queryProxyDTO.getType())) {
        query.addCriteria(Criteria.where("proxyType").is(queryProxyDTO.getType()));
    }
    if(Preconditions.isNotBlank(queryProxyDTO.getIp()) && !"all".equals(queryProxyDTO.getIp())) {
        query.addCriteria(Criteria.where("proxyAddress").regex(".*?"+queryProxyDTO.getIp()+".*"));
    }
    if(queryProxyDTO.getMinPort() != null) {
        query.addCriteria(Criteria.where("proxyPort").gte(queryProxyDTO.getMinPort()).lte(queryProxyDTO.getMaxPort()));
    }
    if(!isGetAll) {
        if(Preconditions.isNotBlank(queryProxyDTO.getSort())) {
            if("asc".equals(queryProxyDTO.getOrder())) {
                query.with(new Sort(Sort.Direction.ASC, queryProxyDTO.getSort()));
            } else {
                query.with(new Sort(Sort.Direction.DESC, queryProxyDTO.getSort()));
            }
        } else {
            query.with(new Sort(Sort.Direction.DESC, "lastSuccessfulTime"));
            query.with(new Sort(Sort.Direction.ASC, "proxyPort"));
        }
        int skip = (queryProxyDTO.getPage() - 1) * queryProxyDTO.getRows();
        query.skip(skip);
        query.limit(queryProxyDTO.getRows());
    }
    return mongoTemplate.find(query, ProxyData.class, Constant.COL_NAME_PROXY);
}
 
Example #19
Source File: ProxyDaoImpl.java    From ProxyPool with Apache License 2.0 5 votes vote down vote up
@Override
public List<ProxyData> takeRandomTenProxy() {

    List<ProxyData> list = mongoTemplate.findAll(ProxyData.class);

    if (Preconditions.isNotBlank(list)) {

        Collections.shuffle(list);
        return list.size()>10?list.subList(0,10):list;
    } else {

        return new ArrayList<>();
    }
}
 
Example #20
Source File: ProxyResourceDaoImpl.java    From ProxyPool with Apache License 2.0 5 votes vote down vote up
@Override
public boolean saveProxyResource(ProxyResource proxyResource) {
    boolean result = false;
    if(Preconditions.isBlank(proxyResource.getResId())) {            //insert
        proxyResource.setResId(commonDao.getNextSequence(Constant.COL_NAME_PROXY_RESOURCE).getSequence());
        proxyResource.setAddTime(new Date().getTime());
        proxyResource.setModTime(new Date().getTime());
        mongoTemplate.save(proxyResource, Constant.COL_NAME_PROXY_RESOURCE);

        result = Preconditions.isNotBlank(proxyResource.getId());
    } else {                                                        //update
        Query query = new Query().addCriteria(Criteria.where("resId").is(proxyResource.getResId()));
        Update update = new Update();
        update.set("webName", proxyResource.getWebName());
        update.set("webUrl", proxyResource.getWebUrl());
        update.set("pageCount", proxyResource.getPageCount());
        update.set("prefix", proxyResource.getPrefix());
        update.set("suffix", proxyResource.getSuffix());
        update.set("parser", proxyResource.getParser());
        update.set("modTime", new Date().getTime());

        WriteResult writeResult = mongoTemplate.updateFirst(query, update, Constant.COL_NAME_PROXY_RESOURCE);

        result = writeResult!=null && writeResult.getN() > 0;
    }
    return result;
}
 
Example #21
Source File: Constant.java    From ProxyPool with Apache License 2.0 5 votes vote down vote up
public static String getUserAgent() {

        if (Preconditions.isNotBlank(uas)) {

            int index=(int)(Math.random() * uas.size());
            return uas.get(index);
        } else {

            return Constant.defaultUAArray[new Random().nextInt(Constant.defaultUAArray.length)];
        }
    }
 
Example #22
Source File: ProxyPageCallable.java    From ProxyPool with Apache License 2.0 5 votes vote down vote up
/**
 * 将下载的proxy放入代理池
 * @param page
 */
private List<Proxy> handle(Page page){

    if (page == null || Preconditions.isBlank(page.getHtml())){
        return new ArrayList<Proxy>();
    }

    List<Proxy> result = new ArrayList<>();

    ProxyListPageParser parser = ProxyListPageParserFactory.getProxyListPageParser(ProxyPool.proxyMap.get(url));
    if (parser!=null) {

        List<Proxy> proxyList = parser.parse(page.getHtml());
        if(Preconditions.isNotBlank(proxyList)) {

            for(Proxy p : proxyList){

                if (!ProxyPool.proxyList.contains(p)) {
                    result.add(p);
                }
            }
        }

    }

    return result;
}
 
Example #23
Source File: SafeAspect.java    From SAF-AOP with Apache License 2.0 5 votes vote down vote up
@Around("methodAnnotatedWithSafe()")
public Object safeMethod(final ProceedingJoinPoint joinPoint) throws Throwable {

    MethodSignature signature = (MethodSignature) joinPoint.getSignature();
    Method method = signature.getMethod();

    Safe safe = method.getAnnotation(Safe.class);

    Object result = null;
    try {
        result = joinPoint.proceed();
    } catch (Throwable e) {
        L.w(getStringFromException(e));

        String callBack = safe.callBack();

        if (Preconditions.isNotBlank(callBack)) {

            try {
                Reflect.on(joinPoint.getTarget()).call(callBack);
            } catch (ReflectException exception) {
                exception.printStackTrace();
                L.e("no method "+callBack);
            }
        }
    }
    return result;
}
 
Example #24
Source File: APIController.java    From ProxyPool with Apache License 2.0 5 votes vote down vote up
@WebLog
@Cacheable(value="proxys")
@RequestMapping(value="/proxys/{count}", method = RequestMethod.GET)
public ResultProxyDataDTO getProxyData(@PathVariable String count) {

    int code = 0;
    String message = "";
    List<ProxyDataDTO> data = new ArrayList<>();
    if(isNumeric(count)) {
        data = proxyDao.findLimitProxy(Integer.parseInt(count));

        if(Preconditions.isNotBlank(data)) {
            code = 200;
            message = "成功返回有效数据";
        } else {
            code = 404;
            message = "无法返回有效数据";
        }
    } else {
        code = 400;
        message = "参数格式无效,请输入数字";
    }

    ResultProxyDataDTO resultProxyDataDTO = new ResultProxyDataDTO();
    resultProxyDataDTO.setCode(code);
    resultProxyDataDTO.setMessage(message);
    resultProxyDataDTO.setData(data);

    return resultProxyDataDTO;
}
 
Example #25
Source File: VertxDownloader.java    From NetDiscovery with Apache License 2.0 5 votes vote down vote up
private WebClientOptions initWebClientOptions(Request request) {

        WebClientOptions options = new WebClientOptions();
        options.setKeepAlive(SpiderConfig.getInstance().isKeepAlive())
                .setReuseAddress(SpiderConfig.getInstance().isReuseAddress())
                .setFollowRedirects(SpiderConfig.getInstance().isFollowRedirects())
                .setConnectTimeout(SpiderConfig.getInstance().getConnectTimeout())
                .setIdleTimeout(SpiderConfig.getInstance().getIdleTimeout())
                .setMaxWaitQueueSize(SpiderConfig.getInstance().getMaxWaitQueueSize());

        if (Preconditions.isNotBlank(request.getUserAgent())) {
            options.setUserAgent(request.getUserAgent());
        }

        if (Preconditions.isNotBlank(request.getProxy())) {

            ProxyOptions proxyOptions = new ProxyOptions();
            proxyOptions.setHost(request.getProxy().getIp());
            proxyOptions.setPort(request.getProxy().getPort());
            options.setProxyOptions(proxyOptions);
        }

        if (Preconditions.isNotBlank(request.getHeader())) {

            header = request.getHeader();
        }

        return options;
    }
 
Example #26
Source File: Request.java    From NetDiscovery with Apache License 2.0 5 votes vote down vote up
/**
 * 从 UserAgent 类中随机获取User-Agent
 */
private void autoUA() {

    this.userAgent = UserAgent.getUserAgent();
    if (Preconditions.isNotBlank(userAgent)) {
        header.put("User-Agent", userAgent);
    }
}
 
Example #27
Source File: Request.java    From NetDiscovery with Apache License 2.0 5 votes vote down vote up
public Request ua(String userAgent) {

        this.userAgent = userAgent;
        if (Preconditions.isNotBlank(userAgent)) {
            header.put("User-Agent", userAgent);
        }

        return this;
    }
 
Example #28
Source File: URLParser.java    From NetDiscovery with Apache License 2.0 5 votes vote down vote up
private String encode(String value) {
    Preconditions.checkNotNull(value);
    try {
        return charset == null ? value : URLEncoder.encode(value, charset);
    } catch (UnsupportedEncodingException e) {
        throw new SpiderException(e);
    }
}
 
Example #29
Source File: URLParser.java    From NetDiscovery with Apache License 2.0 5 votes vote down vote up
private static List<String> getOrCreate(Map<String, List<String>> map, String name) {
    Preconditions.checkNotNull(name);
    List<String> list = map.get(name);
    if (list == null) {
        list = new ArrayList<String>();
        map.put(name, list);
    }
    return list;
}
 
Example #30
Source File: URLParser.java    From NetDiscovery with Apache License 2.0 5 votes vote down vote up
private static LinkedHashMap<String, List<String>> parseQueryString(String query) {
    LinkedHashMap<String, List<String>> params = new LinkedHashMap<String, List<String>>();
    if (Preconditions.isBlank(query)) {
        return params;
    }
    String[] items = query.split("&");
    for (String item : items) {
        String name = substringBefore(item, "=");
        String value = substringAfter(item, "=");
        List<String> values = getOrCreate(params, name);
        values.add(value);
    }
    return params;
}