com.safframework.tony.common.utils.Preconditions Java Examples
The following examples show how to use
com.safframework.tony.common.utils.Preconditions.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SpitalConvAdapter.java From cv4j with Apache License 2.0 | 6 votes |
@Override public void onBindViewHolder(final SpitalConvAdapter.ViewHolder holder, int position) { if (position == 0) { holder.image.setImageBitmap(mBitmap); } else { String filterName = mList.get(position); if (Preconditions.isNotBlank(filterName)) { CommonFilter filter = (CommonFilter)getFilter(filterName); RxImageData.bitmap(mBitmap) // .placeHolder(R.drawable.test_spital_conv) .addFilter(filter) .into(holder.image); } } holder.text.setText(map.get(position)); }
Example #2
Source File: Spider.java From NetDiscovery with Apache License 2.0 | 6 votes |
private Spider() { String queueType = SpiderConfig.getInstance().getQueueType(); if (Preconditions.isNotBlank(queueType)) { switch (queueType) { case Constant.QUEUE_TYPE_DEFAULT: this.queue = new DefaultQueue(); break; case Constant.QUEUE_TYPE_DISRUPTOR: this.queue = new DisruptorQueue(); break; default: break; } } if (this.queue == null) { this.queue = new DefaultQueue(); } initSpiderConfig(); }
Example #3
Source File: UrlParser.java From PicCrawler with Apache License 2.0 | 6 votes |
@Override public List<String> parse(Document doc) { List<String> urls = new ArrayList<>(); Elements links = doc.select("a[href]"); if (Preconditions.isNotBlank(links)) { for (Element src : links) { if (Preconditions.isNotBlank(src.attr("abs:href"))) { String href = src.attr("abs:href"); log.info(href); urls.add(href); } } } return urls; }
Example #4
Source File: CookiesPool.java From NetDiscovery with Apache License 2.0 | 6 votes |
/** * 保存单个cookie字符串 * @param request * @param cookie */ public void saveCookie(Request request, String cookie) { if (Preconditions.isNotBlank(cookie)) { CookiesGroup cookiesGroup = CookiesPool.getInsatance().getCookieGroup(request.getUrlParser().getHost()); List<HttpCookie> httpCookieList = new ArrayList<>(); if (cookiesGroup ==null) { cookiesGroup = new CookiesGroup(request.getUrlParser().getHost()); httpCookieList.addAll(HttpCookie.parse(cookie)); cookiesGroup.putAllCookies(httpCookieList); CookiesPool.getInsatance().addCookieGroup(cookiesGroup); } else { httpCookieList.addAll(HttpCookie.parse(cookie)); cookiesGroup.putAllCookies(httpCookieList); } } }
Example #5
Source File: PicCrawlerClient.java From PicCrawler with Apache License 2.0 | 6 votes |
/** * 下载整个网页的全部图片 * @param url */ public void downloadWebPageImages(String url) { if (Preconditions.isNotBlank(url)) { isWebPage = true; pageParser = new PicParser(); Flowable.just(url) .map(s->httpManager.createHttpWithGet(s)) .map(response->parseHtmlToImages(response,(PicParser)pageParser)) .subscribe(urls -> downloadPics(urls), throwable-> System.out.println(throwable.getMessage())); } }
Example #6
Source File: PicCrawlerClient.java From PicCrawler with Apache License 2.0 | 6 votes |
/** * 下载多个网页的全部图片 * @param urls */ public void downloadWebPageImages(List<String> urls) { if (Preconditions.isNotBlank(urls)) { isWebPage = true; pageParser = new PicParser(); Flowable.fromIterable(urls) .parallel() .map(url->httpManager.createHttpWithGet(url)) .map(response->parseHtmlToImages(response,(PicParser)pageParser)) .sequential() .subscribe(list -> downloadPics(list), throwable-> System.out.println(throwable.getMessage())); } }
Example #7
Source File: Spider.java From NetDiscovery with Apache License 2.0 | 6 votes |
public Spider request(Request... requests) { checkIfRunning(); if (Preconditions.isNotBlank(requests)) { Arrays.asList(requests) .stream() .forEach(request -> { queue.push(request.spiderName(name)); }); signalNewRequest(); } return this; }
Example #8
Source File: CSVPipeline.java From NetDiscovery with Apache License 2.0 | 6 votes |
@Override public void process(ResultItems resultItems) { StringBuilder sb = new StringBuilder(); for (Map.Entry<String, Object> entry : resultItems.getAll().entrySet()) { sb.append(entry.getValue()).append(","); } String[] ss = sb.toString().split(","); if (Preconditions.isNotBlank(ss)) { List<String> dataList = Arrays.asList(ss); SpiderUtils.exportCsv(csvFile,dataList, Charset.forName("GBK")); } }
Example #9
Source File: SpiderUtils.java From NetDiscovery with Apache License 2.0 | 6 votes |
/** * 导出csv * * @param file * @param dataList * @param charset * @return */ public static boolean exportCsv(File file, List<String> dataList, Charset charset) { boolean isSucess = false; FileOutputStream out = null; OutputStreamWriter osw = null; BufferedWriter bw = null; try { out = new FileOutputStream(file); osw = new OutputStreamWriter(out, charset); bw = new BufferedWriter(osw); if (Preconditions.isNotBlank(dataList)) { for (String data : dataList) { bw.append(data).append("\r"); } } isSucess = true; } catch (Exception e) { isSucess = false; } finally { IOUtils.closeQuietly(bw,osw,out); } return isSucess; }
Example #10
Source File: Pipeline.java From NetDiscovery with Apache License 2.0 | 6 votes |
/** * 方便在 pipeline 中往队列中发起爬取任务(进行深度爬取) * @param spider * @param originalRequest 原始的request,新的request可以继承原始request的header信息 * @param url */ public void push(Spider spider, Request originalRequest, String url) { if (spider==null || originalRequest==null || Preconditions.isBlank(url)) { return; } Request request = new Request(url,spider.getName()); // 根据spider的名称来创建request Map<String,String> header = originalRequest.getHeader(); // 从原始request中获取header if (Preconditions.isNotBlank(header)) { // 将原始request的header复制到新的request header.forEach((key,value)->{ request.header(key,value); }); } spider.getQueue().push(request); }
Example #11
Source File: ProxyResourceDaoImpl.java From ProxyPool with Apache License 2.0 | 6 votes |
@Override public boolean saveResourcePlan(ResourcePlan resourcePlan) { boolean result = false; if(resourcePlan.getAddTime() == 0) { //insert resourcePlan.setAddTime(new Date().getTime()); resourcePlan.setModTime(new Date().getTime()); mongoTemplate.save(resourcePlan, Constant.COL_NAME_RESOURCE_PLAN); result = Preconditions.isNotBlank(resourcePlan.getId()); } else { //update Query query = new Query().addCriteria(Criteria.where("_id").is(resourcePlan.getId())); Update update = new Update(); update.set("startPageNum", resourcePlan.getStartPageNum()); update.set("endPageNum", resourcePlan.getEndPageNum()); update.set("modTime", new Date().getTime()); WriteResult writeResult = mongoTemplate.updateFirst(query, update, Constant.COL_NAME_RESOURCE_PLAN); result = writeResult!=null && writeResult.getN() > 0; } return result; }
Example #12
Source File: GridViewFilterAdapter.java From cv4j with Apache License 2.0 | 6 votes |
@Override public void onBindViewHolder(final GridViewFilterAdapter.ViewHolder holder, int position) { String filterName = mList.get(position); if (position == 0) { holder.image.setImageBitmap(mBitmap); } else { if (Preconditions.isNotBlank(filterName)) { CommonFilter filter = (CommonFilter)getFilter(filterName); RxImageData.bitmap(mBitmap) .addFilter(filter) .into(holder.image); } } holder.text.setText(filterName); }
Example #13
Source File: TraceAspect.java From SAF-AOP with Apache License 2.0 | 6 votes |
@Around("methodAnnotatedWithTrace() || constructorAnnotatedTrace()") public Object traceMethod(final ProceedingJoinPoint joinPoint) throws Throwable { MethodSignature methodSignature = (MethodSignature) joinPoint.getSignature(); Trace trace = methodSignature.getMethod().getAnnotation(Trace.class); if (trace!=null && !trace.enable()) { return joinPoint.proceed(); } String className = methodSignature.getDeclaringType().getSimpleName(); String methodName = methodSignature.getName(); final StopWatch stopWatch = new StopWatch(); stopWatch.start(); Object result = joinPoint.proceed(); stopWatch.stop(); if (Preconditions.isBlank(className)) { className = "Anonymous class"; } L.i(className, buildLogMessage(methodName, stopWatch.getElapsedTime())); return result; }
Example #14
Source File: RPCServiceImpl.java From NetDiscovery with Apache License 2.0 | 6 votes |
@Override public SpiderBean detail(String spiderName) { if (Preconditions.isNotBlank(spiderName) && spiders.get(spiderName) != null) { Spider spider = spiders.get(spiderName); SpiderBean entity = new SpiderBean(); entity.setSpiderName(spiderName); entity.setSpiderStatus(spider.getSpiderStatus()); entity.setLeftRequestSize(spider.getQueue().getLeftRequests(spiderName)); entity.setTotalRequestSize(spider.getQueue().getTotalRequests(spiderName)); entity.setConsumedRequestSize(entity.getTotalRequestSize() - entity.getLeftRequestSize()); entity.setQueueType(spider.getQueue().getClass().getSimpleName()); entity.setDownloaderType(spider.getDownloader().getClass().getSimpleName()); return entity; } return null; }
Example #15
Source File: SpiderJob.java From NetDiscovery with Apache License 2.0 | 6 votes |
@Override public void execute(JobExecutionContext context) throws JobExecutionException { log.info("定时任务开始"); log.info("jobName="+context.getJobDetail().getKey().getName()); JobDataMap dataMap = context.getJobDetail().getJobDataMap(); Spider spider = (Spider) dataMap.get("spider"); Request[] requests = (Request[]) dataMap.get("requests"); if (spider!=null && Preconditions.isNotBlank(requests)) { log.info("spiderName="+spider.getName()); if (spider.getSpiderStatus() == Spider.SPIDER_STATUS_INIT || spider.getSpiderStatus() == Spider.SPIDER_STATUS_STOPPED) { spider.run(); } else if (spider.getSpiderStatus() == Spider.SPIDER_STATUS_PAUSE) { spider.resume(); } Stream.of(requests).forEach(request -> spider.getQueue().pushToRunninSpider(request,spider)); } }
Example #16
Source File: SeleniumDownloader.java From NetDiscovery with Apache License 2.0 | 5 votes |
@Override public Maybe<Response> download(Request request) { return Maybe.create(new MaybeOnSubscribe<String>(){ @Override public void subscribe(MaybeEmitter emitter) throws Exception { if (webDriver!=null) { webDriver.get(request.getUrl()); if (Preconditions.isNotBlank(actions)) { actions.forEach( action-> action.perform(webDriver) ); } emitter.onSuccess(webDriver.getPageSource()); } } }) .compose(new DownloaderDelayTransformer(request)) .map(new Function<String, Response>() { @Override public Response apply(String html) throws Exception { Response response = new Response(); response.setContent(html.getBytes()); response.setStatusCode(Constant.OK_STATUS_CODE); response.setContentType(getContentType(webDriver)); return response; } }); }
Example #17
Source File: PicCrawlerClient.java From PicCrawler with Apache License 2.0 | 5 votes |
/** * * @param proxyList 代理的host的列表 * @return */ public PicCrawlerClient addProxyPool(List<Proxy> proxyList) { if (Preconditions.isNotBlank(proxyList)) { httpParamBuilder.addProxyPool(proxyList); } return this; }
Example #18
Source File: ProxyDaoImpl.java From ProxyPool with Apache License 2.0 | 5 votes |
@Override public List<ProxyData> findProxyByCond(QueryProxyDTO queryProxyDTO, boolean isGetAll) { Query query = new Query(); if(Preconditions.isNotBlank(queryProxyDTO.getType()) && !"all".equals(queryProxyDTO.getType())) { query.addCriteria(Criteria.where("proxyType").is(queryProxyDTO.getType())); } if(Preconditions.isNotBlank(queryProxyDTO.getIp()) && !"all".equals(queryProxyDTO.getIp())) { query.addCriteria(Criteria.where("proxyAddress").regex(".*?"+queryProxyDTO.getIp()+".*")); } if(queryProxyDTO.getMinPort() != null) { query.addCriteria(Criteria.where("proxyPort").gte(queryProxyDTO.getMinPort()).lte(queryProxyDTO.getMaxPort())); } if(!isGetAll) { if(Preconditions.isNotBlank(queryProxyDTO.getSort())) { if("asc".equals(queryProxyDTO.getOrder())) { query.with(new Sort(Sort.Direction.ASC, queryProxyDTO.getSort())); } else { query.with(new Sort(Sort.Direction.DESC, queryProxyDTO.getSort())); } } else { query.with(new Sort(Sort.Direction.DESC, "lastSuccessfulTime")); query.with(new Sort(Sort.Direction.ASC, "proxyPort")); } int skip = (queryProxyDTO.getPage() - 1) * queryProxyDTO.getRows(); query.skip(skip); query.limit(queryProxyDTO.getRows()); } return mongoTemplate.find(query, ProxyData.class, Constant.COL_NAME_PROXY); }
Example #19
Source File: ProxyDaoImpl.java From ProxyPool with Apache License 2.0 | 5 votes |
@Override public List<ProxyData> takeRandomTenProxy() { List<ProxyData> list = mongoTemplate.findAll(ProxyData.class); if (Preconditions.isNotBlank(list)) { Collections.shuffle(list); return list.size()>10?list.subList(0,10):list; } else { return new ArrayList<>(); } }
Example #20
Source File: ProxyResourceDaoImpl.java From ProxyPool with Apache License 2.0 | 5 votes |
@Override public boolean saveProxyResource(ProxyResource proxyResource) { boolean result = false; if(Preconditions.isBlank(proxyResource.getResId())) { //insert proxyResource.setResId(commonDao.getNextSequence(Constant.COL_NAME_PROXY_RESOURCE).getSequence()); proxyResource.setAddTime(new Date().getTime()); proxyResource.setModTime(new Date().getTime()); mongoTemplate.save(proxyResource, Constant.COL_NAME_PROXY_RESOURCE); result = Preconditions.isNotBlank(proxyResource.getId()); } else { //update Query query = new Query().addCriteria(Criteria.where("resId").is(proxyResource.getResId())); Update update = new Update(); update.set("webName", proxyResource.getWebName()); update.set("webUrl", proxyResource.getWebUrl()); update.set("pageCount", proxyResource.getPageCount()); update.set("prefix", proxyResource.getPrefix()); update.set("suffix", proxyResource.getSuffix()); update.set("parser", proxyResource.getParser()); update.set("modTime", new Date().getTime()); WriteResult writeResult = mongoTemplate.updateFirst(query, update, Constant.COL_NAME_PROXY_RESOURCE); result = writeResult!=null && writeResult.getN() > 0; } return result; }
Example #21
Source File: Constant.java From ProxyPool with Apache License 2.0 | 5 votes |
public static String getUserAgent() { if (Preconditions.isNotBlank(uas)) { int index=(int)(Math.random() * uas.size()); return uas.get(index); } else { return Constant.defaultUAArray[new Random().nextInt(Constant.defaultUAArray.length)]; } }
Example #22
Source File: ProxyPageCallable.java From ProxyPool with Apache License 2.0 | 5 votes |
/** * 将下载的proxy放入代理池 * @param page */ private List<Proxy> handle(Page page){ if (page == null || Preconditions.isBlank(page.getHtml())){ return new ArrayList<Proxy>(); } List<Proxy> result = new ArrayList<>(); ProxyListPageParser parser = ProxyListPageParserFactory.getProxyListPageParser(ProxyPool.proxyMap.get(url)); if (parser!=null) { List<Proxy> proxyList = parser.parse(page.getHtml()); if(Preconditions.isNotBlank(proxyList)) { for(Proxy p : proxyList){ if (!ProxyPool.proxyList.contains(p)) { result.add(p); } } } } return result; }
Example #23
Source File: SafeAspect.java From SAF-AOP with Apache License 2.0 | 5 votes |
@Around("methodAnnotatedWithSafe()") public Object safeMethod(final ProceedingJoinPoint joinPoint) throws Throwable { MethodSignature signature = (MethodSignature) joinPoint.getSignature(); Method method = signature.getMethod(); Safe safe = method.getAnnotation(Safe.class); Object result = null; try { result = joinPoint.proceed(); } catch (Throwable e) { L.w(getStringFromException(e)); String callBack = safe.callBack(); if (Preconditions.isNotBlank(callBack)) { try { Reflect.on(joinPoint.getTarget()).call(callBack); } catch (ReflectException exception) { exception.printStackTrace(); L.e("no method "+callBack); } } } return result; }
Example #24
Source File: APIController.java From ProxyPool with Apache License 2.0 | 5 votes |
@WebLog @Cacheable(value="proxys") @RequestMapping(value="/proxys/{count}", method = RequestMethod.GET) public ResultProxyDataDTO getProxyData(@PathVariable String count) { int code = 0; String message = ""; List<ProxyDataDTO> data = new ArrayList<>(); if(isNumeric(count)) { data = proxyDao.findLimitProxy(Integer.parseInt(count)); if(Preconditions.isNotBlank(data)) { code = 200; message = "成功返回有效数据"; } else { code = 404; message = "无法返回有效数据"; } } else { code = 400; message = "参数格式无效,请输入数字"; } ResultProxyDataDTO resultProxyDataDTO = new ResultProxyDataDTO(); resultProxyDataDTO.setCode(code); resultProxyDataDTO.setMessage(message); resultProxyDataDTO.setData(data); return resultProxyDataDTO; }
Example #25
Source File: VertxDownloader.java From NetDiscovery with Apache License 2.0 | 5 votes |
private WebClientOptions initWebClientOptions(Request request) { WebClientOptions options = new WebClientOptions(); options.setKeepAlive(SpiderConfig.getInstance().isKeepAlive()) .setReuseAddress(SpiderConfig.getInstance().isReuseAddress()) .setFollowRedirects(SpiderConfig.getInstance().isFollowRedirects()) .setConnectTimeout(SpiderConfig.getInstance().getConnectTimeout()) .setIdleTimeout(SpiderConfig.getInstance().getIdleTimeout()) .setMaxWaitQueueSize(SpiderConfig.getInstance().getMaxWaitQueueSize()); if (Preconditions.isNotBlank(request.getUserAgent())) { options.setUserAgent(request.getUserAgent()); } if (Preconditions.isNotBlank(request.getProxy())) { ProxyOptions proxyOptions = new ProxyOptions(); proxyOptions.setHost(request.getProxy().getIp()); proxyOptions.setPort(request.getProxy().getPort()); options.setProxyOptions(proxyOptions); } if (Preconditions.isNotBlank(request.getHeader())) { header = request.getHeader(); } return options; }
Example #26
Source File: Request.java From NetDiscovery with Apache License 2.0 | 5 votes |
/** * 从 UserAgent 类中随机获取User-Agent */ private void autoUA() { this.userAgent = UserAgent.getUserAgent(); if (Preconditions.isNotBlank(userAgent)) { header.put("User-Agent", userAgent); } }
Example #27
Source File: Request.java From NetDiscovery with Apache License 2.0 | 5 votes |
public Request ua(String userAgent) { this.userAgent = userAgent; if (Preconditions.isNotBlank(userAgent)) { header.put("User-Agent", userAgent); } return this; }
Example #28
Source File: URLParser.java From NetDiscovery with Apache License 2.0 | 5 votes |
private String encode(String value) { Preconditions.checkNotNull(value); try { return charset == null ? value : URLEncoder.encode(value, charset); } catch (UnsupportedEncodingException e) { throw new SpiderException(e); } }
Example #29
Source File: URLParser.java From NetDiscovery with Apache License 2.0 | 5 votes |
private static List<String> getOrCreate(Map<String, List<String>> map, String name) { Preconditions.checkNotNull(name); List<String> list = map.get(name); if (list == null) { list = new ArrayList<String>(); map.put(name, list); } return list; }
Example #30
Source File: URLParser.java From NetDiscovery with Apache License 2.0 | 5 votes |
private static LinkedHashMap<String, List<String>> parseQueryString(String query) { LinkedHashMap<String, List<String>> params = new LinkedHashMap<String, List<String>>(); if (Preconditions.isBlank(query)) { return params; } String[] items = query.split("&"); for (String item : items) { String name = substringBefore(item, "="); String value = substringAfter(item, "="); List<String> values = getOrCreate(params, name); values.add(value); } return params; }