# -*- coding:utf-8 -*-
"""
Verion: 1.0
Author: Helixcs
Time: 2019/12/15
Reference :
https://github.com/SeleniumHQ/selenium/wiki/ChromeDriver#requirements
https://www.srcmake.com/home/selenium-python-chromedriver-ubuntu
wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' | sudo tee /etc/apt/sources.list.d/google-chrome.list
sudo apt-get update
sudo apt-get install google-chrome-stable
"""
import time
import base64
import re
import collections
from zipfile import ZipFile
import numpy as np
import os
import sys
import shutil
from typing import Optional, Union
from urllib3.util.url import parse_url
from urllib3 import PoolManager
from urllib3.contrib.socks import SOCKSProxyManager
from selenium import webdriver
from selenium.webdriver import ChromeOptions
from PIL import Image
from slugify import slugify
WINDOWS = os.name == 'nt'
LINUX = sys.platform.startswith('linux')
DARWIN = sys.platform.startswith('darwin')
def is_blank(value: Optional[Union[int, str, dict, list, bytes, tuple, object]]) -> bool:
if value is None:
return True
if isinstance(value, str):
return True if value is None or value.strip('') == '' else False
if isinstance(value, dict):
return True if len(value) < 1 else False
if isinstance(value, list):
return True if len(value) < 1 else False
if isinstance(value, bytes):
return True if value == b'' else False
# (None,None) ==> False
if isinstance(value, tuple):
if len(value) < 1:
return True
for i in value:
if i is not None:
return False
return True
if isinstance(value, set):
return True if len(value) < 1 else False
return value is None
def is_not_blank(value: Optional[Union[int, str, dict, list, tuple,]]) -> bool:
return not is_blank(value=value)
DEFAULT_GCO_SIZE = 12000
DEFAULT_GCO_OUTPUT_PATH = 'output'
DEFAULT_GCO_PARTIAL_PATH = 'partial'
DEFAULT_GCO_INIT_DELAY = 5
class GoogleArtsCrawlerOption(object):
def __init__(self,
url: str = None,
chrome_options: ChromeOptions = None,
webdriver_execute_path: str = None,
size: int = DEFAULT_GCO_SIZE,
init_delay_time: int = DEFAULT_GCO_INIT_DELAY,
blob_loading_delay_time: int = 2,
output_path: str = DEFAULT_GCO_OUTPUT_PATH,
output_filename: str = None,
partial_tmp_path: str = DEFAULT_GCO_PARTIAL_PATH,
need_download_webdrive: bool = False,
need_clear_cache: bool = True,
is_debug: bool = False):
"""
GoogleArtsCrawlerOption
Usage:
```
GoogleArtsCrawlerOption()
.set_url("https://artsandculture.google.com/asset/madame-moitessier/hQFUe-elM1npbw")
.set_chrome_options(chrome_option)
.set_need_download_webdrive(True)
# .set_webdriver_execute_path("webdriver/chromedriver")
.set_partial_tmp_path("custom_partial_dir")
.set_output_path("custom_output_dir")
.set_output_filename("custom.jpg")
.set_need_clear_cache(True)
.prepare_options()
```
:param url: google arts url.
:param chrome_options: chrome options , visit `https://chromedriver.chromium.org/capabilities` for detail.
:param webdriver_execute_path: webdrive executed path , if you do not set , it will auto download.
:param size: webdrive simulated device size , default 120000.
:param init_delay_time: webdrive request url and set `init_delay_time` after render.
:param blob_loading_delay_time: webdrive get image blob delay time if get filed at first time.
:param output_path: custom output dir , default `output`
:param output_filename: custom output filename, default arts name.
:param need_download_webdrive need download webdrive, default False , it will auto download webdrive if set True.
:param partial_tmp_path: custom partial tmp path , it will be deleted after finish, default `blob`.
:param need_clear_cache: auto clear webdriver download tmp and partial images after finished.
:param is_debug:
"""
self._url = url
self._chrome_options = chrome_options
self._webdriver_execute_path = webdriver_execute_path
self._size = size
self._init_delay_time = init_delay_time
self._blob_loading_delay_time = blob_loading_delay_time
self._output_path = output_path
self._output_filename = output_filename
self._partial_tmp_path = partial_tmp_path
self._is_debug: bool = is_debug
self._need_download_webdrive = need_download_webdrive
self._need_clear_cache = need_clear_cache
pass
def prepare_options(self):
if is_blank(self._url):
raise Exception("GoogleArtsCrawlerOption , url is blank!")
uprs = parse_url(url=self._url)
if not uprs.host == 'artsandculture.google.com':
raise Exception("GoogleArtsCrawlerOption, url netloc is not `artsandculture.google.com`")
self._url = "https://{0}{1}".format(uprs.host, uprs.path)
# download webdriver
if self._webdriver_execute_path is None and self._need_download_webdrive:
default_webdrive_path = "webdriver"
if os.path.isdir(default_webdrive_path):
default_webdrive_files = os.listdir(default_webdrive_path)
if len(default_webdrive_files) > 0:
default_webdrive_execute_file = os.path.join(default_webdrive_path, default_webdrive_files[0])
if os.path.isfile(default_webdrive_execute_file):
print("==> webdriver has exist at {0}".format(default_webdrive_execute_file))
self._webdriver_execute_path = default_webdrive_execute_file
if self._webdriver_execute_path is None:
if WINDOWS:
os_name = "Windows"
webdriver_download_url = "http://chromedriver.storage.googleapis.com/78.0.3904.70/chromedriver_win32.zip"
elif DARWIN:
os_name = "Mac OS"
webdriver_download_url = "http://chromedriver.storage.googleapis.com/78.0.3904.70/chromedriver_mac64.zip"
elif LINUX:
os_name = "Linux"
webdriver_download_url = "http://chromedriver.storage.googleapis.com/78.0.3904.70/chromedriver_linux64.zip"
else:
raise Exception("GoogleArtsCrawlerOptions, unknown platform !")
print("==> current operation system : {0}".format(os_name))
print("==> prepare download webdriver : {0}".format(webdriver_download_url))
default_download_tmp = "tmp"
webdriver_zip_filename = webdriver_download_url.split("/")[-1]
webdriver_local_zip_filepath = os.path.join(default_download_tmp, webdriver_zip_filename)
# not exist
if not os.path.isfile(webdriver_local_zip_filepath):
# http = SOCKSProxyManager('socks5://localhost:1086/')
http = PoolManager()
response = http.request('GET', webdriver_download_url, preload_content=False)
if not os.path.isdir(default_download_tmp):
os.mkdir(default_download_tmp)
with open(webdriver_local_zip_filepath, mode="wb") as fd:
while True:
data = response.read(1024)
if not data:
break
fd.write(data)
response.release_conn()
print("==> webdriver zip file download finished , location at : {0}".format(
os.path.abspath(webdriver_local_zip_filepath)))
else:
print("==> webdriver zip file has existed at {0}".format(webdriver_local_zip_filepath))
with ZipFile(webdriver_local_zip_filepath, 'r') as zipfile:
zipfile.extractall(path=default_webdrive_path)
if self._need_clear_cache:
shutil.rmtree(default_download_tmp)
self._webdriver_execute_path = os.path.join(default_webdrive_path, os.listdir(default_webdrive_path)[0])
if is_blank(self._webdriver_execute_path):
raise Exception("GoogleArtsCrawlerOption , webdriver_execute_path is blank!")
if not os.path.isfile(self._webdriver_execute_path):
raise Exception("GoogleArtsCrawlerOption , webdriver_execute_path is not exist, this is file!")
if LINUX or DARWIN:
os.chmod(self._webdriver_execute_path, 0o777)
# self._chrome_options.binary_location = os.path.abspath(self._webdriver_execute_path)
mobile_emulation = {
"deviceMetrics": {"width": self._size, "height": self._size, "pixelRatio": 1.0},
"userAgent": "Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 "
"(KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19"}
self._chrome_options.add_experimental_option("mobileEmulation", mobile_emulation)
self._chrome_options.add_argument('--no-sandbox')
self._chrome_options.add_argument('--disable-dev-shm-usage')
self._chrome_options.add_argument('--disable-gpu')
self._chrome_options.add_argument("--disable-dev-shm-usage")
self._chrome_options.add_argument("start-maximized")
self._chrome_options.add_argument("disable-infobars")
self._chrome_options.add_argument("--disable-extensions")
if not self._is_debug:
self._chrome_options.add_argument("--headless")
self._output_path = DEFAULT_GCO_OUTPUT_PATH if self._output_path is None else self._output_path
self._size = DEFAULT_GCO_SIZE if self._size is None or self._size < 1 else self._size
self._init_delay_time = DEFAULT_GCO_INIT_DELAY if self._init_delay_time is None or self._init_delay_time < 1 else self._init_delay_time
if not os.path.isdir(self._output_path):
os.makedirs(self._output_path)
if not os.path.isdir(self._partial_tmp_path):
os.makedirs(self._partial_tmp_path)
if self._is_debug:
print("GoogleArtsCrawlerOptions:")
print("==> url:{0}".format(self._url))
print("==> webdriver_execute_path:{0}".format(os.path.abspath(self._webdriver_execute_path)))
print("==> output :{0}".format(os.path.abspath(self._output_path)))
return self
@property
def url(self) -> str:
return self._url
def set_url(self, url):
self._url = url
return self
@property
def chrome_options(self) -> ChromeOptions:
return self._chrome_options
def set_chrome_options(self, chrome_options: ChromeOptions):
self._chrome_options = chrome_options
return self
@property
def webdriver_execute_path(self) -> str:
return self._webdriver_execute_path
def set_webdriver_execute_path(self, webdriver_execute_path: str):
self._webdriver_execute_path = webdriver_execute_path
return self
@property
def size(self) -> int:
return self._size
def set_size(self, size: int):
self._size = size
return self
@property
def init_delay(self) -> int:
return self._init_delay_time
def set_init_delay(self, init_delay_time: int):
self._init_delay_time = init_delay_time
return self
@property
def blob_loading_delay_time(self) -> int:
return self._blob_loading_delay_time
def set_blob_loading_delay_time(self, blob_loading_delay_time: int):
self._blob_loading_delay_time = blob_loading_delay_time
return self
@property
def output_path(self) -> str:
return self._output_path
def set_output_path(self, output_path: str):
self._output_path = output_path
return self
@property
def output_filename(self) -> str:
return self._output_filename
def set_output_filename(self, output_filename):
self._output_filename = output_filename
return self
@property
def partial_tmp_path(self):
return self._partial_tmp_path
def set_partial_tmp_path(self, partial_tmp_path: str):
self._partial_tmp_path = partial_tmp_path
return self
@property
def need_download_webdrive(self):
return self._need_download_webdrive
def set_need_download_webdrive(self, need_download_webdrive: bool):
self._need_download_webdrive = need_download_webdrive
return self
@property
def need_clear_cache(self):
return self._need_clear_cache
def set_need_clear_cache(self, need_clear_cache: bool):
self._need_clear_cache = need_clear_cache
return self
@property
def is_debug(self):
return self._is_debug
def set_debug(self, is_debug: bool):
self._is_debug = is_debug
return self
class GoogleArtsCrawlerProcess(object):
def __init__(self, gaco: GoogleArtsCrawlerOption):
"""
GoogleArtsCrawlerProcess
Usage:
```
GoogleArtsCrawlerProcess(gaco=GoogleArtsCrawlerOption()).process()
```
:param gaco: GoogleArtsCrawlerOption
"""
self._gaco = gaco
print(os.path.abspath(self._gaco.webdriver_execute_path))
self._browser = webdriver.Chrome(options=self._gaco.chrome_options,
executable_path=self._gaco.webdriver_execute_path)
self._local_partial_tmp = None
@property
def gaco(self):
return self._gaco
def process(self):
self._generate_image()
if self._gaco.need_clear_cache:
self._cleanup()
pass
# get blob content from blob:https://xxxxx
def _get_blob_content(self, uri):
"""
Saves blob to base64.
"""
result = self._browser.execute_async_script("""
var uri = arguments[0];
var callback = arguments[1];
var toBase64 = function(buffer){for(var r,n=new Uint8Array(buffer),t=n.length,a=new Uint8Array(4*Math.ceil(t/3)),i=new Uint8Array(64),o=0,c=0;64>c;++c)i[c]="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".charCodeAt(c);for(c=0;t-t%3>c;c+=3,o+=4)r=n[c]<<16|n[c+1]<<8|n[c+2],a[o]=i[r>>18],a[o+1]=i[r>>12&63],a[o+2]=i[r>>6&63],a[o+3]=i[63&r];return t%3===1?(r=n[t-1],a[o]=i[r>>2],a[o+1]=i[r<<4&63],a[o+2]=61,a[o+3]=61):t%3===2&&(r=(n[t-2]<<8)+n[t-1],a[o]=i[r>>10],a[o+1]=i[r>>4&63],a[o+2]=i[r<<2&63],a[o+3]=61),new TextDecoder("ascii").decode(a)};
var xhr = new XMLHttpRequest();
xhr.responseType = 'arraybuffer';
xhr.onload = function(){ callback(toBase64(xhr.response)) };
xhr.onerror = function(){ callback(xhr.status) };
xhr.open('GET', uri);
xhr.send();
""", uri)
if type(result) == int:
raise Exception("Request failed with status %s" % result)
return base64.b64decode(result)
def _pil_grid(self, images: list, max_horiz: int = np.iinfo(int).max) -> Image:
"""
Generates one image out of many blobs.
"""
n_images = len(images)
n_horiz = min(n_images, max_horiz)
h_sizes, v_sizes = [0] * n_horiz, [0] * (n_images // n_horiz)
for i, im in enumerate(images):
h, v = i % n_horiz, i // n_horiz
h_sizes[h] = max(h_sizes[h], im.size[0])
v_sizes[v] = max(v_sizes[v], im.size[1])
h_sizes, v_sizes = np.cumsum([0] + h_sizes), np.cumsum([0] + v_sizes)
im_grid = Image.new('RGB', (h_sizes[-1], v_sizes[-1]), color='white')
for i, im in enumerate(images):
im_grid.paste(im, (h_sizes[i % n_horiz], v_sizes[i // n_horiz]))
return im_grid
def _cleanup(self):
if self._local_partial_tmp is not None:
shutil.rmtree(self._local_partial_tmp)
# 生成切片图，再组合成一张完整图片
def _generate_image(self):
try:
print("==> staring request:{0}".format(self._gaco.url))
self._browser.get(self._gaco.url)
if self._gaco.init_delay is not None and self._gaco.init_delay > 0:
time.sleep(self._gaco.init_delay)
blobs = self._browser.find_elements_by_tag_name('img')
print("==> get total partial images:{0}".format(len(blobs) - 2))
title = slugify(self._browser.title)
columns = []
rows = []
pil_images = []
i = 0
# 重建切片文件夹
local_tmp_path = os.path.join(self._gaco.partial_tmp_path, title)
self._local_partial_tmp = local_tmp_path
if os.path.exists(local_tmp_path):
shutil.rmtree(local_tmp_path)
os.makedirs(local_tmp_path)
for blob in blobs:
if i > 2:
# Get number of rows and columns
style = blob.get_attribute('style')
style_end_index = style.find(');')
# -4 removes "z" translation
style = style[:style_end_index - 4]
style = style.replace('transform: translate3d(', '')
positions = list(map(int, re.findall(r'\d+', style)))
if len(positions) < 2:
# The positions are not available for this image - skip
continue
columns.append(positions[0])
rows.append(positions[1])
# Save blob to file
partial_image_src = blob.get_attribute('src')
while partial_image_src is None:
if self._gaco.blob_loading_delay_time and self._gaco.blob_loading_delay_time > 0:
time.sleep(self._gaco.blob_loading_delay_time)
partial_image_src = blob.get_attribute('src')
partial_image_content = self._get_blob_content(partial_image_src)
local_partial_filename = os.path.join(local_tmp_path, "{0}.jpg".format(i))
print("===> got blob content:{0} to {1}".format(blob.get_attribute('src'), local_partial_filename))
with open(local_partial_filename, 'wb') as fd:
fd.write(partial_image_content)
fd.flush()
# Create PIL objects list
pil_images.append(Image.open(local_partial_filename))
i += 1
finally:
self._browser.close()
print("==> partial images has downloaded, total:{0}".format(len(blobs) - 2))
columns = len(collections.Counter(columns).keys())
rows = len(collections.Counter(rows).keys())
inverted_pil_images = []
# by default images are crawled in vertical direction
# we re-arrange list to create horizontally sorted list
for j in range(0, rows):
for i in range(0, columns):
inverted_pil_images.append(pil_images[(i * rows) + j])
grid = self._pil_grid(inverted_pil_images, columns)
local_full_output_path = os.path.join(self._gaco.output_path,
"{title}.jpg".format(title=title)
if self._gaco.output_filename is None else self._gaco.output_filename)
grid.save(local_full_output_path)
print("==> Image location: {0}".format(local_full_output_path))
inverted_pil_images = None
pil_images = None
from . import GoogleArtsCrawlerProcess, GoogleArtsCrawlerOption
