Skip to content

Commit de0025d

Browse files
bollwyvlbollwyvl
committed
line endings
1 parent 6600bfd commit de0025d

File tree

1 file changed

+151
-151
lines changed

1 file changed

+151
-151
lines changed

‎pyppeteer/chromium_downloader.py‎

Lines changed: 151 additions & 151 deletions
Original file line numberDiff line numberDiff line change
@@ -1,151 +1,151 @@
1-
#!/usr/bin/env python3
2-
# -*- coding: utf-8 -*-
3-
4-
"""Chromium download module."""
5-
6-
import logging
7-
import os
8-
import stat
9-
import sys
10-
from io import BytesIO
11-
from pathlib import Path
12-
from zipfile import ZipFile
13-
14-
import certifi
15-
import urllib3
16-
from pyppeteer import __chromium_revision__, __pyppeteer_home__
17-
from tqdm import tqdm
18-
19-
logger = logging.getLogger(__name__)
20-
21-
DOWNLOADS_FOLDER = Path(__pyppeteer_home__) / 'local-chromium'
22-
DEFAULT_DOWNLOAD_HOST = 'https://storage.googleapis.com'
23-
DOWNLOAD_HOST = os.environ.get('PYPPETEER_DOWNLOAD_HOST', DEFAULT_DOWNLOAD_HOST)
24-
BASE_URL = f'{DOWNLOAD_HOST}/chromium-browser-snapshots'
25-
26-
REVISION = os.environ.get('PYPPETEER_CHROMIUM_REVISION', __chromium_revision__)
27-
28-
NO_PROGRESS_BAR = os.environ.get('PYPPETEER_NO_PROGRESS_BAR', '')
29-
if NO_PROGRESS_BAR.lower() in ('1', 'true'):
30-
NO_PROGRESS_BAR = True # type: ignore
31-
32-
# Windows archive name changed at r591479.
33-
windowsArchive = 'chrome-win' if int(REVISION) > 591479 else 'chrome-win32'
34-
35-
downloadURLs = {
36-
'linux': f'{BASE_URL}/Linux_x64/{REVISION}/chrome-linux.zip',
37-
'mac': f'{BASE_URL}/Mac/{REVISION}/chrome-mac.zip',
38-
'win32': f'{BASE_URL}/Win/{REVISION}/{windowsArchive}.zip',
39-
'win64': f'{BASE_URL}/Win_x64/{REVISION}/{windowsArchive}.zip',
40-
}
41-
42-
chromiumExecutable = {
43-
'linux': DOWNLOADS_FOLDER / REVISION / 'chrome-linux' / 'chrome',
44-
'mac': (DOWNLOADS_FOLDER / REVISION / 'chrome-mac' / 'Chromium.app' / 'Contents' / 'MacOS' / 'Chromium'),
45-
'win32': DOWNLOADS_FOLDER / REVISION / windowsArchive / 'chrome.exe',
46-
'win64': DOWNLOADS_FOLDER / REVISION / windowsArchive / 'chrome.exe',
47-
}
48-
49-
50-
def current_platform() -> str:
51-
"""Get current platform name by short string."""
52-
if sys.platform.startswith('linux'):
53-
return 'linux'
54-
elif sys.platform.startswith('darwin'):
55-
return 'mac'
56-
elif sys.platform.startswith('win') or sys.platform.startswith('msys') or sys.platform.startswith('cyg'):
57-
if sys.maxsize > 2 ** 31 - 1:
58-
return 'win64'
59-
return 'win32'
60-
raise OSError('Unsupported platform: ' + sys.platform)
61-
62-
63-
def get_url() -> str:
64-
"""Get chromium download url."""
65-
return downloadURLs[current_platform()]
66-
67-
68-
def download_zip(url: str) -> BytesIO:
69-
"""Download data from url."""
70-
logger.warning('Starting Chromium download. ' 'Download may take a few minutes.')
71-
72-
with urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) as http:
73-
# Get data from url.
74-
# set preload_content=False means using stream later.
75-
r = http.request('GET', url, preload_content=False)
76-
if r.status >= 400:
77-
raise OSError(f'Chromium downloadable not found at {url}: ' f'Received {r.data.decode()}.\n')
78-
79-
# 10 * 1024
80-
_data = BytesIO()
81-
if NO_PROGRESS_BAR:
82-
for chunk in r.stream(10240):
83-
_data.write(chunk)
84-
else:
85-
try:
86-
total_length = int(r.headers['content-length'])
87-
except (KeyError, ValueError, AttributeError):
88-
total_length = 0
89-
process_bar = tqdm(total=total_length, unit_scale=True, unit='b')
90-
for chunk in r.stream(10240):
91-
_data.write(chunk)
92-
process_bar.update(len(chunk))
93-
process_bar.close()
94-
95-
logger.warning('Chromium download done.')
96-
return _data
97-
98-
99-
def extract_zip(data: BytesIO, path: Path) -> None:
100-
"""Extract zipped data to path."""
101-
# On mac zipfile module cannot extract correctly, so use unzip instead.
102-
if current_platform() == 'mac':
103-
import subprocess
104-
import shutil
105-
106-
zip_path = path / 'chrome.zip'
107-
if not path.exists():
108-
path.mkdir(parents=True)
109-
with zip_path.open('wb') as f:
110-
f.write(data.getvalue())
111-
if not shutil.which('unzip'):
112-
raise OSError('Failed to automatically extract chromium.' f'Please unzip {zip_path} manually.')
113-
proc = subprocess.run(
114-
['unzip', str(zip_path)], cwd=str(path), stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
115-
)
116-
if proc.returncode != 0:
117-
logger.error(proc.stdout.decode())
118-
raise OSError(f'Failed to unzip {zip_path}.')
119-
if chromium_executable().exists() and zip_path.exists():
120-
zip_path.unlink()
121-
else:
122-
with ZipFile(data) as zf:
123-
zf.extractall(str(path))
124-
exec_path = chromium_executable()
125-
if not exec_path.exists():
126-
raise IOError('Failed to extract chromium.')
127-
exec_path.chmod(exec_path.stat().st_mode | stat.S_IXOTH | stat.S_IXGRP | stat.S_IXUSR)
128-
logger.warning(f'chromium extracted to: {path}')
129-
130-
131-
def download_chromium() -> None:
132-
"""Download and extract chromium."""
133-
extract_zip(download_zip(get_url()), DOWNLOADS_FOLDER / REVISION)
134-
135-
136-
def chromium_excutable() -> Path:
137-
"""[Deprecated] miss-spelled function.
138-
Use `chromium_executable` instead.
139-
"""
140-
logger.warning('`chromium_excutable` function is deprecated. ' 'Use `chromium_executable instead.')
141-
return chromium_executable()
142-
143-
144-
def chromium_executable() -> Path:
145-
"""Get path of the chromium executable."""
146-
return chromiumExecutable[current_platform()]
147-
148-
149-
def check_chromium() -> bool:
150-
"""Check if chromium is placed at correct path."""
151-
return chromium_executable().exists()
1+
#!/usr/bin/env python3
2+
# -*- coding: utf-8 -*-
3+
4+
"""Chromium download module."""
5+
6+
import logging
7+
import os
8+
import stat
9+
import sys
10+
from io import BytesIO
11+
from pathlib import Path
12+
from zipfile import ZipFile
13+
14+
import certifi
15+
import urllib3
16+
from pyppeteer import __chromium_revision__, __pyppeteer_home__
17+
from tqdm import tqdm
18+
19+
logger = logging.getLogger(__name__)
20+
21+
DOWNLOADS_FOLDER = Path(__pyppeteer_home__) / 'local-chromium'
22+
DEFAULT_DOWNLOAD_HOST = 'https://storage.googleapis.com'
23+
DOWNLOAD_HOST = os.environ.get('PYPPETEER_DOWNLOAD_HOST', DEFAULT_DOWNLOAD_HOST)
24+
BASE_URL = f'{DOWNLOAD_HOST}/chromium-browser-snapshots'
25+
26+
REVISION = os.environ.get('PYPPETEER_CHROMIUM_REVISION', __chromium_revision__)
27+
28+
NO_PROGRESS_BAR = os.environ.get('PYPPETEER_NO_PROGRESS_BAR', '')
29+
if NO_PROGRESS_BAR.lower() in ('1', 'true'):
30+
NO_PROGRESS_BAR = True # type: ignore
31+
32+
# Windows archive name changed at r591479.
33+
windowsArchive = 'chrome-win' if int(REVISION) > 591479 else 'chrome-win32'
34+
35+
downloadURLs = {
36+
'linux': f'{BASE_URL}/Linux_x64/{REVISION}/chrome-linux.zip',
37+
'mac': f'{BASE_URL}/Mac/{REVISION}/chrome-mac.zip',
38+
'win32': f'{BASE_URL}/Win/{REVISION}/{windowsArchive}.zip',
39+
'win64': f'{BASE_URL}/Win_x64/{REVISION}/{windowsArchive}.zip',
40+
}
41+
42+
chromiumExecutable = {
43+
'linux': DOWNLOADS_FOLDER / REVISION / 'chrome-linux' / 'chrome',
44+
'mac': (DOWNLOADS_FOLDER / REVISION / 'chrome-mac' / 'Chromium.app' / 'Contents' / 'MacOS' / 'Chromium'),
45+
'win32': DOWNLOADS_FOLDER / REVISION / windowsArchive / 'chrome.exe',
46+
'win64': DOWNLOADS_FOLDER / REVISION / windowsArchive / 'chrome.exe',
47+
}
48+
49+
50+
def current_platform() -> str:
51+
"""Get current platform name by short string."""
52+
if sys.platform.startswith('linux'):
53+
return 'linux'
54+
elif sys.platform.startswith('darwin'):
55+
return 'mac'
56+
elif sys.platform.startswith('win') or sys.platform.startswith('msys') or sys.platform.startswith('cyg'):
57+
if sys.maxsize > 2 ** 31 - 1:
58+
return 'win64'
59+
return 'win32'
60+
raise OSError('Unsupported platform: ' + sys.platform)
61+
62+
63+
def get_url() -> str:
64+
"""Get chromium download url."""
65+
return downloadURLs[current_platform()]
66+
67+
68+
def download_zip(url: str) -> BytesIO:
69+
"""Download data from url."""
70+
logger.warning('Starting Chromium download. ' 'Download may take a few minutes.')
71+
72+
with urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) as http:
73+
# Get data from url.
74+
# set preload_content=False means using stream later.
75+
r = http.request('GET', url, preload_content=False)
76+
if r.status >= 400:
77+
raise OSError(f'Chromium downloadable not found at {url}: ' f'Received {r.data.decode()}.\n')
78+
79+
# 10 * 1024
80+
_data = BytesIO()
81+
if NO_PROGRESS_BAR:
82+
for chunk in r.stream(10240):
83+
_data.write(chunk)
84+
else:
85+
try:
86+
total_length = int(r.headers['content-length'])
87+
except (KeyError, ValueError, AttributeError):
88+
total_length = 0
89+
process_bar = tqdm(total=total_length, unit_scale=True, unit='b')
90+
for chunk in r.stream(10240):
91+
_data.write(chunk)
92+
process_bar.update(len(chunk))
93+
process_bar.close()
94+
95+
logger.warning('Chromium download done.')
96+
return _data
97+
98+
99+
def extract_zip(data: BytesIO, path: Path) -> None:
100+
"""Extract zipped data to path."""
101+
# On mac zipfile module cannot extract correctly, so use unzip instead.
102+
if current_platform() == 'mac':
103+
import subprocess
104+
import shutil
105+
106+
zip_path = path / 'chrome.zip'
107+
if not path.exists():
108+
path.mkdir(parents=True)
109+
with zip_path.open('wb') as f:
110+
f.write(data.getvalue())
111+
if not shutil.which('unzip'):
112+
raise OSError('Failed to automatically extract chromium.' f'Please unzip {zip_path} manually.')
113+
proc = subprocess.run(
114+
['unzip', str(zip_path)], cwd=str(path), stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
115+
)
116+
if proc.returncode != 0:
117+
logger.error(proc.stdout.decode())
118+
raise OSError(f'Failed to unzip {zip_path}.')
119+
if chromium_executable().exists() and zip_path.exists():
120+
zip_path.unlink()
121+
else:
122+
with ZipFile(data) as zf:
123+
zf.extractall(str(path))
124+
exec_path = chromium_executable()
125+
if not exec_path.exists():
126+
raise IOError('Failed to extract chromium.')
127+
exec_path.chmod(exec_path.stat().st_mode | stat.S_IXOTH | stat.S_IXGRP | stat.S_IXUSR)
128+
logger.warning(f'chromium extracted to: {path}')
129+
130+
131+
def download_chromium() -> None:
132+
"""Download and extract chromium."""
133+
extract_zip(download_zip(get_url()), DOWNLOADS_FOLDER / REVISION)
134+
135+
136+
def chromium_excutable() -> Path:
137+
"""[Deprecated] miss-spelled function.
138+
Use `chromium_executable` instead.
139+
"""
140+
logger.warning('`chromium_excutable` function is deprecated. ' 'Use `chromium_executable instead.')
141+
return chromium_executable()
142+
143+
144+
def chromium_executable() -> Path:
145+
"""Get path of the chromium executable."""
146+
return chromiumExecutable[current_platform()]
147+
148+
149+
def check_chromium() -> bool:
150+
"""Check if chromium is placed at correct path."""
151+
return chromium_executable().exists()

0 commit comments

Comments
 (0)