#!/usr/bin/env python3 # Copyright 2020 The Emscripten Authors. All rights reserved. # Emscripten is available under two separate licenses, the MIT license and the # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. """Updates the python binaries that we cache store at http://storage.google.com/webassembly. We only supply binaries for windows and macOS, but we do it very different ways for those two OSes. Windows recipe: 1. Download the "embeddable zip file" version of python from python.org 2. Remove .pth file to work around https://bugs.python.org/issue34841 3. Download and install pywin32 in the `site-packages` directory 4. Re-zip and upload to storage.google.com macOS recipe: 1. Clone cpython 2. Use homebrew to install and configure openssl (for static linking!) 3. Build cpython from source and use `make install` to create archive. """ import glob import multiprocessing import os import platform import urllib.request import shutil import subprocess import sys from subprocess import check_call from zip import unzip_cmd, zip_cmd version = '3.9.2' major_minor_version = '.'.join(version.split('.')[:2]) # e.g. '3.9.2' -> '3.9' download_url = 'https://www.nuget.org/api/v2/package/python/%s' % version # This is not part of official Python version, but a repackaging number appended by emsdk # when a version of Python needs to be redownloaded. revision = '4' pywin32_version = '227' pywin32_base = 'https://github.com/mhammond/pywin32/releases/download/b%s/' % pywin32_version upload_base = 'gs://webassembly/emscripten-releases-builds/deps/' def make_python_patch(): pywin32_filename = 'pywin32-%s.win-amd64-py%s.exe' % (pywin32_version, major_minor_version) filename = 'python-%s-amd64.zip' % (version) out_filename = 'python-%s-%s-amd64+pywin32.zip' % (version, revision) if not os.path.exists(pywin32_filename): url = pywin32_base + pywin32_filename print('Downloading pywin32: ' + url) urllib.request.urlretrieve(url, pywin32_filename) if not os.path.exists(filename): print(f'Downloading python: {download_url} to {filename}') urllib.request.urlretrieve(download_url, filename) os.mkdir('python-nuget') check_call(unzip_cmd() + [os.path.abspath(filename)], cwd='python-nuget') os.remove(filename) os.mkdir('pywin32') rtn = subprocess.call(unzip_cmd() + [os.path.abspath(pywin32_filename)], cwd='pywin32') assert rtn in [0, 1] os.mkdir(os.path.join('python-nuget', 'lib')) shutil.move(os.path.join('pywin32', 'PLATLIB'), os.path.join('python-nuget', 'toolss', 'Lib', 'site-packages')) check_call(zip_cmd() + [os.path.join('..', '..', out_filename), '.'], cwd='python-nuget/tools') print('Created: %s' % out_filename) # cleanup if everything went fine shutil.rmtree('python-nuget') shutil.rmtree('pywin32') if '--upload' in sys.argv: upload_url = upload_base + out_filename print('Uploading: ' + upload_url) cmd = ['gsutil', 'cp', '-n', out_filename, upload_url] print(' '.join(cmd)) check_call(cmd) def build_python(): if sys.platform.startswith('darwin'): # Take some rather drastic steps to link openssl and liblzma statically # and avoid linking libintl completely. osname = 'macos' check_call(['brew', 'install', 'openssl', 'xz', 'pkg-config']) if platform.machine() == 'x86_64': prefix = '/usr/local' min_macos_version = '10.11' elif platform.machine() == 'arm64': prefix = '/opt/homebrew' min_macos_version = '11.0' # Append '-x86_64' or '-arm64' depending on current arch. (TODO: Do # this for Linux too, move this below?) osname += '-' + platform.machine() for f in [os.path.join(prefix, 'lib', 'libintl.dylib'), os.path.join(prefix, 'include', 'libintl.h'), os.path.join(prefix, 'opt', 'xz', 'lib', 'liblzma.dylib'), os.path.join(prefix, 'opt', 'openssl', 'lib', 'libssl.dylib'), os.path.join(prefix, 'opt', 'openssl', 'lib', 'libcrypto.dylib')]: if os.path.exists(f): os.remove(f) os.environ['PKG_CONFIG_PATH'] = os.path.join(prefix, 'opt', 'openssl', 'lib', 'pkgconfig') else: osname = 'linux' src_dir = 'cpython' if not os.path.exists(src_dir): check_call(['git', 'clone', 'https://github.com/python/cpython']) check_call(['git', 'checkout', 'v' + version], cwd=src_dir) env = os.environ if sys.platform.startswith('darwin'): # Specify the min OS version we want the build to work on min_macos_version_line = '-mmacosx-version-min=' + min_macos_version build_flags = min_macos_version_line + ' -Werror=partial-availability' # Build against latest SDK, but issue an error if using any API that would not work on the min OS version env = env.copy() env['MACOSX_DEPLOYMENT_TARGET'] = min_macos_version configure_args = ['CFLAGS=' + build_flags, 'CXXFLAGS=' + build_flags, 'LDFLAGS=' + min_macos_version_line] else: configure_args = [] check_call(['./configure'] + configure_args, cwd=src_dir, env=env) check_call(['make', '-j', str(multiprocessing.cpu_count())], cwd=src_dir, env=env) check_call(['make', 'install', 'DESTDIR=install'], cwd=src_dir, env=env) install_dir = os.path.join(src_dir, 'install') # Install requests module. This is needed in particular on macOS to ensure # SSL certificates are available (certifi in installed and used by requests). pybin = os.path.join(src_dir, 'install', 'usr', 'local', 'bin', 'python3') pip = os.path.join(src_dir, 'install', 'usr', 'local', 'bin', 'pip3') check_call([pybin, '-m', 'ensurepip', '--upgrade']) check_call([pybin, pip, 'install', 'requests==2.32.3']) # Install psutil module. This is needed by emrun to track when browser # process quits. check_call([pybin, pip, 'install', 'psutil']) dirname = 'python-%s-%s' % (version, revision) if os.path.isdir(dirname): print('Erasing old build directory ' + dirname) shutil.rmtree(dirname) os.rename(os.path.join(install_dir, 'usr', 'local'), dirname) tarball = 'python-%s-%s-%s.tar.gz' % (version, revision, osname) shutil.rmtree(os.path.join(dirname, 'lib', 'python' + major_minor_version, 'test')) shutil.rmtree(os.path.join(dirname, 'include')) for lib in glob.glob(os.path.join(dirname, 'lib', 'lib*.a')): os.remove(lib) check_call(['tar', 'zcvf', tarball, dirname]) print('Created: %s' % tarball) if '--upload' in sys.argv: print('Uploading: ' + upload_base + tarball) check_call(['gsutil', 'cp', '-n', tarball, upload_base + tarball]) def main(): if sys.platform.startswith('win') or '--win32' in sys.argv: make_python_patch() else: build_python() return 0 if __name__ == '__main__': sys.exit(main())