import glob import sh import subprocess from multiprocessing import cpu_count from os import environ, utime from os.path import dirname, exists, join from pathlib import Path import shutil from pythonforandroid.logger import info, warning, shprint from pythonforandroid.patching import version_starts_with from pythonforandroid.recipe import Recipe, TargetPythonRecipe from pythonforandroid.util import ( current_directory, ensure_dir, walk_valid_filens, BuildInterruptingException, ) NDK_API_LOWER_THAN_SUPPORTED_MESSAGE = ( 'Target ndk-api is {ndk_api}, ' 'but the python3 recipe supports only {min_ndk_api}+' ) class Python3Recipe(TargetPythonRecipe): ''' The python3's recipe ^^^^^^^^^^^^^^^^^^^^ The python 3 recipe can be built with some extra python modules, but to do so, we need some libraries. By default, we ship the python3 recipe with some common libraries, defined in ``depends``. We also support some optional libraries, which are less common that the ones defined in ``depends``, so we added them as optional dependencies (``opt_depends``). Below you have a relationship between the python modules and the recipe libraries:: - _ctypes: you must add the recipe for ``libffi``. - _sqlite3: you must add the recipe for ``sqlite3``. - _ssl: you must add the recipe for ``openssl``. - _bz2: you must add the recipe for ``libbz2`` (optional). - _lzma: you must add the recipe for ``liblzma`` (optional). .. note:: This recipe can be built only against API 21+. .. versionchanged:: 2019.10.06.post0 - Refactored from deleted class ``python.GuestPythonRecipe`` into here - Added optional dependencies: :mod:`~pythonforandroid.recipes.libbz2` and :mod:`~pythonforandroid.recipes.liblzma` .. versionchanged:: 0.6.0 Refactored into class :class:`~pythonforandroid.python.GuestPythonRecipe` ''' version = '3.10.10' url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz' name = 'python3' patches = [ 'patches/pyconfig_detection.patch', 'patches/reproducible-buildinfo.diff', # Python 3.7.1 ('patches/py3.7.1_fix-ctypes-util-find-library.patch', version_starts_with("3.7")), ('patches/py3.7.1_fix-zlib-version.patch', version_starts_with("3.7")), # Python 3.8.1 & 3.9.X ('patches/py3.8.1.patch', version_starts_with("3.8")), ('patches/py3.8.1.patch', version_starts_with("3.9")), ('patches/py3.8.1.patch', version_starts_with("3.10")) ] if shutil.which('lld') is not None: patches = patches + [ ("patches/py3.7.1_fix_cortex_a8.patch", version_starts_with("3.7")), ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.8")), ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.9")), ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.10")) ] depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi'] # those optional depends allow us to build python compression modules: # - _bz2.so # - _lzma.so opt_depends = ['libbz2', 'liblzma'] '''The optional libraries which we would like to get our python linked''' configure_args = ( '--host={android_host}', '--build={android_build}', '--enable-shared', '--enable-ipv6', 'ac_cv_file__dev_ptmx=yes', 'ac_cv_file__dev_ptc=no', '--without-ensurepip', 'ac_cv_little_endian_double=yes', 'ac_cv_header_sys_eventfd_h=no', '--prefix={prefix}', '--exec-prefix={exec_prefix}', '--enable-loadable-sqlite-extensions') '''The configure arguments needed to build the python recipe. Those are used in method :meth:`build_arch` (if not overwritten like python3's recipe does). ''' MIN_NDK_API = 21 '''Sets the minimal ndk api number needed to use the recipe. .. warning:: This recipe can be built only against API 21+, so it means that any class which inherits from class:`GuestPythonRecipe` will have this limitation. ''' stdlib_dir_blacklist = { '__pycache__', 'test', 'tests', 'lib2to3', 'ensurepip', 'idlelib', 'tkinter', } '''The directories that we want to omit for our python bundle''' stdlib_filen_blacklist = [ '*.py', '*.exe', '*.whl', ] '''The file extensions that we want to blacklist for our python bundle''' site_packages_dir_blacklist = { '__pycache__', 'tests' } '''The directories from site packages dir that we don't want to be included in our python bundle.''' site_packages_filen_blacklist = [ '*.py' ] '''The file extensions from site packages dir that we don't want to be included in our python bundle.''' compiled_extension = '.pyc' '''the default extension for compiled python files. .. note:: the default extension for compiled python files has been .pyo for python 2.x-3.4 but as of Python 3.5, the .pyo filename extension is no longer used and has been removed in favour of extension .pyc ''' def __init__(self, *args, **kwargs): self._ctx = None super().__init__(*args, **kwargs) @property def _libpython(self): '''return the python's library name (with extension)''' return 'libpython{link_version}.so'.format( link_version=self.link_version ) @property def link_version(self): '''return the python's library link version e.g. 3.7m, 3.8''' major, minor = self.major_minor_version_string.split('.') flags = '' if major == '3' and int(minor) < 8: flags += 'm' return '{major}.{minor}{flags}'.format( major=major, minor=minor, flags=flags ) def include_root(self, arch_name): return join(self.get_build_dir(arch_name), 'Include') def link_root(self, arch_name): return join(self.get_build_dir(arch_name), 'android-build') def should_build(self, arch): return not Path(self.link_root(arch.arch), self._libpython).is_file() def prebuild_arch(self, arch): super().prebuild_arch(arch) self.ctx.python_recipe = self def get_recipe_env(self, arch=None, with_flags_in_cc=True): env = super().get_recipe_env(arch) env['HOSTARCH'] = arch.command_prefix env['CC'] = arch.get_clang_exe(with_target=True) env['PATH'] = ( '{hostpython_dir}:{old_path}').format( hostpython_dir=self.get_recipe( 'host' + self.name, self.ctx).get_path_to_python(), old_path=env['PATH']) env['CFLAGS'] = ' '.join( [ '-fPIC', '-DANDROID' ] ) env['LDFLAGS'] = env.get('LDFLAGS', '') if shutil.which('lld') is not None: # Note: The -L. is to fix a bug in python 3.7. # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=234409 env['LDFLAGS'] += ' -L. -fuse-ld=lld' else: warning('lld not found, linking without it. ' 'Consider installing lld if linker errors occur.') return env def set_libs_flags(self, env, arch): '''Takes care to properly link libraries with python depending on our requirements and the attribute :attr:`opt_depends`. ''' def add_flags(include_flags, link_dirs, link_libs): env['CPPFLAGS'] = env.get('CPPFLAGS', '') + include_flags env['LDFLAGS'] = env.get('LDFLAGS', '') + link_dirs env['LIBS'] = env.get('LIBS', '') + link_libs if 'sqlite3' in self.ctx.recipe_build_order: info('Activating flags for sqlite3') recipe = Recipe.get_recipe('sqlite3', self.ctx) add_flags(' -I' + recipe.get_build_dir(arch.arch), ' -L' + recipe.get_lib_dir(arch), ' -lsqlite3') if 'libffi' in self.ctx.recipe_build_order: info('Activating flags for libffi') recipe = Recipe.get_recipe('libffi', self.ctx) # In order to force the correct linkage for our libffi library, we # set the following variable to point where is our libffi.pc file, # because the python build system uses pkg-config to configure it. env['PKG_CONFIG_PATH'] = recipe.get_build_dir(arch.arch) add_flags(' -I' + ' -I'.join(recipe.get_include_dirs(arch)), ' -L' + join(recipe.get_build_dir(arch.arch), '.libs'), ' -lffi') if 'openssl' in self.ctx.recipe_build_order: info('Activating flags for openssl') recipe = Recipe.get_recipe('openssl', self.ctx) self.configure_args += \ ('--with-openssl=' + recipe.get_build_dir(arch.arch),) add_flags(recipe.include_flags(arch), recipe.link_dirs_flags(arch), recipe.link_libs_flags()) for library_name in {'libbz2', 'liblzma'}: if library_name in self.ctx.recipe_build_order: info(f'Activating flags for {library_name}') recipe = Recipe.get_recipe(library_name, self.ctx) add_flags(recipe.get_library_includes(arch), recipe.get_library_ldflags(arch), recipe.get_library_libs_flag()) # python build system contains hardcoded zlib version which prevents # the build of zlib module, here we search for android's zlib version # and sets the right flags, so python can be build with android's zlib info("Activating flags for android's zlib") zlib_lib_path = arch.ndk_lib_dir_versioned zlib_includes = self.ctx.ndk.sysroot_include_dir zlib_h = join(zlib_includes, 'zlib.h') try: with open(zlib_h) as fileh: zlib_data = fileh.read() except IOError: raise BuildInterruptingException( "Could not determine android's zlib version, no zlib.h ({}) in" " the NDK dir includes".format(zlib_h) ) for line in zlib_data.split('\n'): if line.startswith('#define ZLIB_VERSION '): break else: raise BuildInterruptingException( 'Could not parse zlib.h...so we cannot find zlib version,' 'required by python build,' ) env['ZLIB_VERSION'] = line.replace('#define ZLIB_VERSION ', '') add_flags(' -I' + zlib_includes, ' -L' + zlib_lib_path, ' -lz') return env def build_arch(self, arch): if self.ctx.ndk_api < self.MIN_NDK_API: raise BuildInterruptingException( NDK_API_LOWER_THAN_SUPPORTED_MESSAGE.format( ndk_api=self.ctx.ndk_api, min_ndk_api=self.MIN_NDK_API ), ) recipe_build_dir = self.get_build_dir(arch.arch) # Create a subdirectory to actually perform the build build_dir = join(recipe_build_dir, 'android-build') ensure_dir(build_dir) # TODO: Get these dynamically, like bpo-30386 does sys_prefix = '/usr/local' sys_exec_prefix = '/usr/local' env = self.get_recipe_env(arch) env = self.set_libs_flags(env, arch) android_build = sh.Command( join(recipe_build_dir, 'config.guess'))().stdout.strip().decode('utf-8') with current_directory(build_dir): if not exists('config.status'): shprint( sh.Command(join(recipe_build_dir, 'configure')), *(' '.join(self.configure_args).format( android_host=env['HOSTARCH'], android_build=android_build, prefix=sys_prefix, exec_prefix=sys_exec_prefix)).split(' '), _env=env) shprint( sh.make, 'all', '-j', str(cpu_count()), 'INSTSONAME={lib_name}'.format(lib_name=self._libpython), _env=env ) # TODO: Look into passing the path to pyconfig.h in a # better way, although this is probably acceptable sh.cp('pyconfig.h', join(recipe_build_dir, 'Include')) def compile_python_files(self, dir): ''' Compile the python files (recursively) for the python files inside a given folder. .. note:: python2 compiles the files into extension .pyo, but in python3, and as of Python 3.5, the .pyo filename extension is no longer used...uses .pyc (https://www.python.org/dev/peps/pep-0488) ''' args = [self.ctx.hostpython] args += ['-OO', '-m', 'compileall', '-b', '-f', dir] subprocess.call(args) def create_python_bundle(self, dirn, arch): """ Create a packaged python bundle in the target directory, by copying all the modules and standard library to the right place. """ # Todo: find a better way to find the build libs folder modules_build_dir = join( self.get_build_dir(arch.arch), 'android-build', 'build', 'lib.linux{}-{}-{}'.format( '2' if self.version[0] == '2' else '', arch.command_prefix.split('-')[0], self.major_minor_version_string )) # Compile to *.pyc the python modules self.compile_python_files(modules_build_dir) # Compile to *.pyc the standard python library self.compile_python_files(join(self.get_build_dir(arch.arch), 'Lib')) # Compile to *.pyc the other python packages (site-packages) self.compile_python_files(self.ctx.get_python_install_dir(arch.arch)) # Bundle compiled python modules to a folder modules_dir = join(dirn, 'modules') c_ext = self.compiled_extension ensure_dir(modules_dir) module_filens = (glob.glob(join(modules_build_dir, '*.so')) + glob.glob(join(modules_build_dir, '*' + c_ext))) info("Copy {} files into the bundle".format(len(module_filens))) for filen in module_filens: info(" - copy {}".format(filen)) shutil.copy2(filen, modules_dir) # zip up the standard library stdlib_zip = join(dirn, 'stdlib.zip') with current_directory(join(self.get_build_dir(arch.arch), 'Lib')): stdlib_filens = list(walk_valid_filens( '.', self.stdlib_dir_blacklist, self.stdlib_filen_blacklist)) if 'SOURCE_DATE_EPOCH' in environ: # for reproducible builds stdlib_filens.sort() timestamp = int(environ['SOURCE_DATE_EPOCH']) for filen in stdlib_filens: utime(filen, (timestamp, timestamp)) info("Zip {} files into the bundle".format(len(stdlib_filens))) shprint(sh.zip, '-X', stdlib_zip, *stdlib_filens) # copy the site-packages into place ensure_dir(join(dirn, 'site-packages')) ensure_dir(self.ctx.get_python_install_dir(arch.arch)) # TODO: Improve the API around walking and copying the files with current_directory(self.ctx.get_python_install_dir(arch.arch)): filens = list(walk_valid_filens( '.', self.site_packages_dir_blacklist, self.site_packages_filen_blacklist)) info("Copy {} files into the site-packages".format(len(filens))) for filen in filens: info(" - copy {}".format(filen)) ensure_dir(join(dirn, 'site-packages', dirname(filen))) shutil.copy2(filen, join(dirn, 'site-packages', filen)) # copy the python .so files into place python_build_dir = join(self.get_build_dir(arch.arch), 'android-build') python_lib_name = 'libpython' + self.link_version shprint( sh.cp, join(python_build_dir, python_lib_name + '.so'), join(self.ctx.bootstrap.dist_dir, 'libs', arch.arch) ) info('Renaming .so files to reflect cross-compile') self.reduce_object_file_names(join(dirn, 'site-packages')) return join(dirn, 'site-packages') recipe = Python3Recipe()