[WIP] toolchain: auto resolve deps

Author: T-Dynamos

pythonforandroid/recipe.py

@@ -979,7 +979,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         env['LANG'] = "en_GB.UTF-8"
 
         # Binaries made by packages installed by pip
-        self.patch_shebangs(self._host_recipe.local_bin, self.real_hostpython_location)
+        self.patch_shebangs(self._host_recipe.local_bin, self._host_recipe.python_exe)
         env["PATH"] = self._host_recipe.local_bin + ":" + self._host_recipe.site_bin + ":" + env["PATH"]
 
         host_env = self.get_hostrecipe_env(arch)
@@ -1022,10 +1022,9 @@ def install_python_package(self, arch, name=None, env=None, is_dir=True):
 
         info('Installing {} into site-packages'.format(self.name))
 
-        hostpython = sh.Command(self.hostpython_location)
         hpenv = env.copy()
         with current_directory(self.get_build_dir(arch.arch)):
-            shprint(hostpython, '-m', 'pip', 'install', '.',
+            shprint(self._host_recipe.pip, 'install', '.',
                     '--compile', '--target',
                     self.ctx.get_python_install_dir(arch.arch),
                     _env=hpenv, *self.setup_extra_args
@@ -1045,8 +1044,7 @@ def hostpython_site_dir(self):
 
     def install_hostpython_package(self, arch):
         env = self.get_hostrecipe_env(arch)
-        real_hostpython = sh.Command(self.real_hostpython_location)
-        shprint(real_hostpython, '-m', 'pip', 'install', '.',
+        shprint(self._host_recipe.pip, 'install', '.',
                 '--compile',
                 '--root={}'.format(self._host_recipe.site_root),
                 _env=env, *self.setup_extra_args)
@@ -1075,8 +1073,7 @@ def install_hostpython_prerequisites(self, packages=None, force_upgrade=True):
             pip_options.append("--upgrade")
         # Use system's pip
         pip_env = self.get_hostrecipe_env()
-        pip_env["HOME"] = "/tmp"
-        shprint(sh.Command(self.real_hostpython_location), "-m", "pip", *pip_options, _env=pip_env)
+        shprint(self._host_recipe.pip, *pip_options, _env=pip_env)
 
     def restore_hostpython_prerequisites(self, packages):
         _packages = []
@@ -1270,10 +1267,14 @@ def get_recipe_env(self, arch, **kwargs):
         return env
 
     def get_wheel_platform_tag(self, arch):
+        # https://peps.python.org/pep-0738/#packaging
+        # official python only supports 64 bit:
+        # android_21_arm64_v8a
+        # android_21_x86_64
         return f"android_{self.ctx.ndk_api}_" + {
-            "armeabi-v7a": "arm",
-            "arm64-v8a": "aarch64",
+            "arm64-v8a": "arm64_v8a",
             "x86_64": "x86_64",
+            "armeabi-v7a": "arm",
             "x86": "i686",
         }[arch.arch]
 

pythonforandroid/recipes/hostpython3/__init__.py

@@ -113,6 +113,14 @@ def site_dir(self):
             f"usr/local/lib/python{p_version.major}.{p_version.minor}/site-packages/"
         )
 
+    @property
+    def _pip(self):
+        return join(self.local_bin, "pip3")
+
+    @property
+    def pip(self):
+        return sh.Command(self._pip)
+
     def build_arch(self, arch):
         env = self.get_recipe_env(arch)
 
@@ -160,10 +168,12 @@ def build_arch(self, arch):
 
         ensure_dir(self.site_root)
         self.ctx.hostpython = self.python_exe
+
         if build_configured:
+
             shprint(
                 sh.Command(self.python_exe), "-m", "ensurepip", "--root", self.site_root, "-U",
-                _env={"HOME": "/tmp"}
+                _env={"PATH": self.local_bin}
             )
 
 

pythonforandroid/toolchain.py

@@ -11,6 +11,7 @@
 from functools import wraps
 import glob
 import logging
+import json
 import os
 from os import environ
 from os.path import (join, dirname, realpath, exists, expanduser, basename)
@@ -48,6 +49,9 @@
     max_build_tool_version,
 )
 
+from packaging.utils import parse_wheel_filename
+from packaging.requirements import Requirement
+
 user_dir = dirname(realpath(os.path.curdir))
 toolchain_dir = dirname(__file__)
 sys.path.insert(0, join(toolchain_dir, "tools", "external"))
@@ -121,6 +125,92 @@ def dist_from_args(ctx, args):
         allow_replace_dist=args.allow_replace_dist)
 
 
+def is_wheel_platform_independent(whl_name):
+    name, version, build, tags = parse_wheel_filename(whl_name)
+    return all(tag.platform == "any" for tag in tags)
+
+
+def process_python_modules(build_order, modules):
+    """
+    idk what this does
+    """
+    modules = list(modules)
+    build_order = list(build_order)
+    _requirement_names = []
+    processed_modules = []
+
+    for module in modules+build_order:
+        try:
+            _requirement_names.append(Requirement(module).name)
+        except Exception:
+            processed_modules.append(module)
+            if module in modules:
+                modules.remove(module)
+
+    if len(processed_modules) > 0:
+        warning(f'Ignored by module resolver : {processed_modules}')
+
+    processed_modules.extend(modules)
+
+    # need a tempfile for pip report output
+    path = os.path.realpath(".pip_report.json")
+
+    if not exists(path):
+        shprint(
+            sh.pip, 'install', *modules,
+            '--dry-run', '--break-system-packages', '--ignore-installed',
+            '--report', path, '-q'
+        )
+
+    with open(path, "r") as f:
+        try:
+            report = json.load(f)
+        except Exception:
+            os.remove(path)
+            return process_python_modules(modules)
+
+    info('Extra resolved pure python dependencies :')
+
+    ignored_str = " (ignored)"
+    # did we find any non pure python package?
+    any_not_pure_python = False
+
+    info(" ")
+    for module in report["install"]:
+
+        mname = module["metadata"]["name"]
+        mver = module["metadata"]["version"]
+        filename = basename(module["download_info"]["url"])
+        pure_python = True
+
+        if (filename.endswith(".whl") and not is_wheel_platform_independent(filename)):
+            any_not_pure_python = True
+            pure_python = False
+
+        # does this module matches any recipe name?
+        if mname.lower() in _requirement_names:
+            continue
+
+        color = Out_Fore.GREEN if pure_python else Out_Fore.RED
+        ignored = "" if pure_python else ignored_str
+
+        info(
+            f"  {color}{mname}{Out_Fore.WHITE} : "
+            f"{Out_Style.BRIGHT}{mver}{Out_Style.RESET_ALL}"
+            f"{ignored}"
+        )
+
+        if pure_python:
+            processed_modules.append(f"{mname}=={mver}")
+    info(" ")
+
+    if any_not_pure_python:
+        warning("Some packages were ignored because they are not pure Python.")
+        warning("To install the ignored packages, explicitly list them in your requirements file.")
+
+    return processed_modules
+
+
 def build_dist_from_args(ctx, dist, args):
     """Parses out any bootstrap related arguments, and uses them to build
     a dist."""
@@ -134,8 +224,9 @@ def build_dist_from_args(ctx, dist, args):
             blacklist=blacklist
         ))
     assert set(build_order).intersection(set(python_modules)) == set()
+
     ctx.recipe_build_order = build_order
-    ctx.python_modules = python_modules
+    ctx.python_modules = process_python_modules(build_order, python_modules)
 
     info('The selected bootstrap is {}'.format(bs.name))
     info_main('# Creating dist with {} bootstrap'.format(bs.name))