Merge branch 'main' into opt

Author: freakboy3742

README.rst

@@ -2,7 +2,8 @@ Python Android Support
 ======================
 
 This is a meta-package for building a version of CPython that can be embedded
-into an Android project. It supports Python versions 3.6, 3.7, and 3.8.
+into an Android project. It supports Python versions 3.7 through 3.10
+(inclusive); experimental support for Py3.11 also exists.
 
 It works by downloading, patching, and building CPython and selected pre-
 requisites, packaging them as linkable dynamic libraries, and packaging
@@ -45,10 +46,15 @@ When you do a local build, you can use the ``support_package = ...`` configurati
 option in a briefcase app's ``pyproject.toml`` to point the app at your local
 support library.
 
-You can run ``python3 test_all_extensions_built.py dist/Python-*-Android-support.zip``
+You can run ``python3 test_all_extensions_built.py dist/Python-*-Android-support.custom.zip``
 to quickly validate that the expected compiled extension modules are available for a
 given build.
 
+You can also use the `Python Support testbed
+<https://github.com/beeware/Python-support-testbed>` project to validate that
+the core pieces of the support package (including modules that have historically
+been problematic) are all present and working.
+
 To run the CPython test suite within an app context, you can add this code to a
 briefcase app::
 
@@ -68,5 +74,3 @@ briefcase app::
 
 Note that you must modify the ``pythonhome`` excludes list for this to work properly,
 which will make the support package larger.
-
-.. _can be downloaded: https://briefcase-support.org/python?platform=android&version=3.7

main.sh

@@ -82,30 +82,40 @@ function build_one_abi() {
     esac
 
     local PYTHON_SOVERSION="${PYTHON_VERSION}"
-    if [ "$PYTHON_VERSION" = "3.7" ] || [ "$PYTHON_VERSION" = "3.6" ] ; then
-        # 3.6 and 3.7 use 3.6m/3.7m
+    local PYTHON_EXTRA_CONFIGURE_FLAGS=""
+    if [ "$PYTHON_VERSION" = "3.7" ] ; then
+        # 3.7 uses 3.7m
         PYTHON_SOVERSION="${PYTHON_VERSION}m"
     fi
 
+    if [ "$PYTHON_VERSION" = "3.11" ] ; then
+        # 3.11 requires --with-build-python
+        PYTHON_EXTRA_CONFIGURE_FLAGS="--with-build-python=python3.11"
+    fi
+
     # We use Docker to run the build. We rely on Docker's build cache to allow the
     # build to be speedy if nothing changed, approx 1-2 seconds for a no-op build.
     # The cache persists even if no Docker "tags" point to the image.
     #
     # We do also use a temporary tag name so that we can use `rsync` to pull some
     # data out of the image.
     #
-    # For debugging the Docker image, you can use the name python-android-support-local:latest.
-    TAG_NAME="python-android-support-local:$(python3 -c 'import random; print(random.randint(0, 1e16))')"
-    DOCKER_BUILDKIT=1 docker build --tag ${TAG_NAME} --tag python-android-support-local:latest \
+    # For debugging the Docker image, you can use the name python-android-support-<version>:<abi>.
+    # e.g. python-android-support-3.7:arm64-v8a
+    TAG_NAME="python-android-support-${PYTHON_VERSION}:${TARGET_ABI_SHORTNAME}"
+    DOCKER_BUILDKIT=1 docker build --platform linux/amd64 --tag ${TAG_NAME} \
         --build-arg PYTHON_VERSION="${PYTHON_VERSION}" --build-arg PYTHON_SOVERSION="${PYTHON_SOVERSION}" \
         --build-arg COMPRESS_LEVEL="${COMPRESS_LEVEL}" --build-arg COMPILER_TRIPLE="${COMPILER_TRIPLE}" \
         --build-arg OPENSSL_BUILD_TARGET="$OPENSSL_BUILD_TARGET" --build-arg TARGET_ABI_SHORTNAME="$TARGET_ABI_SHORTNAME" \
         --build-arg TOOLCHAIN_TRIPLE="$TOOLCHAIN_TRIPLE" --build-arg ANDROID_API_LEVEL="$ANDROID_API_LEVEL" \
+        --build-arg PYTHON_EXTRA_CONFIGURE_FLAGS="$PYTHON_EXTRA_CONFIGURE_FLAGS" \
         -f python.Dockerfile .
     # Extract the build artifacts we need to create our zip file.
     docker run -v "${PWD}"/build/"${PYTHON_VERSION}"/:/mnt/ --rm --entrypoint rsync "$TAG_NAME" -a /opt/python-build/approot/. /mnt/.
     # Extract header files
     docker run -v "${PWD}"/build/"${PYTHON_VERSION}"/app/include/:/mnt/ --rm --entrypoint rsync "$TAG_NAME" -a /opt/python-build/built/python/include/ /mnt/
+    # Extract log files
+    docker run -v "${PWD}"/build/"${PYTHON_VERSION}"/logs/:/mnt/ --rm --entrypoint rsync "$TAG_NAME" -a /opt/python-build/logs/ /mnt/
 
     # Docker creates files as root; reown as the local user
     fix_permissions
@@ -114,8 +124,6 @@ function build_one_abi() {
     mv "${PWD}"/build/"${PYTHON_VERSION}"/app/include/python"${PYTHON_SOVERSION}"/pyconfig.h "${PWD}"/build/"${PYTHON_VERSION}"/app/include/python"${PYTHON_SOVERSION}"/pyconfig-${TARGET_ABI_SHORTNAME}.h
     # Inject a platform-agnostic pyconfig.h wrapper.
     cp "${PWD}/patches/all/pyconfig.h" "${PWD}"/build/"${PYTHON_VERSION}"/app/include/python"${PYTHON_SOVERSION}"/
-    # Remove temporary local tag.
-    docker rmi "$TAG_NAME" > /dev/null
 }
 
 # Download a file into downloads/$name/$filename and verify its sha256sum.
@@ -173,7 +181,7 @@ function main() {
     # DEFAULT_* variables for inclusion into help output.
     local DEFAULT_VERSION="3.7"
     local VERSION="$DEFAULT_VERSION"
-    local DEFAULT_TARGET_ABIS="x86,x86_64,armeabi-v7a,arm64-v8a"
+    local DEFAULT_TARGET_ABIS="x86_64,arm64-v8a"
     local TARGET_ABIS="$DEFAULT_TARGET_ABIS"
     local DEFAULT_COMPRESS_LEVEL="8"
     local COMPRESS_LEVEL="$DEFAULT_COMPRESS_LEVEL"
@@ -200,15 +208,15 @@ function main() {
 
 Build ZIP file of Python resources for Android, including CPython compiled as a .so.
 
--v: Specify Python version to build. For example: -v 3.6
+-v: Specify Python version to build. For example: -v 3.7
     Default: ${DEFAULT_VERSION}
 
 -a: Specify Android ABIs to build, separated by commas. For example: -a x86,arm64-v8a
     Default: ${TARGET_ABIS}
 
 -n: Specify build number. If specified, this gets added to the filename prepended by a dot.
-    For example, -n b5 would create Python-3.6-Android-support.b5.zip when building Python 3.6.
-    By default, e.g. for Python 3.6, we generate the file Python-3.6-Android-support.zip.
+    For example, -n b5 would create Python-3.X-Android-support.b5.zip when building Python 3.X.
+    By default, e.g. for Python 3.X, we generate the file Python-3.X-Android-support.custom.zip.
 
 -z: Specify compression level to use when creating ZIP files. For example, -z 0 is fastest.
     Default: ${DEFAULT_COMPRESS_LEVEL}
@@ -221,7 +229,7 @@ Build ZIP file of Python resources for Android, including CPython compiled as a
     local BUILD_TAG
     if [ -z "${BUILD_NUMBER:-}" ]; then
         echo "Building untagged build"
-        BUILD_TAG=""
+        BUILD_TAG=".custom"
     else
         echo "Building ${BUILD_NUMBER}"
         BUILD_TAG=".${BUILD_NUMBER}"
@@ -230,27 +238,30 @@ Build ZIP file of Python resources for Android, including CPython compiled as a
     echo "Downloading compile-time dependencies."
 
     download jdk "https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u242-b08/OpenJDK8U-jdk_x64_linux_hotspot_8u242b08.tar.gz" "f39b523c724d0e0047d238eb2bb17a9565a60574cf651206c867ee5fc000ab43"
-    download ndk "https://dl.google.com/android/repository/android-ndk-r20b-linux-x86_64.zip" "8381c440fe61fcbb01e209211ac01b519cd6adf51ab1c2281d5daad6ca4c8c8c"
-    download openssl "https://www.openssl.org/source/openssl-1.1.1i.tar.gz" "e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242"
+    download ndk "https://dl.google.com/android/repository/android-ndk-r23b-linux.zip" "c6e97f9c8cfe5b7be0a9e6c15af8e7a179475b7ded23e2d1c1fa0945d6fb4382"
+    download openssl "https://www.openssl.org/source/openssl-1.1.1o.tar.gz" "9384a2b0570dd80358841464677115df785edb941c71211f75076d72fe6b438f"
     download libffi "https://github.com/libffi/libffi/releases/download/v3.3/libffi-3.3.tar.gz" "72fba7922703ddfa7a028d513ac15a85c8d54c8d67f55fa5a4802885dc652056"
     download xz "https://tukaani.org/xz/xz-5.2.5.tar.gz" "f6f4910fd033078738bd82bfba4f49219d03b17eb0794eb91efbae419f4aba10"
     download bzip2 "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz" "ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269"
-    download sqlite3 "http://archive.ubuntu.com/ubuntu/pool/main/s/sqlite3/sqlite3_3.11.0.orig.tar.xz" "79fb8800b8744337d5317270899a5a40612bb76f81517e131bf496c26b044490"
-    download rubicon-java "https://github.com/beeware/rubicon-java/archive/v0.2.5.tar.gz" "23ec17bb5cafec87e286b308f3b0cdd9eb004f59610c9bb37cacfa98ee1ea11c"
+    download sqlite3 "https://github.com/sqlite/sqlite/archive/refs/tags/version-3.35.0.zip" "f85ba70e340428fbf45ed1bf390ddcc622c7f8f4b30e60d063c6b6f8d78924ae"
+    download rubicon-java "https://github.com/beeware/rubicon-java/archive/v0.2.6.tar.gz" "0386d84182b347c0e64947579c3e853e9b72d375094fa6d00942f9a7635ca6d1"
 
     echo "Downloading Python version."
     case "$VERSION" in
-        3.6)
-            download "python-3.6" "https://www.python.org/ftp/python/3.6.12/Python-3.6.12.tar.xz" "70953a9b5d6891d92e65d184c3512126a15814bee15e1eff2ddcce04334e9a99"
-            ;;
         3.7)
-            download "python-3.7" "https://www.python.org/ftp/python/3.7.9/Python-3.7.9.tar.xz" "91923007b05005b5f9bd46f3b9172248aea5abc1543e8a636d59e629c3331b01"
+            download "python-3.7" "https://www.python.org/ftp/python/3.7.13/Python-3.7.13.tar.xz" "99f106275df8899c3e8cb9d7c01ce686c202ef275953301427194693de5bef84"
             ;;
         3.8)
-            download "python-3.8" "https://www.python.org/ftp/python/3.8.7/Python-3.8.7.tar.xz" "ddcc1df16bb5b87aa42ec5d20a5b902f2d088caa269b28e01590f97a798ec50a"
+            download "python-3.8" "https://www.python.org/ftp/python/3.8.13/Python-3.8.13.tar.xz" "6f309077012040aa39fe8f0c61db8c0fa1c45136763299d375c9e5756f09cf57"
             ;;
         3.9)
-            download "python-3.9" "https://www.python.org/ftp/python/3.9.1/Python-3.9.1.tar.xz" "991c3f8ac97992f3d308fefeb03a64db462574eadbff34ce8bc5bb583d9903ff"
+            download "python-3.9" "https://www.python.org/ftp/python/3.9.12/Python-3.9.12.tar.xz" "2cd94b20670e4159c6d9ab57f91dbf255b97d8c1a1451d1c35f4ec1968adf971"
+            ;;
+        3.10)
+            download "python-3.10" "https://www.python.org/ftp/python/3.10.4/Python-3.10.4.tar.xz" "80bf925f571da436b35210886cf79f6eb5fa5d6c571316b73568343451f77a19"
+            ;;
+        3.11)
+            download "python-3.11" "https://www.python.org/ftp/python/3.11.0/Python-3.11.0b1.tar.xz" "dccac9b03dd3fe5cd10bc547579eb0be81a1d8971ec2a866b03dec5391f5ad25"
             ;;
         *)
             echo "Invalid Python version: $VERSION"

patches/3.10/01_python_ssl_module_add_android_certificates

@@ -0,0 +1,48 @@
+From 86b7aee31eb66b99e2e9ff55f83cb2e6393dd8d3 Mon Sep 17 00:00:00 2001
+From: Asheesh Laroia <[email protected]>
+Date: Fri, 19 Jun 2020 16:47:25 -0700
+Subject: [PATCH] Parse certificate authority certs from Android
+ /etc/security/cacerts directory
+
+---
+ Lib/ssl.py           | 11 +++++++++++
+ Lib/test/test_ssl.py |  1 +
+ 2 files changed, 12 insertions(+)
+
+diff --git a/Lib/ssl.py b/Lib/ssl.py
+index 0726caee49..3878bdd8fa 100644
+--- a/Lib/ssl.py
++++ b/Lib/ssl.py
+@@ -572,6 +572,17 @@ class SSLContext(_SSLContext):
+         if sys.platform == "win32":
+             for storename in self._windows_cert_stores:
+                 self._load_windows_store_certs(storename, purpose)
++        if os.path.exists('/etc/security/cacerts'):
++            certs = []
++            for basename in os.listdir('/etc/security/cacerts'):
++                with open('/etc/security/cacerts/' + basename) as fd:
++                    s = fd.read()
++                    if 'END CERTIFICATE' not in s:
++                        continue
++                    lines = s.split('\n')
++                    line_end_certificate = [i for i, line in enumerate(lines) if 'END CERTIFICATE' in line][0]
++                    certs.append('\n'.join(lines[0:line_end_certificate+1]))
++            self.load_verify_locations(None, None, '\n'.join(certs))
+         self.set_default_verify_paths()
+ 
+     if hasattr(_SSLContext, 'minimum_version'):
+diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
+index 0bc0a8c452..3c79debf3e 100644
+--- a/Lib/test/test_ssl.py
++++ b/Lib/test/test_ssl.py
+@@ -1580,6 +1580,7 @@ class ContextTests(unittest.TestCase):
+     @unittest.skipIf(sys.platform == "win32", "not-Windows specific")
+     @unittest.skipIf(IS_LIBRESSL, "LibreSSL doesn't support env vars")
+     def test_load_default_certs_env(self):
++        raise unittest.SkipTest("Skipping this test for Python within an Android app")
+         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+         with support.EnvironmentVarGuard() as env:
+             env["SSL_CERT_DIR"] = CAPATH
+-- 
+2.27.0
+

patches/3.10/Setup.local

@@ -0,0 +1,5 @@
+*disabled*
+
+_gdbm
+_dbm
+grp

patches/3.10/series

@@ -0,0 +1 @@
+01_python_ssl_module_add_android_certificates

patches/3.11/01_python_ssl_module_add_android_certificates

@@ -0,0 +1,48 @@
+From 86b7aee31eb66b99e2e9ff55f83cb2e6393dd8d3 Mon Sep 17 00:00:00 2001
+From: Asheesh Laroia <[email protected]>
+Date: Fri, 19 Jun 2020 16:47:25 -0700
+Subject: [PATCH] Parse certificate authority certs from Android
+ /etc/security/cacerts directory
+
+---
+ Lib/ssl.py           | 11 +++++++++++
+ Lib/test/test_ssl.py |  1 +
+ 2 files changed, 12 insertions(+)
+
+diff --git a/Lib/ssl.py b/Lib/ssl.py
+index 0726caee49..3878bdd8fa 100644
+--- a/Lib/ssl.py
++++ b/Lib/ssl.py
+@@ -572,6 +572,17 @@ class SSLContext(_SSLContext):
+         if sys.platform == "win32":
+             for storename in self._windows_cert_stores:
+                 self._load_windows_store_certs(storename, purpose)
++        if os.path.exists('/etc/security/cacerts'):
++            certs = []
++            for basename in os.listdir('/etc/security/cacerts'):
++                with open('/etc/security/cacerts/' + basename) as fd:
++                    s = fd.read()
++                    if 'END CERTIFICATE' not in s:
++                        continue
++                    lines = s.split('\n')
++                    line_end_certificate = [i for i, line in enumerate(lines) if 'END CERTIFICATE' in line][0]
++                    certs.append('\n'.join(lines[0:line_end_certificate+1]))
++            self.load_verify_locations(None, None, '\n'.join(certs))
+         self.set_default_verify_paths()
+ 
+     if hasattr(_SSLContext, 'minimum_version'):
+diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
+index 0bc0a8c452..3c79debf3e 100644
+--- a/Lib/test/test_ssl.py
++++ b/Lib/test/test_ssl.py
+@@ -1580,6 +1580,7 @@ class ContextTests(unittest.TestCase):
+     @unittest.skipIf(sys.platform == "win32", "not-Windows specific")
+     @unittest.skipIf(IS_LIBRESSL, "LibreSSL doesn't support env vars")
+     def test_load_default_certs_env(self):
++        raise unittest.SkipTest("Skipping this test for Python within an Android app")
+         ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+         with support.EnvironmentVarGuard() as env:
+             env["SSL_CERT_DIR"] = CAPATH
+-- 
+2.27.0
+

patches/3.11/Setup.local

@@ -0,0 +1,5 @@
+*disabled*
+
+_gdbm
+_dbm
+grp

patches/3.11/series

@@ -0,0 +1 @@
+01_python_ssl_module_add_android_certificates

patches/3.7/Setup.local

@@ -0,0 +1,5 @@
+*disabled*
+
+_gdbm
+_dbm
+grp

patches/3.8/Setup.local

@@ -0,0 +1,5 @@
+*disabled*
+
+_gdbm
+_dbm
+grp

patches/3.9/Setup.local

@@ -0,0 +1,5 @@
+*disabled*
+
+_gdbm
+_dbm
+grp