diff --git a/.gitignore b/.gitignore index 571c47c868..f4da3d24c6 100644 --- a/.gitignore +++ b/.gitignore @@ -87,9 +87,10 @@ graalpython/com.oracle.graal.python.test/src/tests/patched_package/build/ graalpython/com.oracle.graal.python.test/src/tests/patched_package/src/patched_package.egg-info graalpython/com.oracle.graal.python.test.integration/target graalpython/graalpy-maven-plugin/target +graalpython/graalpy-gradle-plugin/target graalpython/graalpy-archetype-polyglot-app/target -graalpython/graalpy-micronaut-embedding/target graalpython/com.oracle.graal.python.test/src/tests/standalone/micronaut/hello/target/ graalpython/com.oracle.graal.python.test/src/tests/cpyext/build/ pom-mx.xml .venv +!graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.jar diff --git a/CHANGELOG.md b/CHANGELOG.md index 1559248bfd..9d91cda6c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ language runtime. The main focus is on user-observable behavior of the engine. ## Version 24.2.0 * Updated developer metadata of Maven artifacts. +* Added gradle plugin for polyglot embedding of Python packages into Java. ## Version 24.1.0 * GraalPy is now considered stable for pure Python workloads. While many workloads involving native extension modules work, we continue to consider them experimental. You can use the command-line option `--python.WarnExperimentalFeatures` to enable warnings for such modules at runtime. In Java embeddings the warnings are enabled by default and you can suppress them by setting the context option 'python.WarnExperimentalFeatures' to 'false'. diff --git a/ci.jsonnet b/ci.jsonnet index e7fca179b2..0555b1a006 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -1 +1 @@ -{ "overlay": "f34208d23ccb4436fd16a9e61d0fdbeb12fcb14e" } +{ "overlay": "e7178928f1228c996d4facc3294d5144f7a33776" } diff --git a/graalpython/com.oracle.graal.python.test/src/graalpytest.py b/graalpython/com.oracle.graal.python.test/src/graalpytest.py index 4286d3bd25..61fc8ecc45 100644 --- a/graalpython/com.oracle.graal.python.test/src/graalpytest.py +++ b/graalpython/com.oracle.graal.python.test/src/graalpytest.py @@ -854,6 +854,16 @@ class TextTestResult : class TestSuite: pass +def removeAttr(cls, attr): + if(cls is object): + return + if hasattr(cls, attr): + try: + delattr(cls, attr) + except AttributeError as e: + pass + for b in cls.__bases__: + removeAttr(b, attr) def skip_deselected_test_functions(globals): """ @@ -876,7 +886,7 @@ def skip_deselected_test_functions(globals): if idx % total != batch - 1: n = test_func.__name__ if isinstance(owner, type): - delattr(owner, n) + removeAttr(owner, n) else: del owner[n] diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle new file mode 100644 index 0000000000..ac804e7fcb --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle @@ -0,0 +1,21 @@ +plugins { + id "application" + id 'org.graalvm.python' version '24.2.0' + id "org.graalvm.buildtools.native" version "0.10.2" +} + +repositories { + mavenCentral() +} + +application { + mainClass = "org.example.GraalPy" +} + +dependencies { + implementation("org.graalvm.python:python-community:24.2.0") +} + +run { + enableAssertions = true +} diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts new file mode 100644 index 0000000000..119e1482b1 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/build.gradle.kts @@ -0,0 +1,22 @@ +plugins { + application + id("org.graalvm.python") version "24.2.0" + id("org.graalvm.buildtools.native") version "0.10.2" +} + +repositories { + mavenCentral() +} + +application { + // Define the main class for the application. + mainClass = "org.example.GraalPy" +} + +val r = tasks.run.get() +r.enableAssertions = true +r.outputs.upToDateWhen {false} + +dependencies { + implementation("org.graalvm.python:python-community:24.2.0") +} diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle new file mode 100644 index 0000000000..cd2662dcd4 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle @@ -0,0 +1,7 @@ +pluginManagement { + repositories { + gradlePluginPortal() + } +} + +rootProject.name = "graalpy-gradle-test-project" diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle.kts b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle.kts new file mode 100644 index 0000000000..cd2662dcd4 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/build/settings.gradle.kts @@ -0,0 +1,7 @@ +pluginManagement { + repositories { + gradlePluginPortal() + } +} + +rootProject.name = "graalpy-gradle-test-project" diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.jar b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000..7f93135c49 Binary files /dev/null and b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.jar differ diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.properties b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000000..ec97aa7f8d --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https://services.gradle.org/distributions/gradle-8.9-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists \ No newline at end of file diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradlew b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradlew new file mode 100755 index 0000000000..b740cf1339 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradlew @@ -0,0 +1,249 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradlew.bat b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradlew.bat new file mode 100644 index 0000000000..7101f8e467 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradlew.bat @@ -0,0 +1,92 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/java/org/example/GraalPy.j b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/java/org/example/GraalPy.j new file mode 100644 index 0000000000..6b778e2804 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/java/org/example/GraalPy.j @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.example; + +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.PolyglotException; +import org.graalvm.polyglot.Source; +import org.graalvm.polyglot.Value; +import java.io.IOException; + +import org.graalvm.python.embedding.utils.GraalPyResources; + +public class GraalPy { + private static final String PYTHON = "python"; + + public static void main(String[] args) { + try (Context context = GraalPyResources.createContext()) { + Source source; + try { + source = Source.newBuilder(PYTHON, "import hello", "").internal(true).build(); + } catch (IOException e) { + throw new RuntimeException(e); + } + + context.eval(source); + + // retrieve the python PyHello class + Value pyHelloClass = context.getPolyglotBindings().getMember("PyHello"); + Value pyHello = pyHelloClass.newInstance(); + // and cast it to the Hello interface which matches PyHello + Hello hello = pyHello.as(Hello.class); + hello.hello("java"); + + } catch (PolyglotException e) { + if (e.isExit()) { + System.exit(e.getExitStatus()); + } else { + throw e; + } + } + } +} \ No newline at end of file diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/java/org/example/Hello.j b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/java/org/example/Hello.j new file mode 100644 index 0000000000..61126a408f --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/java/org/example/Hello.j @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.example; + +public interface Hello { + void hello(String txt); +} diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/resources/META-INF/native-image/proxy-config.json b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/resources/META-INF/native-image/proxy-config.json new file mode 100644 index 0000000000..0edc8f9ac2 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/resources/META-INF/native-image/proxy-config.json @@ -0,0 +1,3 @@ +[ + ["org.example.Hello"] +] diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/resources/org.graalvm.python.vfs/src/hello.py b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/resources/org.graalvm.python.vfs/src/hello.py new file mode 100644 index 0000000000..cd39ab8cb2 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/src/main/resources/org.graalvm.python.vfs/src/hello.py @@ -0,0 +1,48 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +import polyglot +from termcolor import colored + +class PyHello: + def hello(self, txt): + colored_text = colored("hello " + str(txt), "red", attrs=["reverse", "blink"]) + print(colored_text) + +# We export the PyHello class to Java as our explicit interface with the Java side +polyglot.export_value("PyHello", PyHello) \ No newline at end of file diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py b/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py index c8dd9bb268..310faf32da 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/test_standalone.py @@ -44,12 +44,22 @@ import urllib.parse import shutil import util +import sys is_enabled = 'ENABLE_STANDALONE_UNITTESTS' in os.environ and os.environ['ENABLE_STANDALONE_UNITTESTS'] == "true" +is_gradle_enabled = 'ENABLE_GRADLE_STANDALONE_UNITTESTS' in os.environ and os.environ['ENABLE_GRADLE_STANDALONE_UNITTESTS'] == "true" GLOBAL_MVN_CMD = [shutil.which('mvn'), "--batch-mode"] VFS_PREFIX = "org.graalvm.python.vfs" +GRAALPY_EMPTY = "graalPy {}" + +GRAALPY_EMPTY_HOME = """ +graalPy { + pythonHome { } +} +""" + def get_gp(): graalpy = util.get_gp() @@ -78,13 +88,26 @@ def get_gp(): def replace_in_file(file, str, replace_str): with open(file, "r") as f: contents = f.read() + assert str in contents with open(file, "w") as f: f.write(contents.replace(str, replace_str)) -class PolyglotAppTest(unittest.TestCase): - +def patch_properties_file(properties_file, distribution_url_override): + if distribution_url_override: + new_lines = [] + with(open(properties_file)) as f: + while line := f.readline(): + line.strip() + if not line.startswith("#") and "distributionUrl" in line: + new_lines.append(f"distributionUrl={distribution_url_override}\n") + else: + new_lines.append(line) + with(open(properties_file, "w")) as f: + f.writelines(new_lines) + +class PolyglotAppTestBase(unittest.TestCase): def setUpClass(self): - if not is_enabled: + if not is_enabled and not is_gradle_enabled: return self.env = os.environ.copy() @@ -155,6 +178,463 @@ def setUpClass(self): assert return_code == 0 break +def append(file, txt): + with open(file, "a") as f: + f.write(txt) + +class PolyglotAppGradleTestBase(PolyglotAppTestBase): + def setUpClass(self): + super().setUpClass() + self.test_prj_path = os.path.join(os.path.dirname(__file__), "gradle", "gradle-test-project") + + def target_dir_name_sufix(self, target_dir): + pass + + def copy_build_files(self, target_dir): + pass + + def packages_termcolor(self, build_file): + pass + + def packages_termcolor_ujson(self): + pass + + def packages_termcolor_resource_dir(self, resources_dir): + pass + + def empty_home_includes(self): + pass + + def home_includes(self): + pass + + def empty_packages(self): + pass + + def generate_app(self, target_dir): + shutil.copytree(self.test_prj_path, target_dir) + for root, dirs, files in os.walk(target_dir): + for file in files: + if file.endswith(".j"): + shutil.move(os.path.join(root, file), os.path.join(root, file[0:len(file)- 1] + "java")) + + patch_properties_file(os.path.join(target_dir, "gradle", "wrapper", "gradle-wrapper.properties"), self.env.get("GRADLE_DISTRIBUTION_URL_OVERRIDE")) + + self.copy_build_files(target_dir) + + @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") + def test_gradle_generated_app(self): + with tempfile.TemporaryDirectory() as tmpdir: + target_dir = os.path.join(str(tmpdir), "generated_app_gradle" + self.target_dir_name_sufix()) + self.generate_app(target_dir) + build_file = os.path.join(target_dir, self.build_file_name) + append(build_file, self.packages_termcolor()) + + gradlew_cmd = util.get_gradle_wrapper(target_dir, self.env) + + # build + cmd = gradlew_cmd + ["build"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("BUILD SUCCESS", out) + + cmd = gradlew_cmd + ["nativeCompile"] + # gradle needs jdk <= 22, but it looks like the 'gradle nativeCompile' cmd does not complain if higher, + # which is fine, because we need to build the native binary with a graalvm build + # and the one we have set in JAVA_HOME is at least jdk24 + # => run without gradle = True + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("BUILD SUCCESS", out) + + # check fileslist.txt + fl_path = os.path.join(target_dir, "build", "resources", "main", VFS_PREFIX, "fileslist.txt") + with open(fl_path) as f: + lines = f.readlines() + assert "/" + VFS_PREFIX + "/\n" in lines, "unexpected output from " + str(cmd) + assert "/" + VFS_PREFIX + "/home/\n" in lines, "unexpected output from " + str(cmd) + assert "/" + VFS_PREFIX + "/home/lib-graalpython/\n" in lines, "unexpected output from " + str(cmd) + assert "/" + VFS_PREFIX + "/home/lib-python/\n" in lines, "unexpected output from " + str(cmd) + + # execute and check native image + cmd = [os.path.join(target_dir, "build", "native", "nativeCompile", "graalpy-gradle-test-project")] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("hello java", out) + + # import struct from python file triggers extract of native extension files in VirtualFileSystem + hello_src = os.path.join(target_dir, "src", "main", "resources", "org.graalvm.python.vfs", "src", "hello.py") + contents = open(hello_src, 'r').read() + with open(hello_src, 'w') as f: + f.write("import struct\n" + contents) + + # rebuild and exec + cmd = gradlew_cmd + ["build", "run"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("BUILD SUCCESS", out) + util.check_ouput("hello java", out) + + #GR-51132 - NoClassDefFoundError when running polyglot app in java mode + util.check_ouput("java.lang.NoClassDefFoundError", out, False) + + @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") + def test_gradle_generated_app_external_resources(self): + with tempfile.TemporaryDirectory() as tmpdir: + target_dir = os.path.join(str(tmpdir), "generated_gradle_app_external_resources" + self.target_dir_name_sufix()) + self.generate_app(target_dir) + build_file = os.path.join(target_dir, self.build_file_name) + + # patch project to use external directory for resources + resources_dir = os.path.join(target_dir, "python-resources") + os.makedirs(resources_dir, exist_ok=True) + src_dir = os.path.join(resources_dir, "src") + os.makedirs(src_dir, exist_ok=True) + # copy hello.py + shutil.copyfile(os.path.join(target_dir, "src", "main", "resources", "org.graalvm.python.vfs", "src", "hello.py"), os.path.join(src_dir, "hello.py")) + shutil.rmtree(os.path.join(target_dir, "src", "main", "resources", "org.graalvm.python.vfs")) + # patch GraalPy.java + replace_in_file(os.path.join(target_dir, "src", "main", "java", "org", "example", "GraalPy.java"), + "package org.example;", + "package org.example;\nimport java.nio.file.Path;") + replace_in_file(os.path.join(target_dir, "src", "main", "java", "org", "example", "GraalPy.java"), + "GraalPyResources.createContext()", + "GraalPyResources.contextBuilder(Path.of(\"python-resources\")).build()") + + # patch build.gradle + append(build_file, self.packages_termcolor_resource_dir(resources_dir)) + + # build + gradle_cmd = util.get_gradle_wrapper(target_dir, self.env) + + cmd = gradle_cmd + ["clean", "build"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("BUILD SUCCESS", out) + + # gradle needs jdk <= 22, but it looks like the 'gradle nativeCompile' cmd does not complain if higher, + # which is fine, because we need to build the native binary with a graalvm build + # and the one we have set in JAVA_HOME is at least jdk24 + # => run without gradle = True + cmd = gradle_cmd + ["nativeCompile"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("BUILD SUCCESS", out) + + # execute and check native image + cmd = [os.path.join(target_dir, "build", "native", "nativeCompile", "graalpy-gradle-test-project")] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir) + util.check_ouput("hello java", out) + + # 2.) check java build and exec + cmd = gradle_cmd + ["run"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("BUILD SUCCESS", out) + util.check_ouput("hello java", out) + + @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") + def test_gradle_fail_without_graalpy_dep(self): + with tempfile.TemporaryDirectory() as tmpdir: + target_dir = os.path.join(str(tmpdir), "gradle_fail_without_graalpy_dep" + self.target_dir_name_sufix()) + self.generate_app(target_dir) + build_file = os.path.join(target_dir, self.build_file_name) + append(build_file, GRAALPY_EMPTY) + + gradle_cmd = util.get_gradle_wrapper(target_dir, self.env) + + replace_in_file(build_file, + "implementation(\"org.graalvm.python:python-community:24.2.0\")", + "// implementation(\"org.graalvm.python:python-community:24.2.0\")") + + cmd = gradle_cmd + ["graalPyResources"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("Missing GraalPy dependency. Please add to your build.gradle either org.graalvm.polyglot:python-community or org.graalvm.polyglot:python", out) + + @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") + def test_gradle_gen_launcher_and_venv(self): + with tempfile.TemporaryDirectory() as tmpdir: + target_dir = os.path.join(str(tmpdir), "gradle_gen_launcher_and_venv" + self.target_dir_name_sufix()) + self.generate_app(target_dir) + + build_file = os.path.join(target_dir, self.build_file_name) + + gradle_cmd = util.get_gradle_wrapper(target_dir, self.env) + + append(build_file, self.packages_termcolor_ujson()) + + cmd = gradle_cmd + ["graalPyResources"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("-m venv", out) + util.check_ouput("-m ensurepip",out) + util.check_ouput("ujson", out) + util.check_ouput("termcolor", out) + + # run again and assert that we do not regenerate the venv + cmd = gradle_cmd + ["graalPyResources"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("-m venv", out, False) + util.check_ouput("-m ensurepip", out, False) + util.check_ouput("ujson", out, False) + util.check_ouput("termcolor", out, False) + + # remove ujson pkg from plugin config and check if unistalled + self.copy_build_files(target_dir) + append(build_file, self.packages_termcolor()) + + cmd = gradle_cmd + ["graalPyResources"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("-m venv", out, False) + util.check_ouput("-m ensurepip", out, False) + util.check_ouput("Uninstalling ujson", out) + util.check_ouput("termcolor", out, False) + + def check_tagfile(self, home, expected): + with open(os.path.join(home, "tagfile")) as f: + lines = f.readlines() + assert lines == expected, "expected tagfile " + str(expected) + ", but got " + str(lines) + + @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") + def test_gradle_check_home(self): + with tempfile.TemporaryDirectory() as tmpdir: + target_dir = os.path.join(str(tmpdir), "check_home_test" + self.target_dir_name_sufix()) + self.generate_app(target_dir) + + build_file_template = os.path.join(os.path.dirname(__file__), "gradle", "build", self.build_file_name) + build_file = os.path.join(target_dir, self.build_file_name) + + gradle_cmd = util.get_gradle_wrapper(target_dir, self.env) + process_resources_cmd = gradle_cmd + ["graalPyResources"] + + # 1. process-resources with no pythonHome config + out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("BUILD SUCCESS", out) + util.check_ouput("Copying std lib to ", out) + + home_dir = os.path.join(target_dir, "build", "generated", "graalpy", "resources", VFS_PREFIX, "home") + assert os.path.exists(home_dir) + assert os.path.exists(os.path.join(home_dir, "lib-graalpython")) + assert os.path.isdir(os.path.join(home_dir, "lib-graalpython")) + assert os.path.exists(os.path.join(home_dir, "lib-python")) + assert os.path.isdir(os.path.join(home_dir, "lib-python")) + assert os.path.exists(os.path.join(home_dir, "tagfile")) + assert os.path.isfile(os.path.join(home_dir, "tagfile")) + self.check_tagfile(home_dir, [f'{self.graalvmVersion}\n', 'include:.*\n']) + + # 2. process-resources with empty pythonHome + self.copy_build_files(target_dir) + append(build_file, GRAALPY_EMPTY_HOME) + out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("BUILD SUCCESS", out) + util.check_ouput("Copying std lib to ", out, False) + self.check_tagfile(home_dir, [f'{self.graalvmVersion}\n', 'include:.*\n']) + + # 3. process-resources with empty pythonHome includes and excludes + self.copy_build_files(target_dir) + append(build_file, self.empty_home_includes()) + out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("BUILD SUCCESS", out) + util.check_ouput("Copying std lib to ", out, False) + self.check_tagfile(home_dir, [f'{self.graalvmVersion}\n', 'include:.*\n']) + + # 4. process-resources with pythonHome includes and excludes + self.copy_build_files(target_dir) + append(build_file, self.home_includes()) + out, return_code = util.run_cmd(process_resources_cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("BUILD SUCCESS", out) + util.check_ouput("Deleting GraalPy home due to changed includes or excludes", out) + util.check_ouput("Copying std lib to ", out) + self.check_tagfile(home_dir, [f'{self.graalvmVersion}\n', 'include:.*__init__.py\n', 'exclude:.*html/__init__.py\n']) + + # 5. check fileslist.txt + # XXX build vs graalPyVFSFilesList task? + out, return_code = util.run_cmd(gradle_cmd + ["build"], self.env, cwd=target_dir, gradle = True) + util.check_ouput("BUILD SUCCESS", out) + fl_path = os.path.join(target_dir, "build", "resources", "main", VFS_PREFIX, "fileslist.txt") + with open(fl_path) as f: + for line in f: + line = f.readline() + # string \n + line = line[:len(line)-1] + if line.endswith("tagfile"): + continue + if not line.startswith("/" + VFS_PREFIX + "/home/") or line.endswith("/"): + continue + assert line.endswith("/__init__.py"), f"expected line to end with /__init__.py, but was '{line}'" + assert not line.endswith("html/__init__.py"), f"expected line to end with html/__init__.py, but was '{line}''" + + @unittest.skipUnless(is_gradle_enabled, "ENABLE_GRADLE_STANDALONE_UNITTESTS is not true") + def test_gradle_empty_packages(self): + with tempfile.TemporaryDirectory() as tmpdir: + target_dir = os.path.join(str(tmpdir), "empty_packages_test" + self.target_dir_name_sufix()) + self.generate_app(target_dir) + build_file = os.path.join(target_dir, self.build_file_name) + + append(build_file, self.empty_packages()) + + gradle_cmd = util.get_gradle_wrapper(target_dir, self.env) + cmd = gradle_cmd + ["graalPyResources"] + out, return_code = util.run_cmd(cmd, self.env, cwd=target_dir, gradle = True) + util.check_ouput("BUILD SUCCESS", out) + +def print_file(file): + print("\n====", file, " ==========================================================================") + with open(file) as f: + while line := f.readline(): + if line.endswith("\n"): + line = line[:len(line) - 1] + print(line) + print("\n========================================================================================") + +class PolyglotAppGradleGroovyTest(PolyglotAppGradleTestBase): + + def setUpClass(self): + super().setUpClass() + self.build_file_name = "build.gradle" + self.settings_file_name = "settings.gradle" + + def target_dir_name_sufix(self): + return "_groovy" + + def copy_build_files(self, target_dir): + build_file = os.path.join(target_dir, self.build_file_name) + shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.build_file_name), build_file) + settings_file = os.path.join(target_dir, self.settings_file_name) + shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.settings_file_name), settings_file) + if custom_repos := os.environ.get("MAVEN_REPO_OVERRIDE"): + mvn_repos = "" + for idx, custom_repo in enumerate(custom_repos.split(",")): + mvn_repos += f"maven {{ url \"{custom_repo}\" }}\n " + replace_in_file(build_file, + "repositories {", f"repositories {{\n mavenLocal()\n {mvn_repos}") + replace_in_file(settings_file, + "repositories {", f"repositories {{\n {mvn_repos}") + + #print_file(build_file) + #print_file(settings_file) + + def packages_termcolor(self): + return """ +graalPy { + packages = ["termcolor"] +} +""" + + def packages_termcolor_ujson(self): + return """ +graalPy { + packages = ["termcolor", "ujson"] +} +""" + + def packages_termcolor_resource_dir(self, resources_dir): + resources_dir = resources_dir if 'win32' != sys.platform else resources_dir.replace("\\", "\\\\") + return f""" +graalPy {{ + packages = ["termcolor"] + pythonResourcesDirectory = file("{resources_dir}") +}} +""" + + def home_includes(self): + return """ +graalPy { + pythonHome { + includes = [".*__init__.py"] + excludes = [".*html/__init__.py"] + } +} +""" + + def empty_home_includes(self): + return """ +graalPy { + pythonHome { + includes = [] + excludes = [] + } +} +""" + + def empty_packages(self): + return """ +graalPy { + packages = [] +} +""" + +class PolyglotAppGradleKotlinTest(PolyglotAppGradleTestBase): + + def setUpClass(self): + super().setUpClass() + self.build_file_name = "build.gradle.kts" + self.settings_file_name = "settings.gradle.kts" + + def target_dir_name_sufix(self): + return "_kotlin" + + def copy_build_files(self, target_dir): + build_file = os.path.join(target_dir, self.build_file_name) + shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.build_file_name), build_file) + settings_file = os.path.join(target_dir, self.settings_file_name) + shutil.copyfile(os.path.join(os.path.dirname(__file__), "gradle", "build", self.settings_file_name), settings_file) + if custom_repos := os.environ.get("MAVEN_REPO_OVERRIDE"): + mvn_repos = "" + for idx, custom_repo in enumerate(custom_repos.split(",")): + mvn_repos += f"maven(url=\"{custom_repo}\")\n " + + replace_in_file(build_file, "repositories {", f"repositories {{\n mavenLocal()\n {mvn_repos}") + replace_in_file(settings_file, "repositories {", f"repositories {{\n {mvn_repos}") + + #print_file(build_file) + #print_file(settings_file) + + def packages_termcolor(self): + return """ +graalPy { + packages.add("termcolor") +} +""" + + def packages_termcolor_ujson(self): + return """ +graalPy { + packages.add("termcolor") + packages.add("ujson") +} +""" + + def packages_termcolor_resource_dir(self, resources_dir): + resources_dir = resources_dir if 'win32' != sys.platform else resources_dir.replace("\\", "\\\\") + return f""" +graalPy {{ + packages.add("termcolor") + pythonResourcesDirectory = file("{resources_dir}") +}} +""" + + def home_includes(self): + return """ +graalPy { + pythonHome { + includes.add(".*__init__.py") + excludes.add(".*html/__init__.py") + } +} +""" + + def empty_home_includes(self): + return """ +graalPy { + pythonHome { + includes + excludes + } +} +""" + + def empty_packages(self): + return """ +graalPy { + packages +} +""" + +class PolyglotAppTest(PolyglotAppTestBase): + def generate_app(self, tmpdir, target_dir, target_name, pom_template=None): cmd = GLOBAL_MVN_CMD + [ "archetype:generate", @@ -175,25 +655,11 @@ def generate_app(self, tmpdir, target_dir, target_name, pom_template=None): util.patch_pom_repositories(os.path.join(target_dir, "pom.xml")) - distribution_url_override = self.env.get("MAVEN_DISTRIBUTION_URL_OVERRIDE") - if distribution_url_override: - mvnw_dir = os.path.join(os.path.dirname(__file__), "mvnw") - - shutil.copy(os.path.join(mvnw_dir, "mvnw"), os.path.join(target_dir, "mvnw")) - shutil.copy(os.path.join(mvnw_dir, "mvnw.cmd"), os.path.join(target_dir, "mvnw.cmd")) - shutil.copytree(os.path.join(mvnw_dir, ".mvn"), os.path.join(target_dir, ".mvn")) - - mvnw_properties = os.path.join(target_dir, ".mvn", "wrapper", "maven-wrapper.properties") - new_lines = [] - with(open(mvnw_properties)) as f: - while line := f.readline(): - line.strip() - if not line.startswith("#") and "distributionUrl" in line: - new_lines.append(f"distributionUrl={distribution_url_override}\n") - else: - new_lines.append(line) - with(open(mvnw_properties, "w")) as f: - f.writelines(new_lines) + mvnw_dir = os.path.join(os.path.dirname(__file__), "mvnw") + shutil.copy(os.path.join(mvnw_dir, "mvnw"), os.path.join(target_dir, "mvnw")) + shutil.copy(os.path.join(mvnw_dir, "mvnw.cmd"), os.path.join(target_dir, "mvnw.cmd")) + shutil.copytree(os.path.join(mvnw_dir, ".mvn"), os.path.join(target_dir, ".mvn")) + patch_properties_file(os.path.join(target_dir, ".mvn", "wrapper", "maven-wrapper.properties"), self.env.get("MAVEN_DISTRIBUTION_URL_OVERRIDE")) @unittest.skipUnless(is_enabled, "ENABLE_STANDALONE_UNITTESTS is not true") def test_generated_app(self): @@ -253,7 +719,7 @@ def test_generated_app_external_resources(self): self.generate_app(tmpdir, target_dir, target_name) # patch project to use external directory for resources - resources_dir = os.path.join(target_dir, "python") + resources_dir = os.path.join(target_dir, "python-resources") os.makedirs(resources_dir, exist_ok=True) src_dir = os.path.join(resources_dir, "src") os.makedirs(src_dir, exist_ok=True) @@ -266,12 +732,12 @@ def test_generated_app_external_resources(self): "package it.pkg;\nimport java.nio.file.Path;") replace_in_file(os.path.join(target_dir, "src", "main", "java", "it", "pkg", "GraalPy.java"), "GraalPyResources.createContext()", - "GraalPyResources.contextBuilder(Path.of(\"python\")).build()") + "GraalPyResources.contextBuilder(Path.of(\"python-resources\")).build()") # patch pom.xml replace_in_file(os.path.join(target_dir, "pom.xml"), "", - "${project.basedir}/python\n") + "${project.basedir}/python-resources\n") mvnw_cmd = util.get_mvn_wrapper(target_dir, self.env) @@ -298,7 +764,6 @@ def test_generated_app_external_resources(self): util.check_ouput("BUILD SUCCESS", out) util.check_ouput("hello java", out) - @unittest.skipUnless(is_enabled, "ENABLE_STANDALONE_UNITTESTS is not true") def test_fail_without_graalpy_dep(self): with tempfile.TemporaryDirectory() as tmpdir: diff --git a/graalpython/com.oracle.graal.python.test/src/tests/standalone/util.py b/graalpython/com.oracle.graal.python.test/src/tests/standalone/util.py index 0a44ef5b84..ab6ff02970 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/standalone/util.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/standalone/util.py @@ -43,20 +43,32 @@ import sys MAVEN_VERSION = "3.9.8" +GRADLE_VERSION = "8.9" -def run_cmd(cmd, env, cwd=None, print_out=False): +def run_cmd(cmd, env, cwd=None, print_out=False, gradle=False): out = [] out.append(f"Executing:\n {cmd=}\n") - process = subprocess.Popen(cmd, env=env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, text=True, errors='backslashreplace') - if print_out: - print("============== output =============") - for line in iter(process.stdout.readline, ""): - out.append(line) + prev_java_home = None + if gradle: + gradle_java_home = env.get("GRADLE_JAVA_HOME") + assert gradle_java_home, "in order to run standalone gradle tests, the 'GRADLE_JAVA_HOME' env var has to be set to a jdk <= 22" + prev_java_home = env["JAVA_HOME"] + env["JAVA_HOME"] = env["GRADLE_JAVA_HOME"] + + try: + process = subprocess.Popen(cmd, env=env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, text=True, errors='backslashreplace') if print_out: - print(line, end="") - if print_out: - print("\n========== end of output ==========") - return "".join(out), process.wait() + print("============== output =============") + for line in iter(process.stdout.readline, ""): + out.append(line) + if print_out: + print(line, end="") + if print_out: + print("\n========== end of output ==========") + return "".join(out), process.wait() + finally: + if prev_java_home: + env["JAVA_HOME"] = prev_java_home def check_ouput(txt, out, contains=True): if contains and txt not in out: @@ -66,12 +78,13 @@ def check_ouput(txt, out, contains=True): print_output(out, f"did not expect '{txt}' in output") assert False -def print_output(out, err_msg): +def print_output(out, err_msg=None): print("============== output =============") for line in out: print(line, end="") print("\n========== end of output ==========") - print("", err_msg, "", sep="\n") + if err_msg: + print("", err_msg, "", sep="\n") def get_mvn_wrapper(project_dir, env): cmd = "mvnw" if 'win32' != sys.platform else "mvnw.cmd" @@ -81,6 +94,13 @@ def get_mvn_wrapper(project_dir, env): check_ouput(MAVEN_VERSION, out) return mvn_cmd +def get_gradle_wrapper(project_dir, env): + gradle_cmd = [os.path.abspath(os.path.join(project_dir, "gradlew" if 'win32' != sys.platform else "gradlew.bat"))] + cmd = gradle_cmd + ["--version"] + out, return_code = run_cmd(cmd, env, cwd=project_dir) + check_ouput(GRADLE_VERSION, out) + return gradle_cmd + def get_gp(): if "PYTHON_STANDALONE_HOME" not in os.environ: print_missing_graalpy_msg() diff --git a/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java b/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java index f348638cfc..8150d8b4e3 100644 --- a/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java +++ b/graalpython/graalpy-maven-plugin/src/main/java/org/graalvm/python/maven/plugin/ManageResourcesMojo.java @@ -41,13 +41,10 @@ package org.graalvm.python.maven.plugin; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; import java.util.*; import java.util.stream.Collectors; @@ -58,54 +55,31 @@ import org.apache.maven.model.Resource; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.plugin.logging.Log; import org.apache.maven.plugins.annotations.*; import org.apache.maven.project.*; import org.eclipse.aether.graph.Dependency; -import org.graalvm.python.embedding.tools.exec.GraalPyRunner; import org.graalvm.python.embedding.tools.vfs.VFSUtils; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.GRAALPY_GROUP_ID; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.LAUNCHER_NAME; import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_HOME; import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_ROOT; import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_VENV; + @Mojo(name = "process-graalpy-resources", defaultPhase = LifecyclePhase.PROCESS_RESOURCES, requiresDependencyCollection = ResolutionScope.COMPILE_PLUS_RUNTIME, requiresDependencyResolution = ResolutionScope.COMPILE_PLUS_RUNTIME) public class ManageResourcesMojo extends AbstractMojo { - private static final String PYTHON_LANGUAGE_ARTIFACT_ID = "python-language"; - private static final String PYTHON_RESOURCES = "python-resources"; private static final String PYTHON_LAUNCHER_ARTIFACT_ID = "python-launcher"; - private static final String GRAALPY_GROUP_ID = "org.graalvm.python"; private static final String POLYGLOT_GROUP_ID = "org.graalvm.polyglot"; private static final String PYTHON_COMMUNITY_ARTIFACT_ID = "python-community"; private static final String PYTHON_ARTIFACT_ID = "python"; private static final String GRAALPY_MAVEN_PLUGIN_ARTIFACT_ID = "graalpy-maven-plugin"; - private static final String GRAALPY_MAIN_CLASS = "com.oracle.graal.python.shell.GraalPythonMain"; - - private static final boolean IS_WINDOWS = System.getProperty("os.name").startsWith("Windows"); - private static final String LAUNCHER = IS_WINDOWS ? "graalpy.exe" : "graalpy.sh"; - - private static final String INCLUDE_PREFIX = "include:"; - - private static final String EXCLUDE_PREFIX = "exclude:"; - - private static final String NATIVE_IMAGE_RESOURCES_CONFIG = """ - { - "resources": { - "includes": [ - {"pattern": "$vfs/.*"} - ] - } - } - """.replace("$vfs", VFS_ROOT); - - private static final String NATIVE_IMAGE_ARGS = "Args = -H:-CopyLanguageResources"; - @Parameter(defaultValue = "${project}", required = true, readonly = true) MavenProject project; @@ -131,10 +105,6 @@ public static class PythonHome { private Set launcherClassPath; - static Path getMetaInfDirectory(MavenProject project) { - return Path.of(project.getBuild().getOutputDirectory(), "META-INF", "native-image", GRAALPY_GROUP_ID, GRAALPY_MAVEN_PLUGIN_ARTIFACT_ID); - } - public void execute() throws MojoExecutionException { if(pythonResourcesDirectory != null) { @@ -166,31 +136,11 @@ public void execute() throws MojoExecutionException { } - private void trim(List l) { - Iterator it = l.iterator(); - while(it.hasNext()) { - String p = it.next(); - if(p == null || p.trim().isEmpty()) { - it.remove(); - } - } - } - private void manageNativeImageConfig() throws MojoExecutionException { - Path metaInf = getMetaInfDirectory(project); - Path resourceConfig = metaInf.resolve("resource-config.json"); - try { - Files.createDirectories(resourceConfig.getParent()); - Files.writeString(resourceConfig, NATIVE_IMAGE_RESOURCES_CONFIG, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - } catch (IOException e) { - throw new MojoExecutionException(String.format("failed to write %s", resourceConfig), e); - } - Path nativeImageProperties = metaInf.resolve("native-image.properties"); try { - Files.createDirectories(nativeImageProperties.getParent()); - Files.writeString(nativeImageProperties, NATIVE_IMAGE_ARGS, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + VFSUtils.writeNativeImageConfig(Path.of(project.getBuild().getOutputDirectory(), "META-INF"), GRAALPY_MAVEN_PLUGIN_ARTIFACT_ID); } catch (IOException e) { - throw new MojoExecutionException(String.format("failed to write %s", nativeImageProperties), e); + throw new MojoExecutionException("failed to create native image configuration files", e); } } @@ -201,7 +151,7 @@ private void manageHome() throws MojoExecutionException { pythonHome.excludes = Collections.emptyList(); } else { if (pythonHome.includes != null) { - trim(pythonHome.includes); + VFSUtils.trim(pythonHome.includes); } if (pythonHome.includes == null || pythonHome.includes.isEmpty()) { pythonHome.includes = Arrays.asList(".*"); @@ -209,73 +159,26 @@ private void manageHome() throws MojoExecutionException { if (pythonHome.excludes == null) { pythonHome.excludes = Collections.emptyList(); } else { - trim(pythonHome.excludes); + VFSUtils.trim(pythonHome.excludes); } } + Path homeDirectory; if(pythonResourcesDirectory == null) { homeDirectory = Path.of(project.getBuild().getOutputDirectory(), VFS_ROOT, VFS_HOME); } else { homeDirectory = Path.of(pythonResourcesDirectory, VFS_HOME); } - var tag = homeDirectory.resolve("tagfile"); - var graalPyVersion = getGraalPyVersion(project); - - List pythonHomeIncludes = toSortedArrayList(pythonHome.includes); - List pythonHomeExcludes = toSortedArrayList(pythonHome.excludes); + List includes = toSortedArrayList(pythonHome.includes); + List excludes = toSortedArrayList(pythonHome.excludes); - if (Files.isReadable(tag)) { - List lines = null; - try { - lines = Files.readAllLines(tag); - } catch (IOException e) { - throw new MojoExecutionException(String.format("failed to read tag file %s", tag), e); - } - if (lines.isEmpty() || !graalPyVersion.equals(lines.get(0))) { - getLog().info(String.format("Stale GraalPy home, updating to %s", graalPyVersion)); - delete(homeDirectory); - } - if (pythonHomeChanged(pythonHomeIncludes, pythonHomeExcludes, lines)) { - getLog().info(String.format("Deleting GraalPy home due to changed includes or excludes")); - delete(homeDirectory); - } - } try { - if (!Files.exists(homeDirectory)) { - getLog().info(String.format("Creating GraalPy %s home in %s", graalPyVersion, homeDirectory)); - Files.createDirectories(homeDirectory.getParent()); - VFSUtils.copyGraalPyHome(calculateLauncherClasspath(project), homeDirectory, pythonHomeIncludes, pythonHomeExcludes, new MavenDelegateLog(getLog())); - } - Files.write(tag, List.of(graalPyVersion), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - write(tag, pythonHomeIncludes, INCLUDE_PREFIX); - write(tag, pythonHomeExcludes, EXCLUDE_PREFIX); - } catch (IOException | InterruptedException e) { + VFSUtils.createHome(homeDirectory, getGraalPyVersion(project), includes, excludes, () -> calculateLauncherClasspath(project), new MavenDelegateLog(getLog()), (s) -> getLog().info(s)); + } catch(IOException e) { throw new MojoExecutionException(String.format("failed to copy graalpy home %s", homeDirectory), e); } } - private boolean pythonHomeChanged(List includes, List excludes, List lines) throws MojoExecutionException { - List prevIncludes = new ArrayList<>(); - List prevExcludes = new ArrayList<>(); - for (int i = 1; i < lines.size(); i++) { - String l = lines.get(i); - if (l.startsWith(INCLUDE_PREFIX)) { - prevIncludes.add(l.substring(INCLUDE_PREFIX.length())); - } else if (l.startsWith(EXCLUDE_PREFIX)) { - prevExcludes.add(l.substring(EXCLUDE_PREFIX.length())); - } - } - prevIncludes = toSortedArrayList(prevIncludes); - prevExcludes = toSortedArrayList(prevExcludes); - return !(prevIncludes.equals(includes) && prevExcludes.equals(excludes)); - } - - private void write(Path tag, List list, String prefix) throws IOException { - if(list != null) { - Files.write(tag, list.stream().map(l -> prefix + l).collect(Collectors.toList()), StandardOpenOption.APPEND); - } - } - private ArrayList toSortedArrayList(List l) { if(l != null) { Collections.sort(l); @@ -284,18 +187,6 @@ private ArrayList toSortedArrayList(List l) { return new ArrayList<>(0); } - private void delete(Path homeDirectory) throws MojoExecutionException { - try { - try (var s = Files.walk(homeDirectory)) { - s.sorted(Comparator.reverseOrder()) - .map(Path::toFile) - .forEach(File::delete); - } - } catch (IOException e) { - new MojoExecutionException(String.format("failed to delete %s", homeDirectory), e); - } - } - private void listGraalPyResources() throws MojoExecutionException { Path vfs = Path.of(project.getBuild().getOutputDirectory(), VFS_ROOT); if (Files.exists(vfs)) { @@ -308,8 +199,6 @@ private void listGraalPyResources() throws MojoExecutionException { } private void manageVenv() throws MojoExecutionException { - generateLaunchers(); - Path venvDirectory; if(pythonResourcesDirectory == null) { venvDirectory = Path.of(project.getBuild().getOutputDirectory(), VFS_ROOT, VFS_VENV); @@ -317,171 +206,36 @@ private void manageVenv() throws MojoExecutionException { venvDirectory = Path.of(pythonResourcesDirectory, VFS_VENV); } - if(packages != null) { - trim(packages); - } - - if (packages == null && pythonResourcesDirectory == null) { - getLog().info(String.format("No venv packages declared, deleting %s", venvDirectory)); - delete(venvDirectory); - return; - } - - var tag = venvDirectory.resolve("contents"); - List installedPackages = new ArrayList(); - var graalPyVersion = getGraalPyVersion(project); - - if (Files.isReadable(tag)) { - List lines = null; - try { - lines = Files.readAllLines(tag); - } catch (IOException e) { - throw new MojoExecutionException(String.format("failed to read tag file %s", tag), e); - } - if (lines.isEmpty() || !graalPyVersion.equals(lines.get(0))) { - getLog().info(String.format("Stale GraalPy venv, updating to %s", graalPyVersion)); + try { + if (packages == null && pythonResourcesDirectory == null) { + getLog().info(String.format("No venv packages declared, deleting %s", venvDirectory)); delete(venvDirectory); - } else { - for (int i = 1; i < lines.size(); i++) { - installedPackages.add(lines.get(i)); - } + return; } - } else { - getLog().info(String.format("Creating GraalPy %s venv", graalPyVersion)); - } - - if (!Files.exists(venvDirectory)) { - runLauncher(getLauncherPath().toString(),"-m", "venv", venvDirectory.toString(), "--without-pip"); - runVenvBin(venvDirectory, "graalpy", "-I", "-m", "ensurepip"); - } - deleteUnwantedPackages(venvDirectory, installedPackages); - installWantedPackages(venvDirectory, installedPackages); - - try { - Files.write(tag, List.of(graalPyVersion), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - Files.write(tag, packages, StandardOpenOption.APPEND); + VFSUtils.createVenv(venvDirectory, new ArrayList(packages), getLauncherPath(), () -> calculateLauncherClasspath(project), getGraalPyVersion(project), new MavenDelegateLog(getLog()), (s) -> getLog().info(s)); } catch (IOException e) { - throw new MojoExecutionException(String.format("failed to write tag file %s", tag), e); - } - } - - private void installWantedPackages(Path venvDirectory, List installedPackages) throws MojoExecutionException { - var pkgsToInstall = new HashSet(packages); - pkgsToInstall.removeAll(installedPackages); - if (pkgsToInstall.isEmpty()) { - return; - } - runPip(venvDirectory, "install", pkgsToInstall.toArray(new String[pkgsToInstall.size()])); - } - - private void deleteUnwantedPackages(Path venvDirectory, List installedPackages) throws MojoExecutionException { - List args = new ArrayList(installedPackages); - args.removeAll(packages); - if (args.isEmpty()) { - return; + throw new MojoExecutionException(String.format("failed to create venv %s", venvDirectory), e); } - args.add(0, "-y"); - runPip(venvDirectory, "uninstall", args.toArray(new String[args.size()])); - } - - private Path getLauncherPath() { - return Paths.get(project.getBuild().getDirectory(), LAUNCHER); } - private void generateLaunchers() throws MojoExecutionException { - getLog().info("Generating GraalPy launchers"); - var launcher = getLauncherPath(); - if (!Files.exists(launcher)) { - var java = Paths.get(System.getProperty("java.home"), "bin", "java"); - var classpath = calculateLauncherClasspath(project); - if (!IS_WINDOWS) { - var script = String.format(""" - #!/usr/bin/env bash - %s -classpath %s %s --python.Executable="$0" "$@" - """, - java, - String.join(File.pathSeparator, classpath), - GRAALPY_MAIN_CLASS); - try { - Files.createDirectories(launcher.getParent()); - Files.writeString(launcher, script); - var perms = Files.getPosixFilePermissions(launcher); - perms.addAll(List.of(PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_EXECUTE)); - Files.setPosixFilePermissions(launcher, perms); - } catch (IOException e) { - throw new MojoExecutionException(String.format("failed to create launcher %s", launcher), e); - } - } else { - // on windows, generate a venv launcher that executes our mvn target - var script = String.format(""" - import os, shutil, struct, venv - from pathlib import Path - vl = os.path.join(venv.__path__[0], 'scripts', 'nt', 'graalpy.exe') - tl = os.path.join(r'%s') - os.makedirs(Path(tl).parent.absolute(), exist_ok=True) - shutil.copy(vl, tl) - cmd = r'%s -classpath "%s" %s' - pyvenvcfg = os.path.join(os.path.dirname(tl), "pyvenv.cfg") - with open(pyvenvcfg, 'w', encoding='utf-8') as f: - f.write('venvlauncher_command = ') - f.write(cmd) - """, - launcher, - java, - String.join(File.pathSeparator, classpath), - GRAALPY_MAIN_CLASS); - File tmp; - try { - tmp = File.createTempFile("create_launcher", ".py"); - } catch (IOException e) { - throw new MojoExecutionException("failed to create tmp launcher", e); - } - tmp.deleteOnExit(); - try (var wr = new FileWriter(tmp)) { - wr.write(script); - } catch (IOException e) { - throw new MojoExecutionException(String.format("failed to write tmp launcher %s", tmp), e); - } - runGraalPy(project, getLog(), tmp.getAbsolutePath()); - } - } - } - - private void runLauncher(String launcherPath, String... args) throws MojoExecutionException { - try { - GraalPyRunner.runLauncher(launcherPath, new MavenDelegateLog(getLog()), args); - } catch(IOException | InterruptedException e) { - throw new MojoExecutionException(String.format("failed to execute launcher command %s", List.of(args))); - } - } - - private void runPip(Path venvDirectory, String command, String... args) throws MojoExecutionException { - try { - GraalPyRunner.runPip(venvDirectory, command, new MavenDelegateLog(getLog()), args); - } catch(IOException | InterruptedException e) { - throw new MojoExecutionException(String.format("failed to execute pip", args), e); - } - } - - private void runVenvBin(Path venvDirectory, String bin, String... args) throws MojoExecutionException { + private void delete(Path homeDirectory) throws MojoExecutionException { try { - GraalPyRunner.runVenvBin(venvDirectory, bin, new MavenDelegateLog(getLog()), args); - } catch(IOException | InterruptedException e) { - throw new MojoExecutionException(String.format("failed to execute venv", args), e); + try (var s = Files.walk(homeDirectory)) { + s.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } catch (IOException e) { + new MojoExecutionException(String.format("failed to delete %s", homeDirectory), e); } } - private void runGraalPy(MavenProject project, Log log, String... args) throws MojoExecutionException { - var classpath = calculateLauncherClasspath(project); - try { - GraalPyRunner.run(classpath, new MavenDelegateLog(log), args); - } catch (IOException | InterruptedException e) { - throw new MojoExecutionException(String.format("failed to run Graalpy launcher"), e); - } + private Path getLauncherPath() { + return Paths.get(project.getBuild().getDirectory(), LAUNCHER_NAME); } - private static String getGraalPyVersion(MavenProject project) throws MojoExecutionException { + private static String getGraalPyVersion(MavenProject project) throws IOException { DefaultArtifact a = (DefaultArtifact) getGraalPyArtifact(project); String version = a.getVersion(); if(a.isSnapshot()) { @@ -499,13 +253,13 @@ private static String getGraalPyVersion(MavenProject project) throws MojoExecuti return version; } - private static Artifact getGraalPyArtifact(MavenProject project) throws MojoExecutionException { + private static Artifact getGraalPyArtifact(MavenProject project) throws IOException { var projectArtifacts = resolveProjectDependencies(project); Artifact graalPyArtifact = projectArtifacts.stream(). filter(a -> isPythonArtifact(a)) .findFirst() .orElse(null); - return Optional.ofNullable(graalPyArtifact).orElseThrow(() -> new MojoExecutionException("Missing GraalPy dependency. Please add to your pom either %s:%s or %s:%s".formatted(POLYGLOT_GROUP_ID, PYTHON_COMMUNITY_ARTIFACT_ID, POLYGLOT_GROUP_ID, PYTHON_ARTIFACT_ID))); + return Optional.ofNullable(graalPyArtifact).orElseThrow(() -> new IOException("Missing GraalPy dependency. Please add to your pom either %s:%s or %s:%s".formatted(POLYGLOT_GROUP_ID, PYTHON_COMMUNITY_ARTIFACT_ID, POLYGLOT_GROUP_ID, PYTHON_ARTIFACT_ID))); } private static boolean isPythonArtifact(Artifact a) { @@ -520,7 +274,7 @@ private static Collection resolveProjectDependencies(MavenProject proj .collect(Collectors.toList()); } - private Set calculateLauncherClasspath(MavenProject project) throws MojoExecutionException { + private Set calculateLauncherClasspath(MavenProject project) throws IOException { if(launcherClassPath == null) { String version = getGraalPyVersion(project); launcherClassPath = new HashSet(); @@ -530,7 +284,7 @@ private Set calculateLauncherClasspath(MavenProject project) throws Mojo getLog().debug("calculateLauncherClasspath based on " + GRAALPY_GROUP_ID + ":" + GRAALPY_MAVEN_PLUGIN_ARTIFACT_ID + ":" + version); DefaultArtifact mvnPlugin = new DefaultArtifact(GRAALPY_GROUP_ID, GRAALPY_MAVEN_PLUGIN_ARTIFACT_ID, version, "compile", "jar", null, new DefaultArtifactHandler("pom")); ProjectBuildingResult result = buildProjectFromArtifact(mvnPlugin); - Artifact graalPyLauncherArtifact = result.getProject().getArtifacts().stream().filter(a ->GRAALPY_GROUP_ID.equals(a.getGroupId()) && PYTHON_LAUNCHER_ARTIFACT_ID.equals(a.getArtifactId())) + Artifact graalPyLauncherArtifact = result.getProject().getArtifacts().stream().filter(a -> GRAALPY_GROUP_ID.equals(a.getGroupId()) && PYTHON_LAUNCHER_ARTIFACT_ID.equals(a.getArtifactId())) .findFirst() .orElse(null); // python-launcher artifact @@ -546,7 +300,7 @@ private Set calculateLauncherClasspath(MavenProject project) throws Mojo return launcherClassPath; } - private Set resolveDependencies(Artifact artifact) throws MojoExecutionException { + private Set resolveDependencies(Artifact artifact) throws IOException { Set dependencies = new HashSet<>(); ProjectBuildingResult result = buildProjectFromArtifact(artifact); for(Dependency d : result.getDependencyResolutionResult().getResolvedDependencies()) { @@ -555,7 +309,7 @@ private Set resolveDependencies(Artifact artifact) throws MojoExecutionE return dependencies; } - private ProjectBuildingResult buildProjectFromArtifact(Artifact artifact) throws MojoExecutionException{ + private ProjectBuildingResult buildProjectFromArtifact(Artifact artifact) throws IOException{ try{ ProjectBuildingRequest buildingRequest = new DefaultProjectBuildingRequest(session.getProjectBuildingRequest()); buildingRequest.setProject(null); @@ -565,7 +319,7 @@ private ProjectBuildingResult buildProjectFromArtifact(Artifact artifact) throws return projectBuilder.build(artifact, buildingRequest); } catch (ProjectBuildingException e) { - throw new MojoExecutionException("Error while building project", e); + throw new IOException("Error while building project", e); } } diff --git a/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/exec/GraalPyRunner.java b/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/exec/GraalPyRunner.java index df47bd3d03..f9bef8a7b3 100644 --- a/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/exec/GraalPyRunner.java +++ b/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/exec/GraalPyRunner.java @@ -61,12 +61,16 @@ public class GraalPyRunner { private static final String EXE_SUFFIX = IS_WINDOWS ? ".exe" : ""; public static void run(Set classpath, SubprocessLog log, String... args) throws IOException, InterruptedException { + run(String.join(File.pathSeparator, classpath), log, args); + } + + public static void run(String classpath, SubprocessLog log, String... args) throws IOException, InterruptedException { String workdir = System.getProperty("exec.workingdir"); Path java = Paths.get(System.getProperty("java.home"), "bin", "java"); List cmd = new ArrayList<>(); cmd.add(java.toString()); cmd.add("-classpath"); - cmd.add(String.join(File.pathSeparator, classpath)); + cmd.add(classpath); cmd.add("com.oracle.graal.python.shell.GraalPythonMain"); cmd.addAll(List.of(args)); var pb = new ProcessBuilder(cmd); diff --git a/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java b/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java index 4fa7b69ea5..b4b95dbf11 100644 --- a/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java +++ b/graalpython/org.graalvm.python.embedding.tools/src/org/graalvm/python/embedding/tools/vfs/VFSUtils.java @@ -51,16 +51,21 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.PosixFilePermission; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Comparator; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; public final class VFSUtils { @@ -69,6 +74,52 @@ public final class VFSUtils { public static final String VFS_VENV = "venv"; public static final String VFS_FILESLIST = "fileslist.txt"; + public static final String GRAALPY_GROUP_ID = "org.graalvm.python"; + + private static final String NATIVE_IMAGE_RESOURCES_CONFIG = """ + { + "resources": { + "includes": [ + {"pattern": "$vfs/.*"} + ] + } + } + """.replace("$vfs", VFS_ROOT); + + private static final String NATIVE_IMAGE_ARGS = "Args = -H:-CopyLanguageResources"; + + private static final String INCLUDE_PREFIX = "include:"; + + private static final String EXCLUDE_PREFIX = "exclude:"; + + private static final boolean IS_WINDOWS = System.getProperty("os.name").startsWith("Windows"); + + public static final String LAUNCHER_NAME = IS_WINDOWS ? "graalpy.exe" : "graalpy.sh"; + + private static final String GRAALPY_MAIN_CLASS = "com.oracle.graal.python.shell.GraalPythonMain"; + + public static void writeNativeImageConfig(Path metaInfRoot, String pluginId) throws IOException { + Path p = metaInfRoot.resolve(Path.of("native-image", GRAALPY_GROUP_ID, pluginId)); + write(p.resolve("resource-config.json"), NATIVE_IMAGE_RESOURCES_CONFIG); + write(p.resolve("native-image.properties"), NATIVE_IMAGE_ARGS); + } + + private static void write(Path config, String txt) throws IOException { + try { + createParentDirectories(config); + Files.writeString(config, txt, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + } catch (IOException e) { + throw new IOException(String.format("failed to write %s", config), e); + } + } + + private static void createParentDirectories(Path path) throws IOException { + Path parent = path.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + } + public static void generateVFSFilesList(Path vfs) throws IOException { Path filesList = vfs.resolve(VFS_FILESLIST); if (!Files.isDirectory(vfs)) { @@ -109,6 +160,89 @@ private static String makeDirPath(Path p) { return ret; } + @FunctionalInterface + public interface LauncherClassPath { + Set get() throws IOException; + } + + public interface Log { + void info(String s); + } + + public static void createHome(Path homeDirectory, String graalPyVersion, List includes, List excludes, LauncherClassPath launcherClassPath, SubprocessLog subprocessLog, Log log) + throws IOException { + + trim(includes); + trim(excludes); + + var tag = homeDirectory.resolve("tagfile"); + + if (Files.isReadable(tag)) { + List lines = null; + try { + lines = Files.readAllLines(tag); + } catch (IOException e) { + throw new IOException(String.format("failed to read tag file %s", tag), e); + } + if (lines.isEmpty() || !graalPyVersion.equals(lines.get(0))) { + log.info(String.format("Stale GraalPy home, updating to %s", graalPyVersion)); + delete(homeDirectory); + } + if (pythonHomeChanged(includes, excludes, lines)) { + log.info(String.format("Deleting GraalPy home due to changed includes or excludes")); + delete(homeDirectory); + } + } + try { + if (!Files.exists(homeDirectory)) { + log.info(String.format("Creating GraalPy %s home in %s", graalPyVersion, homeDirectory)); + createParentDirectories(homeDirectory); + VFSUtils.copyGraalPyHome(launcherClassPath.get(), homeDirectory, includes, excludes, subprocessLog); + } + Files.write(tag, List.of(graalPyVersion), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + write(tag, includes, INCLUDE_PREFIX); + write(tag, excludes, EXCLUDE_PREFIX); + } catch (IOException | InterruptedException e) { + + throw new IOException(String.format("failed to copy graalpy home %s", homeDirectory), e); + } + } + + private static boolean pythonHomeChanged(List includes, List excludes, List lines) { + Set prevIncludes = new HashSet<>(); + Set prevExcludes = new HashSet<>(); + for (int i = 1; i < lines.size(); i++) { + String l = lines.get(i); + if (l.startsWith(INCLUDE_PREFIX)) { + prevIncludes.add(l.substring(INCLUDE_PREFIX.length())); + } else if (l.startsWith(EXCLUDE_PREFIX)) { + prevExcludes.add(l.substring(EXCLUDE_PREFIX.length())); + } + } + boolean includeDidNotChange = prevIncludes.size() == includes.size() && prevIncludes.containsAll(includes); + boolean excludeDidNotChange = prevExcludes.size() == excludes.size() && prevExcludes.containsAll(excludes); + return !(includeDidNotChange && excludeDidNotChange); + } + + private static void write(Path tag, List list, String prefix) throws IOException { + if (list != null) { + Files.write(tag, list.stream().map(l -> prefix + l).collect(Collectors.toList()), StandardOpenOption.APPEND); + } + } + + public static void delete(Path dir) throws IOException { + if (!Files.exists(dir)) { + return; + } + try { + try (var s = Files.walk(dir)) { + s.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } catch (IOException e) { + throw new IOException(String.format("failed to delete %s", dir), e); + } + } + public static void copyGraalPyHome(Set classpath, Path home, Collection pythonHomeIncludes, Collection pythonHomeExcludes, SubprocessLog log) throws IOException, InterruptedException { log.log(String.format("Copying std lib to '%s'\n", home)); @@ -256,4 +390,169 @@ public void log(CharSequence var1) { } } + + public static void createVenv(Path venvDirectory, List packages, Path laucherPath, LauncherClassPath launcherClassPath, String graalPyVersion, SubprocessLog subprocessLog, Log log) + throws IOException { + generateLaunchers(laucherPath, launcherClassPath, subprocessLog, log); + + if (packages != null) { + trim(packages); + } + + var tag = venvDirectory.resolve("contents"); + List installedPackages = new ArrayList<>(); + + if (Files.isReadable(tag)) { + List lines = null; + try { + lines = Files.readAllLines(tag); + } catch (IOException e) { + throw new IOException(String.format("failed to read tag file %s", tag), e); + } + if (lines.isEmpty() || !graalPyVersion.equals(lines.get(0))) { + log.info(String.format("Stale GraalPy venv, updating to %s", graalPyVersion)); + delete(venvDirectory); + } else { + for (int i = 1; i < lines.size(); i++) { + installedPackages.add(lines.get(i)); + } + } + } else { + log.info(String.format("Creating GraalPy %s venv", graalPyVersion)); + } + + if (!Files.exists(venvDirectory)) { + runLauncher(laucherPath.toString(), subprocessLog, "-m", "venv", venvDirectory.toString(), "--without-pip"); + runVenvBin(venvDirectory, "graalpy", subprocessLog, "-I", "-m", "ensurepip"); + } + + if (packages != null) { + deleteUnwantedPackages(venvDirectory, packages, installedPackages, subprocessLog); + installWantedPackages(venvDirectory, packages, installedPackages, subprocessLog); + } + + try { + Files.write(tag, List.of(graalPyVersion), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); + Files.write(tag, packages, StandardOpenOption.APPEND); + } catch (IOException e) { + throw new IOException(String.format("failed to write tag file %s", tag), e); + } + } + + private static void generateLaunchers(Path laucherPath, LauncherClassPath launcherClassPath, SubprocessLog subprocessLog, Log log) throws IOException { + if (!Files.exists(laucherPath)) { + log.info("Generating GraalPy launchers"); + createParentDirectories(laucherPath); + Path java = Paths.get(System.getProperty("java.home"), "bin", "java"); + String classpath = String.join(File.pathSeparator, launcherClassPath.get()); + if (!IS_WINDOWS) { + var script = String.format(""" + #!/usr/bin/env bash + %s -classpath %s %s --python.Executable="$0" "$@" + """, + java, + String.join(File.pathSeparator, classpath), + GRAALPY_MAIN_CLASS); + try { + Files.writeString(laucherPath, script); + var perms = Files.getPosixFilePermissions(laucherPath); + perms.addAll(List.of(PosixFilePermission.OWNER_EXECUTE, PosixFilePermission.GROUP_EXECUTE, PosixFilePermission.OTHERS_EXECUTE)); + Files.setPosixFilePermissions(laucherPath, perms); + } catch (IOException e) { + throw new IOException(String.format("failed to create launcher %s", laucherPath), e); + } + } else { + // on windows, generate a venv launcher that executes our mvn target + var script = String.format(""" + import os, shutil, struct, venv + from pathlib import Path + vl = os.path.join(venv.__path__[0], 'scripts', 'nt', 'graalpy.exe') + tl = os.path.join(r'%s') + os.makedirs(Path(tl).parent.absolute(), exist_ok=True) + shutil.copy(vl, tl) + cmd = r'%s -classpath "%s" %s' + pyvenvcfg = os.path.join(os.path.dirname(tl), "pyvenv.cfg") + with open(pyvenvcfg, 'w', encoding='utf-8') as f: + f.write('venvlauncher_command = ') + f.write(cmd) + """, + laucherPath, + java, + classpath, + GRAALPY_MAIN_CLASS); + File tmp; + try { + tmp = File.createTempFile("create_launcher", ".py"); + } catch (IOException e) { + throw new IOException("failed to create tmp launcher", e); + } + tmp.deleteOnExit(); + try (var wr = new FileWriter(tmp)) { + wr.write(script); + } catch (IOException e) { + throw new IOException(String.format("failed to write tmp launcher %s", tmp), e); + } + + try { + GraalPyRunner.run(classpath, subprocessLog, tmp.getAbsolutePath()); + } catch (InterruptedException e) { + throw new IOException(String.format("failed to run Graalpy launcher"), e); + } + } + } + } + + private static void installWantedPackages(Path venvDirectory, List packages, List installedPackages, SubprocessLog subprocessLog) throws IOException { + Set pkgsToInstall = new HashSet<>(packages); + pkgsToInstall.removeAll(installedPackages); + if (pkgsToInstall.isEmpty()) { + return; + } + runPip(venvDirectory, "install", subprocessLog, pkgsToInstall.toArray(new String[pkgsToInstall.size()])); + } + + private static void deleteUnwantedPackages(Path venvDirectory, List packages, List installedPackages, SubprocessLog subprocessLog) throws IOException { + List args = new ArrayList<>(installedPackages); + args.removeAll(packages); + if (args.isEmpty()) { + return; + } + args.add(0, "-y"); + runPip(venvDirectory, "uninstall", subprocessLog, args.toArray(new String[args.size()])); + } + + private static void runLauncher(String launcherPath, SubprocessLog log, String... args) throws IOException { + try { + GraalPyRunner.runLauncher(launcherPath, log, args); + } catch (IOException | InterruptedException e) { + throw new IOException(String.format("failed to execute launcher command %s", List.of(args))); + } + } + + private static void runPip(Path venvDirectory, String command, SubprocessLog log, String... args) throws IOException { + try { + GraalPyRunner.runPip(venvDirectory, command, log, args); + } catch (IOException | InterruptedException e) { + throw new IOException(String.format("failed to execute pip %s", List.of(args)), e); + } + } + + private static void runVenvBin(Path venvDirectory, String bin, SubprocessLog log, String... args) throws IOException { + try { + GraalPyRunner.runVenvBin(venvDirectory, bin, log, args); + } catch (IOException | InterruptedException e) { + throw new IOException(String.format("failed to execute venv %s", List.of(args)), e); + } + } + + public static List trim(List l) { + Iterator it = l.iterator(); + while (it.hasNext()) { + String p = it.next(); + if (p == null || p.trim().isEmpty()) { + it.remove(); + } + } + return l; + } } diff --git a/graalpython/org.graalvm.python.gradle.plugin/pom.xml b/graalpython/org.graalvm.python.gradle.plugin/pom.xml new file mode 100644 index 0000000000..c7828c84c8 --- /dev/null +++ b/graalpython/org.graalvm.python.gradle.plugin/pom.xml @@ -0,0 +1,174 @@ + + + + + 4.0.0 + + org.graalvm.python + org.graalvm.python.gradle.plugin + jar + + 24.2.0 + http://www.graalvm.org/ + graalpy-gradle-plugin + Handles python related resources in a gradle GraalPy - Java application. + + + 17 + 17 + UTF-8 + 24.2.0 + 6.1.1 + 2.0.13 + + + + + Version from suite + + + env.GRAALPY_VERSION + + + + ${env.GRAALPY_VERSION} + + + + + + + org.codehaus.groovy + groovy + 3.0.22 + provided + + + org.gradle + gradle-core + provided + ${gradle.version} + + + org.gradle + gradle-core-api + provided + ${gradle.version} + + + org.gradle + gradle-model-core + provided + ${gradle.version} + + + org.gradle + gradle-language-java + provided + ${gradle.version} + + + org.gradle + gradle-language-jvm + provided + ${gradle.version} + + + org.gradle + gradle-platform-jvm + provided + ${gradle.version} + + + org.gradle + gradle-base-services + ${gradle.version} + provided + + + org.gradle + gradle-plugins + provided + ${gradle.version} + + + org.gradle + gradle-logging + ${gradle.version} + provided + + + org.slf4j + slf4j-simple + ${slf4j.version} + provided + + + org.graalvm.python + python-embedding-tools + ${graalpy.version} + compile + + + org.graalvm.python + python-launcher + ${graalpy.version} + runtime + + + + + + gradle + https://repo.gradle.org/gradle/libs-releases/ + + true + + + false + + + + + + diff --git a/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java new file mode 100644 index 0000000000..b83fe98447 --- /dev/null +++ b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/GraalPyGradlePlugin.java @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.graalvm.python; + +import org.graalvm.python.dsl.GraalPyExtension; +import org.graalvm.python.tasks.VFSFilesListTask; +import org.graalvm.python.tasks.MetaInfTask; +import org.graalvm.python.tasks.ResourcesTask; +import org.gradle.api.GradleException; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.artifacts.DependencySet; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.tasks.TaskProvider; +import org.gradle.jvm.tasks.Jar; +import org.gradle.language.jvm.tasks.ProcessResources; + +import java.io.File; +import java.util.Collections; +import java.util.List; + +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.GRAALPY_GROUP_ID; + +public abstract class GraalPyGradlePlugin implements Plugin { + private static final String PYTHON_LAUNCHER_ARTIFACT_ID = "python-launcher"; + private static final String PYTHON_EMBEDDING_ARTIFACT_ID = "python-embedding"; + private static final String POLYGLOT_GROUP_ID = "org.graalvm.polyglot"; + private static final String PYTHON_COMMUNITY_ARTIFACT_ID = "python-community"; + private static final String PYTHON_ARTIFACT_ID = "python"; + private static final String GRAALPY_GRADLE_PLUGIN_TASK_GROUP = "graalPy"; + private static final String DEFAULT_RESOURCES_DIRECTORY = "generated" + File.separator + "graalpy" + File.separator + "resources"; + private static final String GRAALPY_META_INF_DIRECTORY = "generated" + File.separator + "graalpy" + File.separator + "META-INF"; + private static final String GRAALPY_RESOURCES_TASK = "graalPyResources"; + private static final String GRAALPY_META_INF_TASK_TASK = "graalPyMetaInf"; + private static final String GRAALPY_VFS_FILESLIST_TASK = "graalPyVFSFilesList"; + + GraalPyExtension extension; + Project project; + + @Override + public void apply(Project project) { + this.project = project; + project.getPluginManager().apply(JavaPlugin.class); + + this.extension = project.getExtensions().create("graalPy", GraalPyExtension.class); + extension.getPythonHome().getIncludes().convention(List.of(".*")); + extension.getPythonHome().getExcludes().convention(Collections.emptyList()); + extension.getPackages().convention(Collections.emptyList()); + + TaskProvider resourcesTask = project.getTasks().register(GRAALPY_RESOURCES_TASK, ResourcesTask.class); + resourcesTask.configure(t -> { + if(extension.getPythonHome().getIncludes().get().isEmpty()) { + t.getIncludes().set(List.of(".*")); + } else { + t.getIncludes().set(extension.getPythonHome().getIncludes()); + } + + t.getExcludes().set(extension.getPythonHome().getExcludes()); + t.getPackages().set(extension.getPackages()); + + if(extension.getPythonResourcesDirectory().isPresent()) { + t.getOutput().set(extension.getPythonResourcesDirectory()); + t.getIncludeVfsRoot().set(false); + } else { + t.getOutput().set(project.getLayout().getBuildDirectory().dir(DEFAULT_RESOURCES_DIRECTORY)); + t.getIncludeVfsRoot().set(true); + } + + t.setGroup(GRAALPY_GRADLE_PLUGIN_TASK_GROUP); + }); + + TaskProvider metaInfTask = project.getTasks().register(GRAALPY_META_INF_TASK_TASK, MetaInfTask.class); + project.getTasks().getByName(JavaPlugin.JAR_TASK_NAME, t -> ((Jar) t).getMetaInf().from(metaInfTask)); + metaInfTask.configure(t -> { + t.getManifestOutputDir().convention(project.getLayout().getBuildDirectory().dir(GRAALPY_META_INF_DIRECTORY)); + t.setGroup(GRAALPY_GRADLE_PLUGIN_TASK_GROUP); + }); + + TaskProvider vfsFilesListTask = project.getTasks().register(GRAALPY_VFS_FILESLIST_TASK, VFSFilesListTask.class); + vfsFilesListTask.configure(t -> { + t.getResourcesDir().convention((((ProcessResources) project.getTasks().getByName(JavaPlugin.PROCESS_RESOURCES_TASK_NAME)).getDestinationDir())); + t.setGroup(GRAALPY_GRADLE_PLUGIN_TASK_GROUP); + }); + project.getTasks().getByName(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, t -> t.finalizedBy(GRAALPY_VFS_FILESLIST_TASK)); + + project.afterEvaluate(p -> { + checkAndAddDependencies(); + if (!extension.getPythonResourcesDirectory().isPresent()) { + ((ProcessResources) project.getTasks().getByName(JavaPlugin.PROCESS_RESOURCES_TASK_NAME)).with(project.copySpec().from(resourcesTask)); + } else { + project.getTasks().getByName(JavaPlugin.CLASSES_TASK_NAME, t -> t.dependsOn(GRAALPY_RESOURCES_TASK)); + } + }); + } + + private void checkAndAddDependencies() { + project.getDependencies().add(JavaPlugin.IMPLEMENTATION_CONFIGURATION_NAME, "%s:%s:%s".formatted(GRAALPY_GROUP_ID, PYTHON_LAUNCHER_ARTIFACT_ID, getGraalPyVersion(project))); + project.getDependencies().add(JavaPlugin.IMPLEMENTATION_CONFIGURATION_NAME, "%s:%s:%s".formatted(GRAALPY_GROUP_ID, PYTHON_EMBEDDING_ARTIFACT_ID, getGraalPyVersion(project))); + } + + public static String getGraalPyVersion(Project project) { + return getGraalPyDependency(project).getVersion(); + } + + public static Dependency getGraalPyDependency(Project project) { + return resolveProjectDependencies(project).stream().filter(GraalPyGradlePlugin::isPythonArtifact).findFirst().orElseThrow(() -> new GradleException("Missing GraalPy dependency. Please add to your build.gradle either %s:%s or %s:%s".formatted(POLYGLOT_GROUP_ID, PYTHON_COMMUNITY_ARTIFACT_ID, POLYGLOT_GROUP_ID, PYTHON_ARTIFACT_ID))); + } + + private static boolean isPythonArtifact(Dependency dependency) { + return (POLYGLOT_GROUP_ID.equals(dependency.getGroup()) || GRAALPY_GROUP_ID.equals(dependency.getGroup())) && + (PYTHON_COMMUNITY_ARTIFACT_ID.equals(dependency.getName()) || PYTHON_ARTIFACT_ID.equals(dependency.getName())); + } + + private static DependencySet resolveProjectDependencies(Project project) { + return project.getConfigurations().getByName("implementation").getAllDependencies(); + } + +} \ No newline at end of file diff --git a/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/GradleLogger.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/GradleLogger.java new file mode 100644 index 0000000000..1612dc9ffa --- /dev/null +++ b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/GradleLogger.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.graalvm.python; + +import org.graalvm.python.embedding.tools.exec.SubprocessLog; +import org.gradle.api.logging.Logger; + +public class GradleLogger implements SubprocessLog { + private Logger logger; + + private GradleLogger(Logger logger) { + this.logger = logger; + } + + @Override + public void subProcessOut(CharSequence out) { + logger.lifecycle(out.toString()); + } + + @Override + public void subProcessErr(CharSequence err) { + logger.warn(err.toString()); + } + + @Override + public void log(CharSequence txt) { + logger.lifecycle(txt.toString()); + } + + @Override + public void log(CharSequence txt, Throwable t) { + logger.lifecycle(txt.toString(), t); + } + + public static GradleLogger of(Logger logger) { + return new GradleLogger(logger); + } +} diff --git a/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java new file mode 100644 index 0000000000..0255c8eefb --- /dev/null +++ b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/dsl/GraalPyExtension.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.graalvm.python.dsl; + +import org.gradle.api.Action; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.provider.ListProperty; +import org.gradle.api.provider.SetProperty; +import org.gradle.api.tasks.Nested; + +public interface GraalPyExtension { + + /** + * External directory supposed to be populated with python resources, namely graalpy stdlib and venv. + * It is either present or not. + */ + DirectoryProperty getPythonResourcesDirectory(); + + /** + * Determines third party python packages to be installed for graalpy usage. + */ + SetProperty getPackages(); + + /** + * Determines what parts of graalpy stdlib are supposed to be available for graalpy. + */ + @Nested + PythonHomeInfo getPythonHome(); + + default void pythonHome(Action action) { + action.execute(getPythonHome()); + } + +} diff --git a/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java new file mode 100644 index 0000000000..70fb487c8a --- /dev/null +++ b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/dsl/PythonHomeInfo.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.graalvm.python.dsl; + +import org.gradle.api.provider.SetProperty; + +/** + * Determines what parts of graalpy stdlib are supposed to be avalable for graalpy. + */ +public interface PythonHomeInfo { + SetProperty getIncludes(); + SetProperty getExcludes(); +} diff --git a/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java new file mode 100644 index 0000000000..2b3c4a927f --- /dev/null +++ b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/MetaInfTask.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.graalvm.python.tasks; + +import org.graalvm.python.embedding.tools.vfs.VFSUtils; +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleScriptException; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.TaskAction; +import java.io.IOException; + +public abstract class MetaInfTask extends DefaultTask { + + private static final String GRAALPY_GRADLE_PLUGIN_ARTIFACT_ID = "org.graalvm.python.gradle.plugin"; + + @OutputDirectory + public abstract DirectoryProperty getManifestOutputDir(); + + @TaskAction + public void exec() { + try { + VFSUtils.writeNativeImageConfig(getManifestOutputDir().get().getAsFile().toPath(), GRAALPY_GRADLE_PLUGIN_ARTIFACT_ID); + } catch (IOException e) { + throw new GradleScriptException("failed to create native image configuration files", e); + } + } +} diff --git a/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/ResourcesTask.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/ResourcesTask.java new file mode 100644 index 0000000000..fa8a4d7b75 --- /dev/null +++ b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/ResourcesTask.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.graalvm.python.tasks; + +import org.graalvm.python.GradleLogger; +import org.graalvm.python.embedding.tools.vfs.VFSUtils; +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.provider.*; +import org.gradle.api.tasks.*; +import org.gradle.api.tasks.Optional; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; + +import static org.graalvm.python.GraalPyGradlePlugin.getGraalPyDependency; +import static org.graalvm.python.GraalPyGradlePlugin.getGraalPyVersion; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.GRAALPY_GROUP_ID; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.LAUNCHER_NAME; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_HOME; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_ROOT; +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_VENV; + + +public abstract class ResourcesTask extends DefaultTask { + + private Set launcherClassPath; + + @Input + @Optional + public abstract Property getIncludeVfsRoot(); + + @Input + public abstract ListProperty getPackages(); + + @Input + public abstract ListProperty getIncludes(); + + @Input + public abstract ListProperty getExcludes(); + + @OutputDirectory + public abstract DirectoryProperty getOutput(); + + @TaskAction + public void exec() { + manageHome(); + manageVenv(); + } + + private void manageHome() { + Path homeDirectory = getHomeDirectory(); + + List includes = new ArrayList<>(getIncludes().get()); + List excludes = new ArrayList<>(getExcludes().get()); + + try { + VFSUtils.createHome(homeDirectory, getGraalPyVersion(getProject()), includes, excludes, () -> calculateLauncherClasspath(), GradleLogger.of(getLogger()), (s) -> getLogger().lifecycle(s)); + } catch (IOException e) { + throw new GradleException(String.format("failed to copy graalpy home %s", homeDirectory), e); + } + } + + private void manageVenv() { + List packages = getPackages().getOrElse(null); + try { + VFSUtils.createVenv(getVenvDirectory(), new ArrayList(packages), getLauncherPath(),() -> calculateLauncherClasspath(), getGraalPyVersion(getProject()), GradleLogger.of(getLogger()), (s) -> getLogger().lifecycle(s)); + } catch (IOException e) { + throw new GradleException(String.format("failed to create venv %s", getVenvDirectory()), e); + } + } + + private Set calculateLauncherClasspath() { + if (launcherClassPath == null) { + var addedPluginDependency = getProject().getConfigurations().getByName("runtimeClasspath").getAllDependencies().stream().filter(d -> d.getGroup().equals(GRAALPY_GROUP_ID) && d.getName().equals("python-launcher") && d.getVersion().equals(getGraalPyVersion(getProject()))).findFirst().orElseThrow(); + launcherClassPath = getProject().getConfigurations().getByName("runtimeClasspath").files(addedPluginDependency).stream().map(File::toString).collect(Collectors.toSet()); + launcherClassPath.addAll(getProject().getConfigurations().getByName("runtimeClasspath").files(getGraalPyDependency(getProject())).stream().map(File::toString).collect(Collectors.toSet())); + } + return launcherClassPath; + } + + private Path getLauncherPath() { + return Paths.get(getProject().getBuildDir().getAbsolutePath(), LAUNCHER_NAME); + } + + private Path getHomeDirectory() { + return getResourceDirectory(VFS_HOME); + } + + private Path getVenvDirectory() { + return getResourceDirectory(VFS_VENV); + } + + private Path getResourceDirectory(String type) { + return Path.of(getOutput().get().getAsFile().toURI()).resolve(getIncludeVfsRoot().getOrElse(true) ? VFS_ROOT : "").resolve(type); + } + +} diff --git a/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/VFSFilesListTask.java b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/VFSFilesListTask.java new file mode 100644 index 0000000000..645b4ea463 --- /dev/null +++ b/graalpython/org.graalvm.python.gradle.plugin/src/main/java/org/graalvm/python/tasks/VFSFilesListTask.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.graalvm.python.tasks; + +import org.graalvm.python.embedding.tools.vfs.VFSUtils; +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleScriptException; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.TaskAction; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.graalvm.python.embedding.tools.vfs.VFSUtils.VFS_ROOT; + +public abstract class VFSFilesListTask extends DefaultTask { + + @InputDirectory + public abstract Property getResourcesDir(); + + @TaskAction + public void exec() { + Path vfs = getVFSDir(); + if (Files.exists(vfs)) { + try { + VFSUtils.generateVFSFilesList(vfs); + } catch (IOException e) { + throw new GradleScriptException(String.format("failed to generate files list in '%s'", vfs), e); + } + } + } + + private Path getVFSDir() { + return Path.of(getResourcesDir().get().toURI()).resolve(VFS_ROOT); + } + +} diff --git a/graalpython/org.graalvm.python.gradle.plugin/src/main/resources/META-INF/gradle-plugins/org.graalvm.python.properties b/graalpython/org.graalvm.python.gradle.plugin/src/main/resources/META-INF/gradle-plugins/org.graalvm.python.properties new file mode 100644 index 0000000000..2b133dff16 --- /dev/null +++ b/graalpython/org.graalvm.python.gradle.plugin/src/main/resources/META-INF/gradle-plugins/org.graalvm.python.properties @@ -0,0 +1,3 @@ +implementation-class=org.graalvm.python.GraalPyGradlePlugin +version=24.2.0 +id=org.graalvm.python diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py index c8915b9959..9c63f1ac43 100644 --- a/mx.graalpython/mx_graalpython.py +++ b/mx.graalpython/mx_graalpython.py @@ -1536,7 +1536,11 @@ def graalpython_gate_runner(args, tasks): standalone_home = graalpy_standalone_home('jvm') mvn_repo_path, version, env = deploy_local_maven_repo() + # in order to run gradle we need a jdk <= 22 + env['GRADLE_JAVA_HOME'] = env.get('JAVA_HOME') + env['ENABLE_STANDALONE_UNITTESTS'] = 'true' + env['ENABLE_GRADLE_STANDALONE_UNITTESTS'] = 'true' env['ENABLE_JBANG_INTEGRATION_UNITTESTS'] ='true' env['JAVA_HOME'] = gvm_jdk env['PYTHON_STANDALONE_HOME'] = standalone_home @@ -1551,6 +1555,10 @@ def graalpython_gate_runner(args, tasks): if "distributionUrl" in urls: env["MAVEN_DISTRIBUTION_URL_OVERRIDE"] = mx_urlrewrites.rewriteurl(urls["distributionUrl"]) + urls = get_wrapper_urls("graalpython/com.oracle.graal.python.test/src/tests/standalone/gradle/gradle-test-project/gradle/wrapper/gradle-wrapper.properties", ["distributionUrl"]) + if "distributionUrl" in urls: + env["GRADLE_DISTRIBUTION_URL_OVERRIDE"] = mx_urlrewrites.rewriteurl(urls["distributionUrl"]) + env["org.graalvm.maven.downloader.version"] = version env["org.graalvm.maven.downloader.repository"] = f"{pathlib.Path(mvn_repo_path).as_uri()}/" diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index dbb644e312..2f334054de 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -1466,5 +1466,17 @@ "tag": ["default", "public"], }, }, + "org.graalvm.python.gradle.plugin": { + "class": "MavenProject", + "subDir": "graalpython", + "noMavenJavadoc": True, + "dependencies": [ + "GRAALPYTHON-LAUNCHER", + "GRAALPYTHON_EMBEDDING_TOOLS", + ], + "maven": { + "tag": ["default", "public"], + }, + }, }, }