From 3305673a511f72e87bff3226a3ee3502dd8acf7d Mon Sep 17 00:00:00 2001 From: Edmund Higham Date: Wed, 29 Jan 2025 15:19:51 -0500 Subject: [PATCH] [repo] generate version.py in Makefile (#14791) This generates the hail version in a way that's importable in python and javascript (for webpages). It simplifies how the version is used in those languages as well as the number of files we need to juggle in CI. This change has no security impact. --- .gitignore | 1 - Makefile | 4 +- batch/batch/front_end/front_end.py | 6 +- batch/test/utils.py | 4 +- build.yaml | 94 +++++-------------- ci/test/resources/build.yaml | 9 -- dev-docs/releasing.md | 23 ++--- dev-docs/services/tls-cookbook.md | 2 +- docker/hailgenetics/Makefile | 2 +- docker/hailgenetics/hailtop/Dockerfile | 1 - hail/.gitignore | 8 +- hail/Makefile | 69 +++++++++----- hail/python/MANIFEST.in | 3 - hail/python/hail/__init__.py | 15 +-- hail/python/hail/backend/py4j_backend.py | 7 +- hail/python/hail/backend/service_backend.py | 7 +- hail/python/hail/context.py | 43 ++------- hail/python/hail/docs/conf.py | 17 +++- hail/python/hail/methods/qc.py | 6 +- hail/python/hailtop/__init__.py | 24 ++--- hail/python/hailtop/batch/backend.py | 4 +- .../hailtop/batch/hail_genetics_images.py | 6 +- hail/python/hailtop/hailctl/__main__.py | 2 +- hail/python/hailtop/hailctl/batch/submit.py | 4 +- hail/python/hailtop/hailctl/hdinsight/cli.py | 8 +- .../python/hailtop/hailctl/hdinsight/start.py | 4 +- hail/python/setup-hailtop.py | 18 +++- hail/python/setup.py | 24 +++-- .../hailtop/batch/test_batch_local_backend.py | 4 +- hail/python/test/hailtop/batch/utils.py | 4 +- hail/version.mk | 22 ----- infra/bootstrap_utils.sh | 4 +- 32 files changed, 186 insertions(+), 263 deletions(-) delete mode 100644 hail/version.mk diff --git a/.gitignore b/.gitignore index b70036bfca8..18de27dacd0 100644 --- a/.gitignore +++ b/.gitignore @@ -33,7 +33,6 @@ hail/python/hail/docs/experimental/hail.experimental.DB.rst hail/python/hailtop/batch/docs/api/ hail/upload-qob-jar hail/upload-qob-test-resources -**/hail_version web_common/web_common/static/css/ docs.tar.gz website/website/static/css/ diff --git a/Makefile b/Makefile index fcb52fafb5b..43cbb87e220 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ HAILGENETICS_IMAGES = $(foreach img,hail vep-grch37-85 vep-grch38-95,hailgenetic CI_IMAGES = ci-utils hail-buildkit base hail-run PRIVATE_REGISTRY_IMAGES = $(patsubst %, pushed-private-%-image, $(SPECIAL_IMAGES) $(SERVICES_PLUS_ADMIN_POD) $(CI_IMAGES) $(HAILGENETICS_IMAGES)) -HAILTOP_VERSION := hail/python/hailtop/hail_version +HAILTOP_VERSION := hail/python/hailtop/version.py SERVICES_IMAGE_DEPS = hail-ubuntu-image $(HAILTOP_VERSION) $(shell git ls-files hail/python/hailtop gear web_common) EMPTY := @@ -110,7 +110,7 @@ generate-pip-lockfiles: ./generate-pip-lockfile.sh ci $(HAILTOP_VERSION): - $(MAKE) -C hail python/hailtop/hail_version + $(MAKE) -C hail python-version-info %-image: IMAGE_NAME = $(patsubst %-image,%,$@):$(TOKEN) diff --git a/batch/batch/front_end/front_end.py b/batch/batch/front_end/front_end.py index 770ebb5b4a6..77f65ea4571 100644 --- a/batch/batch/front_end/front_end.py +++ b/batch/batch/front_end/front_end.py @@ -47,7 +47,7 @@ from gear.database import CallError from gear.profiling import install_profiler_if_requested from gear.time_limited_max_size_cache import TimeLimitedMaxSizeCache -from hailtop import aiotools, dictfix, httpx, uvloopx, version +from hailtop import __version__, aiotools, dictfix, httpx, uvloopx from hailtop.auth import hail_credentials from hailtop.batch_client.globals import MAX_JOB_GROUPS_DEPTH, ROOT_JOB_GROUP_ID from hailtop.batch_client.parse import parse_cpu_in_mcpu, parse_memory_in_bytes, parse_storage_in_bytes @@ -251,7 +251,7 @@ async def get_healthcheck(_) -> web.Response: @routes.get('/api/v1alpha/version') @api_security_headers async def rest_get_version(_) -> web.Response: - return web.Response(text=version()) + return web.Response(text=__version__) @routes.get('/api/v1alpha/cloud') @@ -3490,7 +3490,7 @@ async def swagger(request): @routes.get('/openapi.yaml') @web_security_headers async def openapi(request): - page_context = {'base_path': deploy_config.base_path('batch'), 'spec_version': version()} + page_context = {'base_path': deploy_config.base_path('batch'), 'spec_version': __version__} return await render_template('batch', request, None, 'openapi.yaml', page_context) diff --git a/batch/test/utils.py b/batch/test/utils.py index 2b0e8122ac3..6a5757d10e2 100644 --- a/batch/test/utils.py +++ b/batch/test/utils.py @@ -4,10 +4,10 @@ import hailtop.batch_client.aioclient as aiobc import hailtop.batch_client.client as bc -from hailtop import pip_version +from hailtop import __pip_version__ DOCKER_ROOT_IMAGE = os.environ.get('DOCKER_ROOT_IMAGE', 'ubuntu:22.04') -HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{pip_version()}') +HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{__pip_version__}') @overload diff --git a/build.yaml b/build.yaml index c4c21c5ecb7..327582ddff9 100644 --- a/build.yaml +++ b/build.yaml @@ -22,16 +22,9 @@ steps: cd repo {{ code.checkout_script }} make -C hail python-version-info HAIL_RELEASE_MODE=1 MILLOPTS=--no-server - git rev-parse HEAD > git_version outputs: - from: /io/repo/ to: /repo - - from: /io/repo/hail/python/hail/hail_version - to: /hail_version - - from: /io/repo/hail/python/hail/hail_pip_version - to: /hail_pip_version - - from: /io/repo/git_version - to: /git_version dependsOn: - git_make_bash_image - kind: createNamespace @@ -279,8 +272,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail/python/hailtop/hail_version - from: /repo/gear to: /io/repo/gear dependsOn: @@ -354,8 +345,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail/python/hailtop/hail_version - from: /repo/gear to: /io/repo/gear - from: /repo/web_common @@ -623,8 +612,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail/python/hailtop/hail_version - from: /repo/gear to: /io/repo/gear - from: /repo/web_common @@ -650,8 +637,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail/python/hailtop/hail_version - from: /repo/gear to: /io/repo/gear - from: /repo/web_common @@ -673,8 +658,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail_version dependsOn: - merge_code - hail_ubuntu_image @@ -692,8 +675,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail/python/hailtop/hail_version - from: /repo/gear to: /io/repo/gear - from: /repo/web_common @@ -743,10 +724,8 @@ steps: time retry make wheel # Check wheel size is small enough for pypi (< 200 MiB) - HAIL_PIP_VERSION=$(cat python/hail/hail_pip_version) - WHEEL_PATH="build/deploy/dist/hail-$HAIL_PIP_VERSION-py3-none-any.whl" - du -h $WHEEL_PATH - $(python3 -c "import os; exit(1) if (os.path.getsize('$WHEEL_PATH')) > (200 * 1024 * 1024) else exit(0)") + BYTES=$(du build/deploy/dist/hail-*-py3-none-any.whl | awk '{print $1}') + (( BYTES < 209715200 )) || exit 1 inputs: - from: /repo to: /io/repo @@ -941,16 +920,12 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail/python/hailtop/hail_version - from: /repo/gear to: /io/repo/gear - from: /repo/web_common to: /io/repo/web_common - from: /jvm-entryway.jar to: /io/repo/batch/jvm-entryway/build/libs/jvm-entryway.jar - - from: /hail_version - to: /io/repo/hail_version dependsOn: - merge_code - hail_ubuntu_image @@ -1354,15 +1329,13 @@ steps: mv python/hail/docs/_build/html www/docs/0.2 mv python/hailtop/batch/docs/_build/html www/docs/batch - HAIL_CACHE_VERSION=$(cat python/hail/hail_version) + HAIL_CACHE_VERSION=$(cat env/HAIL_VERSION) find www -iname *.html -type f -exec sed -i -e "s/\.css/\.css\?v\=$HAIL_CACHE_VERSION/" {} +; tar czf /io/www.tar.gz www inputs: - - from: /repo/hail/python - to: /io/hail/python - - from: /hail_version - to: /io/repo/hail/python/hail/hail_version + - from: /repo/hail + to: /io/hail - from: /repo/website/website to: /io/website/website - from: /derived/release/hail/build/deploy/dist @@ -1594,7 +1567,7 @@ steps: # dev deploy elides the hail-is remote, add it and retrieve the tags git remote add hail-is https://github.com/hail-is/hail.git - export HAIL_PIP_VERSION=$(cat /io/hail_pip_version) + export HAIL_PIP_VERSION=$(cat hail/env/HAIL_PIP_VERSION) if git ls-remote --exit-code --tags hail-is $HAIL_PIP_VERSION then @@ -1627,8 +1600,6 @@ steps: to: /io/repo - from: /www.tar.gz to: /io/www.tar.gz - - from: /hail_pip_version - to: /io/hail_pip_version outputs: - from: /io/docs.tar.gz to: /docs.tar.gz @@ -2437,7 +2408,7 @@ steps: {% else %} HAIL_QUERY_JAR_URL={{ global.test_storage_uri }}/{{ default_ns.name }} {% endif %} - HAIL_QUERY_JAR_URL=${HAIL_QUERY_JAR_URL}/jars/$(cat /io/git_version).jar + HAIL_QUERY_JAR_URL=${HAIL_QUERY_JAR_URL}/jars/$(cat /io/REVISION).jar python3 -m hailtop.aiotools.copy 'null' '[{"from": "/io/hail.jar", "to": "'${HAIL_QUERY_JAR_URL}'"}]' secrets: @@ -2448,8 +2419,8 @@ steps: inputs: - from: /derived/release/hail/out/hail/assembly.dest/out.jar to: /io/hail.jar - - from: /git_version - to: /io/git_version + - from: /repo/hail/env/REVISION + to: /io/REVISION dependsOn: - default_ns - deploy_batch @@ -2842,7 +2813,6 @@ steps: cp -R /io/repo/hail/python ./hail/ cp /io/repo/hail/Makefile ./hail/ cp /io/repo/hail/env_var.mk ./hail/ - cp /io/repo/hail/version.mk ./hail/ cp -R /io/repo/web_common ./ cp /io/repo/hail/python/setup-hailtop.py ./hail/python/ @@ -3278,7 +3248,7 @@ steps: gcloud config set project hail-vdc gcloud config set dataproc/region us-central1 - if git ls-remote --exit-code --tags origin $(cat /io/hail_pip_version) + if git ls-remote --exit-code --tags origin $(cat hail/env/HAIL_PIP_VERSION) then echo "tag $HAIL_PIP_VERSION already exists" exit 0 @@ -3293,8 +3263,6 @@ steps: - merge_code - build_hail_jar_and_wheel inputs: - - from: /hail_pip_version - to: /io/hail_pip_version - from: /repo to: /io/repo - from: /derived/release/hail/build/deploy/dist @@ -3324,7 +3292,7 @@ steps: gcloud config set project hail-vdc gcloud config set dataproc/region us-central1 - if git ls-remote --exit-code --tags origin $(cat /io/hail_pip_version) + if git ls-remote --exit-code --tags origin $(cat hail/env/HAIL_PIP_VERSION) then echo "tag $HAIL_PIP_VERSION already exists" exit 0 @@ -3339,8 +3307,6 @@ steps: - merge_code - build_hail_jar_and_wheel inputs: - - from: /hail_pip_version - to: /io/hail_pip_version - from: /repo to: /io/repo - from: /derived/release/hail/build/deploy/dist @@ -3377,7 +3343,7 @@ steps: export GIT_ASKPASS=/io/git-askpass cd /io/repo/hail - export HAIL_PIP_VERSION=$(cat /io/hail_pip_version) + export HAIL_PIP_VERSION=$(cat env/HAIL_PIP_VERSION) if git ls-remote --exit-code --tags origin $HAIL_PIP_VERSION then @@ -3393,9 +3359,9 @@ steps: echo Setting arguments and invoking release.sh. - HAIL_VERSION=$(cat /io/hail_version) \ + HAIL_VERSION=$(cat env/HAIL_VERSION) \ WHEEL="build/deploy/dist/hail-${HAIL_PIP_VERSION}-py3-none-any.whl" \ - GIT_VERSION=$(cat /io/git_version) \ + GIT_VERSION=$(cat env/REVISION) \ REMOTE=origin \ GITHUB_OAUTH_HEADER_FILE=/io/github-oauth \ HAIL_GENETICS_HAIL_IMAGE=docker://{{ hailgenetics_hail_image.image }} \ @@ -3410,12 +3376,6 @@ steps: echo '1' > /io/release-hail-flag inputs: - - from: /hail_version - to: /io/hail_version - - from: /hail_pip_version - to: /io/hail_pip_version - - from: /git_version - to: /io/git_version - from: /repo to: /io/repo - from: /derived/release/hail/out/hail/assembly.dest/out.jar @@ -3480,8 +3440,6 @@ steps: - ci_utils_image - release inputs: - - from: /hail_pip_version - to: /io/hail_pip_version - from: /release-hail-flag to: /io/release-hail-flag - from: /repo @@ -3504,18 +3462,16 @@ steps: exit 0 fi - export HAIL_PIP_VERSION=$(cat /io/hail_pip_version) - export GITHUB_OAUTH_HEADER_FILE=/io/github-oauth - cd /io - printf 'Authorization: token ' > github-oauth - cat /hail-ci-0-1-github-oauth-token/oauth-token >>github-oauth + cat /hail-ci-0-1-github-oauth-token/oauth-token >> /io/github-oauth printf '#!/bin/bash\necho ' > git-askpass - cat /hail-ci-0-1-github-oauth-token/oauth-token >>git-askpass - chmod 755 git-askpass + cat /hail-ci-0-1-github-oauth-token/oauth-token >> /io/git-askpass + chmod 755 /io/git-askpass export GIT_ASKPASS=/io/git-askpass cd /io/repo/hail + export HAIL_PIP_VERSION=$(cat env/HAIL_PIP_VERSION) + export GITHUB_OAUTH_HEADER_FILE=/io/github-oauth bash scripts/make-terra-prs.sh - kind: runImage name: mirror_hailgenetics_images @@ -3539,12 +3495,12 @@ steps: set -x cd /io/docker/hailgenetics - export HAIL_PIP_VERSION=$(cat /io/hail_pip_version) + export HAIL_PIP_VERSION=$(cat /io/HAIL_PIP_VERSION) export DOCKER_PREFIX={{ global.docker_prefix }} bash mirror_images.sh inputs: - - from: /hail_pip_version - to: /io/hail_pip_version + - from: /repo/hail/env/HAIL_PIP_VERSION + to: /io/HAIL_PIP_VERSION - from: /repo/docker/hailgenetics/mirror_images.sh to: /io/docker/hailgenetics/mirror_images.sh - from: /repo/docker/copy_image.sh @@ -3571,8 +3527,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail/python/hailtop/hail_version - from: /repo/gear to: /io/repo/gear - from: /repo/web_common @@ -3745,8 +3699,6 @@ steps: to: /io/repo/hail/python - from: /repo/hail/scripts/benchmark_in_batch.py to: /io/repo/hail/scripts/benchmark_in_batch.py - - from: /hail_version - to: /io/hail_version - from: /release-hail-flag to: /io/release-hail-flag script: | @@ -3762,7 +3714,7 @@ steps: pushd /io/repo/hail BUCKET=hail-benchmarks-2 - HAIL_VERSION=$(cat /io/hail_version) + HAIL_VERSION=$(cat env/HAIL_VERSION) RESULTS_PREFIX="gs://${BUCKET}/{{ scope }}/${HAIL_VERSION}" export HAIL_BATCH_REGIONS={{ global.gcp_region }} diff --git a/ci/test/resources/build.yaml b/ci/test/resources/build.yaml index 9f46a939053..d25d6a9718d 100644 --- a/ci/test/resources/build.yaml +++ b/ci/test/resources/build.yaml @@ -28,16 +28,9 @@ steps: cd repo {{ code.checkout_script }} make -C hail python-version-info - git rev-parse HEAD > git_version outputs: - from: /io/repo/ to: /repo - - from: /io/repo/hail/python/hail/hail_version - to: /hail_version - - from: /io/repo/hail/python/hail/hail_pip_version - to: /hail_pip_version - - from: /io/repo/git_version - to: /git_version dependsOn: - git_make_bash_image - kind: buildImage2 @@ -142,8 +135,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail_version - from: /repo/gear to: /io/repo/gear - kind: buildImage2 diff --git a/dev-docs/releasing.md b/dev-docs/releasing.md index f26a4c8c816..450c16253dc 100644 --- a/dev-docs/releasing.md +++ b/dev-docs/releasing.md @@ -34,9 +34,6 @@ hail-ci-bpk3h bucket. The necessary files are listed under "Sources: " in the "I "release" build step. They should look something like: Sources: - gs://hail-ci-bpk3h/build/9cabeeb4ba047d1722e6f8da0383ab97/hail_version: 1 files, 21 Bytes - gs://hail-ci-bpk3h/build/9cabeeb4ba047d1722e6f8da0383ab97/hail_pip_version: 1 files, 8 Bytes - gs://hail-ci-bpk3h/build/9cabeeb4ba047d1722e6f8da0383ab97/git_version: 1 files, 41 Bytes gs://hail-ci-bpk3h/build/9cabeeb4ba047d1722e6f8da0383ab97/repo: 6272 files, 205.1 MB gs://hail-ci-bpk3h/build/9cabeeb4ba047d1722e6f8da0383ab97/azure-wheel: 1 files, 144.5 MB gs://hail-ci-bpk3h/build/9cabeeb4ba047d1722e6f8da0383ab97/www.tar.gz: 1 files, 43.5 MB @@ -47,14 +44,12 @@ Download all these files except the repo (which you do not need, because you che mkdir $BUILD_TOKEN RELEASE_ARTIFACTS_DIR=$(realpath $BUILD_TOKEN) gcloud storage cp -r \ - gs://hail-ci-bpk3h/build/$BUILD_TOKEN/hail_version \ - gs://hail-ci-bpk3h/build/$BUILD_TOKEN/hail_pip_version \ - gs://hail-ci-bpk3h/build/$BUILD_TOKEN/git_version \ + gs://hail-ci-bpk3h/build/$BUILD_TOKEN/repo/hail/env \ gs://hail-ci-bpk3h/build/$BUILD_TOKEN/azure-wheel \ gs://hail-ci-bpk3h/build/$BUILD_TOKEN/www.tar.gz \ - $BUILD_TOKEN + $RELEASE_ARTIFACTS_DIR -Note that the `-r` is necessary because some of these things like `azure-wheel` are folders. +Note that the `-r` is necessary because some of these things like `env` and `azure-wheel` are folders. Next we need to authenticate with DockerHub. Download the secret and authenticate skopeo with it. `download-secret` is a function stored in `devbin/functions.sh`. @@ -88,9 +83,9 @@ Ensure you have returned to the `hail` sub-folder of the Hail git repository. Now we can construct a `release.sh` invocation. Find the invocation in the "command" part of the "Job Specification" table. It should look like: - bash scripts/release.sh $(cat /io/hail_pip_version) \ - $(cat /io/hail_version) \ - $(cat /io/git_version) \ + bash scripts/release.sh $(cat $RELEASE_ARTIFACTS_DIR/HAIL_PIP_VERSION) \ + $(cat $RELEASE_ARTIFACTS_DIR/HAIL_VERSION) \ + $(cat $RELEASE_ARTIFACTS_DIR/REVISION) \ origin \ /io/repo/hail/build/deploy/dist/hail-*-py3-none-any.whl \ /io/github-oauth \ @@ -112,9 +107,9 @@ We need to make two replacements: It should look something like this: bash scripts/release.sh \ - $(cat $RELEASE_ARTIFACTS_DIR/hail_pip_version) \ - $(cat $RELEASE_ARTIFACTS_DIR/hail_version) \ - $(cat $RELEASE_ARTIFACTS_DIR/git_version) \ + $(cat $RELEASE_ARTIFACTS_DIR/HAIL_PIP_VERSION) \ + $(cat $RELEASE_ARTIFACTS_DIR/HAIL_VERSION) \ + $(cat $RELEASE_ARTIFACTS_DIR/REVISION) \ origin \ /PATH/TO/DOWNLOADED/HAIL-COMMON/WHEEL/hail-0.2.XXX-py3-none-any.whl \ $RELEASE_ARTIFACTS_DIR/github-oauth \ diff --git a/dev-docs/services/tls-cookbook.md b/dev-docs/services/tls-cookbook.md index 0b6070bf065..f14943d2e08 100644 --- a/dev-docs/services/tls-cookbook.md +++ b/dev-docs/services/tls-cookbook.md @@ -122,7 +122,7 @@ kubectl create secret generic \ 3. Update all the service certificates: ``` -make -C $HAIL/hail python/hailtop/hail_version +make -C $HAIL/hail python/hailtop/version.py PYTHONPATH=$HAIL/hail/python \ python3 $HAIL/tls/create_certs.py \ diff --git a/docker/hailgenetics/Makefile b/docker/hailgenetics/Makefile index f839e2af3be..97cff8c8d3d 100644 --- a/docker/hailgenetics/Makefile +++ b/docker/hailgenetics/Makefile @@ -7,5 +7,5 @@ publish-python-dill: mirror-dockerhub-images: DOCKER_PREFIX=$(DOCKER_PREFIX) \ - HAIL_PIP_VERSION=$(shell cat ../../hail/python/hail/hail_pip_version) \ + HAIL_PIP_VERSION=$(shell cat ../../hail/env/HAIL_PIP_VERSION) \ ./mirror_images.sh diff --git a/docker/hailgenetics/hailtop/Dockerfile b/docker/hailgenetics/hailtop/Dockerfile index 19156bd72cf..e2f82873776 100644 --- a/docker/hailgenetics/hailtop/Dockerfile +++ b/docker/hailgenetics/hailtop/Dockerfile @@ -6,5 +6,4 @@ RUN hail-pip-install -r requirements.txt COPY hail/python/setup-hailtop.py /hailtop/setup.py COPY hail/python/MANIFEST.in /hailtop/MANIFEST.in COPY hail/python/hailtop /hailtop/hailtop/ -COPY /hail_version /hailtop/hailtop/hail_version RUN hail-pip-install /hailtop diff --git a/hail/.gitignore b/hail/.gitignore index 506cc43dd2c..79152b40574 100644 --- a/hail/.gitignore +++ b/hail/.gitignore @@ -3,13 +3,10 @@ python/README.md python/dist python/hail.egg-info python/hail/backend/hail-all-spark.jar -python/hail/hail_revision -python/hail/hail_pip_version +python/hail/version.py python/hail/docs/change_log.rst python/hail/docs/_build/* python/hail/docs/_static/hail_version.js -python/hailtop/dist -python/hailtop/hailctl/deploy.yaml python/hail/docs/genetics/hail.genetics.AlleleType.rst python/hail/docs/genetics/hail.genetics.Call.rst python/hail/docs/genetics/hail.genetics.Locus.rst @@ -67,6 +64,9 @@ python/hail/docs/vds/hail.vds.local_to_global.rst python/hail/docs/vds/hail.vds.merge_reference_blocks.rst python/hail/docs/vds/hail.vds.truncate_reference_blocks.rst python/hail/docs/tutorials/iframe_figures/ +python/hailtop/hailctl/deploy.yaml +python/hailtop/dist +python/hailtop/version.py src/main/c/.cxx.vsn src/main/c/headers src/main/c/lib diff --git a/hail/Makefile b/hail/Makefile index 17a616d9e22..1634481c6eb 100644 --- a/hail/Makefile +++ b/hail/Makefile @@ -1,7 +1,25 @@ HAIL_HAIL_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) +REVISION := $(shell git rev-parse HEAD) +ifndef REVISION +$(error "git rev-parse HEAD" failed to produce output) +endif + +SHORT_REVISION := $(shell git rev-parse --short=12 HEAD) +ifndef SHORT_REVISION +$(error "git rev-parse --short=12 HEAD" failed to produce output) +endif + +HAIL_MAJOR_MINOR_VERSION := 0.2 +HAIL_PATCH_VERSION := 133 +HAIL_PIP_VERSION := $(HAIL_MAJOR_MINOR_VERSION).$(HAIL_PATCH_VERSION) +HAIL_VERSION := $(HAIL_PIP_VERSION)-$(SHORT_REVISION) + +SCALA_VERSION ?= 2.12.18 +SPARK_VERSION ?= 3.5.0 + + include env_var.mk -include version.mk .DEFAULT_GOAL := shadowJar MAKEFLAGS += --no-builtin-rules @@ -9,11 +27,10 @@ MAKEFLAGS += --no-builtin-rules $(eval $(call ENV_VAR,REVISION)) $(eval $(call ENV_VAR,SHORT_REVISION)) -$(eval $(call ENV_VAR,BRANCH)) -$(eval $(call ENV_VAR,SPARK_VERSION)) $(eval $(call ENV_VAR,HAIL_PIP_VERSION)) -$(eval $(call ENV_VAR,HAIL_DEBUG_MODE)) -$(eval $(call ENV_VAR,ELASTIC_MAJOR_VERSION)) +$(eval $(call ENV_VAR,HAIL_VERSION)) + +$(eval $(call ENV_VAR,SPARK_VERSION)) HAIL_PYTHON3 ?= python3 PIP ?= $(HAIL_PYTHON3) -m pip @@ -30,11 +47,6 @@ PYTEST_INTER_CLOUD_TARGET ?= test/hailtop/inter_cloud PY_FILES := $(shell git ls-files python) INIT_SCRIPTS := python/hailtop/hailctl/deploy.yaml -PYTHON_VERSION_INFO := python/hail/hail_revision -PYTHON_VERSION_INFO += python/hail/hail_version -PYTHON_VERSION_INFO += python/hail/hail_pip_version -PYTHON_VERSION_INFO += python/hailtop/hail_version -PYTHON_VERSION_INFO += python/hail/docs/_static/hail_version.js PYTHON_JAR := python/hail/backend/hail-all-spark.jar FAST_PYTHON_JAR := python/hail/backend/hail.jar @@ -125,24 +137,31 @@ fs-jvm-test: upload-remote-test-resources .PHONY: jars jars: shadowJar shadowTestJar -.PHONY: python-version-info -python-version-info: $(PYTHON_VERSION_INFO) - -python/hail/hail_revision: env/REVISION - echo $(REVISION) > $@ - -python/hail/hail_version: env/SHORT_REVISION env/HAIL_PIP_VERSION - echo $(HAIL_VERSION) > $@ +PYTHON_VERSION_INFO := \ + python/hail/version.py \ + python/hail/docs/_static/hail_version.js \ + python/hailtop/version.py -python/hail/hail_pip_version: env/HAIL_PIP_VERSION - echo $(HAIL_PIP_VERSION) > $@ +$(PYTHON_VERSION_INFO): env/HAIL_VERSION env/HAIL_PIP_VERSION env/REVISION env/SHORT_REVISION -python/hail/docs/_static/hail_version.js: python/hail/hail_version python/hail/hail_pip_version - printf 'hail_version="$(shell cat python/hail/hail_version)";' > $@ - printf 'hail_pip_version="$(shell cat python/hail/hail_pip_version)"' >> $@ +.PHONY: python-version-info +python-version-info: $(PYTHON_VERSION_INFO) -python/hailtop/hail_version: python/hail/hail_version - cp -f $< $@ +.ONESHELL: $(PYTHON_VERSION_INFO) +python/hail/version.py python/hailtop/version.py: + cat <<- EOF > $@ + # Generated by make on $(shell date). + __pip_version__ = "$(HAIL_PIP_VERSION)" + __revision__ = "$(REVISION)" + __version__ = "$(HAIL_VERSION)" + EOF + +python/hail/docs/_static/hail_version.js: + cat <<- EOF > $@ + # Generated by make on $(shell date). + hail_pip_version = "$(HAIL_PIP_VERSION)"; + hail_version = "$(HAIL_VERSION)"; + EOF python/README.md: ../README.md cp ../README.md python/ diff --git a/hail/python/MANIFEST.in b/hail/python/MANIFEST.in index aa401baf489..468c769a59f 100644 --- a/hail/python/MANIFEST.in +++ b/hail/python/MANIFEST.in @@ -1,8 +1,5 @@ -include hail/hail_pip_version -include hail/hail_version include hail/experimental/datasets.json include hail/backend/hail-all-spark.jar -include hailtop/hail_version include hailtop/hailctl/deploy.yaml include hailtop/py.typed include requirements.txt diff --git a/hail/python/hail/__init__.py b/hail/python/hail/__init__.py index 7b6fffba8b5..a6a2de9acc9 100644 --- a/hail/python/hail/__init__.py +++ b/hail/python/hail/__init__.py @@ -1,7 +1,6 @@ import importlib.resources from pathlib import Path from sys import version_info -from typing import Optional if version_info < (3, 9): raise EnvironmentError(f'Hail requires Python 3.9 or later, found {version_info.major}.{version_info.minor}') @@ -90,7 +89,6 @@ def __resource_str(name: str) -> str: spark_context, stop, tmp_dir, - version, ) from .expr import * # noqa: F403 from .expr import aggregators @@ -98,11 +96,20 @@ def __resource_str(name: str) -> str: from .matrixtable import GroupedMatrixTable, MatrixTable from .methods import * # noqa: F403 from .table import GroupedTable, Table, asc, desc +from .version import __pip_version__, __revision__, __version__ agg = aggregators scan = aggregators.aggregators.ScanFunctions({name: getattr(agg, name) for name in agg.__all__}) + +def version() -> str: + return __version__ + + __all__ = [ + '__pip_version__', + '__revision__', + '__version__', 'init', 'init_local', 'init_batch', @@ -172,10 +179,6 @@ def __resource_str(name: str) -> str: ir.register_functions() ir.register_aggregators() -__pip_version__ = __resource_str('hail_pip_version').strip() -__version__: Optional[str] = None # set by hail.version() -__revision__: Optional[str] = None # set by hail.revision() - import warnings warnings.filterwarnings('once', append=True) diff --git a/hail/python/hail/backend/py4j_backend.py b/hail/python/hail/backend/py4j_backend.py index b9f986e5834..03bc9be5dfe 100644 --- a/hail/python/hail/backend/py4j_backend.py +++ b/hail/python/hail/backend/py4j_backend.py @@ -15,6 +15,7 @@ from hail.expr import construct_expr from hail.ir import JavaIR from hail.utils.java import Env, FatalError, scala_package_object +from hail.version import __version__ from ..hail_logging import Logger from .backend import ActionTag, Backend, fatal_error_from_java_error_triplet @@ -197,15 +198,13 @@ def decode_bytearray(encoded): # This has to go after creating the SparkSession. Unclear why. # Maybe it does its own patch? install_exception_handler() - from hail.context import version - py_version = version() jar_version = self._jhc.version() - if jar_version != py_version: + if jar_version != __version__: raise RuntimeError( f"Hail version mismatch between JAR and Python library\n" f" JAR: {jar_version}\n" - f" Python: {py_version}" + f" Python: {__version__}" ) def jvm(self): diff --git a/hail/python/hail/backend/service_backend.py b/hail/python/hail/backend/service_backend.py index 5a5a2457173..953ce749b05 100644 --- a/hail/python/hail/backend/service_backend.py +++ b/hail/python/hail/backend/service_backend.py @@ -10,13 +10,14 @@ import orjson import hailtop.aiotools.fs as afs -from hail.context import TemporaryDirectory, TemporaryFilename, revision, tmp_dir, version +from hail.context import TemporaryDirectory, TemporaryFilename, tmp_dir from hail.experimental import read_expression, write_expression from hail.expr.expressions.base_expression import Expression from hail.expr.types import HailType from hail.ir import finalize_randomness from hail.ir.renderer import CSERenderer from hail.utils import FatalError +from hail.version import __revision__, __version__ from hailtop import yamlx from hailtop.aiocloud.aiogoogle import GCSRequesterPaysConfiguration, get_gcs_requester_pays_configuration from hailtop.aiotools.fs.exceptions import UnexpectedEOFError @@ -201,7 +202,7 @@ async def create( name_prefix = configuration_of(ConfigVariable.QUERY_NAME_PREFIX, name_prefix, '') batch_attributes: Dict[str, str] = { - 'hail-version': version(), + 'hail-version': __version__, } if name_prefix: batch_attributes['name'] = name_prefix @@ -325,7 +326,7 @@ def fs(self) -> FS: @property def jar_spec(self) -> dict: - return {'type': 'git_revision', 'value': revision()} + return {'type': 'git_revision', 'value': __revision__} @property def logger(self): diff --git a/hail/python/hail/context.py b/hail/python/hail/context.py index 77493ecd083..6c96599a24b 100644 --- a/hail/python/hail/context.py +++ b/hail/python/hail/context.py @@ -15,12 +15,12 @@ from hail.typecheck import dictof, enumeration, nullable, oneof, sequenceof, sized_tupleof, typecheck, typecheck_method from hail.utils import get_env_or_default from hail.utils.java import BackendType, Env, choose_backend, warning +from hail.version import __version__ from hailtop.aiocloud.aiogoogle import GCSRequesterPaysConfiguration, get_gcs_requester_pays_configuration from hailtop.fs.fs import FS from hailtop.hail_event_loop import hail_event_loop from hailtop.utils import secret_alnum_string -from . import __resource_str from .backend.backend import local_jar_information from .builtin_references import BUILTIN_REFERENCES @@ -43,11 +43,10 @@ def _get_local_tmpdir(local_tmpdir): def _get_log(log): if log is None: - py_version = version() log_dir = os.environ.get('HAIL_LOG_DIR') if log_dir is None: log_dir = os.getcwd() - log = hail.utils.timestamp_path(os.path.join(log_dir, 'hail'), suffix=f'-{py_version}.log') + log = hail.utils.timestamp_path(os.path.join(log_dir, 'hail'), suffix=f'-{__version__}.log') return log @@ -104,16 +103,15 @@ def __init__(self, log, quiet, append, tmpdir, local_tmpdir, global_seed, backen self._default_ref: Optional[ReferenceGenome] = None if not quiet: - py_version = version() sys.stderr.write( 'Welcome to\n' ' __ __ <>__\n' ' / /_/ /__ __/ /\n' ' / __ / _ `/ / /\n' - ' /_/ /_/\\_,_/_/_/ version {}\n'.format(py_version) + ' /_/ /_/\\_,_/_/_/ version {}\n'.format(__version__) ) - if py_version.startswith('devel'): + if __version__.startswith('devel'): sys.stderr.write( 'NOTE: This is a beta version. Interfaces may change\n' ' during the beta period. We recommend pulling\n' @@ -633,35 +631,8 @@ def init_local( connect_logger(backend._utils_package_object, 'localhost', 12888) -def version() -> str: - """Get the installed Hail version. - - Returns - ------- - str - """ - if hail.__version__ is None: - hail.__version__ = __resource_str('hail_version').strip() - - return hail.__version__ - - -def revision() -> str: - """Get the installed Hail git revision. - - Returns - ------- - str - """ - if hail.__revision__ is None: - hail.__revision__ = __resource_str('hail_revision').strip() - - return hail.__revision__ - - def _hail_cite_url(): - v = version() - [tag, sha_prefix] = v.split("-") + [tag, sha_prefix] = __version__.split("-") if not local_jar_information().development_mode: # pip installed return f"https://github.com/hail-is/hail/releases/tag/{tag}" @@ -688,7 +659,7 @@ def citation(*, bibtex=False): f" howpublished = {{\\url{{{_hail_cite_url()}}}}}" f"}}" ) - return f"Hail Team. Hail {version()}. {_hail_cite_url()}." + return f"Hail Team. Hail {__version__}. {_hail_cite_url()}." def cite_hail(): @@ -965,4 +936,4 @@ def debug_info(): spark_conf = None if isinstance(Env.backend(), SparkBackend): spark_conf = spark_context()._conf.getAll() - return {'spark_conf': spark_conf, 'local_jar_information': local_jar_information(), 'version': version()} + return {'spark_conf': spark_conf, 'local_jar_information': local_jar_information(), 'version': __version__} diff --git a/hail/python/hail/docs/conf.py b/hail/python/hail/docs/conf.py index a7bfbf2c136..5760713c0bc 100644 --- a/hail/python/hail/docs/conf.py +++ b/hail/python/hail/docs/conf.py @@ -19,8 +19,17 @@ import datetime import os -import re import sys +from importlib.util import module_from_spec, spec_from_file_location + + +def __load_module(name, path): + spec = spec_from_file_location(name, path) + mod = module_from_spec(spec) + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + sys.path.insert(0, os.path.abspath('./_ext')) @@ -106,9 +115,9 @@ # built documents. # # The full version, including alpha/beta/rc tags. -with open('../hail_version') as f: - release = f.read().strip() - version = re.match('[0-9]+.[0-9]+', release)[0] +mod = __load_module('hail.version', '../version.py') +release = mod.__version__ +version = mod.__pip_version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/hail/python/hail/methods/qc.py b/hail/python/hail/methods/qc.py index 2efd7b37944..338937c3fba 100644 --- a/hail/python/hail/methods/qc.py +++ b/hail/python/hail/methods/qc.py @@ -20,7 +20,7 @@ from hail.utils import FatalError from hail.utils.java import Env, info, warning from hail.utils.misc import divide_null, guess_cloud_spark_provider, new_temp_file -from hailtop import pip_version, yamlx +from hailtop import __pip_version__, yamlx from hailtop.config import get_deploy_config from hailtop.utils import async_to_blocking @@ -36,10 +36,10 @@ HAIL_GENETICS_VEP_GRCH37_85_IMAGE = os.environ.get( - 'HAIL_GENETICS_VEP_GRCH37_85_IMAGE', f'hailgenetics/vep-grch37-85:{pip_version()}' + 'HAIL_GENETICS_VEP_GRCH37_85_IMAGE', f'hailgenetics/vep-grch37-85:{__pip_version__}' ) HAIL_GENETICS_VEP_GRCH38_95_IMAGE = os.environ.get( - 'HAIL_GENETICS_VEP_GRCH38_95_IMAGE', f'hailgenetics/vep-grch38-95:{pip_version()}' + 'HAIL_GENETICS_VEP_GRCH38_95_IMAGE', f'hailgenetics/vep-grch38-95:{__pip_version__}' ) diff --git a/hail/python/hailtop/__init__.py b/hail/python/hailtop/__init__.py index 9ee2e7a1b54..32f5c569bbb 100644 --- a/hail/python/hailtop/__init__.py +++ b/hail/python/hailtop/__init__.py @@ -1,19 +1,4 @@ -_VERSION = None - - -def version() -> str: - global _VERSION - if _VERSION is None: - import importlib.resources as r # pylint: disable=import-outside-toplevel - - _VERSION = (r.files(__name__) / 'hail_version').read_text('utf-8').strip() - - return _VERSION - - -def pip_version() -> str: - return version().split('-')[0] - +from hailtop.version import __pip_version__, __version__ IS_NOTEBOOK = None @@ -28,3 +13,10 @@ def is_notebook() -> bool: except (NameError, ModuleNotFoundError): IS_NOTEBOOK = False return IS_NOTEBOOK + + +__all__ = [ + '__pip_version__', + '__version__', + 'is_notebook', +] diff --git a/hail/python/hailtop/batch/backend.py b/hail/python/hailtop/batch/backend.py index ddf65f58692..8686fdb93b1 100644 --- a/hail/python/hailtop/batch/backend.py +++ b/hail/python/hailtop/batch/backend.py @@ -16,7 +16,7 @@ from rich.progress import track import hailtop.batch_client.client as bc -from hailtop import pip_version +from hailtop import __pip_version__ from hailtop.aiocloud.aiogoogle import GCSRequesterPaysConfiguration from hailtop.aiotools.router_fs import RouterAsyncFS from hailtop.aiotools.validators import validate_file @@ -34,7 +34,7 @@ from .globals import DEFAULT_SHELL from .job import PythonJob -HAIL_GENETICS_HAILTOP_IMAGE = os.environ.get('HAIL_GENETICS_HAILTOP_IMAGE', f'hailgenetics/hailtop:{pip_version()}') +HAIL_GENETICS_HAILTOP_IMAGE = os.environ.get('HAIL_GENETICS_HAILTOP_IMAGE', f'hailgenetics/hailtop:{__pip_version__}') HAIL_GENETICS_HAIL_IMAGE = ( os.environ.get('HAIL_GENETICS_HAIL_IMAGE') or hailgenetics_hail_image_for_current_python_version() ) diff --git a/hail/python/hailtop/batch/hail_genetics_images.py b/hail/python/hailtop/batch/hail_genetics_images.py index c5f8b7882aa..97249edf615 100644 --- a/hail/python/hailtop/batch/hail_genetics_images.py +++ b/hail/python/hailtop/batch/hail_genetics_images.py @@ -1,6 +1,6 @@ import sys -from hailtop import pip_version +from hailtop import __pip_version__ HAIL_GENETICS = 'hailgenetics/' HAIL_GENETICS_IMAGES = [ @@ -16,5 +16,5 @@ def hailgenetics_hail_image_for_current_python_version(): f'using a Python version other than 3.9, 3.10, or 3.11 (you are using {version})' ) if version.minor == 9: - return f'hailgenetics/hail:{pip_version()}' - return f'hailgenetics/hail:{pip_version()}-py{version.major}.{version.minor}' + return f'hailgenetics/hail:{__pip_version__}' + return f'hailgenetics/hail:{__pip_version__}-py{version.major}.{version.minor}' diff --git a/hail/python/hailtop/hailctl/__main__.py b/hail/python/hailtop/hailctl/__main__.py index 98e80043868..19695b52557 100644 --- a/hail/python/hailtop/hailctl/__main__.py +++ b/hail/python/hailtop/hailctl/__main__.py @@ -32,7 +32,7 @@ def version(): """Print version information and exit.""" import hailtop # pylint: disable=import-outside-toplevel - print(hailtop.version()) + print(hailtop.__version__) @app.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True}) diff --git a/hail/python/hailtop/hailctl/batch/submit.py b/hail/python/hailtop/hailctl/batch/submit.py index 196b844ab23..234ec62ac52 100644 --- a/hail/python/hailtop/hailctl/batch/submit.py +++ b/hail/python/hailtop/hailctl/batch/submit.py @@ -5,7 +5,7 @@ import orjson -from hailtop import pip_version +from hailtop import __pip_version__ FILE_REGEX = re.compile(r'(?P[^:]+)(:(?P.+))?') @@ -62,7 +62,7 @@ def file_input_to_src_dest(file: str) -> Tuple[str, str, str]: backend = hb.ServiceBackend() b = hb.Batch(name=name, backend=backend) j = b.new_bash_job() - j.image(image_name or os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{pip_version()}')) + j.image(image_name or os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{__pip_version__}')) local_files_to_cloud_files = [] diff --git a/hail/python/hailtop/hailctl/hdinsight/cli.py b/hail/python/hailtop/hailctl/hdinsight/cli.py index 3ad7ae6f3e9..255c9019d81 100644 --- a/hail/python/hailtop/hailctl/hdinsight/cli.py +++ b/hail/python/hailtop/hailctl/hdinsight/cli.py @@ -79,12 +79,10 @@ def start( """ Start an HDInsight cluster configured for Hail. """ - from ... import pip_version # pylint: disable=import-outside-toplevel - - hail_version = pip_version() + from ... import __pip_version__ # pylint: disable=import-outside-toplevel def default_artifact(filename: str) -> str: - return f'https://raw.githubusercontent.com/hail-is/hail/{hail_version}/hail/python/hailtop/hailctl/hdinsight/resources/{filename}' + return f'https://raw.githubusercontent.com/hail-is/hail/{__pip_version__}/hail/python/hailtop/hailctl/hdinsight/resources/{filename}' hdinsight_start( cluster_name, @@ -97,7 +95,7 @@ def default_artifact(filename: str) -> str: install_hail_uri or default_artifact('install-hail.sh'), install_native_deps_uri or default_artifact('install-native-deps.sh'), wheel_uri - or f'https://storage.googleapis.com/hail-common/azure-hdinsight-wheels/hail-{hail_version}-py3-none-any.whl', + or f'https://storage.googleapis.com/hail-common/azure-hdinsight-wheels/hail-{__pip_version__}-py3-none-any.whl', vep, vep_loftee_uri, vep_homo_sapiens_uri, diff --git a/hail/python/hailtop/hailctl/hdinsight/start.py b/hail/python/hailtop/hailctl/hdinsight/start.py index f2e2fa9ec83..335160144b0 100644 --- a/hail/python/hailtop/hailctl/hdinsight/start.py +++ b/hail/python/hailtop/hailctl/hdinsight/start.py @@ -39,7 +39,7 @@ def start( import requests # pylint: disable=import-outside-toplevel import requests.auth # pylint: disable=import-outside-toplevel - from ... import pip_version # pylint: disable=import-outside-toplevel + from ... import __pip_version__ # pylint: disable=import-outside-toplevel from ...utils import secret_alnum_string # pylint: disable=import-outside-toplevel print(f'Starting the cluster {cluster_name}') @@ -138,7 +138,7 @@ def start( sys.exit(1) if vep_config_uri is None: - vep_config_uri = f'https://raw.githubusercontent.com/hail-is/hail/{pip_version()}/hail/python/hailtop/hailctl/hdinsight/resources/vep-{vep}.json' + vep_config_uri = f'https://raw.githubusercontent.com/hail-is/hail/{__pip_version__}/hail/python/hailtop/hailctl/hdinsight/resources/vep-{vep}.json' if vep_loftee_uri is None or vep_homo_sapiens_uri is None: raise ValueError("Both `vep_loftee_uri` and `vep_homo_sapiens_uri` must be specified if `vep` is specified") diff --git a/hail/python/setup-hailtop.py b/hail/python/setup-hailtop.py index 3a59a9b4f29..f98cd35a882 100644 --- a/hail/python/setup-hailtop.py +++ b/hail/python/setup-hailtop.py @@ -1,10 +1,22 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 + +import sys +from importlib.util import module_from_spec, spec_from_file_location from setuptools import find_packages, setup + +def load_module(name, path): + spec = spec_from_file_location(name, path) + mod = module_from_spec(spec) + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + + setup( name='hailtop', - version="0.0.1", + version=load_module('version', 'hailtop/version.py').__pip_version__, author="Hail Team", author_email="hail@broadinstitute.org", description="Top level Hail module.", @@ -15,7 +27,7 @@ }, packages=find_packages('.'), package_dir={'hailtop': 'hailtop'}, - package_data={"hailtop": ["py.typed", "hail_version"], 'hailtop.hailctl': ['hail_version', 'deploy.yaml']}, + package_data={"hailtop": ["py.typed"], 'hailtop.hailctl': ['deploy.yaml']}, classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", diff --git a/hail/python/setup.py b/hail/python/setup.py index ded40cb8d31..1d43379c3ac 100755 --- a/hail/python/setup.py +++ b/hail/python/setup.py @@ -1,15 +1,23 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import os +import sys +from importlib.util import module_from_spec, spec_from_file_location from setuptools import find_packages, setup -with open('hail/hail_pip_version') as f: - hail_pip_version = f.read().strip() - with open("README.md", "r") as fh: long_description = fh.read() + +def load_module(name, path): + spec = spec_from_file_location(name, path) + mod = module_from_spec(spec) + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + + dependencies = [] with open('requirements.txt', 'r') as f: for line in f: @@ -46,7 +54,7 @@ def add_dependencies(fname): setup( name="hail", - version=hail_pip_version, + version=load_module('version', 'hail/version.py').__pip_version__, author="Hail Team", author_email="hail@broadinstitute.org", description="Scalable library for exploring and analyzing genomic data.", @@ -61,10 +69,10 @@ def add_dependencies(fname): packages=find_packages('.'), package_dir={'hail': 'hail', 'hailtop': 'hailtop'}, package_data={ - 'hail': ['hail_pip_version', 'hail_version', 'hail_revision', 'experimental/datasets.json'], + 'hail': ['experimental/datasets.json'], 'hail.backend': ['hail-all-spark.jar'], - 'hailtop': ['hail_version', 'py.typed'], - 'hailtop.hailctl': ['hail_version', 'deploy.yaml'], + 'hailtop': ['py.typed'], + 'hailtop.hailctl': ['deploy.yaml'], }, classifiers=[ "Programming Language :: Python :: 3", diff --git a/hail/python/test/hailtop/batch/test_batch_local_backend.py b/hail/python/test/hailtop/batch/test_batch_local_backend.py index 15e256a6dc8..e96a2cb46b0 100644 --- a/hail/python/test/hailtop/batch/test_batch_local_backend.py +++ b/hail/python/test/hailtop/batch/test_batch_local_backend.py @@ -7,14 +7,14 @@ import pytest -from hailtop import pip_version +from hailtop import __pip_version__ from hailtop.batch import Batch, LocalBackend, ResourceGroup from hailtop.batch.resource import JobResourceFile from hailtop.batch.utils import concatenate DOCKER_ROOT_IMAGE = os.environ.get('DOCKER_ROOT_IMAGE', 'ubuntu:22.04') PYTHON_DILL_IMAGE = 'hailgenetics/python-dill:3.9-slim' -HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{pip_version()}') +HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{__pip_version__}') REQUESTER_PAYS_PROJECT = os.environ.get('GCS_REQUESTER_PAYS_PROJECT') diff --git a/hail/python/test/hailtop/batch/utils.py b/hail/python/test/hailtop/batch/utils.py index 4915962f9c5..928e04f9c17 100644 --- a/hail/python/test/hailtop/batch/utils.py +++ b/hail/python/test/hailtop/batch/utils.py @@ -1,12 +1,12 @@ import inspect import os -from hailtop import pip_version +from hailtop import __pip_version__ from hailtop.batch import Batch DOCKER_ROOT_IMAGE = os.environ.get('DOCKER_ROOT_IMAGE', 'ubuntu:22.04') PYTHON_DILL_IMAGE = 'hailgenetics/python-dill:3.9-slim' -HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{pip_version()}') +HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{__pip_version__}') REQUESTER_PAYS_PROJECT = os.environ.get('GCS_REQUESTER_PAYS_PROJECT') diff --git a/hail/version.mk b/hail/version.mk deleted file mode 100644 index 9754b7a2b4c..00000000000 --- a/hail/version.mk +++ /dev/null @@ -1,22 +0,0 @@ -REVISION := $(shell git rev-parse HEAD) -ifndef REVISION -$(error "git rev-parse HEAD" failed to produce output) -endif - -SHORT_REVISION := $(shell git rev-parse --short=12 HEAD) -ifndef SHORT_REVISION -$(error "git rev-parse --short=12 HEAD" failed to produce output) -endif - -BRANCH := $(shell git rev-parse --abbrev-ref HEAD) -ifndef BRANCH -$(error "git rev-parse --abbrev-ref HEAD" failed to produce output) -endif - -SCALA_VERSION ?= 2.12.18 -SPARK_VERSION ?= 3.5.0 -HAIL_MAJOR_MINOR_VERSION := 0.2 -HAIL_PATCH_VERSION := 133 -HAIL_PIP_VERSION := $(HAIL_MAJOR_MINOR_VERSION).$(HAIL_PATCH_VERSION) -HAIL_VERSION := $(HAIL_PIP_VERSION)-$(SHORT_REVISION) -ELASTIC_MAJOR_VERSION ?= 7 diff --git a/infra/bootstrap_utils.sh b/infra/bootstrap_utils.sh index 14e2325d12e..c4808ac5f2c 100644 --- a/infra/bootstrap_utils.sh +++ b/infra/bootstrap_utils.sh @@ -12,7 +12,7 @@ source $HAIL/devbin/functions.sh copy_images() { make -C $HAIL/docker/third-party copy - make -C $HAIL/hail python/hail/hail_pip_version + make -C $HAIL/hail python/hail/version.py make -C $HAIL/docker/hailgenetics mirror-dockerhub-images } @@ -49,7 +49,7 @@ generate_ssl_certs() { deploy_unmanaged() { export NAMESPACE=default - make -C $HAIL/hail python/hailtop/hail_version + make -C $HAIL/hail python/hailtop/version.py copy_images generate_ssl_certs