diff --git a/.gitignore b/.gitignore index b70036bfca8..18de27dacd0 100644 --- a/.gitignore +++ b/.gitignore @@ -33,7 +33,6 @@ hail/python/hail/docs/experimental/hail.experimental.DB.rst hail/python/hailtop/batch/docs/api/ hail/upload-qob-jar hail/upload-qob-test-resources -**/hail_version web_common/web_common/static/css/ docs.tar.gz website/website/static/css/ diff --git a/Makefile b/Makefile index fcb52fafb5b..43cbb87e220 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ HAILGENETICS_IMAGES = $(foreach img,hail vep-grch37-85 vep-grch38-95,hailgenetic CI_IMAGES = ci-utils hail-buildkit base hail-run PRIVATE_REGISTRY_IMAGES = $(patsubst %, pushed-private-%-image, $(SPECIAL_IMAGES) $(SERVICES_PLUS_ADMIN_POD) $(CI_IMAGES) $(HAILGENETICS_IMAGES)) -HAILTOP_VERSION := hail/python/hailtop/hail_version +HAILTOP_VERSION := hail/python/hailtop/version.py SERVICES_IMAGE_DEPS = hail-ubuntu-image $(HAILTOP_VERSION) $(shell git ls-files hail/python/hailtop gear web_common) EMPTY := @@ -110,7 +110,7 @@ generate-pip-lockfiles: ./generate-pip-lockfile.sh ci $(HAILTOP_VERSION): - $(MAKE) -C hail python/hailtop/hail_version + $(MAKE) -C hail python-version-info %-image: IMAGE_NAME = $(patsubst %-image,%,$@):$(TOKEN) diff --git a/batch/batch/front_end/front_end.py b/batch/batch/front_end/front_end.py index 770ebb5b4a6..77f65ea4571 100644 --- a/batch/batch/front_end/front_end.py +++ b/batch/batch/front_end/front_end.py @@ -47,7 +47,7 @@ from gear.database import CallError from gear.profiling import install_profiler_if_requested from gear.time_limited_max_size_cache import TimeLimitedMaxSizeCache -from hailtop import aiotools, dictfix, httpx, uvloopx, version +from hailtop import __version__, aiotools, dictfix, httpx, uvloopx from hailtop.auth import hail_credentials from hailtop.batch_client.globals import MAX_JOB_GROUPS_DEPTH, ROOT_JOB_GROUP_ID from hailtop.batch_client.parse import parse_cpu_in_mcpu, parse_memory_in_bytes, parse_storage_in_bytes @@ -251,7 +251,7 @@ async def get_healthcheck(_) -> web.Response: @routes.get('/api/v1alpha/version') @api_security_headers async def rest_get_version(_) -> web.Response: - return web.Response(text=version()) + return web.Response(text=__version__) @routes.get('/api/v1alpha/cloud') @@ -3490,7 +3490,7 @@ async def swagger(request): @routes.get('/openapi.yaml') @web_security_headers async def openapi(request): - page_context = {'base_path': deploy_config.base_path('batch'), 'spec_version': version()} + page_context = {'base_path': deploy_config.base_path('batch'), 'spec_version': __version__} return await render_template('batch', request, None, 'openapi.yaml', page_context) diff --git a/batch/test/utils.py b/batch/test/utils.py index 2b0e8122ac3..6a5757d10e2 100644 --- a/batch/test/utils.py +++ b/batch/test/utils.py @@ -4,10 +4,10 @@ import hailtop.batch_client.aioclient as aiobc import hailtop.batch_client.client as bc -from hailtop import pip_version +from hailtop import __pip_version__ DOCKER_ROOT_IMAGE = os.environ.get('DOCKER_ROOT_IMAGE', 'ubuntu:22.04') -HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{pip_version()}') +HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{__pip_version__}') @overload diff --git a/build.yaml b/build.yaml index c4c21c5ecb7..327582ddff9 100644 --- a/build.yaml +++ b/build.yaml @@ -22,16 +22,9 @@ steps: cd repo {{ code.checkout_script }} make -C hail python-version-info HAIL_RELEASE_MODE=1 MILLOPTS=--no-server - git rev-parse HEAD > git_version outputs: - from: /io/repo/ to: /repo - - from: /io/repo/hail/python/hail/hail_version - to: /hail_version - - from: /io/repo/hail/python/hail/hail_pip_version - to: /hail_pip_version - - from: /io/repo/git_version - to: /git_version dependsOn: - git_make_bash_image - kind: createNamespace @@ -279,8 +272,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail/python/hailtop/hail_version - from: /repo/gear to: /io/repo/gear dependsOn: @@ -354,8 +345,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail/python/hailtop/hail_version - from: /repo/gear to: /io/repo/gear - from: /repo/web_common @@ -623,8 +612,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail/python/hailtop/hail_version - from: /repo/gear to: /io/repo/gear - from: /repo/web_common @@ -650,8 +637,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail/python/hailtop/hail_version - from: /repo/gear to: /io/repo/gear - from: /repo/web_common @@ -673,8 +658,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail_version dependsOn: - merge_code - hail_ubuntu_image @@ -692,8 +675,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail/python/hailtop/hail_version - from: /repo/gear to: /io/repo/gear - from: /repo/web_common @@ -743,10 +724,8 @@ steps: time retry make wheel # Check wheel size is small enough for pypi (< 200 MiB) - HAIL_PIP_VERSION=$(cat python/hail/hail_pip_version) - WHEEL_PATH="build/deploy/dist/hail-$HAIL_PIP_VERSION-py3-none-any.whl" - du -h $WHEEL_PATH - $(python3 -c "import os; exit(1) if (os.path.getsize('$WHEEL_PATH')) > (200 * 1024 * 1024) else exit(0)") + BYTES=$(du build/deploy/dist/hail-*-py3-none-any.whl | awk '{print $1}') + (( BYTES < 209715200 )) || exit 1 inputs: - from: /repo to: /io/repo @@ -941,16 +920,12 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail/python/hailtop/hail_version - from: /repo/gear to: /io/repo/gear - from: /repo/web_common to: /io/repo/web_common - from: /jvm-entryway.jar to: /io/repo/batch/jvm-entryway/build/libs/jvm-entryway.jar - - from: /hail_version - to: /io/repo/hail_version dependsOn: - merge_code - hail_ubuntu_image @@ -1354,15 +1329,13 @@ steps: mv python/hail/docs/_build/html www/docs/0.2 mv python/hailtop/batch/docs/_build/html www/docs/batch - HAIL_CACHE_VERSION=$(cat python/hail/hail_version) + HAIL_CACHE_VERSION=$(cat env/HAIL_VERSION) find www -iname *.html -type f -exec sed -i -e "s/\.css/\.css\?v\=$HAIL_CACHE_VERSION/" {} +; tar czf /io/www.tar.gz www inputs: - - from: /repo/hail/python - to: /io/hail/python - - from: /hail_version - to: /io/repo/hail/python/hail/hail_version + - from: /repo/hail + to: /io/hail - from: /repo/website/website to: /io/website/website - from: /derived/release/hail/build/deploy/dist @@ -1594,7 +1567,7 @@ steps: # dev deploy elides the hail-is remote, add it and retrieve the tags git remote add hail-is https://github.com/hail-is/hail.git - export HAIL_PIP_VERSION=$(cat /io/hail_pip_version) + export HAIL_PIP_VERSION=$(cat hail/env/HAIL_PIP_VERSION) if git ls-remote --exit-code --tags hail-is $HAIL_PIP_VERSION then @@ -1627,8 +1600,6 @@ steps: to: /io/repo - from: /www.tar.gz to: /io/www.tar.gz - - from: /hail_pip_version - to: /io/hail_pip_version outputs: - from: /io/docs.tar.gz to: /docs.tar.gz @@ -2437,7 +2408,7 @@ steps: {% else %} HAIL_QUERY_JAR_URL={{ global.test_storage_uri }}/{{ default_ns.name }} {% endif %} - HAIL_QUERY_JAR_URL=${HAIL_QUERY_JAR_URL}/jars/$(cat /io/git_version).jar + HAIL_QUERY_JAR_URL=${HAIL_QUERY_JAR_URL}/jars/$(cat /io/REVISION).jar python3 -m hailtop.aiotools.copy 'null' '[{"from": "/io/hail.jar", "to": "'${HAIL_QUERY_JAR_URL}'"}]' secrets: @@ -2448,8 +2419,8 @@ steps: inputs: - from: /derived/release/hail/out/hail/assembly.dest/out.jar to: /io/hail.jar - - from: /git_version - to: /io/git_version + - from: /repo/hail/env/REVISION + to: /io/REVISION dependsOn: - default_ns - deploy_batch @@ -2842,7 +2813,6 @@ steps: cp -R /io/repo/hail/python ./hail/ cp /io/repo/hail/Makefile ./hail/ cp /io/repo/hail/env_var.mk ./hail/ - cp /io/repo/hail/version.mk ./hail/ cp -R /io/repo/web_common ./ cp /io/repo/hail/python/setup-hailtop.py ./hail/python/ @@ -3278,7 +3248,7 @@ steps: gcloud config set project hail-vdc gcloud config set dataproc/region us-central1 - if git ls-remote --exit-code --tags origin $(cat /io/hail_pip_version) + if git ls-remote --exit-code --tags origin $(cat hail/env/HAIL_PIP_VERSION) then echo "tag $HAIL_PIP_VERSION already exists" exit 0 @@ -3293,8 +3263,6 @@ steps: - merge_code - build_hail_jar_and_wheel inputs: - - from: /hail_pip_version - to: /io/hail_pip_version - from: /repo to: /io/repo - from: /derived/release/hail/build/deploy/dist @@ -3324,7 +3292,7 @@ steps: gcloud config set project hail-vdc gcloud config set dataproc/region us-central1 - if git ls-remote --exit-code --tags origin $(cat /io/hail_pip_version) + if git ls-remote --exit-code --tags origin $(cat hail/env/HAIL_PIP_VERSION) then echo "tag $HAIL_PIP_VERSION already exists" exit 0 @@ -3339,8 +3307,6 @@ steps: - merge_code - build_hail_jar_and_wheel inputs: - - from: /hail_pip_version - to: /io/hail_pip_version - from: /repo to: /io/repo - from: /derived/release/hail/build/deploy/dist @@ -3377,7 +3343,7 @@ steps: export GIT_ASKPASS=/io/git-askpass cd /io/repo/hail - export HAIL_PIP_VERSION=$(cat /io/hail_pip_version) + export HAIL_PIP_VERSION=$(cat env/HAIL_PIP_VERSION) if git ls-remote --exit-code --tags origin $HAIL_PIP_VERSION then @@ -3393,9 +3359,9 @@ steps: echo Setting arguments and invoking release.sh. - HAIL_VERSION=$(cat /io/hail_version) \ + HAIL_VERSION=$(cat env/HAIL_VERSION) \ WHEEL="build/deploy/dist/hail-${HAIL_PIP_VERSION}-py3-none-any.whl" \ - GIT_VERSION=$(cat /io/git_version) \ + GIT_VERSION=$(cat env/REVISION) \ REMOTE=origin \ GITHUB_OAUTH_HEADER_FILE=/io/github-oauth \ HAIL_GENETICS_HAIL_IMAGE=docker://{{ hailgenetics_hail_image.image }} \ @@ -3410,12 +3376,6 @@ steps: echo '1' > /io/release-hail-flag inputs: - - from: /hail_version - to: /io/hail_version - - from: /hail_pip_version - to: /io/hail_pip_version - - from: /git_version - to: /io/git_version - from: /repo to: /io/repo - from: /derived/release/hail/out/hail/assembly.dest/out.jar @@ -3480,8 +3440,6 @@ steps: - ci_utils_image - release inputs: - - from: /hail_pip_version - to: /io/hail_pip_version - from: /release-hail-flag to: /io/release-hail-flag - from: /repo @@ -3504,18 +3462,16 @@ steps: exit 0 fi - export HAIL_PIP_VERSION=$(cat /io/hail_pip_version) - export GITHUB_OAUTH_HEADER_FILE=/io/github-oauth - cd /io - printf 'Authorization: token ' > github-oauth - cat /hail-ci-0-1-github-oauth-token/oauth-token >>github-oauth + cat /hail-ci-0-1-github-oauth-token/oauth-token >> /io/github-oauth printf '#!/bin/bash\necho ' > git-askpass - cat /hail-ci-0-1-github-oauth-token/oauth-token >>git-askpass - chmod 755 git-askpass + cat /hail-ci-0-1-github-oauth-token/oauth-token >> /io/git-askpass + chmod 755 /io/git-askpass export GIT_ASKPASS=/io/git-askpass cd /io/repo/hail + export HAIL_PIP_VERSION=$(cat env/HAIL_PIP_VERSION) + export GITHUB_OAUTH_HEADER_FILE=/io/github-oauth bash scripts/make-terra-prs.sh - kind: runImage name: mirror_hailgenetics_images @@ -3539,12 +3495,12 @@ steps: set -x cd /io/docker/hailgenetics - export HAIL_PIP_VERSION=$(cat /io/hail_pip_version) + export HAIL_PIP_VERSION=$(cat /io/HAIL_PIP_VERSION) export DOCKER_PREFIX={{ global.docker_prefix }} bash mirror_images.sh inputs: - - from: /hail_pip_version - to: /io/hail_pip_version + - from: /repo/hail/env/HAIL_PIP_VERSION + to: /io/HAIL_PIP_VERSION - from: /repo/docker/hailgenetics/mirror_images.sh to: /io/docker/hailgenetics/mirror_images.sh - from: /repo/docker/copy_image.sh @@ -3571,8 +3527,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail/python/hailtop/hail_version - from: /repo/gear to: /io/repo/gear - from: /repo/web_common @@ -3745,8 +3699,6 @@ steps: to: /io/repo/hail/python - from: /repo/hail/scripts/benchmark_in_batch.py to: /io/repo/hail/scripts/benchmark_in_batch.py - - from: /hail_version - to: /io/hail_version - from: /release-hail-flag to: /io/release-hail-flag script: | @@ -3762,7 +3714,7 @@ steps: pushd /io/repo/hail BUCKET=hail-benchmarks-2 - HAIL_VERSION=$(cat /io/hail_version) + HAIL_VERSION=$(cat env/HAIL_VERSION) RESULTS_PREFIX="gs://${BUCKET}/{{ scope }}/${HAIL_VERSION}" export HAIL_BATCH_REGIONS={{ global.gcp_region }} diff --git a/ci/test/resources/build.yaml b/ci/test/resources/build.yaml index 9f46a939053..d25d6a9718d 100644 --- a/ci/test/resources/build.yaml +++ b/ci/test/resources/build.yaml @@ -28,16 +28,9 @@ steps: cd repo {{ code.checkout_script }} make -C hail python-version-info - git rev-parse HEAD > git_version outputs: - from: /io/repo/ to: /repo - - from: /io/repo/hail/python/hail/hail_version - to: /hail_version - - from: /io/repo/hail/python/hail/hail_pip_version - to: /hail_pip_version - - from: /io/repo/git_version - to: /git_version dependsOn: - git_make_bash_image - kind: buildImage2 @@ -142,8 +135,6 @@ steps: to: /io/repo/hail/python/MANIFEST.in - from: /repo/hail/python/hailtop to: /io/repo/hail/python/hailtop - - from: /hail_version - to: /io/repo/hail_version - from: /repo/gear to: /io/repo/gear - kind: buildImage2 diff --git a/dev-docs/releasing.md b/dev-docs/releasing.md index f26a4c8c816..450c16253dc 100644 --- a/dev-docs/releasing.md +++ b/dev-docs/releasing.md @@ -34,9 +34,6 @@ hail-ci-bpk3h bucket. The necessary files are listed under "Sources: " in the "I "release" build step. They should look something like: Sources: - gs://hail-ci-bpk3h/build/9cabeeb4ba047d1722e6f8da0383ab97/hail_version: 1 files, 21 Bytes - gs://hail-ci-bpk3h/build/9cabeeb4ba047d1722e6f8da0383ab97/hail_pip_version: 1 files, 8 Bytes - gs://hail-ci-bpk3h/build/9cabeeb4ba047d1722e6f8da0383ab97/git_version: 1 files, 41 Bytes gs://hail-ci-bpk3h/build/9cabeeb4ba047d1722e6f8da0383ab97/repo: 6272 files, 205.1 MB gs://hail-ci-bpk3h/build/9cabeeb4ba047d1722e6f8da0383ab97/azure-wheel: 1 files, 144.5 MB gs://hail-ci-bpk3h/build/9cabeeb4ba047d1722e6f8da0383ab97/www.tar.gz: 1 files, 43.5 MB @@ -47,14 +44,12 @@ Download all these files except the repo (which you do not need, because you che mkdir $BUILD_TOKEN RELEASE_ARTIFACTS_DIR=$(realpath $BUILD_TOKEN) gcloud storage cp -r \ - gs://hail-ci-bpk3h/build/$BUILD_TOKEN/hail_version \ - gs://hail-ci-bpk3h/build/$BUILD_TOKEN/hail_pip_version \ - gs://hail-ci-bpk3h/build/$BUILD_TOKEN/git_version \ + gs://hail-ci-bpk3h/build/$BUILD_TOKEN/repo/hail/env \ gs://hail-ci-bpk3h/build/$BUILD_TOKEN/azure-wheel \ gs://hail-ci-bpk3h/build/$BUILD_TOKEN/www.tar.gz \ - $BUILD_TOKEN + $RELEASE_ARTIFACTS_DIR -Note that the `-r` is necessary because some of these things like `azure-wheel` are folders. +Note that the `-r` is necessary because some of these things like `env` and `azure-wheel` are folders. Next we need to authenticate with DockerHub. Download the secret and authenticate skopeo with it. `download-secret` is a function stored in `devbin/functions.sh`. @@ -88,9 +83,9 @@ Ensure you have returned to the `hail` sub-folder of the Hail git repository. Now we can construct a `release.sh` invocation. Find the invocation in the "command" part of the "Job Specification" table. It should look like: - bash scripts/release.sh $(cat /io/hail_pip_version) \ - $(cat /io/hail_version) \ - $(cat /io/git_version) \ + bash scripts/release.sh $(cat $RELEASE_ARTIFACTS_DIR/HAIL_PIP_VERSION) \ + $(cat $RELEASE_ARTIFACTS_DIR/HAIL_VERSION) \ + $(cat $RELEASE_ARTIFACTS_DIR/REVISION) \ origin \ /io/repo/hail/build/deploy/dist/hail-*-py3-none-any.whl \ /io/github-oauth \ @@ -112,9 +107,9 @@ We need to make two replacements: It should look something like this: bash scripts/release.sh \ - $(cat $RELEASE_ARTIFACTS_DIR/hail_pip_version) \ - $(cat $RELEASE_ARTIFACTS_DIR/hail_version) \ - $(cat $RELEASE_ARTIFACTS_DIR/git_version) \ + $(cat $RELEASE_ARTIFACTS_DIR/HAIL_PIP_VERSION) \ + $(cat $RELEASE_ARTIFACTS_DIR/HAIL_VERSION) \ + $(cat $RELEASE_ARTIFACTS_DIR/REVISION) \ origin \ /PATH/TO/DOWNLOADED/HAIL-COMMON/WHEEL/hail-0.2.XXX-py3-none-any.whl \ $RELEASE_ARTIFACTS_DIR/github-oauth \ diff --git a/dev-docs/services/tls-cookbook.md b/dev-docs/services/tls-cookbook.md index 0b6070bf065..f14943d2e08 100644 --- a/dev-docs/services/tls-cookbook.md +++ b/dev-docs/services/tls-cookbook.md @@ -122,7 +122,7 @@ kubectl create secret generic \ 3. Update all the service certificates: ``` -make -C $HAIL/hail python/hailtop/hail_version +make -C $HAIL/hail python/hailtop/version.py PYTHONPATH=$HAIL/hail/python \ python3 $HAIL/tls/create_certs.py \ diff --git a/docker/hailgenetics/Makefile b/docker/hailgenetics/Makefile index f839e2af3be..97cff8c8d3d 100644 --- a/docker/hailgenetics/Makefile +++ b/docker/hailgenetics/Makefile @@ -7,5 +7,5 @@ publish-python-dill: mirror-dockerhub-images: DOCKER_PREFIX=$(DOCKER_PREFIX) \ - HAIL_PIP_VERSION=$(shell cat ../../hail/python/hail/hail_pip_version) \ + HAIL_PIP_VERSION=$(shell cat ../../hail/env/HAIL_PIP_VERSION) \ ./mirror_images.sh diff --git a/docker/hailgenetics/hailtop/Dockerfile b/docker/hailgenetics/hailtop/Dockerfile index 19156bd72cf..e2f82873776 100644 --- a/docker/hailgenetics/hailtop/Dockerfile +++ b/docker/hailgenetics/hailtop/Dockerfile @@ -6,5 +6,4 @@ RUN hail-pip-install -r requirements.txt COPY hail/python/setup-hailtop.py /hailtop/setup.py COPY hail/python/MANIFEST.in /hailtop/MANIFEST.in COPY hail/python/hailtop /hailtop/hailtop/ -COPY /hail_version /hailtop/hailtop/hail_version RUN hail-pip-install /hailtop diff --git a/hail/.gitignore b/hail/.gitignore index 506cc43dd2c..79152b40574 100644 --- a/hail/.gitignore +++ b/hail/.gitignore @@ -3,13 +3,10 @@ python/README.md python/dist python/hail.egg-info python/hail/backend/hail-all-spark.jar -python/hail/hail_revision -python/hail/hail_pip_version +python/hail/version.py python/hail/docs/change_log.rst python/hail/docs/_build/* python/hail/docs/_static/hail_version.js -python/hailtop/dist -python/hailtop/hailctl/deploy.yaml python/hail/docs/genetics/hail.genetics.AlleleType.rst python/hail/docs/genetics/hail.genetics.Call.rst python/hail/docs/genetics/hail.genetics.Locus.rst @@ -67,6 +64,9 @@ python/hail/docs/vds/hail.vds.local_to_global.rst python/hail/docs/vds/hail.vds.merge_reference_blocks.rst python/hail/docs/vds/hail.vds.truncate_reference_blocks.rst python/hail/docs/tutorials/iframe_figures/ +python/hailtop/hailctl/deploy.yaml +python/hailtop/dist +python/hailtop/version.py src/main/c/.cxx.vsn src/main/c/headers src/main/c/lib diff --git a/hail/Makefile b/hail/Makefile index 17a616d9e22..1634481c6eb 100644 --- a/hail/Makefile +++ b/hail/Makefile @@ -1,7 +1,25 @@ HAIL_HAIL_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) +REVISION := $(shell git rev-parse HEAD) +ifndef REVISION +$(error "git rev-parse HEAD" failed to produce output) +endif + +SHORT_REVISION := $(shell git rev-parse --short=12 HEAD) +ifndef SHORT_REVISION +$(error "git rev-parse --short=12 HEAD" failed to produce output) +endif + +HAIL_MAJOR_MINOR_VERSION := 0.2 +HAIL_PATCH_VERSION := 133 +HAIL_PIP_VERSION := $(HAIL_MAJOR_MINOR_VERSION).$(HAIL_PATCH_VERSION) +HAIL_VERSION := $(HAIL_PIP_VERSION)-$(SHORT_REVISION) + +SCALA_VERSION ?= 2.12.18 +SPARK_VERSION ?= 3.5.0 + + include env_var.mk -include version.mk .DEFAULT_GOAL := shadowJar MAKEFLAGS += --no-builtin-rules @@ -9,11 +27,10 @@ MAKEFLAGS += --no-builtin-rules $(eval $(call ENV_VAR,REVISION)) $(eval $(call ENV_VAR,SHORT_REVISION)) -$(eval $(call ENV_VAR,BRANCH)) -$(eval $(call ENV_VAR,SPARK_VERSION)) $(eval $(call ENV_VAR,HAIL_PIP_VERSION)) -$(eval $(call ENV_VAR,HAIL_DEBUG_MODE)) -$(eval $(call ENV_VAR,ELASTIC_MAJOR_VERSION)) +$(eval $(call ENV_VAR,HAIL_VERSION)) + +$(eval $(call ENV_VAR,SPARK_VERSION)) HAIL_PYTHON3 ?= python3 PIP ?= $(HAIL_PYTHON3) -m pip @@ -30,11 +47,6 @@ PYTEST_INTER_CLOUD_TARGET ?= test/hailtop/inter_cloud PY_FILES := $(shell git ls-files python) INIT_SCRIPTS := python/hailtop/hailctl/deploy.yaml -PYTHON_VERSION_INFO := python/hail/hail_revision -PYTHON_VERSION_INFO += python/hail/hail_version -PYTHON_VERSION_INFO += python/hail/hail_pip_version -PYTHON_VERSION_INFO += python/hailtop/hail_version -PYTHON_VERSION_INFO += python/hail/docs/_static/hail_version.js PYTHON_JAR := python/hail/backend/hail-all-spark.jar FAST_PYTHON_JAR := python/hail/backend/hail.jar @@ -125,24 +137,31 @@ fs-jvm-test: upload-remote-test-resources .PHONY: jars jars: shadowJar shadowTestJar -.PHONY: python-version-info -python-version-info: $(PYTHON_VERSION_INFO) - -python/hail/hail_revision: env/REVISION - echo $(REVISION) > $@ - -python/hail/hail_version: env/SHORT_REVISION env/HAIL_PIP_VERSION - echo $(HAIL_VERSION) > $@ +PYTHON_VERSION_INFO := \ + python/hail/version.py \ + python/hail/docs/_static/hail_version.js \ + python/hailtop/version.py -python/hail/hail_pip_version: env/HAIL_PIP_VERSION - echo $(HAIL_PIP_VERSION) > $@ +$(PYTHON_VERSION_INFO): env/HAIL_VERSION env/HAIL_PIP_VERSION env/REVISION env/SHORT_REVISION -python/hail/docs/_static/hail_version.js: python/hail/hail_version python/hail/hail_pip_version - printf 'hail_version="$(shell cat python/hail/hail_version)";' > $@ - printf 'hail_pip_version="$(shell cat python/hail/hail_pip_version)"' >> $@ +.PHONY: python-version-info +python-version-info: $(PYTHON_VERSION_INFO) -python/hailtop/hail_version: python/hail/hail_version - cp -f $< $@ +.ONESHELL: $(PYTHON_VERSION_INFO) +python/hail/version.py python/hailtop/version.py: + cat <<- EOF > $@ + # Generated by make on $(shell date). + __pip_version__ = "$(HAIL_PIP_VERSION)" + __revision__ = "$(REVISION)" + __version__ = "$(HAIL_VERSION)" + EOF + +python/hail/docs/_static/hail_version.js: + cat <<- EOF > $@ + # Generated by make on $(shell date). + hail_pip_version = "$(HAIL_PIP_VERSION)"; + hail_version = "$(HAIL_VERSION)"; + EOF python/README.md: ../README.md cp ../README.md python/ diff --git a/hail/python/MANIFEST.in b/hail/python/MANIFEST.in index aa401baf489..468c769a59f 100644 --- a/hail/python/MANIFEST.in +++ b/hail/python/MANIFEST.in @@ -1,8 +1,5 @@ -include hail/hail_pip_version -include hail/hail_version include hail/experimental/datasets.json include hail/backend/hail-all-spark.jar -include hailtop/hail_version include hailtop/hailctl/deploy.yaml include hailtop/py.typed include requirements.txt diff --git a/hail/python/hail/__init__.py b/hail/python/hail/__init__.py index 7b6fffba8b5..a6a2de9acc9 100644 --- a/hail/python/hail/__init__.py +++ b/hail/python/hail/__init__.py @@ -1,7 +1,6 @@ import importlib.resources from pathlib import Path from sys import version_info -from typing import Optional if version_info < (3, 9): raise EnvironmentError(f'Hail requires Python 3.9 or later, found {version_info.major}.{version_info.minor}') @@ -90,7 +89,6 @@ def __resource_str(name: str) -> str: spark_context, stop, tmp_dir, - version, ) from .expr import * # noqa: F403 from .expr import aggregators @@ -98,11 +96,20 @@ def __resource_str(name: str) -> str: from .matrixtable import GroupedMatrixTable, MatrixTable from .methods import * # noqa: F403 from .table import GroupedTable, Table, asc, desc +from .version import __pip_version__, __revision__, __version__ agg = aggregators scan = aggregators.aggregators.ScanFunctions({name: getattr(agg, name) for name in agg.__all__}) + +def version() -> str: + return __version__ + + __all__ = [ + '__pip_version__', + '__revision__', + '__version__', 'init', 'init_local', 'init_batch', @@ -172,10 +179,6 @@ def __resource_str(name: str) -> str: ir.register_functions() ir.register_aggregators() -__pip_version__ = __resource_str('hail_pip_version').strip() -__version__: Optional[str] = None # set by hail.version() -__revision__: Optional[str] = None # set by hail.revision() - import warnings warnings.filterwarnings('once', append=True) diff --git a/hail/python/hail/backend/py4j_backend.py b/hail/python/hail/backend/py4j_backend.py index b9f986e5834..03bc9be5dfe 100644 --- a/hail/python/hail/backend/py4j_backend.py +++ b/hail/python/hail/backend/py4j_backend.py @@ -15,6 +15,7 @@ from hail.expr import construct_expr from hail.ir import JavaIR from hail.utils.java import Env, FatalError, scala_package_object +from hail.version import __version__ from ..hail_logging import Logger from .backend import ActionTag, Backend, fatal_error_from_java_error_triplet @@ -197,15 +198,13 @@ def decode_bytearray(encoded): # This has to go after creating the SparkSession. Unclear why. # Maybe it does its own patch? install_exception_handler() - from hail.context import version - py_version = version() jar_version = self._jhc.version() - if jar_version != py_version: + if jar_version != __version__: raise RuntimeError( f"Hail version mismatch between JAR and Python library\n" f" JAR: {jar_version}\n" - f" Python: {py_version}" + f" Python: {__version__}" ) def jvm(self): diff --git a/hail/python/hail/backend/service_backend.py b/hail/python/hail/backend/service_backend.py index 5a5a2457173..953ce749b05 100644 --- a/hail/python/hail/backend/service_backend.py +++ b/hail/python/hail/backend/service_backend.py @@ -10,13 +10,14 @@ import orjson import hailtop.aiotools.fs as afs -from hail.context import TemporaryDirectory, TemporaryFilename, revision, tmp_dir, version +from hail.context import TemporaryDirectory, TemporaryFilename, tmp_dir from hail.experimental import read_expression, write_expression from hail.expr.expressions.base_expression import Expression from hail.expr.types import HailType from hail.ir import finalize_randomness from hail.ir.renderer import CSERenderer from hail.utils import FatalError +from hail.version import __revision__, __version__ from hailtop import yamlx from hailtop.aiocloud.aiogoogle import GCSRequesterPaysConfiguration, get_gcs_requester_pays_configuration from hailtop.aiotools.fs.exceptions import UnexpectedEOFError @@ -201,7 +202,7 @@ async def create( name_prefix = configuration_of(ConfigVariable.QUERY_NAME_PREFIX, name_prefix, '') batch_attributes: Dict[str, str] = { - 'hail-version': version(), + 'hail-version': __version__, } if name_prefix: batch_attributes['name'] = name_prefix @@ -325,7 +326,7 @@ def fs(self) -> FS: @property def jar_spec(self) -> dict: - return {'type': 'git_revision', 'value': revision()} + return {'type': 'git_revision', 'value': __revision__} @property def logger(self): diff --git a/hail/python/hail/context.py b/hail/python/hail/context.py index 77493ecd083..6c96599a24b 100644 --- a/hail/python/hail/context.py +++ b/hail/python/hail/context.py @@ -15,12 +15,12 @@ from hail.typecheck import dictof, enumeration, nullable, oneof, sequenceof, sized_tupleof, typecheck, typecheck_method from hail.utils import get_env_or_default from hail.utils.java import BackendType, Env, choose_backend, warning +from hail.version import __version__ from hailtop.aiocloud.aiogoogle import GCSRequesterPaysConfiguration, get_gcs_requester_pays_configuration from hailtop.fs.fs import FS from hailtop.hail_event_loop import hail_event_loop from hailtop.utils import secret_alnum_string -from . import __resource_str from .backend.backend import local_jar_information from .builtin_references import BUILTIN_REFERENCES @@ -43,11 +43,10 @@ def _get_local_tmpdir(local_tmpdir): def _get_log(log): if log is None: - py_version = version() log_dir = os.environ.get('HAIL_LOG_DIR') if log_dir is None: log_dir = os.getcwd() - log = hail.utils.timestamp_path(os.path.join(log_dir, 'hail'), suffix=f'-{py_version}.log') + log = hail.utils.timestamp_path(os.path.join(log_dir, 'hail'), suffix=f'-{__version__}.log') return log @@ -104,16 +103,15 @@ def __init__(self, log, quiet, append, tmpdir, local_tmpdir, global_seed, backen self._default_ref: Optional[ReferenceGenome] = None if not quiet: - py_version = version() sys.stderr.write( 'Welcome to\n' ' __ __ <>__\n' ' / /_/ /__ __/ /\n' ' / __ / _ `/ / /\n' - ' /_/ /_/\\_,_/_/_/ version {}\n'.format(py_version) + ' /_/ /_/\\_,_/_/_/ version {}\n'.format(__version__) ) - if py_version.startswith('devel'): + if __version__.startswith('devel'): sys.stderr.write( 'NOTE: This is a beta version. Interfaces may change\n' ' during the beta period. We recommend pulling\n' @@ -633,35 +631,8 @@ def init_local( connect_logger(backend._utils_package_object, 'localhost', 12888) -def version() -> str: - """Get the installed Hail version. - - Returns - ------- - str - """ - if hail.__version__ is None: - hail.__version__ = __resource_str('hail_version').strip() - - return hail.__version__ - - -def revision() -> str: - """Get the installed Hail git revision. - - Returns - ------- - str - """ - if hail.__revision__ is None: - hail.__revision__ = __resource_str('hail_revision').strip() - - return hail.__revision__ - - def _hail_cite_url(): - v = version() - [tag, sha_prefix] = v.split("-") + [tag, sha_prefix] = __version__.split("-") if not local_jar_information().development_mode: # pip installed return f"https://github.com/hail-is/hail/releases/tag/{tag}" @@ -688,7 +659,7 @@ def citation(*, bibtex=False): f" howpublished = {{\\url{{{_hail_cite_url()}}}}}" f"}}" ) - return f"Hail Team. Hail {version()}. {_hail_cite_url()}." + return f"Hail Team. Hail {__version__}. {_hail_cite_url()}." def cite_hail(): @@ -965,4 +936,4 @@ def debug_info(): spark_conf = None if isinstance(Env.backend(), SparkBackend): spark_conf = spark_context()._conf.getAll() - return {'spark_conf': spark_conf, 'local_jar_information': local_jar_information(), 'version': version()} + return {'spark_conf': spark_conf, 'local_jar_information': local_jar_information(), 'version': __version__} diff --git a/hail/python/hail/docs/conf.py b/hail/python/hail/docs/conf.py index a7bfbf2c136..5760713c0bc 100644 --- a/hail/python/hail/docs/conf.py +++ b/hail/python/hail/docs/conf.py @@ -19,8 +19,17 @@ import datetime import os -import re import sys +from importlib.util import module_from_spec, spec_from_file_location + + +def __load_module(name, path): + spec = spec_from_file_location(name, path) + mod = module_from_spec(spec) + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + sys.path.insert(0, os.path.abspath('./_ext')) @@ -106,9 +115,9 @@ # built documents. # # The full version, including alpha/beta/rc tags. -with open('../hail_version') as f: - release = f.read().strip() - version = re.match('[0-9]+.[0-9]+', release)[0] +mod = __load_module('hail.version', '../version.py') +release = mod.__version__ +version = mod.__pip_version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/hail/python/hail/methods/qc.py b/hail/python/hail/methods/qc.py index 2efd7b37944..338937c3fba 100644 --- a/hail/python/hail/methods/qc.py +++ b/hail/python/hail/methods/qc.py @@ -20,7 +20,7 @@ from hail.utils import FatalError from hail.utils.java import Env, info, warning from hail.utils.misc import divide_null, guess_cloud_spark_provider, new_temp_file -from hailtop import pip_version, yamlx +from hailtop import __pip_version__, yamlx from hailtop.config import get_deploy_config from hailtop.utils import async_to_blocking @@ -36,10 +36,10 @@ HAIL_GENETICS_VEP_GRCH37_85_IMAGE = os.environ.get( - 'HAIL_GENETICS_VEP_GRCH37_85_IMAGE', f'hailgenetics/vep-grch37-85:{pip_version()}' + 'HAIL_GENETICS_VEP_GRCH37_85_IMAGE', f'hailgenetics/vep-grch37-85:{__pip_version__}' ) HAIL_GENETICS_VEP_GRCH38_95_IMAGE = os.environ.get( - 'HAIL_GENETICS_VEP_GRCH38_95_IMAGE', f'hailgenetics/vep-grch38-95:{pip_version()}' + 'HAIL_GENETICS_VEP_GRCH38_95_IMAGE', f'hailgenetics/vep-grch38-95:{__pip_version__}' ) diff --git a/hail/python/hailtop/__init__.py b/hail/python/hailtop/__init__.py index 9ee2e7a1b54..32f5c569bbb 100644 --- a/hail/python/hailtop/__init__.py +++ b/hail/python/hailtop/__init__.py @@ -1,19 +1,4 @@ -_VERSION = None - - -def version() -> str: - global _VERSION - if _VERSION is None: - import importlib.resources as r # pylint: disable=import-outside-toplevel - - _VERSION = (r.files(__name__) / 'hail_version').read_text('utf-8').strip() - - return _VERSION - - -def pip_version() -> str: - return version().split('-')[0] - +from hailtop.version import __pip_version__, __version__ IS_NOTEBOOK = None @@ -28,3 +13,10 @@ def is_notebook() -> bool: except (NameError, ModuleNotFoundError): IS_NOTEBOOK = False return IS_NOTEBOOK + + +__all__ = [ + '__pip_version__', + '__version__', + 'is_notebook', +] diff --git a/hail/python/hailtop/batch/backend.py b/hail/python/hailtop/batch/backend.py index ddf65f58692..8686fdb93b1 100644 --- a/hail/python/hailtop/batch/backend.py +++ b/hail/python/hailtop/batch/backend.py @@ -16,7 +16,7 @@ from rich.progress import track import hailtop.batch_client.client as bc -from hailtop import pip_version +from hailtop import __pip_version__ from hailtop.aiocloud.aiogoogle import GCSRequesterPaysConfiguration from hailtop.aiotools.router_fs import RouterAsyncFS from hailtop.aiotools.validators import validate_file @@ -34,7 +34,7 @@ from .globals import DEFAULT_SHELL from .job import PythonJob -HAIL_GENETICS_HAILTOP_IMAGE = os.environ.get('HAIL_GENETICS_HAILTOP_IMAGE', f'hailgenetics/hailtop:{pip_version()}') +HAIL_GENETICS_HAILTOP_IMAGE = os.environ.get('HAIL_GENETICS_HAILTOP_IMAGE', f'hailgenetics/hailtop:{__pip_version__}') HAIL_GENETICS_HAIL_IMAGE = ( os.environ.get('HAIL_GENETICS_HAIL_IMAGE') or hailgenetics_hail_image_for_current_python_version() ) diff --git a/hail/python/hailtop/batch/hail_genetics_images.py b/hail/python/hailtop/batch/hail_genetics_images.py index c5f8b7882aa..97249edf615 100644 --- a/hail/python/hailtop/batch/hail_genetics_images.py +++ b/hail/python/hailtop/batch/hail_genetics_images.py @@ -1,6 +1,6 @@ import sys -from hailtop import pip_version +from hailtop import __pip_version__ HAIL_GENETICS = 'hailgenetics/' HAIL_GENETICS_IMAGES = [ @@ -16,5 +16,5 @@ def hailgenetics_hail_image_for_current_python_version(): f'using a Python version other than 3.9, 3.10, or 3.11 (you are using {version})' ) if version.minor == 9: - return f'hailgenetics/hail:{pip_version()}' - return f'hailgenetics/hail:{pip_version()}-py{version.major}.{version.minor}' + return f'hailgenetics/hail:{__pip_version__}' + return f'hailgenetics/hail:{__pip_version__}-py{version.major}.{version.minor}' diff --git a/hail/python/hailtop/hailctl/__main__.py b/hail/python/hailtop/hailctl/__main__.py index 98e80043868..19695b52557 100644 --- a/hail/python/hailtop/hailctl/__main__.py +++ b/hail/python/hailtop/hailctl/__main__.py @@ -32,7 +32,7 @@ def version(): """Print version information and exit.""" import hailtop # pylint: disable=import-outside-toplevel - print(hailtop.version()) + print(hailtop.__version__) @app.command(context_settings={"allow_extra_args": True, "ignore_unknown_options": True}) diff --git a/hail/python/hailtop/hailctl/batch/submit.py b/hail/python/hailtop/hailctl/batch/submit.py index 196b844ab23..234ec62ac52 100644 --- a/hail/python/hailtop/hailctl/batch/submit.py +++ b/hail/python/hailtop/hailctl/batch/submit.py @@ -5,7 +5,7 @@ import orjson -from hailtop import pip_version +from hailtop import __pip_version__ FILE_REGEX = re.compile(r'(?P[^:]+)(:(?P.+))?') @@ -62,7 +62,7 @@ def file_input_to_src_dest(file: str) -> Tuple[str, str, str]: backend = hb.ServiceBackend() b = hb.Batch(name=name, backend=backend) j = b.new_bash_job() - j.image(image_name or os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{pip_version()}')) + j.image(image_name or os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{__pip_version__}')) local_files_to_cloud_files = [] diff --git a/hail/python/hailtop/hailctl/hdinsight/cli.py b/hail/python/hailtop/hailctl/hdinsight/cli.py index 3ad7ae6f3e9..255c9019d81 100644 --- a/hail/python/hailtop/hailctl/hdinsight/cli.py +++ b/hail/python/hailtop/hailctl/hdinsight/cli.py @@ -79,12 +79,10 @@ def start( """ Start an HDInsight cluster configured for Hail. """ - from ... import pip_version # pylint: disable=import-outside-toplevel - - hail_version = pip_version() + from ... import __pip_version__ # pylint: disable=import-outside-toplevel def default_artifact(filename: str) -> str: - return f'https://raw.githubusercontent.com/hail-is/hail/{hail_version}/hail/python/hailtop/hailctl/hdinsight/resources/{filename}' + return f'https://raw.githubusercontent.com/hail-is/hail/{__pip_version__}/hail/python/hailtop/hailctl/hdinsight/resources/{filename}' hdinsight_start( cluster_name, @@ -97,7 +95,7 @@ def default_artifact(filename: str) -> str: install_hail_uri or default_artifact('install-hail.sh'), install_native_deps_uri or default_artifact('install-native-deps.sh'), wheel_uri - or f'https://storage.googleapis.com/hail-common/azure-hdinsight-wheels/hail-{hail_version}-py3-none-any.whl', + or f'https://storage.googleapis.com/hail-common/azure-hdinsight-wheels/hail-{__pip_version__}-py3-none-any.whl', vep, vep_loftee_uri, vep_homo_sapiens_uri, diff --git a/hail/python/hailtop/hailctl/hdinsight/start.py b/hail/python/hailtop/hailctl/hdinsight/start.py index f2e2fa9ec83..335160144b0 100644 --- a/hail/python/hailtop/hailctl/hdinsight/start.py +++ b/hail/python/hailtop/hailctl/hdinsight/start.py @@ -39,7 +39,7 @@ def start( import requests # pylint: disable=import-outside-toplevel import requests.auth # pylint: disable=import-outside-toplevel - from ... import pip_version # pylint: disable=import-outside-toplevel + from ... import __pip_version__ # pylint: disable=import-outside-toplevel from ...utils import secret_alnum_string # pylint: disable=import-outside-toplevel print(f'Starting the cluster {cluster_name}') @@ -138,7 +138,7 @@ def start( sys.exit(1) if vep_config_uri is None: - vep_config_uri = f'https://raw.githubusercontent.com/hail-is/hail/{pip_version()}/hail/python/hailtop/hailctl/hdinsight/resources/vep-{vep}.json' + vep_config_uri = f'https://raw.githubusercontent.com/hail-is/hail/{__pip_version__}/hail/python/hailtop/hailctl/hdinsight/resources/vep-{vep}.json' if vep_loftee_uri is None or vep_homo_sapiens_uri is None: raise ValueError("Both `vep_loftee_uri` and `vep_homo_sapiens_uri` must be specified if `vep` is specified") diff --git a/hail/python/setup-hailtop.py b/hail/python/setup-hailtop.py index 3a59a9b4f29..f98cd35a882 100644 --- a/hail/python/setup-hailtop.py +++ b/hail/python/setup-hailtop.py @@ -1,10 +1,22 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 + +import sys +from importlib.util import module_from_spec, spec_from_file_location from setuptools import find_packages, setup + +def load_module(name, path): + spec = spec_from_file_location(name, path) + mod = module_from_spec(spec) + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + + setup( name='hailtop', - version="0.0.1", + version=load_module('version', 'hailtop/version.py').__pip_version__, author="Hail Team", author_email="hail@broadinstitute.org", description="Top level Hail module.", @@ -15,7 +27,7 @@ }, packages=find_packages('.'), package_dir={'hailtop': 'hailtop'}, - package_data={"hailtop": ["py.typed", "hail_version"], 'hailtop.hailctl': ['hail_version', 'deploy.yaml']}, + package_data={"hailtop": ["py.typed"], 'hailtop.hailctl': ['deploy.yaml']}, classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", diff --git a/hail/python/setup.py b/hail/python/setup.py index ded40cb8d31..1d43379c3ac 100755 --- a/hail/python/setup.py +++ b/hail/python/setup.py @@ -1,15 +1,23 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import os +import sys +from importlib.util import module_from_spec, spec_from_file_location from setuptools import find_packages, setup -with open('hail/hail_pip_version') as f: - hail_pip_version = f.read().strip() - with open("README.md", "r") as fh: long_description = fh.read() + +def load_module(name, path): + spec = spec_from_file_location(name, path) + mod = module_from_spec(spec) + sys.modules[name] = mod + spec.loader.exec_module(mod) + return mod + + dependencies = [] with open('requirements.txt', 'r') as f: for line in f: @@ -46,7 +54,7 @@ def add_dependencies(fname): setup( name="hail", - version=hail_pip_version, + version=load_module('version', 'hail/version.py').__pip_version__, author="Hail Team", author_email="hail@broadinstitute.org", description="Scalable library for exploring and analyzing genomic data.", @@ -61,10 +69,10 @@ def add_dependencies(fname): packages=find_packages('.'), package_dir={'hail': 'hail', 'hailtop': 'hailtop'}, package_data={ - 'hail': ['hail_pip_version', 'hail_version', 'hail_revision', 'experimental/datasets.json'], + 'hail': ['experimental/datasets.json'], 'hail.backend': ['hail-all-spark.jar'], - 'hailtop': ['hail_version', 'py.typed'], - 'hailtop.hailctl': ['hail_version', 'deploy.yaml'], + 'hailtop': ['py.typed'], + 'hailtop.hailctl': ['deploy.yaml'], }, classifiers=[ "Programming Language :: Python :: 3", diff --git a/hail/python/test/hailtop/batch/test_batch_local_backend.py b/hail/python/test/hailtop/batch/test_batch_local_backend.py index 15e256a6dc8..e96a2cb46b0 100644 --- a/hail/python/test/hailtop/batch/test_batch_local_backend.py +++ b/hail/python/test/hailtop/batch/test_batch_local_backend.py @@ -7,14 +7,14 @@ import pytest -from hailtop import pip_version +from hailtop import __pip_version__ from hailtop.batch import Batch, LocalBackend, ResourceGroup from hailtop.batch.resource import JobResourceFile from hailtop.batch.utils import concatenate DOCKER_ROOT_IMAGE = os.environ.get('DOCKER_ROOT_IMAGE', 'ubuntu:22.04') PYTHON_DILL_IMAGE = 'hailgenetics/python-dill:3.9-slim' -HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{pip_version()}') +HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{__pip_version__}') REQUESTER_PAYS_PROJECT = os.environ.get('GCS_REQUESTER_PAYS_PROJECT') diff --git a/hail/python/test/hailtop/batch/utils.py b/hail/python/test/hailtop/batch/utils.py index 4915962f9c5..928e04f9c17 100644 --- a/hail/python/test/hailtop/batch/utils.py +++ b/hail/python/test/hailtop/batch/utils.py @@ -1,12 +1,12 @@ import inspect import os -from hailtop import pip_version +from hailtop import __pip_version__ from hailtop.batch import Batch DOCKER_ROOT_IMAGE = os.environ.get('DOCKER_ROOT_IMAGE', 'ubuntu:22.04') PYTHON_DILL_IMAGE = 'hailgenetics/python-dill:3.9-slim' -HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{pip_version()}') +HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{__pip_version__}') REQUESTER_PAYS_PROJECT = os.environ.get('GCS_REQUESTER_PAYS_PROJECT') diff --git a/hail/version.mk b/hail/version.mk deleted file mode 100644 index 9754b7a2b4c..00000000000 --- a/hail/version.mk +++ /dev/null @@ -1,22 +0,0 @@ -REVISION := $(shell git rev-parse HEAD) -ifndef REVISION -$(error "git rev-parse HEAD" failed to produce output) -endif - -SHORT_REVISION := $(shell git rev-parse --short=12 HEAD) -ifndef SHORT_REVISION -$(error "git rev-parse --short=12 HEAD" failed to produce output) -endif - -BRANCH := $(shell git rev-parse --abbrev-ref HEAD) -ifndef BRANCH -$(error "git rev-parse --abbrev-ref HEAD" failed to produce output) -endif - -SCALA_VERSION ?= 2.12.18 -SPARK_VERSION ?= 3.5.0 -HAIL_MAJOR_MINOR_VERSION := 0.2 -HAIL_PATCH_VERSION := 133 -HAIL_PIP_VERSION := $(HAIL_MAJOR_MINOR_VERSION).$(HAIL_PATCH_VERSION) -HAIL_VERSION := $(HAIL_PIP_VERSION)-$(SHORT_REVISION) -ELASTIC_MAJOR_VERSION ?= 7 diff --git a/infra/bootstrap_utils.sh b/infra/bootstrap_utils.sh index 14e2325d12e..c4808ac5f2c 100644 --- a/infra/bootstrap_utils.sh +++ b/infra/bootstrap_utils.sh @@ -12,7 +12,7 @@ source $HAIL/devbin/functions.sh copy_images() { make -C $HAIL/docker/third-party copy - make -C $HAIL/hail python/hail/hail_pip_version + make -C $HAIL/hail python/hail/version.py make -C $HAIL/docker/hailgenetics mirror-dockerhub-images } @@ -49,7 +49,7 @@ generate_ssl_certs() { deploy_unmanaged() { export NAMESPACE=default - make -C $HAIL/hail python/hailtop/hail_version + make -C $HAIL/hail python/hailtop/version.py copy_images generate_ssl_certs