Skip to content

Commit c041c11

Browse files
ArrayRecord Teamcopybara-github
ArrayRecord Team
authored and
copybara-github
committed
array_record pip package
PiperOrigin-RevId: 490568092
1 parent 43938f8 commit c041c11

File tree

10 files changed

+203
-0
lines changed

10 files changed

+203
-0
lines changed

‎.bazelrc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
build -c opt
22
build --cxxopt=-std=c++17
33
build --host_cxxopt=-std=c++17
4+
build --experimental_repo_remote_exec
45

56
# TODO(fchern): Use non-hardcode path.
67
build --action_env=PYTHON_BIN_PATH="/usr/bin/python3"
8+
build --action_env=PYTHON_LIB_PATH="/usr/lib/python3"
79
build --repo_env=PYTHON_BIN_PATH="/usr/bin/python3"
810
build --python_path="/usr/bin/python3"

‎WORKSPACE‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ http_archive(
5252
name = "pybind11",
5353
build_file = "@pybind11_bazel//:pybind11.BUILD",
5454
strip_prefix = "pybind11-2.9.2",
55+
sha256 = "d1646e6f70d8a3acb2ddd85ce1ed543b5dd579c68b8fb8e9638282af20edead8",
5556
urls = ["https://github.com/pybind/pybind11/archive/refs/tags/v2.9.2.zip"],
5657
)
5758
load("@pybind11_bazel//:python_configure.bzl", "python_configure")
@@ -133,3 +134,16 @@ http_archive(
133134
strip_prefix = "highwayhash-276dd7b4b6d330e4734b756e97ccfb1b69cc2e12",
134135
urls = ["https://github.com/google/highwayhash/archive/276dd7b4b6d330e4734b756e97ccfb1b69cc2e12.zip"], # 2019-02-22
135136
)
137+
138+
139+
http_archive(
140+
name = "org_tensorflow",
141+
strip_prefix = "tensorflow-2.10.0",
142+
sha256 = "d79a95ede8305f14a10dd0409a1e5a228849039c19ccfb90dfe8367295fd04e0",
143+
urls = ["https://github.com/tensorflow/tensorflow/archive/v2.10.0.zip"],
144+
)
145+
146+
# This import (along with the org_tensorflow archive) is necessary to provide the devtoolset-9 toolchain
147+
load("@org_tensorflow//tensorflow/tools/toolchains/remote_config:configs.bzl", "initialize_rbe_configs") # buildifier: disable=load-on-top
148+
149+
initialize_rbe_configs()

‎__init__.py‎

Whitespace-only changes.

‎oss/README.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Steps to build a new array_record pip package
2+
3+
1. Update the version number in setup.py
4+
5+
2. In workspace, run
6+
```
7+
./array_record/oss/runner.sh
8+
```
9+
10+
3. Wheels are in /tmp/array_record, upload to PyPI

‎oss/build.Dockerfile‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Constructs the environment within which we will build the array_record pip wheels.
2+
#
3+
# From /tmp/array_record,
4+
# ❯ DOCKER_BUILDKIT=1 docker build -t array_record:latest - < oss/build.Dockerfile
5+
# ❯ docker run --rm -it -v /tmp/array_recor:/tmp/array_reco \
6+
# array_record:latest bash
7+
8+
ARG base_image="tensorflow/build:2.10-python3.9"
9+
FROM $base_image
10+
LABEL maintainer="Array_record team <[email protected]>"
11+
12+
ENV DEBIAN_FRONTEND=noninteractive
13+
14+
# Install supplementary Python interpreters
15+
RUN mkdir /tmp/python
16+
RUN --mount=type=cache,target=/var/cache/apt \
17+
apt update && \
18+
apt install -yqq \
19+
apt-utils \
20+
build-essential \
21+
checkinstall \
22+
libffi-dev \
23+
neovim
24+
25+
# 3.9 is the built-in interpreter version in this image.
26+
RUN for v in 3.8.15; do \
27+
wget "https://www.python.org/ftp/python/$v/Python-${v}.tar.xz" && \
28+
rm -rf "/tmp/python${v}" && mkdir -p "/tmp/python${v}" && \
29+
tar xvf "Python-${v}.tar.xz" -C "/tmp/python${v}" && \
30+
cd "/tmp/python${v}/Python-${v}" && \
31+
./configure 2>&1 >/dev/null && \
32+
make -j8 altinstall 2>&1 >/dev/null && \
33+
ln -sf "/usr/local/bin/python${v%.*}" "/usr/bin/python${v%.*}"; \
34+
done
35+
36+
# For each python interpreter, install pip dependencies needed for array_record
37+
RUN --mount=type=cache,target=/root/.cache \
38+
for p in 3.8 3.9; do \
39+
python${p} -m pip install -U pip && \
40+
python${p} -m pip install -U \
41+
absl-py \
42+
auditwheel \
43+
patchelf \
44+
setuptools \
45+
twine \
46+
wheel; \
47+
done
48+
49+
WORKDIR "/tmp/array_record"

‎oss/build_whl.sh‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/bin/bash
2+
# build wheel for python version specified in $PYTHON
3+
4+
set -e -x
5+
6+
export PYTHON_MINOR_VERSION="${PYTHON_MINOR_VERSION}"
7+
PYTHON="python3${PYTHON_MINOR_VERSION:+.$PYTHON_MINOR_VERSION}"
8+
9+
function write_to_bazelrc() {
10+
echo "$1" >> .bazelrc
11+
}
12+
13+
function main() {
14+
# Remove .bazelrc if it already exists
15+
[ -e .bazelrc ] && rm .bazelrc
16+
17+
write_to_bazelrc "build -c opt"
18+
write_to_bazelrc "build --cxxopt=-std=c++17"
19+
write_to_bazelrc "build --host_cxxopt=-std=c++17"
20+
write_to_bazelrc "build --linkopt=\"-lrt -lm\""
21+
write_to_bazelrc "build --experimental_repo_remote_exec"
22+
write_to_bazelrc "build --action_env=PYTHON_BIN_PATH=\"/usr/bin/$PYTHON\""
23+
write_to_bazelrc "build --action_env=PYTHON_LIB_PATH=\"/usr/lib/$PYTHON\""
24+
write_to_bazelrc "build --python_path=\"/usr/bin/$PYTHON\""
25+
26+
bazel clean
27+
bazel build $@ ...
28+
bazel test $@ ...
29+
30+
DEST="/tmp/array_record_pip_pkg"
31+
# Create the directory, then do dirname on a non-existent file inside it to
32+
# give us an absolute paths with tilde characters resolved to the destination
33+
# directory.
34+
mkdir -p "${DEST}"
35+
echo "=== destination directory: ${DEST}"
36+
37+
TMPDIR=$(mktemp -d -t tmp.XXXXXXXXXX)
38+
39+
echo $(date) : "=== Using tmpdir: ${TMPDIR}"
40+
mkdir "${TMPDIR}/array_record"
41+
42+
echo "=== Copy array_record files"
43+
44+
cp setup.py "${TMPDIR}"
45+
cp LICENSE "${TMPDIR}"
46+
rsync -avm -L --exclude="bazel-*/" . "${TMPDIR}/array_record"
47+
rsync -avm -L --include="*.so" --include="*_pb2.py" \
48+
--exclude="*.runfiles" --exclude="*_obj" --include="*/" --exclude="*" \
49+
bazel-bin/cpp "${TMPDIR}/array_record"
50+
rsync -avm -L --include="*.so" --include="*_pb2.py" \
51+
--exclude="*.runfiles" --exclude="*_obj" --include="*/" --exclude="*" \
52+
bazel-bin/python "${TMPDIR}/array_record"
53+
54+
pushd ${TMPDIR}
55+
echo $(date) : "=== Building wheel"
56+
${PYTHON} setup.py bdist_wheel --python-tag py3${PYTHON_MINOR_VERSION}
57+
58+
echo $(date) : "=== Auditing wheel"
59+
auditwheel repair --plat manylinux2014_x86_64 -w dist dist/*.whl
60+
cp dist/*.whl "${DEST}"
61+
popd
62+
63+
echo $(date) : "=== Output wheel file is in: ${DEST}"
64+
}
65+
66+
main "$@"

‎oss/build_whl_runner.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/bin/bash
2+
# run build_whl.sh for different python versions
3+
4+
set -x
5+
6+
for p in 8 9
7+
do
8+
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.${p} 1
9+
10+
PYTHON_MINOR_VERSION=${p} oss/build_whl.sh \
11+
[email protected].${p}_config_cuda//crosstool:toolchain
12+
13+
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.${p} 0
14+
done
15+
16+
cp /tmp/array_record_pip_pkg/*.whl /tmp/array_record/

‎oss/runner.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/bash
2+
# This script copy array_record from internal repo, build a docker, and build pip wheels
3+
4+
set -e -x
5+
6+
export TMP_FOLDER="/tmp/array_record"
7+
8+
[ -f $TMP_FOLDER ] && rm -rf $TMP_FOLDER
9+
copybara array_record/oss/copy.bara.sky g3folder_to_gitfolder ../../ \
10+
--init-history --folder-dir=$TMP_FOLDER --ignore-noop
11+
12+
cd $TMP_FOLDER
13+
DOCKER_BUILDKIT=1 docker build --progress=plain --no-cache \
14+
-t array_record:latest - < oss/build.Dockerfile
15+
16+
docker run --rm -a stdin -a stdout -a stderr \
17+
-v $TMP_FOLDER:/tmp/array_record --name array_record array_record:latest \
18+
bash oss/build_whl_runner.sh
19+
20+
ls $TMP_FOLDER/*.whl

‎python/__init__.py‎

Whitespace-only changes.

‎setup.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
"""Setup.py file for array_record."""
2+
3+
from setuptools import find_packages
4+
from setuptools import setup
5+
6+
setup(
7+
name='array_record',
8+
version='0.1.0',
9+
description=(
10+
'A file format that achieves a new frontier of IO efficiency'
11+
),
12+
author='ArrayRecord team',
13+
author_email='[email protected]',
14+
packages=find_packages(),
15+
include_package_data=True,
16+
package_data={'': ['*.so']},
17+
python_requires='>=3.7',
18+
install_requires=['absl-py'],
19+
url='https://github.com/google/array_record',
20+
license='Apache-2.0',
21+
classifiers=[
22+
'Programming Language :: Python :: 3.8',
23+
'Programming Language :: Python :: 3.9',
24+
],
25+
zip_safe=False,
26+
)

0 commit comments

Comments
 (0)