integration tests for minio rules

.bazelignore

@@ -0,0 +1 @@
+e2e/

e2e/.gitignore

@@ -0,0 +1,2 @@
+.minio.sys/
+mc/share/

e2e/BUILD.archive

@@ -0,0 +1 @@
+exports_files(["file.txt"])

e2e/BUILD.bazel

@@ -0,0 +1,42 @@
+load("@rules_python//python:defs.bzl", "py_binary", "py_library")
+
+py_binary(
+    name = "cloud_archive_test",
+    srcs = ["cloud_archive_test.py"],
+    data = [
+        ":mc_config",
+        "@mc_binary//file",
+    ],
+    deps = [
+        ":minio_server_lib",
+        "@rules_python//python/runfiles",
+    ],
+)
+
+filegroup(
+    name = "testdata",
+    srcs = glob(["testdata/**"]),
+)
+
+filegroup(
+    name = "mc_config",
+    srcs = glob(["mc/**"]),
+)
+
+py_library(
+    name = "minio_server_lib",
+    srcs = ["minio_server.py"],
+    data = [
+        ":testdata",
+        "@minio_binary//file"
+    ],
+    deps = [
+        "@rules_python//python/runfiles",
+    ],
+)
+
+py_binary(
+    name = "minio_server",
+    srcs = ["minio_server.py"],
+    deps = [":minio_server_lib"],
+)

e2e/README.md

@@ -0,0 +1,52 @@
+# Integration Test Repository
+
+```shell
+bazel run //:minio_server
+```
+
+will start a [MinIO](https://docs.min.io) server locally.
+By default, it serves from the `testdata/` directory.
+There is a single bucket named `bucket`.
+
+You can fetch the [AlStor client](https://docs.min.io/enterprise/aistor-object-store/reference/cli/) with `bazel`,
+
+```shell
+bazel build @mc_binary//file
+```
+
+This can be used from the command line by including the output on your `PATH`,
+
+```shell
+export PATH="$(bazel info execution_root)/external/mc_binary/file:$PATH"
+```
+
+The client's config is available in `mc/`.
+There is a single alias named `local` with typical MinIO defaults set.
+Again, we'll update our shell for convenience,
+
+```shell
+export MC_CONFIG_DIR="$PWD/mc"
+```
+
+You should now be able to access the server with `$ mc admin info local`.
+You are now able to fetch the data using `bazel` and the `cloud_archive` rules,
+
+**`minio_file`**:
+
+```shell
+bazel build @test_file//file
+```
+
+**`minio_archive`**:
+
+```shell
+bazel build @test_archive//:file.txt
+```
+
+## Tests
+
+All of the above is done automatically when running the integration tests,
+
+```shell
+bazel run //:cloud_archive_test
+```

e2e/WORKSPACE

@@ -0,0 +1,74 @@
+workspace(name = "my_workspace")
+
+load("@bazel_tools//tools/build_defs/repo:local.bzl", "local_repository")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
+
+local_repository(
+    name = "cloud_archive",
+    path = "..",
+)
+
+load("@cloud_archive//:cloud_archive.bzl", "minio_archive", "minio_file")
+
+minio_archive(
+    name = "test_archive",
+    build_file = "//:BUILD.archive",
+    file_path = "local/bucket/data.tar.gz",
+    sha256 = "cb8c43418acb05e11619478fbb9c0a76827841cc2d1f58ce9b4d7b5093d1c01e",
+)
+
+minio_archive(
+    name = "test_archive_p0_patch",
+    patches = ["@//:patches/p0.patch"],
+    build_file = "//:BUILD.archive",
+    file_path = "local/bucket/data.tar.gz",
+    sha256 = "cb8c43418acb05e11619478fbb9c0a76827841cc2d1f58ce9b4d7b5093d1c01e",
+)
+
+minio_archive(
+    name = "test_archive_p1_patch",
+    patch_args = ["-p1"],
+    patches = ["@//:patches/p1.patch"],
+    build_file = "//:BUILD.archive",
+    file_path = "local/bucket/data.tar.gz",
+    sha256 = "cb8c43418acb05e11619478fbb9c0a76827841cc2d1f58ce9b4d7b5093d1c01e",
+)
+
+minio_file(
+    name = "test_file",
+    file_path = "local/bucket/file.txt",
+    sha256 = "8e686326c475d56c005bcda41fe457e777894b57b98be9b29603779d3f6d50b6",
+)
+
+http_archive(
+    name = "rules_python",
+    sha256 = "fa532d635f29c038a64c8062724af700c30cf6b31174dd4fac120bc561a1a560",
+    strip_prefix = "rules_python-1.5.1",
+    urls = [
+        "https://github.com/bazel-contrib/rules_python/releases/download/1.5.1/rules_python-1.5.1.tar.gz",
+    ],
+)
+
+load("@rules_python//python:repositories.bzl", "py_repositories")
+
+py_repositories()
+
+http_file(
+    name = "minio_binary",
+    urls = [
+        "https://dl.min.io/server/minio/release/linux-amd64/minio",
+    ],
+    downloaded_file_path = "minio",
+    executable = True,
+    sha256 = "eef6581f6509f43ece007a6f2eb4c5e3ce41498c8956e919a7ac7b4b170fa431",
+)
+
+http_file(
+    name = "mc_binary",
+    urls = [
+        "https://dl.min.io/client/mc/release/linux-amd64/mc",
+    ],
+    downloaded_file_path = "mc",
+    executable = True,
+    sha256 = "ea4a453be116071ab1ccbd24eb8755bf0579649f41a7b94ab9e68571bb9f4a1e",
+)

e2e/cloud_archive_test.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import atexit
+import os
+import pathlib
+import shutil
+import socket
+import subprocess
+import tempfile
+import time
+import unittest
+
+import minio_server
+from python.runfiles import Runfiles
+
+MINIO_PORT = 9000
+
+FILE_CONTENT = "go bears"
+PATCHED_CONTENT = "Go Bears!"
+
+# Always run in a new output_base to avoid false-negatives from caching.
+TEMP_OUTPUT_BASE = tempfile.mkdtemp()
+atexit.register(shutil.rmtree, TEMP_OUTPUT_BASE)
+
+
+def wait_for_port(host: str, port: int, timeout: int = 5) -> bool:
+    start = time.time()
+    while time.time() - start < timeout:
+        try:
+            with socket.create_connection((host, port), timeout=1):
+                return True
+        except OSError:  # noqa: PERF203
+            time.sleep(0.1)
+    return False
+
+
+class CloudArchiveTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        r = Runfiles.Create()
+        mc_binary = r.Rlocation("mc_binary/file/mc")
+        mc_config = r.Rlocation("my_workspace/mc/config.json")
+
+        cls.env = {
+            "PATH": f"{os.path.dirname(mc_binary)}:{os.environ['PATH']}",
+            "MC_CONFIG_DIR": os.path.dirname(mc_config),
+        }
+        cls.workspace_dir = os.environ["BUILD_WORKING_DIRECTORY"]
+
+        repository_cache = subprocess.check_output(
+            ["bazel", "info", "repository_cache"],
+            text=True,
+            cwd=cls.workspace_dir,
+        ).strip()
+        cls.bazel_args = [
+            f"--output_base={TEMP_OUTPUT_BASE}",
+        ]
+        cls.bazel_build_args = [
+            f"--repository_cache={repository_cache}",
+        ]
+
+        execution_root = subprocess.check_output(
+            ["bazel", *cls.bazel_args, "info", "execution_root"],
+            text=True,
+            cwd=cls.workspace_dir,
+        ).strip()
+        cls.output_dir = pathlib.Path(execution_root, "external")
+
+    def test_minio(self) -> None:
+        subtests = [
+            (
+                "@test_file//file",
+                FILE_CONTENT,
+                self.output_dir / "test_file" / "file" / "downloaded",
+            ),
+            (
+                "@test_archive//:file.txt",
+                FILE_CONTENT,
+                self.output_dir / "test_archive" / "file.txt",
+            ),
+            (
+                "@test_archive_p0_patch//:file.txt",
+                PATCHED_CONTENT,
+                self.output_dir / "test_archive_p0_patch" / "file.txt",
+            ),
+            (
+                "@test_archive_p1_patch//:file.txt",
+                PATCHED_CONTENT,
+                self.output_dir / "test_archive_p1_patch" / "file.txt",
+            ),
+        ]
+        with minio_server.run(port=MINIO_PORT) as proc:
+            if not wait_for_port("127.0.0.1", MINIO_PORT):
+                proc.kill()
+                print("MinIO server failed to start")
+                raise SystemExit(1)
+
+            for target, expected_content, output_path in subtests:
+                with self.subTest(target=target):
+                    subprocess.check_call(
+                        ["bazel", *self.bazel_args, "build", *self.bazel_build_args, target],
+                        env=self.env,
+                        cwd=self.workspace_dir,
+                    )
+                    assert expected_content in output_path.read_text()
+
+
+if __name__ == "__main__":
+    unittest.main()

e2e/mc/config.json

@@ -0,0 +1,12 @@
+{
+	"version": "10",
+	"aliases": {
+		"local": {
+			"url": "http://127.0.0.1:9000",
+			"accessKey": "minioadmin",
+			"secretKey": "minioadmin",
+			"api": "s3v4",
+			"path": "auto"
+		}
+	}
+}

e2e/minio_server.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+import os
+import subprocess
+import typing
+
+from python.runfiles import Runfiles
+
+MINIO_ACCESS_KEY = "minioadmin"
+MINIO_SECRET_KEY = "minioadmin"  # noqa: S105
+
+
+@contextmanager
+def run(*_: typing.Any, port: int = 9000, wait: bool = False) -> typing.Generator:
+    r = Runfiles.Create()
+    minio_binary = r.Rlocation("minio_binary/file/minio")
+    data_directory = os.path.dirname(
+        os.path.dirname(
+            os.path.dirname(
+                r.Rlocation(
+                    "my_workspace/testdata/bucket/file.txt/xl.meta"  # The particular file is not important.  # noqa: E501
+                )
+            )
+        )
+    )
+
+    env = os.environ.copy()
+    env["MINIO_ROOT_USER"] = MINIO_ACCESS_KEY
+    env["MINIO_ROOT_PASSWORD"] = MINIO_SECRET_KEY
+
+    proc = subprocess.Popen(  # noqa: S603
+        [minio_binary, "server", f"--address=:{port}", data_directory],
+        env=env,
+    )
+
+    try:
+        if wait:
+            proc.wait()
+            return
+        yield proc
+    finally:
+        if not wait:
+            proc.terminate()
+            proc.wait()
+
+
+if __name__ == "__main__":
+    with run(wait=True):
+        pass

e2e/patches/p0.patch

@@ -0,0 +1,7 @@
+diff --git file.txt file.txt
+index 0247de8..18bdb41 100644
+--- file.txt
++++ file.txt
+@@ -1 +1 @@
+-go bears
++Go Bears!

e2e/patches/p1.patch

@@ -0,0 +1,7 @@
+diff --git a/file.txt b/file.txt
+index 0247de8..18bdb41 100644
+--- a/file.txt
++++ b/file.txt
+@@ -1 +1 @@
+-go bears
++Go Bears!