WIP: Build for macOS

Author: marcenacp

.bazelrc

@@ -2,9 +2,8 @@ build -c opt
 build --cxxopt=-std=c++17
 build --host_cxxopt=-std=c++17
 build --experimental_repo_remote_exec
-
-# TODO(fchern): Use non-hardcode path.
-build --action_env=PYTHON_BIN_PATH="/usr/bin/python3"
-build --action_env=PYTHON_LIB_PATH="/usr/lib/python3"
-build --repo_env=PYTHON_BIN_PATH="/usr/bin/python3"
-build --python_path="/usr/bin/python3"
+build --linkopt="-lpthread -lm -framework CoreFoundation"
+build --action_env=PYTHON_BIN_PATH="/Users/pierremarcenac/.pyenv/shims/python"
+build --python_path="/Users/pierremarcenac/.pyenv/shims/python"
+build --action_env=PYTHON_LIB_PATH="/Users/pierremarcenac/.pyenv/versions/python3.10/lib/python3.10/site-packages"
+build --action_env=CXXFLAGS=-stdlib=libc++ --action_env=LDFLAGS=-stdlib=libc++ --action_env=BAZEL_CXXOPTS=-stdlib=libc++ --action_env=BAZEL_LINKOPTS=-lc++:-lm

WORKSPACE

@@ -55,9 +55,9 @@ cc_library(
 # `pybind11_bazel` (https://github.com/pybind/pybind11_bazel): 20230130
 http_archive(
   name = "pybind11_bazel",
-  strip_prefix = "pybind11_bazel-5f458fa53870223a0de7eeb60480dd278b442698",
-  sha256 = "b35f3abc3d52ee5c753fdeeb2b5129b99e796558754ca5d245e28e51c1072a21",
-  urls = ["https://github.com/pybind/pybind11_bazel/archive/5f458fa53870223a0de7eeb60480dd278b442698.tar.gz"],
+  strip_prefix = "pybind11_bazel-master",
+#   sha256 = "b35f3abc3d52ee5c753fdeeb2b5129b99e796558754ca5d245e28e51c1072a21",
+  urls = ["https://github.com/pybind/pybind11_bazel/archive/master.tar.gz"],
 )
 # V2.10.3, 20230130
 http_archive(
@@ -131,9 +131,9 @@ http_archive(
 http_archive(
     name = "highwayhash",
     build_file = "@com_google_riegeli//third_party:highwayhash.BUILD",
-    sha256 = "cf891e024699c82aabce528a024adbe16e529f2b4e57f954455e0bf53efae585",
-    strip_prefix = "highwayhash-276dd7b4b6d330e4734b756e97ccfb1b69cc2e12",
-    urls = ["https://github.com/google/highwayhash/archive/276dd7b4b6d330e4734b756e97ccfb1b69cc2e12.zip"],  # 2019-02-22
+    # sha256 = "cf891e024699c82aabce528a024adbe16e529f2b4e57f954455e0bf53efae585",
+    strip_prefix = "highwayhash-master",
+    urls = ["https://github.com/google/highwayhash/archive/master.zip"],  # 2019-02-22
 )
 
 # Tensorflow, 20230130

cpp/BUILD

@@ -54,6 +54,7 @@ cc_library(
     deps = [
         "@com_google_absl//absl/flags:flag",
         "@eigen3//:eigen3",
+        "@highwayhash//:hh_neon",
     ],
 )
 
@@ -123,6 +124,7 @@ cc_library(
         "@com_google_riegeli//riegeli/chunk_encoding:simple_encoder",
         "@com_google_riegeli//riegeli/chunk_encoding:transpose_encoder",
         "@com_google_riegeli//riegeli/records:records_metadata_cc_proto",
+        "@highwayhash//:hh_neon",
     ],
 )
 
@@ -171,6 +173,7 @@ cc_library(
         "@com_google_riegeli//riegeli/chunk_encoding:chunk_decoder",
         "@com_google_riegeli//riegeli/records:chunk_reader",
         "@com_google_riegeli//riegeli/records:record_position",
+        "@highwayhash//:hh_neon",
     ],
 )
 

cpp/array_record_reader.cc

@@ -317,7 +317,7 @@ absl::Status ArrayRecordReaderBase::ParallelReadRecords(
     return absl::OkStatus();
   }
   uint64_t num_chunk_groups =
-      CeilOfRatio(state_->footer.size(), state_->chunk_group_size);
+      CeilOfRatio<uint64_t>(state_->footer.size(), state_->chunk_group_size);
   const auto reader = get_backing_reader();
   Reader* mutable_reader = const_cast<Reader*>(
       reinterpret_cast<const Reader*>(reader.get()));
@@ -326,7 +326,7 @@ absl::Status ArrayRecordReaderBase::ParallelReadRecords(
         uint64_t chunk_idx_start = buf_idx * state_->chunk_group_size;
         // inclusive index, not the conventional exclusive index.
         uint64_t last_chunk_idx =
-            std::min((buf_idx + 1) * state_->chunk_group_size - 1,
+            std::min<uint64_t>((buf_idx + 1) * state_->chunk_group_size - 1,
                      state_->footer.size() - 1);
         uint64_t buf_len = state_->ChunkEndOffset(last_chunk_idx) -
                            state_->footer[chunk_idx_start].chunk_offset();
@@ -654,7 +654,7 @@ bool ArrayRecordReaderBase::ReadAheadFromBuffer(uint64_t buffer_idx) {
     std::vector<ChunkDecoder> decoders;
     decoders.reserve(state_->chunk_group_size);
     uint64_t chunk_start = buffer_idx * state_->chunk_group_size;
-    uint64_t chunk_end = std::min(state_->footer.size(),
+    uint64_t chunk_end = std::min<uint64_t>(state_->footer.size(),
                                   (buffer_idx + 1) * state_->chunk_group_size);
     const auto reader = get_backing_reader();
     for (uint64_t chunk_idx = chunk_start; chunk_idx < chunk_end; ++chunk_idx) {
@@ -693,7 +693,7 @@ bool ArrayRecordReaderBase::ReadAheadFromBuffer(uint64_t buffer_idx) {
     std::vector<uint64_t> chunk_offsets;
     chunk_offsets.reserve(state_->chunk_group_size);
     uint64_t chunk_start = buffer_to_add * state_->chunk_group_size;
-    uint64_t chunk_end = std::min(
+    uint64_t chunk_end = std::min<uint64_t>(
         state_->footer.size(), (buffer_to_add + 1) * state_->chunk_group_size);
     for (uint64_t chunk_idx = chunk_start; chunk_idx < chunk_end; ++chunk_idx) {
       chunk_offsets.push_back(state_->footer[chunk_idx].chunk_offset());

oss/build_whl.sh

@@ -4,7 +4,7 @@
 set -e -x
 
 export PYTHON_VERSION="${PYTHON_VERSION}"
-PYTHON="python${PYTHON_VERSION}"
+PYTHON="python"
 PYTHON_MAJOR_VERSION=$(${PYTHON} -c 'import sys; print(sys.version_info.major)')
 PYTHON_MINOR_VERSION=$(${PYTHON} -c 'import sys; print(sys.version_info.minor)')
 BAZEL_FLAGS="--crosstool_top="
@@ -21,18 +21,20 @@ function main() {
   write_to_bazelrc "build -c opt"
   write_to_bazelrc "build --cxxopt=-std=c++17"
   write_to_bazelrc "build --host_cxxopt=-std=c++17"
-  write_to_bazelrc "build --linkopt=\"-lrt -lm\""
+  # write_to_bazelrc "build --linkopt=\"-lrt -lm\""
   write_to_bazelrc "build --experimental_repo_remote_exec"
-  write_to_bazelrc "build --action_env=PYTHON_BIN_PATH=\"/usr/bin/${PYTHON}\""
-  write_to_bazelrc "build --action_env=PYTHON_LIB_PATH=\"/usr/lib/${PYTHON}\""
-  write_to_bazelrc "build --python_path=\"/usr/bin/${PYTHON}\""
+  write_to_bazelrc "build --action_env=PYTHON_BIN_PATH=\"/Users/pierremarcenac/.pyenv/shims/python\""
+  write_to_bazelrc "build --action_env=PYTHON_LIB_PATH=\"/Users/pierremarcenac/.pyenv/versions/python3.10/lib/python3.10/site-packages\""
+  write_to_bazelrc "build --python_path=\"/Users/pierremarcenac/.pyenv/shims/python\""
+
+  ${PYTHON} -m pip install absl-py etils wheel
 
   # Using a previous version of Blaze to avoid:
   # https://github.com/bazelbuild/bazel/issues/8622
   export USE_BAZEL_VERSION=5.4.0
-  bazel clean
-  bazel build ${BAZEL_FLAGS} ...
-  bazel test ${BAZEL_FLAGS} --verbose_failures --test_output=errors ...
+  bazel clean --expunge
+  bazel build //...
+  bazel test --verbose_failures --test_output=errors //...
 
   DEST="/tmp/array_record/all_dist"
   # Create the directory, then do dirname on a non-existent file inside it to

python/BUILD

@@ -17,6 +17,7 @@ pybind_extension(
         "@com_google_absl//absl/strings",
         "@com_google_riegeli//riegeli/bytes:fd_reader",
         "@com_google_riegeli//riegeli/bytes:fd_writer",
+        "@highwayhash//:hh_neon",
     ],
 )