MacOSX support: workspace changes and action changes

We no longer build glib from source, but instead require it to be available locally. Also modify other "local system" repositories to point at homebrew directory locations. The workspace changes are not strictly better or worse than the master branch, since users would generally be expected to adjust the local repositories anyway. The new action script uses homebrew for additional library installations. It only builds a small set of targets that work with SDL. The //:game_lib_sdl rule is updated to link with `-framework OpenGL`. Also update the README.md to state the nature of this branch.
google-deepmind · Jan 13, 2021 · fb78a98 · fb78a98
1 parent cfd39c0
commit fb78a98
Show file tree

Hide file tree

Showing 8 changed files with 123 additions and 286 deletions.
diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml
@@ -13,42 +13,58 @@ on:
 
 jobs:
   build-and-test:
-    name: Build and test ${{matrix.cfg.name}} ${{matrix.os.name}}
+    name: Build and test ${{matrix.cfg.name}}
 
-    runs-on: ${{matrix.os.id}}
+    runs-on: ${{matrix.cfg.os}}
 
     strategy:
       matrix:
-        os:
-          - { name: 'Linux', id: 'ubuntu-20.04' }
         cfg:
-          - { name: 'GCC', cc: gcc-10, cxx: g++-10, config: --copt=-Wno-maybe-uninitialized }
-          - { name: 'LLVM+libstdc++', cc: clang, cxx: clang++, config: --copt=-Wno-uninitialized }
-          - { name: 'LLVM+libc++', cc: clang, cxx: clang++, config: --config=libc++ --copt=-Wno-uninitialized }
+          #- { name: 'Linux GCC', os: 'ubuntu-20.04', cc: gcc-10, cxx: g++-10, config: --copt=-Wno-maybe-uninitialized }
+          #- { name: 'Linux LLVM+libstdc++', os: 'ubuntu-20.04', cc: clang, cxx: clang++, config: --copt=-Wno-uninitialized }
+          #- { name: 'Linux LLVM+libc++', os: 'ubuntu-20.04', cc: clang, cxx: clang++, config: --config=libc++ --copt=-Wno-uninitialized }
+          - { name: 'MacOS LLVM+libc++', os: 'macos-10.15', config: --config=libc++ --copt=-Wno-uninitialized }
 
     env:
       CC:  ${{matrix.cfg.cc}}
       CXX: ${{matrix.cfg.cxx}}
 
     steps:
       - uses: actions/checkout@v2
-      - name: install infrastructure
+      - name: install infrastructure (Linux)
+        if: matrix.cfg.os == 'ubuntu-20.04'
         run: |
           sudo apt-get update
           sudo apt-get install gettext python2-dev python3-dev libsdl2-dev libosmesa6-dev python-numpy-dev python3-numpy-dev python-pil python3-pil python-enum34
           sudo find /usr -type f -name Python.h -o -name arrayobject.h -o -name functional -o -name -filesystem
           sudo ln -s /usr/lib/llvm-10/include/c++/v1 /usr/include/c++/v1
-      - name: build
+      - name: install infrastructure (MacOS)
+        if: matrix.cfg.os == 'macos-10.15'
+        run: |
+          brew install SDL2
+          sudo -H pip install numpy
+          sudo -H pip3 install numpy
+          sudo find -L /usr /Library -type f -name SDL.h -o -name glib.h -o -name Python.h -o -name arrayobject.h
+      - name: build (Linux)
+        if: matrix.cfg.os == 'ubuntu-20.04'
         run: |
           bazel --bazelrc=.precommit.bazelrc build ${{matrix.cfg.config}} --copt=-Wno-sign-compare //...
-      - name: run-tests
+      - name: run-tests (Linux)
+        if: matrix.cfg.os == 'ubuntu-20.04'
         timeout-minutes: 45
         run: |
           bazel --bazelrc=.precommit.bazelrc test  ${{matrix.cfg.config}} --copt=-Wno-sign-compare //...
           bazel --bazelrc=.precommit.bazelrc test  ${{matrix.cfg.config}} --copt=-Wno-sign-compare //python/tests:python_module_test.py2 --test_tag_filters="" --test_arg=--verbose
           bazel --bazelrc=.precommit.bazelrc test  ${{matrix.cfg.config}} --copt=-Wno-sign-compare //python/tests:python_module_test.py3 --test_tag_filters="" --test_arg=--verbose
           bazel --bazelrc=.precommit.bazelrc test  ${{matrix.cfg.config}} --copt=-Wno-sign-compare //python/tests:dmenv_module_test.py2 --test_tag_filters="" --test_arg=--verbose
           bazel --bazelrc=.precommit.bazelrc test  ${{matrix.cfg.config}} --copt=-Wno-sign-compare //python/tests:dmenv_module_test.py3 --test_tag_filters="" --test_arg=--verbose
-      - name: run-agent
+      - name: run-agent (Linux)
+        if: matrix.cfg.os == 'ubuntu-20.04'
         run: |
           bazel --bazelrc=.precommit.bazelrc run   ${{matrix.cfg.config}} --copt=-Wno-sign-compare --define headless=osmesa //:python_random_agent
+      - name: build-and-run-tests (MacOS)
+        if: matrix.cfg.os == 'macos-10.15'
+        timeout-minutes: 45
+        run: |
+          bazel --bazelrc=.precommit.bazelrc build ${{matrix.cfg.config}} --copt=-Wno-sign-compare -c opt --dynamic_mode=off --show_progress_rate_limit=10 //:game
+          bazel --bazelrc=.precommit.bazelrc  test ${{matrix.cfg.config}} --copt=-Wno-sign-compare -c opt --dynamic_mode=off --show_progress_rate_limit=10 --test_output=errors //third_party/... //deepmind/... //lua_tests/... $(bazel query //... | grep seed_level_)
diff --git a/BUILD b/BUILD
@@ -814,10 +814,15 @@ cc_library(
     hdrs = ["public/dmlab.h"],
     copts = IOQ3_COMMON_COPTS,
     defines = IOQ3_COMMON_DEFINES,
-    linkopts = [
-        "-lGL",
-        "-lrt",
-    ],
+    linkopts = select({
+        ":is_linux": [
+            "-lGL",
+            "-lrt",
+        ],
+        ":is_macos": [
+            "-framework OpenGL",
+        ],
+    }),
     deps = IOQ3_COMMON_DEPS,
     alwayslink = 1,
 )
@@ -857,10 +862,15 @@ cc_library(
     hdrs = ["public/dmlab.h"],
     copts = IOQ3_COMMON_COPTS,
     defines = IOQ3_COMMON_DEFINES,
-    linkopts = [
-        "-lGL",
-        "-lX11",
-    ],
+    linkopts = select({
+        ":is_linux": [
+            "-lGL",
+            "-lX11",
+        ],
+        ":is_macos": [
+            "-framework OpenGL",
+        ],
+    }),
     deps = IOQ3_COMMON_DEPS,
     alwayslink = 1,
 )
@@ -880,10 +890,15 @@ cc_library(
     hdrs = ["public/dmlab.h"],
     copts = IOQ3_COMMON_COPTS,
     defines = IOQ3_COMMON_DEFINES,
-    linkopts = [
-        "-lEGL",
-        "-lGL",
-    ],
+    linkopts = select({
+        ":is_linux": [
+            "-lEGL",
+            "-lGL",
+        ],
+        ":is_macos": [
+            "-framework OpenGL",
+        ],
+    }),
     deps = IOQ3_COMMON_DEPS + ["//third_party/GL/util:egl_util"],
     alwayslink = 1,
 )
@@ -934,7 +949,10 @@ config_setting(
 
 cc_binary(
     name = "libdmlab_headless_hw.so",
-    linkopts = ["-Wl,--version-script,$(location :dmlab.lds)"],
+    linkopts = select({
+        ":is_linux": ["-Wl,--version-script,$(location :dmlab.lds)"],
+        ":is_macos": [],
+    }),
     linkshared = 1,
     linkstatic = 1,
     visibility = ["//testing:__subpackages__"],
@@ -947,7 +965,10 @@ cc_binary(
 
 cc_binary(
     name = "libdmlab_headless_sw.so",
-    linkopts = ["-Wl,--version-script,$(location :dmlab.lds)"],
+    linkopts = select({
+        ":is_linux": ["-Wl,--version-script,$(location :dmlab.lds)"],
+        ":is_macos": [],
+    }),
     linkshared = 1,
     linkstatic = 1,
     visibility = ["//testing:__subpackages__"],
@@ -990,6 +1011,7 @@ cc_library(
         "dmlab_graphics_sdl": [":game_lib_sdl"],
         "//conditions:default": [":dmlab_so_loader"],
     }),
+    alwayslink = 1,
 )
 
 cc_binary(

diff --git a/README.md b/README.md
@@ -1,3 +1,31 @@
+# Experimental MacOSX port
+
+This is _not_ the main _DeepMind Lab_ branch.
+
+This is an experimental, non-functional development branch that contains
+incompatible changes to make _DeepMind Lab_ work on MacOSX.
+
+**Notable changes:**
+
+* `glib` is no longer compiled from source, but must instead be
+  provided locally.
+* The use of `sendfile` in `public/dmlab_so_loader.cc` has been replaced with
+  `fcopyfile`.
+
+You need to pass `--apple_platform_type=macos` to all Bazel invocations.
+
+You may want to use homebrew to install the following packages:
+`glib`, `sdl2`, `python@2`, `numpy`. We also consume a dependency `-liconv`,
+but that just seems to work.
+
+The headful SDL renderer works (e.g. `//:game`, or `//:python_random_agent`
+with `--define=graphics=sdl`). The headless renderers for OSMesa, GLX and EGL
+do not work yet, apparently because we are missing the relevant library
+dependencies. A native MacOSX "glimp" implementation seems like the best next
+step (but is not complete yet).
+
+<br><br><br>
+
 # <img src="/docs/template/logo.png" alt="DeepMind Lab">
 
 *DeepMind Lab* is a 3D learning environment based on id Software's

diff --git a/WORKSPACE b/WORKSPACE
@@ -62,17 +62,6 @@ http_archive(
     ],
 )
 
-http_archive(
-    name = "glib_archive",
-    build_file = "@//bazel:glib.BUILD",
-    sha256 = "0cbb3d31c9d181bbcc97cba3d9dbe3250f75e2da25e5f7c8bf5a993fe54baf6a",
-    strip_prefix = "glib-2.55.1",
-    urls = [
-        "https://mirror.bazel.build/ftp.gnome.org/pub/gnome/sources/glib/2.55/glib-2.55.1.tar.xz",
-        "https://ftp.gnome.org/pub/gnome/sources/glib/2.55/glib-2.55.1.tar.xz",
-    ],
-)
-
 http_archive(
     name = "jpeg_archive",
     build_file = "@//bazel:jpeg.BUILD",
@@ -160,10 +149,16 @@ http_archive(
 )
 
 # TODO: Replace with hermetic build
+new_local_repository(
+    name = "glib_archive",
+    build_file = "@//bazel:glib.BUILD",
+    path ="/usr/local/Cellar/glib/2.66.2_1",
+)
+
 new_local_repository(
     name = "sdl_system",
     build_file = "@//bazel:sdl.BUILD",
-    path = "/usr",
+    path = "/usr/local/Cellar/sdl2/2.0.12_1",
 )
 
 new_local_repository(