GitHub actions (#73)

Author: christopher-hesse

.github/workflows/main.yml

@@ -0,0 +1,84 @@
+name: ci
+
+on: [push]
+
+jobs:
+  build:
+    name: build ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idruns-on
+        os: ["ubuntu-18.04", "macos-10.15", "windows-2019"]
+        python-version: ["3.7"]
+    steps:
+      - uses: actions/checkout@v2
+      # run vcvars on windows
+      - uses: ilammy/msvc-dev-cmd@v1
+
+      # warning: if something is changed so that the cached dependencies are not correct
+      # this cache will not notice and the build will be slow, to fix this, increment the
+      # version in the key
+      - name: Cache Dependencies
+        id: cache-dependencies
+        uses: actions/cache@v2
+        with:
+          path: cache
+          key: ${{ runner.os }}-dependencies-v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+
+      - name: Install Procgen Build
+        run: python -m pip install -e procgen-build
+
+      - name: Build
+        run: python -u -m procgen_build.build_package
+
+      - name: Upload wheel artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: wheels
+          path: wheelhouse/*
+
+      # - name: Publish distribution 📦 to Test PyPI
+      #   uses: pypa/gh-action-pypi-publish@master
+      #   with:
+      #     password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+      #     repository_url: https://test.pypi.org/legacy/
+          
+      # - name: Publish distribution 📦 to PyPI
+      #   if: startsWith(github.ref, 'refs/tags')
+      #   uses: pypa/gh-action-pypi-publish@master
+      #   with:
+      #     password: ${{ secrets.PYPI_API_TOKEN }}
+
+  dev:
+    name: dev ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idruns-on
+        os: ["ubuntu-18.04", "macos-10.15", "windows-2019"]
+        python-version: ["3.7"]
+    steps:
+      - uses: actions/checkout@v2
+      # run vcvars on windows
+      - uses: ilammy/msvc-dev-cmd@v1
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+
+      - name: Install Procgen Build
+        run: python -m pip install -e procgen-build
+
+      - name: Test
+        run: python -u -m procgen_build.dev_test

.travis.yml

@@ -26,9 +26,10 @@ Supported platforms:
 
 Supported Pythons:
 
-- 3.6 64-bit
 - 3.7 64-bit
 - 3.8 64-bit
+- 3.9 64-bit
+- 3.10 64-bit
 
 Supported CPUs:
 
@@ -40,7 +41,7 @@ First make sure you have a supported version of python:
 
 ```
 # run these commands to check for the correct python version
-python -c "import sys; assert (3,6,0) <= sys.version_info <= (3,9,0), 'python is incorrect version'; print('ok')"
+python -c "import sys; assert (3,7,0) <= sys.version_info <= (3,10,0), 'python is incorrect version'; print('ok')"
 python -c "import platform; assert platform.architecture()[0] == '64bit', 'python is not 64-bit'; print('ok')"
 ```
 
@@ -183,7 +184,7 @@ This returns a list of byte strings representing the state of each game in the v
 
 # Install from Source
 
-If you want to change the environments or create new ones, you should build from source.  You can get miniconda from https://docs.conda.io/en/latest/miniconda.html if you don't have it, or install the dependencies from [`environment.yml`](environment.yml) manually.  On Windows you will also need "Visual Studio 15 2017" installed.
+If you want to change the environments or create new ones, you should build from source.  You can get miniconda from https://docs.conda.io/en/latest/miniconda.html if you don't have it, or install the dependencies from [`environment.yml`](environment.yml) manually.  On Windows you will also need "Visual Studio 16 2019" installed.
 
 ```
 git clone [email protected]:openai/procgen.git

README.md

@@ -4,8 +4,8 @@ channels:
 dependencies:
   - python=3.7.3
   # https://www.anaconda.com/utilizing-the-new-compilers-in-anaconda-distribution-5/
-  - c-compiler=1.0.4
-  - cmake=3.14.0
+  - c-compiler=1.3.0
+  - cmake=3.21.3
   - qt=5.12.5  # conda-forge does not have 5.13.2 available
   - pip
   - pip:

environment.yml

@@ -1,80 +1,19 @@
 import platform
-from urllib.request import urlretrieve
 import os
-import subprocess as sp
-import fnmatch
 
-import blobfile as bf
-
-from .common import run, GCS_BUCKET
+from .common import run
 
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
-# https://stackoverflow.com/a/50135504
-def init_vsvars():
-    print("Initializing environment for Visual Studio")
-
-    vcvars_path = "C:/Program Files (x86)/Microsoft Visual Studio/2017/BuildTools/VC/Auxiliary/Build/vcvars64.bat"
-
-    env_bat_file_path = "setup_build_environment_temp.bat"
-    env_txt_file_path = "build_environment_temp.txt"
-    with open(env_bat_file_path, "w") as env_bat_file:
-        env_bat_file.write('call "%s"\n' % vcvars_path)
-        env_bat_file.write("set > %s\n" % env_txt_file_path)
-
-    os.system(env_bat_file_path)
-    with open(env_txt_file_path, "r") as env_txt_file:
-        lines = env_txt_file.read().splitlines()
-
-    os.remove(env_bat_file_path)
-    os.remove(env_txt_file_path)
-    for line in lines:
-        if "=" not in line:
-            print(f"invalid line {repr(line)}")
-            continue
-        k, v = line.split("=", 1)
-        os.environ[k] = v
-
-
-def get_var(pattern):
-    for key, value in os.environ:
-        if fnmatch.fnmatch(key, pattern):
-            return os.environ[key]
-    return None
-
-
-def setup_google_credentials():
-    # brew install travis
-    # travis login --org
-    # gcloud iam service-accounts create procgen-travis-ci --project <project>
-    # gcloud iam service-accounts keys create /tmp/key.json --iam-account procgen-travis-ci@<project>.iam.gserviceaccount.com
-    # gsutil iam ch serviceAccount:procgen-travis-ci@<project>.iam.gserviceaccount.com:objectAdmin gs://{GCS_BUCKET}
-    # travis encrypt-file --org /tmp/key.json
-    input_path = os.path.join(SCRIPT_DIR, "key.json.enc")
-    output_path = os.path.join(os.getcwd(), "key.json")
-    for h in ["d853b3b05b79", "41b34d34b52c"]:
-        key = os.environ.get(f"encrypted_{h}_key")
-        iv = os.environ.get(f"encrypted_{h}_iv")
-        if key is not None:
-            break
-    if key is None:
-        # being compiled on a fork
-        return False
-    sp.run(["openssl", "aes-256-cbc", "-K", key, "-iv", iv, "-in", input_path, "-out", output_path, "-d"], check=True)
-    os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = output_path
-    return True
-
-
 def main():
-    have_credentials = setup_google_credentials()
-
     os.environ.update(
         {
-            "CIBW_BUILD": "cp36-macosx_x86_64 cp37-macosx_x86_64 cp38-macosx_x86_64 cp36-manylinux_x86_64 cp37-manylinux_x86_64 cp38-manylinux_x86_64 cp36-win_amd64 cp37-win_amd64 cp38-win_amd64",
-            "CIBW_BEFORE_BUILD": "pip install -e procgen-build && python -u -m procgen_build.build_qt --output-dir /tmp/qt5",
+            "CIBW_BUILD": "cp37-macosx_x86_64 cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64 cp37-manylinux_x86_64 cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp37-win_amd64 cp38-win_amd64 cp39-win_amd64 cp310-win_amd64",
+            "CIBW_BEFORE_BUILD": "pip install -r procgen-build/requirements.txt && pip install -e procgen-build && python -u -m procgen_build.build_qt --output-dir /tmp/qt5",
             "CIBW_TEST_EXTRAS": "test",
+            "CIBW_BEFORE_TEST": "pip install -r procgen-build/requirements.txt",
             # the --pyargs option causes pytest to use the installed procgen wheel
             "CIBW_TEST_COMMAND": "pytest --verbose --benchmark-disable --durations=16 --pyargs procgen",
             # this is where build-qt.py will put the files
@@ -83,44 +22,23 @@ def main():
             # "CIBW_BUILD_VERBOSITY": "3",
         }
     )
-    if platform.system() == "Darwin":
-        # cibuildwheel's python copy on mac os x sometimes fails with this error:
-        # [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1076)
-        urlretrieve(
-            "https://curl.haxx.se/ca/cacert.pem",
-            os.environ["TRAVIS_BUILD_DIR"] + "/cacert.pem",
-        )
-        os.environ["SSL_CERT_FILE"] = os.environ["TRAVIS_BUILD_DIR"] + "/cacert.pem"
-    elif platform.system() == "Linux":
-        # since we're inside a docker container, adjust the credentials path to point at the mounted location
-        if have_credentials:
-            os.environ["CIBW_ENVIRONMENT"] = (
-                os.environ["CIBW_ENVIRONMENT"]
-                + " GOOGLE_APPLICATION_CREDENTIALS=/host"
-                + os.environ["GOOGLE_APPLICATION_CREDENTIALS"]
-            )
+    if platform.system() == "Linux":
         if "TRAVIS_TAG" in os.environ:
             # pass TRAVIS_TAG to the container so that it can build wheels with the correct version number
             os.environ["CIBW_ENVIRONMENT"] = (
                 os.environ["CIBW_ENVIRONMENT"]
                 + " TRAVIS_TAG=" + os.environ["TRAVIS_TAG"]
             )
-    elif platform.system() == "Windows":
-        init_vsvars()
+        os.environ["CIBW_ENVIRONMENT"] = (
+            os.environ["CIBW_ENVIRONMENT"]
+            + f" CACHE_DIR=/host{os.getcwd()}/cache"
+        )
+    else:
+        os.environ["CACHE_DIR"] = os.path.join(os.getcwd(), "cache")
 
-    run("pip install cibuildwheel==1.4.1")
+    run("pip install cibuildwheel==2.3.1")
     run("cibuildwheel --output-dir wheelhouse")
 
-    if have_credentials:
-        print("upload wheels", platform.system())
-        input_dir = "wheelhouse"
-        output_dir = f"gs://{GCS_BUCKET}/builds/"
-        for filename in bf.listdir(input_dir):
-            src = bf.join(input_dir, filename)
-            dst = bf.join(output_dir, filename)
-            print(src, "=>", dst)
-            bf.copy(src, dst, overwrite=True)
-
 
 if __name__ == "__main__":
     main()