From dc5c68e495736b9122c9189aa9edfaa9340ee315 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Tue, 8 Aug 2023 17:53:25 +0200 Subject: [PATCH 01/57] Create moma.yml --- .github/workflows/moma.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/moma.yml diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml new file mode 100644 index 00000000..ce5e7e70 --- /dev/null +++ b/.github/workflows/moma.yml @@ -0,0 +1,18 @@ +name: moma Docker + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Build the dev Docker image + run: docker build . --file docker/scripts/dev.Dockerfile --tag moma_dev:$(date +%s) + From 569ea76d0a1a080f6e44880f8402095095d8566c Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Tue, 8 Aug 2023 17:57:45 +0200 Subject: [PATCH 02/57] Update moma.yml --- .github/workflows/moma.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index ce5e7e70..6af436a3 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -14,5 +14,5 @@ jobs: steps: - uses: actions/checkout@v3 - name: Build the dev Docker image - run: docker build . --file docker/scripts/dev.Dockerfile --tag moma_dev:$(date +%s) + run: docker build . --file ./docker/scripts/dev.Dockerfile --tag moma_dev:$(date +%s) From 09f653697debbbd93b8005c7c40c3a556c42cb44 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Tue, 8 Aug 2023 18:04:07 +0200 Subject: [PATCH 03/57] Still working on dockeeeer --- .github/workflows/moma.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index 6af436a3..9b4e2980 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -13,6 +13,14 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Build the dev Docker image - run: docker build . --file ./docker/scripts/dev.Dockerfile --tag moma_dev:$(date +%s) - + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - + name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + file: ./docker/scripts/dev.Dockerfile + push: false + tags: moma_dev:$(date +%s) From 60e797b77963da76b591f0797c0ff571ace4adda Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Tue, 8 Aug 2023 18:05:06 +0200 Subject: [PATCH 04/57] Update moma.yml --- .github/workflows/moma.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index 9b4e2980..67bedd29 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -23,4 +23,4 @@ jobs: context: . file: ./docker/scripts/dev.Dockerfile push: false - tags: moma_dev:$(date +%s) + tags: moma_dev:latest From 93f9c82eeaf3549f3ea4a04a527166ac3f6c641d Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Tue, 8 Aug 2023 18:06:53 +0200 Subject: [PATCH 05/57] aaah --- .github/workflows/moma.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index 67bedd29..e9caea35 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -20,7 +20,7 @@ jobs: name: Build and push uses: docker/build-push-action@v4 with: - context: . + context: ./docker/scripts/ file: ./docker/scripts/dev.Dockerfile push: false tags: moma_dev:latest From 2948c5257e53145c63b6e61f1737ad47df32eb00 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Tue, 8 Aug 2023 18:10:38 +0200 Subject: [PATCH 06/57] Update moma.yml --- .github/workflows/moma.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index e9caea35..068cba11 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -20,7 +20,7 @@ jobs: name: Build and push uses: docker/build-push-action@v4 with: - context: ./docker/scripts/ - file: ./docker/scripts/dev.Dockerfile + context: ./docker/ + file: ./docker/dev.Dockerfile push: false tags: moma_dev:latest From 3542f2816952881bd6fa4112ff1163dd3fac30a7 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 9 Aug 2023 11:30:30 +0200 Subject: [PATCH 07/57] Update moma.yml --- .github/workflows/moma.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index 068cba11..1db63837 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -7,20 +7,23 @@ on: branches: [ "master" ] jobs: - - build: + build-docker: runs-on: ubuntu-latest - steps: - uses: actions/checkout@v3 - - - name: Set up Docker Buildx + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - - name: Build and push + - name: Build and push docker uses: docker/build-push-action@v4 with: context: ./docker/ file: ./docker/dev.Dockerfile push: false tags: moma_dev:latest + build-catkin: + needs: build-docker + runs-on: ubuntu-latest + container: moma_dev:latest + steps: + - run: echo "Hello!" + - run: ls ~/ From 6d25913d893d1f54aeea8e2e8466c8ada44532e4 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 9 Aug 2023 11:31:55 +0200 Subject: [PATCH 08/57] Update moma.yml --- .github/workflows/moma.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index 1db63837..3abc582b 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -12,14 +12,14 @@ jobs: steps: - uses: actions/checkout@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v2 - name: Build and push docker - uses: docker/build-push-action@v4 - with: - context: ./docker/ - file: ./docker/dev.Dockerfile - push: false - tags: moma_dev:latest + uses: docker/build-push-action@v4 + with: + context: ./docker/ + file: ./docker/dev.Dockerfile + push: false + tags: moma_dev:latest build-catkin: needs: build-docker runs-on: ubuntu-latest From 7e4648d6930b1be59697ad3acb136424c07de299 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 9 Aug 2023 13:22:41 +0200 Subject: [PATCH 09/57] Update moma.yml --- .github/workflows/moma.yml | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index 3abc582b..23e59f82 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -1,4 +1,4 @@ -name: moma Docker +name: moma CI & Docker on: push: @@ -6,6 +6,12 @@ on: pull_request: branches: [ "master" ] +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + jobs: build-docker: runs-on: ubuntu-latest @@ -13,17 +19,31 @@ jobs: - uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build and push docker uses: docker/build-push-action@v4 with: context: ./docker/ file: ./docker/dev.Dockerfile - push: false - tags: moma_dev:latest + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max build-catkin: needs: build-docker runs-on: ubuntu-latest - container: moma_dev:latest + container: ${{ steps.meta.outputs.tags }} steps: - - run: echo "Hello!" + - run: echo "${{ steps.meta.outputs.tags }}" - run: ls ~/ From c981617493f7767a35970b8c82e45ceb2b9546bc Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 9 Aug 2023 13:24:17 +0200 Subject: [PATCH 10/57] Update moma.yml --- .github/workflows/moma.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index 23e59f82..ac456792 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -20,16 +20,16 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build and push docker uses: docker/build-push-action@v4 with: From 07775003c830e68e9bdaac39d0f78cb204b7647d Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 9 Aug 2023 13:26:16 +0200 Subject: [PATCH 11/57] Update moma.yml --- .github/workflows/moma.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index ac456792..c0135641 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -46,4 +46,4 @@ jobs: container: ${{ steps.meta.outputs.tags }} steps: - run: echo "${{ steps.meta.outputs.tags }}" - - run: ls ~/ +# - run: ls ~/ From ed204da31a0b5b694e77f7f526101890f742f501 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 9 Aug 2023 13:26:49 +0200 Subject: [PATCH 12/57] Update moma.yml --- .github/workflows/moma.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index c0135641..9428095a 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -43,7 +43,7 @@ jobs: build-catkin: needs: build-docker runs-on: ubuntu-latest - container: ${{ steps.meta.outputs.tags }} + container: ${{ build-docker.steps.meta.outputs.tags }} steps: - - run: echo "${{ steps.meta.outputs.tags }}" + - run: echo "${{ build-docker.steps.meta.outputs.tags }}" # - run: ls ~/ From 2f65f01749d44b933891d96bf33dc2eefdbbe10c Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 9 Aug 2023 13:29:20 +0200 Subject: [PATCH 13/57] Update moma.yml --- .github/workflows/moma.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index 9428095a..bf717c25 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -15,6 +15,8 @@ env: jobs: build-docker: runs-on: ubuntu-latest + outputs: + docker_image_name: ${{ steps.meta.outputs.tags }} steps: - uses: actions/checkout@v3 - name: Set up Docker Buildx @@ -40,10 +42,11 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max + build-catkin: needs: build-docker runs-on: ubuntu-latest - container: ${{ build-docker.steps.meta.outputs.tags }} + container: ${{ needs.build-docker.outputs.docker_image_name }} steps: - - run: echo "${{ build-docker.steps.meta.outputs.tags }}" -# - run: ls ~/ + - run: echo "${{ needs.build-docker.outputs.docker_image_name }}" + - run: ls ~/ From f76f7558e1bd3b915ff236d2f4c91595f8835b62 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 11:31:15 +0200 Subject: [PATCH 14/57] Actually try to build the repo. --- .github/workflows/moma.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index bf717c25..67ea8d82 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -18,7 +18,10 @@ jobs: outputs: docker_image_name: ${{ steps.meta.outputs.tags }} steps: - - uses: actions/checkout@v3 + - name: Check out moma repo + uses: actions/checkout@v3 + with: + path: moma_ws/src - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Log into registry ${{ env.REGISTRY }} @@ -42,11 +45,16 @@ jobs: labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - + build-catkin: needs: build-docker runs-on: ubuntu-latest - container: ${{ needs.build-docker.outputs.docker_image_name }} + container: + image: ${{ needs.build-docker.outputs.docker_image_name }} + volumes: + - $GITHUB_WORKSPACE/moma_ws:/root/moma_ws steps: - run: echo "${{ needs.build-docker.outputs.docker_image_name }}" - - run: ls ~/ + - run: echo $GITHUB_WORKSPACE + - run: cd ~/moma_ws && catkin init && catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release + - run: catkin build --continue \ No newline at end of file From da8888f26a6b7af87be14c0c2d9a42ce8d1a42c2 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 11:33:26 +0200 Subject: [PATCH 15/57] Fix paths? --- .github/workflows/moma.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index 67ea8d82..aa36e90b 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -21,7 +21,7 @@ jobs: - name: Check out moma repo uses: actions/checkout@v3 with: - path: moma_ws/src + path: moma_ws/src/moma - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Log into registry ${{ env.REGISTRY }} @@ -38,8 +38,8 @@ jobs: - name: Build and push docker uses: docker/build-push-action@v4 with: - context: ./docker/ - file: ./docker/dev.Dockerfile + context: ./moma_ws/src/moma/docker/ + file: ./moma_ws/src/moma/docker/dev.Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From f85db6abd2f08ee6134c8832acb2cce90180824e Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 13:37:44 +0200 Subject: [PATCH 16/57] Fix paths again --- .github/workflows/moma.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index aa36e90b..c60f46c5 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -52,9 +52,9 @@ jobs: container: image: ${{ needs.build-docker.outputs.docker_image_name }} volumes: - - $GITHUB_WORKSPACE/moma_ws:/root/moma_ws + - ${{ github.workspace }}/moma_ws:/root/moma_ws steps: - run: echo "${{ needs.build-docker.outputs.docker_image_name }}" - - run: echo $GITHUB_WORKSPACE + - run: echo "${{ github.workspace }}" - run: cd ~/moma_ws && catkin init && catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release - run: catkin build --continue \ No newline at end of file From effad71e5cc7400aaef86e3ce7dec17efcf8098e Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 13:40:52 +0200 Subject: [PATCH 17/57] The path fixing continues --- .github/workflows/moma.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index c60f46c5..8aead320 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -56,5 +56,5 @@ jobs: steps: - run: echo "${{ needs.build-docker.outputs.docker_image_name }}" - run: echo "${{ github.workspace }}" - - run: cd ~/moma_ws && catkin init && catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release + - run: cd /root/moma_ws && catkin init && catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release - run: catkin build --continue \ No newline at end of file From d3c9407066743b4aaaae6ba3985bd09129b19cfd Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 13:51:44 +0200 Subject: [PATCH 18/57] The path fixing continues even more --- .github/workflows/moma.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index 8aead320..cd522656 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -22,6 +22,7 @@ jobs: uses: actions/checkout@v3 with: path: moma_ws/src/moma + submodules: recursive - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Log into registry ${{ env.REGISTRY }} @@ -53,8 +54,13 @@ jobs: image: ${{ needs.build-docker.outputs.docker_image_name }} volumes: - ${{ github.workspace }}/moma_ws:/root/moma_ws + - ${{ github.workspace }}/moma_ws:/github/home/moma_ws steps: - run: echo "${{ needs.build-docker.outputs.docker_image_name }}" - run: echo "${{ github.workspace }}" - - run: cd /root/moma_ws && catkin init && catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release - - run: catkin build --continue \ No newline at end of file + - name: Set up Catkin workspace + run: cd ~/moma_ws && catkin init && catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release + shell: bash + - name: Build the workspace + run: cd ~/moma_ws && catkin build --continue + shell: bash \ No newline at end of file From 7954d17ff7680be72efe71c548460f767c8c41af Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 14:00:54 +0200 Subject: [PATCH 19/57] Debugggg --- .github/workflows/moma.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index cd522656..8a885a1c 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -58,9 +58,11 @@ jobs: steps: - run: echo "${{ needs.build-docker.outputs.docker_image_name }}" - run: echo "${{ github.workspace }}" + - run: ls -la /root/moma_ws + - run: cat /root/.bashrc - name: Set up Catkin workspace - run: cd ~/moma_ws && catkin init && catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release + run: cd /root/moma_ws && catkin init && catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release shell: bash - name: Build the workspace - run: cd ~/moma_ws && catkin build --continue + run: cd /root/moma_ws && catkin build --continue shell: bash \ No newline at end of file From 9e2225cd51ad537261d69ddcc6ccd7ddb1c5e17f Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 14:13:00 +0200 Subject: [PATCH 20/57] I don't understand jobs apparently. --- .github/workflows/moma.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index 8a885a1c..0cd539df 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -52,14 +52,18 @@ jobs: runs-on: ubuntu-latest container: image: ${{ needs.build-docker.outputs.docker_image_name }} - volumes: - - ${{ github.workspace }}/moma_ws:/root/moma_ws - - ${{ github.workspace }}/moma_ws:/github/home/moma_ws steps: - run: echo "${{ needs.build-docker.outputs.docker_image_name }}" - run: echo "${{ github.workspace }}" - run: ls -la /root/moma_ws - run: cat /root/.bashrc + - name: Check out moma repo + uses: actions/checkout@v3 + with: + path: moma_ws/src/moma + submodules: recursive + - run: ls + shell: bash - name: Set up Catkin workspace run: cd /root/moma_ws && catkin init && catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release shell: bash From 66f876fc5b10032960abe90872d976390a58f453 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 14:15:59 +0200 Subject: [PATCH 21/57] Debug the reckoning --- .github/workflows/moma.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index 0cd539df..c3f5c6d3 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -55,8 +55,6 @@ jobs: steps: - run: echo "${{ needs.build-docker.outputs.docker_image_name }}" - run: echo "${{ github.workspace }}" - - run: ls -la /root/moma_ws - - run: cat /root/.bashrc - name: Check out moma repo uses: actions/checkout@v3 with: From 616623c0eabb46fdb3c232b9f4a980ea54561eb6 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 14:23:42 +0200 Subject: [PATCH 22/57] ???? --- .github/workflows/moma.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index c3f5c6d3..9da5c888 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -11,6 +11,7 @@ env: REGISTRY: ghcr.io # github.repository as / IMAGE_NAME: ${{ github.repository }} + CATKIN_WS_PATH: /__w/moma/moma/moma_ws jobs: build-docker: @@ -63,8 +64,15 @@ jobs: - run: ls shell: bash - name: Set up Catkin workspace - run: cd /root/moma_ws && catkin init && catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release + run: | + cd ${{ env.CATKIN_WS_PATH }} + source /root/.bashrc + catkin init + catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release shell: bash - name: Build the workspace - run: cd /root/moma_ws && catkin build --continue + run: | + cd ${{ env.CATKIN_WS_PATH }} | + source /root/.bashrc + catkin build --continue shell: bash \ No newline at end of file From 902edc0c55cba5ab76c744c436d63a73893ad11a Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 14:28:14 +0200 Subject: [PATCH 23/57] ?? --- .github/workflows/moma.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/moma.yml b/.github/workflows/moma.yml index 9da5c888..3eb1f751 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/moma.yml @@ -68,11 +68,12 @@ jobs: cd ${{ env.CATKIN_WS_PATH }} source /root/.bashrc catkin init + catkin config --extend /root/moma_dep_ws/devel catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release shell: bash - name: Build the workspace run: | - cd ${{ env.CATKIN_WS_PATH }} | + cd ${{ env.CATKIN_WS_PATH }} source /root/.bashrc catkin build --continue shell: bash \ No newline at end of file From 4bfff1685ba75119c1f2b430eaf5ede54fbc2535 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 14:45:25 +0200 Subject: [PATCH 24/57] Remove yumi submodule --- .gitmodules | 4 ---- submodules/yumi | 1 - 2 files changed, 5 deletions(-) delete mode 160000 submodules/yumi diff --git a/.gitmodules b/.gitmodules index 6d542ae3..fbe14659 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule "submodules/yumi"] - path = submodules/yumi - url = git@github.com:ethz-asl/yumi.git - branch = asl-devel [submodule "submodules/roboticsgroup_gazebo_plugins"] path = submodules/roboticsgroup_gazebo_plugins url = git@github.com:roboticsgroup/roboticsgroup_gazebo_plugins.git diff --git a/submodules/yumi b/submodules/yumi deleted file mode 160000 index 18a2b064..00000000 --- a/submodules/yumi +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 18a2b06413389b00a09c6a525569f4e106923780 From 919f3c39cae75f1c72b82ccbce2dc790367d5ccf Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 14:49:51 +0200 Subject: [PATCH 25/57] Try this stuff again --- docker/scripts/install_sys_deps.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker/scripts/install_sys_deps.sh b/docker/scripts/install_sys_deps.sh index 4c17e074..f4d312be 100644 --- a/docker/scripts/install_sys_deps.sh +++ b/docker/scripts/install_sys_deps.sh @@ -16,9 +16,5 @@ apt-get -qq update && apt-get install -y python3-catkin-tools python3-vcstool py # Install other system deps apt-get -qq update && apt-get install -y qtbase5-dev bash-completion -# This is for nvblox -pip install --upgrade cmake -apt-get -qq update && apt-get install -y libgoogle-glog-dev libgtest-dev libgflags-dev python3-dev libsqlite3-dev - # Clear cache to keep layer size down rm -rf /var/lib/apt/lists/* From ba78fcfa9665f84d1519fa6242ba01e7f2f197f6 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 15:13:52 +0200 Subject: [PATCH 26/57] Split into CI CD --- .github/workflows/{moma.yml => cd.yml} | 3 +- .github/workflows/ci.yml | 125 +++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 2 deletions(-) rename .github/workflows/{moma.yml => cd.yml} (97%) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/moma.yml b/.github/workflows/cd.yml similarity index 97% rename from .github/workflows/moma.yml rename to .github/workflows/cd.yml index 3eb1f751..08bc95e2 100644 --- a/.github/workflows/moma.yml +++ b/.github/workflows/cd.yml @@ -1,4 +1,4 @@ -name: moma CI & Docker +name: moma CD on: push: @@ -11,7 +11,6 @@ env: REGISTRY: ghcr.io # github.repository as / IMAGE_NAME: ${{ github.repository }} - CATKIN_WS_PATH: /__w/moma/moma/moma_ws jobs: build-docker: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..c62ac460 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,125 @@ +name: moma CI + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + CATKIN_WS_PATH: moma_ws + +jobs: + build-dev: + name: Build dev docker image + runs-on: ubuntu-latest + outputs: + docker_image_name: ${{ steps.meta.outputs.tags }} + steps: + - name: Check out moma repo + uses: actions/checkout@v3 + with: + path: moma_ws/src/moma + submodules: recursive + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + flavor: prefix=dev + - name: Build and push docker + uses: docker/build-push-action@v4 + with: + context: ./moma_ws/src/moma/docker/ + file: ./moma_ws/src/moma/docker/dev.Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-robot: + name: Build robot docker image + runs-on: ubuntu-latest + outputs: + docker_image_name: ${{ steps.meta.outputs.tags }} + steps: + - name: Check out moma repo + uses: actions/checkout@v3 + with: + path: moma_ws/src/moma + submodules: recursive + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + flavor: prefix=robot + - name: Build and push docker + uses: docker/build-push-action@v4 + with: + context: ./moma_ws/src/moma/docker/ + file: ./moma_ws/src/moma/docker/robot.Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Checks out the demo branch and builds a docker image of that. + build-demo: + name: Build robot docker image + runs-on: ubuntu-latest + outputs: + docker_image_name: ${{ steps.meta.outputs.tags }} + steps: + - name: Check out moma repo + uses: actions/checkout@v3 + with: + ref: demo + path: moma_ws/src/moma + submodules: recursive + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + flavor: prefix=demo + - name: Build and push docker + uses: docker/build-push-action@v4 + with: + context: ./moma_ws/src/moma/docker/ + file: ./moma_ws/src/moma/docker/demo.Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max From b1a6a3b8a77f0aada6e694100a269d919e0204e0 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 15:21:22 +0200 Subject: [PATCH 27/57] MAtrix it up --- .github/workflows/cd.yml | 45 ++++--------------- .github/workflows/ci.yml | 96 +++++++++++----------------------------- 2 files changed, 34 insertions(+), 107 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 08bc95e2..2cd1ff86 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,4 +1,4 @@ -name: moma CD +name: moma CI on: push: @@ -13,10 +13,12 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build-docker: + build-dockers: + name: Build docker images + strategy: + matrix: + dockerfile: [dev, robot, demo] runs-on: ubuntu-latest - outputs: - docker_image_name: ${{ steps.meta.outputs.tags }} steps: - name: Check out moma repo uses: actions/checkout@v3 @@ -33,46 +35,17 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Extract Docker metadata id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + uses: docker/metadata-action@v4 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + flavor: prefix=${{ matrix.dockerfile }} - name: Build and push docker uses: docker/build-push-action@v4 with: context: ./moma_ws/src/moma/docker/ - file: ./moma_ws/src/moma/docker/dev.Dockerfile + file: ./moma_ws/src/moma/docker/${{ matrix.dockerfile }}.Dockerfile push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - - build-catkin: - needs: build-docker - runs-on: ubuntu-latest - container: - image: ${{ needs.build-docker.outputs.docker_image_name }} - steps: - - run: echo "${{ needs.build-docker.outputs.docker_image_name }}" - - run: echo "${{ github.workspace }}" - - name: Check out moma repo - uses: actions/checkout@v3 - with: - path: moma_ws/src/moma - submodules: recursive - - run: ls - shell: bash - - name: Set up Catkin workspace - run: | - cd ${{ env.CATKIN_WS_PATH }} - source /root/.bashrc - catkin init - catkin config --extend /root/moma_dep_ws/devel - catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release - shell: bash - - name: Build the workspace - run: | - cd ${{ env.CATKIN_WS_PATH }} - source /root/.bashrc - catkin build --continue - shell: bash \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c62ac460..8367ad0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: moma CI +name: moma CD on: push: @@ -14,8 +14,7 @@ env: CATKIN_WS_PATH: moma_ws jobs: - build-dev: - name: Build dev docker image + build-docker: runs-on: ubuntu-latest outputs: docker_image_name: ${{ steps.meta.outputs.tags }} @@ -35,10 +34,9 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Extract Docker metadata id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - flavor: prefix=dev - name: Build and push docker uses: docker/build-push-action@v4 with: @@ -50,76 +48,32 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - build-robot: - name: Build robot docker image + build-catkin: + needs: build-docker runs-on: ubuntu-latest - outputs: - docker_image_name: ${{ steps.meta.outputs.tags }} - steps: - - name: Check out moma repo - uses: actions/checkout@v3 - with: - path: moma_ws/src/moma - submodules: recursive - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - flavor: prefix=robot - - name: Build and push docker - uses: docker/build-push-action@v4 - with: - context: ./moma_ws/src/moma/docker/ - file: ./moma_ws/src/moma/docker/robot.Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - - # Checks out the demo branch and builds a docker image of that. - build-demo: - name: Build robot docker image - runs-on: ubuntu-latest - outputs: - docker_image_name: ${{ steps.meta.outputs.tags }} + container: + image: ${{ needs.build-docker.outputs.docker_image_name }} steps: + - run: echo "${{ needs.build-docker.outputs.docker_image_name }}" + - run: echo "${{ github.workspace }}" - name: Check out moma repo uses: actions/checkout@v3 with: - ref: demo path: moma_ws/src/moma submodules: recursive - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v4 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - flavor: prefix=demo - - name: Build and push docker - uses: docker/build-push-action@v4 - with: - context: ./moma_ws/src/moma/docker/ - file: ./moma_ws/src/moma/docker/demo.Dockerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + - run: ls + shell: bash + - name: Set up Catkin workspace + run: | + cd ${{ env.CATKIN_WS_PATH }} + source /root/.bashrc + catkin init + catkin config --extend /root/moma_dep_ws/devel + catkin config --cmake-args -DCMAKE_BUILD_TYPE=Release + shell: bash + - name: Build the workspace + run: | + cd ${{ env.CATKIN_WS_PATH }} + source /root/.bashrc + catkin build --continue + shell: bash \ No newline at end of file From ee775c5ebed15685edde8e64ea63d2965edba393 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 15:22:13 +0200 Subject: [PATCH 28/57] Make names match up. --- .github/workflows/cd.yml | 2 +- .github/workflows/ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 2cd1ff86..37162525 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,4 +1,4 @@ -name: moma CI +name: moma CD on: push: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8367ad0e..a77e297b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: moma CD +name: moma CI on: push: From 953f769a441ca88621c4c92ba6977fd2eb6695d7 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 15:28:46 +0200 Subject: [PATCH 29/57] tags??? --- .github/workflows/cd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 37162525..f887d040 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -39,6 +39,7 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} flavor: prefix=${{ matrix.dockerfile }} + tags: type=raw,value=${{ matrix.dockerfile }} - name: Build and push docker uses: docker/build-push-action@v4 with: From 5626a414eda7046438444f718679fab257e70f7f Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 15:33:32 +0200 Subject: [PATCH 30/57] MORE TAGS --- .github/workflows/cd.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index f887d040..308cc181 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -38,7 +38,6 @@ jobs: uses: docker/metadata-action@v4 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - flavor: prefix=${{ matrix.dockerfile }} tags: type=raw,value=${{ matrix.dockerfile }} - name: Build and push docker uses: docker/build-push-action@v4 From c937df31ce1a1fbef6fe12d7cfacf8d72ed514ea Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Thu, 10 Aug 2023 15:53:08 +0200 Subject: [PATCH 31/57] Fix demo docker. --- docker/scripts/install_demo.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/scripts/install_demo.sh b/docker/scripts/install_demo.sh index 7e80d09f..815a7268 100644 --- a/docker/scripts/install_demo.sh +++ b/docker/scripts/install_demo.sh @@ -16,6 +16,7 @@ pip install -r vgn/requirements.txt pip install gdown # Make sure we can unzip too +apt-get update apt-get install unzip # Download the data file from google drive From 12b8059d1da33ce2c58d30e7cc45da4d65d3418f Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Fri, 11 Aug 2023 12:03:12 +0200 Subject: [PATCH 32/57] Make super moma just pulls the demo branch. --- docker/scripts/moma_demo.repos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/scripts/moma_demo.repos b/docker/scripts/moma_demo.repos index 5a10c139..565be57d 100644 --- a/docker/scripts/moma_demo.repos +++ b/docker/scripts/moma_demo.repos @@ -10,4 +10,4 @@ repositories: moma: type: git url: https://github.com/ethz-asl/moma.git - version: 6730e11ca580dd80fb49a553db5394111bf232ff + version: demo From c0ae126f04797aeda6ad487fa934fe7b94c6a731 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Fri, 11 Aug 2023 13:40:49 +0200 Subject: [PATCH 33/57] Hopefully fix the torch downloading issue --- docker/scripts/install_demo.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/scripts/install_demo.sh b/docker/scripts/install_demo.sh index 815a7268..b0af3672 100644 --- a/docker/scripts/install_demo.sh +++ b/docker/scripts/install_demo.sh @@ -9,8 +9,11 @@ vcs import --recursive --input $SCRIPTS_PATH/moma_demo.repos # Upgrade numpy version pip install --upgrade numpy==1.23.0 +# This helps torch download on Github Actions :) +export TMPDIR='/var/tmp' + # Pip install VGN requirements -pip install -r vgn/requirements.txt +pip install --cache-dir=$TMPDIR --build $TMPDIR -r vgn/requirements.txt # Pip install gdown to get google drive files pip install gdown From e1cc6e70c350544986b4c118251d368ded5b3035 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Fri, 11 Aug 2023 14:21:08 +0200 Subject: [PATCH 34/57] Try self-hosted runner --- .github/workflows/cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 308cc181..495ec754 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: dockerfile: [dev, robot, demo] - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Check out moma repo uses: actions/checkout@v3 From 93bbf60a741f61f07a96c4fcac795ae6b4497d0c Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Fri, 11 Aug 2023 14:30:42 +0200 Subject: [PATCH 35/57] Remove submodules from the checkout. --- .github/workflows/cd.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 495ec754..07fda9ac 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -24,7 +24,8 @@ jobs: uses: actions/checkout@v3 with: path: moma_ws/src/moma - submodules: recursive + - name: Workaround for older git client on self-hosted runners + run: cd moma_ws/src/moma && git submodule update --init --recursive - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Log into registry ${{ env.REGISTRY }} From 1e0cf7138c0c06fa3577c79920e92d43931e6ae5 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Fri, 11 Aug 2023 14:37:10 +0200 Subject: [PATCH 36/57] The confusion continues --- .github/workflows/cd.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 07fda9ac..d2a917a9 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -25,7 +25,10 @@ jobs: with: path: moma_ws/src/moma - name: Workaround for older git client on self-hosted runners - run: cd moma_ws/src/moma && git submodule update --init --recursive + run: | + ls + pwd + cd moma_ws/src/moma && git submodule update --init --recursive - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Log into registry ${{ env.REGISTRY }} From dbe9dd96b8a37e3bd186d9c09ff67292a12a461a Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Fri, 11 Aug 2023 14:38:20 +0200 Subject: [PATCH 37/57] ?????? --- .github/workflows/cd.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index d2a917a9..59629e0c 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -28,7 +28,9 @@ jobs: run: | ls pwd - cd moma_ws/src/moma && git submodule update --init --recursive + cd moma_ws/src/moma + ls + git submodule update --init --recursive - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Log into registry ${{ env.REGISTRY }} From b2a220c8b80162d9f47237f117ba66d870583f42 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Fri, 11 Aug 2023 14:49:39 +0200 Subject: [PATCH 38/57] ??? --- .github/workflows/cd.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 59629e0c..12e0a1ff 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -20,6 +20,8 @@ jobs: dockerfile: [dev, robot, demo] runs-on: self-hosted steps: + - name: Fix other git issues + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Check out moma repo uses: actions/checkout@v3 with: From 232bccc39fbfe2c35402db066b48099837ba421d Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Fri, 11 Aug 2023 15:29:33 +0200 Subject: [PATCH 39/57] ..... --- .github/workflows/cd.yml | 12 ++---------- docker/scripts/install_demo.sh | 1 + 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 12e0a1ff..fc3145e7 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -18,21 +18,13 @@ jobs: strategy: matrix: dockerfile: [dev, robot, demo] - runs-on: self-hosted + runs-on: ubuntu-latest steps: - - name: Fix other git issues - run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Check out moma repo uses: actions/checkout@v3 with: path: moma_ws/src/moma - - name: Workaround for older git client on self-hosted runners - run: | - ls - pwd - cd moma_ws/src/moma - ls - git submodule update --init --recursive + submodule: recursive - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Log into registry ${{ env.REGISTRY }} diff --git a/docker/scripts/install_demo.sh b/docker/scripts/install_demo.sh index b0af3672..91238938 100644 --- a/docker/scripts/install_demo.sh +++ b/docker/scripts/install_demo.sh @@ -14,6 +14,7 @@ export TMPDIR='/var/tmp' # Pip install VGN requirements pip install --cache-dir=$TMPDIR --build $TMPDIR -r vgn/requirements.txt +pip install --no-cache-dir torch # Pip install gdown to get google drive files pip install gdown From eff687b5d82532cf5fcce5babe9e1124b6f0acc8 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Fri, 11 Aug 2023 16:19:45 +0200 Subject: [PATCH 40/57] Clean up disk??? --- .github/workflows/cd.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index fc3145e7..cf751b40 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -20,6 +20,12 @@ jobs: dockerfile: [dev, robot, demo] runs-on: ubuntu-latest steps: + - name: Maximize build space + uses: easimon/maximize-build-space@v7 + with: + root-reserve-mb: 512 + swap-size-mb: 1024 + remove-dotnet: 'true' - name: Check out moma repo uses: actions/checkout@v3 with: From 1e8d4f1bf162396f0e89398ed26b1dfd71b8b6fc Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Fri, 11 Aug 2023 16:38:43 +0200 Subject: [PATCH 41/57] Recursive --- .github/workflows/cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index cf751b40..1b99c1f7 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -30,7 +30,7 @@ jobs: uses: actions/checkout@v3 with: path: moma_ws/src/moma - submodule: recursive + submodules: recursive - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Log into registry ${{ env.REGISTRY }} From 850c24f8b564b47dfb45308c9cbc41482c55a221 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Mon, 14 Aug 2023 14:49:51 +0200 Subject: [PATCH 42/57] Try again to get pip to build ok with limtied space. --- docker/scripts/install_demo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/scripts/install_demo.sh b/docker/scripts/install_demo.sh index 91238938..e4b53e9c 100644 --- a/docker/scripts/install_demo.sh +++ b/docker/scripts/install_demo.sh @@ -13,7 +13,7 @@ pip install --upgrade numpy==1.23.0 export TMPDIR='/var/tmp' # Pip install VGN requirements -pip install --cache-dir=$TMPDIR --build $TMPDIR -r vgn/requirements.txt +pip install --no-cache-dir -r vgn/requirements.txt pip install --no-cache-dir torch # Pip install gdown to get google drive files From 36aaac727bc251742ea12fd7e4b3bda58196fdb9 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Mon, 14 Aug 2023 15:30:16 +0200 Subject: [PATCH 43/57] Remove maximizing build space since it seems to mess stuff up. --- .github/workflows/cd.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 1b99c1f7..308cc181 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -20,12 +20,6 @@ jobs: dockerfile: [dev, robot, demo] runs-on: ubuntu-latest steps: - - name: Maximize build space - uses: easimon/maximize-build-space@v7 - with: - root-reserve-mb: 512 - swap-size-mb: 1024 - remove-dotnet: 'true' - name: Check out moma repo uses: actions/checkout@v3 with: From 875c5ed7494437cd9a4cedcd10984db4b9c3268b Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Mon, 14 Aug 2023 16:31:18 +0200 Subject: [PATCH 44/57] Change CD only on push. --- .github/workflows/cd.yml | 2 -- .github/workflows/ci.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 308cc181..d1e4b52a 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -3,8 +3,6 @@ name: moma CD on: push: branches: [ "master" ] - pull_request: - branches: [ "master" ] env: # Use docker.io for Docker Hub if empty diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a77e297b..fb3e992b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,8 +54,6 @@ jobs: container: image: ${{ needs.build-docker.outputs.docker_image_name }} steps: - - run: echo "${{ needs.build-docker.outputs.docker_image_name }}" - - run: echo "${{ github.workspace }}" - name: Check out moma repo uses: actions/checkout@v3 with: From e9d24c9346c6b0bcbdd6aaa65eece7582eb4bced Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Tue, 15 Aug 2023 16:51:35 +0200 Subject: [PATCH 45/57] Try out the self-hosted stuff. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb3e992b..2e302eba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ env: jobs: build-docker: - runs-on: ubuntu-latest + runs-on: [self-hosted, linux] outputs: docker_image_name: ${{ steps.meta.outputs.tags }} steps: @@ -50,7 +50,7 @@ jobs: build-catkin: needs: build-docker - runs-on: ubuntu-latest + runs-on: [self-hosted, linux] container: image: ${{ needs.build-docker.outputs.docker_image_name }} steps: From c2ee1da7b13a1bef9eb43b23221e654c4506b564 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 16 Aug 2023 14:05:27 +0200 Subject: [PATCH 46/57] Add safe directory --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e302eba..dc867662 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,8 @@ jobs: with: path: moma_ws/src/moma submodules: recursive + - name: Git safe directory + run: git config --system --add safe.directory . - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Log into registry ${{ env.REGISTRY }} From 11e4bd7920a3cb2686dd3a61a5310bfeb9048a3d Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 16 Aug 2023 14:06:43 +0200 Subject: [PATCH 47/57] Put the safe directory in the correct place. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc867662..cc131d26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,8 +24,6 @@ jobs: with: path: moma_ws/src/moma submodules: recursive - - name: Git safe directory - run: git config --system --add safe.directory . - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Log into registry ${{ env.REGISTRY }} @@ -56,6 +54,8 @@ jobs: container: image: ${{ needs.build-docker.outputs.docker_image_name }} steps: + - name: Git safe directory (only for running within containers) + run: git config --system --add safe.directory . - name: Check out moma repo uses: actions/checkout@v3 with: From ed6418808f10ed08eb3d47b383f917fe4b387ece Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 16 Aug 2023 14:24:36 +0200 Subject: [PATCH 48/57] Hack workaround for safe directory --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc131d26..809651dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: image: ${{ needs.build-docker.outputs.docker_image_name }} steps: - name: Git safe directory (only for running within containers) - run: git config --system --add safe.directory . + run: git config --system --add safe.directory '*' - name: Check out moma repo uses: actions/checkout@v3 with: From 522442b8d89c2b287d37e8c3ca0088af67c5f685 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 16 Aug 2023 14:39:15 +0200 Subject: [PATCH 49/57] Switch back to github-hosted dockers. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 809651dd..53d4a57a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ env: jobs: build-docker: - runs-on: [self-hosted, linux] + runs-on: ubuntu-latest outputs: docker_image_name: ${{ steps.meta.outputs.tags }} steps: @@ -50,7 +50,7 @@ jobs: build-catkin: needs: build-docker - runs-on: [self-hosted, linux] + runs-on: ubuntu-latest container: image: ${{ needs.build-docker.outputs.docker_image_name }} steps: From d9112ae5652e91422bd23bf0a2c89a6bad3c3f25 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Tue, 10 Oct 2023 15:29:47 +0200 Subject: [PATCH 50/57] Make an attempt to get the demo docker to build properly. --- docker/scripts/install_demo.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/scripts/install_demo.sh b/docker/scripts/install_demo.sh index e4b53e9c..752a76e7 100644 --- a/docker/scripts/install_demo.sh +++ b/docker/scripts/install_demo.sh @@ -7,7 +7,7 @@ cd $MOMA_DEP_WS/src || exit 1 vcs import --recursive --input $SCRIPTS_PATH/moma_demo.repos # Upgrade numpy version -pip install --upgrade numpy==1.23.0 +pip install --no-cache-dir --upgrade numpy==1.23.0 # This helps torch download on Github Actions :) export TMPDIR='/var/tmp' @@ -30,6 +30,7 @@ gdown 1MysYHve3ooWiLq12b58Nm8FWiFBMH-bJ unzip data.zip -d vgn_data mv vgn_data/data/* vgn/assets/ rm -r vgn_data +rm data.zip # Add a rundemo alias: echo 'alias rundemo="roslaunch grasp_demo grasp_demo.launch launch_rviz:=true"' >> ~/.bashrc From 95392e1f60897d0f2a6e9a760e96de0ac6a7f0f1 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Tue, 10 Oct 2023 15:31:02 +0200 Subject: [PATCH 51/57] Re-force CD to run again. --- .github/workflows/cd.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index d1e4b52a..308cc181 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -3,6 +3,8 @@ name: moma CD on: push: branches: [ "master" ] + pull_request: + branches: [ "master" ] env: # Use docker.io for Docker Hub if empty From 292e4c5baab8efe5d935f1f6b893f9b872b786fb Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Tue, 10 Oct 2023 15:49:07 +0200 Subject: [PATCH 52/57] Try to reduce number of pip requirements. --- docker/scripts/install_demo.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/scripts/install_demo.sh b/docker/scripts/install_demo.sh index 752a76e7..5110e910 100644 --- a/docker/scripts/install_demo.sh +++ b/docker/scripts/install_demo.sh @@ -13,8 +13,9 @@ pip install --no-cache-dir --upgrade numpy==1.23.0 export TMPDIR='/var/tmp' # Pip install VGN requirements -pip install --no-cache-dir -r vgn/requirements.txt -pip install --no-cache-dir torch +pip install --no-cache-dir catkin_pkg scipy pandas matplotlib mpi4py open3d pybullet==2.7.9 tqdm +pip cache purge +pip install --no-cache-dir torch pytorch-ignite tensorboard # Pip install gdown to get google drive files pip install gdown From e4cba284c4f876f81ec6854c10561cfa95741f9e Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 11 Oct 2023 11:13:51 +0200 Subject: [PATCH 53/57] We try again, without CUDA this time. --- docker/scripts/install_demo.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/scripts/install_demo.sh b/docker/scripts/install_demo.sh index 5110e910..809a24f8 100644 --- a/docker/scripts/install_demo.sh +++ b/docker/scripts/install_demo.sh @@ -13,9 +13,10 @@ pip install --no-cache-dir --upgrade numpy==1.23.0 export TMPDIR='/var/tmp' # Pip install VGN requirements -pip install --no-cache-dir catkin_pkg scipy pandas matplotlib mpi4py open3d pybullet==2.7.9 tqdm +pip install --no-cache-dir catkin_pkg scipy pandas==0.22.0 matplotlib mpi4py open3d pybullet==2.7.9 tqdm pip cache purge -pip install --no-cache-dir torch pytorch-ignite tensorboard +pip install --no-cache-dir torch==1.2.0+cpu -f https://download.pytorch.org/whl/torch_stable.html +pip install tensorboard #pytorch-ignite # Pip install gdown to get google drive files pip install gdown From 0b7d5257fd86ef8f3ffe12f8103865d0792e337b Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 11 Oct 2023 11:21:26 +0200 Subject: [PATCH 54/57] Newer torch --- docker/scripts/install_demo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/scripts/install_demo.sh b/docker/scripts/install_demo.sh index 809a24f8..f7f8d6fd 100644 --- a/docker/scripts/install_demo.sh +++ b/docker/scripts/install_demo.sh @@ -15,7 +15,7 @@ export TMPDIR='/var/tmp' # Pip install VGN requirements pip install --no-cache-dir catkin_pkg scipy pandas==0.22.0 matplotlib mpi4py open3d pybullet==2.7.9 tqdm pip cache purge -pip install --no-cache-dir torch==1.2.0+cpu -f https://download.pytorch.org/whl/torch_stable.html +pip install --no-cache-dir torch==1.12.0+cpu -f https://download.pytorch.org/whl/torch_stable.html pip install tensorboard #pytorch-ignite # Pip install gdown to get google drive files From 954150bbeb06801d9789462eaddec0cdb8ce58b8 Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 11 Oct 2023 14:48:34 +0200 Subject: [PATCH 55/57] Please open3d --- docker/scripts/install_demo.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/scripts/install_demo.sh b/docker/scripts/install_demo.sh index f7f8d6fd..00633c24 100644 --- a/docker/scripts/install_demo.sh +++ b/docker/scripts/install_demo.sh @@ -13,10 +13,11 @@ pip install --no-cache-dir --upgrade numpy==1.23.0 export TMPDIR='/var/tmp' # Pip install VGN requirements -pip install --no-cache-dir catkin_pkg scipy pandas==0.22.0 matplotlib mpi4py open3d pybullet==2.7.9 tqdm -pip cache purge +pip install --no-cache-dir catkin_pkg scipy matplotlib mpi4py pybullet==2.7.9 tqdm pip install --no-cache-dir torch==1.12.0+cpu -f https://download.pytorch.org/whl/torch_stable.html -pip install tensorboard #pytorch-ignite +pip install --no-cache-dir tensorboard +pip install --no-cache-dir open3d-cpu==0.17.0 + # Pip install gdown to get google drive files pip install gdown From 42fa4ce237d5eec8d07f7be62b074e796e0dd27c Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 11 Oct 2023 15:47:37 +0200 Subject: [PATCH 56/57] 700th time is the charm --- docker/scripts/install_demo.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/scripts/install_demo.sh b/docker/scripts/install_demo.sh index 00633c24..00f265ec 100644 --- a/docker/scripts/install_demo.sh +++ b/docker/scripts/install_demo.sh @@ -16,11 +16,10 @@ export TMPDIR='/var/tmp' pip install --no-cache-dir catkin_pkg scipy matplotlib mpi4py pybullet==2.7.9 tqdm pip install --no-cache-dir torch==1.12.0+cpu -f https://download.pytorch.org/whl/torch_stable.html pip install --no-cache-dir tensorboard -pip install --no-cache-dir open3d-cpu==0.17.0 - +pip install --no-cache-dir open3d==0.13.0 # Pip install gdown to get google drive files -pip install gdown +pip install --no-cache-dir gdown # Make sure we can unzip too apt-get update From 95f228199c05841af93fe26a6066030215de904c Mon Sep 17 00:00:00 2001 From: Helen Oleynikova Date: Wed, 11 Oct 2023 17:05:32 +0200 Subject: [PATCH 57/57] Switch CD back to only building on push. --- .github/workflows/cd.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 308cc181..d1e4b52a 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -3,8 +3,6 @@ name: moma CD on: push: branches: [ "master" ] - pull_request: - branches: [ "master" ] env: # Use docker.io for Docker Hub if empty