Skip to content

Commit f15d54d

Browse files
mattkjames7mattkjames7
authored
Toolchain v6 (#556)
* use v6 of toolchain to build mage, added simple test script to run build manually * remove wget * run tests as memgraph not root * change runner to arm if using arm arch * remove qemu * added docker script for testing within mgbuild * added manual package script for testing * fixes to test scripts for arm * parse arch * use correct ubuntu version * fix arm tag * modify pip call * use new package in test dockerfile * updated dockerfiles * fix test * remove unused script * updated test to set MGBUILD_IMAGE * updated test to set MGBUILD_IMAGE * updated dockerfile * use bash * undo change to dockerfile * undo change to dockerfile * don't remove pip * missing equals * use valid wheel name * added missing argument in workflow * fixes to workflow * use self hosted runners * make push to github optional * updated output path * use self-hosted runners for building and testing mage * correct the ARM64 label * fix python package used in workflow * added arch to PR test matrix * updated to handle both arm and arm64 * try to make tests more consistently use amd64 and arm64 instead of a mixture or arm/arm64 and amd/amd64 * remove toolchain after building * remove unused dockerfile * undo change to pytest.ini * fix spelling in workflow * remove commented code * added comment * removed commented code * complete cleanup * added step to fetch current version of memgraph * update default workflow values * use deps bucket if rc version is being tested * modify defaults * install jq * install jq * use python instead of jq * remove "v" from version * update dockerfiles * handle rc version naming * handle rc version naming * handle rc version naming
1 parent 98252b3 commit f15d54d

12 files changed

+609
-52
lines changed

‎.github/workflows/package_mage.yaml

+4-4
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@ on:
99
mage_build_arch:
1010
type: choice
1111
description: "Mage build architecture"
12-
default: 'amd'
12+
default: 'amd64'
1313
options:
14-
- "amd"
15-
- "arm"
14+
- "amd64"
15+
- "arm64"
1616
mage_build_scope:
1717
type: choice
1818
description: "Mage build scope"
@@ -54,7 +54,7 @@ jobs:
5454
BuildMage:
5555
uses: ./.github/workflows/reusable_package_mage.yaml
5656
with:
57-
arch: ${{ github.event.inputs.mage_build_arch || 'amd' }}
57+
arch: ${{ github.event.inputs.mage_build_arch || 'amd64' }}
5858
mage_build_scope: ${{ github.event.inputs.mage_build_scope || 'all' }}
5959
mage_build_target: ${{ github.event.inputs.mage_build_target || 'prod' }}
6060
mage_build_type: ${{ github.event.inputs.mage_build_type || 'Release' }}

‎.github/workflows/release_artifacts.yaml

+4-4
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ jobs:
2828
runs-on: ubuntu-latest
2929
outputs:
3030
download_link_base: ${{ steps.base_link.outputs.link }}
31-
dowlnoad_binary_name_base: ${{ steps.binary_name.outputs.name }}
31+
download_binary_name_base: ${{ steps.binary_name.outputs.name }}
3232
env:
3333
s3_link_base: ${{ github.event.inputs.official_memgraph == 'true' && 'https://' || 'https://s3.eu-west-1.amazonaws.com' }}
3434
s3_bucket: ${{ github.event.inputs.official_memgraph == 'true' && 'download.memgraph.com' || 'deps.memgraph.io' }}
@@ -51,11 +51,11 @@ jobs:
5151
build_target: ["prod", "dev"]
5252
build_scope: ["with ML", "without ML"]
5353
build_type: ["Release", "RelWithDebInfo"]
54-
build_arch: ["amd", "arm"]
54+
build_arch: ["amd64", "arm64"]
5555
include:
5656
- build_type: "RelWithDebInfo"
5757
rwid_ext: "-relwithdebinfo"
58-
- build_arch: "arm"
58+
- build_arch: "arm64"
5959
arm_ext: "-aarch64"
6060
exclude:
6161
- build_target: "dev"
@@ -70,7 +70,7 @@ jobs:
7070
mage_build_target: "${{ matrix.build_target }}"
7171
mage_build_type: "${{ matrix.build_type }}"
7272
memgraph_version: "${{ github.event.inputs.memgraph_version }}"
73-
memgraph_download_link: "${{ needs.SetupNames.outputs.download_link_base }}${{ matrix.arm_ext }}${{ matrix.rwid_ext }}/${{ needs.SetupNames.outputs.dowlnoad_binary_name_base }}_${{ matrix.build_arch }}64.deb"
73+
memgraph_download_link: "${{ needs.SetupNames.outputs.download_link_base }}${{ matrix.arm_ext }}${{ matrix.rwid_ext }}/${{ needs.SetupNames.outputs.download_binary_name_base }}_${{ matrix.build_arch }}.deb"
7474
shorten_tag: "true"
7575
force_release: "true"
7676
push_to_dockerhub: "false"

‎.github/workflows/reusable_package_mage.yaml

+19-4
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ on:
3838
type: string
3939
description: "Overwrite existing image on dockerhub"
4040
default: true
41+
push_to_github:
42+
type: string
43+
description: "Push the image to GitHub?"
44+
default: false
4145
push_to_dockerhub:
4246
type: string
4347
description: "Push the image to DockerHub?"
@@ -60,12 +64,12 @@ on:
6064
default: false
6165

6266
env:
63-
OS: ${{ inputs.arch == 'arm' && 'ubuntu-24.04-aarch64' || 'ubuntu-24.04' }}
67+
OS: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-aarch64' || 'ubuntu-24.04' }}
6468
DOCKER_REPOSITORY_NAME: memgraph/memgraph-mage
6569

6670
jobs:
6771
BuildAndPush:
68-
runs-on: ${{ inputs.large_runner == 'true' && 'ubuntu-large' || 'ubuntu-latest' }}
72+
runs-on: ${{ (inputs.arch == 'arm64') && fromJSON('["self-hosted", "DockerMgBuild","ARM64"]') || fromJSON('["self-hosted", "DockerMgBuild", "X64"]') }}
6973
steps:
7074
- name: Checkout repository and submodules
7175
uses: actions/checkout@v4
@@ -118,8 +122,8 @@ jobs:
118122
fi
119123
IMAGE_TAG="${MAGE_VERSION}-memgraph-${MEMGRAPH_VERSION}"
120124
ARTIFACT_NAME="mage-${IMAGE_TAG}"
121-
if [[ "${{ inputs.arch }}" == 'arm' ]]; then
122-
ARTIFACT_NAME="${ARTIFACT_NAME}-arm"
125+
if [[ "${{ inputs.arch }}" == 'arm64' ]]; then
126+
ARTIFACT_NAME="${ARTIFACT_NAME}-arm64"
123127
fi
124128
if [[ "${{ env.MAGE_BUILD_SCOPE }}" == 'without ML' ]]; then
125129
ARTIFACT_NAME="${ARTIFACT_NAME}-no-ml"
@@ -185,12 +189,22 @@ jobs:
185189
# --tag ${DOCKER_REPOSITORY_NAME}:${{ inputs.additional_tag }} \
186190
- name: Build docker image
187191
run: |
192+
if [[ "${{ inputs.arch }}" == "arm64" ]]; then
193+
MGBUILD_IMAGE=memgraph/mgbuild:v6_ubuntu-24.04-arm
194+
else
195+
MGBUILD_IMAGE=memgraph/mgbuild:v6_ubuntu-24.04
196+
fi
197+
198+
echo "Using base image: $MGBUILD_IMAGE"
199+
echo "Image Tag: ${{ env.IMAGE_TAG }}"
200+
188201
docker buildx build \
189202
--target ${{ env.MAGE_BUILD_TARGET }} \
190203
--platform linux/${{ inputs.arch }}64 \
191204
--tag ${DOCKER_REPOSITORY_NAME}:${{ env.IMAGE_TAG }} \
192205
--file ${{ env.DOCKERFILE }} \
193206
--build-arg BUILD_TYPE=${{ env.MAGE_BUILD_TYPE }} \
207+
--build-arg MGBUILD_IMAGE=$MGBUILD_IMAGE \
194208
--load .
195209
196210
- name: Save docker image
@@ -199,6 +213,7 @@ jobs:
199213
docker save ${DOCKER_REPOSITORY_NAME}:${{ env.IMAGE_TAG }} > output/${{ env.ARTIFACT_NAME }}.tar.gz
200214
201215
- name: Push to GitHub
216+
if: ${{ inputs.push_to_github == 'true' }}
202217
uses: actions/upload-artifact@v4
203218
with:
204219
name: "${{ env.ARTIFACT_NAME }}"

‎.github/workflows/reusable_test.yml

+36-11
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ on:
1818
memgraph_version:
1919
type: string
2020
description: "Memgraph version built into this image (format: X.Y.Z)"
21-
default: '3.0.0'
21+
required: false
2222
memgraph_download_link:
2323
type: string
2424
description: "Memgraph package download link. Leave empty to use the official download link."
@@ -47,7 +47,7 @@ env:
4747

4848
jobs:
4949
BuildAndTest:
50-
runs-on: [self-hosted, Linux, X64, DockerMgBuild]
50+
runs-on: ${{ (inputs.arch == 'arm64') && fromJSON('["self-hosted", "DockerMgBuild","ARM64"]') || fromJSON('["self-hosted", "DockerMgBuild", "X64"]') }}
5151
steps:
5252
- name: Checkout repository and submodules
5353
uses: actions/checkout@v4
@@ -64,18 +64,37 @@ jobs:
6464
git pull
6565
fi
6666
67-
- name: Set up QEMU
68-
uses: docker/setup-qemu-action@v3
69-
7067
- name: Set up Docker Buildx
7168
id: buildx
7269
uses: docker/setup-buildx-action@v3
7370

71+
- name: Get Latest Tag
72+
run: |
73+
if [[ -z "${{ inputs.memgraph_version }}" ]]; then
74+
latest=$(curl -s https://api.github.com/repos/memgraph/memgraph/tags | python3 -c "import sys, json; print(json.load(sys.stdin)[0]['name'][1:])")
75+
echo "Using latest tag: $latest"
76+
clean_version=$(echo "$latest" | sed -E 's/(.*)-rc.*/\1/')
77+
echo "MEMGRAPH_VERSION_FULL=$latest" >> $GITHUB_ENV
78+
echo "MEMGRAPH_VERSION=$clean_version" >> $GITHUB_ENV
79+
else
80+
echo "Using provided tag: ${{ inputs.memgraph_version }}"
81+
echo "MEMGRAPH_VERSION_FULL=${{ inputs.memgraph_version }}" >> $GITHUB_ENV
82+
clean_version=$(echo "${{ inputs.memgraph_version }}" | sed -E 's/(.*)-rc.*/\1/')
83+
echo "MEMGRAPH_VERSION=$clean_version" >> $GITHUB_ENV
84+
fi
85+
86+
87+
88+
7489
- name: Set up and check memgraph download link
7590
run: |
7691
mg_url=${{ inputs.memgraph_download_link }}
7792
if [[ -z "${{ inputs.memgraph_download_link }}" ]]; then
78-
mg_url="s3://download.memgraph.com/memgraph/v${{ inputs.memgraph_version }}/$OS${{ inputs.arch == 'arm64' && '-aarch64' || '' }}/memgraph_${{ inputs.memgraph_version }}-1_${{ inputs.arch }}.deb"
93+
mg_url="s3://download.memgraph.com/memgraph/v${{ env.MEMGRAPH_VERSION_FULL }}/$OS${{ inputs.arch == 'arm64' && '-aarch64' || '' }}/memgraph_${{ env.MEMGRAPH_VERSION }}-1_${{ inputs.arch }}.deb"
94+
# If the version contains 'rc', swap the domain in the URL
95+
if [[ "${{ env.MEMGRAPH_VERSION_FULL }}" == *"rc"* ]]; then
96+
mg_url=${mg_url/download.memgraph.com/deps.memgraph.io}
97+
fi
7998
fi
8099
echo "Download link: $mg_url"
81100
url_protocol=${mg_url%%://*}
@@ -113,11 +132,17 @@ jobs:
113132

114133
- name: Build and run Memgraph MAGE:${{ inputs.build_target }}
115134
run: |
135+
if [[ "${{ inputs.arch }}" == "arm64" ]]; then
136+
MGBUILD_IMAGE=memgraph/mgbuild:v6_ubuntu-24.04-arm
137+
else
138+
MGBUILD_IMAGE=memgraph/mgbuild:v6_ubuntu-24.04
139+
fi
116140
DOCKER_BUILDKIT=1 docker buildx build \
117141
--tag memgraph-mage:${{ inputs.build_target }} \
118142
--target ${{ inputs.build_target }} \
119143
--platform linux/${{ inputs.arch }} \
120144
--file Dockerfile.${{ inputs.build_scope == 'without ML' && 'no_ML' || 'release' }} \
145+
--build-arg MGBUILD_IMAGE=$MGBUILD_IMAGE \
121146
--load .
122147
123148
docker network create ${{ env.MEMGRAPH_NETWORK }} || true
@@ -134,31 +159,31 @@ jobs:
134159
- name: Rust library tests
135160
if: inputs.build_target == 'dev'
136161
run: |
137-
docker exec -i -u root ${{ env.MAGE_CONTAINER }} bash -c "source ~/.cargo/env && cd /mage/rust/rsmgp-sys && cargo fmt -- --check && cargo test"
162+
docker exec -i -u memgraph ${{ env.MAGE_CONTAINER }} bash -c "source ~/.cargo/env && cd /mage/rust/rsmgp-sys && cargo fmt -- --check && cargo test"
138163
139164
- name: C++ Modules unit tests
140165
if: inputs.build_target == 'dev'
141166
run: |
142-
docker exec -i -u root ${{ env.MAGE_CONTAINER }} bash -c "cd /mage/cpp/build/ && ctest --output-on-failure -j${{ env.CORE_COUNT }}"
167+
docker exec -i -u memgraph ${{ env.MAGE_CONTAINER }} bash -c "cd /mage/cpp/build/ && ctest --output-on-failure -j${{ env.CORE_COUNT }}"
143168
144169
- name: Install Python dependencies
145170
run: |
146-
docker exec -i -u root "$MAGE_CONTAINER" bash -c "pip install -r /mage/python/tests/requirements.txt --break-system-packages"
171+
docker exec -i -u memgraph ${{ env.MAGE_CONTAINER }} bash -c "python3 -m pip install -r /mage/python/tests/requirements.txt --break-system-packages"
147172
148173
- name: Python modules unit tests
149174
if: inputs.build_target == 'dev' && inputs.build_scope != 'without ML'
150175
env:
151176
PYTHONPATH: "$PWD/python"
152177
run: |
153-
docker exec -i -u root ${{ env.MAGE_CONTAINER }} bash -c "cd /mage/python/ && python3 -m pytest ."
178+
docker exec -i -u memgraph ${{ env.MAGE_CONTAINER }} bash -c "cd /mage/python/ && python3 -m pytest ."
154179
155180
- name: Run End-to-end tests
156181
if: inputs.arch != 'arm64'
157182
env:
158183
PYTHONPATH: "$PWD/e2e"
159184
run: |
160185
cd e2e
161-
docker exec -i -u root ${{ env.MAGE_CONTAINER }} bash -c \
186+
docker exec -i -u memgraph ${{ env.MAGE_CONTAINER }} bash -c \
162187
"cd /mage/e2e/ && python3 -m pytest . -k 'not cugraph ${{ inputs.build_scope == 'without ML' && 'and not tgn and not node_classification and not link_prediction' || '' }}'"
163188
164189
- name: Run End-to-end correctness tests

‎.github/workflows/test.yml

+5-5
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,8 @@ jobs:
5757
strategy:
5858
fail-fast: false
5959
matrix:
60-
mg_version: ["3.0.0"]
61-
mg_rc_version: ["rc1"]
60+
mg_version: ["3.1.0"]
61+
mg_rc_version: ["rc4"]
6262
arch: ["amd64", "arm64"]
6363
build_target: ["prod", "dev"]
6464
build_scope: ["with ML", "without ML"]
@@ -77,16 +77,16 @@ jobs:
7777
strategy:
7878
fail-fast: false
7979
matrix:
80-
arch: ["amd64"]
80+
arch: ["amd64","arm64"]
8181
build_target: ["dev"]
8282
build_scope: ["with ML", "without ML"]
8383
uses: ./.github/workflows/reusable_test.yml
8484
with:
8585
arch: "${{ matrix.arch }}"
8686
build_target: "${{ matrix.build_target }}"
8787
build_scope: "${{ matrix.build_scope }}"
88-
memgraph_version: "3.0.0"
89-
memgraph_download_link: "s3://deps.memgraph.io/memgraph-unofficial/04-03-2025-update-mgp-count/ubuntu-24.04/memgraph_3.0.0+60~a932168b3-1_amd64.deb"
88+
memgraph_version:
89+
memgraph_download_link:
9090
memgraph_ref: "master"
9191
memgraph_ref_update: "true"
9292
secrets: inherit

‎Dockerfile.no_ML

+38-9
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
ARG PY_VERSION_DEFAULT=3.12
2+
ARG MGBUILD_IMAGE
23

3-
FROM ubuntu:24.04 as base
4+
FROM $MGBUILD_IMAGE as builder
45

56
USER root
67

@@ -43,37 +44,65 @@ ENV LD_LIBRARY_PATH /usr/lib/memgraph/query_modules
4344
# Memgraph listens for Bolt Protocol on this port by default.
4445
EXPOSE 7687
4546

46-
FROM base as dev
47+
FROM builder as dev
4748

4849
ARG BUILD_TYPE=Release
4950

5051
WORKDIR /mage
5152
COPY . /mage
5253

54+
# hacks so we can do everything under the `memgraph` user
55+
RUN chown -R memgraph: /mage && \
56+
mkdir -pv /usr/lib/memgraph/query_modules && \
57+
chown -R memgraph: /usr/lib/memgraph/query_modules
58+
USER memgraph
5359

5460
#MAGE
55-
RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \
56-
&& export PATH="/root/.cargo/bin:${PATH}" \
61+
SHELL ["/bin/bash", "-c"]
62+
RUN source /opt/toolchain-v6/activate && curl https://sh.rustup.rs -sSf | sh -s -- -y \
63+
&& export PATH="$HOME/.cargo/bin:${PATH}" \
5764
&& python3 -m pip install -r /mage/python/tests/requirements.txt --break-system-packages \
5865
&& python3 -m pip install -r /mage/python/requirements_no_ml.txt --break-system-packages \
5966
&& python3 -m pip install --no-cache-dir -r /usr/lib/memgraph/auth_module/requirements.txt --break-system-packages \
6067
&& python3 /mage/setup build -p /usr/lib/memgraph/query_modules/ --no-ml --cpp-build-flags CMAKE_BUILD_TYPE=${BUILD_TYPE} \
6168
&& chown -R memgraph: /mage/e2e
6269

70+
#remove the toolchain
71+
USER root
72+
RUN rm -rf /opt/toolchain-v6
6373

6474
USER memgraph
6575
ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
6676
CMD [""]
6777

6878

6979

70-
FROM base as prod
80+
FROM ubuntu:24.04 as prod
7181

7282
USER root
73-
ENTRYPOINT []
83+
7484
ARG PY_VERSION_DEFAULT
7585
ENV PY_VERSION ${PY_VERSION_DEFAULT}
7686

87+
# Install only runtime dependencies that are needed
88+
RUN apt-get update && apt-get install -y \
89+
libcurl4 \
90+
libpython${PY_VERSION} \
91+
libssl-dev \
92+
openssl \
93+
python3 \
94+
python3-pip \
95+
g++ \
96+
python3-setuptools \
97+
unixodbc-dev `mage-memgraph` \
98+
libboost-all-dev `mage-memgraph` \
99+
uuid-dev \
100+
gdb \
101+
procps \
102+
libc6-dbg \
103+
--no-install-recommends && \
104+
rm -rf /var/lib/apt/lists/*
105+
77106
#copy modules
78107
COPY --from=dev /usr/lib/memgraph/query_modules/ /usr/lib/memgraph/query_modules/
79108
COPY --from=dev /usr/lib/memgraph/auth_module/ /usr/lib/memgraph/auth_module/
@@ -88,10 +117,10 @@ RUN mv /mage/e2e /e2e/ \
88117
&& rm -rf /mage/* \
89118
&& mv /e2e/ /mage/ \
90119
&& chown -R memgraph: /mage/e2e \
91-
&& export PATH="/usr/local/lib/python${PY_VERSION}:${PATH}" \
92-
&& apt-get -y --purge autoremove clang git curl python3-pip python3-dev cmake build-essential \
93-
&& apt-get clean
120+
&& export PATH="/usr/local/lib/python${PY_VERSION}:${PATH}"
94121

95122
USER memgraph
123+
EXPOSE 7687
124+
96125
ENTRYPOINT ["/usr/lib/memgraph/memgraph"]
97126
CMD [""]

0 commit comments

Comments
 (0)