Skip to content

Commit 60037e7

Browse files
author
Filip Skirtun
authored
Merge branch 'main' into openshift
2 parents fcb5f93 + d6e5ad1 commit 60037e7

File tree

264 files changed

+6038
-404
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

264 files changed

+6038
-404
lines changed

.github/workflows/_helm-e2e.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ jobs:
6666
echo "RELEASE_NAME=${CHART_NAME}$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
6767
echo "NAMESPACE=${CHART_NAME}-$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
6868
echo "ROLLOUT_TIMEOUT_SECONDS=600s" >> $GITHUB_ENV
69+
echo "TEST_TIMEOUT_SECONDS=600s" >> $GITHUB_ENV
6970
echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV
7071
echo "should_cleanup=false" >> $GITHUB_ENV
7172
echo "skip_validate=false" >> $GITHUB_ENV
@@ -110,7 +111,6 @@ jobs:
110111
helm-charts/update_dependency.sh && helm dependency update ${{ env.CHART_FOLDER }}
111112
if ! helm install --create-namespace --namespace $NAMESPACE --wait \
112113
--timeout "$ROLLOUT_TIMEOUT_SECONDS" \
113-
--set autodependency.enabled=true \
114114
--set GOOGLE_API_KEY=${{ env.GOOGLE_API_KEY}} \
115115
--set GOOGLE_CSE_ID=${{ env.GOOGLE_CSE_ID}} \
116116
--values ${{ env.CHART_FOLDER }}/${value_file} \
@@ -130,7 +130,7 @@ jobs:
130130
else
131131
LOG_PATH=/home/$(whoami)/logs
132132
chart=${{ env.CHART_NAME }}
133-
helm test -n $NAMESPACE $RELEASE_NAME --logs |tee ${LOG_PATH}/charts-${chart}.log
133+
helm test -n $NAMESPACE $RELEASE_NAME --logs --timeout "$TEST_TIMEOUT_SECONDS" | tee ${LOG_PATH}/charts-${chart}.log
134134
exit_code=$?
135135
if [ $exit_code -ne 0 ]; then
136136
echo "Chart ${chart} test failed, please check the logs in ${LOG_PATH}!"
Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
name: Memory Bindwidth Exportor Image Build
5+
permissions: read-all
6+
on:
7+
workflow_call:
8+
inputs:
9+
image_repo:
10+
required: false
11+
type: string
12+
image_tag:
13+
required: true
14+
type: string
15+
runner_label:
16+
required: false
17+
type: string
18+
default: 'docker-build-xeon'
19+
outputs:
20+
image_repo:
21+
description: "The image repository used for the image build"
22+
value: ${{ jobs.image-build.outputs.image_repo }}
23+
image_tag:
24+
description: "The image tag used for the image build"
25+
value: ${{ jobs.image-build.outputs.version }}
26+
env:
27+
GOSRC_DIR: "kubernetes-addons/memory-bandwidth-exporter"
28+
29+
jobs:
30+
image-build:
31+
runs-on: ${{ inputs.runner_label }}
32+
outputs:
33+
image_repo: ${{ steps.set_variables.outputs.IMAGE_REPO }}
34+
version: ${{ steps.set_variables.outputs.VERSION }}
35+
steps:
36+
- name: Clean Up Working Directory
37+
run: sudo rm -rf ${{github.workspace}}/*
38+
39+
- name: Get checkout ref
40+
id: get-checkout-ref
41+
run: |
42+
if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
43+
CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge
44+
else
45+
CHECKOUT_REF=${{ github.ref }}
46+
fi
47+
echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT
48+
echo "checkout ref ${CHECKOUT_REF}"
49+
50+
- name: Checkout out Repo
51+
uses: actions/checkout@v4
52+
with:
53+
ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
54+
fetch-depth: 0
55+
56+
- name: Set variables
57+
id: set_variables
58+
env:
59+
imagerepo: ${{ inputs.image_repo }}
60+
run: |
61+
if [[ -z "$imagerepo" ]]; then
62+
echo "DOCKER_REGISTRY=${OPEA_IMAGE_REPO}opea" >> $GITHUB_ENV
63+
echo "IMAGE_REPO=${OPEA_IMAGE_REPO}" >> $GITHUB_OUTPUT
64+
else
65+
echo "DOCKER_REGISTRY=${imagerepo}/opea" >> $GITHUB_ENV
66+
echo "IMAGE_REPO=${imagerepo}/" >> $GITHUB_OUTPUT
67+
fi
68+
echo "VERSION=${{ inputs.image_tag }}" >> $GITHUB_ENV
69+
echo "VERSION=${{ inputs.image_tag }}" >> $GITHUB_OUTPUT
70+
71+
- name: Build image and push
72+
run: |
73+
cd $GOSRC_DIR
74+
make docker.build
75+
make docker.push
76+
77+
- name: Clean up images
78+
if: always()
79+
run: |
80+
# clean up the images
81+
docker rmi ${{ env.DOCKER_REGISTRY }}/memory-bandwidth-exporter:${{ env.VERSION }}
82+
echo y | docker image prune

.github/workflows/pr-chart-e2e.yaml

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ on:
99
types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
1010
paths:
1111
- helm-charts/**
12-
- .github/workflows/chart-e2e.yaml
1312
workflow_dispatch:
1413

1514
concurrency:
@@ -39,19 +38,16 @@ jobs:
3938
merged_commit=$(git log -1 --format='%H')
4039
e2e_charts=$(git diff --name-only ${base_commit} ${merged_commit} | \
4140
grep "^$CHARTS_DIR/" | \
42-
grep -vE 'README.md|common|*.sh' | \
41+
grep -vE 'README.md|CI.md|common|*.sh' | \
4342
cut -d'/' -f2 | sort -u )
4443
common_charts=$(git diff --name-only ${base_commit} ${merged_commit} | \
4544
grep "^$CHARTS_DIR/common" | \
4645
grep -vE 'README.md|*.sh' | \
4746
cut -d'/' -f3 | sort -u )
4847
run_matrix="{\"include\":["
4948
for chart in ${e2e_charts}; do
50-
for file in "$CHARTS_DIR/$chart"/*values.yaml; do
49+
for file in "$CHARTS_DIR/$chart"/ci-*values.yaml; do
5150
if [ -f "$file" ]; then
52-
if [[ "$file" == *"nv-values.yaml" ]] || [[ "$file" == *"hpa-values.yaml" ]]; then
53-
continue
54-
fi
5551
filename=$(basename "$file" .yaml)
5652
if [[ "$filename" == *"gaudi"* ]]; then
5753
run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\"},"
@@ -62,11 +58,8 @@ jobs:
6258
done
6359
done
6460
for chart in ${common_charts}; do
65-
for file in "$CHARTS_DIR/common/$chart"/*values.yaml; do
61+
for file in "$CHARTS_DIR/common/$chart"/ci-*values.yaml; do
6662
if [ -f "$file" ]; then
67-
if [[ "$file" == *"nv-values.yaml" ]] || [[ "$file" == *"hpa-values.yaml" ]]; then
68-
continue
69-
fi
7063
filename=$(basename "$file" .yaml)
7164
if [[ "$filename" == *"gaudi"* ]]; then
7265
run_matrix="${run_matrix}{\"example\":\"${chart}\",\"hardware\":\"gaudi\", \"valuefile\":\"${filename}\",\"directory\":\"common\"},"
Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
name: Check Paths and Hyperlinks
5+
6+
on:
7+
pull_request:
8+
branches: [main]
9+
types: [opened, reopened, ready_for_review, synchronize]
10+
11+
jobs:
12+
check-the-validity-of-hyperlinks-in-README:
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Clean Up Working Directory
16+
run: sudo rm -rf ${{github.workspace}}/*
17+
18+
- name: Checkout Repo GenAIInfra
19+
uses: actions/checkout@v4
20+
with:
21+
fetch-depth: 0
22+
23+
- name: Check the Validity of Hyperlinks
24+
run: |
25+
cd ${{github.workspace}}
26+
fail="FALSE"
27+
merged_commit=$(git log -1 --format='%H')
28+
changed_files="$(git diff --name-status --diff-filter=ARM ${{ github.event.pull_request.base.sha }} ${merged_commit} | awk '/\.md$/ {print $NF}')"
29+
if [ -n "$changed_files" ]; then
30+
for changed_file in $changed_files; do
31+
url_lines=$(grep -H -Eo '\]\(http[s]?://[^)]+\)' "$changed_file" | grep -Ev 'GenAIEval/blob/main') || true
32+
if [ -n "$url_lines" ]; then
33+
for url_line in $url_lines; do
34+
url=$(echo "$url_line"|cut -d '(' -f2 | cut -d ')' -f1|sed 's/\.git$//')
35+
path=$(echo "$url_line"|cut -d':' -f1 | cut -d'/' -f2-)
36+
response=$(curl -L -s -o /dev/null -w "%{http_code}" "$url")
37+
if [ "$response" -ne 200 ]; then
38+
echo "**********Validation failed, try again**********"
39+
response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url")
40+
if [ "$response_retry" -eq 200 ]; then
41+
echo "*****Retry successful*****"
42+
else
43+
echo "Invalid link from ${{github.workspace}}/$path: $url"
44+
fail="TRUE"
45+
fi
46+
fi
47+
done
48+
fi
49+
done
50+
else
51+
echo "No changed .md file."
52+
fi
53+
54+
if [[ "$fail" == "TRUE" ]]; then
55+
exit 1
56+
else
57+
echo "All hyperlinks are valid."
58+
fi
59+
shell: bash
60+
61+
check-the-validity-of-relative-path:
62+
runs-on: ubuntu-latest
63+
steps:
64+
- name: Clean up Working Directory
65+
run: sudo rm -rf ${{github.workspace}}/*
66+
67+
- name: Checkout Repo GenAIInfra
68+
uses: actions/checkout@v4
69+
with:
70+
fetch-depth: 0
71+
72+
- name: Checking Relative Path Validity
73+
run: |
74+
cd ${{github.workspace}}
75+
fail="FALSE"
76+
repo_name=${{ github.event.pull_request.head.repo.full_name }}
77+
if [ "$(echo "$repo_name"|cut -d'/' -f1)" != "opea-project" ]; then
78+
owner=$(echo "${{ github.event.pull_request.head.repo.full_name }}" |cut -d'/' -f1)
79+
branch="https://github.com/$owner/GenAIInfra/tree/${{ github.event.pull_request.head.ref }}"
80+
else
81+
branch="https://github.com/opea-project/GenAIInfra/blob/${{ github.event.pull_request.head.ref }}"
82+
fi
83+
link_head="https://github.com/opea-project/GenAIInfra/blob/main"
84+
85+
merged_commit=$(git log -1 --format='%H')
86+
changed_files="$(git diff --name-status --diff-filter=ARM ${{ github.event.pull_request.base.sha }} ${merged_commit} | awk '/\.md$/ {print $NF}')"
87+
png_lines=$(grep -Eo '\]\([^)]+\)' --include='*.md' -r .|grep -Ev 'http')
88+
if [ -n "$png_lines" ]; then
89+
for png_line in $png_lines; do
90+
refer_path=$(echo "$png_line"|cut -d':' -f1 | cut -d'/' -f2-)
91+
png_path=$(echo "$png_line"|cut -d '(' -f2 | cut -d ')' -f1)
92+
if [[ "${png_path:0:1}" == "/" ]]; then
93+
check_path=$png_path
94+
elif [[ "$png_path" == *#* ]]; then
95+
relative_path=$(echo "$png_path" | cut -d '#' -f1)
96+
if [ -n "$relative_path" ]; then
97+
check_path=$(dirname "$refer_path")/$relative_path
98+
png_path=$(echo "$png_path" | awk -F'#' '{print "#" $2}')
99+
else
100+
check_path=$refer_path
101+
fi
102+
else
103+
check_path=$(dirname "$refer_path")/$png_path
104+
fi
105+
106+
if [ -e "$check_path" ]; then
107+
real_path=$(realpath $check_path)
108+
if [[ "$png_line" == *#* ]]; then
109+
if [ -n "changed_files" ] && echo "$changed_files" | grep -q "^${refer_path}$"; then
110+
url_dev=$branch$(echo "$real_path" | sed 's|.*/GenAIInfra||')$png_path
111+
response=$(curl -I -L -s -o /dev/null -w "%{http_code}" "$url_dev")
112+
if [ "$response" -ne 200 ]; then
113+
echo "**********Validation failed, try again**********"
114+
response_retry=$(curl -s -o /dev/null -w "%{http_code}" "$url_dev")
115+
if [ "$response_retry" -eq 200 ]; then
116+
echo "*****Retry successfully*****"
117+
else
118+
echo "Invalid path from ${{github.workspace}}/$refer_path: $png_path, link: $url_dev"
119+
fail="TRUE"
120+
fi
121+
else
122+
echo "Validation succeed $png_line"
123+
fi
124+
fi
125+
fi
126+
else
127+
echo "$check_path does not exist"
128+
fail="TRUE"
129+
fi
130+
done
131+
fi
132+
133+
if [[ "$fail" == "TRUE" ]]; then
134+
exit 1
135+
else
136+
echo "All hyperlinks are valid."
137+
fi
138+
shell: bash

.github/workflows/push-image-build.yaml renamed to .github/workflows/push-gmc-image-build.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ on:
1313
- "!**.txt"
1414
- "!**.png"
1515
- "!.**"
16-
- .github/workflows/gmc-on-push.yaml
16+
- .github/workflows/push-gmc-image-build.yaml
1717
workflow_dispatch:
1818

1919
concurrency:
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
name: Upgrade memory bandwidth exporter on push event
5+
6+
on:
7+
push:
8+
branches: ["main"]
9+
paths:
10+
- kubernetes-addons/memory-bandwidth-exporter/**
11+
- "!**.md"
12+
- "!**.txt"
13+
- "!**.png"
14+
- "!.**"
15+
- .github/workflows/push-mbe-image-build.yaml
16+
workflow_dispatch:
17+
18+
concurrency:
19+
group: ${{ github.workflow }}-${{ github.ref }}-on-push
20+
cancel-in-progress: true
21+
22+
env:
23+
GOSRC_DIR: "kubernetes-addons/memory-bandwidth-exporter"
24+
25+
jobs:
26+
image-build:
27+
strategy:
28+
matrix:
29+
platform: [xeon, gaudi]
30+
uses: ./.github/workflows/_mbe-image-build.yaml
31+
with:
32+
image_tag: 'latest'
33+
runner_label: 'docker-build-${{ matrix.platform }}'

.github/workflows/scripts/codeScan/bandit.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
# SPDX-License-Identifier: Apache-2.0
55

66
source /GenAIInfra/.github/workflows/scripts/change_color
7-
pip install bandit==1.7.8
7+
pip install --no-cache-dir bandit==1.7.8
88
log_dir=/GenAIInfra/.github/workflows/scripts/codeScan
99
python -m bandit -r -lll -iii /GenAIInfra 2>&1 | tee ${log_dir}/bandit.log
1010
exit_code=$?

.github/workflows/scripts/e2e/chart_test.sh

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,10 @@ function dump_failed_pod_logs() {
6161

6262
function dump_all_pod_logs() {
6363
namespace=$1
64-
echo "-----DUMP POD STATUS AND LOG in NS $namespace------"
64+
echo "-----DUMP POD STATUS AND LOG in NS $namespace-----"
65+
echo "------SUMMARY of POD STATUS in NS $namespace------"
66+
kubectl get pods -n $namespace -o wide
67+
echo "--------------------------------------------------"
6568

6669
pods=$(kubectl get pods -n $namespace -o jsonpath='{.items[*].metadata.name}')
6770
for pod_name in $pods

.github/workflows/scripts/e2e/utils.sh

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,24 @@
22
# Copyright (C) 2024 Intel Corporation
33
# SPDX-License-Identifier: Apache-2.0
44

5+
saved_errexit=False
6+
7+
function turnoff_and_save_errexit() {
8+
if [[ "$-" =~ e ]]; then
9+
saved_errexit=True
10+
set +e
11+
fi
12+
}
13+
14+
15+
function resume_errexit() {
16+
if [ "$saved_errexit" = "True" ]; then
17+
saved_errexit=False
18+
set -e
19+
fi
20+
}
21+
22+
523
function wait_until_pod_ready() {
624
echo "Waiting for the $1 to be ready..."
725
max_retries=60
@@ -53,6 +71,7 @@ function wait_until_all_pod_ready() {
5371
namespace=$1
5472
timeout=$2
5573

74+
turnoff_and_save_errexit
5675
echo "Wait for all pods in NS $namespace to be ready..."
5776
pods=$(kubectl get pods -n $namespace --no-headers | grep -v "Terminating" | awk '{print $1}')
5877
# Loop through each pod
@@ -68,6 +87,7 @@ function wait_until_all_pod_ready() {
6887
exit 1
6988
fi
7089
done
90+
resume_errexit
7191
}
7292

7393
function check_gmc_status() {

0 commit comments

Comments
 (0)