Skip to content

Commit

Permalink
Merge branch 'main' into allocation-drawer-component
Browse files Browse the repository at this point in the history
  • Loading branch information
fabiovincenzi authored Sep 16, 2024
2 parents 7a95785 + 3aa2a22 commit 6c4de29
Show file tree
Hide file tree
Showing 55 changed files with 10,171 additions and 9,027 deletions.
2 changes: 1 addition & 1 deletion .github/actions/setup-go/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ runs:
- id: setup-go
uses: actions/setup-go@v5
with:
go-version: "1.21"
go-version-file: "go.mod"
check-latest: true
cache: false
- id: cache-info
Expand Down
62 changes: 1 addition & 61 deletions .github/workflows/check-required.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,66 +32,6 @@ jobs:
app-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}

- uses: actions/github-script@v7
- uses: G-Research/common-actions/check-required@main
with:
github-token: ${{ steps.app-token.outputs.token }}
script: |
const ghaAppId = 15368;
const ghaName = 'All required checks done';
const myAppId = ${{ secrets.APP_ID }};
const myName = 'All required checks succeeded';
const owner = context.payload.repository.owner.login;
const repo = context.payload.repository.name;
const sha = context.payload.workflow_run.head_sha;
core.info(`List GitHub Actions check runs for ${sha}.`)
const { data: { check_runs: ghaChecks } } = await github.rest.checks.listForRef({
owner: owner,
repo: repo,
ref: sha,
app_id: ghaAppId,
check_name: ghaName,
});
var newCheck = {
owner: owner,
repo: repo,
name: myName,
head_sha: sha,
status: 'in_progress',
started_at: context.payload.workflow_run.created_at,
output: {
title: 'Not all required checks succeeded',
},
};
core.summary.addHeading('The following required checks have been considered:', 3);
ghaChecks.forEach(check => {
core.summary
.addLink(check.name, check.html_url)
.addCodeBlock(JSON.stringify(check, ['status', 'conclusion', 'started_at', 'completed_at'], 2), 'json');
if (check.status === 'completed' && check.conclusion === 'success') {
newCheck.status = 'completed';
newCheck.conclusion = 'success';
newCheck.started_at = check.started_at;
newCheck.completed_at = check.completed_at;
newCheck.output.title = 'All required checks succeeded';
} else if (check.started_at > newCheck.started_at) {
newCheck.started_at = check.started_at;
}
});
if (ghaChecks.length === 0) {
core.summary.addRaw(`No check runs for ${sha} found.`);
}
newCheck.output.summary = core.summary.stringify();
await core.summary.write();
core.info(`Create own check run for ${sha}: ${JSON.stringify(newCheck, null, 2)}.`)
const { data: { html_url } } = await github.rest.checks.create(newCheck);
await core.summary
.addHeading('Check run created:', 3)
.addLink(myName, html_url)
.addCodeBlock(JSON.stringify(newCheck, null, 2), 'json')
.write();
30 changes: 25 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -104,13 +104,36 @@ jobs:
- name: Run Go Integration Tests
run: make integration-tests

codegen:
if: github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
name: Check codegen diff
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Setup Go
uses: ./.github/actions/setup-go
with:
cache-prefix: generate

- name: Generate code
run: make codegen
- name: Check diff
run: git diff --exit-code

build:
if: github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
name: Build (${{ matrix.os }}/${{ matrix.arch }})
needs:
- go-lint
- go-unit-tests
- go-integration-tests
- codegen
strategy:
matrix:
os: [ linux ]
arch: [ amd64, arm64 ]
os: [linux]
arch: [amd64, arm64]
include:
- os: linux
runner: ubuntu-latest
Expand Down Expand Up @@ -160,9 +183,6 @@ jobs:
all-required-checks-done:
name: All required checks done
needs:
- go-lint
- go-unit-tests
- go-integration-tests
- build
runs-on: ubuntu-latest
steps:
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
# UI build files
assets/
!assets/.gitkeep
!web/src/assets/

# OSX
.DS_Store
Expand Down
31 changes: 20 additions & 11 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,14 @@ define yq_get_db
$(shell $(call yq_get, .db.$(1)))
endef

DB_USER ?= $(strip $(call yq_get_db,user))
DB_PASSWORD ?= $(strip $(call yq_get_db,password))
DB_HOST ?= $(strip $(call yq_get_db,host))
DB_PORT ?= $(strip $(call yq_get_db,port))
DB_NAME ?= $(strip $(call yq_get_db,dbname))

define database_url
postgres://$(strip $(call yq_get_db,user)):$(strip $(call yq_get_db,password))@$(strip $(call yq_get_db,host)):$(strip $(call yq_get_db,port))/$(strip $(call yq_get_db,dbname))?sslmode=disable
postgres://$(DB_USER):$(DB_PASSWORD)@$(DB_HOST):$(DB_PORT)/$(DB_NAME)?sslmode=disable
endef

.PHONY: migrate
Expand All @@ -148,8 +154,8 @@ migrate-down: ## migrate down using gomigrate.
##@ Codegen

.PHONY: codegen
codegen: gomock ## generate code using go generate (mocks).
go generate ./...
codegen: mockgen ## generate code using go generate (mocks).
PATH=$(LOCALBIN_TOOLING):$$PATH go generate ./...

##@ Run

Expand Down Expand Up @@ -298,7 +304,8 @@ clean: ## remove generated build artifacts.
rm -rf $(LOCALBIN_APP)

##@ Publish

NODE_VERSION ?= 20
ALPINE_VERSION ?= 3.20
DOCKER_OUTPUT ?= type=docker
DOCKER_TAGS ?= $(IMAGE_TAG)
ifneq ($(origin DOCKER_METADATA), undefined)
Expand Down Expand Up @@ -331,6 +338,8 @@ docker-build: bin/docker clean build ## build docker image using buildx.
--file build/yunikorn-history-server/Dockerfile \
--platform linux/$(ARCH) \
--output $(DOCKER_OUTPUT) \
--build-arg NODE_VERSION=$(NODE_VERSION) \
--build-arg ALPINE_VERSION=$(ALPINE_VERSION) \
$(DOCKER_TAGS) \
.

Expand Down Expand Up @@ -444,7 +453,7 @@ $(GORELEASER): bin/tooling
test -s $(GORELEASER) || GOBIN=$(LOCALBIN_TOOLING) $(GO) install github.com/goreleaser/goreleaser@$(GORELEASER_VERSION)

GOLANGCI_LINT ?= $(LOCALBIN_TOOLING)/golangci-lint
GOLANGCI_LINT_VERSION ?= v1.59.0
GOLANGCI_LINT_VERSION ?= v1.60.2
.PHONY: golangci-lint
golangci-lint: $(GOLANGCI_LINT) ## download golangci-lint locally if necessary.
$(GOLANGCI_LINT): bin/tooling
Expand Down Expand Up @@ -477,12 +486,12 @@ $(YQ): bin/tooling
test -s $(YQ) || curl --silent -L https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_$(OS)_$(ARCH) -o $(LOCALBIN_TOOLING)/yq
chmod +x $(LOCALBIN_TOOLING)/yq

GOMOCK ?= $(LOCALBIN_TOOLING)/gomock
GOMOCK_VERSION ?= v0.4.0
.PHONY: gomock
gomock: $(GOMOCK) ## download uber-go/gomock locally if necessary.
$(GOMOCK): bin/tooling
test -s $(YQ) || GOBIN=$(LOCALBIN_TOOLING) $(GO) install go.uber.org/mock/mockgen@$(GOMOCK_VERSION)
MOCKGEN ?= $(LOCALBIN_TOOLING)/mockgen
MOCKGEN_VERSION ?= v0.4.0
.PHONY: mockgen
mockgen: $(MOCKGEN) ## Download mockgen locally if necessary.
$(MOCKGEN): bin/tooling
test -s $(MOCKGEN) || GOBIN=$(LOCALBIN_TOOLING) $(GO) install go.uber.org/mock/mockgen@$(MOCKGEN_VERSION)

.PHONY: kind
kind: $(KIND) ## download kind locally if necessary.
Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,8 @@ YHS is composed of three main components:
querying capabilities to filter and retrieve specific data.

3. **Web Frontend:** This component enhances the existing Yunikorn Web interface by providing additional features that utilize
the historical data stored by YHS. It is loaded on the application page of Yunikorn Web.
the historical data stored by YHS. It is loaded on the application page of Yunikorn Web. The home page served by YHS
will display instructions on how to connect YHS to the YuniKorn Web.
More details on the web component is available [here](web/README.md).

By integrating these components, YHS provides a comprehensive view of the historical operations of a Yunikorn-managed cluster,
Expand Down
7 changes: 5 additions & 2 deletions build/yunikorn-history-server/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
FROM node:20 as builder
ARG NODE_VERSION=20
ARG ALPINE_VERSION=3.20

FROM node:${NODE_VERSION} as builder

COPY web /build/src

Expand All @@ -8,7 +11,7 @@ RUN npm install -g @angular/cli@17 && \
npm install && \
npm run build

FROM alpine:3.20
FROM alpine:${ALPINE_VERSION}

COPY --from=builder /build/assets /app/assets
COPY bin/app/yunikorn-history-server /app/yunikorn-history-server
Expand Down
22 changes: 22 additions & 0 deletions charts/yunikorn-history-server/templates/NOTES.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
1. Get the application URL by running these commands:
{{- if .Values.ingress.enabled }}
{{- range $host := .Values.ingress.hosts }}
{{- range .paths }}
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
{{- end }}
{{- end }}
{{- else if contains "NodePort" .Values.service.type }}
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "yunikorn-history-server.fullname" . }})
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "yunikorn-history-server.fullname" . }}'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "yunikorn-history-server.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
echo http://$SERVICE_IP:{{ .Values.service.port }}
{{- else if contains "ClusterIP" .Values.service.type }}
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "yunikorn-history-server.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
{{- end }}
61 changes: 61 additions & 0 deletions charts/yunikorn-history-server/templates/ingress.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
{{- if .Values.ingress.enabled -}}
{{- $fullName := include "yunikorn-history-server.fullname" . -}}
{{- $svcPort := .Values.service.port -}}
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
{{- end }}
{{- end }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ $fullName }}
labels:
{{- include "yunikorn-history-server.labels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
ingressClassName: {{ .Values.ingress.className }}
{{- end }}
{{- if .Values.ingress.tls }}
tls:
{{- range .Values.ingress.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
{{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
pathType: {{ .pathType }}
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}
port:
number: {{ $svcPort }}
{{- else }}
serviceName: {{ $fullName }}
servicePort: {{ $svcPort }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
26 changes: 21 additions & 5 deletions charts/yunikorn-history-server/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,22 @@ service:
# -- Service node port
nodePort: 30003

ingress:
# -- Toggle whether to render Ingress resource
enabled: true
# -- Specify which ingress class should be used
className: ""
annotations:
{}
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
tls: []
hosts:
- host: chart-example.local
paths:
- path: /
pathType: ImplementationSpecific

yhs:
# -- YHS port
port: 8989
Expand All @@ -47,7 +63,7 @@ db:
# -- YHS database port
port: "5432"
# -- YHS database name
name: "postgres"
name: "yhs"
# -- YHS database user
user: "postgres"
# -- YHS database password (insecure, use secrets)
Expand All @@ -74,7 +90,7 @@ yunikorn:
protocol: "http"

log:
# -- Output type of the log, if true, log will be output in json format
jsonFormat: true
# -- Log level, one of DEBUG, INFO, WARN, ERROR, DPANIC, PANIC, FATAL
level: "INFO"
# -- Output type of the log, if true, log will be output in json format
jsonFormat: true
# -- Log level, one of DEBUG, INFO, WARN, ERROR, DPANIC, PANIC, FATAL
level: "INFO"
11 changes: 3 additions & 8 deletions cmd/yunikorn-history-server/commands/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,18 @@ package commands

import (
"context"

"github.com/G-Research/yunikorn-history-server/internal/health"

"fmt"
"os/signal"
"syscall"

"github.com/G-Research/yunikorn-history-server/internal/database/postgres"
"github.com/G-Research/yunikorn-history-server/internal/database/repository"

"github.com/oklog/run"

"github.com/spf13/cobra"

"github.com/G-Research/yunikorn-history-server/cmd/yunikorn-history-server/info"

"github.com/G-Research/yunikorn-history-server/internal/config"
"github.com/G-Research/yunikorn-history-server/internal/database/postgres"
"github.com/G-Research/yunikorn-history-server/internal/database/repository"
"github.com/G-Research/yunikorn-history-server/internal/health"
"github.com/G-Research/yunikorn-history-server/internal/log"
"github.com/G-Research/yunikorn-history-server/internal/webservice"
"github.com/G-Research/yunikorn-history-server/internal/yunikorn"
Expand Down
Loading

0 comments on commit 6c4de29

Please sign in to comment.