From 80ea22376ddc8668b8e73ca06a9cc16c0e59ec54 Mon Sep 17 00:00:00 2001 From: Vedant T Date: Mon, 3 Apr 2023 15:40:05 +0530 Subject: [PATCH] Add CI Handler for adding CI files to a repo --- precommit/Default.ci | 8 +++ precommit/Golang.ci | 11 ++++ precommit/Node.ci | 4 ++ precommit/Python.ci | 18 +++++++ scripts/ci.sh | 81 ++++++++++++++++++++++++++++ src/git/config.go | 34 ++++++++++++ src/git/git.go | 122 +++++++++++++++++++++++++++++++++++++++++++ src/main.go | 4 ++ 8 files changed, 282 insertions(+) create mode 100644 precommit/Default.ci create mode 100644 precommit/Golang.ci create mode 100644 precommit/Node.ci create mode 100644 precommit/Python.ci create mode 100644 scripts/ci.sh diff --git a/precommit/Default.ci b/precommit/Default.ci new file mode 100644 index 0000000..e0c58ba --- /dev/null +++ b/precommit/Default.ci @@ -0,0 +1,8 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files \ No newline at end of file diff --git a/precommit/Golang.ci b/precommit/Golang.ci new file mode 100644 index 0000000..295d0e4 --- /dev/null +++ b/precommit/Golang.ci @@ -0,0 +1,11 @@ +- repo: https://github.com/dnephin/pre-commit-golang + rev: master + hooks: + - id: go-fmt + - id: go-vet + - id: go-imports + - id: golangci-lint + - id: go-unit-tests + - id: go-mod-tidy + - id: go-mod-vendor + - id: go-build \ No newline at end of file diff --git a/precommit/Node.ci b/precommit/Node.ci new file mode 100644 index 0000000..5923669 --- /dev/null +++ b/precommit/Node.ci @@ -0,0 +1,4 @@ +- repo: https://github.com/pre-commit/mirrors-eslint + rev: v8.37.0 + hooks: + - id: eslint \ No newline at end of file diff --git a/precommit/Python.ci b/precommit/Python.ci new file mode 100644 index 0000000..06a5bee --- /dev/null +++ b/precommit/Python.ci @@ -0,0 +1,18 @@ +- repo: https://github.com/psf/black + rev: 22.12.0 + hooks: + - id: black + language_version: python3.10 +- repo: https://github.com/pycqa/flake8 + rev: 6.0.0 + hooks: + - id: flake8 +- repo: https://github.com/asottile/reorder_python_imports + rev: v3.9.0 + hooks: + - id: reorder-python-imports +- repo: https://github.com/Lucas-C/pre-commit-hooks-safety + rev: v1.3.0 + hooks: + - id: python-safety-dependencies-check + files: requirements.txt \ No newline at end of file diff --git a/scripts/ci.sh b/scripts/ci.sh new file mode 100644 index 0000000..4d393d9 --- /dev/null +++ b/scripts/ci.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# +# Exit on error. Append "|| true" if you expect an error. +set -o errexit +# Exit on error inside any functions or subshells. +set -o errtrace +# Do not allow use of undefined vars. Use ${VAR:-} to use an undefined VAR +set -o nounset +# Catch the error in case mysqldump fails (but gzip succeeds) in `mysqldump |gzip` +set -o pipefail + +__repo_url=${1} +__repo_branch=${2} + +__repo_name=$(basename ${__repo_url} .git) #get name of git repository +__temp_dir=$(mktemp -d) +__repo_dir="${__temp_dir}/${__repo_name}" +mkdir -p "${__repo_dir}" + +ci_dir_path = "../precommit/CI/" + + +## @brief Pulls repository in the given path +## @param $1 repo url to clone. +## @param $2 path on which to clone the repo. +## @param $3 branch name of the git repository. +pullRepository() { + local repo_url=${1} + local repo_path=${2} + pushd "${repo_path}" + if [ "$#" -gt 2 ] ; then + local repo_branch=${3} + git clone -b "${repo_branch}" "${repo_url}" . + else + git clone "${repo_url}" . + fi + chmod -R 755 "${repo_path}" + echo "Repository successfully pulled - ${repo_url}" + popd +} + + +addCI(){ + local repo_path=${1} + local ci_dir_path=${2} + + pushd "${repo_path}" + cp -r "${ci_dir_path}" . + echo "Git hooks successfully added" + popd +} + + +## @brief Pushes repository to Github +## @param $1 path where updates to the repository are stored. +## @param $2 branch of the repository. +pushRepo(){ + local repo_path=${1} + local repo_branch=${2} + + pushd "${repo_path}" + git add -A . + git commit -m "[DeployBot] Initialized gitsecret and added git hooks" + git pull || true + git push -u origin "${repo_branch}" + echo "Git Repo successfully initialized and pushed" + popd +} + +cleanup() { + local repo_path=$1 + rm -rf "${repo_path}" + rm -rf "${__temp_dir}" + rm -rf "${__hooks_temp_dir}" + echo "Cleanup successfull" +} + +pullRepository "${__repo_url}" "${__repo_dir}" "${__repo_branch}" +addCI "${__repo_dir}" "${ci_dir_path}" +pushRepo "${__repo_dir}" "${__repo_branch}" +cleanup "${__repo_dir}" diff --git a/src/git/config.go b/src/git/config.go index 0c1857e..460a0f6 100644 --- a/src/git/config.go +++ b/src/git/config.go @@ -17,6 +17,12 @@ var ( // githubUserName is the name of the user which is a member of the GitHub //organization. githubAccessToken is a personal access token of this user githubUserName string + + // githubActionToken is used to verify the request coming from GitHub Actions + githubActionToken string + + // Logdir is the directory where all logs will be stored + logDir string ) const ( @@ -26,6 +32,14 @@ const ( // apiURL is the url at which all APIs of github are rooted apiURL = "https://api.github.com" + + // root path of pre-commit files + BASE_FILES_PATH string = "../../precommit/" + // pre-commit config file + PRE_COMMIT_CONFIG string = "../../precommit/CI/pre-commit-config.yaml" + + // CI script name + ciScriptName = "ci.sh" ) // Repository is the type that is used to store information about a repository @@ -35,6 +49,19 @@ type Repository struct { Branches []string `json:"branches"` } +// CIAction : Used for unmarshalling the CI request +type CIAction struct { + Repo string `json:"repo"` + Python string `json:"python"` + Golang string `json:"golang"` + Node string `json:"node"` + Ts string `json:"ts"` + Flutter string `json:"flutter"` + Dart string `json:"dart"` + Docker string `json:"docker"` + Shell string `json:"shell"` +} + func init() { githubSecret = helper.Env("GITHUB_SECRET", "None") if githubSecret == "None" { @@ -50,4 +77,11 @@ func init() { if githubAccessToken == "None" { log.Fatal("GITHUB_ACCESS_TOKEN is not present in environment, Exiting") } + + githubActionToken = helper.Env("GITHUB_ACTION_TOKEN", "None") + if githubActionToken == "None" { + log.Fatal("GITHUB_ACTION_TOKEN is not present in environment, Exiting") + } + + logDir = helper.Env("LOG_DIR", "/var/logs/deploybot/") } diff --git a/src/git/git.go b/src/git/git.go index ef9a453..cca4899 100644 --- a/src/git/git.go +++ b/src/git/git.go @@ -9,7 +9,10 @@ import ( "io/ioutil" "net/http" "net/url" + "os" + "os/exec" "path" + "reflect" "github.com/devclub-iitd/DeployBot/src/helper" log "github.com/sirupsen/logrus" @@ -181,3 +184,122 @@ func CreatedRepo(r *http.Request) (*Repository, int, error) { }, }, 200, nil } + +func validateActionRequest(r *http.Request) bool { + headerValue := r.Header.Get("Authorization") + return "Bearer " + githubActionToken == headerValue +} + +// CIHandler handles the request to add CI Files to a repo +func CIHandler(w http.ResponseWriter, r *http.Request) { + + if !validateActionRequest(r) { + log.Info("Request verification from Github Action: FAILED") + w.WriteHeader(403) + return + } + log.Info("Request verification from Github Action: SUCCESS") + + if r.Method != "POST" { + w.WriteHeader(405) + log.Errorf("received a %s request, expected POST for CI Handler", r.Method) + return + } + + r_body, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(500) + log.Errorf("cannot read request body - %v", err) + return + } + + var ci CIAction + if err := json.Unmarshal(r_body, &ci); err != nil { + w.WriteHeader(500) + log.Errorf("cannot unmarshal request body - %v", err) + return + } + + var ciList []string + // add Default checks + ciList = append(ciList, "Default") + v := reflect.ValueOf(ci) + for i := 0; i < v.NumField(); i++ { + if v.Field(i).String() == "true" { + ciList = append(ciList, v.Type().Field(i).Name) + } + } + addChecks(ciList) + log.Info("Added checks to pre-commit-config.yaml") + + // call a script to add this file to the repo + + // get ssh_url of the repo + repositories, err := Repos() + if err != nil { + log.Error("Error getting repositories %v", err) + return + } + var repo Repository + for _, repository := range repositories { + if repository.Name == ci.Repo { + repo = repository + } + } + if repo.Name == "" { + log.Error("Repository not found") + return + } + + for _, branch := range repo.Branches { + go addCI(repo.URL, repo.Name, branch) + } + +} + +func addChecks(ciList []string) { + // combine the data of all files in ciList to a file named .pre-commit-config.yaml + + // create the file even if it exists + os.Create(PRE_COMMIT_CONFIG) + f, err := os.OpenFile(PRE_COMMIT_CONFIG, os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + log.Error("Error opening file:pre-commit.config") + return + } + + // close the file after the function returns + defer f.Close() + + for i := 0; i < len(ciList); i++ { + // add the data from the file to the pre-commit-config file + + var file_name string = ciList[i] + ".ci" + var file_path string = BASE_FILES_PATH + file_name + + content, err := os.ReadFile(file_path) + if err != nil { + log.Error("Error reading file:" + file_path) + return + } + + // append the data to the pre-commit-config file + _, err = fmt.Fprintf(f, string(content)+"\n") + if err != nil { + fmt.Printf("Error writing to file:%s\n", PRE_COMMIT_CONFIG) + return + } + } +} + +func addCI(repoURL,repoName, branchName string) { + // call the script ciScriptName + log.Infof("Calling script %s for repo:%s and branch:%s",ciScriptName, repoName, branchName) + output,err := exec.Command(ciScriptName, repoURL, branchName).CombinedOutput() + helper.WriteToFile(path.Join(logDir, "git", fmt.Sprintf("%s:%s.txt", repoName, branchName)), string(output)) + if err != nil { + log.Errorf("Add CI to git repo - %s: FAILED - %v", repoURL, err) + } else { + log.Infof("Add CI to git repo - %s: SUCCESS",repoURL) + } +} \ No newline at end of file diff --git a/src/main.go b/src/main.go index c6039f9..eec91a0 100644 --- a/src/main.go +++ b/src/main.go @@ -8,6 +8,7 @@ import ( "github.com/devclub-iitd/DeployBot/src/history" "github.com/devclub-iitd/DeployBot/src/options" "github.com/devclub-iitd/DeployBot/src/slack" + "github.com/devclub-iitd/DeployBot/src/git" "github.com/robfig/cron" log "github.com/sirupsen/logrus" ) @@ -38,6 +39,9 @@ func main() { // Github New repo creation HTTP handler http.HandleFunc("/github/repo/", controllers.RepoHandler) + // Github CI HTTP handler + http.HandleFunc("/github/ci/", git.CIHandler) + // General status and history HTTP handlers http.HandleFunc("/logs/", controllers.LogHandler) http.HandleFunc("/status/", history.StatusHandler)