Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(worker): get available workers #773

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 43 additions & 5 deletions api/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,19 @@ type MetricsQueryParameters struct {
ActiveWorkerCount bool `form:"active_worker_count"`
// InactiveWorkerCount represents total number of inactive workers
InactiveWorkerCount bool `form:"inactive_worker_count"`

// UnregisteredWorkerCount represents total number of workers with a status of unregistered,
UnregisteredWorkerCount bool `form:"unregistered_worker_count"`
// AvailableWorkerCount represents total number of workers with a status of available,
// where worker RunningBuildIDs.length < worker BuildLimit
AvailableWorkerCount bool `form:"available_worker_count"`
// BusyWorkerCount represents total number of workers with a status of busy,
// where worker BuildLimit == worker RunningBuildIDs.length
BusyWorkerCount bool `form:"busy_worker_count"`
// BusyWorkerCount represents total number of workers with a status of maintenance.
MaintenanceWorkerCount bool `form:"maintenance_worker_count"`
// ErrorWorkerCount represents total number of workers with a status of error
ErrorWorkerCount bool `form:"error_worker_count"`
}

// predefine Prometheus metrics else they will be regenerated
Expand Down Expand Up @@ -370,14 +383,19 @@ func recordGauges(c *gin.Context) {

// add worker metrics
var (
buildLimit int64
activeWorkers int64
inactiveWorkers int64
buildLimit int64
activeWorkers int64
inactiveWorkers int64
unregisteredWorkers int64
availableWorkers int64
busyWorkers int64
maintenanceWorkers int64
errorWorkers int64
)

// get worker metrics based on request query parameters
// worker_build_limit, active_worker_count, inactive_worker_count
if q.WorkerBuildLimit || q.ActiveWorkerCount || q.InactiveWorkerCount {
// worker_build_limit, active_worker_count, inactive_worker_count, unregistered_worker_count, available_worker_count, busy_worker_count, maintenance_worker_count, error_worker_count
if q.WorkerBuildLimit || q.ActiveWorkerCount || q.InactiveWorkerCount || q.UnregisteredWorkerCount || q.AvailableWorkerCount || q.BusyWorkerCount || q.MaintenanceWorkerCount || q.ErrorWorkerCount {
// send API call to capture the workers
workers, err := database.FromContext(c).ListWorkers()
if err != nil {
Expand Down Expand Up @@ -411,5 +429,25 @@ func recordGauges(c *gin.Context) {
if q.InactiveWorkerCount {
totals.WithLabelValues("worker", "count", "inactive").Set(float64(inactiveWorkers))
}
// unregistered_worker_count
if q.UnregisteredWorkerCount {
totals.WithLabelValues("worker", "count", "unregistered").Set(float64(unregisteredWorkers))
}
// available_worker_count
if q.AvailableWorkerCount {
totals.WithLabelValues("worker", "count", "available").Set(float64(availableWorkers))
}
// busy_worker_count
if q.BusyWorkerCount {
totals.WithLabelValues("worker", "count", "busy").Set(float64(busyWorkers))
}
// maintenance_worker_count
if q.MaintenanceWorkerCount {
totals.WithLabelValues("worker", "count", "maintenance").Set(float64(maintenanceWorkers))
}
// error_worker_count
if q.ErrorWorkerCount {
totals.WithLabelValues("worker", "count", "error").Set(float64(errorWorkers))
}
}
}
53 changes: 53 additions & 0 deletions api/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,34 @@ func GetWorkers(c *gin.Context) {
c.JSON(http.StatusOK, w)
}

// GetWorkersByStatus represents the API handler to capture a
// list of workers with specified status from the configured backend.
func GetWorkersByStatus(c *gin.Context) {
s := c.Param("status")
// capture middleware values
u := user.Retrieve(c)

// TODO message/error if not valid status or empty string, or they get back all the workers (GetWorkers), how do other endpoints do it? prob use regex to confirm alpha charas only

// update engine logger with API metadata
//
// https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Entry.WithFields
logrus.WithFields(logrus.Fields{
"user": u.GetName(),
}).Info("reading workers")

w, err := database.FromContext(c).ListWorkersByStatus(s)
if err != nil {
retErr := fmt.Errorf("unable to get workers: %w", err)

util.HandleError(c, http.StatusInternalServerError, retErr)

return
}

c.JSON(http.StatusOK, w)
}

// swagger:operation GET /api/v1/workers/{worker} workers GetWorker
//
// Retrieve a worker for the configured backend
Expand Down Expand Up @@ -311,6 +339,31 @@ func UpdateWorker(c *gin.Context) {
w.SetActive(input.GetActive())
}

if len(input.GetStatus()) > 0 {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetStatus undefined (type *library.Worker has no field or method GetStatus)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetStatus undefined (type *library.Worker has no field or method GetStatus)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetStatus undefined (type *library.Worker has no field or method GetStatus)

// update status if set
w.SetStatus(input.GetStatus())
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
w.SetStatus undefined (type *library.Worker has no field or method SetStatus)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetStatus undefined (type *library.Worker has no field or method GetStatus)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
w.SetStatus undefined (type *library.Worker has no field or method SetStatus)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetStatus undefined (type *library.Worker has no field or method GetStatus)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
w.SetStatus undefined (type *library.Worker has no field or method SetStatus)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetStatus undefined (type *library.Worker has no field or method GetStatus)

}

if input.GetLastStatusUpdateAt() > 0 {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetLastStatusUpdateAt undefined (type *library.Worker has no field or method GetLastStatusUpdateAt)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetLastStatusUpdateAt undefined (type *library.Worker has no field or method GetLastStatusUpdateAt)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetLastStatusUpdateAt undefined (type *library.Worker has no field or method GetLastStatusUpdateAt)

// update LastStatusUpdateAt if set
w.SetLastStatusUpdateAt(input.GetLastStatusUpdateAt())
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
w.SetLastStatusUpdateAt undefined (type *library.Worker has no field or method SetLastStatusUpdateAt)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetLastStatusUpdateAt undefined (type *library.Worker has no field or method GetLastStatusUpdateAt)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
w.SetLastStatusUpdateAt undefined (type *library.Worker has no field or method SetLastStatusUpdateAt)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetLastStatusUpdateAt undefined (type *library.Worker has no field or method GetLastStatusUpdateAt)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
w.SetLastStatusUpdateAt undefined (type *library.Worker has no field or method SetLastStatusUpdateAt)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetLastStatusUpdateAt undefined (type *library.Worker has no field or method GetLastStatusUpdateAt)

}

if len(input.GetRunningBuildIDs()) > 0 {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetRunningBuildIDs undefined (type *library.Worker has no field or method GetRunningBuildIDs)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetRunningBuildIDs undefined (type *library.Worker has no field or method GetRunningBuildIDs)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetRunningBuildIDs undefined (type *library.Worker has no field or method GetRunningBuildIDs)

// update RunningBuildIDs if set
w.SetRunningBuildIDs(input.GetRunningBuildIDs())
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
w.SetRunningBuildIDs undefined (type *library.Worker has no field or method SetRunningBuildIDs)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetRunningBuildIDs undefined (type *library.Worker has no field or method GetRunningBuildIDs)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
w.SetRunningBuildIDs undefined (type *library.Worker has no field or method SetRunningBuildIDs)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetRunningBuildIDs undefined (type *library.Worker has no field or method GetRunningBuildIDs)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
w.SetRunningBuildIDs undefined (type *library.Worker has no field or method SetRunningBuildIDs)

}

if input.GetLastBuildFinishedAt() > 0 {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetLastBuildFinishedAt undefined (type *library.Worker has no field or method GetLastBuildFinishedAt)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
too many errors) (typecheck)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
input.GetLastBuildFinishedAt undefined (type *library.Worker has no field or method GetLastBuildFinishedAt)

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚫 [golangci] reported by reviewdog 🐶
too many errors) (typecheck)

// update LastBuildFinishedAt if set
w.SetLastBuildFinishedAt(input.GetLastBuildFinishedAt())
}

if input.GetLastCheckedIn() > 0 {
// update LastCheckedIn if set
w.SetLastCheckedIn(input.GetLastCheckedIn())
}

// send API call to update the worker
err = database.FromContext(c).UpdateWorker(w)
if err != nil {
Expand Down
26 changes: 26 additions & 0 deletions database/worker/count_by_status.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// Copyright (c) 2022 Target Brands, Inc. All rights reserved.
//
// Use of this source code is governed by the LICENSE file in this repository.

package worker

import (
"github.com/go-vela/types/constants"
)

// CountWorkersByStatus gets the count of all workers from the database with the specified status.
func (e *engine) CountWorkersByStatus(status string) (int64, error) {
e.logger.Tracef("getting count of all workers from the database with the specified status")

// variable to store query results
var w int64

// send query to the database and store result in variable
err := e.client.
Table(constants.TableWorker).
Where("status = ?", status).
Count(&w).
Error

return w, err
}
6 changes: 3 additions & 3 deletions database/worker/create_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,9 @@ func TestWorker_Engine_CreateWorker(t *testing.T) {

// ensure the mock expects the query
_mock.ExpectQuery(`INSERT INTO "workers"
("hostname","address","routes","active","last_checked_in","build_limit","id")
VALUES ($1,$2,$3,$4,$5,$6,$7) RETURNING "id"`).
WithArgs("worker_0", "localhost", nil, true, nil, nil, 1).
("hostname","address","routes","active","status","last_status_update_at","running_build_ids","last_build_finished_at","last_checked_in","build_limit","id")
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) RETURNING "id"`).
WithArgs("worker_0", "localhost", nil, true, nil, nil, nil, nil, nil, nil, 1).
WillReturnRows(_rows)

_sqlite := testSqlite(t)
Expand Down
55 changes: 55 additions & 0 deletions database/worker/list_by_status.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
// Copyright (c) 2022 Target Brands, Inc. All rights reserved.
//
// Use of this source code is governed by the LICENSE file in this repository.

package worker

import (
"github.com/go-vela/types/constants"
"github.com/go-vela/types/database"
"github.com/go-vela/types/library"
)

// ListWorkersByStatus gets a list of all workers from the database with the specified status.
func (e *engine) ListWorkersByStatus(status string) ([]*library.Worker, error) {
e.logger.Trace("listing all workers from the database")

// variables to store query results and return value
count := int64(0)
w := new([]database.Worker)
workers := []*library.Worker{}

// count the results
count, err := e.CountWorkersByStatus(status)
if err != nil {
return nil, err
}

// short-circuit if there are no results
if count == 0 {
return workers, nil
}

// send query to the database and store result in variable
err = e.client.
Table(constants.TableWorker).
Where("status = ?", status).
Find(&w).
Error
if err != nil {
return nil, err
}

// iterate through all query results
for _, worker := range *w {
// https://golang.org/doc/faq#closures_and_goroutines
tmp := worker

// convert query result to library type
//
// https://pkg.go.dev/github.com/go-vela/types/database#Worker.ToLibrary
workers = append(workers, tmp.ToLibrary())
}

return workers, nil
}
117 changes: 117 additions & 0 deletions database/worker/list_by_status_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
// Copyright (c) 2022 Target Brands, Inc. All rights reserved.
//
// Use of this source code is governed by the LICENSE file in this repository.

package worker

import (
"reflect"
"testing"

"github.com/DATA-DOG/go-sqlmock"
"github.com/go-vela/types/library"
)

func TestWorker_Engine_GetWorkersByStatus(t *testing.T) {
// setup types
_workerOne := testWorker()
_workerOne.SetID(1)
_workerOne.SetHostname("worker_0")
_workerOne.SetAddress("localhost")
_workerOne.SetActive(true)
_workerOne.SetStatus("available")

_workerTwo := testWorker()
_workerTwo.SetID(2)
_workerTwo.SetHostname("worker_1")
_workerTwo.SetAddress("localhost")
_workerTwo.SetActive(true)
_workerTwo.SetStatus("busy")

_workerThree := testWorker()
_workerThree.SetID(3)
_workerThree.SetHostname("worker_2")
_workerThree.SetAddress("localhost")
_workerThree.SetActive(true)
_workerThree.SetStatus("available")

_postgres, _mock := testPostgres(t)
defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }()

// create expected result in mock
_rows := sqlmock.NewRows([]string{"count"}).AddRow(2)

// ensure the mock expects the query
_mock.ExpectQuery(`SELECT count(*) FROM "workers" WHERE status = $1`).WithArgs("available").WillReturnRows(_rows)

// create expected result in mock
_rows = sqlmock.NewRows(
[]string{"id", "hostname", "address", "routes", "active", "status", "last_checked_in", "build_limit"}).
AddRow(1, "worker_0", "localhost", nil, true, "available", 0, 0).
AddRow(3, "worker_2", "localhost", nil, true, "available", 0, 0)

// ensure the mock expects the query
_mock.ExpectQuery(`SELECT * FROM "workers" WHERE status = $1`).WithArgs("available").WillReturnRows(_rows)

_sqlite := testSqlite(t)
defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }()

err := _sqlite.CreateWorker(_workerOne)
if err != nil {
t.Errorf("unable to create test worker one for sqlite: %v", err)
}

err = _sqlite.CreateWorker(_workerTwo)
if err != nil {
t.Errorf("unable to create test worker two for sqlite: %v", err)
}

err = _sqlite.CreateWorker(_workerThree)
if err != nil {
t.Errorf("unable to create test worker three for sqlite: %v", err)
}

// setup tests
tests := []struct {
failure bool
name string
database *engine
want []*library.Worker
}{
{
failure: false,
name: "postgres",
database: _postgres,
want: []*library.Worker{_workerOne, _workerThree},
},
{
failure: false,
name: "sqlite3",
database: _sqlite,
want: []*library.Worker{_workerOne, _workerThree},
},
}

// run tests
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got, err := test.database.ListWorkersByStatus("available")

if test.failure {
if err == nil {
t.Errorf("ListWorkersByStatus for %s should have returned err", test.name)
}

return
}

if err != nil {
t.Errorf("ListWorkersByStatus for %s returned err: %v", test.name, err)
}

if !reflect.DeepEqual(got, test.want) {
t.Errorf("ListWorkersByStatus for %s is %v, want %v", test.name, got, test.want)
}
})
}
}
2 changes: 2 additions & 0 deletions database/worker/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ type WorkerService interface {
GetWorkerForHostname(string) (*library.Worker, error)
// ListWorkers defines a function that gets a list of all workers.
ListWorkers() ([]*library.Worker, error)
// ListWorkersByStatus defines a function that gets a list of all workers by specified status.
ListWorkersByStatus(string) ([]*library.Worker, error)
// UpdateWorker defines a function that updates an existing worker.
UpdateWorker(*library.Worker) error
}
Loading