Skip to content

Commit c7cdffe

Browse files
author
Andy Ford
committed
fix: improved healthcheck
1 parent 9355bb6 commit c7cdffe

File tree

6 files changed

+78
-48
lines changed

6 files changed

+78
-48
lines changed

‎Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ FROM builder_base AS development
1717
EXPOSE 80
1818

1919
# Health check
20-
HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 CMD [ "grpc_health_probe", "-addr", "localhost:80", "-connect-timeout", "250ms", "-rpc-timeout", "100ms" ]
20+
HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 CMD [ "grpc_health_probe", "-addr", "localhost:80", "-connect-timeout", "100ms", "-rpc-timeout", "250ms" ]
2121

2222
# Create the user
2323
RUN adduser --uid 1000 appuser
@@ -77,7 +77,7 @@ COPY --from=builder_base /usr/local/bin/grpc_health_probe /usr/local/bin/grpc_he
7777
EXPOSE 80
7878

7979
# Health check
80-
HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 CMD [ "grpc_health_probe", "-addr", "localhost:80", "-connect-timeout", "250ms", "-rpc-timeout", "100ms" ]
80+
HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 CMD [ "grpc_health_probe", "-addr", "localhost:80", "-connect-timeout", "100ms", "-rpc-timeout", "250ms" ]
8181

8282
USER appuser
8383

‎internal/db/mongo.go

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,6 @@ func NewMongo() (*Mongo, error) {
3636
return nil, err
3737
}
3838

39-
// Ping mongo
40-
pingErr := client.Ping(ctx, nil)
41-
if pingErr != nil {
42-
log.Errorf("Failed to ping mongo: %v", pingErr)
43-
return nil, pingErr
44-
}
45-
4639
// Create necessary indexes in mongo
4740
collection := client.Database(os.Getenv("MONGO_DB")).Collection("discord_messages")
4841
_, indexErr := collection.Indexes().CreateOne(
@@ -246,15 +239,6 @@ func (m *Mongo) GetDiscordMessageByClientRequestId(clientRequestId string) (*Dis
246239
return &result, nil
247240
}
248241

249-
/**
250-
* Pings the mongo database to check if it is up.
251-
*/
252-
func (m *Mongo) Ping() error {
253-
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
254-
defer cancel()
255-
return m.Client.Ping(ctx, nil)
256-
}
257-
258242
/**
259243
* Disconnects from the mongo database.
260244
*/

‎internal/discord/scheduler.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,14 @@ import (
99

1010
type Scheduler interface {
1111
ScheduleMessage(id string)
12+
Ready() bool
1213
}
1314

1415
type DiscordScheduler struct {
1516
channel chan string
1617
mongo *db.Mongo
1718
discord Discord
19+
ready bool
1820

1921
GoRoutineWaitGroup *sync.WaitGroup
2022
}
@@ -27,10 +29,12 @@ func NewDiscordScheduler(mongo *db.Mongo, discordInterface Discord) *DiscordSche
2729
mongo: mongo,
2830
discord: discordInterface,
2931
channel: make(chan string, 50),
32+
ready: false,
3033
GoRoutineWaitGroup: &sync.WaitGroup{},
3134
}
3235

3336
go func(schedulerToProcess *DiscordScheduler) {
37+
schedulerToProcess.ready = true
3438
schedulerToProcess.processChannel()
3539
}(scheduler)
3640

@@ -46,6 +50,10 @@ func (d *DiscordScheduler) ScheduleMessage(id string) {
4650
d.channel <- id
4751
}
4852

53+
func (d *DiscordScheduler) Ready() bool {
54+
return d.ready
55+
}
56+
4957
/**
5058
* Called by the scheduler's goroutine to process messages from the channel asynchonously to the
5159
* request that scheduled them.

‎internal/discord/scheduler_test.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
pb "ecfmp/discord/proto/discord/gen/pb-go/ecfmp.vatsim.net/grpc/discord"
88
"os"
99
"testing"
10+
"time"
1011

1112
log "github.com/sirupsen/logrus"
1213
"github.com/stretchr/testify/assert"
@@ -127,3 +128,21 @@ func Test_ItUpdatesMessagesFromVersions(t *testing.T) {
127128

128129
assert.Equal(t, "some-client-request-id", mongoMessage.LastClientRequestPublished)
129130
}
131+
132+
func Test_ItReturnsReadyStatus(t *testing.T) {
133+
testMongo, _, scheduler := SetupTest(t)
134+
defer testMongo.tearDown()
135+
136+
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
137+
defer cancel()
138+
for {
139+
if ctx.Err() != nil {
140+
t.Errorf("Timed out waiting for scheduler to be ready")
141+
break
142+
}
143+
144+
if scheduler.Ready() {
145+
break
146+
}
147+
}
148+
}

‎internal/grpc/grpc.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -165,10 +165,8 @@ func (server *server) Update(ctx context.Context, in *pb_discord.UpdateRequest)
165165
* Implements the Check method of the HealthServer interface
166166
*/
167167
func (server *server) Check(ctx context.Context, in *pb_health.HealthCheckRequest) (*pb_health.HealthCheckResponse, error) {
168-
mongoPingErr := server.mongo.Ping()
169-
if mongoPingErr != nil {
170-
log.Errorf("Failed to ping mongo: %v", mongoPingErr)
171-
return nil, status.Error(codes.Unavailable, "Failed to ping mongo")
168+
if !server.scheduler.Ready() {
169+
return &pb_health.HealthCheckResponse{Status: pb_health.HealthCheckResponse_NOT_SERVING}, nil
172170
}
173171

174172
return &pb_health.HealthCheckResponse{Status: pb_health.HealthCheckResponse_SERVING}, nil

‎internal/grpc/grpc_test.go

Lines changed: 47 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ type TestMongo struct {
3131
}
3232

3333
type MockScheduler struct {
34+
isReady bool
3435
callCount int
3536
callId string
3637
}
@@ -40,7 +41,11 @@ func (scheduler *MockScheduler) ScheduleMessage(id string) {
4041
scheduler.callId = id
4142
}
4243

43-
func SetupTest(t *testing.T, realInterceptor bool) (TestMongo, *MockScheduler) {
44+
func (scheduler *MockScheduler) Ready() bool {
45+
return scheduler.isReady
46+
}
47+
48+
func SetupTest(t *testing.T, realInterceptor bool, schedulerReady bool) (TestMongo, *MockScheduler) {
4449
// Turn off logging except for fatals
4550
log.SetLevel(log.FatalLevel)
4651

@@ -53,7 +58,9 @@ func SetupTest(t *testing.T, realInterceptor bool) (TestMongo, *MockScheduler) {
5358
mongo.Client.Database(os.Getenv("MONGO_DB")).Collection("discord_messages").Drop(context.Background())
5459

5560
// Mock scheduler
56-
scheduler := &MockScheduler{}
61+
scheduler := &MockScheduler{
62+
isReady: schedulerReady,
63+
}
5764

5865
var interceptor ecfmp_grpc.AuthInterceptor
5966
if realInterceptor {
@@ -114,7 +121,7 @@ func setupGrpcClient() TestClient {
114121
}
115122

116123
func Test_ItCreatesADiscordMessage(t *testing.T) {
117-
mongo, scheduler := SetupTest(t, false)
124+
mongo, scheduler := SetupTest(t, false, true)
118125
defer mongo.tearDown()
119126

120127
grpcClient := setupGrpcClient()
@@ -177,7 +184,7 @@ func Test_ItCreatesADiscordMessage(t *testing.T) {
177184
}
178185

179186
func Test_ItCreatesADiscordMessageWithNoContent(t *testing.T) {
180-
mongo, scheduler := SetupTest(t, false)
187+
mongo, scheduler := SetupTest(t, false, true)
181188
defer mongo.tearDown()
182189

183190
grpcClient := setupGrpcClient()
@@ -240,7 +247,7 @@ func Test_ItCreatesADiscordMessageWithNoContent(t *testing.T) {
240247
}
241248

242249
func Test_ItAllowsCreateIfAuthenticated(t *testing.T) {
243-
mongo, _ := SetupTest(t, true)
250+
mongo, _ := SetupTest(t, true, true)
244251
defer mongo.tearDown()
245252

246253
grpcClient := setupGrpcClient()
@@ -289,7 +296,7 @@ func Test_ItAllowsCreateIfAuthenticated(t *testing.T) {
289296
}
290297

291298
func Test_ItForbidsCreateIfNotAuthenticated(t *testing.T) {
292-
mongo, _ := SetupTest(t, true)
299+
mongo, _ := SetupTest(t, true, true)
293300
defer mongo.tearDown()
294301

295302
grpcClient := setupGrpcClient()
@@ -336,7 +343,7 @@ func Test_ItForbidsCreateIfNotAuthenticated(t *testing.T) {
336343
}
337344

338345
func Test_ItReturnsPrexistingIdIfRequestAlreadyExists(t *testing.T) {
339-
mongo, scheduler := SetupTest(t, false)
346+
mongo, scheduler := SetupTest(t, false, true)
340347
defer mongo.tearDown()
341348

342349
grpcClient := setupGrpcClient()
@@ -358,7 +365,7 @@ func Test_ItReturnsPrexistingIdIfRequestAlreadyExists(t *testing.T) {
358365
}
359366

360367
func Test_ItRejectsRequestsThatDontHaveAClientRequestId(t *testing.T) {
361-
mongo, scheduler := SetupTest(t, false)
368+
mongo, scheduler := SetupTest(t, false, true)
362369
defer mongo.tearDown()
363370

364371
grpcClient := setupGrpcClient()
@@ -374,7 +381,7 @@ func Test_ItRejectsRequestsThatDontHaveAClientRequestId(t *testing.T) {
374381
}
375382

376383
func Test_ItRejectsRequestsThatHaveAnEmptyClientRequestId(t *testing.T) {
377-
mongo, scheduler := SetupTest(t, false)
384+
mongo, scheduler := SetupTest(t, false, true)
378385
defer mongo.tearDown()
379386

380387
grpcClient := setupGrpcClient()
@@ -392,7 +399,7 @@ func Test_ItRejectsRequestsThatHaveAnEmptyClientRequestId(t *testing.T) {
392399
}
393400

394401
func Test_ItRejectsAMessageThatHasMissingChannel(t *testing.T) {
395-
mongo, scheduler := SetupTest(t, false)
402+
mongo, scheduler := SetupTest(t, false, true)
396403
defer mongo.tearDown()
397404

398405
grpcClient := setupGrpcClient()
@@ -435,7 +442,7 @@ func Test_ItRejectsAMessageThatHasMissingChannel(t *testing.T) {
435442
}
436443

437444
func Test_ItRejectsAMessageThatHasEmptyChannel(t *testing.T) {
438-
mongo, scheduler := SetupTest(t, false)
445+
mongo, scheduler := SetupTest(t, false, true)
439446
defer mongo.tearDown()
440447

441448
grpcClient := setupGrpcClient()
@@ -479,7 +486,7 @@ func Test_ItRejectsAMessageThatHasEmptyChannel(t *testing.T) {
479486
}
480487

481488
func Test_ItRejectsAMessageThatHasMissingFieldName(t *testing.T) {
482-
mongo, scheduler := SetupTest(t, false)
489+
mongo, scheduler := SetupTest(t, false, true)
483490
defer mongo.tearDown()
484491

485492
grpcClient := setupGrpcClient()
@@ -523,7 +530,7 @@ func Test_ItRejectsAMessageThatHasMissingFieldName(t *testing.T) {
523530
}
524531

525532
func Test_ItRejectsAMessageThatHasMissingFieldValue(t *testing.T) {
526-
mongo, scheduler := SetupTest(t, false)
533+
mongo, scheduler := SetupTest(t, false, true)
527534
defer mongo.tearDown()
528535

529536
grpcClient := setupGrpcClient()
@@ -567,7 +574,7 @@ func Test_ItRejectsAMessageThatHasMissingFieldValue(t *testing.T) {
567574
}
568575

569576
func Test_ItDoesAHealthCheck(t *testing.T) {
570-
mongo, _ := SetupTest(t, false)
577+
mongo, _ := SetupTest(t, false, true)
571578
defer mongo.tearDown()
572579

573580
grpcClient := setupGrpcClient()
@@ -580,8 +587,22 @@ func Test_ItDoesAHealthCheck(t *testing.T) {
580587
assert.Equal(t, resp.Status, pb_health.HealthCheckResponse_SERVING)
581588
}
582589

590+
func Test_ItDoesAFailedHealthCheck(t *testing.T) {
591+
mongo, _ := SetupTest(t, false, false)
592+
defer mongo.tearDown()
593+
594+
grpcClient := setupGrpcClient()
595+
defer grpcClient.close()
596+
597+
client := pb_health.NewHealthClient(grpcClient.conn)
598+
599+
resp, err := client.Check(context.Background(), &pb_health.HealthCheckRequest{})
600+
assert.Nil(t, err)
601+
assert.Equal(t, resp.Status, pb_health.HealthCheckResponse_NOT_SERVING)
602+
}
603+
583604
func Test_ItDoesAHealthCheckIfUnauthenticated(t *testing.T) {
584-
mongo, _ := SetupTest(t, true)
605+
mongo, _ := SetupTest(t, true, true)
585606
defer mongo.tearDown()
586607

587608
grpcClient := setupGrpcClient()
@@ -595,7 +616,7 @@ func Test_ItDoesAHealthCheckIfUnauthenticated(t *testing.T) {
595616
}
596617

597618
func Test_ItUpdatesAMessage(t *testing.T) {
598-
mongo, scheduler := SetupTest(t, false)
619+
mongo, scheduler := SetupTest(t, false, true)
599620
defer mongo.tearDown()
600621

601622
grpcClient := setupGrpcClient()
@@ -663,7 +684,7 @@ func Test_ItUpdatesAMessage(t *testing.T) {
663684
}
664685

665686
func Test_ItUpdatesAMessageWithNoContent(t *testing.T) {
666-
mongo, scheduler := SetupTest(t, false)
687+
mongo, scheduler := SetupTest(t, false, true)
667688
defer mongo.tearDown()
668689

669690
grpcClient := setupGrpcClient()
@@ -731,7 +752,7 @@ func Test_ItUpdatesAMessageWithNoContent(t *testing.T) {
731752
}
732753

733754
func Test_ItUpdatesAMessageIfAuthenticated(t *testing.T) {
734-
mongo, _ := SetupTest(t, true)
755+
mongo, _ := SetupTest(t, true, true)
735756
defer mongo.tearDown()
736757

737758
grpcClient := setupGrpcClient()
@@ -785,7 +806,7 @@ func Test_ItUpdatesAMessageIfAuthenticated(t *testing.T) {
785806
}
786807

787808
func Test_ItRejectsAMessageIfNotAuthenticated(t *testing.T) {
788-
mongo, _ := SetupTest(t, true)
809+
mongo, _ := SetupTest(t, true, true)
789810
defer mongo.tearDown()
790811

791812
grpcClient := setupGrpcClient()
@@ -835,7 +856,7 @@ func Test_ItRejectsAMessageIfNotAuthenticated(t *testing.T) {
835856
}
836857

837858
func Test_ItDoesntUpdateAMessageNotFound(t *testing.T) {
838-
mongo, scheduler := SetupTest(t, false)
859+
mongo, scheduler := SetupTest(t, false, true)
839860
defer mongo.tearDown()
840861

841862
grpcClient := setupGrpcClient()
@@ -856,7 +877,7 @@ func Test_ItDoesntUpdateAMessageNotFound(t *testing.T) {
856877
}
857878

858879
func Test_ItDoesntUpdateAMessageNoIdSpecified(t *testing.T) {
859-
mongo, scheduler := SetupTest(t, false)
880+
mongo, scheduler := SetupTest(t, false, true)
860881
defer mongo.tearDown()
861882

862883
grpcClient := setupGrpcClient()
@@ -877,7 +898,7 @@ func Test_ItDoesntUpdateAMessageNoIdSpecified(t *testing.T) {
877898
}
878899

879900
func Test_ItRejectsAnUpdateThatHasMissingFieldName(t *testing.T) {
880-
mongo, scheduler := SetupTest(t, false)
901+
mongo, scheduler := SetupTest(t, false, true)
881902
defer mongo.tearDown()
882903

883904
grpcClient := setupGrpcClient()
@@ -924,7 +945,7 @@ func Test_ItRejectsAnUpdateThatHasMissingFieldName(t *testing.T) {
924945
}
925946

926947
func Test_ItRejectsAnUpdateThatHasMissingFieldValue(t *testing.T) {
927-
mongo, scheduler := SetupTest(t, false)
948+
mongo, scheduler := SetupTest(t, false, true)
928949
defer mongo.tearDown()
929950

930951
grpcClient := setupGrpcClient()
@@ -971,7 +992,7 @@ func Test_ItRejectsAnUpdateThatHasMissingFieldValue(t *testing.T) {
971992
}
972993

973994
func Test_ItDoesntUpdateAMessageClientRequestIdEmpty(t *testing.T) {
974-
mongo, scheduler := SetupTest(t, false)
995+
mongo, scheduler := SetupTest(t, false, true)
975996
defer mongo.tearDown()
976997

977998
grpcClient := setupGrpcClient()
@@ -992,7 +1013,7 @@ func Test_ItDoesntUpdateAMessageClientRequestIdEmpty(t *testing.T) {
9921013
}
9931014

9941015
func Test_ItDoesntUpdateAMessageClientRequestIdMissing(t *testing.T) {
995-
mongo, scheduler := SetupTest(t, false)
1016+
mongo, scheduler := SetupTest(t, false, true)
9961017
defer mongo.tearDown()
9971018

9981019
grpcClient := setupGrpcClient()
@@ -1011,7 +1032,7 @@ func Test_ItDoesntUpdateAMessageClientRequestIdMissing(t *testing.T) {
10111032
}
10121033

10131034
func Test_ItCanCreateThenUpdateAMessage(t *testing.T) {
1014-
mongo, scheduler := SetupTest(t, false)
1035+
mongo, scheduler := SetupTest(t, false, true)
10151036
defer mongo.tearDown()
10161037

10171038
grpcClient := setupGrpcClient()

0 commit comments

Comments
 (0)