Skip to content

orchestrator: include current/max counts in ResourceExhausted error messages #3584

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AdaAibaby wants to merge 1 commit into
base: main
Choose a base branch
from
Open

orchestrator: include current/max counts in ResourceExhausted error messages #3584

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions packages/orchestrator/pkg/server/sandboxes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ
if runningSandboxes >= maxRunningSandboxesPerNode {
telemetry.ReportEvent(ctx, "max number of running sandboxes reached")

return nil, status.Errorf(codes.ResourceExhausted, "max number of running sandboxes on node reached (%d), please retry", maxRunningSandboxesPerNode)
return nil, status.Errorf(codes.ResourceExhausted, "max number of running sandboxes on node reached: current=%d, max=%d, please retry", runningSandboxes, maxRunningSandboxesPerNode)
}

// Check if we've reached the max number of starting instances on this node
Expand All @@ -166,7 +166,7 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ
if !acquired {
telemetry.ReportEvent(ctx, "too many starting sandboxes on node")

return nil, status.Errorf(codes.ResourceExhausted, "too many sandboxes starting on this node, please retry")
return nil, status.Errorf(codes.ResourceExhausted, "too many sandboxes starting on this node: current=%d, max=%d, please retry", s.startingSandboxes.Current(), s.startingSandboxes.Limit())
}
}
defer s.startingSandboxes.Release(1)
Expand Down
2 changes: 1 addition & 1 deletion packages/orchestrator/pkg/server/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ func (s *Server) waitForAcquire(ctx context.Context) error {
if err != nil {
telemetry.ReportEvent(ctx, "too many resuming sandboxes on node")

return status.Errorf(codes.ResourceExhausted, "too many sandboxes resuming on this node, please retry")
return status.Errorf(codes.ResourceExhausted, "too many sandboxes resuming on this node: current=%d, max=%d, please retry", s.startingSandboxes.Current(), s.startingSandboxes.Limit())
}

return nil
Expand Down
14 changes: 14 additions & 0 deletions packages/shared/pkg/utils/resizable_semaphore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,20 @@ func (s *AdjustableSemaphore) SetLimit(limit int64) error {
return nil
}

func (s *AdjustableSemaphore) Current() int64 {
s.mu.Lock()
defer s.mu.Unlock()

return s.used
}

func (s *AdjustableSemaphore) Limit() int64 {
s.mu.Lock()
defer s.mu.Unlock()

return s.limit
}

func (s *AdjustableSemaphore) Release(n int64) {
if n <= 0 {
panic("Release: n must be > 0")
Expand Down