diff --git a/packages/orchestrator/pkg/server/sandboxes.go b/packages/orchestrator/pkg/server/sandboxes.go index a20995aabb..f3ade545dc 100644 --- a/packages/orchestrator/pkg/server/sandboxes.go +++ b/packages/orchestrator/pkg/server/sandboxes.go @@ -152,7 +152,7 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ if runningSandboxes >= maxRunningSandboxesPerNode { telemetry.ReportEvent(ctx, "max number of running sandboxes reached") - return nil, status.Errorf(codes.ResourceExhausted, "max number of running sandboxes on node reached (%d), please retry", maxRunningSandboxesPerNode) + return nil, status.Errorf(codes.ResourceExhausted, "max number of running sandboxes on node reached: current=%d, max=%d, please retry", runningSandboxes, maxRunningSandboxesPerNode) } // Check if we've reached the max number of starting instances on this node @@ -166,7 +166,7 @@ func (s *Server) Create(ctx context.Context, req *orchestrator.SandboxCreateRequ if !acquired { telemetry.ReportEvent(ctx, "too many starting sandboxes on node") - return nil, status.Errorf(codes.ResourceExhausted, "too many sandboxes starting on this node, please retry") + return nil, status.Errorf(codes.ResourceExhausted, "too many sandboxes starting on this node: current=%d, max=%d, please retry", s.startingSandboxes.Current(), s.startingSandboxes.Limit()) } } defer s.startingSandboxes.Release(1) diff --git a/packages/orchestrator/pkg/server/utils.go b/packages/orchestrator/pkg/server/utils.go index 3ef314b3e6..8f82ebacce 100644 --- a/packages/orchestrator/pkg/server/utils.go +++ b/packages/orchestrator/pkg/server/utils.go @@ -22,7 +22,7 @@ func (s *Server) waitForAcquire(ctx context.Context) error { if err != nil { telemetry.ReportEvent(ctx, "too many resuming sandboxes on node") - return status.Errorf(codes.ResourceExhausted, "too many sandboxes resuming on this node, please retry") + return status.Errorf(codes.ResourceExhausted, "too many sandboxes resuming on this node: current=%d, max=%d, please retry", s.startingSandboxes.Current(), s.startingSandboxes.Limit()) } return nil diff --git a/packages/shared/pkg/utils/resizable_semaphore.go b/packages/shared/pkg/utils/resizable_semaphore.go index db1542335d..ccfdfc299b 100644 --- a/packages/shared/pkg/utils/resizable_semaphore.go +++ b/packages/shared/pkg/utils/resizable_semaphore.go @@ -92,6 +92,20 @@ func (s *AdjustableSemaphore) SetLimit(limit int64) error { return nil } +func (s *AdjustableSemaphore) Current() int64 { + s.mu.Lock() + defer s.mu.Unlock() + + return s.used +} + +func (s *AdjustableSemaphore) Limit() int64 { + s.mu.Lock() + defer s.mu.Unlock() + + return s.limit +} + func (s *AdjustableSemaphore) Release(n int64) { if n <= 0 { panic("Release: n must be > 0")