Repository navigation
node: keep trusted keys the control plane still lists (#916) #917
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -118,11 +118,46 @@ var ( | |
|
|
||
| var ErrFatalAuth = errors.New("fatal authentication error") | ||
|
|
||
| // TrustedKey is a control plane public key the node trusts. ReceivedAt is | ||
| // the last time the control plane vouched for the key, by listing it in /keys | ||
| // or announcing it in a rotation event; key pruning ages keys from it. | ||
| type TrustedKey struct { | ||
| Key ed25519.PublicKey | ||
| ReceivedAt time.Time | ||
| } | ||
|
|
||
| // neverVouched is the ReceivedAt of a key the control plane has not vouched | ||
| // for in this trust set's lifetime: older than any real one, and, unlike the | ||
| // zero time, kept as it is through the store, which reads a missing time as | ||
| // now. | ||
| var neverVouched = time.Unix(0, 0).UTC() | ||
|
|
||
| // StaleIdentityError is Start's error when the stored identity fails the | ||
| // role check against every trusted key and neither recovery renewed it. It | ||
| // unwraps to the role check's error and to both recovery errors, so a caller | ||
| // can tell a control plane that refused the identity (a *RefreshError with | ||
| // its status) from one that never answered. | ||
| type StaleIdentityError struct { | ||
| Role string | ||
| RoleErr error | ||
| RefreshErr error | ||
| RecoveryErr error | ||
| } | ||
|
|
||
| func (e *StaleIdentityError) Error() string { | ||
| return fmt.Sprintf("loaded identity fails role requirement %q (autonomous refresh: %v; refresh-token recovery: %v): %v", e.Role, e.RefreshErr, e.RecoveryErr, e.RoleErr) | ||
| } | ||
|
|
||
| func (e *StaleIdentityError) Unwrap() []error { | ||
| var errs []error | ||
| for _, err := range []error{e.RoleErr, e.RefreshErr, e.RecoveryErr} { | ||
| if err != nil { | ||
| errs = append(errs, err) | ||
| } | ||
| } | ||
| return errs | ||
| } | ||
|
Comment on lines
+135
to
+159
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. #916 does not need this type, and nothing in this PR reads it. Please remove it here, together with its test, and send it in a separate PR with the code that uses it. |
||
|
|
||
| type nodeRelayACL struct { | ||
| node *AgentMeshNode | ||
| } | ||
|
|
@@ -483,7 +518,15 @@ func NewAgentMeshNode(cfg Options) (*AgentMeshNode, error) { | |
| } | ||
| } | ||
| if len(cfg.ControlPlanePubKey) > 0 && !containsTrustedKey(trustedKeys, cfg.ControlPlanePubKey) { | ||
| trustedKeys = append(trustedKeys, TrustedKey{Key: cfg.ControlPlanePubKey, ReceivedAt: time.Now()}) | ||
| // The configured key is only as fresh as the enrollment or /info | ||
| // answer that stored it. Next to persisted keys it must not count as | ||
| // just vouched for, or it would outrank the control plane's newer | ||
| // keys when pruning picks the newest; alone, it is all there is. | ||
| vouched := neverVouched | ||
| if len(trustedKeys) == 0 { | ||
| vouched = time.Now() | ||
| } | ||
| trustedKeys = append(trustedKeys, TrustedKey{Key: cfg.ControlPlanePubKey, ReceivedAt: vouched}) | ||
|
Comment on lines
+521
to
+529
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is a different problem from #916. Please remove it from this PR, with |
||
| } | ||
|
|
||
| node := &AgentMeshNode{ | ||
|
|
@@ -579,7 +622,7 @@ func (n *AgentMeshNode) Start(ctx context.Context) error { | |
| } else { | ||
| logger.Warnf("Autonomous refresh refused (%v); attempting recovery via stored refresh token", refreshErr) | ||
| if recErr := n.ReEnrollWithRefreshToken(ctx); recErr != nil { | ||
| return fmt.Errorf("loaded identity fails role requirement %q (autonomous refresh: %v; refresh-token recovery: %v): %w", n.config.RequiredRole, refreshErr, recErr, roleErr) | ||
| return &StaleIdentityError{Role: n.config.RequiredRole, RoleErr: roleErr, RefreshErr: refreshErr, RecoveryErr: recErr} | ||
| } | ||
| logger.Info("Identity recovered via refresh-token re-enrollment.") | ||
| } | ||
|
|
@@ -1979,29 +2022,52 @@ func (n *AgentMeshNode) handleKeyRotationEvent(event *api.MeshEvent) { | |
| n.triggerControlPlaneSync() | ||
| } | ||
|
|
||
| // pruneTrustedKeys drops keys older than gracePeriod but always keeps the | ||
| // most recently received one: the current signing key has no successor until | ||
| // a rotation event arrives, so ageing it out would empty the trust set and | ||
| // leave the node unable to verify any peer or router. | ||
| func pruneTrustedKeys(keys []TrustedKey, now time.Time, gracePeriod time.Duration) []TrustedKey { | ||
| // pruneTrustedKeys drops keys the control plane has not vouched for within | ||
| // gracePeriod, as when it stopped listing them or the node could not reach | ||
| // it, but always keeps the most recently vouched-for ones: the current | ||
| // signing key has no successor until a rotation event arrives, so ageing it | ||
| // out would empty the trust set and leave the node unable to verify any peer | ||
| // or router. A /keys answer vouches for all its keys at once and says | ||
| // nothing about which is current, so every key with the latest time is kept, | ||
| // never one of them by its place in the list. It also keeps every key keep reports, which is the key that | ||
| // signed the node's own identity: the control plane lists all its valid keys | ||
| // alike, so the newest vouched-for key need not be the one a refresh signed | ||
| // with, and dropping that one after a long time offline leaves a valid | ||
| // identity that fails the check at the next start, when only the control | ||
| // plane can renew it. | ||
| func pruneTrustedKeys(keys []TrustedKey, now time.Time, gracePeriod time.Duration, keep func(ed25519.PublicKey) bool) []TrustedKey { | ||
|
Comment on lines
+2025
to
+2038
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please remove the
The loop below stays as it is, without // pruneTrustedKeys drops the keys the control plane last vouched for more
// than gracePeriod ago, but always keeps the ones with the latest time: a
// node that cannot reach the control plane must not age out its whole trust
// set and lose the means to verify any peer or router. A /keys answer vouches
// for all its keys at once and its order does not say which is current, so
// the latest time keeps all of them, not the first one in the list.
func pruneTrustedKeys(keys []TrustedKey, now time.Time, gracePeriod time.Duration) []TrustedKey {In n.keysMu.Lock()
before := n.trustedKeys
pruned := pruneTrustedKeys(before, time.Now(), gracePeriod)
changed := len(pruned) != len(before)
n.trustedKeys = pruned
snapshot := append([]TrustedKey(nil), pruned...)
n.keysMu.Unlock()
for _, tk := range before {
if !containsTrustedKey(pruned, tk.Key) {
logger.Infow("[KeyPruning] dropped control plane key", "key", fmt.Sprintf("%x", tk.Key), "last_vouched", tk.ReceivedAt.UTC().Format(time.RFC3339), "grace_period", gracePeriod.String())
}
} |
||
| if len(keys) == 0 { | ||
| return keys | ||
| } | ||
| newest := 0 | ||
| for i, tk := range keys { | ||
| if tk.ReceivedAt.After(keys[newest].ReceivedAt) { | ||
| newest = i | ||
| latest := keys[0].ReceivedAt | ||
| for _, tk := range keys { | ||
| if tk.ReceivedAt.After(latest) { | ||
| latest = tk.ReceivedAt | ||
| } | ||
| } | ||
| var active []TrustedKey | ||
| for i, tk := range keys { | ||
| if i == newest || now.Sub(tk.ReceivedAt) <= gracePeriod { | ||
| for _, tk := range keys { | ||
| if tk.ReceivedAt.Equal(latest) || now.Sub(tk.ReceivedAt) <= gracePeriod || (keep != nil && keep(tk.Key)) { | ||
| active = append(active, tk) | ||
| } | ||
| } | ||
| return active | ||
| } | ||
|
|
||
| // signedOwnIdentity reports whether key signed the node's current identity. | ||
| func (n *AgentMeshNode) signedOwnIdentity(key ed25519.PublicKey) bool { | ||
| token := n.GetIdentity() | ||
| if len(token) == 0 { | ||
| return false | ||
| } | ||
| b, err := biscuit.Unmarshal(token) | ||
| if err != nil { | ||
| return false | ||
| } | ||
| _, err = b.Authorizer(key) | ||
| return err == nil | ||
| } | ||
|
|
||
| func (n *AgentMeshNode) startKeyPruning(ctx context.Context, gracePeriod time.Duration) { | ||
| if gracePeriod <= 0 { | ||
| return | ||
|
|
@@ -2014,7 +2080,7 @@ func (n *AgentMeshNode) startKeyPruning(ctx context.Context, gracePeriod time.Du | |
| case <-ticker.C: | ||
| logger.Info("[KeyPruning] Pruning expired keys...") | ||
| n.keysMu.Lock() | ||
| pruned := pruneTrustedKeys(n.trustedKeys, time.Now(), gracePeriod) | ||
| pruned := pruneTrustedKeys(n.trustedKeys, time.Now(), gracePeriod, n.signedOwnIdentity) | ||
| changed := len(pruned) != len(n.trustedKeys) | ||
| n.trustedKeys = pruned | ||
| snapshot := append([]TrustedKey(nil), pruned...) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -353,7 +353,7 @@ func TestPruneTrustedKeys(t *testing.T) { | |
|
|
||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
| got := pruneTrustedKeys(tt.in, now, grace) | ||
| got := pruneTrustedKeys(tt.in, now, grace, nil) | ||
| if len(got) != len(tt.want) { | ||
| t.Fatalf("got %d keys, want %d", len(got), len(tt.want)) | ||
| } | ||
|
|
@@ -366,6 +366,45 @@ func TestPruneTrustedKeys(t *testing.T) { | |
| } | ||
| } | ||
|
|
||
| // TestKeyPruningKeepsTheIdentitySigner covers a node whose identity was | ||
| // signed by a key it learned before the newest one it holds, as when the | ||
| // control plane's newest key reached it earlier or its stored key was added | ||
| // again at a restart: pruning must not drop the signer, or the next start | ||
| // finds a valid identity that no trusted key verifies. | ||
| func TestKeyPruningKeepsTheIdentitySigner(t *testing.T) { | ||
|
Comment on lines
+369
to
+374
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. For the reduced change, two tests are sufficient. Both fail on
// The node got the current key at enrollment and the retiring key from a
// later sync. While /keys lists both, pruning must keep both, whatever
// the time since the node first saw each (#916). A key that /keys no
// longer lists is dropped at the sync.
t.Run("a key /keys still lists outlives the grace period", func(t *testing.T) {
expiredPub, _ := mustGenerateKey(t)
srv := keysServer(t, []ed25519.PublicKey{retiredPub, currentPub}, []ed25519.PrivateKey{retiredPriv, currentPriv})
defer srv.Close()
n := newNode(t)
enrolled := time.Now().Add(-25 * time.Hour)
n.trustedKeys = []TrustedKey{{Key: currentPub, ReceivedAt: enrolled}, {Key: expiredPub, ReceivedAt: enrolled}}
if err := n.syncTrustedKeys(context.Background(), srv.URL); err != nil {
t.Fatalf("syncTrustedKeys: %v", err)
}
if containsTrustedKey(n.trustedKeys, expiredPub) {
t.Error("a key /keys does not list stayed in the trust set")
}
got := pruneTrustedKeys(n.trustedKeys, time.Now(), 24*time.Hour)
if len(got) != 2 || !containsTrustedKey(got, retiredPub) || !containsTrustedKey(got, currentPub) {
t.Errorf("pruning kept %d keys, want the retired and the current key that /keys lists", len(got))
}
})
sameSync := k(48 * time.Hour) {"every key of the latest sync survives", []TrustedKey{stale, staler, sameSync}, []TrustedKey{stale, sameSync}},Please remove |
||
| now := time.Now() | ||
| signerPub, signerPriv := mustGenerateKey(t) | ||
| otherPub, _ := mustGenerateKey(t) | ||
| stalePub, _ := mustGenerateKey(t) | ||
| priv, _, err := crypto.GenerateEd25519Key(nil) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| pid, err := peer.IDFromPrivateKey(priv) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| token, err := identity.MintBootstrapBiscuitToken(signerPriv, pid, api.RoleNode, now.Add(24*time.Hour), nil, nil) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| n := &AgentMeshNode{} | ||
| n.SetIdentityCache(token) | ||
|
|
||
| keys := []TrustedKey{ | ||
| {Key: signerPub, ReceivedAt: now.Add(-72 * time.Hour)}, | ||
| {Key: stalePub, ReceivedAt: now.Add(-48 * time.Hour)}, | ||
| {Key: otherPub, ReceivedAt: now.Add(-time.Minute)}, | ||
| } | ||
| got := pruneTrustedKeys(keys, now, 24*time.Hour, n.signedOwnIdentity) | ||
| if len(got) != 2 || !got[0].Key.Equal(signerPub) || !got[1].Key.Equal(otherPub) { | ||
| t.Fatalf("got %d keys, want the identity's signer and the newest key", len(got)) | ||
| } | ||
| if !n.signedOwnIdentity(signerPub) || n.signedOwnIdentity(otherPub) { | ||
| t.Fatal("signedOwnIdentity does not tell the signer from another key") | ||
| } | ||
| } | ||
|
|
||
| // TestHandleAuthHandshake covers the admission office: the relay ACL trusts | ||
| // authPeers, so the cached instant must be the token's own expiry, and a token | ||
| // that is expired or bound to another peer must leave no entry at all (#296). | ||
|
|
@@ -860,3 +899,83 @@ func TestFindProvidersSkipsRevoked(t *testing.T) { | |
| t.Errorf("revoked peer returned as provider: %v", providers) | ||
| } | ||
| } | ||
|
|
||
| // TestStaleIdentityErrorUnwraps covers what a caller needs from a failed | ||
| // start: the refresh status when the control plane answered, and no status | ||
| // when it did not. | ||
| func TestStaleIdentityErrorUnwraps(t *testing.T) { | ||
| refused := &StaleIdentityError{Role: api.RoleNode, RoleErr: errors.New("biscuit: invalid signature"), RefreshErr: &RefreshError{StatusCode: 401, Message: "refused"}, RecoveryErr: errors.New("no refresh token")} | ||
| var re *RefreshError | ||
| if !errors.As(refused, &re) || re.StatusCode != 401 { | ||
| t.Fatalf("errors.As found no refresh status in %v", refused) | ||
| } | ||
| offline := &StaleIdentityError{Role: api.RoleNode, RoleErr: errors.New("biscuit: invalid signature"), RefreshErr: context.DeadlineExceeded} | ||
| if errors.As(offline, &re) || !errors.Is(offline, context.DeadlineExceeded) { | ||
| t.Fatalf("offline error unwraps wrong: %v", offline) | ||
| } | ||
| } | ||
|
|
||
| // TestKeyPruningKeepsEveryLatestKey covers a /keys answer that vouched for | ||
| // a retiring and a current key at once, followed by an outage longer than | ||
| // the grace period: the answer's order says nothing about which key is | ||
| // current, so pruning keeps both rather than whichever came first. | ||
| func TestKeyPruningKeepsEveryLatestKey(t *testing.T) { | ||
| now := time.Now() | ||
| synced := now.Add(-48 * time.Hour) | ||
| retiring, _ := mustGenerateKey(t) | ||
| current, _ := mustGenerateKey(t) | ||
| dropped, _ := mustGenerateKey(t) | ||
| keys := []TrustedKey{{Key: retiring, ReceivedAt: synced}, {Key: current, ReceivedAt: synced}, {Key: dropped, ReceivedAt: synced.Add(-time.Hour)}} | ||
| got := pruneTrustedKeys(keys, now, 24*time.Hour, func(k ed25519.PublicKey) bool { return k.Equal(retiring) }) | ||
| if len(got) != 2 || !got[0].Key.Equal(retiring) || !got[1].Key.Equal(current) { | ||
| t.Fatalf("got %d keys, want the retiring and the current key of the last sync", len(got)) | ||
| } | ||
| } | ||
|
|
||
| // TestFallbackKeyStaysOldThroughTheStore covers the configured key re-added | ||
| // next to persisted keys at a start: it must stay older than them after the | ||
| // set is saved and loaded again, or the next start would take it for the | ||
| // newest key and prune the real rotation key. | ||
| func TestFallbackKeyStaysOldThroughTheStore(t *testing.T) { | ||
| store, err := NewStore(t.TempDir()) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| t.Cleanup(func() { | ||
| if err := store.Close(); err != nil { | ||
| t.Errorf("close store: %v", err) | ||
| } | ||
| }) | ||
| rotated, _ := mustGenerateKey(t) | ||
| configured, _ := mustGenerateKey(t) | ||
| learned := time.Now().Add(-30 * time.Hour).Truncate(time.Second) | ||
| if err := store.SaveTrustedKeys([]TrustedKey{{Key: rotated, ReceivedAt: learned}}); err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| priv, _, err := crypto.GenerateEd25519Key(nil) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| n, err := NewAgentMeshNode(Options{PrivKey: priv, Store: store, ControlPlanePubKey: configured}) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| // A rotation event persists the set before any sync succeeds. | ||
| n.persistTrustedKeys(n.trustedKeys) | ||
| loaded, err := store.LoadTrustedKeys() | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| for _, tk := range loaded { | ||
| switch { | ||
| case tk.Key.Equal(configured) && !tk.ReceivedAt.Equal(neverVouched): | ||
| t.Fatalf("configured key loaded as vouched for at %s", tk.ReceivedAt) | ||
| case tk.Key.Equal(rotated) && !tk.ReceivedAt.Equal(learned): | ||
| t.Fatalf("rotation key loaded with %s, want %s", tk.ReceivedAt, learned) | ||
| } | ||
| } | ||
| got := pruneTrustedKeys(loaded, time.Now(), 24*time.Hour, nil) | ||
| if len(got) != 1 || !got[0].Key.Equal(rotated) { | ||
| t.Fatalf("pruning kept %d keys, want only the rotation key", len(got)) | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
existinghas no effect now, and the function does not merge. Please delete this function andTestMergeTrustedKeys, and give the keys their time insyncTrustedKeys:syncTrustedKeysstill readsexisting, to verify the answer.