Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 5 additions & 11 deletions internal/node/controlplane.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
package node

import (
"bytes"
"context"
"crypto/ed25519"
"time"
Expand Down Expand Up @@ -87,19 +86,14 @@ func FetchControlPlaneInfo(ctx context.Context, controlPlaneURL string) (*api.Co
}

// mergeTrustedKeys replaces the stored trust set with the authoritative set
// from /keys, preserving ReceivedAt for keys already known so local grace
// pruning keeps working across syncs.
// from /keys. The control plane lists every key that is still valid, its
// current key and any in their rotation grace period, so each listed key's
// ReceivedAt becomes now: local pruning then ages a key from the last time
// the control plane vouched for it, and never drops one it still lists.
func mergeTrustedKeys(existing []TrustedKey, fetched []ed25519.PublicKey, now time.Time) []TrustedKey {
merged := make([]TrustedKey, 0, len(fetched))
for _, key := range fetched {
tk := TrustedKey{Key: key, ReceivedAt: now}
for _, old := range existing {
if bytes.Equal(old.Key, key) {
tk.ReceivedAt = old.ReceivedAt
break
}
}
merged = append(merged, tk)
merged = append(merged, TrustedKey{Key: key, ReceivedAt: now})
}
return merged
}
Comment on lines 88 to 99

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

existing has no effect now, and the function does not merge. Please delete this function and TestMergeTrustedKeys, and give the keys their time in syncTrustedKeys:

	// The answer lists every key the control plane still accepts, so it
	// vouches for each of them now: pruning ages a key from the last answer
	// that listed it, not from the first time the node saw it.
	now := time.Now()
	synced := make([]TrustedKey, 0, len(keys))
	for _, key := range keys {
		synced = append(synced, TrustedKey{Key: key, ReceivedAt: now})
	}

syncTrustedKeys still reads existing, to verify the answer.

Expand Down
4 changes: 2 additions & 2 deletions internal/node/controlplane_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ func TestMergeTrustedKeys(t *testing.T) {
if len(got) != 2 {
t.Fatalf("expected 2 keys, got %d", len(got))
}
if !got[0].Key.Equal(keyA) || !got[0].ReceivedAt.Equal(earlier) {
t.Errorf("known key must keep its ReceivedAt: got %+v", got[0])
if !got[0].Key.Equal(keyA) || !got[0].ReceivedAt.Equal(now) {
t.Errorf("a key the control plane still lists must get the sync time: got %+v", got[0])
}
if !got[1].Key.Equal(keyB) || !got[1].ReceivedAt.Equal(now) {
t.Errorf("new key must get the sync time: got %+v", got[1])
Expand Down
94 changes: 80 additions & 14 deletions internal/node/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,11 +118,46 @@ var (

var ErrFatalAuth = errors.New("fatal authentication error")

// TrustedKey is a control plane public key the node trusts. ReceivedAt is
// the last time the control plane vouched for the key, by listing it in /keys
// or announcing it in a rotation event; key pruning ages keys from it.
type TrustedKey struct {
Key ed25519.PublicKey
ReceivedAt time.Time
}

// neverVouched is the ReceivedAt of a key the control plane has not vouched
// for in this trust set's lifetime: older than any real one, and, unlike the
// zero time, kept as it is through the store, which reads a missing time as
// now.
var neverVouched = time.Unix(0, 0).UTC()

// StaleIdentityError is Start's error when the stored identity fails the
// role check against every trusted key and neither recovery renewed it. It
// unwraps to the role check's error and to both recovery errors, so a caller
// can tell a control plane that refused the identity (a *RefreshError with
// its status) from one that never answered.
type StaleIdentityError struct {
Role string
RoleErr error
RefreshErr error
RecoveryErr error
}

func (e *StaleIdentityError) Error() string {
return fmt.Sprintf("loaded identity fails role requirement %q (autonomous refresh: %v; refresh-token recovery: %v): %v", e.Role, e.RefreshErr, e.RecoveryErr, e.RoleErr)
}

func (e *StaleIdentityError) Unwrap() []error {
var errs []error
for _, err := range []error{e.RoleErr, e.RefreshErr, e.RecoveryErr} {
if err != nil {
errs = append(errs, err)
}
}
return errs
}
Comment on lines +135 to +159

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#916 does not need this type, and nothing in this PR reads it. Please remove it here, together with its test, and send it in a separate PR with the code that uses it.


type nodeRelayACL struct {
node *AgentMeshNode
}
Expand Down Expand Up @@ -483,7 +518,15 @@ func NewAgentMeshNode(cfg Options) (*AgentMeshNode, error) {
}
}
if len(cfg.ControlPlanePubKey) > 0 && !containsTrustedKey(trustedKeys, cfg.ControlPlanePubKey) {
trustedKeys = append(trustedKeys, TrustedKey{Key: cfg.ControlPlanePubKey, ReceivedAt: time.Now()})
// The configured key is only as fresh as the enrollment or /info
// answer that stored it. Next to persisted keys it must not count as
// just vouched for, or it would outrank the control plane's newer
// keys when pruning picks the newest; alone, it is all there is.
vouched := neverVouched
if len(trustedKeys) == 0 {
vouched = time.Now()
}
trustedKeys = append(trustedKeys, TrustedKey{Key: cfg.ControlPlanePubKey, ReceivedAt: vouched})
Comment on lines +521 to +529

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a different problem from #916. LoadMeshConfig returns the first stored key, so the node comes here only when --control-plane-public-key gives a key that is not in the stored set. The change also writes the epoch as a special value into the persisted MemberCredential, which the SDKs load too.

Please remove it from this PR, with neverVouched and its test. If you can reproduce the problem, open an issue with the steps.

}

node := &AgentMeshNode{
Expand Down Expand Up @@ -579,7 +622,7 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
} else {
logger.Warnf("Autonomous refresh refused (%v); attempting recovery via stored refresh token", refreshErr)
if recErr := n.ReEnrollWithRefreshToken(ctx); recErr != nil {
return fmt.Errorf("loaded identity fails role requirement %q (autonomous refresh: %v; refresh-token recovery: %v): %w", n.config.RequiredRole, refreshErr, recErr, roleErr)
return &StaleIdentityError{Role: n.config.RequiredRole, RoleErr: roleErr, RefreshErr: refreshErr, RecoveryErr: recErr}
}
logger.Info("Identity recovered via refresh-token re-enrollment.")
}
Expand Down Expand Up @@ -1979,29 +2022,52 @@ func (n *AgentMeshNode) handleKeyRotationEvent(event *api.MeshEvent) {
n.triggerControlPlaneSync()
}

// pruneTrustedKeys drops keys older than gracePeriod but always keeps the
// most recently received one: the current signing key has no successor until
// a rotation event arrives, so ageing it out would empty the trust set and
// leave the node unable to verify any peer or router.
func pruneTrustedKeys(keys []TrustedKey, now time.Time, gracePeriod time.Duration) []TrustedKey {
// pruneTrustedKeys drops keys the control plane has not vouched for within
// gracePeriod, as when it stopped listing them or the node could not reach
// it, but always keeps the most recently vouched-for ones: the current
// signing key has no successor until a rotation event arrives, so ageing it
// out would empty the trust set and leave the node unable to verify any peer
// or router. A /keys answer vouches for all its keys at once and says
// nothing about which is current, so every key with the latest time is kept,
// never one of them by its place in the list. It also keeps every key keep reports, which is the key that
// signed the node's own identity: the control plane lists all its valid keys
// alike, so the newest vouched-for key need not be the one a refresh signed
// with, and dropping that one after a long time offline leaves a valid
// identity that fails the check at the next start, when only the control
// plane can renew it.
func pruneTrustedKeys(keys []TrustedKey, now time.Time, gracePeriod time.Duration, keep func(ed25519.PublicKey) bool) []TrustedKey {
Comment on lines +2025 to +2038

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please remove the keep parameter and signedOwnIdentity:

The loop below stays as it is, without keep. The comment becomes shorter, for example:

// pruneTrustedKeys drops the keys the control plane last vouched for more
// than gracePeriod ago, but always keeps the ones with the latest time: a
// node that cannot reach the control plane must not age out its whole trust
// set and lose the means to verify any peer or router. A /keys answer vouches
// for all its keys at once and its order does not say which is current, so
// the latest time keeps all of them, not the first one in the list.
func pruneTrustedKeys(keys []TrustedKey, now time.Time, gracePeriod time.Duration) []TrustedKey {

In startKeyPruning, log each key that pruning drops:

				n.keysMu.Lock()
				before := n.trustedKeys
				pruned := pruneTrustedKeys(before, time.Now(), gracePeriod)
				changed := len(pruned) != len(before)
				n.trustedKeys = pruned
				snapshot := append([]TrustedKey(nil), pruned...)
				n.keysMu.Unlock()
				for _, tk := range before {
					if !containsTrustedKey(pruned, tk.Key) {
						logger.Infow("[KeyPruning] dropped control plane key", "key", fmt.Sprintf("%x", tk.Key), "last_vouched", tk.ReceivedAt.UTC().Format(time.RFC3339), "grace_period", gracePeriod.String())
					}
				}

if len(keys) == 0 {
return keys
}
newest := 0
for i, tk := range keys {
if tk.ReceivedAt.After(keys[newest].ReceivedAt) {
newest = i
latest := keys[0].ReceivedAt
for _, tk := range keys {
if tk.ReceivedAt.After(latest) {
latest = tk.ReceivedAt
}
}
var active []TrustedKey
for i, tk := range keys {
if i == newest || now.Sub(tk.ReceivedAt) <= gracePeriod {
for _, tk := range keys {
if tk.ReceivedAt.Equal(latest) || now.Sub(tk.ReceivedAt) <= gracePeriod || (keep != nil && keep(tk.Key)) {
active = append(active, tk)
}
}
return active
}

// signedOwnIdentity reports whether key signed the node's current identity.
func (n *AgentMeshNode) signedOwnIdentity(key ed25519.PublicKey) bool {
token := n.GetIdentity()
if len(token) == 0 {
return false
}
b, err := biscuit.Unmarshal(token)
if err != nil {
return false
}
_, err = b.Authorizer(key)
return err == nil
}

func (n *AgentMeshNode) startKeyPruning(ctx context.Context, gracePeriod time.Duration) {
if gracePeriod <= 0 {
return
Expand All @@ -2014,7 +2080,7 @@ func (n *AgentMeshNode) startKeyPruning(ctx context.Context, gracePeriod time.Du
case <-ticker.C:
logger.Info("[KeyPruning] Pruning expired keys...")
n.keysMu.Lock()
pruned := pruneTrustedKeys(n.trustedKeys, time.Now(), gracePeriod)
pruned := pruneTrustedKeys(n.trustedKeys, time.Now(), gracePeriod, n.signedOwnIdentity)
changed := len(pruned) != len(n.trustedKeys)
n.trustedKeys = pruned
snapshot := append([]TrustedKey(nil), pruned...)
Expand Down
121 changes: 120 additions & 1 deletion internal/node/node_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,7 @@ func TestPruneTrustedKeys(t *testing.T) {

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := pruneTrustedKeys(tt.in, now, grace)
got := pruneTrustedKeys(tt.in, now, grace, nil)
if len(got) != len(tt.want) {
t.Fatalf("got %d keys, want %d", len(got), len(tt.want))
}
Expand All @@ -366,6 +366,45 @@ func TestPruneTrustedKeys(t *testing.T) {
}
}

// TestKeyPruningKeepsTheIdentitySigner covers a node whose identity was
// signed by a key it learned before the newest one it holds, as when the
// control plane's newest key reached it earlier or its stored key was added
// again at a restart: pruning must not drop the signer, or the next start
// finds a valid identity that no trusted key verifies.
func TestKeyPruningKeepsTheIdentitySigner(t *testing.T) {
Comment on lines +369 to +374

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For the reduced change, two tests are sufficient. Both fail on main.

  1. A subtest in TestSyncTrustedKeys for the sequence of the issue:
	// The node got the current key at enrollment and the retiring key from a
	// later sync. While /keys lists both, pruning must keep both, whatever
	// the time since the node first saw each (#916). A key that /keys no
	// longer lists is dropped at the sync.
	t.Run("a key /keys still lists outlives the grace period", func(t *testing.T) {
		expiredPub, _ := mustGenerateKey(t)
		srv := keysServer(t, []ed25519.PublicKey{retiredPub, currentPub}, []ed25519.PrivateKey{retiredPriv, currentPriv})
		defer srv.Close()
		n := newNode(t)
		enrolled := time.Now().Add(-25 * time.Hour)
		n.trustedKeys = []TrustedKey{{Key: currentPub, ReceivedAt: enrolled}, {Key: expiredPub, ReceivedAt: enrolled}}
		if err := n.syncTrustedKeys(context.Background(), srv.URL); err != nil {
			t.Fatalf("syncTrustedKeys: %v", err)
		}
		if containsTrustedKey(n.trustedKeys, expiredPub) {
			t.Error("a key /keys does not list stayed in the trust set")
		}
		got := pruneTrustedKeys(n.trustedKeys, time.Now(), 24*time.Hour)
		if len(got) != 2 || !containsTrustedKey(got, retiredPub) || !containsTrustedKey(got, currentPub) {
			t.Errorf("pruning kept %d keys, want the retired and the current key that /keys lists", len(got))
		}
	})
  1. A row in TestPruneTrustedKeys. Add it at the end, because the table sets tests[3].want by index:
	sameSync := k(48 * time.Hour)
		{"every key of the latest sync survives", []TrustedKey{stale, staler, sameSync}, []TrustedKey{stale, sameSync}},

Please remove TestKeyPruningKeepsTheIdentitySigner, TestKeyPruningKeepsEveryLatestKey, TestFallbackKeyStaysOldThroughTheStore and TestStaleIdentityErrorUnwraps together with the code that they test.

now := time.Now()
signerPub, signerPriv := mustGenerateKey(t)
otherPub, _ := mustGenerateKey(t)
stalePub, _ := mustGenerateKey(t)
priv, _, err := crypto.GenerateEd25519Key(nil)
if err != nil {
t.Fatal(err)
}
pid, err := peer.IDFromPrivateKey(priv)
if err != nil {
t.Fatal(err)
}
token, err := identity.MintBootstrapBiscuitToken(signerPriv, pid, api.RoleNode, now.Add(24*time.Hour), nil, nil)
if err != nil {
t.Fatal(err)
}
n := &AgentMeshNode{}
n.SetIdentityCache(token)

keys := []TrustedKey{
{Key: signerPub, ReceivedAt: now.Add(-72 * time.Hour)},
{Key: stalePub, ReceivedAt: now.Add(-48 * time.Hour)},
{Key: otherPub, ReceivedAt: now.Add(-time.Minute)},
}
got := pruneTrustedKeys(keys, now, 24*time.Hour, n.signedOwnIdentity)
if len(got) != 2 || !got[0].Key.Equal(signerPub) || !got[1].Key.Equal(otherPub) {
t.Fatalf("got %d keys, want the identity's signer and the newest key", len(got))
}
if !n.signedOwnIdentity(signerPub) || n.signedOwnIdentity(otherPub) {
t.Fatal("signedOwnIdentity does not tell the signer from another key")
}
}

// TestHandleAuthHandshake covers the admission office: the relay ACL trusts
// authPeers, so the cached instant must be the token's own expiry, and a token
// that is expired or bound to another peer must leave no entry at all (#296).
Expand Down Expand Up @@ -860,3 +899,83 @@ func TestFindProvidersSkipsRevoked(t *testing.T) {
t.Errorf("revoked peer returned as provider: %v", providers)
}
}

// TestStaleIdentityErrorUnwraps covers what a caller needs from a failed
// start: the refresh status when the control plane answered, and no status
// when it did not.
func TestStaleIdentityErrorUnwraps(t *testing.T) {
refused := &StaleIdentityError{Role: api.RoleNode, RoleErr: errors.New("biscuit: invalid signature"), RefreshErr: &RefreshError{StatusCode: 401, Message: "refused"}, RecoveryErr: errors.New("no refresh token")}
var re *RefreshError
if !errors.As(refused, &re) || re.StatusCode != 401 {
t.Fatalf("errors.As found no refresh status in %v", refused)
}
offline := &StaleIdentityError{Role: api.RoleNode, RoleErr: errors.New("biscuit: invalid signature"), RefreshErr: context.DeadlineExceeded}
if errors.As(offline, &re) || !errors.Is(offline, context.DeadlineExceeded) {
t.Fatalf("offline error unwraps wrong: %v", offline)
}
}

// TestKeyPruningKeepsEveryLatestKey covers a /keys answer that vouched for
// a retiring and a current key at once, followed by an outage longer than
// the grace period: the answer's order says nothing about which key is
// current, so pruning keeps both rather than whichever came first.
func TestKeyPruningKeepsEveryLatestKey(t *testing.T) {
now := time.Now()
synced := now.Add(-48 * time.Hour)
retiring, _ := mustGenerateKey(t)
current, _ := mustGenerateKey(t)
dropped, _ := mustGenerateKey(t)
keys := []TrustedKey{{Key: retiring, ReceivedAt: synced}, {Key: current, ReceivedAt: synced}, {Key: dropped, ReceivedAt: synced.Add(-time.Hour)}}
got := pruneTrustedKeys(keys, now, 24*time.Hour, func(k ed25519.PublicKey) bool { return k.Equal(retiring) })
if len(got) != 2 || !got[0].Key.Equal(retiring) || !got[1].Key.Equal(current) {
t.Fatalf("got %d keys, want the retiring and the current key of the last sync", len(got))
}
}

// TestFallbackKeyStaysOldThroughTheStore covers the configured key re-added
// next to persisted keys at a start: it must stay older than them after the
// set is saved and loaded again, or the next start would take it for the
// newest key and prune the real rotation key.
func TestFallbackKeyStaysOldThroughTheStore(t *testing.T) {
store, err := NewStore(t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := store.Close(); err != nil {
t.Errorf("close store: %v", err)
}
})
rotated, _ := mustGenerateKey(t)
configured, _ := mustGenerateKey(t)
learned := time.Now().Add(-30 * time.Hour).Truncate(time.Second)
if err := store.SaveTrustedKeys([]TrustedKey{{Key: rotated, ReceivedAt: learned}}); err != nil {
t.Fatal(err)
}
priv, _, err := crypto.GenerateEd25519Key(nil)
if err != nil {
t.Fatal(err)
}
n, err := NewAgentMeshNode(Options{PrivKey: priv, Store: store, ControlPlanePubKey: configured})
if err != nil {
t.Fatal(err)
}
// A rotation event persists the set before any sync succeeds.
n.persistTrustedKeys(n.trustedKeys)
loaded, err := store.LoadTrustedKeys()
if err != nil {
t.Fatal(err)
}
for _, tk := range loaded {
switch {
case tk.Key.Equal(configured) && !tk.ReceivedAt.Equal(neverVouched):
t.Fatalf("configured key loaded as vouched for at %s", tk.ReceivedAt)
case tk.Key.Equal(rotated) && !tk.ReceivedAt.Equal(learned):
t.Fatalf("rotation key loaded with %s, want %s", tk.ReceivedAt, learned)
}
}
got := pruneTrustedKeys(loaded, time.Now(), 24*time.Hour, nil)
if len(got) != 1 || !got[0].Key.Equal(rotated) {
t.Fatalf("pruning kept %d keys, want only the rotation key", len(got))
}
}
Loading