diff --git a/Makefile b/Makefile index f7ad015..e2010c8 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,8 @@ all: lint test $(binary) .PHONY: lint lint: $(godeps) golangci-lint run + go vet ./... + govulncheck ./... .PHONY: test test: $(godeps) diff --git a/README.md b/README.md index 751124c..9616725 100644 --- a/README.md +++ b/README.md @@ -55,5 +55,7 @@ system, take a look at our [research paper](https://arxiv.org/abs/2206.04123). * [How to use nitriding](doc/usage.md) * [System architecture](doc/architecture.md) +* [HTTP API](doc/http-api.md) +* [Horizontal scaling](doc/key-synchronization.md) * [Example application](example/) * [Setup enclave EC2 host](doc/setup.md) diff --git a/attestation.go b/attestation.go index 2dbd402..b86fce4 100644 --- a/attestation.go +++ b/attestation.go @@ -7,24 +7,19 @@ import ( "fmt" "github.com/hf/nitrite" - "github.com/hf/nsm" - "github.com/hf/nsm/request" ) const ( - nonceLen = 20 // The size of a nonce in bytes. - nonceNumDigits = nonceLen * 2 // The number of hex digits in a nonce. - maxAttDocLen = 5000 // A (reasonable?) upper limit for attestation doc lengths. - hashPrefix = "sha256:" - hashSeparator = ";" + hashPrefix = "sha256:" + hashSeparator = ";" ) var ( - errBadForm = "failed to parse POST form data" - errNoNonce = "could not find nonce in URL query parameters" - errBadNonceFormat = fmt.Sprintf("unexpected nonce format; must be %d-digit hex string", nonceNumDigits) - errFailedAttestation = "failed to obtain attestation document from hypervisor" - errProfilingSet = "attestation disabled because profiling is enabled" + errBadForm = errors.New("failed to parse POST form data") + errNoNonce = errors.New("could not find nonce in URL query parameters") + errBadNonceFormat = fmt.Errorf("unexpected nonce format; must be %d-digit hex string", nonceLen*2) + errFailedAttestation = errors.New("failed to obtain attestation document from hypervisor") + errProfilingSet = errors.New("attestation disabled because profiling is enabled") // getPCRValues is a variable pointing to a function that returns PCR // values. Using a variable allows us to easily mock the function in our @@ -55,7 +50,7 @@ func (a *AttestationHashes) Serialize() []byte { // _getPCRValues returns the enclave's platform configuration register (PCR) // values. func _getPCRValues() (map[uint][]byte, error) { - rawAttDoc, err := attest(nil, nil, nil) + rawAttDoc, err := newNitroAttester().createAttstn(nil) if err != nil { return nil, err } @@ -76,6 +71,12 @@ func arePCRsIdentical(ourPCRs, theirPCRs map[uint][]byte) bool { } for pcr, ourValue := range ourPCRs { + // PCR4 contains a hash over the parent's instance ID. Our enclaves run + // on different parent instances; PCR4 will therefore always differ: + // https://docs.aws.amazon.com/enclaves/latest/user/set-up-attestation.html + if pcr == 4 { + continue + } theirValue, exists := theirPCRs[pcr] if !exists { return false @@ -86,33 +87,3 @@ func arePCRsIdentical(ourPCRs, theirPCRs map[uint][]byte) bool { } return true } - -// attest takes as input a nonce, user-provided data and a public key, and then -// asks the Nitro hypervisor to return a signed attestation document that -// contains all three values. -func attest(nonce, userData, publicKey []byte) ([]byte, error) { - s, err := nsm.OpenDefaultSession() - if err != nil { - return nil, err - } - defer func() { - if err = s.Close(); err != nil { - elog.Printf("Attestation: Failed to close default NSM session: %s", err) - } - }() - - res, err := s.Send(&request.Attestation{ - Nonce: nonce, - UserData: userData, - PublicKey: publicKey, - }) - if err != nil { - return nil, err - } - - if res.Attestation == nil || res.Attestation.Document == nil { - return nil, errors.New("NSM device did not return an attestation") - } - - return res.Attestation.Document, nil -} diff --git a/attestation_test.go b/attestation_test.go index a79e223..efe7da7 100644 --- a/attestation_test.go +++ b/attestation_test.go @@ -20,6 +20,12 @@ func TestArePCRsIdentical(t *testing.T) { t.Fatal("Failed to recognize identical PCRs as such.") } + // PCR4 should be ignored. + pcr1[4], pcr2[4] = []byte("foo"), []byte("bar") + if !arePCRsIdentical(pcr1, pcr2) { + t.Fatal("Failed to recognize identical PCRs as such.") + } + // Add a new PCR value, so our two maps are no longer identical. pcr1[2] = []byte("barfoo") if arePCRsIdentical(pcr1, pcr2) { @@ -49,7 +55,7 @@ func TestAttestationHashes(t *testing.T) { rec := httptest.NewRecorder() buf := bytes.NewBufferString(base64.StdEncoding.EncodeToString(appKeyHash[:])) req := httptest.NewRequest(http.MethodPost, pathHash, buf) - e.privSrv.Handler.ServeHTTP(rec, req) + e.intSrv.Handler.ServeHTTP(rec, req) s := e.hashes.Serialize() expectedLen := sha256.Size*2 + len(hashPrefix)*2 + len(hashSeparator) diff --git a/attester.go b/attester.go new file mode 100644 index 0000000..2180d15 --- /dev/null +++ b/attester.go @@ -0,0 +1,197 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + + "github.com/hf/nitrite" + "github.com/hf/nsm" + "github.com/hf/nsm/request" +) + +var ( + errPCRMismatch = errors.New("PCR values differ") + errNonceMismatch = errors.New("nonce is unexpected") + errNoAttstnFromNSM = errors.New("NSM device did not return an attestation") + padding = []byte("dummy") +) + +// attester defines functions for the creation and verification of attestation +// documents. Making this an interface helps with testing: It allows us to +// implement a dummy attester that works without the AWS Nitro hypervisor. +type attester interface { + createAttstn(auxInfo) ([]byte, error) + verifyAttstn([]byte, nonce) (auxInfo, error) +} + +type auxInfo interface{} + +// workerAuxInfo holds the auxilitary information of an attestation document +// requested by clients. +type clientAuxInfo struct { + clientNonce nonce + attestationHashes []byte +} + +// workerAuxInfo holds the auxiliary information of the worker's attestation +// document. +type workerAuxInfo struct { + WorkersNonce nonce `json:"workers_nonce"` + LeadersNonce nonce `json:"leaders_nonce"` + PublicKey []byte `json:"public_key"` +} + +// leaderAuxInfo holds the auxiliary information of the leader's attestation +// document. +type leaderAuxInfo struct { + WorkersNonce nonce `json:"workers_nonce"` + HashOfEncrypted []byte `json:"hash_of_encrypted"` +} + +// dummyAttester helps with local testing. The interface simply turns +// auxiliary information into JSON, and does not do any cryptography. +type dummyAttester struct{} + +// newDummyAttester returns a new dummyAttester. +func newDummyAttester() *dummyAttester { + return new(dummyAttester) +} + +func (*dummyAttester) createAttstn(aux auxInfo) ([]byte, error) { + return json.Marshal(aux) +} + +func (*dummyAttester) verifyAttstn(doc []byte, n nonce) (auxInfo, error) { + var ( + w workerAuxInfo + l leaderAuxInfo + ) + + // First, assume we're dealing with a worker's auxiliary information. + if err := json.Unmarshal(doc, &w); err != nil { + return nil, err + } + if w.PublicKey != nil { + if n.b64() != w.LeadersNonce.b64() { + return nil, errNonceMismatch + } + return &w, nil + } + + // Next, let's assume it's a leader. + if err := json.Unmarshal(doc, &l); err != nil { + return nil, err + } + if l.HashOfEncrypted != nil { + if n.b64() != l.WorkersNonce.b64() { + return nil, errNonceMismatch + } + return &l, nil + } + + return nil, errors.New("invalid auxiliary information") +} + +// nitroAttester implements the attester interface by drawing on the AWS Nitro +// Enclave hypervisor. +type nitroAttester struct{} + +// newNitroAttester returns a new nitroAttester. +func newNitroAttester() *nitroAttester { + return new(nitroAttester) +} + +// createAttstn asks the AWS Nitro Enclave hypervisor for an attestation +// document that contains the given auxiliary information. +func (*nitroAttester) createAttstn(aux auxInfo) ([]byte, error) { + var nonce, userData, publicKey []byte + + // Prepare our auxiliary information. If the public key field is unused, we + // pad it with dummy bytes because the nitrite package (which we use to + // verify attestation documents) expects all three fields to be set. + switch v := aux.(type) { + case *workerAuxInfo: + nonce = v.LeadersNonce[:] + userData = v.WorkersNonce[:] + publicKey = v.PublicKey + case *leaderAuxInfo: + nonce = v.WorkersNonce[:] + userData = v.HashOfEncrypted + publicKey = padding + case *clientAuxInfo: + nonce = v.clientNonce[:] + userData = v.attestationHashes + publicKey = padding + } + + s, err := nsm.OpenDefaultSession() + if err != nil { + return nil, err + } + defer s.Close() + + res, err := s.Send(&request.Attestation{ + Nonce: nonce, + UserData: userData, + PublicKey: publicKey, + }) + if err != nil { + return nil, err + } + if res.Attestation == nil || res.Attestation.Document == nil { + return nil, errNoAttstnFromNSM + } + + return res.Attestation.Document, nil +} + +// verifyAttstn verifies the given attestation document and, if successful, +// returns the document's auxiliary information. +func (*nitroAttester) verifyAttstn(doc []byte, ourNonce nonce) (auxInfo, error) { + // First, verify the remote enclave's attestation document. + opts := nitrite.VerifyOptions{CurrentTime: currentTime()} + their, err := nitrite.Verify(doc, opts) + if err != nil { + return nil, err + } + + // Verify that the remote enclave's PCR values (e.g., the image ID) are + // identical to ours. + ourPCRs, err := getPCRValues() + if err != nil { + return nil, err + } + if !arePCRsIdentical(ourPCRs, their.Document.PCRs) { + return nil, errPCRMismatch + } + + // Verify that the remote enclave's attestation document contains the nonce + // that we asked it to embed. + theirNonce, err := sliceToNonce(their.Document.Nonce) + if err != nil { + return nil, err + } + if ourNonce != theirNonce { + return nil, errNonceMismatch + } + + // If the "public key" field contains padding, we know that we're + // dealing with a leader's auxiliary information. + if bytes.Equal(their.Document.PublicKey, padding) { + return &leaderAuxInfo{ + WorkersNonce: theirNonce, + HashOfEncrypted: their.Document.UserData, + }, nil + } + + workersNonce, err := sliceToNonce(their.Document.UserData) + if err != nil { + return nil, err + } + return &workerAuxInfo{ + WorkersNonce: workersNonce, + LeadersNonce: theirNonce, + PublicKey: their.Document.PublicKey, + }, nil +} diff --git a/attester_test.go b/attester_test.go new file mode 100644 index 0000000..68d1ced --- /dev/null +++ b/attester_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "bytes" + "errors" + "testing" + + "github.com/hf/nitrite" +) + +func TestDummyAttestation(t *testing.T) { + var ( + d = newDummyAttester() + workersNonce = nonce{1, 2, 3} + hashOfEncrypted = []byte("this is a hash") + ) + + attstn, err := d.createAttstn(&leaderAuxInfo{ + WorkersNonce: workersNonce, + HashOfEncrypted: hashOfEncrypted, + }) + failOnErr(t, err) + + aux, err := d.verifyAttstn(attstn, workersNonce) + failOnErr(t, err) + + leaderAux := aux.(*leaderAuxInfo) + if leaderAux.WorkersNonce != workersNonce { + t.Fatal("Extracted unexpected workers nonce.") + } + if !bytes.Equal(leaderAux.HashOfEncrypted, hashOfEncrypted) { + t.Fatalf("Extracted unexpected hash over encrypted keys.") + } +} + +func TestVerifyNitroAttstn(t *testing.T) { + var n = newNitroAttester() + _, err := n.verifyAttstn([]byte("foobar"), nonce{}) + assertEqual(t, errors.Is(err, nitrite.ErrBadCOSESign1Structure), true) +} + +func TestCreateNitroAttstn(t *testing.T) { + var n = newNitroAttester() + _, err := n.createAttstn(nil) + assertEqual(t, err != nil, true) +} diff --git a/cache.go b/cache.go index 3e0d43e..5c461bf 100644 --- a/cache.go +++ b/cache.go @@ -5,10 +5,6 @@ import ( "time" ) -const ( - defaultItemExpiry = time.Minute -) - // cache implements a simple cache whose items expire. type cache struct { sync.RWMutex diff --git a/certcache.go b/certcache.go index 3de03db..e233e5a 100644 --- a/certcache.go +++ b/certcache.go @@ -2,15 +2,45 @@ package main import ( "context" + "crypto/tls" + "errors" "sync" "golang.org/x/crypto/acme/autocert" ) +var errUninitializedCert = errors.New("certificate not yet initialized") + +// certRetriever stores an HTTPS certificate and implements the GetCertificate +// function signature, which allows our Web servers to retrieve the +// certificate when clients connect: +// https://pkg.go.dev/crypto/tls#Config +type certRetriever struct { + sync.Mutex // Guards cert. + cert *tls.Certificate +} + +func (c *certRetriever) set(cert *tls.Certificate) { + c.Lock() + defer c.Unlock() + + c.cert = cert +} + +func (c *certRetriever) get(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { + c.Lock() + defer c.Unlock() + + if c.cert == nil { + return nil, errUninitializedCert + } + return c.cert, nil +} + // certCache implements the autocert.Cache interface. type certCache struct { - sync.RWMutex - cache map[string][]byte + sync.RWMutex // Guards cache. + cache map[string][]byte } func newCertCache() *certCache { diff --git a/certcache_test.go b/certcache_test.go index a591bc8..0b386df 100644 --- a/certcache_test.go +++ b/certcache_test.go @@ -3,12 +3,23 @@ package main import ( "bytes" "context" + "crypto/tls" "errors" "testing" "golang.org/x/crypto/acme/autocert" ) +func TestPrematureGet(t *testing.T) { + var ( + err error + r = new(certRetriever) + ) + + _, err = r.get(new(tls.ClientHelloInfo)) + assertEqual(t, err, errUninitializedCert) +} + func TestGet(t *testing.T) { var err error var key = "foo" diff --git a/doc/http-api.md b/doc/http-api.md new file mode 100644 index 0000000..47fe911 --- /dev/null +++ b/doc/http-api.md @@ -0,0 +1,112 @@ +# Nitriding's HTTP API + +## External endpoints, reachable to the Internet + +* `GET /enclave` Returns an index page explaining that this code runs + inside an enclave. + The enclave responds with status code `200 OK`. + +* `GET /enclave/attestation?nonce={nonce}` Returns an attestation document + containing the given nonce. + `nonce` must be a 20-byte nonce encoded in 40 hexadecimal digits. + The attestation document is encoded using Base64. + If all goes well, the enclave responds with status code `200 OK`. + +* `GET /enclave/config` Returns nitriding's configuration. + The enclave responds with status code `200 OK`. + +* `GET /enclave/debug` If enabled, returns profiling information. + If nitriding is invoked with the `-debug` command line flag, + it exposes this endpoint to make profiling information available. + If all goes well, the enclave responds with status code `200 OK`. + +## External endpoints, reachable to other enclaves + +* `GET /enclave/sync?nonce={nonce}` Exposed by workers, the leader talks to this endpoint to initiate key synchronization. + `nonce` must be a 20-byte nonce encoded in 40 hexadecimal digits. + If all goes well, the worker responds with status code `200 OK` and the following JSON-formatted body: + ``` + { + "document": "{Base64-encoded attestation document}", + } + ``` + +* `POST /enclave/sync` Exposed by workers, the leader talks to this endpoint to + complete key synchronization. + + The request must contain the following JSON-formatted body: + ``` + { + "document": "{Base64-encoded attestation document}", + "encrypted_keys": "{Base64-encoded, encrypted enclave keys}", + } + ``` + If all goes well, the worker responds with status code `200 OK`. + +* `POST /enclave/heartbeat` Exposed by the leader, workers periodically send a heartbeat to this endpoint. + The request must contain the following JSON-formatted body: + ``` + { + "hashed_keys": "{hashed_keys}", + "worker_hostname": "{worker_hostname}", + } + ``` + `worker_hostname` contains the worker's EC2-internal hostname, e.g., `ip-12-34-56-78.us-east-2.compute.internal`. + `hashed_keys` contains the Base64-encoded SHA-256 hash over the worker's enclave key material. + If all goes well, the leader responds with status code `200 OK`. + +* `GET /enclave/leader?nonce={nonce}` Exposed by all enclaves, this endpoint + helps enclaves figure out who the leader is. + `nonce` must be a 20-byte nonce encoded in 40 hexadecimal digits. + All enclaves create a random `nonce` and send it to the leader's endpoint. + If the leader notices that it's talking to itself (by comparing the received nonce to its previously-generated nonce), + it designates itself as the leader. + After that, the leader responds with status code `410 Gone`. + Workers know that they are workers when they receive status code `410 Gone`. + Before that, the leader responds with status code `200 OK`. + While workers expose this endpoint too, they should never receive any requests. + +## Internal endpoints, reachable to the application + +* `GET /enclave/ready` Used by the enclave application to signal its readiness. + When nitriding is invoked with the command line argument `-wait-for-app`, + it refrains from starting its external Web servers until the application + signals its readiness by calling this endpoint, after which nitriding starts + the external Web servers. + The first invocation of this endpoint returns status code `200 OK`. + Subsequent invocations return status code `410 Gone`. + +* `GET /enclave/state` Returns the application's state in the response body. + This endpoint allows an application to retrieve state + (e.g., confidential key material) that was previously set by the "leader" application. + If synchronization is not enabled via the `-fqdn-leader` command line + argument, the endpoint responds with status code `403 Forbidden`. + If synchronization is enabled but leader designation is currently in progress, + the endpoint responds with status code `503 Service Unavailable`. + If synchronization is enabled and the enclave is the leader, + the endpoint responds with status code `410 Gone`. + Finally, if synchronization is enabled _and_ the enclave is a worker, + the endpoint returns the application's state in the response body and + responds with status code `200 OK`. + The application's state is returned without encoding, + using the `application/octet-stream` content type. + +* `PUT /enclave/state` Sets the application's state. + This endpoint allows the "leader" application to set state that is + subsequently synchronized with worker enclaves. + If synchronization is not enabled via the `-fqdn-leader` command line + argument, the endpoint responds with status code `403 Forbidden`. + If synchronization is enabled but leader designation is currently in progress, + the endpoint responds with status code `503 Service Unavailable`. + If synchronization is enabled and the enclave is a worker, + the endpoint responds with status code `410 Gone`. + Finally, if synchronization is enabled _and_ the enclave is the leader, + the endpoint saves the state that's set in the request body and + responds with status code `200 OK`. + +* `POST /enclave/hash` Allows the application to set a hash that's included in + attestation documents. + The enclave application can invoke this endpoint to submit a SHA-256 hash that + nitriding is subsequently going to include in attestation documents. + The Base64-encoded SHA-256 hash must be given in the request body. + If all goes well, the endpoint responds with status code `200 OK`. \ No newline at end of file diff --git a/doc/key-synchronization.md b/doc/key-synchronization.md new file mode 100644 index 0000000..2a6f137 --- /dev/null +++ b/doc/key-synchronization.md @@ -0,0 +1,149 @@ +# Enclave key synchronization + +Nitriding supports horizontal scaling, i.e., it allows for the synchronization +of key material among identical enclaves. Key material consists of both +_nitriding_ and _application_ keys: + +1. Nitriding's key material is the self-signed HTTPS certificate (both public + and private key) that provides the confidential channel between clients and + the enclave. +2. The application's key material is application-specific. Nitriding is + agnostic to the structure of this key material and treats it as arbitrary + bytes. + +All of the above must be synced among enclaves. + +For enclave key synchronization to work, there must be a _single leader +enclave_ and _one or more worker enclaves_. The leader's sole job is to +create key material and make itself available for synchronizing this key +material with worker enclaves. Worker enclaves do the actual work, i.e., +process user requests. Before doing any work though, workers must register +themselves with the leader, which triggers key synchronization. + +To set up key synchronization, several steps are necessary: + +* Use the `-fqdn-leader` command line flag on both the leader and the worker. + Note that the leader and worker images _must be identical_. The leader is + only willing to synchronize key material with _identical enclaves_. +* Practically speaking, the leader is meant to run in a separate k8s deployment + from the workers. + +## Protocol + +1. The leader creates a 20-byte nonce $\textrm{nonce}_l$ and sends it to the + worker as part of a `GET` request. +2. Upon receiving $\textrm{nonce}_l$, the worker creates its own 20-byte nonce + $\textrm{nonce}_w$ and an ephemeral asymmetric key pair $K_e = \(sk, pk\)$, + which we generate with Go's `crypto/nacl/box` package. The worker now asks + its hypervisor to create an attestation document $A_w$ containing + $\textrm{nonce}_l$, $\textrm{nonce}_w$, and $pk$. The worker responds to the + leader's `GET` request with $A_w$. +3. Having received $A_w$, the leader now verifies that... + 1. ...the attestation document is signed by the AWS Nitro Enclave hypervisor. + This stops attackers from sending spoofed attestation documents. + 2. ...the attestation document contains $\textrm{nonce}_l$. This stops + attackers from replaying old attestation documents. Note that attestation + documents [always contain the enclave's image ID](https://docs.aws.amazon.com/enclaves/latest/user/set-up-attestation.html). + 3. ...the attestation document's platform configuration registers are + identical to the leader's registers. This stops attackers from using + modified enclaves to extract the sensitive key material. +4. The leader is now convinced that it's dealing with an authentic worker + enclave. In the next and final interaction, the leader encrypts its sensitive + enclave keys $K_s$ using the worker's ephemeral public key $pk$, resulting in + $E = \textrm{Enc}(K_s, pk)$. The leader then asks its hypervisor to create an + attestation document $A_l$ containing $\textrm{nonce}_w$ and SHA-256($E$). + The leader sends $A_l$ and $E$ to the worker in a separate `POST` request. +6. Upon receiving $A_l$, the worker first verifies the attestation document + (same as above), and decrypts $E$ using $sk$, revealing in $K_s$, the + sensitive enclave keys. At this point, key synchronization is complete. +7. After key synchronization, workers send a periodic heartbeat to the leader in + a `POST` request. The request's body contains a Base64-encoded SHA-256 hash + over $K_s$. This allows the leader to verify if the worker's keys are still + up-to-date. If not, the leader initiates key-synchronization using the + protocol as above. + +## Security considerations + +The sensitive key material $K_s$ is protected as follows: + +* Communication between leader and worker enclaves happens over AWS's Virtual + Private Cloud (VPC). We therefore expose the endpoints for key + synchronization over a separate Web server that's not reachable over the + Internet. + +* Leader and worker enclaves use HTTPS as an underlying secure channel. Note + that the authenticity of our HTTPS certificates is rooted in the + hypervisor-signed attestation documents; not in a certificate authority. + +* Worker enclaves create an ephemeral key pair that's used to encrypt key + material using Go's `crypto/nacl/box` API. Even if an attacker can snoop on + the VPC network _and_ compromise the confidentiality of our HTTPS connection, + enclave keys are still protected by this ephemeral key pair. + +The leader only synchronizes with workers that run _identical_ code. The leader +therefore has assurance that workers are not going to game the system. + +```mermaid +sequenceDiagram + box rgba(100, 100, 100, .1) Leader enclave + participant leaderApp as Enclave application + participant leader as Leader enclave + end + box rgba(100, 100, 100, .1) Worker enclave + participant worker as Worker enclave + participant workerApp as Enclave application + end + +leader->>leader: Generate HTTPS certificate +leaderApp->>leaderApp: Generate key material + +Note over leader,worker: Enclaves designate the leader +worker->>+leader: GET /enclave/leader (nonce_w) +leader-->>-worker: OK +worker->>worker: Did not call itself: worker +leader->>leader: GET /enclave/leader (nonce_l) +leader->>leader: Did call itself: leader + +Note over leaderApp,leader: Application sets its key material +leaderApp->>+leader: PUT /enclave/state (key material) +leader->>leader: Save key material +leader-->>-leaderApp: OK + +Note over leader,worker: Worker announces itself to leader +worker->>+leader: POST /enclave/heartbeat +leader->>leader: Register new worker +leader-->>-worker: OK + +Note over leader,worker: Leader initiates key synchronization +leader->>leader: Create nonce +leader->>+worker: GET /enclave/sync (nonce_l) +worker->>worker: Create attestation, nonce, and ephemeral keys +worker-->>-leader: OK (Attestation(nonce_l, nonce_w, pk)) + +leader->>leader: Verify & create attestation +leader->>+worker: POST /enclave/sync (Attestation(nonce_w, SHA-256(E(keys, pk))), E(keys, pk)) +worker->>worker: Verify attestation & install keys +worker-->>-leader: OK + +worker->>worker: Install HTTPS certificate + +Note over worker,workerApp: Application retrieves key material +workerApp->>+worker: GET /enclave/state +worker->>worker: Retrieve key material +worker-->>-workerApp: OK (key material) +workerApp->>workerApp: Install key material + +Note over leader, worker: Worker starts heartbeat loop + +loop Heartbeat + worker->>+leader: POST /enclave/heartbeat (Hash(key material)) + leader-->>-worker: OK +end + +Note over leaderApp: Application updates its key material +leaderApp->>+leader: PUT /enclave/state (key material) +leader->>leader: Save key material +leader-->>-leaderApp: OK + +note over leader,worker: Leader initiates key re-synchronization as above +``` \ No newline at end of file diff --git a/enclave.go b/enclave.go index de8622d..06857ea 100644 --- a/enclave.go +++ b/enclave.go @@ -1,20 +1,16 @@ package main import ( + "bytes" "context" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" "crypto/sha256" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "encoding/hex" "encoding/json" "encoding/pem" "errors" "fmt" - "math/big" "net" "net/http" "net/http/httputil" @@ -42,7 +38,6 @@ const ( parentCID = 3 // The following paths are handled by nitriding. pathRoot = "/enclave" - pathNonce = "/enclave/nonce" pathAttestation = "/enclave/attestation" pathState = "/enclave/state" pathSync = "/enclave/sync" @@ -50,31 +45,40 @@ const ( pathReady = "/enclave/ready" pathProfiling = "/enclave/debug" pathConfig = "/enclave/config" + pathLeader = "/enclave/leader" + pathHeartbeat = "/enclave/heartbeat" // All other paths are handled by the enclave application's Web server if // it exists. pathProxy = "/*" + // The states the enclave can be in relating to key synchronization. + noSync = 0 // The enclave is not configured to synchronize keys. + inProgress = 1 // Leader designation is in progress. + isLeader = 2 // The enclave is the leader. + isWorker = 3 // The enclave is a worker. ) var ( - errNoKeyMaterial = errors.New("no key material registered") errCfgMissingFQDN = errors.New("given config is missing FQDN") errCfgMissingPort = errors.New("given config is missing port") ) // Enclave represents a service running inside an AWS Nitro Enclave. type Enclave struct { - sync.RWMutex - cfg *Config - pubSrv *http.Server - privSrv *http.Server - promSrv *http.Server - revProxy *httputil.ReverseProxy - hashes *AttestationHashes - promRegistry *prometheus.Registry - metrics *metrics - nonceCache *cache - keyMaterial any - ready, stop chan bool + attester + sync.Mutex // Guard syncState. + cfg *Config + syncState int + extPubSrv, extPrivSrv *http.Server + intSrv *http.Server + promSrv *http.Server + revProxy *httputil.ReverseProxy + hashes *AttestationHashes + promRegistry *prometheus.Registry + metrics *metrics + workers *workerManager + keys *enclaveKeys + httpsCert *certRetriever + ready, stop chan struct{} } // Config represents the configuration of our enclave service. @@ -84,26 +88,37 @@ type Config struct { // is required. FQDN string - // ExtPort contains the TCP port that the Web server should + // FQDNLeader contains the fully qualified domain name of the leader + // enclave, which coordinates enclave synchronization. Only set this field + // if horizontal scaling is required. + FQDNLeader string + + // ExtPubPort contains the TCP port that the public Web server should // listen on, e.g. 443. This port is not *directly* reachable by the // Internet but the EC2 host's proxy *does* forward Internet traffic to // this port. This field is required. - ExtPort uint16 + ExtPubPort uint16 + + // ExtPrivPort contains the TCP port that the non-public Web server should + // listen on. The Web server behind this port exposes confidential + // endpoints and is therefore only meant to be reachable by the enclave + // administrator but *not* the public Internet. + ExtPrivPort uint16 + + // IntPort contains the enclave-internal TCP port of the Web server that + // provides an HTTP API to the enclave application. This field is + // required. + IntPort uint16 // UseVsockForExtPort must be set to true if direct communication // between the host and Web server via VSOCK is desired. The daemon will listen - // on the enclave's VSOCK address and the port defined in ExtPort. + // on the enclave's VSOCK address and the port defined in ExtPubPort. UseVsockForExtPort bool // DisableKeepAlives must be set to true if keep-alive connections // should be disabled for the HTTPS service. DisableKeepAlives bool - // IntPort contains the enclave-internal TCP port of the Web server that - // provides an HTTP API to the enclave application. This field is - // required. - IntPort uint16 - // HostProxyPort indicates the TCP port of the proxy application running on // the EC2 host. Note that VSOCK ports are 32 bits large. This field is // required. @@ -174,7 +189,7 @@ type Config struct { // Validate returns an error if required fields in the config are not set. func (c *Config) Validate() error { - if c.ExtPort == 0 || c.IntPort == 0 || c.HostProxyPort == 0 { + if c.ExtPubPort == 0 || c.IntPort == 0 || c.HostProxyPort == 0 { return errCfgMissingPort } if c.FQDN == "" { @@ -183,6 +198,12 @@ func (c *Config) Validate() error { return nil } +// isScalingEnabled returns true if horizontal enclave scaling is enabled in our +// enclave configuration. +func (c *Config) isScalingEnabled() bool { + return c.FQDNLeader != "" +} + // String returns a string representation of the enclave's configuration. func (c *Config) String() string { s, err := json.MarshalIndent(c, "", " ") @@ -200,11 +221,16 @@ func NewEnclave(cfg *Config) (*Enclave, error) { reg := prometheus.NewRegistry() e := &Enclave{ - cfg: cfg, - pubSrv: &http.Server{ + attester: &nitroAttester{}, + cfg: cfg, + extPubSrv: &http.Server{ + Handler: chi.NewRouter(), + }, + extPrivSrv: &http.Server{ + Addr: fmt.Sprintf(":%d", cfg.ExtPrivPort), Handler: chi.NewRouter(), }, - privSrv: &http.Server{ + intSrv: &http.Server{ Addr: fmt.Sprintf("127.0.0.1:%d", cfg.IntPort), Handler: chi.NewRouter(), }, @@ -212,12 +238,14 @@ func NewEnclave(cfg *Config) (*Enclave, error) { Addr: fmt.Sprintf(":%d", cfg.PrometheusPort), Handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg}), }, + httpsCert: &certRetriever{}, + keys: &enclaveKeys{}, promRegistry: reg, metrics: newMetrics(reg, cfg.PrometheusNamespace), - nonceCache: newCache(defaultItemExpiry), hashes: new(AttestationHashes), - stop: make(chan bool), - ready: make(chan bool), + workers: newWorkerManager(time.Minute), + stop: make(chan struct{}), + ready: make(chan struct{}), } // Increase the maximum number of idle connections per host. This is @@ -228,34 +256,43 @@ func NewEnclave(cfg *Config) (*Enclave, error) { http.DefaultTransport.(*http.Transport).MaxIdleConns = 500 if cfg.Debug { - e.pubSrv.Handler.(*chi.Mux).Use(middleware.Logger) - e.privSrv.Handler.(*chi.Mux).Use(middleware.Logger) + e.attester = &dummyAttester{} + e.extPubSrv.Handler.(*chi.Mux).Use(middleware.Logger) + e.extPrivSrv.Handler.(*chi.Mux).Use(middleware.Logger) + e.intSrv.Handler.(*chi.Mux).Use(middleware.Logger) } if cfg.PrometheusPort > 0 { - e.pubSrv.Handler.(*chi.Mux).Use(e.metrics.middleware) - e.privSrv.Handler.(*chi.Mux).Use(e.metrics.middleware) + e.extPubSrv.Handler.(*chi.Mux).Use(e.metrics.middleware) + e.extPrivSrv.Handler.(*chi.Mux).Use(e.metrics.middleware) + e.intSrv.Handler.(*chi.Mux).Use(e.metrics.middleware) } if cfg.UseProfiling { - e.pubSrv.Handler.(*chi.Mux).Mount(pathProfiling, middleware.Profiler()) + e.extPubSrv.Handler.(*chi.Mux).Mount(pathProfiling, middleware.Profiler()) } if cfg.DisableKeepAlives { - e.pubSrv.SetKeepAlivesEnabled(false) + e.extPubSrv.SetKeepAlivesEnabled(false) + } + if cfg.isScalingEnabled() { + e.setSyncState(inProgress) } - // Register public HTTP API. - m := e.pubSrv.Handler.(*chi.Mux) - m.Get(pathAttestation, attestationHandler(e.cfg.UseProfiling, e.hashes)) - m.Get(pathNonce, nonceHandler(e)) + // Register external public HTTP API. + m := e.extPubSrv.Handler.(*chi.Mux) + m.Get(pathAttestation, attestationHandler(e.cfg.UseProfiling, e.hashes, e.attester)) m.Get(pathRoot, rootHandler(e.cfg)) - m.Post(pathSync, respSyncHandler(e)) m.Get(pathConfig, configHandler(e.cfg)) + // Register external but private HTTP API. + m = e.extPrivSrv.Handler.(*chi.Mux) + m.Handle(pathSync, asWorker(e.setupWorkerPostSync, e.attester)) + // Register enclave-internal HTTP API. - m = e.privSrv.Handler.(*chi.Mux) - m.Get(pathSync, reqSyncHandler(e)) - m.Get(pathReady, readyHandler(e)) - m.Get(pathState, getStateHandler(e)) - m.Put(pathState, putStateHandler(e)) + m = e.intSrv.Handler.(*chi.Mux) + if cfg.WaitForApp { + m.Get(pathReady, readyHandler(e.ready)) + } + m.Get(pathState, getStateHandler(e.getSyncState, e.keys)) + m.Put(pathState, putStateHandler(e.attester, e.getSyncState, e.keys, e.workers)) m.Post(pathHash, hashHandler(e)) // Configure our reverse proxy if the enclave application exposes an HTTP @@ -263,7 +300,7 @@ func NewEnclave(cfg *Config) (*Enclave, error) { if cfg.AppWebSrv != nil { e.revProxy = httputil.NewSingleHostReverseProxy(cfg.AppWebSrv) e.revProxy.BufferPool = newBufPool() - e.pubSrv.Handler.(*chi.Mux).Handle(pathProxy, e.revProxy) + e.extPubSrv.Handler.(*chi.Mux).Handle(pathProxy, e.revProxy) // If we expose Prometheus metrics, we keep track of the HTTP backend's // responses. if cfg.PrometheusPort > 0 { @@ -278,7 +315,10 @@ func NewEnclave(cfg *Config) (*Enclave, error) { // Start starts the Nitro Enclave. If something goes wrong, the function // returns an error. func (e *Enclave) Start() error { - var err error + var ( + err error + leader = e.getLeader(pathHeartbeat) + ) errPrefix := "failed to start Nitro Enclave" if inEnclave { @@ -309,16 +349,177 @@ func (e *Enclave) Start() error { return fmt.Errorf("%s: %w", errPrefix, err) } + if !e.cfg.isScalingEnabled() { + return nil + } + + // Check if we are the leader. + if !e.weAreLeader() { + elog.Println("Obtaining worker's hostname.") + worker := getSyncURL(getHostnameOrDie(), e.cfg.ExtPrivPort) + err = asWorker(e.setupWorkerPostSync, e.attester).registerWith(leader, worker) + if err != nil { + elog.Fatalf("Error syncing with leader: %v", err) + } + } + + return nil +} + +// getSyncState returns the enclave's key synchronization state. +func (e *Enclave) getSyncState() int { + e.Lock() + defer e.Unlock() + return e.syncState +} + +// setSyncState sets the enclave's key synchronization state. +func (e *Enclave) setSyncState(state int) { + e.Lock() + defer e.Unlock() + e.syncState = state +} + +// weAreLeader figures out if the enclave is the leader or worker. +func (e *Enclave) weAreLeader() (result bool) { + var ( + err error + ourNonce nonce + weAreLeader = make(chan struct{}, 1) + areWeLeader = make(chan bool) + errChan = make(chan error) + leader = e.getLeader(pathLeader) + ) + defer func() { + elog.Printf("We are leader: %v", result) + if result { + e.setSyncState(isLeader) + } else { + e.setSyncState(isWorker) + } + }() + + ourNonce, err = newNonce() + if err != nil { + elog.Fatalf("Error creating new nonce: %v", err) + } + + m := e.extPrivSrv.Handler.(*chi.Mux) + m.Get(pathLeader, getLeaderHandler(ourNonce, weAreLeader)) + // Reset the handler as we no longer have a need for it. + defer m.Get(pathLeader, + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusGone) + }, + ) + + timeout := time.NewTicker(10 * time.Second) + for { + go makeLeaderRequest(leader, ourNonce, areWeLeader, errChan) + select { + case <-e.stop: + return + case <-errChan: + elog.Println("Not yet able to talk to leader designation endpoint.") + time.Sleep(time.Second) + continue + case result = <-areWeLeader: + return + case <-weAreLeader: + e.setupLeader() + result = true + return + case <-timeout.C: + elog.Fatal("Timed out talking to leader designation endpoint.") + } + } +} + +// setupWorkerPostSync performs necessary post-key synchronization tasks like +// installing the given enclave keys and starting the heartbeat loop. +func (e *Enclave) setupWorkerPostSync(keys *enclaveKeys) error { + e.keys.set(keys) + cert, err := tls.X509KeyPair(keys.NitridingCert, keys.NitridingKey) + if err != nil { + return err + } + e.httpsCert.set(&cert) + + // Start our heartbeat. + worker := getSyncURL(getHostnameOrDie(), e.cfg.ExtPrivPort) + go e.workerHeartbeat(worker) + return nil } +// setupLeader performs necessary setup tasks like starting the worker event +// loop and installing leader-specific HTTP handlers. +func (e *Enclave) setupLeader() { + go e.workers.start(e.stop) + // Make leader-specific endpoint available. + e.extPrivSrv.Handler.(*chi.Mux).Post(pathHeartbeat, heartbeatHandler(e)) + elog.Println("Set up leader endpoint and started worker event loop.") +} + +// workerHeartbeat periodically talks to the leader enclave to 1) let the leader +// know that we're still alive, and 2) to compare key material. If it turns out +// that the leader has different key material than the worker, the worker +// re-registers itself, which triggers key re-synchronization. +func (e *Enclave) workerHeartbeat(worker *url.URL) { + elog.Println("Starting worker's heartbeat loop.") + defer elog.Println("Exiting worker's heartbeat loop.") + var ( + leader = e.getLeader(pathHeartbeat) + timer = time.NewTicker(time.Minute) + hbBody = heartbeatRequest{ + WorkerHostname: worker.Host, + } + ) + + for { + select { + case <-e.stop: + return + case <-timer.C: + hbBody.HashedKeys = e.keys.hashAndB64() + body, err := json.Marshal(hbBody) + if err != nil { + elog.Printf("Error marshalling heartbeat request: %v", err) + e.metrics.heartbeats.With(badHb(err)).Inc() + continue + } + + resp, err := newUnauthenticatedHTTPClient().Post( + leader.String(), + "text/plain", + bytes.NewReader(body), + ) + if err != nil { + elog.Printf("Error posting heartbeat to leader: %v", err) + e.metrics.heartbeats.With(badHb(err)).Inc() + continue + } + if resp.StatusCode != http.StatusOK { + e.metrics.heartbeats.With(badHb(fmt.Errorf("got status code %d", resp.StatusCode))).Inc() + elog.Printf("Leader responded to heartbeat with status code %d.", resp.StatusCode) + continue + } + elog.Println("Successfully sent heartbeat to leader.") + e.metrics.heartbeats.With(goodHb).Inc() + } + } +} + // Stop stops the enclave. func (e *Enclave) Stop() error { close(e.stop) - if err := e.privSrv.Shutdown(context.Background()); err != nil { + if err := e.intSrv.Shutdown(context.Background()); err != nil { + return err + } + if err := e.extPubSrv.Shutdown(context.Background()); err != nil { return err } - if err := e.pubSrv.Shutdown(context.Background()); err != nil { + if err := e.extPrivSrv.Shutdown(context.Background()); err != nil { return err } if err := e.promSrv.Shutdown(context.Background()); err != nil { @@ -331,9 +532,9 @@ func (e *Enclave) Stop() error { // via AF_INET or AF_VSOCK. func (e *Enclave) getExtListener() (net.Listener, error) { if e.cfg.UseVsockForExtPort { - return vsock.Listen(uint32(e.cfg.ExtPort), nil) + return vsock.Listen(uint32(e.cfg.ExtPubPort), nil) } else { - return net.Listen("tcp", fmt.Sprintf(":%d", e.cfg.ExtPort)) + return net.Listen("tcp", fmt.Sprintf(":%d", e.cfg.ExtPubPort)) } } @@ -350,13 +551,20 @@ func (e *Enclave) startWebServers() error { }() } - elog.Printf("Starting public (%s) and private (%s) Web servers.", e.pubSrv.Addr, e.privSrv.Addr) go func() { - err := e.privSrv.ListenAndServe() + elog.Printf("Starting internal Web server at %s.", e.intSrv.Addr) + err := e.intSrv.ListenAndServe() if err != nil && !errors.Is(err, http.ErrServerClosed) { elog.Fatalf("Private Web server error: %v", err) } }() + go func() { + elog.Printf("Starting external private Web server at %s.", e.extPrivSrv.Addr) + err := e.extPrivSrv.ListenAndServeTLS("", "") + if err != nil && !errors.Is(err, http.ErrServerClosed) { + elog.Fatalf("External private Web server error: %v", err) + } + }() go func() { // If desired, don't launch our Internet-facing Web server until the // application signalled that it's ready. @@ -370,78 +578,38 @@ func (e *Enclave) startWebServers() error { elog.Fatalf("Failed to listen on external port: %v", err) } - err = e.pubSrv.ServeTLS(listener, "", "") + elog.Printf("Starting external public Web server at :%d.", e.cfg.ExtPubPort) + err = e.extPubSrv.ServeTLS(listener, "", "") if err != nil && !errors.Is(err, http.ErrServerClosed) { - elog.Fatalf("Public Web server error: %v", err) + elog.Fatalf("External public Web server error: %v", err) } }() return nil } -// genSelfSignedCert creates and returns a self-signed TLS certificate based on -// the given FQDN. Some of the code below was taken from: -// https://eli.thegreenplace.net/2021/go-https-servers-with-tls/ +// genSelfSignedCert creates and installs a self-signed certificate. func (e *Enclave) genSelfSignedCert() error { - privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - return err - } - elog.Println("Generated private key for self-signed certificate.") - - serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) - serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) - if err != nil { - return err - } - elog.Println("Generated serial number for self-signed certificate.") - - template := x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{ - Organization: []string{certificateOrg}, - }, - DNSNames: []string{e.cfg.FQDN}, - NotBefore: time.Now(), - NotAfter: time.Now().Add(certificateValidity), - KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - } - - derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + cert, key, err := createCertificate(e.cfg.FQDN) if err != nil { return err } - elog.Println("Created certificate from template.") - pemCert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) - if pemCert == nil { - return errors.New("failed to encode certificate to PEM") - } - // Determine and set the certificate's fingerprint because we need to add - // the fingerprint to our Nitro attestation document. - if err := e.setCertFingerprint(pemCert); err != nil { + if err := e.setCertFingerprint(cert); err != nil { return err } + e.keys.setNitridingKeys(key, cert) - privBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) - if err != nil { - elog.Fatalf("Unable to marshal private key: %v", err) - } - pemKey := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}) - if pemKey == nil { - elog.Fatal("Failed to encode key to PEM.") - } - - cert, err := tls.X509KeyPair(pemCert, pemKey) + tlsCert, err := tls.X509KeyPair(cert, key) if err != nil { return err } - - e.pubSrv.TLSConfig = &tls.Config{ - Certificates: []tls.Certificate{cert}, + e.httpsCert.set(&tlsCert) + e.extPubSrv.TLSConfig = &tls.Config{ + GetCertificate: e.httpsCert.get, } + // Both servers share a TLS config. + e.extPrivSrv.TLSConfig = e.extPubSrv.TLSConfig.Clone() return nil } @@ -470,7 +638,7 @@ func (e *Enclave) setupAcme() error { Prompt: autocert.AcceptTOS, HostPolicy: autocert.HostWhitelist([]string{e.cfg.FQDN}...), } - e.pubSrv.TLSConfig = certManager.TLSConfig() + e.extPubSrv.TLSConfig = certManager.TLSConfig() go func() { var rawData []byte @@ -501,7 +669,7 @@ func (e *Enclave) setCertFingerprint(rawData []byte) error { if e.cfg.MockCertFp != "" { hash, err := hex.DecodeString(e.cfg.MockCertFp) if err != nil { - return errors.New("Failed to decode mock certificate fingerprint hex") + return errors.New("failed to decode mock certificate fingerprint hex") } copy(e.hashes.tlsKeyHash[:], hash) return nil @@ -529,27 +697,25 @@ func (e *Enclave) setCertFingerprint(rawData []byte) error { return nil } -// SetKeyMaterial registers the enclave's key material (e.g., secret encryption -// keys) as being ready to be synchronized to other, identical enclaves. Note -// that the key material's underlying data structure must be marshallable to -// JSON. -// -// This is only necessary if you intend to scale enclaves horizontally. If you -// will only ever run a single enclave, ignore this function. -func (e *Enclave) SetKeyMaterial(keyMaterial any) { - e.Lock() - defer e.Unlock() - - e.keyMaterial = keyMaterial +// getLeader returns the leader enclave's URL. +func (e *Enclave) getLeader(path string) *url.URL { + return &url.URL{ + Scheme: "https", + Host: fmt.Sprintf("%s:%d", e.cfg.FQDNLeader, e.cfg.ExtPrivPort), + Path: path, + } } -// KeyMaterial returns the key material or, if none was registered, an error. -func (e *Enclave) KeyMaterial() (any, error) { - e.RLock() - defer e.RUnlock() - - if e.keyMaterial == nil { - return nil, errNoKeyMaterial +// getWorker takes as input the worker's heartbeat request payload and returns +// the worker's URL. +func (e *Enclave) getWorker(hb *heartbeatRequest) (*url.URL, error) { + var ( + host string + err error + ) + host, _, err = net.SplitHostPort(hb.WorkerHostname) + if err != nil { + return nil, err } - return e.keyMaterial, nil + return getSyncURL(host, e.cfg.ExtPrivPort), nil } diff --git a/enclave_keys.go b/enclave_keys.go new file mode 100644 index 0000000..24560c3 --- /dev/null +++ b/enclave_keys.go @@ -0,0 +1,82 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "sync" +) + +// enclaveKeys holds key material for nitriding itself (the HTTPS certificate) +// and for the enclave application (whatever the application wants to "store" +// in nitriding). These keys are meant to be managed by a leader enclave and -- +// if horizontal scaling is required -- synced to worker enclaves. The struct +// implements getters and setters that allow for thread-safe setting and getting +// of members. +type enclaveKeys struct { + sync.Mutex + NitridingKey []byte `json:"nitriding_key"` + NitridingCert []byte `json:"nitriding_cert"` + AppKeys []byte `json:"app_keys"` +} + +func (e1 *enclaveKeys) equal(e2 *enclaveKeys) bool { + e1.Lock() + e2.Lock() + defer e1.Unlock() + defer e2.Unlock() + + return bytes.Equal(e1.NitridingCert, e2.NitridingCert) && + bytes.Equal(e1.NitridingKey, e2.NitridingKey) && + bytes.Equal(e1.AppKeys, e2.AppKeys) +} + +func (e *enclaveKeys) setAppKeys(appKeys []byte) { + e.Lock() + defer e.Unlock() + + e.AppKeys = appKeys +} + +func (e *enclaveKeys) setNitridingKeys(key, cert []byte) { + e.Lock() + defer e.Unlock() + + e.NitridingKey = key + e.NitridingCert = cert +} + +func (e *enclaveKeys) set(newKeys *enclaveKeys) { + e.setAppKeys(newKeys.AppKeys) + e.setNitridingKeys(newKeys.NitridingKey, newKeys.NitridingCert) +} + +func (e *enclaveKeys) copy() *enclaveKeys { + e.Lock() + defer e.Unlock() + + return &enclaveKeys{ + NitridingKey: e.NitridingKey, + NitridingCert: e.NitridingCert, + AppKeys: e.AppKeys, + } +} + +func (e *enclaveKeys) getAppKeys() []byte { + e.Lock() + defer e.Unlock() + + return e.AppKeys +} + +// hashAndB64 returns the Base64-encoded hash over our key material. The +// resulting string is not confidential as it's impractical to reverse the key +// material. +func (e *enclaveKeys) hashAndB64() string { + e.Lock() + defer e.Unlock() + + keys := append(append(e.NitridingCert, e.NitridingKey...), e.AppKeys...) + hash := sha256.Sum256(keys) + return base64.StdEncoding.EncodeToString(hash[:]) +} diff --git a/enclave_keys_test.go b/enclave_keys_test.go new file mode 100644 index 0000000..28952e7 --- /dev/null +++ b/enclave_keys_test.go @@ -0,0 +1,84 @@ +package main + +import ( + "bytes" + "testing" +) + +// newTestKeys returns arbitrary keys that we use for testing. +func newTestKeys(t *testing.T) *enclaveKeys { + t.Helper() + var testKeys = &enclaveKeys{ + AppKeys: []byte("AppTestKeys"), + } + cert, key, err := createCertificate("example.com") + if err != nil { + t.Fatal(err) + } + testKeys.setNitridingKeys(key, cert) + return testKeys +} + +func TestSetKeys(t *testing.T) { + var ( + keys enclaveKeys + appKeys = []byte("AppKeys") + testKeys = newTestKeys(t) + ) + + // Ensure that the application keys are set correctly. + keys = enclaveKeys{} + keys.setAppKeys(appKeys) + if !bytes.Equal(keys.AppKeys, appKeys) { + t.Fatal("Application keys not set correctly.") + } + + // Ensure that the nitriding keys are set correctly. + keys = enclaveKeys{} + keys.setNitridingKeys(testKeys.NitridingKey, testKeys.NitridingCert) + if !bytes.Equal(keys.NitridingKey, testKeys.NitridingKey) { + t.Fatal("Nitriding key not set correctly.") + } + if !bytes.Equal(keys.NitridingCert, testKeys.NitridingCert) { + t.Fatal("Nitriding cert not set correctly.") + } + + // Ensure that a new set of keys is set correctly. + keys = enclaveKeys{} + keys.set(testKeys) + if !keys.equal(testKeys) { + t.Fatal("Enclave keys not set correctly.") + } +} + +func TestGetKeys(t *testing.T) { + var ( + testKeys = newTestKeys(t) + appKeys = testKeys.getAppKeys() + keys = testKeys.copy() + ) + + // Ensure that the application key is retrieved correctly. + if !bytes.Equal(appKeys, testKeys.AppKeys) { + t.Fatal("Application keys not retrieved correctly.") + } + + // Ensure that a new set of keys is retrieved correctly. + if !keys.equal(testKeys) { + t.Fatal("Enclave keys not retrieved correctly.") + } +} + +func TestModifyCloneObject(t *testing.T) { + var ( + keys = newTestKeys(t) + clonedKeys = keys.copy() + ) + + // Make sure that setting the clone's application keys does not affect the + // original object. + keys.setAppKeys([]byte("foobar")) + if bytes.Equal(keys.getAppKeys(), clonedKeys.getAppKeys()) { + t.Fatal("Cloned object must not affect original object.") + } +} diff --git a/enclave_test.go b/enclave_test.go index c33a096..624411e 100644 --- a/enclave_test.go +++ b/enclave_test.go @@ -6,11 +6,12 @@ import ( var defaultCfg = Config{ FQDN: "example.com", - ExtPort: 50000, - IntPort: 50001, + ExtPubPort: 50000, + ExtPrivPort: 50001, + IntPort: 50002, HostProxyPort: 1024, UseACME: false, - Debug: false, + Debug: true, FdCur: 1024, FdMax: 4096, WaitForApp: true, @@ -46,7 +47,8 @@ func TestValidateConfig(t *testing.T) { } // Set the remaining required fields. - c.ExtPort = 1 + c.ExtPubPort = 1 + c.ExtPrivPort = 1 c.IntPort = 1 c.HostProxyPort = 1 if err = c.Validate(); err != nil { @@ -60,21 +62,3 @@ func TestGenSelfSignedCert(t *testing.T) { t.Fatalf("Failed to create self-signed certificate: %s", err) } } - -func TestKeyMaterial(t *testing.T) { - e := createEnclave(&defaultCfg) - k := struct{ Foo string }{"foobar"} - - if _, err := e.KeyMaterial(); err != errNoKeyMaterial { - t.Fatal("Expected error because we're trying to retrieve non-existing key material.") - } - - e.SetKeyMaterial(k) - r, err := e.KeyMaterial() - if err != nil { - t.Fatalf("Failed to retrieve key material: %s", err) - } - if r != k { - t.Fatal("Retrieved key material is unexpected.") - } -} diff --git a/go.mod b/go.mod index 1be032d..b820fbe 100644 --- a/go.mod +++ b/go.mod @@ -4,53 +4,37 @@ go 1.20 require ( github.com/containers/gvisor-tap-vsock v0.5.0 + github.com/go-chi/chi v1.5.4 github.com/go-chi/chi/v5 v5.0.8 github.com/hf/nitrite v0.0.0-20211104000856-f9e0dcc73703 github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9 github.com/mdlayher/vsock v1.2.1 github.com/milosgajdos/tenus v0.0.3 github.com/prometheus/client_golang v1.15.1 - github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091 github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 github.com/vishvananda/netlink v1.2.1-beta.2 golang.org/x/crypto v0.6.0 golang.org/x/sys v0.7.0 - gvisor.dev/gvisor v0.0.0-20230218065217-1b534ef82cec ) require ( - github.com/bazelbuild/rules_go v0.38.1 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cilium/ebpf v0.9.3 // indirect - github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/libcontainer v2.2.1+incompatible // indirect github.com/fxamacker/cbor/v2 v2.4.0 // indirect - github.com/go-chi/chi v1.5.4 // indirect - github.com/godbus/dbus/v5 v5.0.4 // indirect - github.com/gofrs/flock v0.8.0 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/btree v1.1.2 // indirect - github.com/google/subcommands v1.0.2-0.20190508160503-636abe8753b8 // indirect - github.com/kr/pty v1.1.4-0.20190131011033-7dc38fb350b1 // indirect github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mdlayher/socket v0.4.1 // indirect - github.com/mohae/deepcopy v0.0.0-20170308212314-bb9b5e7adda9 // indirect - github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/vishvananda/netns v0.0.4 // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/mod v0.8.0 // indirect golang.org/x/net v0.9.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/text v0.9.0 // indirect - golang.org/x/time v0.3.0 // indirect google.golang.org/protobuf v1.30.0 // indirect ) diff --git a/go.sum b/go.sum index c91772c..abf29cb 100644 --- a/go.sum +++ b/go.sum @@ -1,913 +1,99 @@ -bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM= -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/bazelbuild/rules_go v0.27.0/go.mod h1:MC23Dc/wkXEyk3Wpq6lCqz0ZAYOZDw2DR5y3N1q2i7M= -github.com/bazelbuild/rules_go v0.30.0 h1:kX4jVcstqrsRqKPJSn2mq2o+TI21edRzEJSrEOMQtr0= -github.com/bazelbuild/rules_go v0.30.0/go.mod h1:MC23Dc/wkXEyk3Wpq6lCqz0ZAYOZDw2DR5y3N1q2i7M= -github.com/bazelbuild/rules_go v0.38.1 h1:YGNsLhWe18Ielebav7cClP3GMwBxBE+xEArLHtmXDx8= -github.com/bazelbuild/rules_go v0.38.1/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cenkalti/backoff v1.1.1-0.20190506075156-2146c9339422 h1:8eZxmY1yvxGHzdzTEhI09npjMVGzNAdrqzruTX6jcK4= -github.com/cenkalti/backoff v1.1.1-0.20190506075156-2146c9339422/go.mod h1:b6Nc7NRH5C4aCISLry0tLnTjcuTEvoiqcWDdsU0sOGM= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= -github.com/cilium/ebpf v0.4.0 h1:QlHdikaxALkqWasW8hAC1mfR0jdmvbfaBdBPFmRSglA= -github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.9.3 h1:5KtxXZU+scyERvkJMEm16TbScVvuuMrlhPly78ZMbSc= -github.com/cilium/ebpf v0.9.3/go.mod h1:w27N4UjpaQ9X/DGrSugxUG+H+NhgntDuPb5lCzxCn8A= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= -github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.2.1/go.mod h1:wCYX+dRqZdImhGucXOqTQn05AhX6EUDaGEMUzTFFpLg= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= -github.com/containers/gvisor-tap-vsock v0.4.0 h1:sPu8OWiboCBJE/VJyxKOikslCI3sAApNfl3jPXdzBRw= -github.com/containers/gvisor-tap-vsock v0.4.0/go.mod h1:2daFkw9Qp3NTz7+SLEIeSEZsQbxvJVoPVF5WtRy7h/4= github.com/containers/gvisor-tap-vsock v0.5.0 h1:hoCkrfQ96tjek2BtiW1BHy50zAQCzkqeiAQY96y6NLk= github.com/containers/gvisor-tap-vsock v0.5.0/go.mod h1:jrnI5plQtmys5LEKpXcCCrLqZlrHsozQg0V2Jw1UG74= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/stream-metadata-go v0.3.0/go.mod h1:RTjQyHgO/G37oJ3qnqYK6Z4TPZ5EsaabOtfMjVXmgko= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libcontainer v2.2.1+incompatible h1:++SbbkCw+X8vAd4j2gOCzZ2Nn7s2xFALTf7LZKmM1/0= github.com/docker/libcontainer v2.2.1+incompatible/go.mod h1:osvj61pYsqhNCMLGX31xr7klUBhHb/ZBuXS0o1Fvwbw= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fanliao/go-promise v0.0.0-20141029170127-1890db352a72/go.mod h1:PjfxuH4FZdUyfMdtBio2lsRr1AKEaVPwelzuHuh8Lqc= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fxamacker/cbor/v2 v2.2.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg= -github.com/go-chi/chi/v5 v5.0.7 h1:rDTPXLDHGATaeHvVlLcR4Qe0zftYethFucbjVQ1PxU8= -github.com/go-chi/chi/v5 v5.0.7/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0= github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY= -github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= -github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/subcommands v1.0.2-0.20190508160503-636abe8753b8 h1:8nlgEAjIalk6uj/CGKCdOO8CQqTeysvcW4RFZ6HbkGM= -github.com/google/subcommands v1.0.2-0.20190508160503-636abe8753b8/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= -github.com/google/tcpproxy v0.0.0-20200125044825-b6bb9b5b8252/go.mod h1:DavVbd41y+b7ukKDmlnPR4nGYmkWXR6vHUkjQNiHPBs= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.4.0/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= -github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hf/nitrite v0.0.0-20211104000856-f9e0dcc73703 h1:oTi0zYvHo1sfk5sevGc4LrfgpLYB6cIhP/HllCUGcZ8= github.com/hf/nitrite v0.0.0-20211104000856-f9e0dcc73703/go.mod h1:ycRhVmo6wegyEl6WN+zXOHUTJvB0J2tiuH88q/McTK8= github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9 h1:pU32bJGmZwF4WXb9Yaz0T8vHDtIPVxqDOdmYdwTQPqw= github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9/go.mod h1:MJsac5D0fKcNWfriUERtln6segcGfD6Nu0V5uGBbPf8= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hugelgupf/socketpair v0.0.0-20190730060125-05d35a94e714/go.mod h1:2Goc3h8EklBH5mspfHFxBnEoURQCGzQQH1ga9Myjvis= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/insomniacslk/dhcp v0.0.0-20210812084645-decc701b3665/go.mod h1:h+MxyHxRg9NH3terB1nfRIUaQEcI0XOVkdR9LNBlp8E= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= -github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= -github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok= -github.com/jsimonetti/rtnetlink v0.0.0-20201110080708-d2c240429e6c/go.mod h1:huN4d1phzjhlOsNIjFsw2SVRbwIHj3fJDMEU2SDPTmg= -github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.4-0.20190131011033-7dc38fb350b1 h1:zc0R6cOw98cMengLA0fvU55mqbnN7sd/tBMLzSejp+M= -github.com/kr/pty v1.1.4-0.20190131011033-7dc38fb350b1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2 h1:DZMFueDbfz6PNc1GwDRA8+6lBx1TB9UnxDQliCqR73Y= github.com/linuxkit/virtsock v0.0.0-20220523201153-1a23e78aa7a2/go.mod h1:SWzULI85WerrFt3u+nIm5F9l7EvxZTKQvd0InF3nmgM= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7/go.mod h1:U6ZQobyTjI/tJyq2HG+i/dfSoFUt8/aZCM+GKtmFk/Y= -github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= -github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= -github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY= -github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZmb2XgV7o= -github.com/mdlayher/raw v0.0.0-20190606142536-fef19f00fc18/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg= -github.com/mdlayher/raw v0.0.0-20191009151244-50f2db8cc065/go.mod h1:7EpbotpCmVZcu+KCX4g9WaRNuu11uyhiW7+Le1dKawg= -github.com/mdlayher/socket v0.2.0/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E= -github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw= -github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc= github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= -github.com/mdlayher/vsock v1.1.1/go.mod h1:Y43jzcy7KM3QB+/FK15pfqGxDMCMzUXWegEfIbSM18U= -github.com/mdlayher/vsock v1.2.0 h1:klRY9lndjmg6k/QWbX/ucQ3e2JFRm1M7vfG9hijbQ0A= -github.com/mdlayher/vsock v1.2.0/go.mod h1:w4kdSTQB9p1l/WwGmAs0V62qQ869qRYoongwgN+Y1HE= github.com/mdlayher/vsock v1.2.1 h1:pC1mTJTvjo1r9n9fbm7S1j04rCgCzhCOS5DY0zqHlnQ= github.com/mdlayher/vsock v1.2.1/go.mod h1:NRfCibel++DgeMD8z/hP+PPTjlNJsdPOmxcnENvE+SE= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/milosgajdos/tenus v0.0.3 h1:jmaJzwaY1DUyYVD0lM4U+uvP2kkEg1VahDqRFxIkVBE= github.com/milosgajdos/tenus v0.0.3/go.mod h1:eIjx29vNeDOYWJuCnaHY2r4fq5egetV26ry3on7p8qY= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mohae/deepcopy v0.0.0-20170308212314-bb9b5e7adda9 h1:Sha2bQdoWE5YQPTlJOL31rmce94/tYi113SlFo1xQ2c= -github.com/mohae/deepcopy v0.0.0-20170308212314-bb9b5e7adda9/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc90/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a h1:9iT75RHhYHWwWRlVWU7wnmtFulYcURCglzQOpT+cAF8= -github.com/opencontainers/runtime-spec v1.0.3-0.20211123151946-c2389c3cb60a/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.1.0-rc.1 h1:wHa9jroFfKGQqFHj0I1fMRKLl0pfj+ynAqBxo3v6u9w= -github.com/opencontainers/runtime-spec v1.1.0-rc.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091 h1:1zN6ImoqhSJhN8hGXFaJlSC8msLmIbX8bFqOfWLKw0w= -github.com/songgao/packets v0.0.0-20160404182456-549a10cd4091/go.mod h1:N20Z5Y8oye9a7HmytmZ+tr8Q2vlP0tAHP13kTHzwvQY= github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= -github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/u-root/uio v0.0.0-20210528114334-82958018845c/go.mod h1:LpEX5FO/cB+WF4TYGY1V5qktpaZLkKkSegbr0V4eYXA= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vishvananda/netlink v1.0.1-0.20190930145447-2ec5bdc52b86/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.1 h1:JDkWS7Axy5ziNM3svylLhpSgqjPDb+BgVUbXoDo+iPw= -github.com/vishvananda/netns v0.0.1/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= -github.com/vishvananda/netns v0.0.3 h1:WxY6MpgIdDMQX50UJ7bPIRJdBCOeUV6XtW8dZZja988= -github.com/vishvananda/netns v0.0.3/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190419010253-1f3472d942ba/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190418153312-f0ce4c0180be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606122018-79a91cf218c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= -golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105210202-9ed45478a130/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210722135532-667f2b7c528f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.42.0-dev.0.20211020220737-f00baa6c3c84/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.2-0.20230118093459-a9481185b34d h1:qp0AnQCvRCMlu9jBjtdbTaaEmThIgZOrbVyDEOcmKhQ= -google.golang.org/protobuf v1.28.2-0.20230118093459-a9481185b34d/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gvisor.dev/gvisor v0.0.0-20220121190119-4f2d380c8b55/go.mod h1:vmN0Pug/s8TJmpnt30DvrEfZ5vDl52psGLU04tFuK2U= -gvisor.dev/gvisor v0.0.0-20221020013634-8e6a0b996cdf h1:odIYzLMj6x99QKT69c6O8Cc3BNas3oUZPPw74hUzsS0= -gvisor.dev/gvisor v0.0.0-20221020013634-8e6a0b996cdf/go.mod h1:D0iRe6RVONyvN6uEi/rqBtONyitX5GaHMDDbeMzwgiE= -gvisor.dev/gvisor v0.0.0-20230120050049-cc0dc87fa27d h1:AlEztVdR4/JpZ6H6jLn4emZjz0GxNtwZky6nlePV674= -gvisor.dev/gvisor v0.0.0-20230120050049-cc0dc87fa27d/go.mod h1:94x/o/BlxPAbw4phqHRac0/IzpcQRUP7ZQldDWV3TKU= -gvisor.dev/gvisor v0.0.0-20230120050912-b6da4fed55f0 h1:Yzd9ezp0lE3VtdTQUdkM7IJAeafJ/Ycpg+22jWOC2LA= -gvisor.dev/gvisor v0.0.0-20230120050912-b6da4fed55f0/go.mod h1:94x/o/BlxPAbw4phqHRac0/IzpcQRUP7ZQldDWV3TKU= -gvisor.dev/gvisor v0.0.0-20230218064137-f11778abbab6 h1:HFyFcXT7q8D4iklzDg7eLPojzeRgYZRfMfc53Vig8Gw= -gvisor.dev/gvisor v0.0.0-20230218064137-f11778abbab6/go.mod h1:pzr6sy8gDLfVmDAg8OYrlKvGEHw5C3PGTiBXBTCx76Q= -gvisor.dev/gvisor v0.0.0-20230218065217-1b534ef82cec h1:C3Ox17AYBEjJx+FatCp61xBTxxWs4YFf6gFhb4nD/C4= -gvisor.dev/gvisor v0.0.0-20230218065217-1b534ef82cec/go.mod h1:pzr6sy8gDLfVmDAg8OYrlKvGEHw5C3PGTiBXBTCx76Q= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.16.13/go.mod h1:QWu8UWSTiuQZMMeYjwLs6ILu5O74qKSJ0c+4vrchDxs= -k8s.io/apimachinery v0.16.13/go.mod h1:4HMHS3mDHtVttspuuhrJ1GGr/0S9B6iWYWZ57KnnZqQ= -k8s.io/apimachinery v0.16.14-rc.0/go.mod h1:4HMHS3mDHtVttspuuhrJ1GGr/0S9B6iWYWZ57KnnZqQ= -k8s.io/client-go v0.16.13/go.mod h1:UKvVT4cajC2iN7DCjLgT0KVY/cbY6DGdUCyRiIfws5M= -k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/kube-openapi v0.0.0-20200410163147-594e756bea31/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/handlers.go b/handlers.go index d054438..3018a21 100644 --- a/handlers.go +++ b/handlers.go @@ -3,31 +3,41 @@ package main import ( "crypto/sha256" "encoding/base64" - "encoding/hex" + "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "strings" + "sync" ) const ( // The maximum length of the key material (in bytes) that enclave // applications can PUT to our HTTP API. maxKeyMaterialLen = 1024 * 1024 + // The maximum length (in bytes) of a heartbeat's request body: + // 44 bytes for the Base64-encoded SHA-256 hash, 255 bytes for the domain + // name, and another 128 bytes for the port and the surrounding JSON. + maxHeartbeatBody = 44 + 255 + 128 // The HTML for the enclave's index page. indexPage = "This host runs inside an AWS Nitro Enclave.\n" ) var ( - errFailedReqBody = errors.New("failed to read request body") - errFailedGetState = errors.New("failed to retrieve saved state") - errNoAddr = errors.New("parameter 'addr' not found") - errBadSyncAddr = errors.New("invalid 'addr' parameter for sync") - errHashWrongSize = errors.New("given hash is of invalid size") + errFailedReqBody = errors.New("failed to read request body") + errHashWrongSize = errors.New("given hash is of invalid size") + errNoBase64 = errors.New("no Base64 given") + errDesignationInProgress = errors.New("leader designation in progress") + errEndpointGone = errors.New("endpoint not meant to be used") + errKeySyncDisabled = errors.New("key synchronization is disabled") ) +func errNo200(code int) error { + return fmt.Errorf("peer responded with HTTP code %d", code) +} + func formatIndexPage(appURL *url.URL) string { page := indexPage if appURL != nil { @@ -46,62 +56,31 @@ func rootHandler(cfg *Config) http.HandlerFunc { } } -// reqSyncHandler returns a handler that lets the enclave application request -// state synchronization, which copies the given remote enclave's state into -// our state. -// -// This is an enclave-internal endpoint that can only be accessed by the -// trusted enclave application. -// -// FIXME: https://github.com/brave/nitriding-daemon/issues/10 -func reqSyncHandler(e *Enclave) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - q := r.URL.Query() - // The 'addr' parameter must have the following form: - // https://example.com:443 - addrs, ok := q["addr"] - if !ok { - http.Error(w, errNoAddr.Error(), http.StatusBadRequest) - return - } - addr := addrs[0] - - // Are we dealing with a well-formed URL? - if _, err := url.Parse(addr); err != nil { - http.Error(w, errBadSyncAddr.Error(), http.StatusBadRequest) - return - } - - if err := RequestKeys(addr, e.KeyMaterial); err != nil { - http.Error(w, fmt.Sprintf("failed to synchronize state: %v", err), http.StatusInternalServerError) - return - } - w.WriteHeader(http.StatusOK) - } -} - // getStateHandler returns a handler that lets the enclave application retrieve // previously-set state. // // This is an enclave-internal endpoint that can only be accessed by the // trusted enclave application. -func getStateHandler(e *Enclave) http.HandlerFunc { +func getStateHandler(getSyncState func() int, keys *enclaveKeys) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/octet-stream") - s, err := e.KeyMaterial() - if err != nil { - http.Error(w, errFailedGetState.Error(), http.StatusInternalServerError) - return - } - n, err := w.Write(s.([]byte)) - if err != nil { - elog.Printf("Error writing state to client: %v", err) - return - } - expected := len(s.([]byte)) - if n != expected { - elog.Printf("Only wrote %d out of %d-byte state to client.", n, expected) - return + switch getSyncState() { + case noSync: + http.Error(w, errKeySyncDisabled.Error(), http.StatusForbidden) + case isLeader: + http.Error(w, errEndpointGone.Error(), http.StatusGone) + case inProgress: + http.Error(w, errDesignationInProgress.Error(), http.StatusServiceUnavailable) + case isWorker: + w.Header().Set("Content-Type", "application/octet-stream") + appKeys := keys.getAppKeys() + n, err := w.Write(appKeys) + if err != nil { + elog.Fatalf("Error writing state to client: %v", err) + } + expected := len(appKeys) + if n != expected { + elog.Fatalf("Only wrote %d out of %d-byte state to client.", n, expected) + } } } } @@ -112,15 +91,42 @@ func getStateHandler(e *Enclave) http.HandlerFunc { // // This is an enclave-internal endpoint that can only be accessed by the // trusted enclave application. -func putStateHandler(e *Enclave) http.HandlerFunc { +func putStateHandler( + a attester, + getSyncState func() int, + enclaveKeys *enclaveKeys, + workers *workerManager, +) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - body, err := io.ReadAll(newLimitReader(r.Body, maxKeyMaterialLen)) - if err != nil { - http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) - return + switch getSyncState() { + case noSync: + http.Error(w, errKeySyncDisabled.Error(), http.StatusForbidden) + case isWorker: + http.Error(w, errEndpointGone.Error(), http.StatusGone) + case inProgress: + http.Error(w, errDesignationInProgress.Error(), http.StatusServiceUnavailable) + case isLeader: + keys, err := io.ReadAll(newLimitReader(r.Body, maxKeyMaterialLen)) + if err != nil { + http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) + return + } + enclaveKeys.setAppKeys(keys) + w.WriteHeader(http.StatusOK) + + // The leader's application keys have changed. Re-synchronize the key + // material with all registered workers. If synchronization fails for a + // given worker, unregister it. + elog.Printf("Application keys have changed. Re-synchronizing with %d worker(s).", + workers.length()) + go workers.forAll( + func(worker *url.URL) { + if err := asLeader(enclaveKeys, a).syncWith(worker); err != nil { + workers.unregister(worker) + } + }, + ) } - e.SetKeyMaterial(body) - w.WriteHeader(http.StatusOK) } } @@ -163,17 +169,18 @@ func hashHandler(e *Enclave) http.HandlerFunc { // signal that it's ready, instructing nitriding to start its Internet-facing // Web server. We initially gate access to the Internet-facing API to avoid // the issuance of unexpected attestation documents that lack the application's -// hash because the application couldn't register it in time. The downside is -// that state synchronization among enclaves does not work until the -// application signalled its readiness. While not ideal, we chose to ignore -// this for now. +// hash because the application couldn't register it in time. // // This is an enclave-internal endpoint that can only be accessed by the // trusted enclave application. -func readyHandler(e *Enclave) http.HandlerFunc { +func readyHandler(ready chan struct{}) http.HandlerFunc { + var once sync.Once return func(w http.ResponseWriter, r *http.Request) { - close(e.ready) - w.WriteHeader(http.StatusOK) + once.Do(func() { + close(ready) + w.WriteHeader(http.StatusOK) + }) + w.WriteHeader(http.StatusGone) } } @@ -192,36 +199,94 @@ func configHandler(cfg *Config) http.HandlerFunc { // subsequently asks its hypervisor for an attestation document that contains // both the nonce and the hashes in the given struct. The resulting // Base64-encoded attestation document is then returned to the requester. -func attestationHandler(useProfiling bool, hashes *AttestationHashes) http.HandlerFunc { +func attestationHandler(useProfiling bool, hashes *AttestationHashes, a attester) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if useProfiling { - http.Error(w, errProfilingSet, http.StatusServiceUnavailable) + http.Error(w, errProfilingSet.Error(), http.StatusServiceUnavailable) return } - if err := r.ParseForm(); err != nil { - http.Error(w, errBadForm, http.StatusBadRequest) + + n, err := getNonceFromReq(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) return } - nonce := r.URL.Query().Get("nonce") - if nonce == "" { - http.Error(w, errNoNonce, http.StatusBadRequest) + rawDoc, err := a.createAttstn(&clientAuxInfo{ + clientNonce: n, + attestationHashes: hashes.Serialize(), + }) + if err != nil { + http.Error(w, errFailedAttestation.Error(), http.StatusInternalServerError) return } - nonce = strings.ToLower(nonce) - // Decode hex-encoded nonce. - rawNonce, err := hex.DecodeString(nonce) + b64Doc := base64.StdEncoding.EncodeToString(rawDoc) + fmt.Fprintln(w, b64Doc) + } +} + +func heartbeatHandler(e *Enclave) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + hb heartbeatRequest + syncAndRegister = func(keys *enclaveKeys, worker *url.URL) { + if err := asLeader(keys, e.attester).syncWith(worker); err == nil { + e.workers.register(worker) + } + } + ) + + body, err := io.ReadAll(newLimitReader(r.Body, maxHeartbeatBody)) if err != nil { - http.Error(w, errBadNonceFormat, http.StatusBadRequest) + http.Error(w, errFailedReqBody.Error(), http.StatusInternalServerError) + return + } + if err := json.Unmarshal(body, &hb); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) return } + worker, err := e.getWorker(&hb) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + elog.Printf("Heartbeat from worker %s.", worker.Host) + ourKeysHash, theirKeysHash := e.keys.hashAndB64(), hb.HashedKeys + if ourKeysHash != theirKeysHash { + elog.Printf("Worker's keys are invalid. Re-synchronizing.") + go syncAndRegister(e.keys, worker) + } else { + e.workers.register(worker) + } + w.WriteHeader(http.StatusOK) + } +} - rawDoc, err := attest(rawNonce, hashes.Serialize(), nil) +func getLeaderHandler(ourNonce nonce, weAreLeader chan struct{}) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var ( + err error + theirNonce nonce + ) + theirNonce, err = getNonceFromReq(r) if err != nil { - http.Error(w, errFailedAttestation, http.StatusInternalServerError) + http.Error(w, err.Error(), http.StatusBadRequest) return } - b64Doc := base64.StdEncoding.EncodeToString(rawDoc) - fmt.Fprintln(w, b64Doc) + + if ourNonce == theirNonce { + if len(weAreLeader) == 0 { + weAreLeader <- struct{}{} + } + } else { + // We may end up in this branch for two reasons: + // 1. We're the leader and a worker beat us to talking to this + // endpoint. + // 2. We're a worker and some other entity in the private network is + // talking to this endpoint. That shouldn't happen. + elog.Println("Received nonce that does not match our own.") + } + w.WriteHeader(http.StatusOK) } } diff --git a/handlers_test.go b/handlers_test.go index 2f8bfb2..01e0776 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "crypto/tls" "encoding/base64" + "encoding/json" "errors" "fmt" "io" @@ -12,13 +13,14 @@ import ( "net/http/httptest" "net/http/httputil" "net/url" + "strings" + "sync" "syscall" "testing" "time" ) -// makeRequestFor is a helper function that creates an HTTP request. -func makeRequestFor(srv *http.Server) func(method, path string, body io.Reader) *http.Response { +func makeReqToSrv(srv *http.Server) func(method, path string, body io.Reader) *http.Response { return func(method, path string, body io.Reader) *http.Response { req := httptest.NewRequest(method, path, body) rec := httptest.NewRecorder() @@ -27,6 +29,15 @@ func makeRequestFor(srv *http.Server) func(method, path string, body io.Reader) } } +func makeReqToHandler(handler http.HandlerFunc) func(method, path string, body io.Reader) *http.Response { + return func(method, path string, body io.Reader) *http.Response { + req := httptest.NewRequest(method, path, body) + w := httptest.NewRecorder() + handler(w, req) + return w.Result() + } +} + // newResp is a helper function that creates an HTTP response. func newResp(status int, body string) *http.Response { return &http.Response{ @@ -35,6 +46,26 @@ func newResp(status int, body string) *http.Response { } } +func retState(state int) func() int { + return func() int { + return state + } +} + +// keysToHeartbeat turns the given keys into a Buffer that contains a heartbeat +// request. +func keysToHeartbeat(t *testing.T, keys *enclaveKeys) *bytes.Buffer { + t.Helper() + blob, err := json.Marshal(&heartbeatRequest{ + HashedKeys: keys.hashAndB64(), + WorkerHostname: "localhost:1234", + }) + if err != nil { + t.Fatal(err) + } + return bytes.NewBuffer(blob) +} + // assertResponse ensures that the two given HTTP responses are (almost) // identical. We only check the HTTP status code and the response body. // If the expected response has no body, we only compare the status code. @@ -67,7 +98,7 @@ func assertResponse(t *testing.T, actual, expected *http.Response) { } func TestRootHandler(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).pubSrv) + makeReq := makeReqToSrv(createEnclave(&defaultCfg).extPubSrv) assertResponse(t, makeReq(http.MethodGet, pathRoot, nil), @@ -79,7 +110,7 @@ func TestRootHandler(t *testing.T) { // instructing it to spin up its Internet-facing Web server. func signalReady(t *testing.T, e *Enclave) { t.Helper() - makeReq := makeRequestFor(e.privSrv) + makeReq := makeReqToSrv(e.intSrv) assertResponse(t, makeReq(http.MethodGet, pathReady, nil), @@ -93,53 +124,98 @@ func signalReady(t *testing.T, e *Enclave) { time.Sleep(100 * time.Millisecond) } -func TestSyncHandler(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).privSrv) +func TestGetStateHandler(t *testing.T) { + var keys = newTestKeys(t) + makeReq := makeReqToHandler(getStateHandler(retState(noSync), keys)) assertResponse(t, - makeReq(http.MethodGet, pathSync, nil), - newResp(http.StatusBadRequest, errNoAddr.Error()), + makeReq(http.MethodGet, pathState, nil), + newResp(http.StatusForbidden, errKeySyncDisabled.Error()), ) + makeReq = makeReqToHandler(getStateHandler(retState(isLeader), keys)) assertResponse(t, - makeReq(http.MethodGet, pathSync+"?addr=:foo", nil), - newResp(http.StatusBadRequest, errBadSyncAddr.Error()), + makeReq(http.MethodGet, pathState, nil), + newResp(http.StatusGone, errEndpointGone.Error()), ) + makeReq = makeReqToHandler(getStateHandler(retState(isWorker), keys)) assertResponse(t, - makeReq(http.MethodGet, pathSync+"?addr=foobar", nil), - newResp(http.StatusInternalServerError, ""), // The exact error is convoluted, so we skip comparison. + makeReq(http.MethodGet, pathState, nil), + newResp(http.StatusOK, string(keys.getAppKeys())), + ) + + makeReq = makeReqToHandler(getStateHandler(retState(inProgress), keys)) + assertResponse(t, + makeReq(http.MethodGet, pathState, nil), + newResp(http.StatusServiceUnavailable, errDesignationInProgress.Error()), ) } -func TestStateHandlers(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).privSrv) +func TestPutStateHandler(t *testing.T) { + var ( + tooLargeKey = make([]byte, maxKeyMaterialLen+1) + almostTooLargeKey = make([]byte, maxKeyMaterialLen) + a = &dummyAttester{} + keys = newTestKeys(t) + stop = make(chan struct{}) + workers = newWorkerManager(time.Second) + ) + go workers.start(stop) + defer close(stop) + + makeReq := makeReqToHandler(putStateHandler(a, retState(noSync), keys, workers)) + assertResponse(t, + makeReq(http.MethodPut, pathState, strings.NewReader("appKeys")), + newResp(http.StatusForbidden, errKeySyncDisabled.Error()), + ) + + makeReq = makeReqToHandler(putStateHandler(a, retState(isWorker), keys, workers)) + assertResponse(t, + makeReq(http.MethodPut, pathState, strings.NewReader("appKeys")), + newResp(http.StatusGone, errEndpointGone.Error()), + ) - tooLargeKey := make([]byte, 1024*1024+1) + makeReq = makeReqToHandler(putStateHandler(a, retState(inProgress), keys, workers)) + assertResponse(t, + makeReq(http.MethodPut, pathState, strings.NewReader("appKeys")), + newResp(http.StatusServiceUnavailable, errDesignationInProgress.Error()), + ) + + makeReq = makeReqToHandler(putStateHandler(a, retState(isLeader), keys, workers)) assertResponse(t, makeReq(http.MethodPut, pathState, bytes.NewReader(tooLargeKey)), newResp(http.StatusInternalServerError, errFailedReqBody.Error()), ) - - // As long as we don't hit our (generous) upload limit, we always expect an - // HTTP 200 response. - almostTooLargeKey := make([]byte, 1024*1024) assertResponse(t, makeReq(http.MethodPut, pathState, bytes.NewReader(almostTooLargeKey)), newResp(http.StatusOK, ""), ) +} + +func TestGetPutStateHandlers(t *testing.T) { + var ( + a = &dummyAttester{} + keys = newTestKeys(t) + appKeys = "application keys" + stop = make(chan struct{}) + workers = newWorkerManager(time.Second) + ) + go workers.start(stop) + defer close(stop) - // Subsequent calls to the endpoint overwrite the previous call. - expected := []byte("foobar") + // Set application state. + makeReq := makeReqToHandler(putStateHandler(a, retState(isLeader), keys, workers)) assertResponse(t, - makeReq(http.MethodPut, pathState, bytes.NewReader(expected)), + makeReq(http.MethodPut, pathState, strings.NewReader(appKeys)), newResp(http.StatusOK, ""), ) - // Now retrieve the state and make sure that it's what we sent earlier. + // Retrieve previously-set application state. + makeReq = makeReqToHandler(getStateHandler(retState(isWorker), keys)) assertResponse(t, makeReq(http.MethodGet, pathState, nil), - newResp(http.StatusOK, string(expected)), + newResp(http.StatusOK, appKeys), ) } @@ -172,7 +248,7 @@ func TestProxyHandler(t *testing.T) { // Skip certificate validation because we are using a self-signed // certificate in this test. http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - nitridingSrv := "https://127.0.0.1" + e.pubSrv.Addr + nitridingSrv := fmt.Sprintf("https://127.0.0.1:%d", e.cfg.ExtPubPort) // Request the enclave's index page. Nitriding is going to return it. resp, err := http.Get(nitridingSrv + pathRoot) @@ -194,7 +270,7 @@ func TestHashHandler(t *testing.T) { validHash := [sha256.Size]byte{} validHashB64 := base64.StdEncoding.EncodeToString(validHash[:]) e := createEnclave(&defaultCfg) - makeReq := makeRequestFor(e.privSrv) + makeReq := makeReqToSrv(e.intSrv) // Send invalid Base64. assertResponse(t, @@ -242,7 +318,7 @@ func TestReadiness(t *testing.T) { } defer e.Stop() //nolint:errcheck - nitridingSrv := fmt.Sprintf("https://127.0.0.1:%d", e.cfg.ExtPort) + nitridingSrv := fmt.Sprintf("https://127.0.0.1:%d", e.cfg.ExtPubPort) u := nitridingSrv + pathRoot // Make sure that the Internet-facing Web server is already running because // we didn't ask nitriding to wait for the application. The Web server may @@ -269,7 +345,7 @@ func TestReadiness(t *testing.T) { }(t, u) } -func TestReadyHandler(t *testing.T) { +func TestReadinessWithWaitForUp(t *testing.T) { cfg := defaultCfg cfg.WaitForApp = true e := createEnclave(&cfg) @@ -279,7 +355,7 @@ func TestReadyHandler(t *testing.T) { defer e.Stop() //nolint:errcheck // Check if the Internet-facing Web server is running. - nitridingSrv := fmt.Sprintf("https://127.0.0.1:%d", e.cfg.ExtPort) + nitridingSrv := fmt.Sprintf("https://127.0.0.1:%d", e.cfg.ExtPubPort) _, err := http.Get(nitridingSrv + pathRoot) if !errors.Is(err, syscall.ECONNREFUSED) { t.Fatal("Expected 'connection refused'.") @@ -296,20 +372,47 @@ func TestReadyHandler(t *testing.T) { } } +func TestReadyHandler(t *testing.T) { + var ( + ready = make(chan struct{}) + wg sync.WaitGroup + ) + wg.Add(1) + go func() { + defer wg.Done() + <-ready + }() + + makeReq := makeReqToHandler(readyHandler(ready)) + assertResponse(t, + makeReq(http.MethodGet, pathReady, nil), + newResp(http.StatusOK, ""), + ) + wg.Wait() + + // Subsequent calls should return 410 Gone. + assertResponse(t, + makeReq(http.MethodGet, pathReady, nil), + newResp(http.StatusGone, ""), + ) +} + func TestAttestationHandlerWhileProfiling(t *testing.T) { cfg := defaultCfg cfg.UseProfiling = true - makeReq := makeRequestFor(createEnclave(&cfg).pubSrv) + makeReq := makeReqToSrv(createEnclave(&cfg).extPubSrv) // Ensure that the attestation handler aborts if profiling is enabled. assertResponse(t, makeReq(http.MethodGet, pathAttestation, nil), - newResp(http.StatusServiceUnavailable, errProfilingSet), + newResp(http.StatusServiceUnavailable, errProfilingSet.Error()), ) } func TestAttestationHandler(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).pubSrv) + prodCfg := defaultCfg + prodCfg.Debug = false + makeReq := makeReqToSrv(createEnclave(&prodCfg).extPubSrv) assertResponse(t, makeReq(http.MethodPost, pathAttestation, nil), @@ -318,12 +421,12 @@ func TestAttestationHandler(t *testing.T) { assertResponse(t, makeReq(http.MethodGet, pathAttestation, nil), - newResp(http.StatusBadRequest, errNoNonce), + newResp(http.StatusBadRequest, errNoNonce.Error()), ) assertResponse(t, makeReq(http.MethodGet, pathAttestation+"?nonce=foobar", nil), - newResp(http.StatusBadRequest, errBadNonceFormat), + newResp(http.StatusBadRequest, errBadNonceFormat.Error()), ) // If we are not inside an enclave, attestation is going to result in an @@ -331,16 +434,114 @@ func TestAttestationHandler(t *testing.T) { if !inEnclave { assertResponse(t, makeReq(http.MethodGet, pathAttestation+"?nonce=0000000000000000000000000000000000000000", nil), - newResp(http.StatusInternalServerError, errFailedAttestation), + newResp(http.StatusInternalServerError, errFailedAttestation.Error()), ) } } func TestConfigHandler(t *testing.T) { - makeReq := makeRequestFor(createEnclave(&defaultCfg).pubSrv) + makeReq := makeReqToSrv(createEnclave(&defaultCfg).extPubSrv) assertResponse(t, makeReq(http.MethodGet, pathConfig, nil), newResp(http.StatusOK, defaultCfg.String()), ) } + +func TestHeartbeatHandler(t *testing.T) { + var ( + e = createEnclave(&defaultCfg) + keys = newTestKeys(t) + makeReq = makeReqToSrv(e.extPrivSrv) + ) + e.setupLeader() + e.keys.set(keys) + + tooLargeBuf := bytes.NewBuffer(make([]byte, maxHeartbeatBody+1)) + assertResponse(t, + makeReq(http.MethodPost, pathHeartbeat, tooLargeBuf), + newResp(http.StatusInternalServerError, errFailedReqBody.Error()), + ) + + assertResponse(t, + makeReq(http.MethodPost, pathHeartbeat, keysToHeartbeat(t, keys)), + newResp(http.StatusOK, ""), + ) +} + +func TestHeartbeatHandlerWithSync(t *testing.T) { + var ( + wg = sync.WaitGroup{} + leaderEnclave = createEnclave(&defaultCfg) + makeReq = makeReqToSrv(leaderEnclave.extPrivSrv) + workerKeys = newTestKeys(t) + setWorkerKeys = func(keys *enclaveKeys) error { + defer wg.Done() + workerKeys.set(keys) + return nil + } + worker = asWorker(setWorkerKeys, &dummyAttester{}) + workerSrv = httptest.NewTLSServer(worker) + ) + defer workerSrv.Close() + if err := leaderEnclave.Start(); err != nil { + t.Fatal(err) + } + leaderEnclave.setupLeader() + wg.Add(1) + + // Mock two functions to make the leader enclave talk to our test server. + newUnauthenticatedHTTPClient = workerSrv.Client + getSyncURL = func(host string, port uint16) *url.URL { + u, err := url.Parse(workerSrv.URL) + if err != nil { + t.Fatal(err) + } + return u + } + + assertEqual(t, leaderEnclave.workers.length(), 0) + + // Send a heartbeat to the leader. The heartbeat's keys don't match the + // leader's keys, which results in the leader initiating key + // synchronization. + assertResponse(t, + makeReq(http.MethodPost, pathHeartbeat, keysToHeartbeat(t, workerKeys)), + newResp(http.StatusOK, ""), + ) + + // Wait until the worker's keys were set and make sure that the keys were + // synchronized successfully. + wg.Wait() + assertEqual(t, leaderEnclave.keys.equal(workerKeys), true) +} + +func TestGetLeaderHandler(t *testing.T) { + var ( + weAreLeader = make(chan struct{}) + unexpectedSuffix = "?nonce=0000000000000000000000000000000000000000" + ) + nonce, err := newNonce() + failOnErr(t, err) + + // Don't provide the expected nonce. + makeReq := makeReqToHandler(getLeaderHandler(nonce, weAreLeader)) + assertResponse(t, + makeReq(http.MethodGet, pathLeader, nil), + newResp(http.StatusBadRequest, errNoNonce.Error()), + ) + + // Send an unexpected nonce. + assertResponse(t, + makeReq(http.MethodGet, pathLeader+unexpectedSuffix, nil), + newResp(http.StatusOK, ""), + ) + + // Send the expected nonce. + go func() { <-weAreLeader }() + suffix := fmt.Sprintf("?nonce=%x", nonce) + assertResponse(t, + makeReq(http.MethodGet, pathLeader+suffix, nil), + newResp(http.StatusOK, ""), + ) +} diff --git a/keysync_initiator.go b/keysync_initiator.go deleted file mode 100644 index 79bd9b2..0000000 --- a/keysync_initiator.go +++ /dev/null @@ -1,212 +0,0 @@ -package main - -// AWS Nitro Enclave attestation documents contain three fields (called -// "nonce", "user data", and "public key") that can be set by the requester. -// We are using those fields as follows: -// -// When the requesting enclave sends a request to the remote enclave, it sets -// the following fields in the attestation document: -// -// Attestation document( -// Nonce: Remote enclave's nonce -// User data: Requesting enclave's nonce -// Public key: Requesting enclave's NaCl box public key -// ) -// -// The remote enclave then generates its own attestation document containing -// the following fields: -// -// Attestation document( -// Nonce: The nonce the requester provided in its attestation document -// User data: Encrypted key material -// Public key: -// ) - -import ( - "bytes" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "strings" - - "github.com/hf/nitrite" - "golang.org/x/crypto/nacl/box" -) - -// RequestKeys asks a remote enclave to share its key material with us, which -// is then written to the provided variable. -// -// This is only necessary if you intend to scale enclaves horizontally. If -// you will only ever run a single enclave, ignore this function. -func RequestKeys(addr string, keyMaterial any) error { - errStr := "failed to request key material" - - // First, request a nonce from the remote enclave. - theirNonce, err := requestNonce(addr) - if err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - // Now, create our own nonce. - ourNonce, err := newNonce() - if err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - // Next, create a key that the remote enclave is going to use to encrypt - // its key material. - boxKey, err := newBoxKey() - if err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - // Now create an attestation document containing our nonce, the remote - // enclave's nonce, and the key material that they remote enclave is - // supposed to use. - ourAttDoc, err := attest(theirNonce[:], ourNonce[:], boxKey.pubKey[:]) - if err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - // Send our attestation document to the remote enclave, and get theirs in - // return. - theirAttDoc, err := requestAttDoc(addr, ourAttDoc) - if err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - // Finally, verify the attestation document and extract the key material. - if err := processAttDoc(theirAttDoc, &ourNonce, boxKey, keyMaterial); err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - return nil -} - -// requestNonce requests a nonce from the remote enclave specified by 'addr'. -func requestNonce(addr string) (nonce, error) { - errStr := "failed to fetch nonce from remote enclave" - - endpoint := fmt.Sprintf("%s%s", addr, pathNonce) - resp, err := http.Get(endpoint) - if err != nil { - return nonce{}, fmt.Errorf("%s: %w", errStr, err) - } - defer resp.Body.Close() - - // Add an extra byte to account for the "\n". - maxReadLen := base64.StdEncoding.EncodedLen(nonceLen) + 1 - body, err := io.ReadAll(newLimitReader(resp.Body, maxReadLen)) - if err != nil { - return nonce{}, fmt.Errorf("%s: %w", errStr, err) - } - - // Decode the Base64-encoded nonce. - raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(body))) - if err != nil { - return nonce{}, fmt.Errorf("%s: %w", errStr, err) - } - - if len(raw) != nonceLen { - return nonce{}, errors.New("remote enclave's nonce has incorrect length") - } - var n nonce - copy(n[:], raw) - return n, nil -} - -// requestAttDoc takes as input the remote enclave's address (e.g., -// ) and our attestation document. The function then -// submits our attestation document to the remote enclave, and returns the -// remote enclave's attestation document. -func requestAttDoc(addr string, ourAttDoc []byte) ([]byte, error) { - errStr := "failed to fetch attestation doc from remote enclave" - - endpoint := fmt.Sprintf("%s%s", addr, pathSync) - - // Finally, send our attestation document to the remote enclave. If - // everything works out, the remote enclave is going to respond with its - // attestation document. - b64AttDoc := base64.StdEncoding.EncodeToString(ourAttDoc) - resp, err := http.Post( - endpoint, - "text/plain", - bytes.NewBufferString(b64AttDoc), - ) - if err != nil { - return nil, fmt.Errorf("%s: %w", errStr, err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%s: expected status code %d but got %d", errStr, http.StatusOK, resp.StatusCode) - } - - maxReadLen := base64.StdEncoding.EncodedLen(maxAttDocLen) - body, err := io.ReadAll(newLimitReader(resp.Body, maxReadLen)) - if err != nil { - return nil, fmt.Errorf("%s: %w", errStr, err) - } - - theirAttDoc, err := base64.StdEncoding.DecodeString(string(body)) - if err != nil { - return nil, fmt.Errorf("%s: %w", errStr, err) - } - - return theirAttDoc, nil -} - -// processAttDoc first verifies that the remote enclave's attestation document -// is authentic, and then attempts to decrypt and extract the key material that -// the remote enclave provided in its attestation document. -func processAttDoc( - theirAttDoc []byte, - ourNonce *nonce, - boxKey *boxKey, - keyMaterial any, -) error { - errStr := "failed to process attestation doc from remote enclave" - // Verify the remote enclave's attestation document before doing anything - // with it. - opts := nitrite.VerifyOptions{CurrentTime: currentTime()} - their, err := nitrite.Verify(theirAttDoc, opts) - if err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - // Are the PCR values (i.e. image IDs) identical? - ourPCRs, err := getPCRValues() - if err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - if !arePCRsIdentical(ourPCRs, their.Document.PCRs) { - return fmt.Errorf("%s: PCR values of remote enclave not identical to ours", errStr) - } - - // Now verify that the remote enclave's attestation document contains the - // nonce that we provided earlier. - if !bytes.Equal(their.Document.Nonce, ourNonce[:]) { - return fmt.Errorf("%s: expected nonce %x but got %x", - errStr, ourNonce[:], their.Document.Nonce) - } - - // Attempt to decrypt the key material. - decrypted, ok := box.OpenAnonymous( - nil, - their.Document.UserData, - boxKey.pubKey, - boxKey.privKey) - if !ok { - return fmt.Errorf("%s: failed to decrypt key material", errStr) - } - - // Finally, write the JSON-encoded key material to the provided interface. - if err := json.Unmarshal(decrypted, keyMaterial); err != nil { - return fmt.Errorf("%s: %w", errStr, err) - } - - return nil -} diff --git a/keysync_initiator_test.go b/keysync_initiator_test.go deleted file mode 100644 index ebf3146..0000000 --- a/keysync_initiator_test.go +++ /dev/null @@ -1,238 +0,0 @@ -package main - -import ( - "encoding/base64" - "errors" - "fmt" - "net/http" - "net/http/httptest" - "testing" - "time" -) - -func TestRequestNonce(t *testing.T) { - expNonce := nonce{ - 0x14, 0x56, 0x82, 0x13, 0x1f, 0xff, 0x9c, 0xf7, 0xeb, 0xb6, - 0x9e, 0x7b, 0xea, 0x29, 0x16, 0x49, 0xeb, 0x03, 0xa2, 0x47, - } - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, expNonce.B64()) - })) - defer srv.Close() - - retNonce, err := requestNonce(srv.URL) - if err != nil { - t.Fatalf("Failed to request nonce: %s", err) - } - if expNonce != retNonce { - t.Fatal("Returned nonce not as expected.") - } -} - -func TestRequestNonceDoS(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - nonce1 := nonce{} - nonce2 := nonce{} - fmt.Fprintf(w, "%s%s", nonce1.B64(), nonce2.B64()) - })) - defer srv.Close() - - _, err := requestNonce(srv.URL) - if !errors.Is(err, errTooMuchToRead) { - t.Fatalf("Expected error %q but got %q.", errTooMuchToRead, err) - } -} - -func TestRequestAttDoc(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, "foobar") - })) - defer srv.Close() - - _, err := requestAttDoc(srv.URL, []byte{}) - if err == nil { - t.Fatal("Client code should have rejected non-Base64 data but didn't.") - } -} - -func TestRequestAttDocDoS(t *testing.T) { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - maxReadLen := base64.StdEncoding.EncodedLen(maxAttDocLen) - // Send one byte more than the client is willing to read. - buf := make([]byte, maxReadLen+1) - fmt.Fprint(w, string(buf)) - })) - defer srv.Close() - - _, err := requestAttDoc(srv.URL, []byte{}) - if !errors.Is(err, errTooMuchToRead) { - t.Fatalf("Expected error %q but got %q.", errTooMuchToRead, err) - } -} - -func TestProcessAttDoc(t *testing.T) { - // Mock functions for our tests to pass. - getPCRValues = func() (map[uint][]byte, error) { - return respAttInfo.pcr, nil - } - currentTime = func() time.Time { return respAttInfo.attDocTime } - - rawAttDoc, err := base64.StdEncoding.DecodeString(respAttInfo.attDoc) - if err != nil { - t.Fatalf("Failed to Base64-decode attestation document: %s", err) - } - keyMaterial := struct { - SecretKey string `json:"secret_key"` - }{} - - if err := processAttDoc( - rawAttDoc, - &respAttInfo.nonce, - &boxKey{ - pubKey: &respAttInfo.pubKey, - privKey: &respAttInfo.privKey, - }, - &keyMaterial, - ); err != nil { - t.Fatalf("Failed to verify valid attestation document: %s", err) - } - - // Make sure that processAttDoc successfully decrypted and recovered the - // secret key material, "foobar". - if keyMaterial.SecretKey != "foobar" { - t.Fatalf("Expected secret key 'foobar' but got %q.", keyMaterial.SecretKey) - } -} - -var respAttInfo = &remoteAttInfo{ - pubKey: [boxKeyLen]byte{ - 213, 156, 108, 34, 179, 183, 69, 26, 209, 218, 58, 186, 9, 32, 237, - 253, 46, 80, 36, 200, 169, 239, 97, 200, 17, 188, 203, 99, 151, 40, - 10, 113, - }, - privKey: [boxKeyLen]byte{ - 74, 137, 121, 11, 209, 38, 48, 48, 167, 157, 184, 58, 2, 110, 9, 204, - 174, 148, 243, 154, 191, 74, 118, 90, 11, 240, 246, 131, 187, 157, - 157, 25, - }, - nonce: nonce{}, - pcr: map[uint][]byte{ - 0: { - 0xb0, 0x61, 0xbc, 0xe3, 0x1a, 0x85, 0x50, 0xc2, 0x4c, 0xb8, - 0xc1, 0xdc, 0x0e, 0x53, 0x98, 0xe5, 0xc8, 0x0f, 0xab, 0xa6, - 0x7f, 0x75, 0xfd, 0x3b, 0x06, 0x21, 0xc0, 0xb8, 0x66, 0x36, - 0xfc, 0xe0, 0xd6, 0x4c, 0x4d, 0x7d, 0x37, 0x47, 0x89, 0x08, - 0xe1, 0xf8, 0xfc, 0xe9, 0xdf, 0x66, 0xe1, 0xb9}, - 1: { - 0xbc, 0xdf, 0x05, 0xfe, 0xfc, 0xca, 0xa8, 0xe5, 0x5b, 0xf2, - 0xc8, 0xd6, 0xde, 0xe9, 0xe7, 0x9b, 0xbf, 0xf3, 0x1e, 0x34, - 0xbf, 0x28, 0xa9, 0x9a, 0xa1, 0x9e, 0x6b, 0x29, 0xc3, 0x7e, - 0xe8, 0x0b, 0x21, 0x4a, 0x41, 0x4b, 0x76, 0x07, 0x23, 0x6e, - 0xdf, 0x26, 0xfc, 0xb7, 0x86, 0x54, 0xe6, 0x3f}, - 2: { - 0x6a, 0xe6, 0x79, 0x76, 0xd7, 0x40, 0x38, 0x0d, 0x50, 0x64, - 0x36, 0x91, 0xac, 0x3a, 0xae, 0xbb, 0xa6, 0x0f, 0x27, 0xd7, - 0xb8, 0xa0, 0xe1, 0xa9, 0xea, 0xf2, 0x38, 0x6d, 0x25, 0xee, - 0xab, 0x88, 0x1c, 0x09, 0xac, 0xc5, 0xc8, 0x09, 0xeb, 0xec, - 0xf9, 0x9b, 0x49, 0x71, 0x05, 0xf6, 0xcb, 0x5b}, - 3: null, - 4: { - 0xd8, 0xa8, 0xe8, 0xee, 0xe9, 0x6d, 0x81, 0xb7, 0x7a, 0x25, - 0x14, 0x10, 0xb7, 0xa9, 0xb1, 0x80, 0x78, 0x76, 0x53, 0xf1, - 0x25, 0xd1, 0xdb, 0xca, 0x79, 0x68, 0x5c, 0x93, 0xfb, 0x88, - 0x5b, 0x33, 0x5e, 0x0b, 0x8d, 0x17, 0x2c, 0x98, 0x21, 0xa8, - 0x62, 0x51, 0x5a, 0x60, 0x3c, 0xc3, 0x3a, 0xb2}, - 5: null, - 6: null, - 7: null, - 8: null, - 9: null, - 10: null, - 11: null, - 12: null, - 13: null, - 14: null, - 15: null, - }, - attDocTime: mustParse("2022-07-27T05:00:00Z"), - // The following attestation document was generated on 2022-07-27 and - // contains a nonce (set to all 0 bytes) and user data (contains encrypted - // key information). - attDoc: ` -hEShATgioFkRG6lpbW9kdWxlX2lkeCdpLTA4MDk4NDk3MTBiZjFiNjFiLWVuYzAxODIzZDY0M2U2Mzl -hYTBmZGlnZXN0ZlNIQTM4NGl0aW1lc3RhbXAbAAABgj1kWVpkcGNyc7AAWDCwYbzjGoVQwky4wdwOU5 -jlyA+rpn91/TsGIcC4Zjb84NZMTX03R4kI4fj86d9m4bkBWDC83wX+/Mqo5VvyyNbe6eebv/MeNL8oq -Zqhnmspw37oCyFKQUt2ByNu3yb8t4ZU5j8CWDBq5nl210A4DVBkNpGsOq67pg8n17ig4anq8jhtJe6r -iBwJrMXICevs+ZtJcQX2y1sDWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAEWDDYqOju6W2Bt3olFBC3qbGAeHZT8SXR28p5aFyT+4hbM14LjRcsmCGoYlFaYDzDOr -IFWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGWDAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHWDAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAKWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAALWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAMWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANWDAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOWDAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPWDAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABrY2VydGlmaWNhdGVZAn8wggJ7MIICAaADAgECAhAB -gj1kPmOaoAAAAABi4JzCMAoGCCqGSM49BAMDMIGOMQswCQYDVQQGEwJVUzETMBEGA1UECAwKV2FzaGl -uZ3RvbjEQMA4GA1UEBwwHU2VhdHRsZTEPMA0GA1UECgwGQW1hem9uMQwwCgYDVQQLDANBV1MxOTA3Bg -NVBAMMMGktMDgwOTg0OTcxMGJmMWI2MWIudXMtZWFzdC0yLmF3cy5uaXRyby1lbmNsYXZlczAeFw0yM -jA3MjcwMjAyMzlaFw0yMjA3MjcwNTAyNDJaMIGTMQswCQYDVQQGEwJVUzETMBEGA1UECAwKV2FzaGlu -Z3RvbjEQMA4GA1UEBwwHU2VhdHRsZTEPMA0GA1UECgwGQW1hem9uMQwwCgYDVQQLDANBV1MxPjA8BgN -VBAMMNWktMDgwOTg0OTcxMGJmMWI2MWItZW5jMDE4MjNkNjQzZTYzOWFhMC51cy1lYXN0LTIuYXdzMH -YwEAYHKoZIzj0CAQYFK4EEACIDYgAE7il+oEijv6hrLpdsG4T/TbjSNxla6LnM2/2IGyCFNblCghVv1 -VNv7JF1zu+pP4jT7VbeVEj2z5T0lQMc/bLLxXUcbVlaA8qzAIX5yTkwAA53zU6m7frzvWVwdSuSNvXw -ox0wGzAMBgNVHRMBAf8EAjAAMAsGA1UdDwQEAwIGwDAKBggqhkjOPQQDAwNoADBlAjEAiKzrNPjQug4 -lt4wfSuIxvyr4BoiS0en2pLM7NtI9QnQKwXKT7V1Rk4oKr7zVBeiJAjAMnKjSMZn3cID2nL55qgoeCF -0PXntyuGXwkh8J5bsN5BUKP38CiqmONjvyxPOiQWpoY2FidW5kbGWEWQIVMIICETCCAZagAwIBAgIRA -PkxdWgbkK/hHUbMtOTn+FYwCgYIKoZIzj0EAwMwSTELMAkGA1UEBhMCVVMxDzANBgNVBAoMBkFtYXpv -bjEMMAoGA1UECwwDQVdTMRswGQYDVQQDDBJhd3Mubml0cm8tZW5jbGF2ZXMwHhcNMTkxMDI4MTMyODA -1WhcNNDkxMDI4MTQyODA1WjBJMQswCQYDVQQGEwJVUzEPMA0GA1UECgwGQW1hem9uMQwwCgYDVQQLDA -NBV1MxGzAZBgNVBAMMEmF3cy5uaXRyby1lbmNsYXZlczB2MBAGByqGSM49AgEGBSuBBAAiA2IABPwCV -OumCMHzaHDimtqQvkY4MpJzbolL//Zy2YlES1BR5TSksfbb48C8WBoyt7F2Bw7eEtaaP+ohG2bnUs99 -0d0JX28TcPQXCEPZ3BABIeTPYwEoCWZEh8l5YoQwTcU/9KNCMEAwDwYDVR0TAQH/BAUwAwEB/zAdBgN -VHQ4EFgQUkCW1DdkFR+eWw5b6cp3PmanfS5YwDgYDVR0PAQH/BAQDAgGGMAoGCCqGSM49BAMDA2kAMG -YCMQCjfy+Rocm9Xue4YnwWmNJVA44fA0P5W2OpYow9OYCVRaEevL8uO1XYru5xtMPWrfMCMQCi85sWB -bJwKKXdS6BptQFuZbT73o/gBh1qUxl/nNr12UO8Yfwr6wPLb+6NIwLz3/ZZAsIwggK+MIICRKADAgEC -AhA990CB9kGNMZfvZChwdlgnMAoGCCqGSM49BAMDMEkxCzAJBgNVBAYTAlVTMQ8wDQYDVQQKDAZBbWF -6b24xDDAKBgNVBAsMA0FXUzEbMBkGA1UEAwwSYXdzLm5pdHJvLWVuY2xhdmVzMB4XDTIyMDcyNTA2ND -gwOFoXDTIyMDgxNDA3NDgwOFowZDELMAkGA1UEBhMCVVMxDzANBgNVBAoMBkFtYXpvbjEMMAoGA1UEC -wwDQVdTMTYwNAYDVQQDDC05ZjJmYTNhYWRlZTBhMzZhLnVzLWVhc3QtMi5hd3Mubml0cm8tZW5jbGF2 -ZXMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASJrZH+NsENK8xQ+r4qIT56spgyQ0rqLuQOUv7CmfHg19Z -giX4k1tUbTIGc1hFVphxMaahoM6N3e1mBRMkX9Y/gxYmPnSrom/cq6BnWW8yYWpocaFuXqq/VjOJ9Ba -TcO4qjgdUwgdIwEgYDVR0TAQH/BAgwBgEB/wIBAjAfBgNVHSMEGDAWgBSQJbUN2QVH55bDlvpync+Zq -d9LljAdBgNVHQ4EFgQUPZaPBc+bF0kz5B3MQi4KoWP+hRIwDgYDVR0PAQH/BAQDAgGGMGwGA1UdHwRl -MGMwYaBfoF2GW2h0dHA6Ly9hd3Mtbml0cm8tZW5jbGF2ZXMtY3JsLnMzLmFtYXpvbmF3cy5jb20vY3J -sL2FiNDk2MGNjLTdkNjMtNDJiZC05ZTlmLTU5MzM4Y2I2N2Y4NC5jcmwwCgYIKoZIzj0EAwMDaAAwZQ -IxANMhikJw9gtb6vBdlgVKT1gOgX8g8HhmL764kGCqNUcQEx87vPMhiVamVtUsCIB/awIwbV4Neqsy1 -H4Cq3JWZG9lR2+D8s+nMCVDpUlThEK2K8p0EJP5lPF8N5e0V8oZuA0JWQMYMIIDFDCCApqgAwIBAgIQ -NF6Fd19DpZgwKsWQtzHX8TAKBggqhkjOPQQDAzBkMQswCQYDVQQGEwJVUzEPMA0GA1UECgwGQW1hem9 -uMQwwCgYDVQQLDANBV1MxNjA0BgNVBAMMLTlmMmZhM2FhZGVlMGEzNmEudXMtZWFzdC0yLmF3cy5uaX -Ryby1lbmNsYXZlczAeFw0yMjA3MjYxMTQ1MDBaFw0yMjA4MDEwNDQ0NTlaMIGJMTwwOgYDVQQDDDM0M -TE4MGUyNmU3ZWNjNWY0LnpvbmFsLnVzLWVhc3QtMi5hd3Mubml0cm8tZW5jbGF2ZXMxDDAKBgNVBAsM -A0FXUzEPMA0GA1UECgwGQW1hem9uMQswCQYDVQQGEwJVUzELMAkGA1UECAwCV0ExEDAOBgNVBAcMB1N -lYXR0bGUwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARay8thWSHgPinGev1LSpKiNMhpY3uGlzdNtGrl4D -vRC3tYm3e4y1WC8zjR96rPEPqPaImtmJ4GuXUC8oP5u1g4Cr4jFqqL4KwvwvZFeOhY5FdIEidNByaFQ -1PmdLWM7OGjgeowgecwEgYDVR0TAQH/BAgwBgEB/wIBATAfBgNVHSMEGDAWgBQ9lo8Fz5sXSTPkHcxC -LgqhY/6FEjAdBgNVHQ4EFgQUQjKkC8oNyrWpWkdVSGw8wOOYa9IwDgYDVR0PAQH/BAQDAgGGMIGABgN -VHR8EeTB3MHWgc6Bxhm9odHRwOi8vY3JsLXVzLWVhc3QtMi1hd3Mtbml0cm8tZW5jbGF2ZXMuczMudX -MtZWFzdC0yLmFtYXpvbmF3cy5jb20vY3JsLzI1NDY2N2JmLWY2ZDMtNDNlZS1iMGNiLTYyZWNmZWNiZ -TZmMS5jcmwwCgYIKoZIzj0EAwMDaAAwZQIxAKuYyI19bA8mHLo88O1epcirSbOfK348e6SbhdyJazZb -cIkko5zyvgKmskjACB2IpwIwVo0cIeP+2C4L+5CW8iVr5DrRVhtESi+qta4DzYNJlUXl2X3HiV23fqz -2/3XY9uyqWQKCMIICfjCCAgSgAwIBAgIUfFo+I5v6VGh7k5qouGsLv7Mv57owCgYIKoZIzj0EAwMwgY -kxPDA6BgNVBAMMMzQxMTgwZTI2ZTdlY2M1ZjQuem9uYWwudXMtZWFzdC0yLmF3cy5uaXRyby1lbmNsY -XZlczEMMAoGA1UECwwDQVdTMQ8wDQYDVQQKDAZBbWF6b24xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJX -QTEQMA4GA1UEBwwHU2VhdHRsZTAeFw0yMjA3MjYxNzIyMThaFw0yMjA3MjcxNzIyMThaMIGOMQswCQY -DVQQGEwJVUzETMBEGA1UECAwKV2FzaGluZ3RvbjEQMA4GA1UEBwwHU2VhdHRsZTEPMA0GA1UECgwGQW -1hem9uMQwwCgYDVQQLDANBV1MxOTA3BgNVBAMMMGktMDgwOTg0OTcxMGJmMWI2MWIudXMtZWFzdC0yL -mF3cy5uaXRyby1lbmNsYXZlczB2MBAGByqGSM49AgEGBSuBBAAiA2IABHauNrI7BTIweN+zwPt+cchE -nzuRwHLILTAHh3OTa47tKPrx5siwKIwhkjOvzAN82o4MzgUmqtfQ0yrntfrox2be5qzKx7U26aatS5G -JR/STHSjtoeKZn5FLMYysMJM00KMmMCQwEgYDVR0TAQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAg -QwCgYIKoZIzj0EAwMDaAAwZQIxAK5vbx5ZauD2RpeK2+v3u37cc9imCrMvF1JY4zbZ3ZZQ8UYa/HjnP -iB3pGd8whiA7wIwNiE2h4KKQEhF4Ory87EpxJCT39uXxVByr5TWQ89Ruj1rB2JSXU1psJ8GxlCkcBVD -anB1YmxpY19rZXn2aXVzZXJfZGF0YVhI4nBS+mXu8vE1TpAF0X8GLthggVJB44h4AnNzMvCtD3Qlagn -FYcA3/G9DXSk1uaaVLTm/O4nVtbjo4MaU8C2rqO94hvbTrml7ZW5vbmNlVAAAAAAAAAAAAAAAAAAAAA -AAAAAAWGDMVPwPgNQE0B4IvYVyzsWa6IguwPxu4RrKW7SzNkcv9b0RySXdAAPD071+Ju6Ic8Pr4EOyd -ac+wcqKQm4ZH3U5+yel2+YU33Tq/WvX1Ra2xmQsgQj3xqcL9XMBbdmNW8M=`, -} diff --git a/keysync_responder.go b/keysync_responder.go deleted file mode 100644 index 186ee53..0000000 --- a/keysync_responder.go +++ /dev/null @@ -1,128 +0,0 @@ -package main - -import ( - cryptoRand "crypto/rand" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "strings" - - "github.com/hf/nitrite" - "golang.org/x/crypto/nacl/box" -) - -var ( - errFailedNonce = errors.New("failed to create nonce") - errNoBase64 = errors.New("failed to Base64-decode attestation document") - errFailedVerify = errors.New("failed to verify attestation document") - errFailedRespBody = errors.New("failed to read response body") - errFailedPCR = errors.New("failed to get PCR values") - errFailedFindNonce = errors.New("could not find provided nonce") - errInvalidBoxKeys = errors.New("invalid box key material") - errPCRNotIdentical = errors.New("remote enclave's PCR values not identical") -) - -// nonceHandler returns a HandlerFunc that creates a new nonce and returns it -// to the client. -func nonceHandler(e *Enclave) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - nonce, err := newNonce() - if err != nil { - http.Error(w, errFailedNonce.Error(), http.StatusInternalServerError) - return - } - - e.nonceCache.Add(nonce.B64()) - fmt.Fprintln(w, nonce.B64()) - } -} - -// respSyncHandler returns a HandlerFunc that shares our secret key material -// with the requesting enclave -- after authentication, of course. -func respSyncHandler(e *Enclave) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - var ourNonce, theirNonce nonce - - maxReadLen := base64.StdEncoding.EncodedLen(maxAttDocLen) - body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, int64(maxReadLen))) - if err != nil { - http.Error(w, errFailedRespBody.Error(), http.StatusInternalServerError) - return - } - theirRawAttDoc, err := base64.StdEncoding.DecodeString(strings.TrimSpace(string(body))) - if err != nil { - http.Error(w, errNoBase64.Error(), http.StatusInternalServerError) - return - } - - // Verify the remote enclave's attestation document before touching it. - opts := nitrite.VerifyOptions{CurrentTime: currentTime()} - res, err := nitrite.Verify(theirRawAttDoc, opts) - if err != nil { - http.Error(w, errFailedVerify.Error(), http.StatusUnauthorized) - return - } - theirAttDoc := res.Document - - // Are the PCR values (i.e. image IDs) identical? - ourPCRs, err := getPCRValues() - if err != nil { - http.Error(w, errFailedPCR.Error(), http.StatusInternalServerError) - return - } - if !arePCRsIdentical(ourPCRs, theirAttDoc.PCRs) { - http.Error(w, errPCRNotIdentical.Error(), http.StatusUnauthorized) - return - } - - // Did we actually issue the nonce that the remote enclave provided? - copy(ourNonce[:], theirAttDoc.Nonce) - if !e.nonceCache.Exists(ourNonce.B64()) { - http.Error(w, errFailedFindNonce.Error(), http.StatusUnauthorized) - return - } - - // If we made it this far, we're convinced that we're talking to an - // identical enclave. Now get the remote enclave's nonce, which is in - // the attestation document's "user data" field. - copy(theirNonce[:], theirAttDoc.UserData) - - if len(theirAttDoc.PublicKey) != boxKeyLen { - http.Error(w, errInvalidBoxKeys.Error(), http.StatusBadRequest) - return - } - theirBoxPubKey := &[boxKeyLen]byte{} - copy(theirBoxPubKey[:], theirAttDoc.PublicKey[:]) - - // Encrypt our key material with the provided key. - jsonKeyMaterial, err := json.Marshal(e.keyMaterial) - if err != nil { - http.Error(w, "failed to marshal key material", http.StatusInternalServerError) - return - } - var encrypted []byte - if encrypted, err = box.SealAnonymous( - nil, - jsonKeyMaterial, - theirBoxPubKey, - cryptoRand.Reader, - ); err != nil { - http.Error(w, "failed to encrypt key material", http.StatusInternalServerError) - return - } - - // Encapsulate the remote enclave's nonce and the encrypted key - // material in an attestation document and send it back. - ourAttDoc, err := attest(theirNonce[:], encrypted, nil) - if err != nil { - http.Error(w, errFailedAttestation, http.StatusInternalServerError) - return - } - - b64AttDoc := base64.StdEncoding.EncodeToString(ourAttDoc) - fmt.Fprint(w, b64AttDoc) - } -} diff --git a/keysync_responder_test.go b/keysync_responder_test.go deleted file mode 100644 index b700a2e..0000000 --- a/keysync_responder_test.go +++ /dev/null @@ -1,251 +0,0 @@ -package main - -import ( - "bytes" - "crypto/rand" - "encoding/base64" - "errors" - "io" - "net/http" - "net/http/httptest" - "strings" - "time" - - "testing" -) - -func queryHandler(handler http.HandlerFunc, path string, reader io.Reader) *http.Response { - req := httptest.NewRequest(http.MethodGet, path, reader) - rec := httptest.NewRecorder() - handler(rec, req) - res := rec.Result() - defer res.Body.Close() - return res -} - -func TestNonceHandler(t *testing.T) { - enclave := createEnclave(&defaultCfg) - res := queryHandler(nonceHandler(enclave), pathNonce, bytes.NewReader([]byte{})) - - // Did the operation succeed? - if res.StatusCode != http.StatusOK { - t.Fatalf("Expected status code %d but got %d.", http.StatusOK, res.StatusCode) - } - - // Did we get what looks like a nonce? - b64Nonce, err := io.ReadAll(res.Body) - failOnErr(t, err) - rawNonce, err := base64.StdEncoding.DecodeString(string(b64Nonce)) - if err != nil { - t.Fatalf("Failed to decode Base64-encoded nonce: %s", err) - } - if len(rawNonce) != nonceLen { - t.Fatalf("Expected nonce length %d but got %d.", nonceLen, len(rawNonce)) - } - - // Was the nonce added to the enclave's nonce cache? - if !enclave.nonceCache.Exists(strings.TrimSpace(string(b64Nonce))) { - t.Fatal("Nonce was not added to enclave's nonce cache.") - } -} - -func TestNonceHandlerIfErr(t *testing.T) { - cryptoRead = func(b []byte) (n int, err error) { - return 0, errors.New("not enough randomness") - } - defer func() { - cryptoRead = rand.Read - }() - - res := queryHandler( - nonceHandler(createEnclave(&defaultCfg)), - pathNonce, - bytes.NewReader([]byte{}), - ) - - // Did the operation fail? - if res.StatusCode != http.StatusInternalServerError { - t.Fatalf("Expected status code %d but got %d.", http.StatusInternalServerError, res.StatusCode) - } - - // Did we get the correct error string? - errMsg, err := io.ReadAll(res.Body) - failOnErr(t, err) - if strings.TrimSpace(string(errMsg)) != errFailedNonce.Error() { - t.Fatalf("Expected error message %q but got %q.", errFailedNonce.Error(), errMsg) - } -} - -func TestRespSyncHandlerForBadReqs(t *testing.T) { - var res *http.Response - enclave := createEnclave(&defaultCfg) - - // Send non-Base64 bogus data. - res = queryHandler(respSyncHandler(enclave), pathSync, strings.NewReader("foobar!")) - assertResponse(t, res, newResp(http.StatusInternalServerError, errNoBase64.Error())) - - // Send Base64-encoded bogus data. - res = queryHandler(respSyncHandler(enclave), pathSync, strings.NewReader("Zm9vYmFyCg==")) - assertResponse(t, res, newResp(http.StatusUnauthorized, errFailedVerify.Error())) -} - -func TestRespSyncHandler(t *testing.T) { - var res *http.Response - enclave := createEnclave(&defaultCfg) - enclave.nonceCache.Add(initAttInfo.nonce.B64()) - - // Mock functions for our tests to pass. - getPCRValues = func() (map[uint][]byte, error) { - return initAttInfo.pcr, nil - } - currentTime = func() time.Time { return initAttInfo.attDocTime } - - res = queryHandler(respSyncHandler(enclave), pathSync, strings.NewReader(initAttInfo.attDoc)) - // On a non-enclave platform, the responder code will get as far as to - // request its attestation document. - assertResponse(t, res, newResp(http.StatusInternalServerError, errFailedAttestation)) -} - -func TestRespSyncHandlerDoS(t *testing.T) { - var res *http.Response - enclave := createEnclave(&defaultCfg) - - // Send more data than the handler should be willing to read. - maxSize := base64.StdEncoding.EncodedLen(maxAttDocLen) - body := make([]byte, maxSize+1) - res = queryHandler(respSyncHandler(enclave), pathSync, bytes.NewReader(body)) - assertResponse(t, res, newResp(http.StatusInternalServerError, errFailedRespBody.Error())) -} - -var initAttInfo = &remoteAttInfo{ - pubKey: [boxKeyLen]byte{ - 213, 156, 108, 34, 179, 183, 69, 26, 209, 218, 58, 186, 9, 32, 237, - 253, 46, 80, 36, 200, 169, 239, 97, 200, 17, 188, 203, 99, 151, 40, - 10, 113, - }, - privKey: [boxKeyLen]byte{ - 74, 137, 121, 11, 209, 38, 48, 48, 167, 157, 184, 58, 2, 110, 9, 204, - 174, 148, 243, 154, 191, 74, 118, 90, 11, 240, 246, 131, 187, 157, - 157, 25, - }, - nonce: nonce{}, - pcr: map[uint][]byte{ - 0: { - 0xda, 0x54, 0x6f, 0x8d, 0xda, 0x37, 0x52, 0x19, 0x45, 0xdf, 0x4a, - 0x6d, 0x3e, 0x39, 0x70, 0x63, 0x58, 0x8c, 0xd5, 0xf8, 0x70, 0xaa, - 0xa0, 0x7a, 0x62, 0xe9, 0x67, 0xb2, 0x54, 0xd5, 0xf8, 0x17, 0x6d, - 0xaa, 0x96, 0xec, 0x83, 0xcd, 0xc5, 0x40, 0x2b, 0x0b, 0x52, 0x7a, - 0x16, 0x24, 0x72, 0xb5}, - 1: { - 0xbc, 0xdf, 0x05, 0xfe, 0xfc, 0xca, 0xa8, 0xe5, 0x5b, 0xf2, 0xc8, - 0xd6, 0xde, 0xe9, 0xe7, 0x9b, 0xbf, 0xf3, 0x1e, 0x34, 0xbf, 0x28, - 0xa9, 0x9a, 0xa1, 0x9e, 0x6b, 0x29, 0xc3, 0x7e, 0xe8, 0x0b, 0x21, - 0x4a, 0x41, 0x4b, 0x76, 0x07, 0x23, 0x6e, 0xdf, 0x26, 0xfc, 0xb7, - 0x86, 0x54, 0xe6, 0x3f}, - 2: { - 0x45, 0xaa, 0xd9, 0xf5, 0xc3, 0x9a, 0x90, 0x5b, 0x9f, 0xef, 0xac, - 0x05, 0x56, 0x87, 0x0a, 0x20, 0xd1, 0x6f, 0x3f, 0x3c, 0x21, 0xcf, - 0x93, 0x3e, 0x60, 0x64, 0xff, 0xf9, 0x24, 0xaf, 0x9c, 0x13, 0xed, - 0x26, 0xab, 0x6d, 0x56, 0x3e, 0x27, 0x2b, 0x85, 0xe7, 0xc3, 0x17, - 0x0f, 0x01, 0xac, 0xda}, - 3: null, - 4: { - 0xd8, 0xa8, 0xe8, 0xee, 0xe9, 0x6d, 0x81, 0xb7, 0x7a, 0x25, 0x14, - 0x10, 0xb7, 0xa9, 0xb1, 0x80, 0x78, 0x76, 0x53, 0xf1, 0x25, 0xd1, - 0xdb, 0xca, 0x79, 0x68, 0x5c, 0x93, 0xfb, 0x88, 0x5b, 0x33, 0x5e, - 0x0b, 0x8d, 0x17, 0x2c, 0x98, 0x21, 0xa8, 0x62, 0x51, 0x5a, 0x60, - 0x3c, 0xc3, 0x3a, 0xb2}, - 5: null, - 6: null, - 7: null, - 8: null, - 9: null, - 10: null, - 11: null, - 12: null, - 13: null, - 14: null, - 15: null, - }, - attDocTime: mustParse("2022-08-04T20:00:00Z"), - // The following attestation document was generated on 2022-08-04 and - // contains a nonce (set to all 0 bytes), user data (contains a nonce and - // is set to all 0 bytes), and a public key (contains an NaCl public key). - attDoc: ` -hEShATgioFkRB6lpbW9kdWxlX2lkeCdpLTA4MDk4NDk3MTBiZjFiNjFiLWVuYzAxODI2YTQ0OWEwMGI -xN2FmZGlnZXN0ZlNIQTM4NGl0aW1lc3RhbXAbAAABgmpE949kcGNyc7AAWDDaVG+N2jdSGUXfSm0+OX -BjWIzV+HCqoHpi6WeyVNX4F22qluyDzcVAKwtSehYkcrUBWDC83wX+/Mqo5VvyyNbe6eebv/MeNL8oq -Zqhnmspw37oCyFKQUt2ByNu3yb8t4ZU5j8CWDBFqtn1w5qQW5/vrAVWhwog0W8/PCHPkz5gZP/5JK+c -E+0mq21WPicrhefDFw8BrNoDWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAEWDDYqOju6W2Bt3olFBC3qbGAeHZT8SXR28p5aFyT+4hbM14LjRcsmCGoYlFaYDzDOr -IFWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAGWDAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHWDAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAJWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAKWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAALWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAMWDAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAANWDAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAOWDAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPWDAAAAAAAAAAAAAAAAAAAAAAAAAAA -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABrY2VydGlmaWNhdGVZAoAwggJ8MIICAaADAgECAhAB -gmpEmgCxegAAAABi7BnHMAoGCCqGSM49BAMDMIGOMQswCQYDVQQGEwJVUzETMBEGA1UECAwKV2FzaGl -uZ3RvbjEQMA4GA1UEBwwHU2VhdHRsZTEPMA0GA1UECgwGQW1hem9uMQwwCgYDVQQLDANBV1MxOTA3Bg -NVBAMMMGktMDgwOTg0OTcxMGJmMWI2MWIudXMtZWFzdC0yLmF3cy5uaXRyby1lbmNsYXZlczAeFw0yM -jA4MDQxOTExMDBaFw0yMjA4MDQyMjExMDNaMIGTMQswCQYDVQQGEwJVUzETMBEGA1UECAwKV2FzaGlu -Z3RvbjEQMA4GA1UEBwwHU2VhdHRsZTEPMA0GA1UECgwGQW1hem9uMQwwCgYDVQQLDANBV1MxPjA8BgN -VBAMMNWktMDgwOTg0OTcxMGJmMWI2MWItZW5jMDE4MjZhNDQ5YTAwYjE3YS51cy1lYXN0LTIuYXdzMH -YwEAYHKoZIzj0CAQYFK4EEACIDYgAEpy6eKLNsGy1mhV9SjR5Yj1Wn3wGmX87HinGw/jjpz/Ij3JsGO -HoF0Ve7wtVGgHxT0MjRh/1a45Zd39zpWMyc06tiN6ZM9S9GKws23tPr826TGE9PNB4jQhsNv8gHEJT3 -ox0wGzAMBgNVHRMBAf8EAjAAMAsGA1UdDwQEAwIGwDAKBggqhkjOPQQDAwNpADBmAjEA8SBbh3YYlv/ -XZPttIR9m43jTNkgUHkWyB9hxhWkVEjnfb3MDqAPFhMh5BFoArDD0AjEAj3XawBSe5AK9842TdW/mt+ -C0e/OSZpaFAJqvTAX9MNX3wSEm/Jron+wtoVb+DecTaGNhYnVuZGxlhFkCFTCCAhEwggGWoAMCAQICE -QD5MXVoG5Cv4R1GzLTk5/hWMAoGCCqGSM49BAMDMEkxCzAJBgNVBAYTAlVTMQ8wDQYDVQQKDAZBbWF6 -b24xDDAKBgNVBAsMA0FXUzEbMBkGA1UEAwwSYXdzLm5pdHJvLWVuY2xhdmVzMB4XDTE5MTAyODEzMjg -wNVoXDTQ5MTAyODE0MjgwNVowSTELMAkGA1UEBhMCVVMxDzANBgNVBAoMBkFtYXpvbjEMMAoGA1UECw -wDQVdTMRswGQYDVQQDDBJhd3Mubml0cm8tZW5jbGF2ZXMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAT8A -lTrpgjB82hw4prakL5GODKSc26JS//2ctmJREtQUeU0pLH22+PAvFgaMrexdgcO3hLWmj/qIRtm51LP -fdHdCV9vE3D0FwhD2dwQASHkz2MBKAlmRIfJeWKEME3FP/SjQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQY -DVR0OBBYEFJAltQ3ZBUfnlsOW+nKdz5mp30uWMA4GA1UdDwEB/wQEAwIBhjAKBggqhkjOPQQDAwNpAD -BmAjEAo38vkaHJvV7nuGJ8FpjSVQOOHwND+VtjqWKMPTmAlUWhHry/LjtV2K7ucbTD1q3zAjEAovObF -gWycCil3UugabUBbmW0+96P4AYdalMZf5za9dlDvGH8K+sDy2/ujSMC89/2WQLBMIICvTCCAkSgAwIB -AgIQQpblfNs/3yOBCWXcu04/WDAKBggqhkjOPQQDAzBJMQswCQYDVQQGEwJVUzEPMA0GA1UECgwGQW1 -hem9uMQwwCgYDVQQLDANBV1MxGzAZBgNVBAMMEmF3cy5uaXRyby1lbmNsYXZlczAeFw0yMjA4MDQwNT -Q4MDhaFw0yMjA4MjQwNjQ4MDhaMGQxCzAJBgNVBAYTAlVTMQ8wDQYDVQQKDAZBbWF6b24xDDAKBgNVB -AsMA0FXUzE2MDQGA1UEAwwtOWUyOTllNTRmZTE2M2Q1YS51cy1lYXN0LTIuYXdzLm5pdHJvLWVuY2xh -dmVzMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEzjYIWIyVfPvNjCsLf8VS2P1R1lmaMox7vIOWVU5sfCp -/kyhzz1RLlKTPZLXRfpVZWT8F58ygN3AAzjqOfS8HzWRwfmH0kTsP9T/U2kLYIEYlEPv7Qw98U/1+Wx -VpP94zo4HVMIHSMBIGA1UdEwEB/wQIMAYBAf8CAQIwHwYDVR0jBBgwFoAUkCW1DdkFR+eWw5b6cp3Pm -anfS5YwHQYDVR0OBBYEFPjkq4ZrPqs7R5KtXk3YYASqnwhwMA4GA1UdDwEB/wQEAwIBhjBsBgNVHR8E -ZTBjMGGgX6BdhltodHRwOi8vYXdzLW5pdHJvLWVuY2xhdmVzLWNybC5zMy5hbWF6b25hd3MuY29tL2N -ybC9hYjQ5NjBjYy03ZDYzLTQyYmQtOWU5Zi01OTMzOGNiNjdmODQuY3JsMAoGCCqGSM49BAMDA2cAMG -QCMHXCswA211klCMLW+p3sD8sces9/WEEuIxeaQ1lwKfbCW9yWN7ynujRz01+W378qBgIwXoEP9UIQ2 -p7oC7+HJYp/GNiFrYg4mwEETh75CWX38CFEIyZtbx9abI8pb3bQ+zaZWQMZMIIDFTCCApugAwIBAgIR -ANSXsgCDg5YY9m4w3HzTIQ8wCgYIKoZIzj0EAwMwZDELMAkGA1UEBhMCVVMxDzANBgNVBAoMBkFtYXp -vbjEMMAoGA1UECwwDQVdTMTYwNAYDVQQDDC05ZTI5OWU1NGZlMTYzZDVhLnVzLWVhc3QtMi5hd3Mubm -l0cm8tZW5jbGF2ZXMwHhcNMjIwODA0MTU1MDI2WhcNMjIwODEwMTY1MDI1WjCBiTE8MDoGA1UEAwwzN -2MwM2Q3ZjMxM2IzZDdiOC56b25hbC51cy1lYXN0LTIuYXdzLm5pdHJvLWVuY2xhdmVzMQwwCgYDVQQL -DANBV1MxDzANBgNVBAoMBkFtYXpvbjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAldBMRAwDgYDVQQHDAd -TZWF0dGxlMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEjh5qz+Cx8rHDYKvh1E7GqR+GDG/g5CzjPBiu+p -krFdYe8AN58lfwHv2+YN6i+lOmjpjFADefv6yBS7Va7Ddj6DB3cJWcOhOlKqIyDZpZ4yDeG5H2TvxGi -IR+1vFPFDKZo4HqMIHnMBIGA1UdEwEB/wQIMAYBAf8CAQEwHwYDVR0jBBgwFoAU+OSrhms+qztHkq1e -TdhgBKqfCHAwHQYDVR0OBBYEFGLAF3Hq2WorIwNXLsBHMR4s0uD1MA4GA1UdDwEB/wQEAwIBhjCBgAY -DVR0fBHkwdzB1oHOgcYZvaHR0cDovL2NybC11cy1lYXN0LTItYXdzLW5pdHJvLWVuY2xhdmVzLnMzLn -VzLWVhc3QtMi5hbWF6b25hd3MuY29tL2NybC83MjJlMzIxYy1mYTdmLTRjZjQtYjljMS00YzQ0YzFiN -2M3OWQuY3JsMAoGCCqGSM49BAMDA2gAMGUCMCC/N/6QnA+LQgtLMZhqSXcq8stbOQZ7PTZ6uOK6XcO2 -FC6huMamexK3bkjXQ9tUzgIxANwt5DWIAvBA1hfn1wBl7gQqz1bSlenLqz0ZFyxFW4sT0/rur4ui7OG -JCF5IG4P8zVkCgTCCAn0wggIEoAMCAQICFHPGskW4/ej8wLV8S9yhPGlYOibuMAoGCCqGSM49BAMDMI -GJMTwwOgYDVQQDDDM3YzAzZDdmMzEzYjNkN2I4LnpvbmFsLnVzLWVhc3QtMi5hd3Mubml0cm8tZW5jb -GF2ZXMxDDAKBgNVBAsMA0FXUzEPMA0GA1UECgwGQW1hem9uMQswCQYDVQQGEwJVUzELMAkGA1UECAwC -V0ExEDAOBgNVBAcMB1NlYXR0bGUwHhcNMjIwODA0MTcyMjMyWhcNMjIwODA1MTcyMjMyWjCBjjELMAk -GA1UEBhMCVVMxEzARBgNVBAgMCldhc2hpbmd0b24xEDAOBgNVBAcMB1NlYXR0bGUxDzANBgNVBAoMBk -FtYXpvbjEMMAoGA1UECwwDQVdTMTkwNwYDVQQDDDBpLTA4MDk4NDk3MTBiZjFiNjFiLnVzLWVhc3QtM -i5hd3Mubml0cm8tZW5jbGF2ZXMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAR2rjayOwUyMHjfs8D7fnHI -RJ87kcByyC0wB4dzk2uO7Sj68ebIsCiMIZIzr8wDfNqODM4FJqrX0NMq57X66Mdm3uasyse1NummrUu -RiUf0kx0o7aHimZ+RSzGMrDCTNNCjJjAkMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAg -IEMAoGCCqGSM49BAMDA2cAMGQCMCyFFdZ9EjHxnY9dnOnTevkwJFOYEmLsSQAzl2D6X64LpuuKunhnr -VGEE8wz7lxwZgIwaRQAmr0Ke/l2wNI5UcWoov7VVYoVgbA/VudK7x85KMoqRH+N/IRXcgQWAlw3pZv8 -anB1YmxpY19rZXlYINWcbCKzt0Ua0do6ugkg7f0uUCTIqe9hyBG8y2OXKApxaXVzZXJfZGF0YVQAAAA -AAAAAAAAAAAAAAAAAAAAAAGVub25jZVQAAAAAAAAAAAAAAAAAAAAAAAAAAFhgUx278a0ygjrmIGtSIe -WlqX/lNr5cx9VZAFb5rFJ6igkZsFxwmk764LQEcCE7sifYTKvf/4jpKGdTw+wwmu+Ekdqhi0rmm7dgG -PxIqEgb+JWZGN+Ke5HwVMMGoboAYCXw`, -} diff --git a/keysync_shared.go b/keysync_shared.go deleted file mode 100644 index d24d040..0000000 --- a/keysync_shared.go +++ /dev/null @@ -1,54 +0,0 @@ -package main - -import ( - cryptoRand "crypto/rand" - "encoding/base64" - "time" - - "golang.org/x/crypto/nacl/box" -) - -const ( - boxKeyLen = 32 // NaCl box's private and public key length. -) - -var ( - // Instead of using rand.Read or time.Now directly, we use the following - // variables to enable mocking as part of our unit tets. - cryptoRead = cryptoRand.Read - currentTime = func() time.Time { return time.Now().UTC() } -) - -// nonce represents a nonce that's used to prove the freshness of an enclave's -// attestation document. -type nonce [nonceLen]byte - -// boxKey represents key material for NaCl's box, i.e., a private and a public -// key. -type boxKey struct { - pubKey *[boxKeyLen]byte - privKey *[boxKeyLen]byte -} - -// newBoxKey creates and returns a key pair for use with box. -func newBoxKey() (*boxKey, error) { - pubKey, privKey, err := box.GenerateKey(cryptoRand.Reader) - if err != nil { - return nil, err - } - return &boxKey{pubKey: pubKey, privKey: privKey}, nil -} - -// newNonce creates and returns a cryptographically secure, random nonce. -func newNonce() (nonce, error) { - var n nonce - if _, err := cryptoRead(n[:]); err != nil { - return nonce{}, err - } - return n, nil -} - -// B64 returns a Base64-encoded representation of the nonce. -func (n *nonce) B64() string { - return base64.StdEncoding.EncodeToString(n[:]) -} diff --git a/keysync_shared_test.go b/keysync_shared_test.go deleted file mode 100644 index 37d6249..0000000 --- a/keysync_shared_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package main - -import ( - "crypto/rand" - "errors" - "testing" - "time" -) - -var ( - null = make([]byte, 48) // An empty PCR value. -) - -// remoteAttInfo contains everything that we need to verify a remote enclave's -// attestation information. -type remoteAttInfo struct { - pubKey [boxKeyLen]byte - privKey [boxKeyLen]byte - nonce nonce - pcr map[uint][]byte - attDocTime time.Time - attDoc string -} - -func mustParse(timeStr string) time.Time { - t, err := time.Parse(time.RFC3339, timeStr) - if err != nil { - panic(err) - } - return t -} - -func failOnErr(t *testing.T, err error) { - t.Helper() - if err != nil { - t.Fatal(err) - } -} - -func TestBoxKeyRandomness(t *testing.T) { - k1, err := newBoxKey() - failOnErr(t, err) - k2, err := newBoxKey() - failOnErr(t, err) - - // It's notoriously difficult to test if something is truly random. Here, - // we simply make sure that two subsequently generated key pairs are not - // identical. That's a low bar to pass but better than nothing. - if k1.privKey == k2.privKey { - t.Error("Private keys of two separate box keys are identical.") - } - if k1.pubKey == k2.pubKey { - t.Error("Public keys of two separate box keys are identical.") - } -} - -func TestNonce(t *testing.T) { - n1, err := newNonce() - failOnErr(t, err) - n2, err := newNonce() - failOnErr(t, err) - - if n1 == n2 { - t.Error("Two separately generated nonces are identical.") - } - if n1.B64() == n2.B64() { - t.Error("Two separately generated Base64-encoded nonces are identical.") - } -} - -func TestErrors(t *testing.T) { - // Make cryptoRead always return an error, and check if functions propagate - // that error. - ourError := errors.New("not enough randomness") - cryptoRead = func(b []byte) (n int, err error) { - return 0, ourError - } - defer func() { - cryptoRead = rand.Read - }() - - if _, err := newNonce(); err == nil { - t.Error("Failed to return error") - if !errors.Is(err, ourError) { - t.Error("Propagated error does not contain expected error string.") - } - } -} diff --git a/main.go b/main.go index 5168d90..23f314b 100644 --- a/main.go +++ b/main.go @@ -33,13 +33,15 @@ func init() { } func main() { - var fqdn, appURL, appWebSrv, appCmd, prometheusNamespace, mockCertFp string - var extPort, intPort, hostProxyPort, prometheusPort uint + var fqdn, fqdnLeader, appURL, appWebSrv, appCmd, prometheusNamespace, mockCertFp string + var extPubPort, extPrivPort, intPort, hostProxyPort, prometheusPort uint var useACME, waitForApp, useProfiling, useVsockForExtPort, disableKeepAlives, debug bool var err error flag.StringVar(&fqdn, "fqdn", "", "FQDN of the enclave application (e.g., \"example.com\").") + flag.StringVar(&fqdnLeader, "fqdn-leader", "", + "FQDN of the leader enclave (e.g., \"leader.example.com\"). Setting this enables key synchronization.") flag.StringVar(&appURL, "appurl", "", "Code repository of the enclave application (e.g., \"github.com/foo/bar\").") flag.StringVar(&appWebSrv, "appwebsrv", "", @@ -48,8 +50,10 @@ func main() { "Launch enclave application via the given command.") flag.StringVar(&prometheusNamespace, "prometheus-namespace", "", "Prometheus namespace for exported metrics.") - flag.UintVar(&extPort, "extport", 443, - "Nitriding's HTTPS port. Must match port forwarding rules on EC2 host.") + flag.UintVar(&extPubPort, "ext-pub-port", 443, + "Nitriding's external, public HTTPS port. Must match port forwarding rules on EC2 host.") + flag.UintVar(&extPrivPort, "ext-priv-port", 444, + "Nitriding's external, non-public HTTPS port. Must match port forwarding rules on the EC2 host.") flag.BoolVar(&disableKeepAlives, "disable-keep-alives", false, "Disables keep-alive connections for the HTTPS service.") flag.BoolVar(&useVsockForExtPort, "vsock-ext", false, @@ -67,7 +71,7 @@ func main() { flag.BoolVar(&waitForApp, "wait-for-app", false, "Start Internet-facing Web server only after application signals its readiness.") flag.BoolVar(&debug, "debug", false, - "Print debug messages.") + "Print extra debug messages and use dummy attester for testing outside enclaves.") flag.StringVar(&mockCertFp, "mock-cert-fp", "", "Mock certificate fingerprint to use in attestation documents (hexadecimal)") flag.Parse() @@ -75,9 +79,12 @@ func main() { if fqdn == "" { elog.Fatalf("-fqdn must be set.") } - if extPort < 1 || extPort > math.MaxUint16 { + if extPubPort < 1 || extPubPort > math.MaxUint16 { elog.Fatalf("-extport must be in interval [1, %d]", math.MaxUint16) } + if extPrivPort < 1 || extPrivPort > math.MaxUint16 { + elog.Fatalf("-extPrivPort must be in interval [1, %d]", math.MaxUint16) + } if intPort < 1 || intPort > math.MaxUint16 { elog.Fatalf("-intport must be in interval [1, %d]", math.MaxUint16) } @@ -93,10 +100,12 @@ func main() { c := &Config{ FQDN: fqdn, - ExtPort: uint16(extPort), + FQDNLeader: fqdnLeader, + ExtPubPort: uint16(extPubPort), + ExtPrivPort: uint16(extPrivPort), + IntPort: uint16(intPort), UseVsockForExtPort: useVsockForExtPort, DisableKeepAlives: disableKeepAlives, - IntPort: uint16(intPort), PrometheusPort: uint16(prometheusPort), PrometheusNamespace: prometheusNamespace, HostProxyPort: uint32(hostProxyPort), @@ -120,6 +129,9 @@ func main() { } c.AppWebSrv = u } + if debug { + elog.Println("WARNING: Using debug mode, which must not be enabled in production!") + } enclave, err := NewEnclave(c) if err != nil { diff --git a/metrics.go b/metrics.go index 3889362..68a2643 100644 --- a/metrics.go +++ b/metrics.go @@ -18,10 +18,22 @@ const ( notAvailable = "n/a" ) +var ( + goodHb = prometheus.Labels{ + respErr: notAvailable, + } + badHb = func(err error) prometheus.Labels { + return prometheus.Labels{ + respErr: err.Error(), + } + } +) + // metrics contains our Prometheus metrics. type metrics struct { reqs *prometheus.CounterVec proxiedReqs *prometheus.CounterVec + heartbeats *prometheus.CounterVec } // newMetrics initializes our Prometheus metrics. @@ -44,9 +56,18 @@ func newMetrics(reg prometheus.Registerer, namespace string) *metrics { }, []string{reqPath, reqMethod, respStatus, respErr}, ), + heartbeats: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: "heartbeats", + Help: "Heartbeats sent to the leader enclave", + }, + []string{respErr}, + ), } reg.MustRegister(m.proxiedReqs) reg.MustRegister(m.reqs) + reg.MustRegister(m.heartbeats) reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{ Namespace: namespace, diff --git a/metrics_test.go b/metrics_test.go index 13e4351..3cf6320 100644 --- a/metrics_test.go +++ b/metrics_test.go @@ -18,7 +18,7 @@ func TestHandlerMetrics(t *testing.T) { // installed. c.PrometheusPort = 80 enclave := createEnclave(&c) - makeReq := makeRequestFor(enclave.pubSrv) + makeReq := makeReqToSrv(enclave.extPubSrv) // GET /enclave/config assertResponse(t, @@ -50,7 +50,7 @@ func TestHandlerMetrics(t *testing.T) { ), float64(1)) // POST /enclave/hash - makeReq = makeRequestFor(enclave.privSrv) + makeReq = makeReqToSrv(enclave.intSrv) assertResponse(t, makeReq(http.MethodPost, pathHash, bytes.NewBufferString("foo")), newResp(http.StatusBadRequest, errNoBase64.Error()), diff --git a/nonce.go b/nonce.go new file mode 100644 index 0000000..7237fc6 --- /dev/null +++ b/nonce.go @@ -0,0 +1,32 @@ +package main + +import ( + "encoding/base64" + "errors" +) + +const nonceLen = 20 // The size of a nonce in bytes. + +var errNotEnoughRead = errors.New("failed to read enough random bytes") + +// nonce represents a nonce that's used to prove the freshness of an enclave's +// attestation document. +type nonce [nonceLen]byte + +// newNonce returns a cryptographically secure, random nonce. +func newNonce() (nonce, error) { + var newNonce nonce + n, err := cryptoRead(newNonce[:]) + if err != nil { + return nonce{}, err + } + if n != nonceLen { + return nonce{}, errNotEnoughRead + } + return newNonce, nil +} + +// b64 returns a Base64-encoded representation of the nonce. +func (n *nonce) b64() string { + return base64.StdEncoding.EncodeToString(n[:]) +} diff --git a/nonce_test.go b/nonce_test.go new file mode 100644 index 0000000..1181104 --- /dev/null +++ b/nonce_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "crypto/rand" + "errors" + "testing" +) + +func failOnErr(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatalf("Expected no error but got %v.", err) + } +} + +func TestNonce(t *testing.T) { + nonce1, err := newNonce() + failOnErr(t, err) + nonce2, err := newNonce() + failOnErr(t, err) + + if nonce1 == nonce2 { + t.Fatal("Two separate nonces should not be identical.") + } + if nonce1.b64() == nonce2.b64() { + t.Fatal("Two separate, Base64-encoded nonces should not be identical.") + } +} + +func TestNonceErrors(t *testing.T) { + defer func() { + cryptoRead = rand.Read + }() + + // Make cryptoRead return an error. + ourError := errors.New("not enough randomness") + cryptoRead = func(b []byte) (n int, err error) { + return 0, ourError + } + if _, err := newNonce(); !errors.Is(err, ourError) { + t.Fatal("Propagated error does not contain expected error string.") + } + + // Make cryptoRead return an insufficient number of random bytes. + cryptoRead = func(b []byte) (n int, err error) { + return nonceLen - 1, nil + } + if _, err := newNonce(); !errors.Is(err, errNotEnoughRead) { + t.Fatalf("Expected error %v but got %v.", errNotEnoughRead, err) + } +} diff --git a/proxy.go b/proxy.go index de7937e..4149e4f 100644 --- a/proxy.go +++ b/proxy.go @@ -23,13 +23,12 @@ var ( // runNetworking calls the function that sets up our networking environment. // If anything fails, we try again after a brief wait period. -func runNetworking(c *Config, stop chan bool) { +func runNetworking(c *Config, stop chan struct{}) { var err error for { if err = setupNetworking(c, stop); err == nil { return } - elog.Printf("TAP tunnel to EC2 host failed: %v. Restarting.", err) time.Sleep(time.Second) } } @@ -42,10 +41,7 @@ func runNetworking(c *Config, stop chan bool) { // 3. Establish a connection with the proxy running on the host. // 4. Spawn goroutines to forward traffic between the TAP device and the proxy // running on the host. -func setupNetworking(c *Config, stop chan bool) error { - elog.Println("Setting up networking between host and enclave.") - defer elog.Println("Tearing down networking between host and enclave.") - +func setupNetworking(c *Config, stop chan struct{}) error { // Establish connection with the proxy running on the EC2 host. endpoint := fmt.Sprintf("vsock://%d:%d/connect", parentCID, c.HostProxyPort) conn, path, err := transport.Dial(endpoint) diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..2ce914b --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,24 @@ +# Key synchronization test scripts + +This directory contains scripts that help with testing key synchronization +outside the context of Nitro Enclaves. The scripts assume that we have at least +two "enclaves" that run on separate IP addresses. + +First, change the `leader` variable inside config.sh to match the IP address of +your local leader "enclave". + +To start the leader "enclave", run on machine A: + + ./launch-enclave.sh + +To designate this "enclave" as the leader, run on machine A: + + ./make-leader.sh + +To start a worker "enclave", run on machine B: + + ./launch-enclave.sh + +To update the leader's key material, run on machine A: + + ./update-app-keys.sh \ No newline at end of file diff --git a/scripts/config.sh b/scripts/config.sh new file mode 100644 index 0000000..72c116e --- /dev/null +++ b/scripts/config.sh @@ -0,0 +1,4 @@ +leader="192.168.1.3" # Change this to match your local leader "enclave" address. +ext_pub_port=8443 +ext_priv_port=8444 +int_port=8445 \ No newline at end of file diff --git a/scripts/get-app-keys.sh b/scripts/get-app-keys.sh new file mode 100755 index 0000000..01d93d3 --- /dev/null +++ b/scripts/get-app-keys.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +source "$(dirname $0)/config.sh" + +curl --request GET \ + --include \ + "http://localhost:${int_port}/enclave/state" diff --git a/scripts/launch-enclave.sh b/scripts/launch-enclave.sh new file mode 100755 index 0000000..0559cd2 --- /dev/null +++ b/scripts/launch-enclave.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +source "$(dirname $0)/config.sh" + +nitriding-daemon \ + -debug \ + -fqdn localhost \ + -fqdn-leader "$leader" \ + -ext-pub-port "$ext_pub_port" \ + -ext-priv-port "$ext_priv_port" \ + -intport "$int_port" \ No newline at end of file diff --git a/scripts/make-leader.sh b/scripts/make-leader.sh new file mode 100755 index 0000000..ef03d7c --- /dev/null +++ b/scripts/make-leader.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +source "$(dirname $0)/config.sh" + +curl --insecure \ + --include \ + "https://localhost:${ext_priv_port}/enclave/leader" \ No newline at end of file diff --git a/scripts/set-app-keys.sh b/scripts/set-app-keys.sh new file mode 100755 index 0000000..ca367a9 --- /dev/null +++ b/scripts/set-app-keys.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +source "$(dirname $0)/config.sh" + +curl --request PUT \ + --include \ + "http://localhost:${int_port}/enclave/state" --data 'NewAppKeys' \ No newline at end of file diff --git a/sync_leader.go b/sync_leader.go new file mode 100644 index 0000000..32812c9 --- /dev/null +++ b/sync_leader.go @@ -0,0 +1,142 @@ +package main + +import ( + "bytes" + cryptoRand "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + + "golang.org/x/crypto/nacl/box" +) + +var ( + errExpectedEmptyKeys = errors.New("expected encrypted keys to be unset") +) + +// leaderSync holds the state and code that we need for a one-off sync with a +// worker enclave. +type leaderSync struct { + attester + keys *enclaveKeys +} + +// asLeader returns a new leaderSync struct. +func asLeader(keys *enclaveKeys, a attester) *leaderSync { + return &leaderSync{ + attester: a, + keys: keys, + } +} + +// syncWith makes the leader initiate key synchronization with the given worker +// enclave. +func (s *leaderSync) syncWith(worker *url.URL) (err error) { + var ( + reqBody attstnBody + encrypted []byte + ) + defer func() { + if err == nil { + elog.Printf("Successfully synced with worker %s.", worker.Host) + } else { + elog.Printf("Error syncing with worker %s: %v", worker.Host, err) + } + }() + + // Step 1: Create a nonce that the worker must embed in its attestation + // document, to prevent replay attacks. + nonce, err := newNonce() + if err != nil { + return err + } + + // Step 2: Request the worker's attestation document, and provide the + // previously-generated nonce. + reqURL := *worker + reqURL.RawQuery = fmt.Sprintf("nonce=%x", nonce) + resp, err := newUnauthenticatedHTTPClient().Get(reqURL.String()) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return errNo200(resp.StatusCode) + } + + // Step 3: Verify the worker's attestation document and extract its + // auxiliary information. + maxReadLen := base64.StdEncoding.EncodedLen(maxAttstnBodyLen) + jsonBody, err := io.ReadAll(newLimitReader(resp.Body, maxReadLen)) + if err != nil { + return err + } + defer resp.Body.Close() + + if err := json.Unmarshal(jsonBody, &reqBody); err != nil { + return err + } + if len(reqBody.EncryptedKeys) != 0 { + return errExpectedEmptyKeys + } + attstnDoc, err := base64.StdEncoding.DecodeString(reqBody.Document) + if err != nil { + return err + } + aux, err := s.verifyAttstn(attstnDoc, nonce) + if err != nil { + return err + } + workerAux := aux.(*workerAuxInfo) + + // Step 4: Encrypt the leader's enclave keys with the ephemeral public key + // that the worker put into its auxiliary information. + pubKey := &[boxKeyLen]byte{} + copy(pubKey[:], workerAux.PublicKey[:]) + jsonKeys, err := json.Marshal(s.keys.copy()) + if err != nil { + return err + } + encrypted, err = box.SealAnonymous(nil, jsonKeys, pubKey, cryptoRand.Reader) + if err != nil { + return err + } + + // Step 5: Create the leader's auxiliary information, consisting of the + // worker's nonce and a hash of the encrypted enclave keys. + hash := sha256.Sum256(encrypted) + leaderAux := &leaderAuxInfo{ + WorkersNonce: workerAux.WorkersNonce, + HashOfEncrypted: hash[:], + } + attstnDoc, err = s.createAttstn(leaderAux) + if err != nil { + return err + } + + // Step 6: Send the leader's attestation document to the worker. + jsonBody, err = json.Marshal(&attstnBody{ + Document: base64.StdEncoding.EncodeToString(attstnDoc), + EncryptedKeys: base64.StdEncoding.EncodeToString(encrypted), + }) + if err != nil { + return err + } + resp, err = newUnauthenticatedHTTPClient().Post( + worker.String(), + "text/plain", + bytes.NewReader(jsonBody), + ) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return errNo200(resp.StatusCode) + } + + return nil +} diff --git a/sync_shared.go b/sync_shared.go new file mode 100644 index 0000000..719903b --- /dev/null +++ b/sync_shared.go @@ -0,0 +1,47 @@ +package main + +import ( + cryptoRand "crypto/rand" + "time" + + "golang.org/x/crypto/nacl/box" +) + +const ( + maxAttstnBodyLen = 1 << 14 // Upper limit for attestation body length. + boxKeyLen = 32 // NaCl box's private and public key length. +) + +var ( + // Instead of using rand.Read or time.Now directly, we use the following + // variables to enable mocking as part of our unit tets. + cryptoRead = cryptoRand.Read + currentTime = func() time.Time { return time.Now().UTC() } +) + +// attstnBody contains a JSON-formatted, Base64-encoded attestation document and +// encrypted key material. The leader and worker use this struct to exchange +// attestation documents. +type attstnBody struct { + Document string `json:"document"` + EncryptedKeys string `json:"encrypted_keys"` +} + +// boxKey represents key material for NaCl's box, i.e., a private and a public +// key. +type boxKey struct { + pubKey *[boxKeyLen]byte + privKey *[boxKeyLen]byte +} + +// newBoxKey returns a key pair for use with box. +func newBoxKey() (*boxKey, error) { + pubKey, privKey, err := box.GenerateKey(cryptoRand.Reader) + if err != nil { + return nil, err + } + return &boxKey{ + pubKey: pubKey, + privKey: privKey, + }, nil +} diff --git a/sync_shared_test.go b/sync_shared_test.go new file mode 100644 index 0000000..881e7fb --- /dev/null +++ b/sync_shared_test.go @@ -0,0 +1,22 @@ +package main + +import ( + "testing" +) + +func TestBoxKeyRandomness(t *testing.T) { + k1, err := newBoxKey() + failOnErr(t, err) + k2, err := newBoxKey() + failOnErr(t, err) + + // It's notoriously difficult to test if something is truly random. Here, + // we simply make sure that two subsequently generated key pairs are not + // identical. That's a low bar to pass but better than nothing. + if k1.privKey == k2.privKey { + t.Error("Private keys of two separate box keys are identical.") + } + if k1.pubKey == k2.pubKey { + t.Error("Public keys of two separate box keys are identical.") + } +} diff --git a/sync_worker.go b/sync_worker.go new file mode 100644 index 0000000..5a895f5 --- /dev/null +++ b/sync_worker.go @@ -0,0 +1,231 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "golang.org/x/crypto/nacl/box" +) + +var ( + errInProgress = errors.New("key sync already in progress") + errFailedToDecrypt = errors.New("error decrypting enclave keys") + errHashNotInAttstn = errors.New("hash of encrypted keys not in attestation document") +) + +// workerSync holds the state and code that we need for a one-off sync with a +// leader enclave. workerSync implements the http.Handler interface because the +// sync protocol requires two endpoints on the worker. +type workerSync struct { + attester + setupWorker func(*enclaveKeys) error + ephemeralKeys chan *boxKey + nonce chan nonce +} + +// asWorker returns a new workerSync object. +func asWorker( + setupWorker func(*enclaveKeys) error, + a attester, +) *workerSync { + return &workerSync{ + attester: a, + setupWorker: setupWorker, + nonce: make(chan nonce, 1), + ephemeralKeys: make(chan *boxKey, 1), + } +} + +type heartbeatRequest struct { + HashedKeys string `json:"hashed_keys"` + WorkerHostname string `json:"worker_hostname"` +} + +// registerWith registers the given worker with the given leader enclave. +func (s *workerSync) registerWith(leader, worker *url.URL) error { + elog.Println("Attempting to sync with leader.") + + errChan := make(chan error) + register := func(e chan error) { + body, err := json.Marshal(heartbeatRequest{WorkerHostname: worker.Host}) + if err != nil { + e <- err + return + } + resp, err := newUnauthenticatedHTTPClient().Post(leader.String(), "text/plain", bytes.NewBuffer(body)) + if err != nil { + e <- err + return + } + if resp.StatusCode != http.StatusOK { + e <- fmt.Errorf("leader returned HTTP code %d", resp.StatusCode) + return + } + e <- nil + } + go register(errChan) + + // Keep on trying every five seconds, for a minute. + retry := time.NewTicker(5 * time.Second) + timeout := time.NewTicker(time.Minute) + for { + select { + case err := <-errChan: + if err == nil { + elog.Println("Successfully registered with leader.") + return nil + } + elog.Printf("Error registering with leader: %v", err) + case <-timeout.C: + return errors.New("timed out syncing with leader") + case <-retry.C: + go register(errChan) + } + } +} + +func (s *workerSync) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + s.initSync(w, r) + } else if r.Method == http.MethodPost { + s.finishSync(w, r) + } +} + +// initSync responds to the leader's request for initiating key synchronization. +func (s *workerSync) initSync(w http.ResponseWriter, r *http.Request) { + elog.Println("Received leader's request to initiate key sync.") + + // There must not be more than one key synchronization attempt at any given + // time. Abort if we get another request while key synchronization is still + // in progress. + if len(s.ephemeralKeys) > 0 { + http.Error(w, errInProgress.Error(), http.StatusTooManyRequests) + return + } + + // Extract the leader's nonce from the URL, which must look like this: + // https://example.com/enclave/sync?nonce=[HEX-ENCODED-NONCE] + leadersNonce, err := getNonceFromReq(r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } + + // Create the worker's nonce and store it in our channel, so we can later + // verify it. + workersNonce, err := newNonce() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + s.nonce <- workersNonce + + // Create an ephemeral key that the leader is going to use to encrypt + // its enclave keys. + boxKey, err := newBoxKey() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + s.ephemeralKeys <- boxKey + + // Create and return the worker's Base64-encoded attestation document. + attstnDoc, err := s.createAttstn(&workerAuxInfo{ + WorkersNonce: workersNonce, + LeadersNonce: leadersNonce, + PublicKey: boxKey.pubKey[:], + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + respBody, err := json.Marshal(&attstnBody{ + Document: base64.StdEncoding.EncodeToString(attstnDoc), + }) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + fmt.Fprintln(w, string(respBody)) +} + +// finishSync responds to the leader's final request before key synchronization +// is complete. +func (s *workerSync) finishSync(w http.ResponseWriter, r *http.Request) { + var ( + reqBody attstnBody + keys enclaveKeys + ) + elog.Println("Received leader's request to complete key sync.") + + // Read the leader's Base64-encoded attestation document. + maxReadLen := base64.StdEncoding.EncodedLen(maxAttstnBodyLen) + jsonBody, err := io.ReadAll(newLimitReader(r.Body, maxReadLen)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if err := json.Unmarshal(jsonBody, &reqBody); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + } + attstnDoc, err := base64.StdEncoding.DecodeString(reqBody.Document) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Verify attestation document and obtain its auxiliary information. + aux, err := s.verifyAttstn(attstnDoc, <-s.nonce) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + leaderAux := aux.(*leaderAuxInfo) + encrypted, err := base64.StdEncoding.DecodeString(reqBody.EncryptedKeys) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Make sure that the hash of the encrypted key material is present in the + // attestation document. + hash := sha256.Sum256(encrypted) + if !bytes.Equal(hash[:], leaderAux.HashOfEncrypted) { + http.Error(w, errHashNotInAttstn.Error(), http.StatusBadRequest) + return + } + + ephemeralKey := <-s.ephemeralKeys + // Decrypt the leader's enclave keys, which are encrypted with the + // public key that we provided earlier. + decrypted, ok := box.OpenAnonymous( + nil, + encrypted, + ephemeralKey.pubKey, + ephemeralKey.privKey) + if !ok { + http.Error(w, errFailedToDecrypt.Error(), http.StatusBadRequest) + return + } + + // Install the leader's enclave keys. + if err := json.Unmarshal(decrypted, &keys); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if err := s.setupWorker(&keys); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + elog.Fatalf("Failed to install enclave keys: %v", err) + } + + elog.Printf("Successfully synced keys %s with leader.", keys.hashAndB64()) +} diff --git a/sync_worker_test.go b/sync_worker_test.go new file mode 100644 index 0000000..8e2d575 --- /dev/null +++ b/sync_worker_test.go @@ -0,0 +1,77 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" +) + +// leaderKeys holds arbitrary keys that we use for testing. +var leaderKeys = &enclaveKeys{ + NitridingKey: []byte("NitridingTestKey"), + NitridingCert: []byte("NitridingTestCert"), + AppKeys: []byte("AppTestKeys"), +} + +func initLeaderKeysCert(t *testing.T) { + t.Helper() + cert, key, err := createCertificate("example.com") + if err != nil { + t.Fatal(err) + } + leaderKeys.setNitridingKeys(key, cert) +} + +func TestSuccessfulRegisterWith(t *testing.T) { + e := createEnclave(&defaultCfg) + hasRegistered := false + + srv := httptest.NewTLSServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + hasRegistered = true + w.WriteHeader(http.StatusOK) + }), + ) + leader, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("Error creating test server URL: %v", err) + } + worker := &url.URL{ + Host: "localhost", + } + + err = asWorker(e.setupWorkerPostSync, &dummyAttester{}).registerWith(leader, worker) + if err != nil { + t.Fatalf("Error registering with leader: %v", err) + } + if !hasRegistered { + t.Fatal("Worker did not register with leader.") + } +} + +func TestSuccessfulSync(t *testing.T) { + // For key synchronization to be successful, we need actual certificates in + // the leader keys. + initLeaderKeysCert(t) + + // Set up the worker. + worker := createEnclave(&defaultCfg) + srv := httptest.NewTLSServer( + asWorker(worker.setupWorkerPostSync, &dummyAttester{}), + ) + workerURL, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("Error creating test server URL: %v", err) + } + + if err = asLeader(leaderKeys, &dummyAttester{}).syncWith(workerURL); err != nil { + t.Fatalf("Error syncing with leader: %v", err) + } + + // Make sure that the keys were synced correctly. + if !worker.keys.equal(leaderKeys) { + t.Fatalf("Keys differ between worker and leader:\n%v (worker)\n%v (leader)", + leaderKeys, worker.keys) + } +} diff --git a/util.go b/util.go new file mode 100644 index 0000000..8478e3c --- /dev/null +++ b/util.go @@ -0,0 +1,259 @@ +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/pem" + "errors" + "fmt" + "io" + "math/big" + "net" + "net/http" + "net/url" + "strings" + "time" +) + +const ( + // The endpoint of AWS's Instance Metadata Service, which allows an enclave + // to learn its internal hostname: + // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html + metadataSvcToken = "http://169.254.169.254/latest/api/token" + metadataSvcInfo = "http://169.254.169.254/latest/meta-data/local-hostname" +) + +var ( + errBadSliceLen = errors.New("slice is not of same length as nonce") + newUnauthenticatedHTTPClient = func() *http.Client { + return _newUnauthenticatedHTTPClient() + } + getSyncURL = func(host string, port uint16) *url.URL { + return _getSyncURL(host, port) + } +) + +// _getSyncURL turns the given host and port into a URL that a leader enclave +// can sync with. +var _getSyncURL = func(host string, port uint16) *url.URL { + return &url.URL{ + Scheme: "https", + Host: fmt.Sprintf("%s:%d", host, port), + Path: pathSync, + } +} + +// _newUnauthenticatedHTTPClient returns an HTTP client that skips HTTPS +// certificate validation. In the context of nitriding, this is fine because +// all we need is a *confidential* channel; not an authenticated channel. +// Authentication is handled on the next layer, using attestation documents. +func _newUnauthenticatedHTTPClient() *http.Client { + transport := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + return &http.Client{ + Transport: transport, + Timeout: 3 * time.Second, + } +} + +// createCertificate creates a self-signed certificate and returns the +// PEM-encoded certificate and key. Some of the code below was taken from: +// https://eli.thegreenplace.net/2021/go-https-servers-with-tls/ +func createCertificate(fqdn string) (cert []byte, key []byte, err error) { + privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, err + } + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return nil, nil, err + } + + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{certificateOrg}, + }, + DNSNames: []string{fqdn}, + NotBefore: time.Now(), + NotAfter: time.Now().Add(certificateValidity), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + derBytes, err := x509.CreateCertificate( + rand.Reader, + &template, + &template, + &privateKey.PublicKey, + privateKey, + ) + if err != nil { + return nil, nil, err + } + + pemCert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + if pemCert == nil { + return nil, nil, errors.New("error encoding cert as PEM") + } + + privBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + return nil, nil, err + } + pemKey := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}) + if pemKey == nil { + return nil, nil, errors.New("error encoding key as PEM") + } + + return pemCert, pemKey, nil +} + +// sliceToNonce copies the given slice into a nonce and returns the nonce. +func sliceToNonce(s []byte) (nonce, error) { + var n nonce + + if len(s) != nonceLen { + return nonce{}, errBadSliceLen + } + + copy(n[:], s[:nonceLen]) + return n, nil +} + +// getHostnameOrDie returns the "enclave"'s hostname (or IP address) or dies +// trying. If inside an enclave, we query AWS's Instance Metadata Service. If +// outside an enclave, we pick whatever IP address the operating system would +// choose when talking to a public IP address. +func getHostnameOrDie() (hostname string) { + defer func() { + elog.Printf("Determined our hostname: %s", hostname) + }() + var err error + + if !inEnclave { + hostname = getLocalAddr() + return + } + + // We cannot easily tell when all components are in place to receive + // incoming connections. We therefore make five attempts to get our + // hostname from IMDS while waiting for one second in between attempts. + const retries = 5 + for i := 0; i < retries; i++ { + hostname, err = getLocalEC2Hostname() + if err == nil { + return + } + time.Sleep(time.Second) + } + if err != nil { + elog.Fatalf("Error obtaining hostname from IMDSv2: %v", err) + } + return +} + +func getLocalAddr() string { + const target = "1.1.1.1:53" + conn, err := net.Dial("udp", target) + if err != nil { + elog.Fatalf("Error dialing %s: %v", target, err) + } + defer conn.Close() + + host, _, err := net.SplitHostPort(conn.LocalAddr().String()) + if err != nil { + elog.Fatalf("Error extracing host: %v", err) + } + return host +} + +func getLocalEC2Hostname() (string, error) { + const ( + maxTokenLen = 100 + maxHostnameLen = 255 + ) + // IMDSv2, which we are using, is session-oriented (God knows why), so we + // first obtain a session token from the service. + req, err := http.NewRequest(http.MethodPut, metadataSvcToken, nil) + if err != nil { + return "", err + } + req.Header.Set("X-aws-ec2-metadata-token-ttl-seconds", "10") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + body, err := io.ReadAll(newLimitReader(resp.Body, maxTokenLen)) + if err != nil { + return "", err + } + token := string(body) + + // Having obtained the session token, we can now make the actual metadata + // request. + req, err = http.NewRequest(http.MethodGet, metadataSvcInfo, nil) + if err != nil { + return "", err + } + req.Header.Set("X-aws-ec2-metadata-token", token) + resp, err = http.DefaultClient.Do(req) + if err != nil { + return "", err + } + body, err = io.ReadAll(newLimitReader(resp.Body, maxHostnameLen)) + if err != nil { + return "", err + } + return string(body), nil +} + +func getNonceFromReq(r *http.Request) (nonce, error) { + if err := r.ParseForm(); err != nil { + return nonce{}, errBadForm + } + + strNonce := r.URL.Query().Get("nonce") + if strNonce == "" { + return nonce{}, errNoNonce + } + strNonce = strings.ToLower(strNonce) + // Decode hex-encoded nonce. + rawNonce, err := hex.DecodeString(strNonce) + if err != nil { + return nonce{}, errBadNonceFormat + } + + n, err := sliceToNonce(rawNonce) + if err != nil { + return nonce{}, err + } + return n, nil +} + +func makeLeaderRequest(leader *url.URL, ourNonce nonce, areWeLeader chan bool, errChan chan error) { + elog.Println("Attempting to talk to leader designation endpoint.") + + reqURL := *leader + reqURL.RawQuery = fmt.Sprintf("nonce=%x", ourNonce[:]) + resp, err := newUnauthenticatedHTTPClient().Get(reqURL.String()) + if err != nil { + errChan <- err + return + } + if resp.StatusCode == http.StatusGone { + // The leader already knows that it's the leader, and it's not us. + areWeLeader <- false + return + } + errChan <- fmt.Errorf("leader designation endpoint returned %d", resp.StatusCode) +} diff --git a/util_test.go b/util_test.go new file mode 100644 index 0000000..0c834e3 --- /dev/null +++ b/util_test.go @@ -0,0 +1,13 @@ +package main + +import "testing" + +func TestSliceToNonce(t *testing.T) { + var err error + + _, err = sliceToNonce([]byte("foo")) + assertEqual(t, err, errBadSliceLen) + + _, err = sliceToNonce(make([]byte, nonceLen)) + assertEqual(t, err, nil) +} diff --git a/workers.go b/workers.go new file mode 100644 index 0000000..39935e7 --- /dev/null +++ b/workers.go @@ -0,0 +1,100 @@ +package main + +import ( + "net/url" + "time" +) + +// workerManager manages worker enclaves. +type workerManager struct { + timeout time.Duration + reg, unreg chan *url.URL + len chan int + forAllFunc chan func(*url.URL) +} + +// workers maps worker enclaves (identified by a URL) to a timestamp that keeps +// track of when we last got a heartbeat from the worker. +type workers map[url.URL]time.Time + +func newWorkerManager(timeout time.Duration) *workerManager { + return &workerManager{ + timeout: timeout, + reg: make(chan *url.URL), + unreg: make(chan *url.URL), + len: make(chan int), + forAllFunc: make(chan func(*url.URL)), + } +} + +// start starts the worker manager's event loop. +func (w *workerManager) start(stop chan struct{}) { + var ( + set = make(workers) + timer = time.NewTicker(w.timeout) + ) + elog.Println("Starting worker event loop.") + defer elog.Println("Stopping worker event loop.") + + for { + select { + case <-stop: + return + + case <-timer.C: + now := time.Now() + for worker, lastSeen := range set { + if now.Sub(lastSeen) > w.timeout { + delete(set, worker) + elog.Printf("Pruned %s from worker set.", worker.Host) + } + } + + case worker := <-w.reg: + set[*worker] = time.Now() + elog.Printf("(Re-)registered worker %s; %d worker(s) now registered.", + worker.Host, len(set)) + + case worker := <-w.unreg: + delete(set, *worker) + elog.Printf("Unregistered worker %s; %d worker(s) left.", + worker.Host, len(set)) + + case f := <-w.forAllFunc: + w.runForAll(f, set) + + case <-w.len: + w.len <- len(set) + } + } +} + +// runForAll runs the given function over all workers in our set. For key +// synchronization, this should never take more than a couple seconds. +func (w *workerManager) runForAll(f func(*url.URL), set workers) { + for worker := range set { + go f(&worker) + } +} + +// length returns the number of workers that are currently registered. +func (w *workerManager) length() int { + w.len <- 0 // Signal to the event loop that we want the length. + return <-w.len +} + +// forAll runs the given function over all registered workers. +func (w *workerManager) forAll(f func(*url.URL)) { + w.forAllFunc <- f +} + +// register registers a new worker enclave. It is safe to repeatedly register +// the same worker enclave. +func (w *workerManager) register(worker *url.URL) { + w.reg <- worker +} + +// unregister unregisters the given worker enclave. +func (w *workerManager) unregister(worker *url.URL) { + w.unreg <- worker +} diff --git a/workers_test.go b/workers_test.go new file mode 100644 index 0000000..bf489dc --- /dev/null +++ b/workers_test.go @@ -0,0 +1,77 @@ +package main + +import ( + "net/url" + "sync" + "testing" + "time" +) + +func TestWorkerRegistration(t *testing.T) { + var ( + w = newWorkerManager(time.Minute) + stop = make(chan struct{}) + ) + go w.start(stop) + defer close(stop) + + // Identical URLs are only tracked once. + worker1 := url.URL{Host: "foo"} + w.register(&worker1) + w.register(&worker1) + assertEqual(t, w.length(), 1) + + worker2 := url.URL{Host: "bar"} + w.register(&worker2) + assertEqual(t, w.length(), 2) + + w.unregister(&worker1) + w.unregister(&worker2) + // It should be safe to unregister a non-existing worker. + w.unregister(&worker2) + assertEqual(t, w.length(), 0) + + // Nothing should happen when attempting to unregister a non-existing + // worker. + w.unregister(&url.URL{Host: "does-not-exist"}) +} + +func TestForAll(t *testing.T) { + var ( + w = newWorkerManager(time.Millisecond) + stop = make(chan struct{}) + wg = sync.WaitGroup{} + mutex = sync.Mutex{} + total = 0 + ) + go w.start(stop) + defer close(stop) + + w.register(&url.URL{Host: "foo"}) + w.register(&url.URL{Host: "bar"}) + assertEqual(t, w.length(), 2) + + wg.Add(2) + w.forAll( + func(w *url.URL) { + mutex.Lock() + defer mutex.Unlock() + defer wg.Done() + total += 1 + }, + ) + wg.Wait() + assertEqual(t, total, 2) +} + +func TestIneffectiveForAll(t *testing.T) { + var ( + w = newWorkerManager(time.Minute) + stop = make(chan struct{}) + ) + go w.start(stop) + defer close(stop) + + // Make sure that forAll finishes for an empty worker set. + w.forAll(func(_ *url.URL) {}) +}