Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion docs/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,8 @@ Key mechanisms (all under `pkg/sandbox/`):
(with SNI/Host-inspecting TCP firewall for domain allow/deny lists). Slots are pooled and
reused; slot allocation is coordinated through Consul KV.
- **Sandbox proxy** (:5007, `pkg/proxy/`): reverse-proxies incoming traffic from client-proxy to
the sandbox's slot IP and requested port, enforcing per-sandbox traffic access tokens.
the sandbox's slot IP and requested port over HTTP or configured HTTPS, enforcing per-sandbox
traffic access tokens. HTTPS backends may use self-signed certificates.
- Writes sandbox lifecycle **events** and cgroup **host stats** to ClickHouse; exports metrics via OTel.

### Envd (`packages/envd`)
Expand Down
384 changes: 194 additions & 190 deletions packages/api/internal/api/api.gen.go

Large diffs are not rendered by default.

40 changes: 40 additions & 0 deletions packages/api/internal/handlers/sandbox_create.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/e2b-dev/infra/packages/db/pkg/types"
"github.com/e2b-dev/infra/packages/db/queries"
"github.com/e2b-dev/infra/packages/shared/pkg/clusters"
"github.com/e2b-dev/infra/packages/shared/pkg/consts"
"github.com/e2b-dev/infra/packages/shared/pkg/featureflags"
"github.com/e2b-dev/infra/packages/shared/pkg/ginutils"
"github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator"
Expand All @@ -48,6 +49,7 @@ const (
// Network validation error messages
ErrMsgDomainsRequireBlockAll = "When specifying allowed domains in allow out, you must include 'ALL_TRAFFIC' in deny out to block all other traffic."

maxHTTPSPorts = 128
maxNetworkRuleDomains = 10
maxNetworkRuleTransformsPerDomain = 1
maxNetworkRuleDomainLen = 128
Expand Down Expand Up @@ -212,6 +214,7 @@ func (a *APIStore) PostSandboxes(c *gin.Context) {
Ingress: &types.SandboxNetworkIngressConfig{
AllowPublicAccess: n.AllowPublicTraffic,
MaskRequestHost: n.MaskRequestHost,
HTTPSPorts: sharedUtils.DerefOrDefault(n.HttpsPorts, nil),
},
Egress: &types.SandboxNetworkEgressConfig{
AllowedAddresses: sharedUtils.DerefOrDefault(n.AllowOut, nil),
Expand Down Expand Up @@ -631,6 +634,43 @@ func validateNetworkConfig(ctx context.Context, featureFlags featureFlagsClient,
return nil
}

if httpsPorts := network.HttpsPorts; httpsPorts != nil {
if len(*httpsPorts) > maxHTTPSPorts {
return &api.APIError{
Code: http.StatusBadRequest,
Err: fmt.Errorf("too many HTTPS ports: %d (max %d)", len(*httpsPorts), maxHTTPSPorts),
ClientMsg: fmt.Sprintf("HTTPS ports can have at most %d entries.", maxHTTPSPorts),
}
}

seen := make(map[uint32]struct{}, len(*httpsPorts))
for _, port := range *httpsPorts {
if port == 0 || port > 65535 {
return &api.APIError{
Code: http.StatusBadRequest,
Err: fmt.Errorf("invalid HTTPS port %d", port),
ClientMsg: fmt.Sprintf("HTTPS port must be between 1 and 65535: %d", port),
}
}
if port == uint32(consts.DefaultEnvdServerPort) {
return &api.APIError{
Code: http.StatusBadRequest,
Err: errors.New("envd port cannot use HTTPS backend routing"),
ClientMsg: fmt.Sprintf("HTTPS backend routing is not supported for reserved port %d", port),
}
}
if _, ok := seen[port]; ok {
return &api.APIError{
Code: http.StatusBadRequest,
Err: fmt.Errorf("duplicate HTTPS port %d", port),
ClientMsg: fmt.Sprintf("HTTPS port %d is specified more than once.", port),
}
}

seen[port] = struct{}{}
}
}
Comment thread
jakubno marked this conversation as resolved.

if maskRequestHost := network.MaskRequestHost; maskRequestHost != nil {
hostname, _, err := splitHostPortOptional(*maskRequestHost)
if err != nil {
Expand Down
73 changes: 73 additions & 0 deletions packages/api/internal/handlers/sandbox_create_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/e2b-dev/infra/packages/db/pkg/testutils"
dbtypes "github.com/e2b-dev/infra/packages/db/pkg/types"
"github.com/e2b-dev/infra/packages/db/queries"
"github.com/e2b-dev/infra/packages/shared/pkg/consts"
"github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator"
"github.com/e2b-dev/infra/packages/shared/pkg/id"
redis_utils "github.com/e2b-dev/infra/packages/shared/pkg/redis"
Expand Down Expand Up @@ -105,6 +106,65 @@ func TestValidateNetworkConfig(t *testing.T) {
network: &api.SandboxNetworkConfig{},
wantErr: false,
},
{
name: "valid HTTPS ports",
network: &api.SandboxNetworkConfig{
HttpsPorts: &[]uint32{443, 8443},
},
wantErr: false,
},
{
name: "too many HTTPS ports",
network: &api.SandboxNetworkConfig{
HttpsPorts: func() *[]uint32 {
ports := make([]uint32, maxHTTPSPorts+1)
for i := range ports {
ports[i] = uint32(i + 1)
}

return &ports
}(),
},
wantErr: true,
wantCode: http.StatusBadRequest,
wantErrMsg: fmt.Sprintf("HTTPS ports can have at most %d entries.", maxHTTPSPorts),
},
{
name: "duplicate HTTPS port is invalid",
network: &api.SandboxNetworkConfig{
HttpsPorts: &[]uint32{443, 443},
},
wantErr: true,
wantCode: http.StatusBadRequest,
wantErrMsg: "HTTPS port 443 is specified more than once.",
},
{
name: "zero HTTPS port is invalid",
network: &api.SandboxNetworkConfig{
HttpsPorts: &[]uint32{0},
},
wantErr: true,
wantCode: http.StatusBadRequest,
wantErrMsg: "HTTPS port must be between 1 and 65535: 0",
},
{
name: "out-of-range HTTPS port is invalid",
network: &api.SandboxNetworkConfig{
HttpsPorts: &[]uint32{65536},
},
wantErr: true,
wantCode: http.StatusBadRequest,
wantErrMsg: "HTTPS port must be between 1 and 65535: 65536",
},
{
name: "envd HTTPS port is invalid",
network: &api.SandboxNetworkConfig{
HttpsPorts: &[]uint32{uint32(consts.DefaultEnvdServerPort)},
},
wantErr: true,
wantCode: http.StatusBadRequest,
wantErrMsg: fmt.Sprintf("HTTPS backend routing is not supported for reserved port %d", consts.DefaultEnvdServerPort),
},
{
name: "valid deny_out with CIDR",
network: &api.SandboxNetworkConfig{
Expand Down Expand Up @@ -302,6 +362,19 @@ func TestValidateNetworkConfig(t *testing.T) {
}
}

func TestDBNetworkConfigToAPIHTTPSPorts(t *testing.T) {
t.Parallel()

httpsPorts := []uint32{443, 8443}
result := dbNetworkConfigToAPI(&dbtypes.SandboxNetworkConfig{
Ingress: &dbtypes.SandboxNetworkIngressConfig{HTTPSPorts: httpsPorts},
})

require.NotNil(t, result)
require.NotNil(t, result.HttpsPorts)
assert.Equal(t, httpsPorts, *result.HttpsPorts)
}

func TestOrchestrator_convertVolumeMounts(t *testing.T) {
t.Parallel()

Expand Down
3 changes: 3 additions & 0 deletions packages/api/internal/handlers/sandbox_get.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ func dbNetworkConfigToAPI(network *dbtypes.SandboxNetworkConfig) *api.SandboxNet
if ingress := network.Ingress; ingress != nil {
result.AllowPublicTraffic = ingress.AllowPublicAccess
result.MaskRequestHost = ingress.MaskRequestHost
if ingress.HTTPSPorts != nil {
result.HttpsPorts = &ingress.HTTPSPorts
}
Comment thread
jakubno marked this conversation as resolved.
}

if egress := network.Egress; egress != nil {
Expand Down
1 change: 1 addition & 0 deletions packages/api/internal/orchestrator/create_instance.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ func buildNetworkConfig(network *types.SandboxNetworkConfig, allowInternetAccess

if network != nil && network.Ingress != nil {
orchNetwork.Ingress.MaskRequestHost = network.Ingress.MaskRequestHost
orchNetwork.Ingress.HttpsPorts = network.Ingress.HTTPSPorts
}

// Handle the case where internet access is explicitly disabled
Expand Down
12 changes: 12 additions & 0 deletions packages/api/internal/orchestrator/create_instance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
sandboxredis "github.com/e2b-dev/infra/packages/api/internal/sandbox/storage/redis"
teamtypes "github.com/e2b-dev/infra/packages/auth/pkg/types"
authqueries "github.com/e2b-dev/infra/packages/db/pkg/auth/queries"
dbtypes "github.com/e2b-dev/infra/packages/db/pkg/types"
"github.com/e2b-dev/infra/packages/db/queries"
"github.com/e2b-dev/infra/packages/shared/pkg/featureflags"
redis_utils "github.com/e2b-dev/infra/packages/shared/pkg/redis"
Expand Down Expand Up @@ -98,6 +99,17 @@ func testTeam() *teamtypes.Team {
}
}

func TestBuildNetworkConfigHTTPSPorts(t *testing.T) {
t.Parallel()

httpsPorts := []uint32{443, 8443}
network := buildNetworkConfig(&dbtypes.SandboxNetworkConfig{
Ingress: &dbtypes.SandboxNetworkIngressConfig{HTTPSPorts: httpsPorts},
}, nil, nil)

assert.Equal(t, httpsPorts, network.GetIngress().GetHttpsPorts())
}

// TestCreateSandbox_StaleDataAfterConcurrentPause exercises CreateSandbox with
// two sequential resume attempts where the snapshot changes between them.
//
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ func (n *Node) GetSandboxes(ctx context.Context) ([]sandbox.Sandbox, error) {
network.Ingress = &types.SandboxNetworkIngressConfig{
AllowPublicAccess: new(networkTrafficAccessToken == nil),
MaskRequestHost: ingress.MaskRequestHost,
HTTPSPorts: ingress.GetHttpsPorts(),
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,35 @@ func TestGetSandboxes_RestoresAutoPauseFilesystemOnly(t *testing.T) {
assert.True(t, got["fs-only"], "filesystem-only auto-pause policy must survive an orchestrator re-sync")
assert.False(t, got["memory"], "memory auto-pause policy must survive an orchestrator re-sync")
}

func TestGetSandboxesRestoresHTTPSPorts(t *testing.T) {
t.Parallel()

httpsPorts := []uint32{443, 8443}
now := time.Now()
node := NewTestNode("test-node", api.NodeStatusReady, 0, 4)
node.SetSandboxClient(&mockSandboxListClient{
resp: &orchestrator.SandboxListResponse{
Sandboxes: []*orchestrator.RunningSandbox{{
StartTime: timestamppb.New(now),
EndTime: timestamppb.New(now.Add(time.Hour)),
Config: &orchestrator.SandboxConfig{
SandboxId: "sandbox",
TemplateId: "template",
TeamId: uuid.NewString(),
BuildId: uuid.NewString(),
Network: &orchestrator.SandboxNetworkConfig{
Ingress: &orchestrator.SandboxNetworkIngressConfig{HttpsPorts: httpsPorts},
},
},
}},
},
})

sandboxes, err := node.GetSandboxes(t.Context())
require.NoError(t, err)
require.Len(t, sandboxes, 1)
require.NotNil(t, sandboxes[0].Network)
require.NotNil(t, sandboxes[0].Network.Ingress)
assert.Equal(t, httpsPorts, sandboxes[0].Network.Ingress.HTTPSPorts)
}
5 changes: 3 additions & 2 deletions packages/db/pkg/types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,9 @@ type SandboxNetworkEgressConfig struct {
const AllowPublicAccessDefault = true

type SandboxNetworkIngressConfig struct {
AllowPublicAccess *bool `json:"allowPublicAccess,omitempty"`
MaskRequestHost *string `json:"maskRequestHost,omitempty"`
AllowPublicAccess *bool `json:"allowPublicAccess,omitempty"`
MaskRequestHost *string `json:"maskRequestHost,omitempty"`
HTTPSPorts []uint32 `json:"httpsPorts,omitempty"`
}

type SandboxNetworkConfig struct {
Expand Down
22 changes: 22 additions & 0 deletions packages/db/pkg/types/types_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,3 +124,25 @@ func TestPausedSandboxConfig_LegacyRowDefaultsToMemoryAutoPause(t *testing.T) {
assert.False(t, decoded.AutoPauseFilesystemOnly, "legacy rows must default to a memory auto-pause")
assert.True(t, decoded.FilesystemOnly, "unrelated fields must still decode")
}

func TestPausedSandboxConfigHTTPSPortsRoundTrip(t *testing.T) {
t.Parallel()

httpsPorts := []uint32{443, 8443}
v, err := PausedSandboxConfig{
Version: PausedSandboxConfigVersion,
Network: &SandboxNetworkConfig{
Ingress: &SandboxNetworkIngressConfig{HTTPSPorts: httpsPorts},
},
}.Value()
require.NoError(t, err)

raw, ok := v.(string)
require.True(t, ok)

var decoded PausedSandboxConfig
require.NoError(t, json.Unmarshal([]byte(raw), &decoded))
require.NotNil(t, decoded.Network)
require.NotNil(t, decoded.Network.Ingress)
assert.Equal(t, httpsPorts, decoded.Network.Ingress.HTTPSPorts)
}
1 change: 1 addition & 0 deletions packages/orchestrator/orchestrator.proto
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ message SandboxNetworkEgressConfig {
message SandboxNetworkIngressConfig {
optional string traffic_access_token = 1;
optional string mask_request_host = 2;
repeated uint32 https_ports = 3;
}

message SandboxCreateRequest {
Expand Down
22 changes: 18 additions & 4 deletions packages/orchestrator/pkg/proxy/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/e2b-dev/infra/packages/shared/pkg/connlimit"
"github.com/e2b-dev/infra/packages/shared/pkg/consts"
"github.com/e2b-dev/infra/packages/shared/pkg/featureflags"
"github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator"
"github.com/e2b-dev/infra/packages/shared/pkg/logger"
reverseproxy "github.com/e2b-dev/infra/packages/shared/pkg/proxy"
"github.com/e2b-dev/infra/packages/shared/pkg/proxy/pool"
Expand All @@ -36,6 +37,18 @@ const (

var _ sandbox.MapSubscriber = (*SandboxProxy)(nil)

// schemeForPort returns the backend scheme for a sandbox port: "https" when the
// port is configured in the ingress HTTPS ports, "http" otherwise.
func schemeForPort(ingress *orchestrator.SandboxNetworkIngressConfig, port uint64) string {
for _, httpsPort := range ingress.GetHttpsPorts() {
if uint64(httpsPort) == port {
return "https"
}
}

return "http"
}

type SandboxProxy struct {
proxy *reverseproxy.Proxy
limiter *connlimit.ConnectionLimiter
Expand Down Expand Up @@ -98,7 +111,7 @@ func NewSandboxProxy(meterProvider metric.MeterProvider, port uint16, sandboxes
}

url := &url.URL{
Scheme: "http",
Scheme: schemeForPort(ingress, port),
Host: net.JoinHostPort(sbx.Slot.HostIPString(), strconv.FormatUint(port, 10)),
}

Expand All @@ -118,9 +131,10 @@ func NewSandboxProxy(meterProvider metric.MeterProvider, port uint16, sandboxes
IncludeSandboxIdInProxyErrorLogger: true,
// We need to include id unique to sandbox to prevent reuse of connection to the same IP:port pair by different sandboxes reusing the network slot.
// We are not using sandbox id to prevent removing connections based on sandbox id (pause/resume race condition).
ConnectionKey: sbx.LifecycleID,
RequestLogger: logger,
MaskRequestHost: maskRequestHost,
ConnectionKey: sbx.LifecycleID,
RequestLogger: logger,
MaskRequestHost: maskRequestHost,
InsecureSkipTLSVerify: true,
}, nil
},
connLimitConfig,
Expand Down
Loading
Loading