package main import ( "context" "crypto/tls" "crypto/x509" "encoding/json" "encoding/pem" "io" "log" "maps" "net/http" "regexp" "slices" "strconv" "strings" "testing" "time" "github.com/moby/moby/api/types/container" "github.com/shellhub-io/shellhub/pkg/api/authorizer" "github.com/shellhub-io/shellhub/pkg/api/query" "github.com/shellhub-io/shellhub/pkg/api/requests" "github.com/shellhub-io/shellhub/pkg/api/responses" "github.com/shellhub-io/shellhub/pkg/models" "github.com/shellhub-io/shellhub/pkg/testimage" "github.com/shellhub-io/shellhub/tests/environment" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" ) const ( deviceServiceHost = "127.0.0.1" intruderUsername = "intruder" intruderNamespaceName = "intruderspace" intruderNamespace = "00000000-0000-4000-0000-000000000022" deviceTLSDomain = "device.example.com" webEndpointDomain = "localhost" unknownAddress = "0123456789abcdef0123456789abcdef" webEndpointNotFound = "web endpoint not found" gatewayRootCertificate = "/data/pki/authorities/local/root.crt" finiteTTL = 60 ) var ( webEndpointAddress = regexp.MustCompile(`^[0-9a-f]{32}$`) unknownDeviceUID = strings.Repeat("0", 64) ) type deviceService struct { name string httpPort int tlsPort int } func (s deviceService) plain(uid string, ttl int) environment.WebEndpointCreate { return environment.WebEndpointCreate{UID: uid, Host: deviceServiceHost, Port: s.httpPort, TTL: ttl} } func (s deviceService) secure(uid string, verify bool) environment.WebEndpointCreate { return environment.WebEndpointCreate{ UID: uid, Host: deviceServiceHost, Port: s.tlsPort, TTL: -1, TLS: &environment.WebEndpointTLS{Enabled: true, Verify: verify, Domain: deviceTLSDomain}, } } type deviceSeen struct { Name string `json:"name"` Host string `json:"host"` URI string `json:"uri"` TLS bool `json:"tls"` } // TestEnterpriseWebEndpoints covers web endpoints on an enterprise instance that enables them: what // the API refuses to create, the address and lifetime of what it creates, the gateway carrying a // request sent to an endpoint's address through the agent into an HTTP or TLS service on the // device, the deprecated tunnel API over the same endpoints, a device merge handing a device's // endpoints to the device it merged into, the gateway refusing an endpoint whose device is no // longer accepted and its deletion taking the endpoint with it, the list refusing a filter or sort // it does not take, the roles that may not create or delete an endpoint, and a connector carrying a // request for a container's loopback to the container's own address. The device's services listen // in the agents' network namespace, or on another host of the stack's network, and answer with the // name they were started with and what they received. func TestEnterpriseWebEndpoints(t *testing.T) { t.Parallel() ctx := context.Background() compose := newEnterpriseEnvironment(t, ctx, environment.New(t, run).WithEnv("SHELLHUB_WEB_ENDPOINTS", "true")) _, device := startAcceptedAgent(t, ctx, compose) first := startDeviceService(t, compose, deviceService{name: "first", httpPort: 8001, tlsPort: 8443}) second := startDeviceService(t, compose, deviceService{name: "second", httpPort: 8002, tlsPort: 8444}) t.Run("creating", func(t *testing.T) { testWebEndpointCreation(t, compose, device, first.plain(device.UID, -1)) }) t.Run("proxying", func(t *testing.T) { testWebEndpointProxy(t, compose, device, first, second) }) t.Run("lifetime", func(t *testing.T) { testWebEndpointLifetime(t, compose, first.plain(device.UID, -1), second.plain(device.UID, -1)) }) t.Run("deprecated tunnel API", func(t *testing.T) { testTunnelAPI(t, compose, device, second) }) t.Run("another namespace", func(t *testing.T) { testWebEndpointAnotherNamespace(t, ctx, compose, device, first) }) t.Run("device merge", func(t *testing.T) { testWebEndpointDeviceMerge(t, ctx, compose, first) }) t.Run("device leaving", func(t *testing.T) { testWebEndpointDeviceLeaving(t, ctx, compose, first) }) t.Run("listing", func(t *testing.T) { testWebEndpointListing(t, compose) }) t.Run("roles", func(t *testing.T) { testWebEndpointRoles(t, compose, device, first) }) t.Run("connector", func(t *testing.T) { testWebEndpointConnector(t, ctx, compose) }) } // TestEnterpriseWebEndpointsDisabled covers an enterprise instance that leaves web endpoints off, // as it does by default: neither the API nor the gateway serves them. func TestEnterpriseWebEndpointsDisabled(t *testing.T) { t.Parallel() compose := newEnterpriseEnvironment(t, context.Background(), environment.New(t, run)) t.Run("the API serves no web endpoint route", func(t *testing.T) { requests := []struct { method string path string }{ {method: http.MethodGet, path: "/api/web-endpoints"}, {method: http.MethodPost, path: "/api/web-endpoints"}, {method: http.MethodDelete, path: "/api/web-endpoints/" + unknownAddress}, {method: http.MethodGet, path: "/api/devices/" + unknownDeviceUID + "/tunnels"}, {method: http.MethodPost, path: "/api/devices/" + unknownDeviceUID + "/tunnels"}, } for _, req := range requests { resp, err := compose.R(t.Context()).SetBody(map[string]any{}).Execute(req.method, req.path) require.NoError(t, err) assert.Equal(t, http.StatusNotFound, resp.StatusCode(), "%s %s: %s", req.method, req.path, resp.String()) } }) t.Run("the gateway serves no site for an endpoint's address", func(t *testing.T) { resp := visitWebEndpoint(t, compose, unknownAddress+"."+webEndpointDomain, "/") assert.Equal(t, http.StatusNotFound, resp.StatusCode) assert.Empty(t, resp.Body) }) } // TestEnterpriseWebEndpointsTLS covers the gateway serving web endpoints over HTTPS on a domain no // public authority signs for: it presents a certificate its own authority issued for the address, // and carries the request to the server. func TestEnterpriseWebEndpointsTLS(t *testing.T) { t.Parallel() compose := environment.New(t, run). WithEdition(environment.EditionEnterprise). WithEnv("SHELLHUB_WEB_ENDPOINTS", "true"). WithAutoSSL(). Up(context.Background()) t.Cleanup(compose.Down) roots := x509.NewCertPool() roots.AddCert(gatewayAuthority(t, compose)) var resp *environment.WebEndpointResponse require.EventuallyWithT(t, func(tt *assert.CollectT) { var err error resp, err = compose.VisitWebEndpointOverTLS(t.Context(), &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, unknownAddress+"."+webEndpointDomain, "/") assert.NoError(tt, err) }, 30*time.Second, time.Second) assert.Equal(t, http.StatusForbidden, resp.StatusCode) assert.Contains(t, resp.Body, webEndpointNotFound) } // TestEnterpriseWebEndpointsDNS01 covers the gateway proving the web endpoints wildcard over DNS-01 // through the acme-dns provider: it writes the challenge to an acme-dns server that answers for the // web endpoint domain, waits until that server serves it, and then presents the wildcard // certificate the ACME CA issued for an endpoint's address, carrying the request to the server. func TestEnterpriseWebEndpointsDNS01(t *testing.T) { t.Parallel() compose := environment.New(t, run). WithEdition(environment.EditionEnterprise). WithEnv("SHELLHUB_WEB_ENDPOINTS", "true"). WithACMEDNS(). Up(context.Background()) t.Cleanup(compose.Down) host := unknownAddress + "." + environment.ACMEDNSWebEndpointsDomain config := &tls.Config{ ServerName: host, InsecureSkipVerify: true, //nolint:gosec // Pebble's root is generated when it starts; the issuer is what the case reads MinVersion: tls.VersionTLS12, } var served *x509.Certificate require.EventuallyWithT(t, func(tt *assert.CollectT) { var err error served, err = servedCertificate(t.Context(), compose.HTTPSAddress(), config) if assert.NoError(tt, err, "the gateway holds no certificate for %s yet", host) { assert.True(tt, strings.HasPrefix(served.Issuer.CommonName, "Pebble Intermediate CA"), "issued by %q", served.Issuer.CommonName) } }, 3*time.Minute, 2*time.Second) assert.Equal(t, []string{"*." + environment.ACMEDNSWebEndpointsDomain}, served.DNSNames) require.NoError(t, served.VerifyHostname(host)) resp, err := compose.VisitWebEndpointOverTLS(t.Context(), config, host, "/") require.NoError(t, err) assert.Equal(t, http.StatusForbidden, resp.StatusCode) assert.Contains(t, resp.Body, webEndpointNotFound) } func testWebEndpointCreation(t *testing.T, compose *environment.DockerCompose, device *models.Device, valid environment.WebEndpointCreate) { t.Helper() refusals := []struct { description string change func(req *environment.WebEndpointCreate) status int }{ {description: "a host that is not an IP address", change: func(req *environment.WebEndpointCreate) { req.Host = "localhost" }, status: http.StatusBadRequest}, {description: "port 0", change: func(req *environment.WebEndpointCreate) { req.Port = 0 }, status: http.StatusBadRequest}, {description: "a TTL of 0", change: func(req *environment.WebEndpointCreate) { req.TTL = 0 }, status: http.StatusBadRequest}, {description: "a TTL below -1", change: func(req *environment.WebEndpointCreate) { req.TTL = -2 }, status: http.StatusBadRequest}, {description: "a device the namespace does not hold", change: func(req *environment.WebEndpointCreate) { req.UID = unknownDeviceUID }, status: http.StatusNotFound}, {description: "TLS without a domain", change: func(req *environment.WebEndpointCreate) { req.TLS = &environment.WebEndpointTLS{Enabled: true} }, status: http.StatusBadRequest}, } for _, tc := range refusals { t.Run("an endpoint for "+tc.description+" is refused", func(t *testing.T) { req := valid tc.change(&req) requireCreateRefused(t, compose, req, tc.status) }) } t.Run("an endpoint for a device not yet accepted is refused", func(t *testing.T) { pending := enrollPendingDevice(t, compose, "web-endpoint-pending", "02:00:00:00:22:01") req := valid req.UID = pending.UID requireCreateRefused(t, compose, req, http.StatusForbidden) }) t.Run("a token whose namespace was deleted cannot create an endpoint", func(t *testing.T) { const username, password = "orphan", "password" compose.NewUser(t, username, "orphan@shellhub.example.com", password) compose.NewNamespace(t, username, "orphanspace", "00000000-0000-4000-0000-000000002201") token := compose.AuthUser(t, username, password).Token compose.DeleteNamespace(t, "orphanspace") resp, err := compose.Anonymous(t.Context()).SetAuthToken(token).SetBody(valid).Post("/api/web-endpoints") require.NoError(t, err) assert.Equal(t, http.StatusUnauthorized, resp.StatusCode(), resp.String()) }) t.Run("an endpoint answers under a 32 hex digit address of the web endpoint domain", func(t *testing.T) { endpoint := compose.CreateWebEndpoint(t, valid) assert.Regexp(t, webEndpointAddress, endpoint.Address) assert.Equal(t, endpoint.Address+"."+webEndpointDomain, endpoint.FullAddress) assert.Equal(t, ShellHubNamespace, endpoint.Namespace) assert.Equal(t, device.UID, endpoint.DeviceUID) assert.Equal(t, -1, endpoint.TTL) assert.True(t, endpoint.ExpiresIn.IsZero(), "an endpoint with a TTL of -1 has no expiry") }) t.Run("the same device, host and port keep their endpoint and get a new address once it is deleted", func(t *testing.T) { endpoint := compose.CreateWebEndpoint(t, valid) assert.Equal(t, endpoint.Address, compose.CreateWebEndpoint(t, valid).Address) resp, err := compose.DeleteWebEndpoint(t.Context(), endpoint.Address) require.NoError(t, err) require.Equal(t, http.StatusOK, resp.StatusCode(), resp.String()) recreated := compose.CreateWebEndpoint(t, valid) assert.Regexp(t, webEndpointAddress, recreated.Address) assert.NotEqual(t, endpoint.Address, recreated.Address) }) } func testWebEndpointProxy(t *testing.T, compose *environment.DockerCompose, device *models.Device, first, second deviceService) { t.Helper() t.Run("a request to an endpoint's address reaches the device's service with its path and the address as Host", func(t *testing.T) { endpoint := compose.CreateWebEndpoint(t, first.plain(device.UID, -1)) seen := requireDeviceAnswer(t, compose, endpoint.FullAddress, "/some/path?query=value") assert.Equal(t, first.name, seen.Name) assert.Equal(t, "/some/path?query=value", seen.URI) assert.Equal(t, endpoint.FullAddress, seen.Host) assert.False(t, seen.TLS) }) t.Run("requests to two endpoints each reach their own service", func(t *testing.T) { one := compose.CreateWebEndpoint(t, first.plain(device.UID, -1)) two := compose.CreateWebEndpoint(t, second.plain(device.UID, -1)) for range 3 { assert.Equal(t, first.name, requireDeviceAnswer(t, compose, one.FullAddress, "/").Name) assert.Equal(t, second.name, requireDeviceAnswer(t, compose, two.FullAddress, "/").Name) } }) t.Run("a request to an endpoint for another host on the device's network reaches that host", func(t *testing.T) { neighbour := deviceService{name: "neighbour", httpPort: 8003, tlsPort: 8445} host := startDeviceServiceOnStackNetwork(t, compose, neighbour, nil) endpoint := compose.CreateWebEndpoint(t, environment.WebEndpointCreate{UID: device.UID, Host: host, Port: neighbour.httpPort, TTL: -1}) assert.Equal(t, neighbour.name, requireDeviceAnswer(t, compose, endpoint.FullAddress, "/").Name) }) t.Run("a request to a deleted endpoint's address is refused", func(t *testing.T) { endpoint := compose.CreateWebEndpoint(t, first.plain(device.UID, -1)) requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") resp, err := compose.DeleteWebEndpoint(t.Context(), endpoint.Address) require.NoError(t, err) require.Equal(t, http.StatusOK, resp.StatusCode(), resp.String()) requireEndpointRefused(t, compose, endpoint.FullAddress) }) t.Run("a request to an address no endpoint has is refused", func(t *testing.T) { requireEndpointRefused(t, compose, unknownAddress+"."+webEndpointDomain) }) t.Run("an endpoint over TLS that does not verify reaches a self-signed service, naming its domain as Host", func(t *testing.T) { endpoint := compose.CreateWebEndpoint(t, first.secure(device.UID, false)) seen := requireDeviceAnswer(t, compose, endpoint.FullAddress, "/secure?query=value") assert.Equal(t, first.name, seen.Name) assert.True(t, seen.TLS) assert.Equal(t, "/secure?query=value", seen.URI) assert.Equal(t, deviceTLSDomain, seen.Host) }) t.Run("an endpoint over TLS that verifies refuses a self-signed service", func(t *testing.T) { endpoint := compose.CreateWebEndpoint(t, first.secure(device.UID, true)) resp := visitWebEndpoint(t, compose, endpoint.FullAddress, "/") assert.Equal(t, http.StatusInternalServerError, resp.StatusCode) assert.Contains(t, resp.Body, "failed to connect to the port on device") }) t.Run("a subdomain that is not an address is not served", func(t *testing.T) { for _, path := range []string{"/api/info", "/internal/metrics"} { resp := visitWebEndpoint(t, compose, "console."+webEndpointDomain, path) assert.Equal(t, http.StatusNotFound, resp.StatusCode, "%s: %s", path, resp.Body) } }) t.Run("a request to the instance's own domain still reaches the API", func(t *testing.T) { resp := visitWebEndpoint(t, compose, webEndpointDomain, "/api/info") assert.Equal(t, http.StatusOK, resp.StatusCode, resp.Body) }) } func testWebEndpointLifetime(t *testing.T, compose *environment.DockerCompose, endless, elsewhere environment.WebEndpointCreate) { t.Helper() withTTL := func(ttl int) environment.WebEndpointCreate { req := endless req.TTL = ttl return req } t.Run("an endpoint never accessed expires its TTL after its creation", func(t *testing.T) { const ttl = 3600 endpoint := listedWebEndpoint(t, compose, compose.CreateWebEndpoint(t, withTTL(ttl)).Address) assert.WithinDuration(t, endpoint.CreatedAt.Add(ttl*time.Second), endpoint.ExpiresIn, time.Millisecond) }) t.Run("an accessed endpoint expires its TTL after its last access", func(t *testing.T) { const ttl = 3600 endpoint := compose.CreateWebEndpoint(t, withTTL(ttl)) compose.AgeWebEndpoint(t, endpoint.Address, 2*time.Second) created := listedWebEndpoint(t, compose, endpoint.Address) before := wallClock.Now() requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") after := wallClock.Now() accessed := listedWebEndpoint(t, compose, endpoint.Address) assert.True(t, accessed.ExpiresIn.After(created.ExpiresIn.Add(time.Second)), "the expiry %s did not move past %s", accessed.ExpiresIn, created.ExpiresIn) assert.WithinRange(t, accessed.ExpiresIn.Add(-ttl*time.Second), before.Add(-time.Second), after.Add(time.Second)) }) t.Run("an endpoint answers until its TTL runs out and is refused after it", func(t *testing.T) { endpoint := compose.CreateWebEndpoint(t, withTTL(finiteTTL)) requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") compose.MoveWebEndpointLastAccessBack(t, endpoint.Address, (finiteTTL+1)*time.Second) requireEndpointRefused(t, compose, endpoint.FullAddress) }) t.Run("an endpoint accessed within its TTL keeps answering past it", func(t *testing.T) { endpoint := compose.CreateWebEndpoint(t, withTTL(finiteTTL)) requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") compose.AgeWebEndpoint(t, endpoint.Address, (finiteTTL+1)*time.Second) requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") compose.MoveWebEndpointLastAccessBack(t, endpoint.Address, (finiteTTL+1)*time.Second) requireEndpointRefused(t, compose, endpoint.FullAddress) }) t.Run("an endpoint with a TTL of -1 keeps answering after one with a TTL has expired", func(t *testing.T) { finite := compose.CreateWebEndpoint(t, withTTL(finiteTTL)) infinite := compose.CreateWebEndpoint(t, elsewhere) requireRefusedOnceCreatedPastTTL(t, compose, finite) requireDeviceAnswer(t, compose, infinite.FullAddress, "/") assert.True(t, listedWebEndpoint(t, compose, infinite.Address).ExpiresIn.IsZero()) }) t.Run("re-creating an expired endpoint restarts its TTL from the re-creation", func(t *testing.T) { endpoint := compose.CreateWebEndpoint(t, withTTL(finiteTTL)) requireRefusedOnceCreatedPastTTL(t, compose, endpoint) before := wallClock.Now() recreated := compose.CreateWebEndpoint(t, withTTL(finiteTTL)) after := wallClock.Now() require.Equal(t, endpoint.Address, recreated.Address) listed := listedWebEndpoint(t, compose, recreated.Address) assert.WithinRange(t, listed.CreatedAt, before.Add(-time.Second), after.Add(time.Second)) assert.WithinDuration(t, listed.CreatedAt.Add(finiteTTL*time.Second), listed.ExpiresIn, time.Millisecond) requireDeviceAnswer(t, compose, recreated.FullAddress, "/") }) t.Run("re-creating an endpoint idle past its TTL answers again", func(t *testing.T) { endpoint := compose.CreateWebEndpoint(t, withTTL(finiteTTL)) requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") compose.MoveWebEndpointLastAccessBack(t, endpoint.Address, (finiteTTL+1)*time.Second) requireEndpointRefused(t, compose, endpoint.FullAddress) recreated := compose.CreateWebEndpoint(t, withTTL(finiteTTL)) require.Equal(t, endpoint.Address, recreated.Address) requireDeviceAnswer(t, compose, recreated.FullAddress, "/") }) t.Run("re-creating an endpoint takes the TTL it is re-created with", func(t *testing.T) { endpoint := compose.CreateWebEndpoint(t, withTTL(finiteTTL)) recreated := compose.CreateWebEndpoint(t, withTTL(-1)) require.Equal(t, endpoint.Address, recreated.Address) assert.Equal(t, -1, recreated.TTL) assert.True(t, listedWebEndpoint(t, compose, recreated.Address).ExpiresIn.IsZero()) compose.AgeWebEndpoint(t, recreated.Address, (finiteTTL+1)*time.Second) requireDeviceAnswer(t, compose, recreated.FullAddress, "/") }) t.Run("an expired endpoint can be deleted", func(t *testing.T) { endpoint := compose.CreateWebEndpoint(t, withTTL(finiteTTL)) requireRefusedOnceCreatedPastTTL(t, compose, endpoint) resp, err := compose.DeleteWebEndpoint(t.Context(), endpoint.Address) require.NoError(t, err) require.Equal(t, http.StatusOK, resp.StatusCode(), resp.String()) for _, listed := range compose.ListWebEndpoints(t) { assert.NotEqual(t, endpoint.Address, listed.Address) } }) } func testTunnelAPI(t *testing.T, compose *environment.DockerCompose, device *models.Device, service deviceService) { t.Helper() resp, err := compose.PostTunnel(t.Context(), device.UID, environment.TunnelCreate{Host: deviceServiceHost, Port: service.httpPort, TTL: -1}) require.NoError(t, err) require.Equal(t, http.StatusOK, resp.StatusCode(), resp.String()) tunnels := compose.ListTunnels(t, device.UID) require.Len(t, tunnels, 1) tunnel := tunnels[0] t.Cleanup(func() { _, _ = compose.DeleteTunnel(context.WithoutCancel(t.Context()), device.UID, tunnel.Address) }) assert.Regexp(t, webEndpointAddress, tunnel.Address) assert.Equal(t, tunnel.Address+"."+webEndpointDomain, tunnel.FullAddress) assert.Equal(t, device.UID, tunnel.Device) assert.Equal(t, deviceServiceHost, tunnel.Host) assert.Equal(t, service.httpPort, tunnel.Port) assert.Equal(t, -1, tunnel.TTL) assert.Equal(t, service.name, requireDeviceAnswer(t, compose, tunnel.FullAddress, "/").Name) resp, err = compose.DeleteTunnel(t.Context(), device.UID, tunnel.Address) require.NoError(t, err) require.Equal(t, http.StatusOK, resp.StatusCode(), resp.String()) assert.Empty(t, compose.ListTunnels(t, device.UID)) requireEndpointRefused(t, compose, tunnel.FullAddress) } func testWebEndpointAnotherNamespace(t *testing.T, ctx context.Context, compose *environment.DockerCompose, device *models.Device, service deviceService) { t.Helper() endpoint := compose.CreateWebEndpoint(t, service.plain(device.UID, -1)) requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") owner := compose.AuthUser(t, ShellHubUsername, ShellHubPassword) compose.NewUser(t, intruderUsername, "intruder@ossystems.com.br", ShellHubPassword) compose.NewNamespace(t, intruderUsername, intruderNamespaceName, intruderNamespace) compose.JWT(compose.AuthUser(t, intruderUsername, ShellHubPassword).Token) t.Cleanup(func() { compose.JWT(owner.Token) }) intruderKey := compose.CreateProvisioningKey(t, &requests.CreateProvisioningKey{Name: "intruder", Mode: string(models.ProvisioningKeyModeManual)}) _, foreign := startAcceptedAgent(t, ctx, compose, NewAgentContainerWithProvisioningKey(intruderKey.Key), NewAgentContainerWithIdentity("020000002203"), NewAgentContainerWithHostname("intruding"), ) resp, err := compose.DeleteTunnel(t.Context(), foreign.UID, endpoint.Address) require.NoError(t, err) assert.Equal(t, http.StatusNotFound, resp.StatusCode(), "deleting through a device of its own: %s", resp.String()) resp, err = compose.DeleteWebEndpoint(t.Context(), endpoint.Address) require.NoError(t, err) assert.Equal(t, http.StatusNotFound, resp.StatusCode(), "deleting by address: %s", resp.String()) assert.Equal(t, service.name, requireDeviceAnswer(t, compose, endpoint.FullAddress, "/").Name) } func testWebEndpointDeviceMerge(t *testing.T, ctx context.Context, compose *environment.DockerCompose, service deviceService) { t.Helper() same := []NewAgentContainerOption{NewAgentContainerWithIdentity("020000002202"), NewAgentContainerWithHostname("merging")} oldAgent, old := startAcceptedAgent(t, ctx, compose, same...) endpoint := compose.CreateWebEndpoint(t, service.plain(old.UID, -1)) requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") require.NoError(t, oldAgent.Stop(ctx, nil)) compose.AwaitDeviceOffline(t, old.UID) _, merged := startAcceptedAgent(t, ctx, compose, same...) require.NotEqual(t, old.UID, merged.UID) assert.Equal(t, merged.UID, listedWebEndpoint(t, compose, endpoint.Address).DeviceUID) assert.Equal(t, service.name, requireDeviceAnswer(t, compose, endpoint.FullAddress, "/").Name) } func testWebEndpointDeviceLeaving(t *testing.T, ctx context.Context, compose *environment.DockerCompose, service deviceService) { t.Helper() agent, device := startAcceptedAgent(t, ctx, compose, NewAgentContainerWithIdentity("020000002204"), NewAgentContainerWithHostname("leaving")) endpoint := compose.CreateWebEndpoint(t, service.plain(device.UID, -1)) requireDeviceAnswer(t, compose, endpoint.FullAddress, "/") require.NoError(t, agent.Stop(ctx, nil)) compose.AwaitDeviceOffline(t, device.UID) removed := t.Run("an endpoint whose device was removed stays listed and is refused", func(t *testing.T) { compose.DeleteDevice(t, device.UID) require.Equal(t, models.DeviceStatusRemoved, requireDevice(t, compose, device.UID).Status) listedWebEndpoint(t, compose, endpoint.Address) requireEndpointRefused(t, compose, endpoint.FullAddress) }) if !removed { return } pending := t.Run("an endpoint whose removed device is back to pending is refused", func(t *testing.T) { require.NoError(t, agent.Start(ctx)) awaitDeviceStatus(t, compose, device.UID, models.DeviceStatusPending) requireEndpointRefused(t, compose, endpoint.FullAddress) }) if !pending { return } rejected := t.Run("an endpoint whose pending device was rejected is refused", func(t *testing.T) { compose.UpdateDeviceStatus(t, device.UID, environment.DeviceActionReject) require.Equal(t, models.DeviceStatusRejected, requireDevice(t, compose, device.UID).Status) requireEndpointRefused(t, compose, endpoint.FullAddress) }) if !rejected { return } t.Run("deleting a rejected device deletes its endpoints", func(t *testing.T) { require.NoError(t, agent.Stop(ctx, nil)) listedWebEndpoint(t, compose, endpoint.Address) compose.DeleteDevice(t, device.UID) _, resp, err := compose.GetDevice(t.Context(), device.UID) require.NoError(t, err) require.Equal(t, http.StatusNotFound, resp.StatusCode(), resp.String()) for _, listed := range compose.ListWebEndpoints(t) { assert.NotEqual(t, endpoint.Address, listed.Address) } requireEndpointRefused(t, compose, endpoint.FullAddress) }) } func testWebEndpointListing(t *testing.T, compose *environment.DockerCompose) { t.Helper() refusals := []struct { description string param string value string reason error }{ {description: "a filter that is not encoded JSON", param: "filter", value: "not a filter", reason: query.ErrFilterInvalid}, {description: "a filter on a field it does not take", param: "filter", value: propertyFilter(t, "port", "eq", 8001), reason: query.ErrFilterPropertyInvalid}, {description: "a filter with an operator its field does not take", param: "filter", value: propertyFilter(t, "device_id", "contains", "0"), reason: query.ErrFilterPropertyInvalid}, {description: "a sort on a field it does not take", param: "sort_by", value: "device_id", reason: query.ErrSorterFieldInvalid}, } for _, tc := range refusals { t.Run("a list with "+tc.description+" is refused", func(t *testing.T) { var refusal responses.Error resp, err := compose.R(t.Context()).SetQueryParam(tc.param, tc.value).SetError(&refusal).Get("/api/web-endpoints") require.NoError(t, err) require.Equal(t, http.StatusBadRequest, resp.StatusCode(), resp.String()) assert.Equal(t, map[string]string{tc.param: tc.reason.Error()}, refusal.Fields) }) } } func testWebEndpointRoles(t *testing.T, compose *environment.DockerCompose, device *models.Device, service deviceService) { t.Helper() endpoint := compose.CreateWebEndpoint(t, service.plain(device.UID, -1)) unused := environment.WebEndpointCreate{UID: device.UID, Host: deviceServiceHost, Port: 9001, TTL: -1} requests := []struct { method string path string body any }{ {method: http.MethodPost, path: "/api/web-endpoints", body: unused}, {method: http.MethodPost, path: "/api/devices/" + device.UID + "/tunnels", body: environment.TunnelCreate{Host: unused.Host, Port: unused.Port, TTL: unused.TTL}}, {method: http.MethodDelete, path: "/api/web-endpoints/" + endpoint.Address}, {method: http.MethodDelete, path: "/api/devices/" + device.UID + "/tunnels/" + endpoint.Address}, } for _, role := range []authorizer.Role{authorizer.RoleOperator, authorizer.RoleObserver} { t.Run("a member with the "+string(role)+" role can neither create nor delete an endpoint", func(t *testing.T) { member := newMember(t, compose, "web-"+string(role), role) listed := len(compose.ListWebEndpoints(t)) for _, req := range requests { resp, err := compose.Anonymous(t.Context()).SetAuthToken(member.Token).SetBody(req.body).Execute(req.method, req.path) require.NoError(t, err) assert.Equal(t, http.StatusForbidden, resp.StatusCode(), "%s %s: %s", req.method, req.path, resp.String()) } assert.Len(t, compose.ListWebEndpoints(t), listed) assert.Equal(t, service.name, requireDeviceAnswer(t, compose, endpoint.FullAddress, "/").Name) }) } t.Run("a member without the device connect permission cannot read an endpoint's address", func(t *testing.T) { require.False(t, authorizer.RoleObserver.HasPermission(authorizer.DeviceConnect)) member := newMember(t, compose, "web-reader", authorizer.RoleObserver) for _, path := range []string{"/api/web-endpoints", "/api/devices/" + device.UID + "/tunnels"} { resp, err := compose.Anonymous(t.Context()).SetAuthToken(member.Token).Get(path) require.NoError(t, err) assert.Equal(t, http.StatusForbidden, resp.StatusCode(), "GET %s: %s", path, resp.String()) } }) t.Run("a member with the device connect permission reads an endpoint's address", func(t *testing.T) { require.True(t, authorizer.RoleOperator.HasPermission(authorizer.DeviceConnect)) member := newMember(t, compose, "web-connector", authorizer.RoleOperator) assert.True(t, slices.ContainsFunc(compose.ListWebEndpointsAs(t, member.Token), func(e environment.WebEndpoint) bool { return e.Address == endpoint.Address })) assert.True(t, slices.ContainsFunc(compose.ListTunnelsAs(t, device.UID, member.Token), func(e environment.Tunnel) bool { return e.Address == endpoint.Address })) }) } func awaitDeviceStatus(t *testing.T, compose *environment.DockerCompose, uid string, status models.DeviceStatus) { t.Helper() require.EventuallyWithT(t, func(tt *assert.CollectT) { device, resp, err := compose.GetDevice(t.Context(), uid) if assert.NoError(tt, err) && assert.Equal(tt, http.StatusOK, resp.StatusCode()) { assert.Equal(tt, status, device.Status) } }, 30*time.Second, time.Second) } func testWebEndpointConnector(t *testing.T, ctx context.Context, compose *environment.DockerCompose) { t.Helper() label := "web-" + randomHex(t) containerService := deviceService{name: "container", httpPort: 8006, tlsPort: 8446} startDeviceServiceOnStackNetwork(t, compose, containerService, map[string]string{connectorLabel: label}) startConnector(t, ctx, compose, connectorLabel+"="+label) device := awaitContainerWithStatus(t, compose, models.DeviceStatusPending) compose.UpdateDeviceStatus(t, device.UID, environment.DeviceActionAccept) compose.AwaitDeviceOnline(t, device.UID) t.Run("an endpoint for a container's loopback reaches the service on the container's address", func(t *testing.T) { endpoint := compose.CreateWebEndpoint(t, containerService.plain(device.UID, -1)) assert.Equal(t, containerService.name, requireDeviceAnswer(t, compose, endpoint.FullAddress, "/").Name) }) } func startDeviceService(t *testing.T, compose *environment.DockerCompose, service deviceService) deviceService { t.Helper() relay := compose.Service(environment.ServiceRelay).GetContainerID() startDeviceHTTP(t, service, deviceServiceHost, func(req *testcontainers.ContainerRequest) { req.HostConfigModifier = func(hc *container.HostConfig) { hc.NetworkMode = container.NetworkMode("container:" + relay) } }) return service } func startDeviceServiceOnStackNetwork(t *testing.T, compose *environment.DockerCompose, service deviceService, labels map[string]string) string { t.Helper() network := stackNetwork(t, compose) c := startDeviceHTTP(t, service, "0.0.0.0", func(req *testcontainers.ContainerRequest) { req.Networks = []string{network} maps.Copy(req.Labels, labels) }) ip, err := c.ContainerIP(t.Context()) require.NoError(t, err) return ip } func startDeviceHTTP(t *testing.T, service deviceService, listenHost string, place func(*testcontainers.ContainerRequest)) testcontainers.Container { t.Helper() buildArgs, err := testimage.BuildArgs("..") require.NoError(t, err) req := testcontainers.ContainerRequest{ FromDockerfile: testcontainers.FromDockerfile{ Repo: "device-http", Tag: "test", Context: "cmd/device-http", KeepImage: true, BuildArgs: buildArgs, }, Env: map[string]string{ "NAME": service.name, "HTTP_ADDRESS": listenHost + ":" + strconv.Itoa(service.httpPort), "TLS_ADDRESS": listenHost + ":" + strconv.Itoa(service.tlsPort), "TLS_DOMAIN": deviceTLSDomain, }, Labels: run.Labels(), WaitingFor: wait.ForLog("listening"), } place(&req) c, err := testcontainers.GenericContainer(t.Context(), testcontainers.GenericContainerRequest{ ContainerRequest: req, Started: true, Logger: log.New(io.Discard, "", log.LstdFlags), }) require.NoError(t, err) t.Cleanup(func() { _ = c.Terminate(context.WithoutCancel(t.Context())) }) return c } func gatewayAuthority(t *testing.T, compose *environment.DockerCompose) *x509.Certificate { t.Helper() var data []byte require.EventuallyWithT(t, func(tt *assert.CollectT) { reader, err := compose.Service(environment.ServiceServer).CopyFileFromContainer(t.Context(), gatewayRootCertificate) if !assert.NoError(tt, err) { return } defer reader.Close() //nolint:errcheck // a failed read is retried; the close changes nothing data, err = io.ReadAll(reader) assert.NoError(tt, err) }, 30*time.Second, time.Second) block, _ := pem.Decode(data) require.NotNil(t, block, "the gateway's root certificate is not PEM") certificate, err := x509.ParseCertificate(block.Bytes) require.NoError(t, err) return certificate } func listedWebEndpoint(t *testing.T, compose *environment.DockerCompose, address string) environment.WebEndpoint { t.Helper() for _, endpoint := range compose.ListWebEndpoints(t) { if endpoint.Address == address { return endpoint } } require.Failf(t, "the web endpoint is not listed", "address %s", address) return environment.WebEndpoint{} } func requireCreateRefused(t *testing.T, compose *environment.DockerCompose, req environment.WebEndpointCreate, status int) { t.Helper() listed := len(compose.ListWebEndpoints(t)) resp, err := compose.PostWebEndpoint(t.Context(), req, nil) require.NoError(t, err) require.Equal(t, status, resp.StatusCode(), resp.String()) assert.Len(t, compose.ListWebEndpoints(t), listed, "a refused create left an endpoint behind") } func visitWebEndpoint(t *testing.T, compose *environment.DockerCompose, host, uri string) *environment.WebEndpointResponse { t.Helper() resp, err := compose.VisitWebEndpoint(t.Context(), host, uri) require.NoError(t, err) return resp } func requireDeviceAnswer(t *testing.T, compose *environment.DockerCompose, host, uri string) deviceSeen { t.Helper() resp := visitWebEndpoint(t, compose, host, uri) require.Equal(t, http.StatusOK, resp.StatusCode, resp.Body) seen := deviceSeen{} require.NoError(t, json.Unmarshal([]byte(resp.Body), &seen), resp.Body) return seen } func requireEndpointRefused(t *testing.T, compose *environment.DockerCompose, host string) { t.Helper() resp := visitWebEndpoint(t, compose, host, "/") require.Equal(t, http.StatusForbidden, resp.StatusCode, resp.Body) assert.Contains(t, resp.Body, webEndpointNotFound) } func requireRefusedOnceCreatedPastTTL(t *testing.T, compose *environment.DockerCompose, endpoint environment.WebEndpoint) { t.Helper() compose.AgeWebEndpoint(t, endpoint.Address, time.Duration(endpoint.TTL+1)*time.Second) requireEndpointRefused(t, compose, endpoint.FullAddress) }