package main import ( "context" "crypto/tls" "crypto/x509" "encoding/json" "flag" "fmt" "net" "os/signal" "runtime" "syscall" "time" "net/http" "net/url" "os" "sort" "strings" operatorv1 "github.com/openshift/api/operator/v1" authopts "github.com/openshift/console/cmd/bridge/config/auth" "github.com/openshift/console/cmd/bridge/config/session" "github.com/openshift/console/pkg/auth" "github.com/openshift/console/pkg/controllers" "github.com/openshift/console/pkg/flags" "github.com/openshift/console/pkg/knative" "github.com/openshift/console/pkg/olm" "github.com/openshift/console/pkg/proxy" "github.com/openshift/console/pkg/server" "github.com/openshift/console/pkg/serverconfig" oscrypto "github.com/openshift/library-go/pkg/crypto" ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics/server" "github.com/patrickmn/go-cache" "golang.org/x/net/http2" kruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" klog "k8s.io/klog/v2" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) const ( k8sInClusterCA = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" k8sInClusterBearerToken = "/var/run/secrets/kubernetes.io/serviceaccount/token" catalogdHost = "catalogd-service.openshift-catalogd.svc:443" // Well-known location of the tenant aware Thanos service for OpenShift exposing the query and query_range endpoints. This is only accessible in-cluster. // Thanos proxies requests to both cluster monitoring and user workload monitoring prometheus instances. openshiftThanosTenancyHost = "thanos-querier.openshift-monitoring.svc:9092" // Well-known location of the tenant aware Thanos service for OpenShift exposing the rules endpoint. This is only accessible in-cluster. // Thanos proxies requests to the cluster monitoring and user workload monitoring prometheus instances as well as Thanos ruler instances. openshiftThanosTenancyForRulesHost = "thanos-querier.openshift-monitoring.svc:9093" // Well-known location of the Thanos service for OpenShift. This is only accessible in-cluster. // This is used for non-tenant global query requests // proxying to both cluster monitoring and user workload monitoring prometheus instances. openshiftThanosHost = "thanos-querier.openshift-monitoring.svc:9091" // Well-known location of Alert Manager service for OpenShift. This is only accessible in-cluster. openshiftAlertManagerHost = "alertmanager-main.openshift-monitoring.svc:9094" // Default location of the tenant aware Alert Manager service for OpenShift. This is only accessible in-cluster. openshiftAlertManagerTenancyHost = "alertmanager-main.openshift-monitoring.svc:9092" // Well-known location of the GitOps service. This is only accessible in-cluster openshiftGitOpsHost = "cluster.openshift-gitops.svc:8080" // Well-known location of the cluster proxy service. This is only accessible in-cluster openshiftClusterProxyHost = "cluster-proxy-addon-user.multicluster-engine.svc:9092" clusterManagementURL = "https://api.openshift.com/" defaultCacheDuration = 5 * time.Minute defaultCacheCleanup = 30 * time.Minute ) func main() { // Initialize controller-runtime logger, needed for the OLM handler log.SetLogger(zap.New()) fs := flag.NewFlagSet("bridge", flag.ExitOnError) klog.InitFlags(fs) defer klog.Flush() authOptions := authopts.NewAuthOptions() authOptions.AddFlags(fs) sessionOptions := session.NewSessionOptions() sessionOptions.AddFlags(fs) // Define commandline / env / config options fs.String("config", "", "The YAML config file.") fListen := fs.String("listen", "http://0.0.0.0:9000", "") fBaseAddress := fs.String("base-address", "", "Format: ://domainOrIPAddress[:port]. Example: https://openshift.example.com.") fAdditionalBaseAddresses := fs.String("additional-base-addresses", "", "Comma-separated additional console base URLs for multi-domain support.") fBasePath := fs.String("base-path", "/", "") // See https://github.com/openshift/service-serving-cert-signer fServiceCAFile := fs.String("service-ca-file", "", "CA bundle for OpenShift services signed with the service signing certificates.") fK8sMode := fs.String("k8s-mode", "in-cluster", "in-cluster | off-cluster") fK8sModeOffClusterEndpoint := fs.String("k8s-mode-off-cluster-endpoint", "", "URL of the Kubernetes API server.") fK8sModeOffClusterSkipVerifyTLS := fs.Bool("k8s-mode-off-cluster-skip-verify-tls", false, "DEV ONLY. When true, skip verification of certs presented by k8s API server.") fK8sModeOffClusterThanos := fs.String("k8s-mode-off-cluster-thanos", "", "DEV ONLY. URL of the cluster's Thanos server.") fK8sModeOffClusterAlertmanager := fs.String("k8s-mode-off-cluster-alertmanager", "", "DEV ONLY. URL of the cluster's AlertManager server.") fK8sModeOffClusterCatalogd := fs.String("k8s-mode-off-cluster-catalogd", "", "DEV ONLY. URL of the cluster's catalogd server.") fK8sModeOffClusterServiceAccountBearerTokenFile := fs.String("k8s-mode-off-cluster-service-account-bearer-token-file", "", "DEV ONLY. bearer token file for the service account used for internal K8s API server calls.") fK8sAuth := fs.String("k8s-auth", "", "this option is deprecated, setting it has no effect") fK8sModeOffClusterGitOps := fs.String("k8s-mode-off-cluster-gitops", "", "DEV ONLY. URL of the GitOps backend service") fRedirectPort := fs.Int("redirect-port", 0, "Port number under which the console should listen for custom hostname redirect.") fLogLevel := fs.String("log-level", "", "level of logging information by package (pkg=level).") fPublicDir := fs.String("public-dir", "./frontend/public/dist", "directory containing static web assets.") fTLSCertFile := fs.String("tls-cert-file", "", "TLS certificate. If the certificate is signed by a certificate authority, the certFile should be the concatenation of the server's certificate followed by the CA's certificate.") fTLSKeyFile := fs.String("tls-key-file", "", "The TLS certificate key.") fCAFile := fs.String("ca-file", "", "PEM File containing trusted certificates of trusted CAs. If not present, the system's Root CAs will be used.") _ = fs.String("kubectl-client-id", "", "DEPRECATED: setting this does not do anything.") _ = fs.String("kubectl-client-secret", "", "DEPRECATED: setting this does not do anything.") _ = fs.String("kubectl-client-secret-file", "", "DEPRECATED: setting this does not do anything.") fK8sPublicEndpoint := fs.String("k8s-public-endpoint", "", "Endpoint to use to communicate to the API server.") fBranding := fs.String("branding", "okd", "Console branding for the masthead logo and title. One of okd, openshift, ocp, online, dedicated, azure, or rosa. Defaults to okd.") fCustomProductName := fs.String("custom-product-name", "", "Custom product name for console branding.") customLogoFlags := serverconfig.LogosKeyValue{} fs.Var(&customLogoFlags, "custom-logo-files", "List of custom product images used for branding of console's logo in the Masthead and 'About' modal.\n"+ "Each entry consist of theme type (Dark | Light ) as a key and the path to the image file used for the given theme as its value.\n"+ "Example --custom-logo-files Dark=./foo/dark-image.png,Light=./foo/light-image.png") customFaviconFlags := serverconfig.LogosKeyValue{} fs.Var(&customFaviconFlags, "custom-favicon-files", "List of custom images used for branding of console's favicon.\n"+ "Each entry consist of theme type (Dark | Light ) as a key and the path to the image file used for the given theme as its value.\n"+ "Example --custom-favicon-files Dark=./foo/dark-image.png,Light=./foo/light-image.png") fStatuspageID := fs.String("statuspage-id", "", "Unique ID assigned by statuspage.io page that provides status info.") fDocumentationBaseURL := fs.String("documentation-base-url", "", "The base URL for documentation links.") fAlertmanagerUserWorkloadHost := fs.String("alermanager-user-workload-host", openshiftAlertManagerHost, "Location of the Alertmanager service for user-defined alerts.") fAlertmanagerTenancyHost := fs.String("alermanager-tenancy-host", openshiftAlertManagerTenancyHost, "Location of the tenant-aware Alertmanager service.") fAlermanagerPublicURL := fs.String("alermanager-public-url", "", "Public URL of the cluster's AlertManager server.") fGrafanaPublicURL := fs.String("grafana-public-url", "", "Public URL of the cluster's Grafana server.") fPrometheusPublicURL := fs.String("prometheus-public-url", "", "Public URL of the cluster's Prometheus server.") fThanosPublicURL := fs.String("thanos-public-url", "", "Public URL of the cluster's Thanos server.") enabledPlugins := serverconfig.MultiKeyValue{} fs.Var(&enabledPlugins, "plugins", "List of plugin entries that are enabled for the console. Each entry consist of plugin-name as a key and plugin-endpoint as a value.") fPluginsOrder := fs.String("plugins-order", "", "List of plugin names which determines the order in which plugin extensions will be resolved.") fPluginProxy := fs.String("plugin-proxy", "", "Defines various service types to which will console proxy plugins requests. (JSON as string)") fI18NamespacesFlags := fs.String("i18n-namespaces", "", "List of namespaces separated by comma. Example --i18n-namespaces=plugin__acm,plugin__kubevirt") fCSPMode := fs.String("csp-mode", string(serverconfig.CSPModeReportOnly), "CSP header mode: 'report-only' emits Content-Security-Policy-Report-Only header; 'enforce' emits Content-Security-Policy header") consoleCSPFlags := serverconfig.MultiKeyValue{} fs.Var(&consoleCSPFlags, "content-security-policy", "List of CSP directives that are enabled for the console. Each entry consist of csp-directive-name as a key and csp-directive-value as a value. Example --content-security-policy script-src='localhost:9000',font-src='localhost:9001'") telemetryFlags := serverconfig.MultiKeyValue{} fs.Var(&telemetryFlags, "telemetry", "Telemetry configuration that can be used by console plugins. Each entry should be a key=value pair.") fLoadTestFactor := fs.Int("load-test-factor", 0, "DEV ONLY. The factor used to multiply k8s API list responses for load testing purposes.") fDevCatalogCategories := fs.String("developer-catalog-categories", "", "Allow catalog categories customization. (JSON as string)") fDevCatalogTypes := fs.String("developer-catalog-types", "", "Allow enabling/disabling of sub-catalog types from the developer catalog. (JSON as string)") fUserSettingsLocation := fs.String("user-settings-location", "configmap", "DEV ONLY. Define where the user settings should be stored. (configmap | localstorage).") fQuickStarts := fs.String("quick-starts", "", "Allow customization of available ConsoleQuickStart resources in console. (JSON as string)") fAddPage := fs.String("add-page", "", "DEV ONLY. Allow add page customization. (JSON as string)") fProjectAccessClusterRoles := fs.String("project-access-cluster-roles", "", "The list of Cluster Roles assignable for the project access page. (JSON as string)") fPerspectives := fs.String("perspectives", "", "Allow enabling/disabling of perspectives in the console. (JSON as string)") fCapabilities := fs.String("capabilities", "", "Allow enabling/disabling of capabilities in the console. (JSON as string)") fControlPlaneTopology := fs.String("control-plane-topology-mode", "", "Defines the topology mode of the control-plane nodes (External | HighlyAvailable | HighlyAvailableArbiter | DualReplica | SingleReplica)") fReleaseVersion := fs.String("release-version", "", "Defines the release version of the cluster") fNodeArchitectures := fs.String("node-architectures", "", "List of node architectures. Example --node-architecture=amd64,arm64") fNodeOperatingSystems := fs.String("node-operating-systems", "", "List of node operating systems. Example --node-operating-system=linux,windows") fCopiedCSVsDisabled := fs.Bool("copied-csvs-disabled", false, "Flag to indicate if OLM copied CSVs are disabled.") fTechPreview := fs.Bool("tech-preview", false, "Enable console Technology Preview features.") fOLMLifecycleMetadata := fs.Bool("olm-lifecycle-metadata", false, "Enable OLM Operator lifecycle and compatibility features.") cfg, err := serverconfig.Parse(fs, os.Args[1:], "BRIDGE") if err != nil { fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } if err := serverconfig.Validate(fs); err != nil { fmt.Fprintln(os.Stderr, err.Error()) os.Exit(1) } if *fTechPreview { klog.Warning("Technology Preview features are enabled. These features are experimental and not supported for production use. If you encounter issues, send feedback through the usual support or bug-reporting channels.") } authOptions.ApplyConfig(&cfg.Auth) sessionOptions.ApplyConfig(&cfg.Session) baseURL, err := flags.ValidateFlagIsURL("base-address", *fBaseAddress, true) flags.FatalIfFailed(err) if !strings.HasPrefix(*fBasePath, "/") || !strings.HasSuffix(*fBasePath, "/") { flags.FatalIfFailed(flags.NewInvalidFlagError("base-path", "value must start and end with slash")) } baseURL.Path = *fBasePath var additionalBaseURLs []*url.URL if *fAdditionalBaseAddresses != "" { for _, addr := range strings.Split(*fAdditionalBaseAddresses, ",") { addr = strings.TrimSpace(addr) if addr == "" { continue } u, err := url.Parse(addr) if err != nil { klog.Fatalf("invalid additional base address %q: %v", addr, err) } if u.Scheme == "" || u.Host == "" { klog.Fatalf("additional base address %q must be an absolute URL with scheme and host", addr) } u.Path = *fBasePath additionalBaseURLs = append(additionalBaseURLs, u) } } documentationBaseURL := &url.URL{} if *fDocumentationBaseURL != "" { if !strings.HasSuffix(*fDocumentationBaseURL, "/") { flags.FatalIfFailed(flags.NewInvalidFlagError("documentation-base-url", "value must end with slash")) } documentationBaseURL, err = flags.ValidateFlagIsURL("documentation-base-url", *fDocumentationBaseURL, false) flags.FatalIfFailed(err) } alertManagerPublicURL, err := flags.ValidateFlagIsURL("alermanager-public-url", *fAlermanagerPublicURL, true) flags.FatalIfFailed(err) grafanaPublicURL, err := flags.ValidateFlagIsURL("grafana-public-url", *fGrafanaPublicURL, true) flags.FatalIfFailed(err) prometheusPublicURL, err := flags.ValidateFlagIsURL("prometheus-public-url", *fPrometheusPublicURL, true) flags.FatalIfFailed(err) thanosPublicURL, err := flags.ValidateFlagIsURL("thanos-public-url", *fThanosPublicURL, true) flags.FatalIfFailed(err) branding := *fBranding if branding == "origin" { branding = "okd" } switch branding { case "okd": case "openshift": case "ocp": case "online": case "dedicated": case "azure": case "rosa": default: flags.FatalIfFailed(flags.NewInvalidFlagError("branding", "value must be one of okd, openshift, ocp, online, dedicated, azure, or rosa")) } i18nNamespaces := []string{} if *fI18NamespacesFlags != "" { for _, str := range strings.Split(*fI18NamespacesFlags, ",") { str = strings.TrimSpace(str) if str == "" { flags.FatalIfFailed(flags.NewInvalidFlagError("i18n-namespaces", "list must contain name of i18n namespaces separated by comma")) } i18nNamespaces = append(i18nNamespaces, str) } } enabledPluginsOrder := []string{} if *fPluginsOrder != "" { for _, str := range strings.Split(*fPluginsOrder, ",") { str = strings.TrimSpace(str) if str == "" { flags.FatalIfFailed(flags.NewInvalidFlagError("plugins-order", "list must contain names of plugins separated by comma")) } if enabledPlugins[str] == "" { flags.FatalIfFailed(flags.NewInvalidFlagError("plugins-order", "list must only contain currently enabled plugins")) } enabledPluginsOrder = append(enabledPluginsOrder, str) } } else if len(enabledPlugins) > 0 { for plugin := range enabledPlugins { enabledPluginsOrder = append(enabledPluginsOrder, plugin) } } if len(enabledPluginsOrder) > 0 { klog.Infoln("Console plugins are enabled in following order:") for _, pluginName := range enabledPluginsOrder { klog.Infof(" - %s", pluginName) } } nodeArchitectures := []string{} if *fNodeArchitectures != "" { for _, str := range strings.Split(*fNodeArchitectures, ",") { str = strings.TrimSpace(str) if str == "" { flags.FatalIfFailed(flags.NewInvalidFlagError("node-architectures", "list must contain name of node architectures separated by comma")) } nodeArchitectures = append(nodeArchitectures, str) } } nodeOperatingSystems := []string{} if *fNodeOperatingSystems != "" { for _, str := range strings.Split(*fNodeOperatingSystems, ",") { str = strings.TrimSpace(str) if str == "" { flags.FatalIfFailed(flags.NewInvalidFlagError("node-operating-systems", "list must contain name of node architectures separated by comma")) } nodeOperatingSystems = append(nodeOperatingSystems, str) } } capabilities := []operatorv1.Capability{} if *fCapabilities != "" { err = json.Unmarshal([]byte(*fCapabilities), &capabilities) if err != nil { klog.Fatalf("Error unmarshaling capabilities JSON: %v", err) } } if len(telemetryFlags) > 0 { keys := make([]string, 0, len(telemetryFlags)) for name := range telemetryFlags { keys = append(keys, name) } sort.Strings(keys) klog.Infoln("Console telemetry options:") for _, k := range keys { klog.Infof(" - %s %s", k, telemetryFlags[k]) } } srv := &server.Server{ PublicDir: *fPublicDir, BaseURL: baseURL, AdditionalBaseURLs: additionalBaseURLs, Branding: branding, CustomProductName: *fCustomProductName, CustomLogoFiles: customLogoFlags, CustomFaviconFiles: customFaviconFlags, ControlPlaneTopology: *fControlPlaneTopology, StatuspageID: *fStatuspageID, DocumentationBaseURL: documentationBaseURL, AlertManagerUserWorkloadHost: *fAlertmanagerUserWorkloadHost, AlertManagerTenancyHost: *fAlertmanagerTenancyHost, AlertManagerPublicURL: alertManagerPublicURL, GrafanaPublicURL: grafanaPublicURL, PrometheusPublicURL: prometheusPublicURL, ThanosPublicURL: thanosPublicURL, LoadTestFactor: *fLoadTestFactor, DevCatalogCategories: *fDevCatalogCategories, DevCatalogTypes: *fDevCatalogTypes, UserSettingsLocation: *fUserSettingsLocation, EnabledPlugins: enabledPlugins, EnabledPluginsOrder: enabledPluginsOrder, I18nNamespaces: i18nNamespaces, PluginProxy: *fPluginProxy, ContentSecurityPolicy: consoleCSPFlags, CSPMode: serverconfig.CSPMode(*fCSPMode), QuickStarts: *fQuickStarts, AddPage: *fAddPage, ProjectAccessClusterRoles: *fProjectAccessClusterRoles, Perspectives: *fPerspectives, Telemetry: telemetryFlags, ReleaseVersion: *fReleaseVersion, NodeArchitectures: nodeArchitectures, NodeOperatingSystems: nodeOperatingSystems, K8sMode: *fK8sMode, CopiedCSVsDisabled: *fCopiedCSVsDisabled, TechPreview: *fTechPreview, OLMLifecycleMetadata: *fOLMLifecycleMetadata, Capabilities: capabilities, } completedAuthnOptions, err := authOptions.Complete() if err != nil { klog.Fatalf("failed to complete authentication options: %v", err) os.Exit(1) } completedSessionOptions, err := sessionOptions.Complete(completedAuthnOptions.AuthType) if err != nil { klog.Fatalf("failed to complete session options: %v", err) os.Exit(1) } // if !in-cluster (dev) we should not pass these values to the frontend // is used by catalog-utils.ts if *fK8sMode == "in-cluster" { srv.GOARCH = runtime.GOARCH srv.GOOS = runtime.GOOS } // Blacklisted headers srv.ProxyHeaderDenyList = []string{"Cookie", "X-CSRFToken"} if *fLogLevel != "" { klog.Warningf("DEPRECATED: --log-level is now deprecated, use verbosity flag --v=Level instead") } var ( // Hold on to raw certificates so we can render them in kubeconfig files. k8sCertPEM []byte ) if *fK8sModeOffClusterSkipVerifyTLS && *fK8sMode != "off-cluster" { flags.FatalIfFailed(flags.NewInvalidFlagError("k8s-mode-off-cluster-skip-verify-tls", "can only be set when -k8s-mode=off-cluster")) } var k8sEndpoint *url.URL switch *fK8sMode { case "in-cluster": k8sEndpoint = &url.URL{Scheme: "https", Host: "kubernetes.default.svc"} var err error k8sCertPEM, err = os.ReadFile(k8sInClusterCA) if err != nil { klog.Fatalf("Error inferring Kubernetes config from environment: %v", err) } rootCAs := x509.NewCertPool() if !rootCAs.AppendCertsFromPEM(k8sCertPEM) { klog.Fatal("No CA found for the API server") } tlsConfig := oscrypto.SecureTLSConfig(&tls.Config{ RootCAs: rootCAs, }) srv.InternalProxiedK8SClientConfig = &rest.Config{ Host: k8sEndpoint.String(), BearerTokenFile: k8sInClusterBearerToken, TLSClientConfig: rest.TLSClientConfig{ CAFile: k8sInClusterCA, }, } srv.K8sProxyConfig = &proxy.Config{ TLSClientConfig: tlsConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: k8sEndpoint, } // If running in an OpenShift cluster, set up a proxy to the prometheus-k8s service running in the openshift-monitoring namespace. if *fServiceCAFile != "" { serviceCertPEM, err := os.ReadFile(*fServiceCAFile) if err != nil { klog.Fatalf("failed to read service-ca.crt file: %v", err) } serviceProxyRootCAs := x509.NewCertPool() if !serviceProxyRootCAs.AppendCertsFromPEM(serviceCertPEM) { klog.Fatal("no CA found for Kubernetes services") } serviceProxyTLSConfig := oscrypto.SecureTLSConfig(&tls.Config{ RootCAs: serviceProxyRootCAs, }) srv.ServiceClient = newServiceHTTPClient(serviceProxyTLSConfig) srv.CatalogdProxyConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: &url.URL{Scheme: "https", Host: catalogdHost}, } srv.ThanosProxyConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: &url.URL{Scheme: "https", Host: openshiftThanosHost, Path: "/api"}, } srv.ThanosTenancyProxyConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: &url.URL{Scheme: "https", Host: openshiftThanosTenancyHost, Path: "/api"}, } srv.ThanosTenancyProxyForRulesConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: &url.URL{Scheme: "https", Host: openshiftThanosTenancyForRulesHost, Path: "/api"}, } srv.AlertManagerProxyConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: &url.URL{Scheme: "https", Host: openshiftAlertManagerHost, Path: "/api"}, } srv.AlertManagerUserWorkloadProxyConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: &url.URL{Scheme: "https", Host: *fAlertmanagerUserWorkloadHost, Path: "/api"}, } srv.AlertManagerTenancyProxyConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: &url.URL{Scheme: "https", Host: *fAlertmanagerTenancyHost, Path: "/api"}, } srv.TerminalProxyTLSConfig = serviceProxyTLSConfig srv.PluginsProxyTLSConfig = serviceProxyTLSConfig srv.GitOpsProxyConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: &url.URL{Scheme: "https", Host: openshiftGitOpsHost}, } } case "off-cluster": k8sEndpoint, err = flags.ValidateFlagIsURL("k8s-mode-off-cluster-endpoint", *fK8sModeOffClusterEndpoint, false) flags.FatalIfFailed(err) k8sProxyTLSConfig, serviceProxyTLSConfig := offClusterProxyTLSConfigs(*fCAFile, *fServiceCAFile, *fK8sModeOffClusterSkipVerifyTLS) srv.ServiceClient = newServiceHTTPClient(serviceProxyTLSConfig) srv.InternalProxiedK8SClientConfig = &rest.Config{ Host: k8sEndpoint.String(), Transport: &http.Transport{TLSClientConfig: k8sProxyTLSConfig}, } if *fK8sModeOffClusterServiceAccountBearerTokenFile != "" { srv.InternalProxiedK8SClientConfig.BearerTokenFile = *fK8sModeOffClusterServiceAccountBearerTokenFile } srv.K8sProxyConfig = &proxy.Config{ TLSClientConfig: k8sProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: k8sEndpoint, UseProxyFromEnvironment: true, } if *fK8sModeOffClusterCatalogd != "" { offClusterCatalogdURL, err := flags.ValidateFlagIsURL("k8s-mode-off-cluster-catalogd", *fK8sModeOffClusterCatalogd, false) flags.FatalIfFailed(err) srv.CatalogdProxyConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: offClusterCatalogdURL, } } if *fK8sModeOffClusterThanos != "" { offClusterThanosURL, err := flags.ValidateFlagIsURL("k8s-mode-off-cluster-thanos", *fK8sModeOffClusterThanos, false) flags.FatalIfFailed(err) offClusterThanosURL.Path += "/api" srv.ThanosTenancyProxyConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: offClusterThanosURL, } srv.ThanosTenancyProxyForRulesConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: offClusterThanosURL, } srv.ThanosProxyConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: offClusterThanosURL, } } if *fK8sModeOffClusterAlertmanager != "" { offClusterAlertManagerURL, err := flags.ValidateFlagIsURL("k8s-mode-off-cluster-alertmanager", *fK8sModeOffClusterAlertmanager, false) flags.FatalIfFailed(err) offClusterAlertManagerURL.Path += "/api" srv.AlertManagerProxyConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: offClusterAlertManagerURL, } srv.AlertManagerTenancyProxyConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: offClusterAlertManagerURL, } srv.AlertManagerUserWorkloadProxyConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: offClusterAlertManagerURL, } } srv.TerminalProxyTLSConfig = serviceProxyTLSConfig srv.PluginsProxyTLSConfig = serviceProxyTLSConfig if *fK8sModeOffClusterGitOps != "" { offClusterGitOpsURL, err := flags.ValidateFlagIsURL("k8s-mode-off-cluster-gitops", *fK8sModeOffClusterGitOps, false) flags.FatalIfFailed(err) srv.GitOpsProxyConfig = &proxy.Config{ TLSClientConfig: serviceProxyTLSConfig, HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: offClusterGitOpsURL, } } default: flags.FatalIfFailed(flags.NewInvalidFlagError("k8s-mode", "must be one of: in-cluster, off-cluster")) } // Set up signal handler for graceful shutdown on first interrupt // This context will be used by both the controller manager and HTTP server ctx, cancel := context.WithCancel(context.Background()) sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) go func() { sig := <-sigCh klog.Infof("Received shutdown signal: %v. Gracefully shutting down...", sig) cancel() }() // Controllers are behind Tech Preview flag if *fTechPreview { controllerManagerMetricsOptions := ctrlmetrics.Options{ // Disable the metrics server for now. We can enable it later if we want and make it a configurable flag. BindAddress: "0", } mgr, err := ctrl.NewManager(srv.InternalProxiedK8SClientConfig, ctrl.Options{ Scheme: kruntime.NewScheme(), Metrics: controllerManagerMetricsOptions, }) if err != nil { klog.Errorf("problem creating main controller manager: %v", err) } cache := cache.New(defaultCacheDuration, defaultCacheCleanup) catalogService := olm.NewCatalogService(srv.ServiceClient, srv.CatalogdProxyConfig, cache) srv.CatalogService = catalogService if err = controllers.NewClusterCatalogReconciler(mgr, catalogService).SetupWithManager(mgr); err != nil { klog.Errorf("failed to start ClusterCatalog reconciler: %v", err) } klog.Info("starting controller manager") go func() { // Controller manager will be stopped when signal context is cancelled if err := mgr.Start(ctx); err != nil { klog.Errorf("problem running manager: %v", err) } }() } apiServerEndpoint := *fK8sPublicEndpoint if apiServerEndpoint == "" { apiServerEndpoint = srv.K8sProxyConfig.Endpoint.String() } srv.KubeAPIServerURL = apiServerEndpoint // Embedded into kubeconfig files downloaded via /api/kubeconfig so clients // can verify the API server's serving certificate. Empty in off-cluster mode, // in which case downloaded kubeconfigs fall back to insecure-skip-tls-verify. srv.K8sCertPEM = k8sCertPEM clusterManagementURL, err := url.Parse(clusterManagementURL) if err != nil { klog.Fatalf("failed to parse %q", clusterManagementURL) } srv.ClusterManagementProxyConfig = &proxy.Config{ TLSClientConfig: oscrypto.SecureTLSConfig(&tls.Config{}), HeaderBlacklist: srv.ProxyHeaderDenyList, Endpoint: clusterManagementURL, } if len(*fK8sAuth) > 0 { klog.Warning("DEPRECATED: --k8s-auth is deprecated and setting it has no effect") } internalProxiedK8SRT, err := rest.TransportFor(srv.InternalProxiedK8SClientConfig) if err != nil { klog.Fatalf("Failed to create k8s HTTP client: %v", err) } srv.MonitoringDashboardConfigMapLister = server.NewResourceLister( &url.URL{ Scheme: k8sEndpoint.Scheme, Host: k8sEndpoint.Host, Path: k8sEndpoint.Path + "/api/v1/namespaces/openshift-config-managed/configmaps", RawQuery: url.Values{ "labelSelector": {"console.openshift.io/dashboard=true"}, }.Encode(), }, internalProxiedK8SRT, nil, ) srv.KnativeEventSourceCRDLister = server.NewResourceLister( &url.URL{ Scheme: k8sEndpoint.Scheme, Host: k8sEndpoint.Host, Path: k8sEndpoint.Path + "/apis/apiextensions.k8s.io/v1/customresourcedefinitions", RawQuery: url.Values{ "labelSelector": {"duck.knative.dev/source=true"}, }.Encode(), }, internalProxiedK8SRT, knative.EventSourceFilter, ) srv.KnativeChannelCRDLister = server.NewResourceLister( &url.URL{ Scheme: k8sEndpoint.Scheme, Host: k8sEndpoint.Host, Path: k8sEndpoint.Path + "/apis/apiextensions.k8s.io/v1/customresourcedefinitions", RawQuery: url.Values{ "labelSelector": {"duck.knative.dev/addressable=true,messaging.knative.dev/subscribable=true"}, }.Encode(), }, internalProxiedK8SRT, knative.ChannelFilter, ) srv.AnonymousInternalProxiedK8SRT, err = rest.TransportFor(anonymousK8SClientConfig(srv.InternalProxiedK8SClientConfig, *fCAFile, *fK8sModeOffClusterSkipVerifyTLS)) if err != nil { klog.Fatalf("Failed to create anonymous k8s HTTP client: %v", err) } srv.AuthMetrics = auth.NewMetrics(srv.AnonymousInternalProxiedK8SRT) caCertFilePath := *fCAFile if *fK8sMode == "in-cluster" { caCertFilePath = k8sInClusterCA } tokenReviewer, err := auth.NewTokenReviewer(srv.InternalProxiedK8SClientConfig) if err != nil { klog.Fatalf("failed to create token reviewer: %v", err) } srv.TokenReviewer = tokenReviewer if err := completedAuthnOptions.ApplyTo(srv, k8sEndpoint, caCertFilePath, completedSessionOptions); err != nil { klog.Fatalf("failed to apply configuration to server: %v", err) } listenURL, err := flags.ValidateFlagIsURL("listen", *fListen, false) flags.FatalIfFailed(err) switch listenURL.Scheme { case "http": case "https": flags.FatalIfFailed(flags.ValidateFlagNotEmpty("tls-cert-file", *fTLSCertFile)) flags.FatalIfFailed(flags.ValidateFlagNotEmpty("tls-key-file", *fTLSKeyFile)) default: flags.FatalIfFailed(flags.NewInvalidFlagError("listen", "scheme must be one of: http, https")) } handler, err := srv.HTTPHandler(ctx) if err != nil { klog.Fatalf("failed to set up HTTP handler: %v", err) } httpsrv := &http.Server{Handler: handler} if listenURL.Scheme == "https" { if err := http2.ConfigureServer(httpsrv, &http2.Server{}); err != nil { klog.Fatalf("failed to configure HTTP/2: %v", err) } klog.Info("HTTP/2 enabled") } listener, err := listen(listenURL.Scheme, listenURL.Host, *fTLSCertFile, *fTLSKeyFile, cfg.ServingInfo.MinTLSVersion, cfg.ServingInfo.CipherSuites, cfg.ServingInfo.TLSGroups) if err != nil { klog.Fatalf("Failed to start the server listener: %v. Verify the --tls-cert-file and --tls-key-file flags and the servingInfo TLS settings (minTLSVersion, cipherSuites, tlsGroups).", err) } defer listener.Close() go func() { <-ctx.Done() klog.Infof("Shutting down HTTP server...") // Use a timeout context for graceful shutdown shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) defer shutdownCancel() if err = httpsrv.Shutdown(shutdownCtx); err != nil { klog.Errorf("Error shutting down server: %v", err) } }() if *fRedirectPort != 0 { go func() { // Listen on passed port number to be redirected to the console redirectServer := http.NewServeMux() redirectServer.HandleFunc("/", func(res http.ResponseWriter, req *http.Request) { redirectURL := &url.URL{ Scheme: srv.BaseURL.Scheme, Host: srv.BaseURL.Host, RawQuery: req.URL.RawQuery, Path: req.URL.Path, } http.Redirect(res, req, redirectURL.String(), http.StatusMovedPermanently) }) redirectPort := fmt.Sprintf(":%d", *fRedirectPort) klog.Infof("Listening on %q for custom hostname redirect...", redirectPort) klog.Fatal(http.ListenAndServe(redirectPort, redirectServer)) }() } httpsrv.Serve(listener) } // anonymousK8SClientConfig derives the anonymous variant of the off-cluster // proxied k8s client config. rest.AnonymousClientConfig drops the source // config's Transport (which carries the off-cluster proxy's RootCAs and // InsecureSkipVerify setting), so when -ca-file is set we re-supply trust via // CAFile and re-apply insecure skip-verify. This lets rest.TransportFor build // a CA-aware transport; without it the anonymous transport falls back to the // system trust store and can't verify an API server signed by a private CA -- // which breaks the consumers of this transport (the user-settings handler and // the login-role metrics). caFile is only applied when the source config // doesn't already carry a CAFile, so in-cluster mode's service-account CA is // left untouched. Insecure is set first, and CAFile only when the result // isn't already insecure: rest.TransportFor rejects a config that carries // both a CA and Insecure, and -k8s-mode-off-cluster-skip-verify-tls=true is // commonly paired with -ca-file (e.g. examples/run-bridge.sh), which would // otherwise crash the bridge on startup. func anonymousK8SClientConfig(config *rest.Config, caFile string, insecureSkipVerify bool) *rest.Config { anonymous := rest.AnonymousClientConfig(config) anonymous.Insecure = anonymous.Insecure || insecureSkipVerify if caFile != "" && anonymous.CAFile == "" && !anonymous.Insecure { anonymous.CAFile = caFile } return anonymous } // mustLoadCAPool reads a PEM CA bundle file and returns a cert pool, exiting // fatally on any read/parse error. func mustLoadCAPool(caFile string) *x509.CertPool { caPEM, err := os.ReadFile(caFile) if err != nil { klog.Fatalf("Failed to read ca file %q: %v", caFile, err) } pool := x509.NewCertPool() if !pool.AppendCertsFromPEM(caPEM) { klog.Fatalf("No CA found in ca file %q", caFile) } return pool } // newServiceHTTPClient builds the client used for direct (non-reverse-proxy) // calls to OpenShift services -- currently just the catalogd client's own // requests to a catalog's baseURL (pkg/olm.CatalogdClient). It needs the same // environment-proxy support as proxy.NewProxy (pkg/proxy/proxy.go), which // every proxy.Config-based service proxy already gets: in a HyperShift // control-plane-side deployment, guest-cluster services are only reachable // through the konnectivity socks5 proxy exposed via HTTP(S)_PROXY, and // without this, catalogd requests fail outright instead of tunneling. func newServiceHTTPClient(tlsConfig *tls.Config) *http.Client { return &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, TLSClientConfig: tlsConfig, }, } } // offClusterProxyTLSConfigs builds the two TLS trust domains needed by // off-cluster mode: the guest KAS is signed by the CA at caFile, while // OpenShift services (Thanos, Alertmanager, terminal, plugins, catalogd, // gitops) present service-serving certs signed by a different CA // (service-ca). serviceCAFile supplies that second trust, mirroring the // in-cluster branch; when it's empty the service config falls back to the // KAS config so existing off-cluster callers that only pass -ca-file are // unaffected. func offClusterProxyTLSConfigs(caFile, serviceCAFile string, insecureSkipVerify bool) (k8sProxyTLSConfig, serviceProxyTLSConfig *tls.Config) { k8sProxyTLSConfig = oscrypto.SecureTLSConfig(&tls.Config{ InsecureSkipVerify: insecureSkipVerify, }) if caFile != "" { k8sProxyTLSConfig.RootCAs = mustLoadCAPool(caFile) } if serviceCAFile == "" { return k8sProxyTLSConfig, k8sProxyTLSConfig } serviceProxyTLSConfig = oscrypto.SecureTLSConfig(&tls.Config{ InsecureSkipVerify: insecureSkipVerify, RootCAs: mustLoadCAPool(serviceCAFile), }) return k8sProxyTLSConfig, serviceProxyTLSConfig } func listen(scheme, host, certFile, keyFile, minTLSVersion string, cipherSuites, groups []string) (net.Listener, error) { klog.Infof("Binding to %s...", host) if scheme == "http" { klog.Info("Serving over plain HTTP without TLS because the listen address scheme is \"http\"; traffic will not be encrypted. To enable TLS, use an \"https\" listen address together with --tls-cert-file and --tls-key-file.") return net.Listen("tcp", host) } klog.Info("Using TLS") tlsConfig := &tls.Config{ NextProtos: []string{"h2", "http/1.1"}, GetCertificate: func(_ *tls.ClientHelloInfo) (*tls.Certificate, error) { klog.V(4).Infof("Getting TLS certs.") cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { return nil, err } return &cert, nil }, } if minTLSVersion != "" { minVersion, err := oscrypto.TLSVersion(minTLSVersion) if err != nil { return nil, fmt.Errorf("invalid minTLSVersion %q: %w", minTLSVersion, err) } tlsConfig.MinVersion = minVersion } if len(cipherSuites) > 0 { ciphers := make([]uint16, 0, len(cipherSuites)) for _, cipherName := range cipherSuites { cipher, err := oscrypto.CipherSuite(cipherName) if err != nil { return nil, fmt.Errorf("invalid cipher suite %q: %w", cipherName, err) } ciphers = append(ciphers, cipher) } tlsConfig.CipherSuites = ciphers } if len(groups) > 0 { curves, unsupported := tlsCurvePreferences(groups) if len(unsupported) > 0 { klog.Warningf("Ignoring TLS groups not supported by this console build: %v. Remove them from the TLSSecurityProfile or replace them with supported groups (X25519, secp256r1, secp384r1, secp521r1, X25519MLKEM768, SecP256r1MLKEM768, SecP384r1MLKEM1024).", unsupported) } if len(curves) > 0 { tlsConfig.CurvePreferences = curves } else { klog.Warningf("The console does not support any of the configured TLS groups %v and is using the default TLS groups instead. To use specific groups, configure supported groups in the TLSSecurityProfile.", groups) } } return tls.Listen("tcp", host, tlsConfig) }