// Copyright (C) 2023 Percona LLC // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . // Package main provides the entry point for the update application. package main import ( "bytes" "context" "database/sql" _ "expvar" // register /debug/vars "fmt" "html/template" "log" "net" "net/http" _ "net/http/pprof" //nolint:gosec // register /debug/pprof "net/url" "os" "os/signal" "sort" "strconv" "strings" "sync" "time" _ "github.com/ClickHouse/clickhouse-go/v2" "github.com/alecthomas/kingpin/v2" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" grpc_validator "github.com/grpc-ecosystem/go-grpc-middleware/validator" grpc_gateway "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/pkg/errors" metrics "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" prom "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "golang.org/x/sync/semaphore" "golang.org/x/sys/unix" "google.golang.org/grpc" "google.golang.org/grpc/backoff" channelz "google.golang.org/grpc/channelz/service" "google.golang.org/grpc/credentials/insecure" // Installing the gzip encoding registers it as an available compressor. // GRPC will automatically negotiate and use gzip if the client supports it. _ "google.golang.org/grpc/encoding/gzip" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/reflection" "google.golang.org/protobuf/encoding/protojson" "gopkg.in/reform.v1" "gopkg.in/reform.v1/dialects/postgresql" accesscontrolv1 "github.com/percona/pmm/api/accesscontrol/v1beta1" actionsv1 "github.com/percona/pmm/api/actions/v1" advisorsv1 "github.com/percona/pmm/api/advisors/v1" agentpb "github.com/percona/pmm/api/agent/pb" agentv1 "github.com/percona/pmm/api/agent/v1" alertingv1 "github.com/percona/pmm/api/alerting/v1" backupv1 "github.com/percona/pmm/api/backup/v1" dumpv1beta1 "github.com/percona/pmm/api/dump/v1beta1" hav1beta1 "github.com/percona/pmm/api/ha/v1beta1" inventoryv1 "github.com/percona/pmm/api/inventory/v1" managementv1 "github.com/percona/pmm/api/management/v1" rtav1 "github.com/percona/pmm/api/realtimeanalytics/v1" serverv1 "github.com/percona/pmm/api/server/v1" uieventsv1 "github.com/percona/pmm/api/uievents/v1" userv1 "github.com/percona/pmm/api/user/v1" "github.com/percona/pmm/managed/models" "github.com/percona/pmm/managed/services/agents" agentgrpc "github.com/percona/pmm/managed/services/agents/grpc" "github.com/percona/pmm/managed/services/alerting" "github.com/percona/pmm/managed/services/backup" "github.com/percona/pmm/managed/services/checks" "github.com/percona/pmm/managed/services/config" //nolint:staticcheck "github.com/percona/pmm/managed/services/dump" "github.com/percona/pmm/managed/services/grafana" "github.com/percona/pmm/managed/services/ha" "github.com/percona/pmm/managed/services/inventory" inventorygrpc "github.com/percona/pmm/managed/services/inventory/grpc" "github.com/percona/pmm/managed/services/management" managementbackup "github.com/percona/pmm/managed/services/management/backup" "github.com/percona/pmm/managed/services/management/common" managementdump "github.com/percona/pmm/managed/services/management/dump" managementgrpc "github.com/percona/pmm/managed/services/management/grpc" "github.com/percona/pmm/managed/services/minio" "github.com/percona/pmm/managed/services/nomad" "github.com/percona/pmm/managed/services/qan" "github.com/percona/pmm/managed/services/realtimeanalytics" "github.com/percona/pmm/managed/services/scheduler" "github.com/percona/pmm/managed/services/server" "github.com/percona/pmm/managed/services/supervisord" "github.com/percona/pmm/managed/services/telemetry" "github.com/percona/pmm/managed/services/telemetry/uievents" "github.com/percona/pmm/managed/services/user" "github.com/percona/pmm/managed/services/versioncache" "github.com/percona/pmm/managed/services/victoriametrics" "github.com/percona/pmm/managed/services/vmalert" "github.com/percona/pmm/managed/utils/clean" "github.com/percona/pmm/managed/utils/distribution" "github.com/percona/pmm/managed/utils/envvars" "github.com/percona/pmm/managed/utils/interceptors" platformClient "github.com/percona/pmm/managed/utils/platform" pmmerrors "github.com/percona/pmm/utils/errors" "github.com/percona/pmm/utils/logger" "github.com/percona/pmm/utils/sqlmetrics" "github.com/percona/pmm/version" ) var ( interfaceToBind = envvars.GetInterfaceToBind() gRPCAddr = net.JoinHostPort(interfaceToBind, "7771") http1Addr = net.JoinHostPort(interfaceToBind, "7772") debugAddr = net.JoinHostPort(interfaceToBind, "7773") ) const ( shutdownTimeout = 3 * time.Second gRPCMessageMaxSize = 100 * 1024 * 1024 cleanInterval = 10 * time.Minute cleanOlderThan = 30 * time.Minute defaultContextTimeout = 10 * time.Second pProfProfileDuration = 30 * time.Second pProfTraceDuration = 10 * time.Second clickhouseMaxIdleConns = 5 clickhouseMaxOpenConns = 10 distributionInfoFilePath = "/srv/pmm-distribution" osInfoFilePath = "/proc/version" ) var pprofSemaphore = semaphore.NewWeighted(1) func addLogsHandler(mux *http.ServeMux, logs *server.Logs) { l := logrus.WithField("component", "logs.zip") mux.HandleFunc("/v1/server/logs.zip", func(rw http.ResponseWriter, req *http.Request) { var lineCount int64 contextTimeout := defaultContextTimeout // increase context timeout if pprof query parameter exist in request pprofQueryParameter, err := strconv.ParseBool(req.FormValue("pprof")) if err != nil { l.Debug("Unable to read 'pprof' query param. Using default: pprof=false") } var pprofConfig *server.PprofConfig if pprofQueryParameter { if !pprofSemaphore.TryAcquire(1) { rw.WriteHeader(http.StatusLocked) _, err := rw.Write([]byte("Pprof is already running. Please try again later.")) if err != nil { l.Errorf("%+v", err) } return } defer pprofSemaphore.Release(1) contextTimeout += pProfProfileDuration + pProfTraceDuration pprofConfig = &server.PprofConfig{ ProfileDuration: pProfProfileDuration, TraceDuration: pProfTraceDuration, } } lc := req.FormValue("line-count") if lc != "" { lineCount, err = strconv.ParseInt(lc, 10, 32) if err != nil { l.Error(err) http.Error(rw, fmt.Sprintf("Unable to parse line-count parameter: %s", err), http.StatusBadRequest) return } } // fail-safe ctx, cancel := context.WithTimeout(req.Context(), contextTimeout) defer cancel() filename := fmt.Sprintf("pmm-server_%s.zip", time.Now().UTC().Format("2006-01-02_15-04")) rw.Header().Set(`Access-Control-Allow-Origin`, `*`) rw.Header().Set(`Content-Type`, `application/zip`) rw.Header().Set(`Content-Disposition`, `attachment; filename="`+filename+`"`) ctx = logger.Set(ctx, "logs") if err := logs.Zip(ctx, rw, pprofConfig, int(lineCount)); err != nil { l.Errorf("%+v", err) } }) } type gRPCServerDeps struct { db *reform.DB ha *ha.Service checksService *checks.Service config *config.Config agentsRegistry *agents.Registry handler *agents.Handler actions *agents.ActionsService agentService *agents.AgentService jobsService *agents.JobsService connectionCheck *agents.ConnectionChecker serviceInfoBroker *agents.ServiceInfoBroker agentsStateUpdater *agents.StateUpdater externalExporterStatusSvc *agents.ExternalExporterStatusService grafanaClient *grafana.Client templatesService *alerting.Service backupService *backup.Service dumpService *dump.Service compatibilityService *backup.CompatibilityService backupRemovalService *backup.RemovalService pbmPITRService *backup.PBMPITRService vmClient *metrics.Client minioClient *minio.Client settings *models.Settings platformClient *platformClient.Client schedulerService *scheduler.Service supervisord *supervisord.Service server *server.Server uieventsService *uievents.Service versionCache *versioncache.Service vmdb *victoriametrics.Service vmalert *vmalert.Service } // runGRPCServer runs gRPC server until context is canceled, then gracefully stops it. // //nolint:lll func runGRPCServer(ctx context.Context, deps *gRPCServerDeps) { l := logrus.WithField("component", "gRPC") l.Infof("Starting server on http://%s/ ...", gRPCAddr) grpcMetrics := interceptors.NewServerMetricsWithExtension(&interceptors.GRPCMetricsExtension{}) prom.MustRegister(grpcMetrics) gRPCServer := grpc.NewServer( grpc.MaxRecvMsgSize(gRPCMessageMaxSize), grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer( interceptors.UnaryAdd(grpcMetrics.UnaryServerInterceptor()), interceptors.UnaryServiceEnabledInterceptor(), grpc_validator.UnaryServerInterceptor())), grpc.StreamInterceptor(grpc_middleware.ChainStreamServer( interceptors.Stream(grpcMetrics.StreamServerInterceptor()), interceptors.StreamServiceEnabledInterceptor(), grpc_validator.StreamServerInterceptor())), ) if l.Logger.GetLevel() >= logrus.DebugLevel { l.Debug("Reflection and channelz are enabled.") reflection.Register(gRPCServer) channelz.RegisterChannelzServiceToServer(gRPCServer) l.Debug("RPC response latency histogram enabled.") grpcMetrics.EnableHandlingTimeHistogram() } serverv1.RegisterServerServiceServer(gRPCServer, deps.server) agentv1.RegisterAgentServiceServer(gRPCServer, agentgrpc.NewAgentServer(deps.handler)) agentpb.RegisterAgentServer(gRPCServer, agentgrpc.NewAgentPBServer(deps.handler)) nodesSvc := inventory.NewNodesService(deps.db, deps.agentsRegistry, deps.agentsStateUpdater, deps.vmdb) agentsSvc := inventory.NewAgentsService( deps.db, deps.agentsRegistry, deps.agentsStateUpdater, deps.vmdb, deps.connectionCheck, deps.serviceInfoBroker, deps.agentService) mgmtBackupService := managementbackup.NewBackupsService(deps.db, deps.backupService, deps.compatibilityService, deps.schedulerService, deps.backupRemovalService, deps.pbmPITRService) mgmtRestoreService := managementbackup.NewRestoreService(deps.db, deps.backupService, deps.schedulerService) mgmtServices := common.NewMgmtServices(mgmtBackupService, mgmtRestoreService) servicesSvc := inventory.NewServicesService(deps.db, deps.agentsRegistry, deps.agentsStateUpdater, deps.vmdb, deps.versionCache, mgmtServices) inventoryv1.RegisterNodesServiceServer(gRPCServer, inventorygrpc.NewNodesServer(nodesSvc)) inventoryv1.RegisterServicesServiceServer(gRPCServer, inventorygrpc.NewServicesServer(servicesSvc)) inventoryv1.RegisterAgentsServiceServer(gRPCServer, inventorygrpc.NewAgentsServer(agentsSvc)) managementSvc := management.NewManagementService(deps.db, deps.agentsRegistry, deps.agentsStateUpdater, deps.connectionCheck, deps.serviceInfoBroker, deps.vmdb, deps.versionCache, deps.grafanaClient, v1.NewAPI(*deps.vmClient)) managementv1.RegisterManagementServiceServer(gRPCServer, managementSvc) actionsv1.RegisterActionsServiceServer(gRPCServer, managementgrpc.NewActionsServer(deps.actions, deps.db)) advisorsv1.RegisterAdvisorServiceServer(gRPCServer, management.NewChecksAPIService(deps.checksService)) accesscontrolv1.RegisterAccessControlServiceServer(gRPCServer, management.NewAccessControlService(deps.db)) alertingv1.RegisterAlertingServiceServer(gRPCServer, deps.templatesService) backupv1.RegisterBackupServiceServer(gRPCServer, mgmtBackupService) backupv1.RegisterLocationsServiceServer(gRPCServer, managementbackup.NewLocationsService(deps.db, deps.minioClient)) backupv1.RegisterRestoreServiceServer(gRPCServer, mgmtRestoreService) dumpv1beta1.RegisterDumpServiceServer(gRPCServer, managementdump.New(deps.db, deps.grafanaClient, deps.dumpService)) userv1.RegisterUserServiceServer(gRPCServer, user.NewUserService(deps.db, deps.grafanaClient)) uieventsv1.RegisterUIEventsServiceServer(gRPCServer, deps.uieventsService) hav1beta1.RegisterHAServiceServer(gRPCServer, ha.NewHAServer(deps.ha)) // Register RTA service with in-memory store rtaStore := realtimeanalytics.NewStore() rtaSvc := realtimeanalytics.NewService(deps.db, deps.agentsRegistry, deps.agentsStateUpdater, rtaStore) rtav1.RegisterRealtimeAnalyticsServiceServer(gRPCServer, rtaSvc) rtav1.RegisterCollectorServiceServer(gRPCServer, rtaSvc) // Start RTA store cleanup goroutine go rtaStore.Run(ctx) // run server until it is stopped gracefully or not listener, err := net.Listen("tcp", gRPCAddr) if err != nil { l.Panic(err) } go func() { for { err = gRPCServer.Serve(listener) if err == nil || errors.Is(err, grpc.ErrServerStopped) { break } l.Errorf("Failed to serve: %s", err) } l.Info("Server stopped.") }() <-ctx.Done() // try to stop server gracefully, then not ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) //nolint:contextcheck go func() { <-ctx.Done() gRPCServer.Stop() }() gRPCServer.GracefulStop() cancel() } type http1ServerDeps struct { logs *server.Logs authServer *grafana.AuthServer } // runHTTP1Server runs grpc-gateway and other HTTP 1.1 APIs (like auth_request and logs.zip) // until context is canceled, then gracefully stops it. func runHTTP1Server(ctx context.Context, deps *http1ServerDeps) { l := logrus.WithField("component", "JSON") l.Infof("Starting server on http://%s/ ...", http1Addr) marshaller := &grpc_gateway.JSONPb{ MarshalOptions: protojson.MarshalOptions{ UseEnumNumbers: false, EmitUnpopulated: true, UseProtoNames: true, Indent: " ", }, UnmarshalOptions: protojson.UnmarshalOptions{ DiscardUnknown: true, }, } proxyMux := grpc_gateway.NewServeMux( grpc_gateway.WithMarshalerOption(grpc_gateway.MIMEWildcard, marshaller), grpc_gateway.WithErrorHandler(pmmerrors.PMMHTTPErrorHandler), grpc_gateway.WithRoutingErrorHandler(pmmerrors.PMMRoutingErrorHandler)) opts := []grpc.DialOption{ grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(gRPCMessageMaxSize)), } // Create a shared gRPC connection for handlers that use Register*Handler sharedConn, err := grpc.NewClient(gRPCAddr, opts...) if err != nil { l.Panic(err) } go func() { <-ctx.Done() if err := sharedConn.Close(); err != nil { l.Errorf("Failed to close the shared gRPC connection: %s", err) } }() // Register services using Register*Handler type registrar func(context.Context, *grpc_gateway.ServeMux, *grpc.ClientConn) error for _, r := range []registrar{ serverv1.RegisterServerServiceHandler, inventoryv1.RegisterNodesServiceHandler, inventoryv1.RegisterServicesServiceHandler, inventoryv1.RegisterAgentsServiceHandler, managementv1.RegisterManagementServiceHandler, actionsv1.RegisterActionsServiceHandler, advisorsv1.RegisterAdvisorServiceHandler, accesscontrolv1.RegisterAccessControlServiceHandler, alertingv1.RegisterAlertingServiceHandler, backupv1.RegisterBackupServiceHandler, backupv1.RegisterLocationsServiceHandler, backupv1.RegisterRestoreServiceHandler, dumpv1beta1.RegisterDumpServiceHandler, rtav1.RegisterRealtimeAnalyticsServiceHandler, uieventsv1.RegisterUIEventsServiceHandler, userv1.RegisterUserServiceHandler, hav1beta1.RegisterHAServiceHandler, } { if err := r(ctx, proxyMux, sharedConn); err != nil { l.Panic(err) } } mux := http.NewServeMux() addLogsHandler(mux, deps.logs) mux.Handle("/auth_request", deps.authServer) mux.Handle("/", proxyMux) server := &http.Server{ //nolint:gosec Addr: http1Addr, ErrorLog: log.New(os.Stderr, "runJSONServer: ", 0), Handler: mux, } go func() { if err := server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { l.Panic(err) } l.Info("Server stopped.") }() <-ctx.Done() ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) if err := server.Shutdown(ctx); err != nil { //nolint:contextcheck l.Errorf("Failed to shutdown gracefully: %s", err) } cancel() } // runDebugServer runs debug server until context is canceled, then gracefully stops it. // TODO merge with HTTP1 server? https://jira.percona.com/browse/PMM-4326 func runDebugServer(ctx context.Context) { handler := promhttp.HandlerFor(prom.DefaultGatherer, promhttp.HandlerOpts{ ErrorLog: logrus.WithField("component", "metrics"), ErrorHandling: promhttp.ContinueOnError, }) http.Handle("/debug/metrics", promhttp.InstrumentMetricHandler(prom.DefaultRegisterer, handler)) l := logrus.WithField("component", "debug") handlers := []string{ "/debug/metrics", // by http.Handle above "/debug/vars", // by expvar "/debug/requests", // by golang.org/x/net/trace imported by google.golang.org/grpc "/debug/events", // by golang.org/x/net/trace imported by google.golang.org/grpc "/debug/pprof", // by net/http/pprof } for i, h := range handlers { handlers[i] = "http://" + debugAddr + h } var buf bytes.Buffer err := template.Must(template.New("debug").Parse(` `)).Execute(&buf, handlers) if err != nil { l.Panic(err) } http.HandleFunc("/debug", func(rw http.ResponseWriter, _ *http.Request) { rw.Write(buf.Bytes()) //nolint:errcheck }) l.Infof("Starting server on http://%s/debug\nRegistered handlers:\n\t%s", debugAddr, strings.Join(handlers, "\n\t")) server := &http.Server{ //nolint:gosec Addr: debugAddr, ErrorLog: log.New(os.Stderr, "runDebugServer: ", 0), } go func() { if err := server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { l.Panic(err) } l.Info("Server stopped.") }() <-ctx.Done() ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) if err := server.Shutdown(ctx); err != nil { //nolint:contextcheck l.Errorf("Failed to shutdown gracefully: %s", err) } cancel() } type setupDeps struct { sqlDB *sql.DB ha *ha.Service supervisord *supervisord.Service vmdb *victoriametrics.Service vmalert *vmalert.Service server *server.Server l *logrus.Entry } // setup performs setup tasks that depend on database. func setup(ctx context.Context, deps *setupDeps) bool { l := reform.NewPrintfLogger(deps.l.Debugf) db := reform.NewDB(deps.sqlDB, postgresql.Dialect, l) // log and ignore validation errors; fail on other errors deps.l.Infof("Updating settings...") env := os.Environ() sort.Strings(env) if errs := deps.server.UpdateSettingsFromEnv(ctx, env); len(errs) != 0 { // This should be impossible in the normal workflow. // An invalid environment variable must be caught with pmm-managed-init // and the docker run must be terminated. deps.l.Errorln("Failed to update settings from environment:") for _, e := range errs { deps.l.Errorln(e) } return false } deps.l.Infof("Updating supervisord configuration...") settings, err := models.GetSettings(db.Querier) if err != nil { deps.l.Warnf("Failed to get settings: %s.", err) return false } err = deps.supervisord.UpdateConfiguration(settings) if err != nil { deps.l.Warnf("Failed to update supervisord configuration: %s.", err) return false } deps.l.Infof("Checking VictoriaMetrics...") if err = deps.vmdb.IsReady(ctx); err != nil { deps.l.Warnf("Failed to check VictoriaMetrics readiness: %+v.", err) return false } deps.vmdb.RequestConfigurationUpdate() deps.l.Infof("Checking VMAlert...") if err = deps.vmalert.IsReady(ctx); err != nil { deps.l.Warnf("VMAlert problem: %+v.", err) return false } deps.vmalert.RequestConfigurationUpdate() deps.l.Info("Setup completed.") return true } func getQANClient(ctx context.Context, sqlDB *sql.DB, dbName, qanAPIAddr string) *qan.Client { bc := backoff.DefaultConfig bc.MaxDelay = time.Second opts := []grpc.DialOption{ grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithConnectParams(grpc.ConnectParams{ Backoff: bc, }), grpc.WithUserAgent("pmm-managed/" + version.Version), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(gRPCMessageMaxSize)), } // Without grpc.WithBlock() DialContext returns an error only if something very wrong with address or options; // it does not return an error of connection failure but tries to reconnect in the background. conn, err := grpc.DialContext(ctx, qanAPIAddr, opts...) if err != nil { logrus.Fatalf("Failed to connect QAN API %s: %s.", qanAPIAddr, err) } l := logrus.WithField("component", "reform/qan") reformL := sqlmetrics.NewReform("postgres", dbName+"/qan", l.Tracef) prom.MustRegister(reformL) db := reform.NewDB(sqlDB, postgresql.Dialect, reformL) return qan.NewClient(conn, db) } func migrateDB(ctx context.Context, sqlDB *sql.DB, params models.SetupDBParams) { l := logrus.WithField("component", "migration") params.Logf = l.Debugf params.SetupFixtures = models.SetupFixtures const timeout = 5 * time.Minute timeoutCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() for { select { case <-timeoutCtx.Done(): l.Fatalf("Could not migrate DB: timeout") default: } l.Infof("Migrating database...") _, err := models.SetupDB(timeoutCtx, sqlDB, params) if err == nil { l.Infof("Database migration completed.") return } l.Warnf("Failed to migrate database: %s.", err) time.Sleep(time.Second) } } // newClickhouseDB return a new Clickhouse db. func newClickhouseDB(dsn string, maxIdleConns, maxOpenConns int) (*sql.DB, error) { db, err := sql.Open("clickhouse", dsn) if err != nil { return nil, errors.Wrap(err, "Failed to open connection to QAN DB") } db.SetConnMaxLifetime(0) db.SetMaxIdleConns(maxIdleConns) db.SetMaxOpenConns(maxOpenConns) return db, nil } func main() { //nolint:maintidx,cyclop // empty version breaks much of pmm-managed logic if version.Version == "" { panic("pmm-managed version is not set during build.") } log.SetFlags(0) log.SetPrefix("stdlog: ") kingpin.Version(version.FullInfo()) kingpin.HelpFlag.Short('h') victoriaMetricsURLF := kingpin.Flag("victoriametrics-url", "VictoriaMetrics base URL").Envar("PMM_VM_URL"). Default(models.VMBaseURL).String() victoriaMetricsVMAlertURLF := kingpin.Flag("victoriametrics-vmalert-url", "VictoriaMetrics VMAlert base URL").Envar("PMM_VM_ALERT_URL"). Default("http://127.0.0.1:8880/").String() victoriaMetricsConfigF := kingpin.Flag("victoriametrics-config", "VictoriaMetrics scrape configuration file path"). Default("/etc/victoriametrics-promscrape.yml").String() grafanaAddrF := kingpin.Flag("grafana-addr", "Grafana HTTP API address").Default("127.0.0.1:3000").String() qanAPIAddrF := kingpin.Flag("qan-api-addr", "QAN API gRPC API address").Default("127.0.0.1:9911").String() postgresAddrF := kingpin.Flag("postgres-addr", "PostgreSQL address"). Default(models.DefaultPostgreSQLAddr). Envar("PMM_POSTGRES_ADDR"). String() postgresDBNameF := kingpin.Flag("postgres-name", "PostgreSQL database name"). Default("pmm-managed"). Envar("PMM_POSTGRES_DBNAME"). String() postgresDBUsernameF := kingpin.Flag("postgres-username", "PostgreSQL database username"). Default("pmm-managed"). Envar("PMM_POSTGRES_USERNAME"). String() postgresSSLModeF := kingpin.Flag("postgres-ssl-mode", "PostgreSQL SSL mode"). Default(models.DisableSSLMode). Envar("PMM_POSTGRES_SSL_MODE"). Enum(models.DisableSSLMode, models.RequireSSLMode, models.VerifyCaSSLMode, models.VerifyFullSSLMode) postgresSSLCAPathF := kingpin.Flag("postgres-ssl-ca-path", "PostgreSQL SSL CA root certificate path"). Envar("PMM_POSTGRES_SSL_CA_PATH"). String() postgresDBPasswordF := kingpin.Flag("postgres-password", "PostgreSQL database password"). Default("pmm-managed"). Envar("PMM_POSTGRES_DBPASSWORD"). String() postgresSSLKeyPathF := kingpin.Flag("postgres-ssl-key-path", "PostgreSQL SSL key path"). Envar("PMM_POSTGRES_SSL_KEY_PATH"). String() postgresSSLCertPathF := kingpin.Flag("postgres-ssl-cert-path", "PostgreSQL SSL certificate path"). Envar("PMM_POSTGRES_SSL_CERT_PATH"). String() haEnabled := kingpin.Flag("ha-enable", "Enable HA"). Envar("PMM_HA_ENABLE"). Bool() haNodeID := kingpin.Flag("ha-node-id", "HA Node ID"). Envar("PMM_HA_NODE_ID"). String() haAdvertiseAddress := kingpin.Flag("ha-advertise-address", "HA Advertise address"). Envar("PMM_HA_ADVERTISE_ADDRESS"). String() haPeers := kingpin.Flag("ha-peers", "HA Peers"). Envar("PMM_HA_PEERS"). String() haRaftPort := kingpin.Flag("ha-raft-port", "HA raft port"). Envar("PMM_HA_RAFT_PORT"). Default("9760"). Int() haGossipPort := kingpin.Flag("ha-gossip-port", "HA gossip port"). Envar("PMM_HA_GOSSIP_PORT"). Default("9761"). Int() haGrafanaGossipPort := kingpin.Flag("ha-grafana-gossip-port", "HA Grafana gossip port"). Envar("PMM_HA_GRAFANA_GOSSIP_PORT"). Default("9762"). Int() supervisordConfigDirF := kingpin.Flag("supervisord-config-dir", "Supervisord configuration directory").Required().String() logLevelF := kingpin.Flag("log-level", "Set logging level").Envar("PMM_LOG_LEVEL").Default("info").Enum("trace", "debug", "info", "warn", "error", "fatal") debugF := kingpin.Flag("debug", "Enable debug logging").Envar("PMM_DEBUG").Bool() traceF := kingpin.Flag("trace", "[DEPRECATED] Enable trace logging (implies debug)").Envar("PMM_TRACE").Bool() clickHouseDatabaseF := kingpin.Flag("clickhouse-name", "Clickhouse database name").Default("pmm").Envar("PMM_CLICKHOUSE_DATABASE").String() clickhouseAddrF := kingpin.Flag("clickhouse-addr", "Clickhouse database address").Default("127.0.0.1:9000").Envar("PMM_CLICKHOUSE_ADDR").String() clickhouseUsernameF := kingpin.Flag("clickhouse-username", "Clickhouse database user").Default("default").Envar("PMM_CLICKHOUSE_USER").String() clickhousePasswordF := kingpin.Flag("clickhouse-password", "Clickhouse database user password").Default("clickhouse").Envar("PMM_CLICKHOUSE_PASSWORD").String() watchtowerHostF := kingpin.Flag("watchtower-host", "Watchtower host").Default("http://watchtower:8080").Envar("PMM_WATCHTOWER_HOST").URL() // Nomad garbage collection flags nomadGCIntervalF := kingpin.Flag("nomad-gc-interval", "Interval at which Nomad attempts to garbage collect terminal allocation directories."). Default("1m"). Envar("PMM_NOMAD_GC_INTERVAL"). Duration() nomadGCDiskUsageThresholdF := kingpin.Flag("nomad-gc-disk-usage-threshold", "Disk usage percent which Nomad tries to maintain by garbage collecting terminal allocations."). Default("80"). Envar("PMM_NOMAD_GC_DISK_USAGE_THRESHOLD"). Int() nomadGCInodeUsageThresholdF := kingpin.Flag("nomad-gc-inode-usage-threshold", "Inode usage percent which Nomad tries to maintain by garbage collecting terminal allocations."). Default("70"). Envar("PMM_NOMAD_GC_INODE_USAGE_THRESHOLD"). Int() nomadGCMaxAllocsF := kingpin.Flag("nomad-gc-max-allocs", "Maximum number of allocations which a client will track before triggering a garbage collection of terminal allocations."). Default("50"). Envar("PMM_NOMAD_GC_MAX_ALLOCS"). Int() nomadGCParallelDestroysF := kingpin.Flag("nomad-gc-parallel-destroys", "Maximum number of parallel destroys allowed by the garbage collector."). Default("2"). Envar("PMM_NOMAD_GC_PARALLEL_DESTROYS"). Int() kingpin.Parse() logger.SetupGlobalLogger() level := parseLoggerConfig(*logLevelF, *debugF, *traceF) logrus.SetLevel(level) if level == logrus.TraceLevel { grpclog.SetLoggerV2(&logger.GRPC{Entry: logrus.WithField("component", "grpclog")}) logrus.SetReportCaller(true) } logrus.Infof("Log level: %s.", logrus.GetLevel()) l := logrus.WithField("component", "main") ctx, cancel := context.WithCancel(context.Background()) ctx = logger.Set(ctx, "main") defer l.Info("Done.") var nodes []string if *haPeers != "" { nodes = strings.Split(*haPeers, ",") } haParams := &models.HAParams{ Enabled: *haEnabled, NodeID: *haNodeID, AdvertiseAddress: *haAdvertiseAddress, Nodes: nodes, RaftPort: *haRaftPort, GossipPort: *haGossipPort, GrafanaGossipPort: *haGrafanaGossipPort, } haService := ha.New(haParams) cfg := config.NewService() if err := cfg.Load(); err != nil { l.Panicf("Failed to load config: %+v", err) } // in order to reproduce postgres behaviour. if *postgresSSLModeF == models.RequireSSLMode && *postgresSSLCAPathF != "" { *postgresSSLModeF = models.VerifyCaSSLMode } ds := cfg.Config.Services.Telemetry.DataSources pmmdb := ds.PmmDBSelect pmmdb.Credentials.Username = *postgresDBUsernameF pmmdb.Credentials.Password = *postgresDBPasswordF pmmdb.DSN.Scheme = "postgres" pmmdb.DSN.Host = *postgresAddrF pmmdb.DSN.DB = *postgresDBNameF q := make(url.Values) q.Set("sslmode", *postgresSSLModeF) if *postgresSSLModeF != models.DisableSSLMode { q.Set("sslrootcert", *postgresSSLCAPathF) q.Set("sslcert", *postgresSSLCertPathF) q.Set("sslkey", *postgresSSLKeyPathF) } pmmdb.DSN.Params = q.Encode() grafanadb := ds.GrafanaDBSelect grafanadb.DSN.Scheme = "postgres" grafanadb.DSN.Host = *postgresAddrF grafanadb.DSN.DB = "grafana" grafanadb.DSN.Params = q.Encode() chURI := url.URL{ Scheme: "tcp", User: url.UserPassword(*clickhouseUsernameF, *clickhousePasswordF), Host: *clickhouseAddrF, Path: *clickHouseDatabaseF, } qanDB := ds.QanDBSelect qanDB.DSN = chURI.String() ds.VM.Address = *victoriaMetricsURLF vmParams, err := models.NewVictoriaMetricsParams( models.BasePrometheusConfigPath, *victoriaMetricsURLF) if err != nil { l.Panicf("cannot load victoriametrics params problem: %+v", err) } setupParams := models.SetupDBParams{ Address: *postgresAddrF, Name: *postgresDBNameF, Username: *postgresDBUsernameF, Password: *postgresDBPasswordF, SSLMode: *postgresSSLModeF, SSLCAPath: *postgresSSLCAPathF, SSLKeyPath: *postgresSSLKeyPathF, SSLCertPath: *postgresSSLCertPathF, HANodeID: *haNodeID, HAPeers: nodes, } sqlDB, err := models.OpenDB(setupParams) if err != nil { l.Panicf("Failed to connect to database: %+v", err) } defer sqlDB.Close() //nolint:errcheck if *haEnabled { models.AgentConfigFilePath = "/srv/pmm-agent/config/pmm-agent.yaml" } migrateDB(ctx, sqlDB, setupParams) prom.MustRegister(sqlmetrics.NewCollector("postgres", *postgresDBNameF, sqlDB)) reformL := sqlmetrics.NewReform("postgres", *postgresDBNameF, logrus.WithField("component", "reform").Tracef) prom.MustRegister(reformL) db := reform.NewDB(sqlDB, postgresql.Dialect, reformL) // Generate unique PMM Server ID if it's not already. err = models.SetPMMServerID(db) if err != nil { l.Panicf("failed to set PMM Server ID") } cleaner := clean.New(db) externalRules := vmalert.NewExternalRules() vmdb, err := victoriametrics.NewVictoriaMetrics(*victoriaMetricsConfigF, db, vmParams, haService) if err != nil { l.Panicf("VictoriaMetrics service problem: %+v", err) } vmalert, err := vmalert.NewVMAlert(externalRules, *victoriaMetricsVMAlertURLF) if err != nil { l.Panicf("VictoriaMetrics VMAlert service problem: %+v", err) } prom.MustRegister(vmalert) minioClient := minio.New() qanClient := getQANClient(ctx, sqlDB, *postgresDBNameF, *qanAPIAddrF) agentsRegistry := agents.NewRegistry(db, vmParams, haService) // TODO remove once PMM cluster is Active-Active // TODO kick non-pmm-server agents only // haService.AddLeaderService(ha.NewStandardService( // "agentsRegistry", // func(_ context.Context) error { return nil }, // func() { agentsRegistry.KickAll(ctx) })) pbmPITRService := backup.NewPBMPITRService() backupRemovalService := backup.NewRemovalService(db, pbmPITRService) backupRetentionService := backup.NewRetentionService(db, backupRemovalService) prom.MustRegister(agentsRegistry) inventoryMetrics := inventory.NewInventoryMetrics(db, agentsRegistry) inventoryMetricsCollector := inventory.NewInventoryMetricsCollector(inventoryMetrics) prom.MustRegister(inventoryMetricsCollector) connectionCheck := agents.NewConnectionChecker(agentsRegistry) serviceInfoBroker := agents.NewServiceInfoBroker(agentsRegistry) updater := server.NewUpdater(*watchtowerHostF, gRPCMessageMaxSize, db) logs := server.NewLogs(version.FullInfo(), updater, vmParams) supervisord := supervisord.New( *supervisordConfigDirF, &models.Params{ VMParams: vmParams, PGParams: &models.PGParams{ Addr: *postgresAddrF, DBName: *postgresDBNameF, DBUsername: *postgresDBUsernameF, DBPassword: *postgresDBPasswordF, SSLMode: *postgresSSLModeF, SSLCAPath: *postgresSSLCAPathF, SSLKeyPath: *postgresSSLKeyPathF, SSLCertPath: *postgresSSLCertPathF, }, HAParams: haParams, }) // Keep the agent always running, even on follower nodes. err = supervisord.StartSupervisedService("pmm-agent") if err != nil { l.Warnf("Could not start pmm-agent: %s", err) } platformAddress, err := envvars.GetPlatformAddress() if err != nil { l.Fatal(err) } platformClient, err := platformClient.NewClient(platformAddress) if err != nil { l.Fatalf("Could not create telemetry client: %s", err) } uieventsService := uievents.New() uieventsService.ScheduleCleanup(ctx) telemetryExtensions := map[telemetry.ExtensionType]telemetry.Extension{ telemetry.UIEventsExtension: uieventsService, } dus := distribution.NewService(distributionInfoFilePath, osInfoFilePath, l) telemetry, err := telemetry.NewService(db, platformClient, version.Version, dus, cfg.Config.Services.Telemetry, telemetryExtensions) if err != nil { l.Fatalf("Could not create telemetry service: %s", err) } grafanaClient := grafana.NewClient(*grafanaAddrF) prom.MustRegister(grafanaClient) nomadClientConfig := &models.NomadClient{ GCInterval: *nomadGCIntervalF, GCDiskUsageThreshold: *nomadGCDiskUsageThresholdF, GCInodeUsageThreshold: *nomadGCInodeUsageThresholdF, GCMaxAllocs: *nomadGCMaxAllocsF, GCParallelDestroys: *nomadGCParallelDestroysF, } nomad, err := nomad.New(db, nomadClientConfig) if err != nil { l.Fatalf("Could not create Nomad client: %s", err) } jobsService := agents.NewJobsService(db, agentsRegistry, backupRetentionService) agentsStateUpdater := agents.NewStateUpdater(db, agentsRegistry, vmdb, vmParams, nomad) agentsHandler := agents.NewHandler(db, qanClient, vmdb, agentsRegistry, agentsStateUpdater, jobsService) actionsService := agents.NewActionsService(qanClient, agentsRegistry) vmClient, err := metrics.NewClient(metrics.Config{Address: *victoriaMetricsURLF}) if err != nil { l.Fatalf("Could not create Victoria Metrics client: %s", err) } clickhouseClient, err := newClickhouseDB(qanDB.DSN, clickhouseMaxIdleConns, clickhouseMaxOpenConns) if err != nil { l.Fatalf("Could not create Clickhouse client: %s", err) } externalExporterStatusSvc := agents.NewExternalExporterStatusService(db, v1.NewAPI(vmClient)) checksService := checks.New(db, actionsService, v1.NewAPI(vmClient), clickhouseClient) prom.MustRegister(checksService) alertingService, err := alerting.NewService(db, grafanaClient) if err != nil { l.Fatalf("Could not create alerting service: %s", err) } alertingService.CollectTemplates(ctx) agentService := agents.NewAgentService(agentsRegistry) versioner := agents.NewVersionerService(agentsRegistry) compatibilityService := backup.NewCompatibilityService(db, versioner) backupService := backup.NewService(db, jobsService, agentService, compatibilityService, pbmPITRService) backupMetricsCollector := backup.NewMetricsCollector(db) prom.MustRegister(backupMetricsCollector) schedulerService := scheduler.New(db, backupService) versionCache := versioncache.New(db, versioner) dumpService := dump.New(db, &dump.URLs{ ClickhouseURL: chURI.String(), VMURL: *victoriaMetricsURLF, }) serverParams := &server.Params{ DB: db, VMDB: vmdb, VMAlert: vmalert, AgentsStateUpdater: agentsStateUpdater, ChecksService: checksService, TemplatesService: alertingService, Supervisord: supervisord, TelemetryService: telemetry, GrafanaClient: grafanaClient, VMAlertExternalRules: externalRules, Updater: updater, Dus: dus, HAService: haService, Nomad: nomad, QANClient: qanClient, } server, err := server.NewServer(serverParams) if err != nil { l.Panicf("Server problem: %+v", err) } // handle unix signals terminationSignals := make(chan os.Signal, 1) signal.Notify(terminationSignals, unix.SIGTERM, unix.SIGINT) updateSignals := make(chan os.Signal, 1) signal.Notify(updateSignals, unix.SIGHUP) go func() { for { select { case s := <-terminationSignals: signal.Stop(terminationSignals) l.Warnf("Got %s, shutting down...", unix.SignalName(s.(unix.Signal))) //nolint:forcetypeassert cancel() return case s := <-updateSignals: l.Infof("Got %s, reloading configuration...", unix.SignalName(s.(unix.Signal))) //nolint:forcetypeassert err := server.UpdateConfigurations(ctx) if err != nil { l.Warnf("Couldn't reload configuration: %s", err) } else { l.Info("Configuration reloaded") } } } }() // try synchronously once, then retry in the background deps := &setupDeps{ sqlDB: sqlDB, ha: haService, supervisord: supervisord, vmdb: vmdb, vmalert: vmalert, server: server, l: logrus.WithField("component", "setup"), } if !setup(ctx, deps) { go func() { const delay = 2 * time.Second for { deps.l.Warnf("Retrying in %s.", delay) sleepCtx, sleepCancel := context.WithTimeout(ctx, delay) <-sleepCtx.Done() sleepCancel() if ctx.Err() != nil { return } if setup(ctx, deps) { return } } }() } settings, err := models.GetSettings(sqlDB) if err != nil { l.Fatalf("Failed to get settings: %+v.", err) } authServer := grafana.NewAuthServer(grafanaClient, db) l.Info("Starting services...") var wg sync.WaitGroup wg.Go(func() { authServer.Run(ctx) }) wg.Go(func() { vmalert.Run(ctx) }) wg.Go(func() { vmdb.Run(ctx) }) wg.Add(1) go func() { defer wg.Done() externalExporterStatusSvc.Run(ctx) }() haService.AddLeaderService(ha.NewContextService("checks", func(ctx context.Context) error { checksService.Run(ctx) return nil })) wg.Go(func() { supervisord.Run(ctx) }) wg.Go(func() { updater.Run(ctx) }) wg.Add(1) haService.AddLeaderService(ha.NewContextService("telemetry", func(ctx context.Context) error { defer wg.Done() telemetry.Run(ctx) return nil })) haService.AddLeaderService(ha.NewContextService("scheduler", func(ctx context.Context) error { schedulerService.Run(ctx) return nil })) haService.AddLeaderService(ha.NewContextService("versionCache", func(ctx context.Context) error { versionCache.Run(ctx) return nil })) wg.Go(func() { runGRPCServer(ctx, &gRPCServerDeps{ actions: actionsService, agentService: agentService, agentsRegistry: agentsRegistry, agentsStateUpdater: agentsStateUpdater, externalExporterStatusSvc: externalExporterStatusSvc, backupRemovalService: backupRemovalService, backupService: backupService, checksService: checksService, compatibilityService: compatibilityService, config: &cfg.Config, connectionCheck: connectionCheck, db: db, dumpService: dumpService, grafanaClient: grafanaClient, handler: agentsHandler, ha: haService, jobsService: jobsService, minioClient: minioClient, pbmPITRService: pbmPITRService, platformClient: platformClient, schedulerService: schedulerService, server: server, serviceInfoBroker: serviceInfoBroker, settings: settings, supervisord: supervisord, templatesService: alertingService, uieventsService: uieventsService, versionCache: versionCache, vmalert: vmalert, vmClient: &vmClient, vmdb: vmdb, }) }) wg.Go(func() { runHTTP1Server(ctx, &http1ServerDeps{ logs: logs, authServer: authServer, }) }) wg.Go(func() { runDebugServer(ctx) }) haService.AddLeaderService(ha.NewContextService("cleaner", func(ctx context.Context) error { cleaner.Run(ctx, cleanInterval, cleanOlderThan) return nil })) wg.Add(1) go func() { defer wg.Done() err := haService.Run(ctx) if err != nil { l.Panicf("cannot start high availability service: %+v", err) } }() wg.Wait() } func parseLoggerConfig(level string, debug, trace bool) logrus.Level { if trace { return logrus.TraceLevel } if debug { return logrus.DebugLevel } if level != "" { parsedLevel, err := logrus.ParseLevel(level) if err != nil { logrus.Warnf("Cannot parse logging level: %s, error: %v", level, err) } else { return parsedLevel } } return logrus.InfoLevel }