// Copyright Istio Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package kubeinject import ( "bytes" "context" "crypto/tls" "crypto/x509" "encoding/json" "errors" "fmt" "io" "net/http" "os" "sort" "strings" "time" "github.com/hashicorp/go-multierror" "github.com/spf13/cobra" admission "k8s.io/api/admission/v1" admissionv1beta1 "k8s.io/api/admission/v1beta1" admissionregistration "k8s.io/api/admissionregistration/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" v1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/kubectl/pkg/polymorphichelpers" "k8s.io/kubectl/pkg/util/podutils" "sigs.k8s.io/yaml" "istio.io/api/label" meshconfig "istio.io/api/mesh/v1alpha1" "istio.io/istio/istioctl/pkg/cli" "istio.io/istio/istioctl/pkg/clioptions" "istio.io/istio/istioctl/pkg/util" "istio.io/istio/operator/pkg/render" "istio.io/istio/pkg/config/mesh" "istio.io/istio/pkg/kube" "istio.io/istio/pkg/kube/inject" "istio.io/istio/pkg/log" "istio.io/istio/pkg/version" ) const ( configMapKey = "mesh" injectConfigMapKey = "config" valuesConfigMapKey = "values" ) type ExternalInjector struct { client kube.CLIClient clientConfig *admissionregistration.WebhookClientConfig injectorAddress string } func (e ExternalInjector) GetKubeClient() kube.Client { return e.client } func (e ExternalInjector) Inject(pod *corev1.Pod, deploymentNS string) ([]byte, error) { cc := e.clientConfig if cc == nil { return nil, nil } var address string if cc.URL != nil { address = *cc.URL } var certPool *x509.CertPool if len(cc.CABundle) > 0 { certPool = x509.NewCertPool() certPool.AppendCertsFromPEM(cc.CABundle) } else { var err error certPool, err = x509.SystemCertPool() if err != nil { return nil, err } } tlsClientConfig := &tls.Config{RootCAs: certPool, MinVersion: tls.VersionTLS12} client := http.Client{ Timeout: time.Second * 5, Transport: &http.Transport{ TLSClientConfig: tlsClientConfig, }, } if cc.Service != nil { svc, err := e.client.Kube().CoreV1().Services(cc.Service.Namespace).Get(context.Background(), cc.Service.Name, metav1.GetOptions{}) if err != nil { return nil, err } namespace, selector, err := polymorphichelpers.SelectorsForObject(svc) if err != nil { if e.injectorAddress == "" { return nil, fmt.Errorf("cannot attach to %T: %v", svc, err) } address = fmt.Sprintf("https://%s:%d%s", e.injectorAddress, *cc.Service.Port, *cc.Service.Path) } else { pod, err := GetFirstPod(e.client.Kube().CoreV1(), namespace, selector.String()) if err != nil { return nil, err } webhookPort := cc.Service.Port podPort := 15017 for _, v := range svc.Spec.Ports { if v.Port == *webhookPort { podPort = v.TargetPort.IntValue() break } } f, err := e.client.NewPortForwarder(pod.Name, pod.Namespace, "", 0, podPort) if err != nil { return nil, err } if err := f.Start(); err != nil { return nil, err } address = fmt.Sprintf("https://%s%s", f.Address(), *cc.Service.Path) defer func() { f.Close() f.WaitForStop() }() } tlsClientConfig.ServerName = fmt.Sprintf("%s.%s.%s", cc.Service.Name, cc.Service.Namespace, "svc") } else if isMCPAddr(address) { var err error client.Transport, err = mcpTransport(context.TODO(), client.Transport) if err != nil { return nil, err } } podBytes, err := json.Marshal(pod) if pod.Namespace != "" { deploymentNS = pod.Namespace } if err != nil { return nil, err } rev := &admission.AdmissionReview{ TypeMeta: metav1.TypeMeta{ APIVersion: admission.SchemeGroupVersion.String(), Kind: "AdmissionReview", }, Request: &admission.AdmissionRequest{ Object: runtime.RawExtension{Raw: podBytes}, Kind: metav1.GroupVersionKind{ Group: admission.GroupName, Version: admission.SchemeGroupVersion.Version, Kind: "AdmissionRequest", }, Resource: metav1.GroupVersionResource{}, SubResource: "", RequestKind: nil, RequestResource: nil, RequestSubResource: "", Name: pod.Name, Namespace: deploymentNS, }, Response: nil, } revBytes, err := json.Marshal(rev) if err != nil { return nil, err } resp, err := client.Post(address, "application/json", bytes.NewBuffer(revBytes)) if err != nil { return nil, err } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } var obj runtime.Object var ar *kube.AdmissionReview out, _, err := deserializer.Decode(body, nil, obj) if err != nil { return nil, fmt.Errorf("could not decode body: %v", err) } ar, err = kube.AdmissionReviewKubeToAdapter(out) if err != nil { return nil, fmt.Errorf("could not decode object: %v", err) } return ar.Response.Patch, nil } var ( runtimeScheme = func() *runtime.Scheme { r := runtime.NewScheme() r.AddKnownTypes(admissionv1beta1.SchemeGroupVersion, &admissionv1beta1.AdmissionReview{}) r.AddKnownTypes(admission.SchemeGroupVersion, &admission.AdmissionReview{}) return r }() codecs = serializer.NewCodecFactory(runtimeScheme) deserializer = codecs.UniversalDeserializer() ) // GetFirstPod returns a pod matching the namespace and label selector // and the number of all pods that match the label selector. // This is forked from polymorphichelpers.GetFirstPod to not watch and instead return an error if no pods are found func GetFirstPod(client v1.CoreV1Interface, namespace string, selector string) (*corev1.Pod, error) { options := metav1.ListOptions{LabelSelector: selector} sortBy := func(pods []*corev1.Pod) sort.Interface { return sort.Reverse(podutils.ActivePods(pods)) } podList, err := client.Pods(namespace).List(context.TODO(), options) if err != nil { return nil, err } pods := make([]*corev1.Pod, 0, len(podList.Items)) for i := range podList.Items { pod := podList.Items[i] pods = append(pods, &pod) } if len(pods) > 0 { sort.Sort(sortBy(pods)) return pods[0], nil } return nil, fmt.Errorf("no pods matching selector %q found in namespace %q", selector, namespace) } func getMeshConfigFromConfigMap(ctx cli.Context, command, revision string) (*meshconfig.MeshConfig, error) { client, err := ctx.CLIClient() if err != nil { return nil, err } if meshConfigMapName == defaultMeshConfigMapName && revision != "" { meshConfigMapName = fmt.Sprintf("%s-%s", defaultMeshConfigMapName, revision) } meshConfigMap, err := client.Kube().CoreV1().ConfigMaps(ctx.IstioNamespace()).Get(context.TODO(), meshConfigMapName, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("could not read valid configmap %q from namespace %q: %v - "+ "Use --meshConfigFile or re-run "+command+" with `-i and ensure valid MeshConfig exists", meshConfigMapName, ctx.IstioNamespace(), err) } // values in the data are strings, while proto might use a // different data type. therefore, we have to get a value by a // key configYaml, exists := meshConfigMap.Data[configMapKey] if !exists { return nil, fmt.Errorf("missing configuration map key %q", configMapKey) } cfg, err := mesh.ApplyMeshConfigDefaults(configYaml) if err != nil { err = multierror.Append(err, fmt.Errorf("istioctl version %s cannot parse mesh config. Install istioctl from the latest Istio release", version.Info.Version)) } return cfg, err } // grabs the raw values from the ConfigMap. These are encoded as JSON. func GetValuesFromConfigMap(ctx cli.Context, revision string) (string, error) { client, err := ctx.CLIClient() if err != nil { return "", err } if revision != "" { injectConfigMapName = fmt.Sprintf("%s-%s", defaultInjectConfigMapName, revision) } meshConfigMap, err := client.Kube().CoreV1().ConfigMaps(ctx.IstioNamespace()).Get(context.TODO(), injectConfigMapName, metav1.GetOptions{}) if err != nil { return "", fmt.Errorf("could not find valid configmap %q from namespace %q: %v - "+ "Use --valuesFile or re-run kube-inject with `-i and ensure istio-sidecar-injector configmap exists", injectConfigMapName, ctx.IstioNamespace(), err) } valuesData, exists := meshConfigMap.Data[valuesConfigMapKey] if !exists { return "", fmt.Errorf("missing configuration map key %q in %q", valuesConfigMapKey, injectConfigMapName) } return valuesData, nil } func readInjectConfigFile(f []byte) (inject.RawTemplates, error) { var injectConfig inject.Config err := yaml.Unmarshal(f, &injectConfig) if err != nil || len(injectConfig.RawTemplates) == 0 { // This must be a direct template, instead of an inject.Config. We support both formats return map[string]string{inject.SidecarTemplateName: string(f)}, nil } cfg, err := inject.UnmarshalConfig(f) if err != nil { return nil, err } return cfg.RawTemplates, err } func getInjectConfigFromConfigMap(ctx cli.Context, revision string) (inject.RawTemplates, error) { client, err := ctx.CLIClient() if err != nil { return nil, err } if injectConfigMapName == defaultInjectConfigMapName && revision != "" { injectConfigMapName = fmt.Sprintf("%s-%s", defaultInjectConfigMapName, revision) } meshConfigMap, err := client.Kube().CoreV1().ConfigMaps(ctx.IstioNamespace()).Get(context.TODO(), injectConfigMapName, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("could not find valid configmap %q from namespace %q: %v - "+ "Use --injectConfigFile or re-run kube-inject with `-i ` and ensure istio-sidecar-injector configmap exists", injectConfigMapName, ctx.IstioNamespace(), err) } // values in the data are strings, while proto might use a // different data type. therefore, we have to get a value by a // key injectData, exists := meshConfigMap.Data[injectConfigMapKey] if !exists { return nil, fmt.Errorf("missing configuration map key %q in %q", injectConfigMapKey, injectConfigMapName) } injectConfig, err := inject.UnmarshalConfig([]byte(injectData)) if err != nil { return nil, fmt.Errorf("unable to convert data from configmap %q: %v", injectConfigMapName, err) } log.Debugf("using inject template from configmap %q", injectConfigMapName) return injectConfig.RawTemplates, nil } func setUpExternalInjector(ctx cli.Context, revision, injectorAddress string) (*ExternalInjector, error) { e := &ExternalInjector{} client, err := ctx.CLIClient() if err != nil { return e, err } if revision == "" { revision = "default" } whcList, err := client.Kube().AdmissionregistrationV1().MutatingWebhookConfigurations().List(context.TODO(), metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", label.IoIstioRev.Name, revision)}) if err != nil { return e, fmt.Errorf("could not find valid mutatingWebhookConfiguration %q from cluster %v", whcName, err) } if whcList != nil && len(whcList.Items) != 0 { for _, wh := range whcList.Items[0].Webhooks { if strings.HasSuffix(wh.Name, defaultWebhookName) { return &ExternalInjector{client, &wh.ClientConfig, injectorAddress}, nil } } } return e, fmt.Errorf("could not find valid mutatingWebhookConfiguration %q from cluster", defaultWebhookName) } func validateFlags() error { var err error if inFilename == "" { err = multierror.Append(err, errors.New("filename not specified (see --filename or -f)")) } if meshConfigFile == "" && meshConfigMapName == "" && iopFilename == "" { err = multierror.Append(err, errors.New("--meshConfigFile or --meshConfigMapName or --operatorFileName must be set")) } return err } func setupKubeInjectParameters(cliContext cli.Context, sidecarTemplate *inject.RawTemplates, valuesConfig *string, revision, injectorAddress string, ) (*ExternalInjector, *meshconfig.MeshConfig, error) { var err error // Get configs from IOP files firstly, and if not exists, get configs from files and configmaps. values, meshConfig, err := getIOPConfigs() if err != nil { return nil, nil, err } if meshConfig == nil { if meshConfigFile != "" { if meshConfig, err = mesh.ReadMeshConfig(meshConfigFile); err != nil { return nil, nil, err } } else { if meshConfig, err = getMeshConfigFromConfigMap(cliContext, "kube-inject", revision); err != nil { return nil, nil, err } } } injector := &ExternalInjector{} if injectConfigFile != "" { injectionConfig, err := os.ReadFile(injectConfigFile) // nolint: vetshadow if err != nil { return nil, nil, err } injectConfig, err := readInjectConfigFile(injectionConfig) if err != nil { return nil, nil, multierror.Append(err, fmt.Errorf("loading --injectConfigFile")) } *sidecarTemplate = injectConfig } else { injector, err = setUpExternalInjector(cliContext, revision, injectorAddress) if err != nil || injector.clientConfig == nil { log.Warnf("failed to get injection config from mutatingWebhookConfigurations %q, will fall back to "+ "get injection from the injection configmap %q : %v", whcName, defaultInjectWebhookConfigName, err) if *sidecarTemplate, err = getInjectConfigFromConfigMap(cliContext, revision); err != nil { return nil, nil, err } } return injector, meshConfig, nil } if values != "" { *valuesConfig = values } if valuesConfig == nil || *valuesConfig == "" { if valuesFile != "" { valuesConfigBytes, err := os.ReadFile(valuesFile) // nolint: vetshadow if err != nil { return nil, nil, err } *valuesConfig = string(valuesConfigBytes) } else if *valuesConfig, err = GetValuesFromConfigMap(cliContext, revision); err != nil { return nil, nil, err } } return injector, meshConfig, err } // getIOPConfigs gets the configs in IOPs. func getIOPConfigs() (string, *meshconfig.MeshConfig, error) { var meshConfig *meshconfig.MeshConfig var valuesConfig string if iopFilename != "" { iop, err := render.MergeInputs([]string{iopFilename}, nil, nil) if err != nil { return "", nil, err } v, _ := iop.GetPathMap("spec.values") valuesConfig = v.JSON() if mc, f := iop.GetPathMap("spec.meshConfig"); f { meshConfig, err = mesh.ApplyMeshConfigDefaults(mc.JSON()) if err != nil { return "", nil, err } } } return valuesConfig, meshConfig, nil } var ( inFilename string outFilename string meshConfigFile string meshConfigMapName string valuesFile string injectConfigFile string injectConfigMapName string whcName string iopFilename string ) const ( defaultMeshConfigMapName = "istio" defaultInjectConfigMapName = "istio-sidecar-injector" defaultInjectWebhookConfigName = "istio-sidecar-injector" defaultWebhookName = "sidecar-injector.istio.io" ) func InjectCommand(cliContext cli.Context) *cobra.Command { var opts clioptions.ControlPlaneOptions var centralOpts clioptions.CentralControlPlaneOptions injectCmd := &cobra.Command{ Use: "kube-inject", Short: "Inject Istio sidecar into Kubernetes pod resources", Long: ` kube-inject manually injects the Istio sidecar into Kubernetes workloads. Unsupported resources are left unmodified so it is safe to run kube-inject over a single file that contains multiple Service, ConfigMap, Deployment, etc. definitions for a complex application. When in doubt re-run istioctl kube-inject on deployments to get the most up-to-date changes. It's best to do kube-inject when the resource is initially created. `, Example: ` # Update resources on the fly before applying. kubectl apply -f <(istioctl kube-inject -f ) # Create a persistent version of the deployment with Istio sidecar injected. istioctl kube-inject -f deployment.yaml -o deployment-injected.yaml # Update an existing deployment. kubectl get deployment -o yaml | istioctl kube-inject -f - | kubectl apply -f - # Capture cluster configuration for later use with kube-inject kubectl -n istio-system get cm istio-sidecar-injector -o jsonpath="{.data.config}" > /tmp/inj-template.tmpl kubectl -n istio-system get cm istio -o jsonpath="{.data.mesh}" > /tmp/mesh.yaml kubectl -n istio-system get cm istio-sidecar-injector -o jsonpath="{.data.values}" > /tmp/values.json # Use kube-inject based on captured configuration istioctl kube-inject -f samples/bookinfo/platform/kube/bookinfo.yaml \ --injectConfigFile /tmp/inj-template.tmpl \ --meshConfigFile /tmp/mesh.yaml \ --valuesFile /tmp/values.json `, RunE: func(c *cobra.Command, _ []string) (err error) { if err = validateFlags(); err != nil { return err } var reader io.Reader if inFilename == "-" { reader = os.Stdin } else { var in *os.File if in, err = os.Open(inFilename); err != nil { return err } reader = in defer func() { if errClose := in.Close(); errClose != nil { log.Errorf("Error: close file from %s, %s", inFilename, errClose) // don't overwrite the previous error if err == nil { err = errClose } } }() } var writer io.Writer if outFilename == "" { writer = c.OutOrStdout() } else { var out *os.File if out, err = os.Create(outFilename); err != nil { return err } writer = out defer func() { if errClose := out.Close(); errClose != nil { log.Errorf("Error: close file from %s, %s", outFilename, errClose) // don't overwrite the previous error if err == nil { err = errClose } } }() } var valuesConfig string var sidecarTemplate inject.RawTemplates var meshConfig *meshconfig.MeshConfig rev := cliContext.RevisionOrDefault(opts.Revision) // if the revision is "default", render templates with an empty revision if rev == util.DefaultRevisionName { rev = "" } injectorAddress := centralOpts.Xds index := strings.IndexByte(injectorAddress, ':') if index != -1 { injectorAddress = injectorAddress[:index] } injector, meshConfig, err := setupKubeInjectParameters(cliContext, &sidecarTemplate, &valuesConfig, rev, injectorAddress) if err != nil { return err } if injector.client == nil && meshConfig == nil { return fmt.Errorf( "failed to get injection config from mutatingWebhookConfigurations and injection configmap - " + "check injection configmap or pass --revision flag", ) } var warnings []string templs, err := inject.ParseTemplates(sidecarTemplate) if err != nil { return err } vc, err := inject.NewValuesConfig(valuesConfig) if err != nil { return err } retval := inject.IntoResourceFile(injector, templs, vc, rev, meshConfig, reader, writer, func(warning string) { warnings = append(warnings, warning) }) if len(warnings) > 0 { fmt.Fprintln(c.ErrOrStderr()) } for _, warning := range warnings { fmt.Fprintln(c.ErrOrStderr(), warning) } return retval }, PersistentPreRunE: func(c *cobra.Command, args []string) error { // istioctl kube-inject is typically redirected to a .yaml file; // the default for log messages should be stderr, not stdout root := c.Root() if root != nil { _ = c.Root().PersistentFlags().Set("log_target", "stderr") } if c.Parent() != nil && c.Parent().PersistentPreRunE != nil { return c.Parent().PersistentPreRunE(c, args) } return nil }, } injectCmd.PersistentFlags().StringVar(&meshConfigFile, "meshConfigFile", "", "Mesh configuration filename. Takes precedence over --meshConfigMapName if set") injectCmd.PersistentFlags().StringVar(&injectConfigFile, "injectConfigFile", "", "Injection configuration filename. Cannot be used with --injectConfigMapName") injectCmd.PersistentFlags().StringVar(&valuesFile, "valuesFile", "", "Injection values configuration filename.") injectCmd.PersistentFlags().StringVarP(&inFilename, "filename", "f", "", "Input Kubernetes resource filename") injectCmd.PersistentFlags().StringVarP(&outFilename, "output", "o", "", "Modified output Kubernetes resource filename") injectCmd.PersistentFlags().StringVar(&iopFilename, "operatorFileName", "", "Path to file containing IstioOperator custom resources. If configs from files like "+ "meshConfigFile, valuesFile are provided, they will be overridden by iop config values.") injectCmd.PersistentFlags().StringVar(&meshConfigMapName, "meshConfigMapName", defaultMeshConfigMapName, fmt.Sprintf("ConfigMap name for Istio mesh configuration, key should be %q", configMapKey)) injectCmd.PersistentFlags().StringVar(&injectConfigMapName, "injectConfigMapName", defaultInjectConfigMapName, fmt.Sprintf("ConfigMap name for Istio sidecar injection, key should be %q.", injectConfigMapKey)) _ = injectCmd.PersistentFlags().MarkHidden("injectConfigMapName") injectCmd.PersistentFlags().StringVar(&whcName, "webhookConfig", defaultInjectWebhookConfigName, "MutatingWebhookConfiguration name for Istio") opts.AttachControlPlaneFlags(injectCmd) centralOpts.AttachControlPlaneFlags(injectCmd) return injectCmd }