package management import ( "context" "fmt" "os" "path/filepath" "regexp" "strconv" "strings" "time" "github.com/hyperledger/fabric-lib-go/bccsp/sw" cb "github.com/hyperledger/fabric-protos-go-apiv2/common" ab "github.com/hyperledger/fabric-protos-go-apiv2/orderer" "github.com/hyperledger/fabric-x-common/api/committerpb" "github.com/hyperledger/fabric-x-common/api/msppb" "github.com/hyperledger/fabric-x-common/msp" "github.com/hyperledger/fabric-x-common/protoutil" "github.com/hyperledger/fabric-x-common/tools/configtxlator/update" "github.com/hyperledger/fabric-x-common/tools/pkg/comm" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/emptypb" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" fabricxv1alpha1 "github.com/kfsoftware/fabric-x-operator/api/v1alpha1" "github.com/kfsoftware/fabric-x-operator/internal/controller/utils" ) // sendKnownCertConfigUpdate sends a channel configuration update that adds // certPEM to the specified org's FabricMSPConfig.known_certs. It fetches // the current config from the orderer (falling back to the genesis block), // computes the delta, signs it with the bootstrap admin identity, and // broadcasts it to the first available orderer router. func sendKnownCertConfigUpdate( ctx context.Context, k8s client.Client, caName, namespace, channelID, orgName string, certPEM []byte, ) (string, error) { log.Info("sending config update for known cert", "channel", channelID, "org", orgName, "certSize", len(certPEM)) // 1. Resolve orderer router endpoint and set up admin signer first, // because we need both to fetch the live config and to broadcast. orderer, tlsCA, err := resolveOrdererRouter(ctx, k8s, namespace) if err != nil { return "", fmt.Errorf("resolve orderer router: %w", err) } signer, err := setupAdminSigner(ctx, k8s, caName, namespace) if err != nil { return "", fmt.Errorf("setup admin signer: %w", err) } // 2. Load the current channel config. Prefer the committer sidecar // because its ledger is kept in sync with the orderer and has // accurate version numbers. Fall back to the genesis block secret // if no sidecar is available yet. var oldConfig *cb.Config oldConfig, err = fetchLatestConfigFromSidecar(ctx, k8s, namespace) if err != nil { log.Info("could not fetch live config from sidecar, falling back to genesis block", "error", err) oldConfig, err = loadConfigFromGenesisSecret(ctx, k8s, caName, namespace) if err != nil { return "", fmt.Errorf("load config from genesis secret: %w", err) } } const maxVersionRetries = 10 for attempt := 0; attempt < maxVersionRetries; attempt++ { newConfig := proto.Clone(oldConfig).(*cb.Config) if err := addKnownCert(newConfig, orgName, certPEM); err != nil { return "", fmt.Errorf("add known cert to config: %w", err) } env, err := buildConfigUpdateEnvelope(oldConfig, newConfig, channelID, signer) if err != nil { return "", fmt.Errorf("build config update envelope: %w", err) } txID, err := broadcastToOrderer(ctx, orderer, env, tlsCA) if err == nil { log.Info("config update broadcasted successfully", "orderer", orderer, "channel", channelID, "tx_id", txID, "msp", orgName, "attempt", attempt) return txID, nil } // Parse the error to extract the exact key and current version. // If we cannot parse it, or the key is not found in the config, // give up. adjusted := adjustConfigVersionFromError(oldConfig, err.Error()) if !adjusted { return "", fmt.Errorf("broadcast to orderer %s: %w", orderer, err) } log.Info("config update version mismatch, adjusting key version and retrying", "orderer", orderer, "attempt", attempt+1) } return "", fmt.Errorf("exhausted %d version retries; orderer config versions may have diverged too far from genesis", maxVersionRetries) } // fetchLatestConfigFromSidecar discovers the first CommitterSidecar in the // namespace, dials its gRPC BlockQueryService (port 5050), fetches the latest // config block from the sidecar's local ledger, and extracts the current // channel Config. The sidecar keeps its ledger in sync with the orderer, so // the returned config has accurate version numbers. func fetchLatestConfigFromSidecar(ctx context.Context, k8s client.Client, namespace string) (*cb.Config, error) { // 1. Discover a committer sidecar. scList := &fabricxv1alpha1.CommitterSidecarList{} if err := k8s.List(ctx, scList, client.InNamespace(namespace)); err != nil { return nil, fmt.Errorf("list committer sidecars: %w", err) } if len(scList.Items) == 0 { return nil, fmt.Errorf("no committer sidecars found in namespace %s", namespace) } sc := scList.Items[0] scName := sc.Name // 2. Dial the sidecar gRPC service (plaintext – cluster-internal traffic). addr := fmt.Sprintf("%s-service.%s.svc.cluster.local:5050", scName, namespace) conn, err := grpc.Dial(addr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return nil, fmt.Errorf("dial sidecar %s: %w", addr, err) } defer conn.Close() client := committerpb.NewBlockQueryServiceClient(conn) // 4. Get chain height. info, err := client.GetBlockchainInfo(ctx, &emptypb.Empty{}) if err != nil { return nil, fmt.Errorf("get blockchain info from sidecar: %w", err) } if info.Height == 0 { return nil, fmt.Errorf("sidecar ledger is empty") } // 5. Fetch the latest block. latestBlockNum := info.Height - 1 latestBlock, err := client.GetBlockByNumber(ctx, &committerpb.BlockNumber{Number: latestBlockNum}) if err != nil { return nil, fmt.Errorf("get latest block %d from sidecar: %w", latestBlockNum, err) } // 6. Find the index of the last config block. configIdx, err := protoutil.GetLastConfigIndexFromBlock(latestBlock) if err != nil { return nil, fmt.Errorf("get last config index from latest block: %w", err) } // 7. Fetch the actual config block. configBlock, err := client.GetBlockByNumber(ctx, &committerpb.BlockNumber{Number: configIdx}) if err != nil { return nil, fmt.Errorf("get config block %d from sidecar: %w", configIdx, err) } // 8. Extract cb.Config from the config block (same chain as genesis). return extractConfigFromBlock(configBlock) } // extractConfigFromBlock unmarshals a block that contains a ConfigEnvelope // and returns the embedded cb.Config. func extractConfigFromBlock(block *cb.Block) (*cb.Config, error) { if len(block.Data.Data) == 0 { return nil, fmt.Errorf("block has no data") } env := &cb.Envelope{} if err := proto.Unmarshal(block.Data.Data[0], env); err != nil { return nil, fmt.Errorf("unmarshal envelope: %w", err) } payload := &cb.Payload{} if err := proto.Unmarshal(env.Payload, payload); err != nil { return nil, fmt.Errorf("unmarshal payload: %w", err) } configEnv := &cb.ConfigEnvelope{} if err := proto.Unmarshal(payload.Data, configEnv); err != nil { return nil, fmt.Errorf("unmarshal config envelope: %w", err) } if configEnv.Config == nil { return nil, fmt.Errorf("config envelope has nil Config") } if configEnv.Config.ChannelGroup == nil { return nil, fmt.Errorf("config has nil ChannelGroup") } return configEnv.Config, nil } // loadConfigFromGenesisSecret fetches the genesis block secret and extracts // the embedded channel configuration. func loadConfigFromGenesisSecret(ctx context.Context, k8s client.Client, caName, namespace string) (*cb.Config, error) { secretName := caName + "-genesis-block" secret := &corev1.Secret{} if err := k8s.Get(ctx, types.NamespacedName{Name: secretName, Namespace: namespace}, secret); err != nil { return nil, fmt.Errorf("get secret %s/%s: %w", namespace, secretName, err) } blockKey := "genesis.block" blockData, ok := secret.Data[blockKey] if !ok { return nil, fmt.Errorf("key %s not found in secret %s/%s", blockKey, namespace, secretName) } block := &cb.Block{} if err := proto.Unmarshal(blockData, block); err != nil { return nil, fmt.Errorf("unmarshal genesis block: %w", err) } if len(block.Data.Data) == 0 { return nil, fmt.Errorf("genesis block has no data") } envelope := &cb.Envelope{} if err := proto.Unmarshal(block.Data.Data[0], envelope); err != nil { return nil, fmt.Errorf("unmarshal envelope: %w", err) } payload := &cb.Payload{} if err := proto.Unmarshal(envelope.Payload, payload); err != nil { return nil, fmt.Errorf("unmarshal payload: %w", err) } configEnv := &cb.ConfigEnvelope{} if err := proto.Unmarshal(payload.Data, configEnv); err != nil { return nil, fmt.Errorf("unmarshal config envelope: %w", err) } if configEnv.Config == nil { return nil, fmt.Errorf("genesis block config envelope has nil Config") } if configEnv.Config.ChannelGroup == nil { return nil, fmt.Errorf("genesis block config has nil ChannelGroup") } return configEnv.Config, nil } // addKnownCert appends certPEM to the org's FabricMSPConfig.known_certs in // both the Application and Orderer channel groups. Fabric requires the same // MSPID to have identical config across all groups. func addKnownCert(config *cb.Config, orgName string, certPEM []byte) error { if config == nil || config.ChannelGroup == nil { return fmt.Errorf("config or ChannelGroup is nil") } updated := false if app := config.ChannelGroup.Groups["Application"]; app != nil { if org := app.Groups[orgName]; org != nil { if err := appendCertToMSP(org, certPEM); err != nil { return fmt.Errorf("update Application MSP for %s: %w", orgName, err) } updated = true } } if orderer := config.ChannelGroup.Groups["Orderer"]; orderer != nil { if org := orderer.Groups[orgName]; org != nil { if err := appendCertToMSP(org, certPEM); err != nil { return fmt.Errorf("update Orderer MSP for %s: %w", orgName, err) } updated = true } } if !updated { return fmt.Errorf("org %s not found in Application or Orderer config", orgName) } return nil } // appendCertToMSP mutates the MSP value inside a ConfigGroup by appending // certPEM to FabricMSPConfig.known_certs. func appendCertToMSP(org *cb.ConfigGroup, certPEM []byte) error { mspVal := org.Values["MSP"] if mspVal == nil { return fmt.Errorf("no MSP value") } mspConfig := &msppb.FabricMSPConfig{} if err := proto.Unmarshal(mspVal.Value, mspConfig); err != nil { return fmt.Errorf("unmarshal FabricMSPConfig: %w", err) } mspConfig.KnownCerts = append(mspConfig.KnownCerts, certPEM) b, err := proto.Marshal(mspConfig) if err != nil { return fmt.Errorf("marshal FabricMSPConfig: %w", err) } mspVal.Value = b return nil } // patchWriteSetVersions walks the readSet / writeSet trees bottom-up and // increments writeSet group versions for any group whose contents differ from // the readSet. This works around a bug in fabric-x-common v0.2.6 where // computeGroupUpdate does not set updatedMembers when existing values or // sub-groups are modified, so parent group versions are never incremented. func patchWriteSetVersions(readSet, writeSet *cb.ConfigGroup) bool { changed := false // Values added or modified in writeSet but absent in readSet indicate change. for name := range writeSet.Values { if _, ok := readSet.Values[name]; !ok { changed = true } } // Policies added or modified. for name := range writeSet.Policies { if _, ok := readSet.Policies[name]; !ok { changed = true } } // Recurse into sub-groups. for name, writeGroup := range writeSet.Groups { readGroup, ok := readSet.Groups[name] if !ok { changed = true continue } childChanged := patchWriteSetVersions(readGroup, writeGroup) if childChanged { changed = true } } if changed { writeSet.Version = readSet.Version + 1 } return changed } // buildConfigUpdateEnvelope computes the delta between old and new config, // signs it with the admin identity, and wraps it in an outer Envelope. func buildConfigUpdateEnvelope( oldConfig, newConfig *cb.Config, channelID string, signer msp.SigningIdentity, ) (*cb.Envelope, error) { cu, err := update.Compute(oldConfig, newConfig) if err != nil { return nil, fmt.Errorf("compute config update: %w", err) } cu.ChannelId = channelID cue := &cb.ConfigUpdateEnvelope{ ConfigUpdate: protoutil.MarshalOrPanic(cu), Signatures: []*cb.ConfigSignature{}, } // Sign ConfigUpdateEnvelope with admin identity. sid, err := signer.Serialize() if err != nil { return nil, fmt.Errorf("serialize signer: %w", err) } sigHdr := protoutil.NewSignatureHeaderOrPanic(signer) sigHdr.Creator = sid cs := &cb.ConfigSignature{ SignatureHeader: protoutil.MarshalOrPanic(sigHdr), } toSign := append(cs.SignatureHeader, cue.ConfigUpdate...) cs.Signature, err = signer.Sign(toSign) if err != nil { return nil, fmt.Errorf("sign config update: %w", err) } cue.Signatures = append(cue.Signatures, cs) cueBytes := protoutil.MarshalOrPanic(cue) // Outer envelope (CONFIG_UPDATE). ch := protoutil.MakeChannelHeader(cb.HeaderType_CONFIG_UPDATE, 0, channelID, 0) protoutil.SetTxID(ch, sigHdr) payload := &cb.Payload{ Header: protoutil.MakePayloadHeader(ch, sigHdr), Data: cueBytes, } payloadBytes := protoutil.MarshalOrPanic(payload) sig, err := signer.Sign(payloadBytes) if err != nil { return nil, fmt.Errorf("sign outer envelope: %w", err) } return &cb.Envelope{Payload: payloadBytes, Signature: sig}, nil } // broadcastToOrderer sends the envelope to the orderer via gRPC broadcast and // returns the transaction ID from the broadcast response. func broadcastToOrderer(ctx context.Context, orderer string, env *cb.Envelope, tlsCA []byte) (string, error) { clientConfig := comm.ClientConfig{ DialTimeout: 5 * time.Second, } if len(tlsCA) > 0 { clientConfig.SecOpts = comm.SecureOptions{ UseTLS: true, ServerRootCAs: [][]byte{tlsCA}, } } conn, err := clientConfig.Dial(orderer) if err != nil { return "", fmt.Errorf("dial orderer: %w", err) } defer conn.Close() abc, err := ab.NewAtomicBroadcastClient(conn).Broadcast(ctx) if err != nil { return "", fmt.Errorf("create broadcast client: %w", err) } if err := abc.Send(env); err != nil { return "", fmt.Errorf("send envelope: %w", err) } status, err := abc.Recv() if err != nil { return "", fmt.Errorf("recv status: %w", err) } if status.GetStatus() != cb.Status_SUCCESS { return "", fmt.Errorf("broadcast failed: status=%v info=%s", status.GetStatus(), status.GetInfo()) } return status.GetInfo(), nil } // versionMismatchRe matches the two error patterns the orderer returns for // version mismatches: // 1. WriteSet mismatch: "attempt to set key [Value] /Channel/... to version 1, but key is at version 1" // 2. ReadSet mismatch: "proposed update requires that key [Group] /Channel be at version 1, but it is currently at version 0" var versionMismatchRe = regexp.MustCompile(`(?:attempt to set key|proposed update requires that key) \[(Value|Group|Policy)\]\s+(.+?) (?:to|be at) version \d+, but (?:key is at|it is currently at) version (\d+)`) // adjustConfigVersionFromError parses a Fabric orderer version-mismatch error, // extracts the key path and the current version on the orderer, and updates // that key's Version in config to match. It returns true when the config was // successfully adjusted. func adjustConfigVersionFromError(config *cb.Config, errMsg string) bool { matches := versionMismatchRe.FindStringSubmatch(errMsg) if matches == nil { return false } keyType := matches[1] path := strings.TrimSpace(matches[2]) currentVersion, parseErr := strconv.ParseUint(matches[3], 10, 64) if parseErr != nil { return false } return setConfigVersion(config, keyType, path, currentVersion) } // setConfigVersion walks config.ChannelGroup following path and sets the // target key's Version. path always starts with "/Channel". func setConfigVersion(config *cb.Config, keyType, path string, version uint64) bool { parts := strings.Split(path, "/") // Drop the empty string before the leading "/" var clean []string for _, p := range parts { if p != "" { clean = append(clean, p) } } if len(clean) == 0 || clean[0] != "Channel" || config.ChannelGroup == nil { return false } current := config.ChannelGroup for i := 1; i < len(clean); i++ { isLast := i == len(clean)-1 if isLast { switch keyType { case "Value": if v, ok := current.Values[clean[i]]; ok { v.Version = version return true } case "Policy": if p, ok := current.Policies[clean[i]]; ok { p.Version = version return true } case "Group": if g, ok := current.Groups[clean[i]]; ok { g.Version = version return true } } return false } if g, ok := current.Groups[clean[i]]; ok { current = g } else { return false } } // Path is exactly "/Channel" if keyType == "Group" && len(clean) == 1 { current.Version = version return true } return false } // resolveOrdererRouter lists OrdererGroups in the namespace and returns the // gRPC endpoint of the first party's router service. It also attempts to // return the CA TLS certificate for secure dialing. func resolveOrdererRouter(ctx context.Context, k8s client.Client, namespace string) (string, []byte, error) { ogList := &fabricxv1alpha1.OrdererGroupList{} if err := k8s.List(ctx, ogList, client.InNamespace(namespace)); err != nil { return "", nil, fmt.Errorf("list orderer groups: %w", err) } if len(ogList.Items) == 0 { return "", nil, fmt.Errorf("no orderer groups found in namespace %s", namespace) } og := ogList.Items[0] routerName := og.Name + "-router" orderer := utils.GetServiceFQDNWithSuffix(routerName, "service", namespace) + ":7150" // Try to fetch the CA TLS cert from the CA's tls-crypto secret. // The secret name follows the convention -tls-crypto where // ca-name is derived from the OrdererGroup enrollment settings. var tlsCA []byte if og.Spec.Enrollment != nil && og.Spec.Enrollment.Sign.CA.CATLS != nil { caName := og.Spec.Enrollment.Sign.CA.CAName if caName == "" { caName = og.Spec.Enrollment.Sign.CA.CAHost } secretName := caName + "-tls-crypto" secret := &corev1.Secret{} if err := k8s.Get(ctx, types.NamespacedName{Name: secretName, Namespace: namespace}, secret); err == nil { if data, ok := secret.Data["tls.crt"]; ok { tlsCA = data } } } return orderer, tlsCA, nil } // setupAdminSigner creates a temporary MSP directory from the admin cert // secret (-admin-cert), initializes the MSP, and returns the // default signing identity. func setupAdminSigner(ctx context.Context, k8s client.Client, caName, namespace string) (msp.SigningIdentity, error) { adminSecretName := caName + "-admin-cert" adminSecret := &corev1.Secret{} if err := k8s.Get(ctx, types.NamespacedName{Name: adminSecretName, Namespace: namespace}, adminSecret); err != nil { return nil, fmt.Errorf("get admin secret %s/%s: %w", namespace, adminSecretName, err) } certPEM, ok := adminSecret.Data["cert.pem"] if !ok { return nil, fmt.Errorf("cert.pem not found in admin secret %s", adminSecretName) } keyPEM, ok := adminSecret.Data["key.pem"] if !ok { return nil, fmt.Errorf("key.pem not found in admin secret %s", adminSecretName) } caPEM, ok := adminSecret.Data["cacert.pem"] if !ok { // Fallback: some secrets use ca.pem caPEM, ok = adminSecret.Data["ca.pem"] if !ok { return nil, fmt.Errorf("cacert.pem/ca.pem not found in admin secret %s", adminSecretName) } } // Parse MSPID from the CA or assume from secret labels. mspid := string(adminSecret.Data["msp_id"]) if mspid == "" { mspid = "Org1MSP" // fallback — caller should ensure correct MSPID } tmpDir, err := os.MkdirTemp("", "admin-msp-*") if err != nil { return nil, fmt.Errorf("create temp dir: %w", err) } defer os.RemoveAll(tmpDir) mspPath := filepath.Join(tmpDir, "msp") dirs := []string{ filepath.Join(mspPath, "signcerts"), filepath.Join(mspPath, "keystore"), filepath.Join(mspPath, "cacerts"), } for _, d := range dirs { if err := os.MkdirAll(d, 0755); err != nil { return nil, fmt.Errorf("mkdir %s: %w", d, err) } } if err := os.WriteFile(filepath.Join(mspPath, "signcerts", "cert.pem"), certPEM, 0644); err != nil { return nil, err } if err := os.WriteFile(filepath.Join(mspPath, "keystore", "priv_sk"), keyPEM, 0600); err != nil { return nil, err } if err := os.WriteFile(filepath.Join(mspPath, "cacerts", "ca.pem"), caPEM, 0644); err != nil { return nil, err } configYAML := `NodeOUs: Enable: true ClientOUIdentifier: Certificate: cacerts/ca.pem OrganizationalUnitIdentifier: client PeerOUIdentifier: Certificate: cacerts/ca.pem OrganizationalUnitIdentifier: peer AdminOUIdentifier: Certificate: cacerts/ca.pem OrganizationalUnitIdentifier: admin OrdererOUIdentifier: Certificate: cacerts/ca.pem OrganizationalUnitIdentifier: orderer ` if err := os.WriteFile(filepath.Join(mspPath, "config.yaml"), []byte(configYAML), 0644); err != nil { return nil, err } mspConf, err := msp.GetLocalMspConfig(mspPath, nil, mspid) if err != nil { return nil, fmt.Errorf("get local msp config: %w", err) } dir := filepath.Join(mspPath, "keystore") ks, err := sw.NewFileBasedKeyStore(nil, dir, true) if err != nil { return nil, fmt.Errorf("create keystore: %w", err) } cp, err := sw.NewDefaultSecurityLevelWithKeystore(ks) if err != nil { return nil, fmt.Errorf("create crypto provider: %w", err) } thisMSP, err := msp.New(&msp.BCCSPNewOpts{NewBaseOpts: msp.NewBaseOpts{Version: msp.MSPv1_0}}, cp) if err != nil { return nil, fmt.Errorf("create msp: %w", err) } if err := thisMSP.Setup(mspConf); err != nil { return nil, fmt.Errorf("setup msp: %w", err) } signer, err := thisMSP.GetDefaultSigningIdentity() if err != nil { return nil, fmt.Errorf("get default signing identity: %w", err) } return signer, nil }