package clone import ( "context" "errors" "fmt" "io" "io/fs" "os" "path/filepath" "strings" ) // settleSubmodules leaves each submodule directory under dst checked out at // its gitlink or empty, as an uninitialized submodule appears, and removes // what remains of submodules HEAD no longer declares. Leftovers are found // from the submodule repositories under .git/modules rather than from what // changed, so a retry after a failed or canceled update finds them again. // Submodules nested deeper than maxDepth are emptied. func settleSubmodules(ctx context.Context, policy Retry, dst string, maxDepth int) error { root, err := os.OpenRoot(dst) if err != nil { return err } defer func() { _ = root.Close() }() return settleLevel(ctx, policy, root, pendingSubmodules{dir: dst, depth: 1}, maxDepth) } func settleLevel(ctx context.Context, policy Retry, root *os.Root, p pendingSubmodules, maxDepth int) error { links, err := submoduleGitlinks(ctx, policy, p.dir) if err != nil { return err } if err := removeLeftovers(ctx, policy, root, p, declaredPaths(links)); err != nil { return err } for _, link := range links { rel := joinSlash(p.prefix, link.path) ok := false if p.depth <= maxDepth { if ok, err = checkedOutAt(ctx, policy, pathFromGit(p.dir, link.path), link.commit); err != nil { return err } } if !ok { if err := emptySubmodule(root, rel); err != nil { return err } continue } child := pendingSubmodules{dir: pathFromGit(p.dir, link.path), prefix: rel, depth: p.depth + 1} if err := settleLevel(ctx, policy, root, child, maxDepth); err != nil { return err } } return nil } // removeLeftovers clears the worktrees of p.dir's submodule repositories that // are not at a declared submodule path. Git leaves a populated submodule // directory in place when a checkout deletes, moves or replaces its gitlink. // Once a worktree is cleared, its repository's core.worktree is unset so a // later update leaves that path alone; Git sets it again if the submodule // returns. func removeLeftovers(ctx context.Context, policy Retry, root *os.Root, p pendingSubmodules, declared map[string]bool) error { worktrees, err := submoduleWorktrees(ctx, policy, p.dir) if err != nil { return err } var leftovers []moduleWorktree var paths []string for _, w := range worktrees { if !declared[w.path] { leftovers = append(leftovers, w) paths = append(paths, w.path) } } if len(leftovers) == 0 { return nil } env := []string{"GIT_LITERAL_PATHSPECS=1"} out, err := policy.Run(ctx, p.dir, env, append([]string{"ls-tree", "-z", "HEAD", "--"}, paths...)...) if err != nil { return gitFailure(ctx, "list submodule leftovers", out, err) } kinds := map[string]string{} for entry := range strings.SplitSeq(strings.TrimSuffix(out, "\x00"), "\x00") { metadata, treePath, ok := strings.Cut(entry, "\t") if fields := strings.Fields(metadata); ok && len(fields) == 3 { kinds[treePath] = fields[1] } } for _, leftover := range leftovers { if err := clearLeftover(ctx, policy, root, p, leftover.path, kinds[leftover.path]); err != nil { return err } out, err := policy.Run(ctx, "", nil, "config", "--file", filepath.Join(leftover.gitDir, "config"), "--unset", "core.worktree") if err != nil { return gitFailure(ctx, "detach stale submodule "+joinSlash(p.prefix, leftover.path), out, err) } } return nil } // clearLeftover removes what remains at the slash path leftover under p.dir, // given the kind of object HEAD now tracks there. func clearLeftover(ctx context.Context, policy Retry, root *os.Root, p pendingSubmodules, leftover, kind string) error { rel := joinSlash(p.prefix, leftover) info, err := lstatBelow(root, rel) if err != nil || info == nil || !info.IsDir() { return err } switch kind { case "": if err := root.RemoveAll(filepath.FromSlash(rel)); err != nil { return fmt.Errorf("remove stale submodule %s: %w", rel, err) } case "commit": return emptySubmodule(root, rel) case "tree": // git clean keeps the tracked files but not the old gitfile, which // Git ignores inside a tracked directory. out, err := policy.Run(ctx, p.dir, []string{"GIT_LITERAL_PATHSPECS=1"}, "clean", "-ffdxq", "--", leftover) if err != nil { return gitFailure(ctx, "clean stale submodule "+rel, out, err) } if err := root.RemoveAll(filepath.FromSlash(rel + "/.git")); err != nil { return fmt.Errorf("remove stale submodule %s: %w", rel, err) } } return nil } func declaredPaths(links []gitlink) map[string]bool { declared := make(map[string]bool, len(links)) for _, link := range links { declared[link.path] = true } return declared } type moduleWorktree struct { gitDir string path string // slash path relative to the checkout that owns gitDir } // submoduleWorktrees returns where the submodule repositories under dir's Git // directory are checked out. func submoduleWorktrees(ctx context.Context, policy Retry, dir string) ([]moduleWorktree, error) { gitDir, ok := checkoutGitDir(dir) if !ok { return nil, nil } base, err := filepath.Abs(dir) if err != nil { return nil, err } modules := filepath.Join(gitDir, "modules") var worktrees []moduleWorktree err = filepath.WalkDir(modules, func(path string, entry fs.DirEntry, err error) error { if err != nil { if path == modules && errors.Is(err, fs.ErrNotExist) { return nil } return err } if !entry.IsDir() || !isGitDir(path) { return nil } out, err := policy.Run(ctx, "", nil, "config", "-z", "--file", filepath.Join(path, "config"), "--get", "core.worktree") if err != nil && gitExitCode(err) != 1 { return gitFailure(ctx, "read submodule worktree", out, err) } if err == nil { if w, ok := worktreeBelow(base, path, strings.TrimSuffix(out, "\x00")); ok { worktrees = append(worktrees, moduleWorktree{gitDir: path, path: w}) } } // Nested submodules belong to the submodule's own level. return filepath.SkipDir }) return worktrees, err } // checkoutGitDir returns the Git directory of the checkout in dir, following // a gitfile. func checkoutGitDir(dir string) (string, bool) { dotGit := filepath.Join(dir, ".git") info, err := os.Lstat(dotGit) if err != nil { return "", false } if info.IsDir() { return dotGit, true } if !info.Mode().IsRegular() { return "", false } content, err := os.ReadFile(dotGit) if err != nil { return "", false } // Git strips only the line ending; spaces belong to the path. gitDir, ok := strings.CutPrefix(strings.TrimRight(string(content), "\r\n"), "gitdir: ") if !ok { return "", false } if !filepath.IsAbs(gitDir) { gitDir = filepath.Join(dir, gitDir) } return gitDir, true } func isGitDir(path string) bool { for _, name := range []string{"HEAD", "config"} { info, err := os.Lstat(filepath.Join(path, name)) if err != nil || !info.Mode().IsRegular() { return false } } return true } // worktreeBelow resolves a core.worktree value against gitDir and returns it // as a slash path under base. func worktreeBelow(base, gitDir, worktree string) (string, bool) { if worktree == "" { return "", false } if !filepath.IsAbs(worktree) { worktree = filepath.Join(gitDir, worktree) } abs, err := filepath.Abs(worktree) if err != nil { return "", false } rel, err := filepath.Rel(base, abs) if err != nil { return "", false } return SanitizePath(filepath.ToSlash(rel)) } // emptySubmodule replaces the directory at the slash path rel with an empty // one. The submodule's repository under .git/modules is kept for a later // update. func emptySubmodule(root *os.Root, rel string) error { info, err := lstatBelow(root, rel) if err != nil || info == nil { return err } name := filepath.FromSlash(rel) if info.IsDir() { dir, err := root.Open(name) if err != nil { return fmt.Errorf("remove stale submodule %s: %w", rel, err) } _, err = dir.ReadDir(1) _ = dir.Close() if errors.Is(err, io.EOF) { return nil } } if err := root.RemoveAll(name); err != nil { return fmt.Errorf("remove stale submodule %s: %w", rel, err) } if err := root.Mkdir(name, dirPerm); err != nil { return fmt.Errorf("remove stale submodule %s: %w", rel, err) } return nil } // lstatBelow returns the entry at the slash path rel, or nil when it is // absent or reached through anything but real directories. os.Root follows a // symlink that stays inside the root, which could otherwise point a removal // at .git. func lstatBelow(root *os.Root, rel string) (fs.FileInfo, error) { parts := strings.Split(rel, "/") for i := 1; i <= len(parts); i++ { info, err := root.Lstat(filepath.FromSlash(strings.Join(parts[:i], "/"))) if errors.Is(err, fs.ErrNotExist) { return nil, nil } if err != nil { return nil, fmt.Errorf("inspect stale submodule %s: %w", rel, err) } if i == len(parts) { return info, nil } if !info.IsDir() { return nil, nil } } return nil, nil } func gitFailure(ctx context.Context, action, out string, err error) error { if ctxErr := ctx.Err(); ctxErr != nil { return ctxErr } return fmt.Errorf("%s: %s: %w", action, redactOutput(out), err) }