package imports import ( "context" "crypto/sha256" "encoding/json" "errors" "fmt" "os" "path/filepath" "sort" "strings" "sync" "time" "github.com/bmatcuk/doublestar/v4" "github.com/hashicorp/go-getter" errUtils "github.com/cloudposse/atmos/errors" "github.com/cloudposse/atmos/pkg/cache" "github.com/cloudposse/atmos/pkg/downloader" "github.com/cloudposse/atmos/pkg/duration" "github.com/cloudposse/atmos/pkg/perf" "github.com/cloudposse/atmos/pkg/policy/guard" "github.com/cloudposse/atmos/pkg/schema" ) const ( // DefaultDownloadTimeout is the timeout for downloading remote imports. defaultDownloadTimeout = 60 * time.Second ) var ( errRemoteSubdirTraversal = errors.New("subdirectory component contains path traversal out of the repository") remoteStackExtensions = []string{".yaml", ".yml", ".yaml.tmpl", ".yml.tmpl"} ) // RemoteImportMatch is a resolved remote stack import ready for local processing. type RemoteImportMatch struct { Path string Key string BasePath string } // RemoteImporter handles downloading stack imports from remote URLs. type RemoteImporter struct { atmosConfig *schema.AtmosConfiguration downloader downloader.FileDownloader cache *cache.FileCache memCache map[string]string // In-memory cache for session. matchCache map[string][]RemoteImportMatch // sessionFetched tracks source repos already fetched in this process so that a // repo shared by many subdir imports is cloned at most once per invocation, // regardless of TTL. Guarded by sourceMu. sessionFetched map[string]bool memMu sync.RWMutex sourceMu sync.Mutex } // RemoteImporterOption is a functional option for configuring RemoteImporter. type RemoteImporterOption func(*RemoteImporter) // WithCache sets a custom FileCache (useful for testing). func WithCache(c *cache.FileCache) RemoteImporterOption { defer perf.Track(nil, "imports.WithCache")() return func(r *RemoteImporter) { r.cache = c } } // WithDownloader sets a custom downloader (useful for testing). func WithDownloader(d downloader.FileDownloader) RemoteImporterOption { defer perf.Track(nil, "imports.WithDownloader")() return func(r *RemoteImporter) { r.downloader = d } } // NewRemoteImporter creates a new RemoteImporter. func NewRemoteImporter(atmosConfig *schema.AtmosConfiguration, opts ...RemoteImporterOption) (*RemoteImporter, error) { defer perf.Track(atmosConfig, "imports.NewRemoteImporter")() // Create default cache using XDG Base Directory Specification. fileCache, err := cache.NewFileCache("stack-imports") if err != nil { return nil, err } // Create default downloader using go-getter. fd := downloader.NewGoGetterDownloader(atmosConfig) r := &RemoteImporter{ atmosConfig: atmosConfig, downloader: fd, cache: fileCache, memCache: make(map[string]string), matchCache: make(map[string][]RemoteImportMatch), sessionFetched: make(map[string]bool), } // Apply options. for _, opt := range opts { opt(r) } return r, nil } // uriToTempName generates a unique temp filename for a URI using SHA256 hashing. func uriToTempName(uri string) string { // codeql[go/weak-sensitive-data-hashing] -- hashes an import URI to derive a // deterministic temp-download filename; never used for credential // verification or storage. hash := sha256.Sum256([]byte(uri)) return fmt.Sprintf(".download.%x", hash[:8]) } // downloadMetaCacheKey derives a FileCache key for a download's freshness metadata, // distinct from the content key (uri) itself so the two never collide. func downloadMetaCacheKey(uri string) string { return "meta:" + uri } // downloadCacheFresh reports whether a previously downloaded uri is still within ttl, // per the freshness metadata written by the last real fetch. An empty ttl, missing // metadata, or an unparsable/expired timestamp are all treated as "not fresh". func (r *RemoteImporter) downloadCacheFresh(uri, ttl string) bool { if ttl == "" { return false } metaBytes, exists, err := r.cache.Get(downloadMetaCacheKey(uri)) if err != nil || !exists { return false } var meta sourceMetadata if err := json.Unmarshal(metaBytes, &meta); err != nil { return false } expired, err := duration.IsExpired(meta.UpdatedAt, ttl) if err != nil || expired { return false } return true } // writeDownloadMeta records that uri was just fetched, for downloadCacheFresh's next check. // A write failure is non-critical: the next Download call will just treat the entry as // stale and re-fetch, matching GetOrFetch's own best-effort cache-write handling. func (r *RemoteImporter) writeDownloadMeta(uri string) { data, err := json.Marshal(sourceMetadata{SourceURI: uri, UpdatedAt: time.Now()}) if err != nil { return } _ = r.cache.Set(downloadMetaCacheKey(uri), data) } // Download fetches a remote import and returns the local path. // Downloads are cached by URL hash to avoid redundant downloads. // Uses file locking and atomic writes for safe concurrent access. // // Honors the same atmosConfig.Imports.TTL default Resolve does. An empty ttl preserves // this cache's historical behavior: once downloaded, a URL is reused indefinitely within // this cache directory (no expiry) until the cache is cleared. A non-empty ttl instead // expires the cached content after ttl, matching the freshness semantics ensureSourceDir // uses for git-subdir imports, so the same setting governs both caching mechanisms. func (r *RemoteImporter) Download(uri string) (result string, resultErr error) { defer perf.Track(nil, "imports.RemoteImporter.Download")() return r.download(context.Background(), uri) } // download is Download for one caller context: the context carries the per-fetch environment of a // named identity (see downloader.WithIdentity). func (r *RemoteImporter) download(ctx context.Context, uri string) (result string, resultErr error) { if err := guard.Source(ctx, "import", uri, ""); err != nil { return "", err } defer func() { if resultErr == nil { guard.RegisterRemote(result, uri) } }() return r.downloadUnguarded(ctx, uri) } func (r *RemoteImporter) downloadUnguarded(ctx context.Context, uri string) (result string, resultErr error) { defer perf.Track(nil, "imports.RemoteImporter.Download")() if !IsRemote(uri) { return "", errUtils.Build(errUtils.ErrInvalidRemoteImport). WithExplanation("URI is not a remote URL"). WithContext("uri", uri). Err() } ttl := "" if r.atmosConfig != nil { ttl = r.atmosConfig.Imports.TTL } if cachedPath := r.cachedDownload(uri); cachedPath != "" { return cachedPath, nil } // A ttl is configured but the cached content (if any) is stale: evict it so // GetOrFetch below treats this as a miss instead of reusing stale content. if ttl != "" && !r.downloadCacheFresh(uri, ttl) { _ = r.cache.Delete(uri) } // Use GetOrFetch to properly handle file locking and atomic writes. // This ensures safe concurrent access across multiple atmos processes. fetched := false _, err := r.cache.GetOrFetch(uri, func() ([]byte, error) { fetched = true return r.fetchDownloadBytes(ctx, uri) }) if err != nil { return "", errUtils.Build(errUtils.ErrDownloadRemoteImport). WithCause(err). WithContext("uri", uri). WithHint("Check network connectivity and verify the URL is accessible"). Err() } // Record freshness only when a real fetch just happened, matching fetchSourceDir's // write-only-on-fetch semantics (not sliding expiration on every cache hit). if fetched && ttl != "" { r.writeDownloadMeta(uri) } // Get the final cached path. destPath, _ := r.cache.GetPath(uri) // Add to memory cache. r.memMu.Lock() r.memCache[uri] = destPath r.memMu.Unlock() return destPath, nil } // Resolve fetches a remote import and returns all local stack files it resolves to. // // Honors the global atmosConfig.Imports.TTL default (the same setting stack-manifest imports // fall back to via ResolveRemoteImportNested) for cross-run reuse of the cloned source repo; an // unset TTL refreshes the clone once per invocation, as before. func (r *RemoteImporter) Resolve(uri string) ([]RemoteImportMatch, error) { defer perf.Track(nil, "imports.RemoteImporter.Resolve")() ttl := "" if r.atmosConfig != nil { ttl = r.atmosConfig.Imports.TTL } return r.resolve(context.Background(), uri, ttl) } // resolve fetches a remote import and returns all local stack files it resolves to. // The ttl controls cross-run reuse of the cloned source repo for git subdir imports // (see ensureSourceDir); an empty ttl refreshes the clone once per invocation. func (r *RemoteImporter) resolve(ctx context.Context, uri, ttl string) (result []RemoteImportMatch, resultErr error) { if err := guard.Source(ctx, "import", uri, ""); err != nil { return nil, err } defer func() { if resultErr == nil { for _, match := range result { guard.RegisterRemote(match.Path, uri) if match.BasePath != "" { guard.RegisterRemote(match.BasePath, uri) } } } }() return r.resolveUnguarded(ctx, uri, ttl) } // resolveUnguarded resolves a remote import to local stack files without the source-policy guard. // Git and oci:// imports with a "//" path are resolved from a shared source directory (file, directory, or // glob); any other remote URI is downloaded as a single file. func (r *RemoteImporter) resolveUnguarded(ctx context.Context, uri, ttl string) (result []RemoteImportMatch, resultErr error) { defer perf.Track(nil, "imports.RemoteImporter.resolve")() if !IsRemote(uri) { return nil, errUtils.Build(errUtils.ErrInvalidRemoteImport). WithExplanation("URI is not a remote URL"). WithContext("uri", uri). Err() } if matches, ok := r.cachedMatches(uri); ok { return matches, nil } isOCI := IsOCIURI(uri) if !isOCI && !IsGitURI(uri) { return r.downloadMatches(ctx, uri) } sourceURI, subdir := getter.SourceDirSubdir(uri) if isOCI && subdir == "" { // An OCI artifact is a set of layers, never a single file, so the file to import // must be selected with the go-getter "//" subdirectory delimiter. return nil, errUtils.Build(errUtils.ErrOCIImportMissingPath). WithContext("uri", uri). WithHint("Append the path of the file inside the artifact, e.g. `oci://ghcr.io/org/stacks:v1//catalog/vpc.yaml`"). Err() } if subdir == "" { return r.downloadMatches(ctx, uri) } matches, err := r.resolveSourceSubdir(ctx, uri, sourceURI, subdir, ttl) if err != nil { return nil, err } r.storeMatches(uri, matches) return matches, nil } // downloadMatches downloads uri as a single file and returns it as the import's one match. func (r *RemoteImporter) downloadMatches(ctx context.Context, uri string) ([]RemoteImportMatch, error) { path, err := r.download(ctx, uri) if err != nil { return nil, err } matches := []RemoteImportMatch{{Path: path, Key: uri}} r.storeMatches(uri, matches) return matches, nil } func (r *RemoteImporter) cachedMatches(uri string) ([]RemoteImportMatch, bool) { r.memMu.RLock() cached, ok := r.matchCache[uri] r.memMu.RUnlock() if !ok { return nil, false } for _, match := range cached { if _, err := os.Stat(match.Path); err == nil { continue } r.memMu.Lock() delete(r.matchCache, uri) for _, stale := range cached { delete(r.memCache, stale.Key) } r.memMu.Unlock() return nil, false } return cloneMatches(cached), true } func (r *RemoteImporter) storeMatches(uri string, matches []RemoteImportMatch) { r.memMu.Lock() defer r.memMu.Unlock() r.matchCache[uri] = cloneMatches(matches) } func cloneMatches(matches []RemoteImportMatch) []RemoteImportMatch { cloned := make([]RemoteImportMatch, len(matches)) copy(cloned, matches) return cloned } // resolveSourceSubdir resolves a git or oci:// import that targets a subdirectory (or glob) within // a repo. It sources files from the shared, deduplicated clone produced by // ensureSourceDir (cloned once per source repo per invocation, persisted across runs // when a TTL is set) rather than re-cloning into a throwaway temp dir per import. The // resolved files are copied into the file cache so the returned paths stay stable and // the default ("local" nested imports) semantics are unchanged. // // Ambient credential brokers (e.g. Atmos Pro github/sts) are provisioned inside // ensureSourceDir, before its detectGitSource call — see the note there. func (r *RemoteImporter) resolveSourceSubdir(ctx context.Context, originalURI, sourceURI, subdir, ttl string) ([]RemoteImportMatch, error) { sourceRoot, err := r.ensureSourceDir(ctx, sourceURI, ttl) if err != nil { return nil, err } files, err := resolveStackFiles(sourceRoot, subdir) if err != nil { return nil, err } matches := make([]RemoteImportMatch, 0, len(files)) for _, file := range files { rel, err := filepath.Rel(sourceRoot, file) if err != nil { return nil, err } key := originalURI + "#" + filepath.ToSlash(rel) cachedPath, err := r.cacheFile(key, file) if err != nil { return nil, err } matches = append(matches, RemoteImportMatch{Path: cachedPath, Key: key}) } return matches, nil } func (r *RemoteImporter) detectGitSource(ctx context.Context, sourceURI string) string { atmosConfig := r.atmosConfig if atmosConfig == nil { atmosConfig = &schema.AtmosConfiguration{} } detector := downloader.NewCustomGitDetectorContext(ctx, atmosConfig, sourceURI) detected, ok, err := detector.Detect(sourceURI, "") if err != nil || !ok { return sourceURI } return detected } // cacheFile copies sourcePath into the file cache and returns the cached path. The cache entry is // content-addressed (see contentAddressedKey): each revision of a file gets its own path, so a stale // copy is never served after the source is re-fetched, and a copy another atmos process is reading is // never rewritten underneath it by a process that fetched a newer revision. func (r *RemoteImporter) cacheFile(key, sourcePath string) (string, error) { r.memMu.RLock() if cachedPath, ok := r.memCache[key]; ok { r.memMu.RUnlock() if _, err := os.Stat(cachedPath); err == nil { return cachedPath, nil } r.memMu.Lock() delete(r.memCache, key) r.memMu.Unlock() } else { r.memMu.RUnlock() } content, err := os.ReadFile(sourcePath) if err != nil { return "", err } storeKey := contentAddressedKey(key, content) if _, err := r.cache.GetOrFetch(storeKey, func() ([]byte, error) { return content, nil }); err != nil { return "", err } destPath, _ := r.cache.GetPath(storeKey) r.memMu.Lock() r.memCache[key] = destPath r.memMu.Unlock() return destPath, nil } // contentAddressedKey derives the file-cache key for one revision of an imported file from its // "uri#path/to/file" import key by inserting a digest of the content into the fragment, e.g. // "uri#/path/to/file.yaml". The file extension is still taken from the fragment, which the // cache needs to preserve the config file type. func contentAddressedKey(key string, content []byte) string { // codeql[go/weak-sensitive-data-hashing] -- content-addresses a cached import file; the digest is // only a cache key and is never used for credential verification or storage. sum := sha256.Sum256(content) digest := fmt.Sprintf("%x", sum[:8]) if idx := strings.Index(key, "#"); idx != -1 { return key[:idx+1] + digest + "/" + key[idx+1:] } return key + "#" + digest } func resolveStackFiles(root, subdir string) ([]string, error) { if subdir == "" { return nil, fmt.Errorf("%w: empty git subdirectory", errUtils.ErrFailedToFindImport) } if hasGlobMeta(subdir) { return resolveGlob(root, subdir) } cleanSubdir, err := cleanRemoteSubdir(subdir) if err != nil { return nil, err } target := filepath.Join(root, filepath.FromSlash(cleanSubdir)) if filepath.Ext(cleanSubdir) == "" { for _, ext := range remoteStackExtensions { candidate := target + ext if info, statErr := os.Stat(candidate); statErr == nil && !info.IsDir() { return []string{candidate}, nil } } } if info, statErr := os.Stat(target); statErr == nil { if info.IsDir() { return resolveDirectory(root, target) } return []string{target}, nil } return nil, errUtils.Build(errUtils.ErrFailedToFindImport). WithContext("pattern", filepath.ToSlash(target)). Err() } func resolveDirectory(root, dir string) ([]string, error) { patterns := make([]string, 0, len(remoteStackExtensions)) rel, err := filepath.Rel(root, dir) if err != nil { return nil, err } rel = filepath.ToSlash(rel) for _, ext := range remoteStackExtensions { patterns = append(patterns, filepath.ToSlash(filepath.Join(rel, "**", "*"+ext))) } return resolveGlobPatterns(root, patterns) } func resolveGlob(root, pattern string) ([]string, error) { cleanPattern, err := cleanRemoteSubdir(pattern) if err != nil { return nil, err } return resolveGlobPatterns(root, []string{cleanPattern}) } func resolveGlobPatterns(root string, patterns []string) ([]string, error) { seen := make(map[string]struct{}) var files []string for _, pattern := range patterns { matches, err := doublestar.Glob(os.DirFS(root), filepath.ToSlash(pattern)) if err != nil { return nil, err } for _, match := range matches { fullPath := filepath.Join(root, filepath.FromSlash(match)) info, err := os.Stat(fullPath) if err != nil || info.IsDir() { continue } rel, err := filepath.Rel(root, fullPath) if err != nil { return nil, err } normalized := filepath.ToSlash(rel) if _, ok := seen[normalized]; ok { continue } seen[normalized] = struct{}{} files = append(files, fullPath) } } if len(files) == 0 { return nil, errUtils.Build(errUtils.ErrFailedToFindImport). WithContext("pattern", strings.Join(patterns, ",")). Err() } sort.Slice(files, func(i, j int) bool { left, _ := filepath.Rel(root, files[i]) right, _ := filepath.Rel(root, files[j]) return filepath.ToSlash(left) < filepath.ToSlash(right) }) return files, nil } func cleanRemoteSubdir(subdir string) (string, error) { cleaned := filepath.ToSlash(filepath.Clean(filepath.FromSlash(strings.TrimPrefix(subdir, "/")))) if cleaned == "." || cleaned == "" { return ".", nil } if strings.HasPrefix(cleaned, "../") || cleaned == ".." { return "", errRemoteSubdirTraversal } return cleaned, nil } func hasGlobMeta(path string) bool { return strings.ContainsAny(path, "*?[") } // ClearCache removes all cached imports. func (r *RemoteImporter) ClearCache() error { defer perf.Track(nil, "imports.RemoteImporter.ClearCache")() r.memMu.Lock() defer r.memMu.Unlock() // Clear in-memory cache. r.memCache = make(map[string]string) r.matchCache = make(map[string][]RemoteImportMatch) r.sourceMu.Lock() r.sessionFetched = make(map[string]bool) r.sourceMu.Unlock() // Clear persistent cache. return r.cache.Clear() } // global remote importer instance (lazily initialized). var ( globalImporter *RemoteImporter globalImporterOnce sync.Once globalImporterErr error ) // getGlobalImporter returns the global RemoteImporter instance. func getGlobalImporter(atmosConfig *schema.AtmosConfiguration) (*RemoteImporter, error) { globalImporterOnce.Do(func() { globalImporter, globalImporterErr = NewRemoteImporter(atmosConfig) }) return globalImporter, globalImporterErr } // DownloadRemoteImport is a convenience function that uses the global importer. func DownloadRemoteImport(atmosConfig *schema.AtmosConfiguration, uri string) (string, error) { defer perf.Track(atmosConfig, "imports.DownloadRemoteImport")() importer, err := getGlobalImporter(atmosConfig) if err != nil { return "", err } return importer.Download(uri) } // ResolveRemoteImport is a convenience function that uses the global importer. func ResolveRemoteImport(atmosConfig *schema.AtmosConfiguration, uri string) ([]RemoteImportMatch, error) { defer perf.Track(atmosConfig, "imports.ResolveRemoteImport")() importer, err := getGlobalImporter(atmosConfig) if err != nil { return nil, err } return importer.Resolve(uri) } func (r *RemoteImporter) cachedDownload(uri string) string { // Check in-memory cache first (session-level cache) -- always wins within one atmos // invocation regardless of ttl, matching ensureSourceDir's within-run dedup. r.memMu.RLock() if cachedPath, ok := r.memCache[uri]; ok { r.memMu.RUnlock() // Verify the file still exists. if _, err := os.Stat(cachedPath); err == nil { return cachedPath } // File was deleted, remove from memory cache and re-download. r.memMu.Lock() delete(r.memCache, uri) r.memMu.Unlock() } else { r.memMu.RUnlock() } return "" } func (r *RemoteImporter) fetchDownloadBytes(ctx context.Context, uri string) ([]byte, error) { // The cache owns locking and the final atomic write. tempPath := filepath.Join(r.cache.BaseDir(), uriToTempName(uri)) defer os.Remove(tempPath) if err := r.fetch(ctx, uri, tempPath, downloader.ClientModeFile); err != nil { return nil, err } return os.ReadFile(tempPath) }