package downloader
import (
"context"
"fmt"
"net/url"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
errUtils "github.com/cloudposse/atmos/errors"
"github.com/cloudposse/atmos/pkg/github"
log "github.com/cloudposse/atmos/pkg/logger"
"github.com/cloudposse/atmos/pkg/perf"
"github.com/cloudposse/atmos/pkg/policy/guard"
"github.com/cloudposse/atmos/pkg/schema"
)
const schemeSeparator = "://"
// CustomGitDetector intercepts Git URLs (for GitHub, Bitbucket, GitLab, etc.)
// and transforms them into a proper URL for cloning, optionally injecting tokens.
type CustomGitDetector struct {
ctx context.Context
atmosConfig *schema.AtmosConfiguration
source string
}
// NewCustomGitDetector creates a new CustomGitDetector with the provided configuration and source URL.
func NewCustomGitDetector(atmosConfig *schema.AtmosConfiguration, source string) *CustomGitDetector {
return NewCustomGitDetectorContext(context.Background(), atmosConfig, source)
}
// NewCustomGitDetectorContext propagates caller cancellation to credential lookup.
func NewCustomGitDetectorContext(ctx context.Context, atmosConfig *schema.AtmosConfiguration, source string) *CustomGitDetector {
defer perf.Track(atmosConfig, "downloader.NewCustomGitDetectorContext")()
return &CustomGitDetector{
ctx: ctx,
atmosConfig: atmosConfig,
source: source,
}
}
// Detect implements the getter.Detector interface for go-getter v1.
func (d *CustomGitDetector) Detect(src, _ string) (string, bool, error) {
log.Debug("CustomGitDetector.Detect called")
if len(src) == 0 {
return "", false, nil
}
// Ensure the URL has an explicit scheme.
src = d.ensureScheme(src)
// Parse the URL to extract the host and path.
parsedURL, err := url.Parse(src)
if err != nil {
maskedSrc, _ := maskBasicAuth(src)
log.Debug("Failed to parse URL", keyURL, maskedSrc, "error", err)
return "", false, fmt.Errorf("%w: %q: %w", errUtils.ErrParseURL, maskedSrc, err)
}
// If no host is detected, this is likely a local file path.
// Skip custom processing so that go getter handles it as is.
if parsedURL.Host == "" {
log.Debug("No host detected in URL, skipping custom git detection", keyURL, src)
return "", false, nil
}
if err := guard.Source(d.ctx, "download", parsedURL.String(), ""); err != nil {
return "", false, err
}
// Normalize the path.
d.normalizePath(parsedURL)
// Adjust host check to support GitHub, Bitbucket, GitLab, etc.
// Use Hostname() to extract just the hostname without port (e.g., "github.com:22" → "github.com")
// for the provider-specific (github.com/gitlab.com/bitbucket.org) comparisons below. rawHost
// keeps the port: it is passed to the configured-GHES check (isConfiguredGitHubHost) instead,
// since a GHES host reachable only on a non-default port (e.g. "ghes.example.com:8443") would
// otherwise never match its own RepoEndpoints.Host, which also keeps that port (see
// github.Endpoints.Host's doc comment).
rawHost := parsedURL.Host
host := strings.ToLower(parsedURL.Hostname())
if !isSupportedHost(host, rawHost) {
log.Debug("Skipping token injection for an unsupported host", keyHost, rawHost)
return "", false, nil
}
d.normalizeRepositorySubdirPath(parsedURL)
// Check if token injection is enabled for this host and inject if appropriate.
// atmosConfig may be nil (e.g. callers that don't need token injection), in which
// case there is nothing to inject.
if d.atmosConfig != nil && shouldInjectTokenForHost(host, rawHost, &d.atmosConfig.Settings) {
log.Debug("Token injection enabled for host", keyHost, rawHost)
d.injectToken(parsedURL, host, rawHost)
} else {
log.Debug("Token injection disabled for host", keyHost, rawHost)
}
// Note: URI normalization (including adding //.) is now handled by normalizeVendorURI
// in the vendor processing pipeline, so we don't need to adjust subdirectory here
// Set "depth=1" for a shallow clone if not specified.
q := parsedURL.Query()
if _, exists := q["depth"]; !exists {
q.Set("depth", "1")
}
parsedURL.RawQuery = q.Encode()
finalURL := "git::" + parsedURL.String()
maskedFinal, err := maskBasicAuth(strings.TrimPrefix(finalURL, "git::"))
if err != nil {
log.Debug("Masking failed", "error", err)
} else {
log.Debug("Final transformation", "url", "git::"+maskedFinal)
}
return finalURL, true, nil
}
const (
// Named constants for regex match indices.
matchIndexUser = 1
matchIndexHost = 3
matchIndexPath = 4
matchIndexSuffix = 5
matchIndexExtra = 6
keyURL = "url"
keyHost = "host"
hostGitHub = "github.com"
hostGitLab = "gitlab.com"
hostBitbucket = "bitbucket.org"
)
const GitPrefix = "git::"
// isConfiguredGitHubHost reports whether host is public github.com or the GitHub Enterprise
// Server host configured via GITHUB_SERVER_URL, so token injection, default-username
// selection, and host support all recognize a GHES host exactly like github.com. Host is the
// already-lowercased, portless hostname (see Detect's strings.ToLower(parsedURL.Hostname()))
// used for the literal "github.com" comparison, which is intentionally case-sensitive,
// matching isSupportedHost's existing contract. RawHost is the full authority including any
// port (parsedURL.Host) and is used only for the GHES IsHost check: RepoEndpoints.Host keeps a
// non-default port (see its doc comment), so a GHES host reachable only on a non-default port
// (e.g. "ghes.example.com:8443") would otherwise never match its own configured URL if compared
// with the portless host instead. RepoEndpoints is only consulted when GHES is actually
// configured (endpoints.Host differs from the default): RepoEndpoints.IsHost normalizes case
// internally, and since it defaults to "github.com" when unset, calling it unconditionally
// would silently make the literal comparison case-insensitive too, which the pinned tests below
// reject.
func isConfiguredGitHubHost(host, rawHost string) bool {
if host == hostGitHub {
return true
}
endpoints := github.RepoEndpoints()
return endpoints.Host != hostGitHub && endpoints.IsHost(rawHost)
}
// isSupportedHost checks if the host is a supported Git hosting provider.
// This is a pure function that can be easily tested.
func isSupportedHost(host, rawHost string) bool {
return isConfiguredGitHubHost(host, rawHost) || host == hostBitbucket || host == hostGitLab
}
// shouldInjectTokenForHost checks if token injection is enabled for the given host.
// This is a pure function that encapsulates the logic of checking inject settings per host.
func shouldInjectTokenForHost(host, rawHost string, settings *schema.AtmosSettings) bool {
switch {
case isConfiguredGitHubHost(host, rawHost):
return settings.InjectGithubToken
case host == hostBitbucket:
return settings.InjectBitbucketToken
case host == hostGitLab:
return settings.InjectGitlabToken
default:
return false
}
}
// needsTokenInjection checks if a URL needs token injection.
// A URL needs token injection if it doesn't already have user credentials.
// This is a pure function that can be easily tested.
func needsTokenInjection(parsedURL *url.URL) bool {
return parsedURL.User == nil
}
// ensureScheme checks for an explicit scheme and rewrites SCP-style URLs if needed.
// Also removes any existing "git::" prefix (required for the dry-run mode to operate correctly).
func (d *CustomGitDetector) ensureScheme(src string) string {
// Strip any existing "git::" prefix
src = strings.TrimPrefix(src, GitPrefix)
if !strings.Contains(src, schemeSeparator) {
if newSrc, rewritten := rewriteSCPURL(src); rewritten {
maskedOld, _ := maskBasicAuth(src)
maskedNew, _ := maskBasicAuth(newSrc)
log.Debug("Rewriting SCP-style SSH URL", "old_url", maskedOld, "new_url", maskedNew)
return newSrc
}
src = "https://" + src
maskedSrc, _ := maskBasicAuth(src)
log.Debug("Defaulting to https scheme", keyURL, maskedSrc)
}
return src
}
func rewriteSCPURL(src string) (string, bool) {
scpPattern := regexp.MustCompile(`^(([\w.-]+)@)?([\w.-]+\.[\w.-]+):([\w./-]+)(\.git)?(.*)$`)
if scpPattern.MatchString(src) {
matches := scpPattern.FindStringSubmatch(src)
newSrc := "ssh://"
user := matches[matchIndexUser] // This includes the "@" if present.
host := matches[matchIndexHost]
// Only for SSH vendoring (i.e. when rewriting an SCP URL), inject default username (git)
// for known hosts. Unlike isConfiguredGitHubHost's other call sites, host here comes
// straight from the regex match (not pre-lowercased), so the github.com comparison
// stays case-insensitive via EqualFold; RepoEndpoints.Hostname() normalizes case itself.
// SCP syntax ("[user@]host:path") has no way to carry a port, so a configured GHES host
// on a non-default port (e.g. "ghe.example.com:8443") must be compared against
// RepoEndpoints().Hostname() (portless) rather than IsHost (which keeps the port and
// would never match).
if user == "" && (strings.EqualFold(host, hostGitHub) || strings.EqualFold(host, github.RepoEndpoints().Hostname()) ||
strings.EqualFold(host, hostGitLab) ||
strings.EqualFold(host, hostBitbucket)) {
user = "git@"
}
newSrc += user + host + "/" + matches[matchIndexPath]
if matches[matchIndexSuffix] != "" {
newSrc += matches[matchIndexSuffix]
}
if matches[matchIndexExtra] != "" {
newSrc += matches[matchIndexExtra]
}
return newSrc, true
}
return "", false
}
// normalizePath converts the URL path to use forward slashes.
func (d *CustomGitDetector) normalizePath(parsedURL *url.URL) {
unescapedPath, err := url.PathUnescape(parsedURL.Path)
if err == nil {
parsedURL.Path = filepath.ToSlash(unescapedPath)
} else {
parsedURL.Path = filepath.ToSlash(parsedURL.Path)
}
}
// normalizeRepositorySubdirPath makes go-getter repository subdir imports unambiguous.
// Native Atmos imports commonly use `github.com/org/repo//path/file.yaml`. Without the
// explicit `.git` suffix, go-getter can split the source at the wrong `//` and try to clone
// `github.com/org` instead of `github.com/org/repo`.
func (d *CustomGitDetector) normalizeRepositorySubdirPath(parsedURL *url.URL) {
repoPath, subdir, ok := strings.Cut(parsedURL.Path, "//")
if !ok || subdir == "" || strings.HasSuffix(repoPath, ".git") {
return
}
segments := strings.Split(strings.Trim(repoPath, "/"), "/")
if len(segments) < 2 {
return
}
segments[len(segments)-1] += ".git"
parsedURL.Path = "/" + strings.Join(segments, "/") + "//" + subdir
}
// injectToken injects a token into the URL if available.
// User-specified credentials in the URL always take precedence over automatic injection. Host
// is the lowercased, portless hostname; rawHost is the full authority (with port, if any),
// forwarded to isConfiguredGitHubHost so a GHES host on a non-default port is still recognized.
func (d *CustomGitDetector) injectToken(parsedURL *url.URL, host, rawHost string) {
// If URL already has user credentials, respect them and skip injection. This is checked
// first: it is the most specific condition and applies regardless of scheme (an
// "ssh://git@..." source carries its own user, for example).
if !needsTokenInjection(parsedURL) {
maskedURL, _ := maskBasicAuth(parsedURL.String())
log.Debug("Skipping token injection: URL already has user credentials", keyURL, maskedURL)
return
}
// Only inject over HTTPS: the token is embedded as URL userinfo, so an "http://" (or any
// other non-https) scheme would send it in cleartext. This matters most for a
// GITHUB_SERVER_URL/GHES host explicitly configured with a non-https scheme, but applies
// equally to any manually-typed "http://" source for github.com, gitlab.com, or bitbucket.org.
if parsedURL.Scheme != "https" {
log.Debug("Skipping token injection: URL scheme is not https", "scheme", parsedURL.Scheme, keyHost, host)
return
}
// If an auth broker (e.g. github/sts) has already exported a GIT_CONFIG_* insteadOf rewrite that
// matches this host/owner, skip URL injection: injecting userinfo here would prevent git's
// insteadOf left-side from matching, silently shadowing the broker's correct minted token.
if brokerInsteadOfMatchesURLContext(d.ctx, parsedURL, host) {
maskedURL, _ := maskBasicAuth(parsedURL.String())
log.Debug("Skipping token injection: a broker GIT_CONFIG insteadOf already covers this host/owner; letting git's rewrite win", keyURL, maskedURL)
return
}
token, tokenSource := d.resolveToken(host, rawHost)
if token != "" {
defaultUsername := d.getDefaultUsername(host, rawHost)
parsedURL.User = url.UserPassword(defaultUsername, token)
maskedURL, _ := maskBasicAuth(parsedURL.String())
log.Debug("Injected token", "env", tokenSource, keyURL, maskedURL)
} else {
log.Debug("No token found for injection", keyHost, host)
}
}
// brokerInsteadOfMatchesURLContext reports whether a live GIT_CONFIG_* insteadOf rewrite (exported by an
// auth broker, e.g. github/sts) would match this URL's host and owner — "match" being git's own
// term for insteadOf: any URL beginning with the rewrite's value is rewritten (git help config,
// url..insteadOf). When it matches, URL token injection must be skipped so git's rewrite —
// carrying the correct minted token — wins instead of being shadowed by userinfo in the URL.
//
// It is evaluated per fetch (the ctx parameter): it reads GIT_CONFIG_* from the fetch's per-fetch
// environment (see WithEnvironment) first, then from the process environment. A named identity's `insteadOf` rewrite is exported to its Git subprocesses through
// that per-fetch environment, so it must be visible here too or URL token injection would shadow it.
func brokerInsteadOfMatchesURLContext(ctx context.Context, parsedURL *url.URL, host string) bool {
count, err := strconv.Atoi(fetchEnvironmentValue(ctx, "GIT_CONFIG_COUNT"))
if err != nil || count <= 0 {
return false
}
owner := firstPathSegment(parsedURL.Path)
if owner == "" {
return false
}
for i := 0; i < count; i++ {
if gitConfigInsteadOfMatches(ctx, i, host, owner) {
return true
}
}
return false
}
// gitConfigInsteadOfMatches reports whether the i-th GIT_CONFIG_* entry is an https `insteadOf`
// rewrite whose host and owner match. It covers both broker materialization modes:
// - env mode: an inline `url..insteadOf` key whose value is the rewrite target.
// - file mode: an `include.path` key pointing at a gitconfig file whose `[url ...] insteadOf`
// rewrites live inside the file (so the inline-key check below can never see them).
//
// Only an https rewrite matches an https clone — an ssh-only rewrite must not suppress injection.
func gitConfigInsteadOfMatches(ctx context.Context, i int, host, owner string) bool {
idx := strconv.Itoa(i)
key := fetchEnvironmentValue(ctx, "GIT_CONFIG_KEY_"+idx)
value := fetchEnvironmentValue(ctx, "GIT_CONFIG_VALUE_"+idx)
// File mode: the rewrites are inside the included gitconfig, not in this env entry.
if key == gitConfigIncludePath {
return fileInsteadOfMatches(value, host, owner)
}
// Env mode: an inline `url..insteadOf` rewrite whose value is the target URL.
if !strings.HasPrefix(key, "url.") || !strings.HasSuffix(key, ".insteadOf") {
return false
}
return insteadOfValueMatches(value, host, owner)
}
// gitConfigIncludePath is the GIT_CONFIG key the broker emits in file mode (GitConfigModeFile) to
// pull in the on-disk gitconfig that carries the `insteadOf` rewrites.
const gitConfigIncludePath = "include.path"
// insteadOfValueMatches reports whether an `insteadOf` value is an https rewrite target whose host
// and owner match. Only https matches an https clone — an ssh-only rewrite must not suppress injection.
func insteadOfValueMatches(value, host, owner string) bool {
insteadOf, err := url.Parse(value)
if err != nil || insteadOf.Scheme != "https" {
return false
}
return strings.ToLower(insteadOf.Hostname()) == host && firstPathSegment(insteadOf.Path) == owner
}
// fileInsteadOfMatches scans the gitconfig referenced by a GIT_CONFIG include.path entry for an
// https `insteadOf` rewrite whose host and owner match. File mode keeps the broker's rewrites off
// the environment and inside this file, so the inline-key guard cannot see them; without this the
// guard would always miss a file-mode rewrite and let ambient injection shadow the minted token.
func fileInsteadOfMatches(path, host, owner string) bool {
if path == "" {
return false
}
// The path is the broker-exported GIT_CONFIG include.path (trusted config, mirrors the other
// broker-env reads here), not user input.
data, err := os.ReadFile(path) //nolint:gosec // G703: path is the trusted broker include.path.
if err != nil {
return false
}
for _, line := range strings.Split(string(data), "\n") {
value, ok := cutInsteadOfDirective(strings.TrimSpace(line))
if !ok {
continue
}
if insteadOfValueMatches(value, host, owner) {
return true
}
}
return false
}
// cutInsteadOfDirective extracts the value of a gitconfig `insteadOf = ` line, or reports
// false when the line is not an insteadOf directive. Git config keys are case-insensitive.
func cutInsteadOfDirective(line string) (string, bool) {
eq := strings.IndexByte(line, '=')
if eq < 0 {
return "", false
}
if !strings.EqualFold(strings.TrimSpace(line[:eq]), "insteadOf") {
return "", false
}
return strings.TrimSpace(line[eq+1:]), true
}
// firstPathSegment returns the lowercased first non-empty path segment (the owner/org), or "".
func firstPathSegment(path string) string {
for _, seg := range strings.Split(path, "/") {
if seg != "" {
return strings.ToLower(seg)
}
}
return ""
}
// resolveToken returns the token and its source based on the host. RawHost (the full
// authority, with port) is forwarded to isConfiguredGitHubHost for the GHES check. The
// per-provider token cascades are split into their own helpers (resolveGitHubToken,
// resolveBitbucketToken, resolveGitLabToken) to keep this dispatcher's cyclomatic complexity low.
func (d *CustomGitDetector) resolveToken(host, rawHost string) (string, string) {
switch {
case isConfiguredGitHubHost(host, rawHost):
return d.resolveGitHubToken()
case host == hostBitbucket:
return d.resolveBitbucketToken()
case host == hostGitLab:
return d.resolveGitLabToken()
}
return "", ""
}
// resolveGitHubToken returns the GitHub/GHES token and its source.
// It prefers ATMOS_* prefixed tokens but falls back to standard tokens if not set.
func (d *CustomGitDetector) resolveGitHubToken() (string, string) {
// Prefer ATMOS_PRO_GITHUB_TOKEN (Atmos Pro-brokered), then ATMOS_GITHUB_TOKEN, then GITHUB_TOKEN.
if d.atmosConfig.Settings.AtmosProGithubToken != "" {
return d.atmosConfig.Settings.AtmosProGithubToken, "ATMOS_PRO_GITHUB_TOKEN"
}
// The broker os.Setenv's the minted token after startup (after Settings is populated), so
// read the live env here as well — mirrors pkg/http/client.go.
//nolint:forbidigo // Live env fallback for the broker-set token; mirrors pkg/http/client.go.
if token := os.Getenv("ATMOS_PRO_GITHUB_TOKEN"); token != "" {
return token, "ATMOS_PRO_GITHUB_TOKEN"
}
if d.atmosConfig.Settings.AtmosGithubToken != "" {
return d.atmosConfig.Settings.AtmosGithubToken, "ATMOS_GITHUB_TOKEN"
}
if d.atmosConfig.Settings.GithubToken != "" {
return d.atmosConfig.Settings.GithubToken, "GITHUB_TOKEN"
}
// Last resort: fall back to `gh auth token`, matching the fallback github.GetGitHubToken()
// already uses for plain HTTPS/API fetches, so a developer who's only run `gh auth login`
// doesn't need a separate token for private-repo git:: imports/vendoring/module fetches too.
ctx := d.ctx
if ctx == nil {
ctx = context.Background()
}
if token := github.GetGitHubTokenFromCLIContext(ctx); token != "" {
return token, "GH_CLI"
}
return "", ""
}
// resolveBitbucketToken returns the Bitbucket token and its source, preferring
// ATMOS_BITBUCKET_TOKEN and falling back to BITBUCKET_TOKEN.
func (d *CustomGitDetector) resolveBitbucketToken() (string, string) {
if d.atmosConfig.Settings.AtmosBitbucketToken != "" {
return d.atmosConfig.Settings.AtmosBitbucketToken, "ATMOS_BITBUCKET_TOKEN"
}
return d.atmosConfig.Settings.BitbucketToken, "BITBUCKET_TOKEN"
}
// resolveGitLabToken returns the GitLab token and its source, preferring ATMOS_GITLAB_TOKEN
// and falling back to GITLAB_TOKEN.
func (d *CustomGitDetector) resolveGitLabToken() (string, string) {
if d.atmosConfig.Settings.AtmosGitlabToken != "" {
return d.atmosConfig.Settings.AtmosGitlabToken, "ATMOS_GITLAB_TOKEN"
}
return d.atmosConfig.Settings.GitlabToken, "GITLAB_TOKEN"
}
// getDefaultUsername returns the default username for token injection based on the host.
// RawHost (the full authority, with port) is forwarded to isConfiguredGitHubHost for the GHES
// check.
func (d *CustomGitDetector) getDefaultUsername(host, rawHost string) string {
switch {
case isConfiguredGitHubHost(host, rawHost):
return "x-access-token"
case host == hostGitLab:
return "oauth2"
case host == hostBitbucket:
defaultUsername := d.atmosConfig.Settings.BitbucketUsername
if defaultUsername == "" {
return "x-token-auth"
}
return defaultUsername
default:
return "x-access-token"
}
}