"""Credential-bearing HTTP header masking for agent.redact. API-key style headers (``x-api-key``…) and Cookie / Set-Cookie headers in raw header lines, curl ``-H`` arguments, mid-line log text and JSON-serialized request dumps. Authorization headers stay in agent/redact.py (they carry a scheme word). """ import re # API-key style headers (single opaque value, no scheme word): non-vendor-prefix # values would otherwise leak when a curl command is echoed into tool output. SECRET_HEADER_NAME_LIST = ("x-api-key", "x-goog-api-key", "api-key", "apikey", "x-api-token", "x-auth-token", "x-access-token") _SECRET_HEADER_NAMES = rf"(?:{'|'.join(SECRET_HEADER_NAME_LIST)})" _SECRET_HEADER_LINE_RE = re.compile( rf"(^[ \t]*(?:[<>][ \t]+)?{_SECRET_HEADER_NAMES}[ \t]*:[ \t]*)([^\r\n]+)", re.IGNORECASE | re.MULTILINE, ) _SECRET_HEADER_INLINE_RE = re.compile( # ``$`` and ``&``/``;``/``|`` stay in the value (``x-api-key: &opaque…`` / ``a|opaque…`` in a # log is one credential) unless whitespace, end of text or another separator follows # (``&& ls``, ``; next``, ``||true``). A leading ``<``/``(`` is part of a log value # (````); later ``<>()`` are redirections/grouping and end the value. rf"({_SECRET_HEADER_NAMES}[ \t]*:[ \t]*)" rf"((?:[<(&;|]+(?![\s;&|<>()]|$))?(?:[^\s\"';&|<>()`\\]|[;&|](?![\s;&|<>()]|$))+)", re.IGNORECASE, ) _SECRET_HEADER_DOUBLE_QUOTED_RE = re.compile( rf'((?:^|[ \t])(?:-H(?:(?:[ \t]|\\\r?\n)*|=)' rf'|--header(?:(?:[ \t]|\\\r?\n)+|=))")' rf'({_SECRET_HEADER_NAMES}:[ \t]*)' r'((?:\\\r?\n|\\.|[^"\\\r\n])+)(")', re.IGNORECASE, ) _SECRET_HEADER_SINGLE_QUOTED_RE = re.compile( rf"((?:^|[ \t])(?:-H(?:(?:[ \t]|\\\r?\n)*|=)" rf"|--header(?:(?:[ \t]|\\\r?\n)+|=))')" rf"({_SECRET_HEADER_NAMES}:[ \t]*)([^'\r\n]+)(')", re.IGNORECASE, ) _SECRET_HEADER_UNQUOTED_CURL_RE = re.compile( rf"((?:^|[ \t])(?:-H(?:(?:[ \t]|\\\r?\n)*|=)" rf"|--header(?:(?:[ \t]|\\\r?\n)+|=)){_SECRET_HEADER_NAMES}:)" r"((?:\\[^\r\n]|[^ \t\r\n;&|<>()$`\\])+)", re.IGNORECASE | re.MULTILINE, ) # Same secret headers serialized in JSON form (``"x-api-key": ""``). The # bare-header regex above cannot match this shape — the closing quote sits # between the name and the colon — and ``_JSON_FIELD_RE``'s key list omits the # hyphenated header names, so opaque custom/local-backend keys (no vendor prefix # for ``_PREFIX_RE`` to catch) leak verbatim from serialized request dumps. This # mirrors the JSON-field redaction shape, so it is applied under the same # ``code_file`` gate. _SECRET_HEADER_JSON_RE = re.compile( # ``(?:\\.|[^"\\])`` consumes JSON string contents escape-aware, so a value # containing an escaped quote (``"ab\"cd"``) is masked whole rather than # truncated at the embedded quote (which would leave the tail in cleartext). rf'("{_SECRET_HEADER_NAMES}"\s*:\s*")((?:\\[^\r\n]|[^"\\\r\n])+)(")', re.IGNORECASE, ) # Fail closed for a recognized JSON header whose string value is truncated at # a line boundary. The primary regex above handles all valid JSON strings; this # fallback is deliberately line-bounded and its alternatives are disjoint so # malformed diagnostic output cannot trigger unbounded backtracking. _SECRET_HEADER_JSON_UNTERMINATED_RE = re.compile( rf'("{_SECRET_HEADER_NAMES}"\s*:\s*")((?:\\[^\r\n]|[^"\\\r\n])*(?:\\)?)(?=\r?\n|\Z)', re.IGNORECASE | re.MULTILINE, ) # Cookie: / Set-Cookie: headers carry session, auth and CSRF tokens and were # previously unhandled. The full cookie payload is masked while the header name # is preserved. A ``name=value`` pair is required so plain prose ("the cookie: # …") is left untouched. # Restrict raw Cookie matching to an actual header line. A bounded cookie-name # class avoids repeatedly scanning whole suffixes for '=' on malformed input. _COOKIE_HEADER_LINE_RE = re.compile( r"(^[ \t]*(?:[<>][ \t]+)?(?:Set-)?Cookie:[ \t]*)" r"([!#$%&'*+\-.^_`|~0-9A-Za-z]+=[^\r\n]*)", re.IGNORECASE | re.MULTILINE, ) # Shell commands commonly embed a header in a quoted ``-H`` argument. Separate # delimiter-specific patterns cannot scan across another candidate opener, so # repeated malformed fragments remain linear rather than quadratic. _COOKIE_HEADER_DOUBLE_QUOTED_RE = re.compile( r'((?:^|[ \t])(?:-H(?:(?:[ \t]|\\\r?\n)*|=)' r'|--header(?:(?:[ \t]|\\\r?\n)+|=))")' r'((?:Set-)?Cookie:[ \t]*)' r'([!#$%&\'*+\-.^_`|~0-9A-Za-z]+=(?:\\\r?\n|\\.|[^"\\\r\n])*)(")', re.IGNORECASE, ) _COOKIE_HEADER_SINGLE_QUOTED_RE = re.compile( r"((?:^|[ \t])(?:-H(?:(?:[ \t]|\\\r?\n)*|=)" r"|--header(?:(?:[ \t]|\\\r?\n)+|=))')" r"((?:Set-)?Cookie:[ \t]*)([!#$%&'*+\-.^_`|~0-9A-Za-z]+=[^'\r\n]*)(')", re.IGNORECASE, ) # An unquoted ``-H Cookie:name=value`` argument cannot contain whitespace, so # its token boundary is unambiguous and needs no open-ended delimiter search. _COOKIE_HEADER_UNQUOTED_CURL_RE = re.compile( r"((?:^|[ \t])(?:-H(?:(?:[ \t]|\\\r?\n)*|=)" r"|--header(?:(?:[ \t]|\\\r?\n)+|=))" r"(?:Set-)?Cookie:[ \t]*)" r"([!#$%&'*+\-.^_`|~0-9A-Za-z]+=" r"(?:\\[^\r\n]|[^ \t\r\n;&|<>()$`\\])*)", re.IGNORECASE | re.MULTILINE, ) # The on-disk request/response dump sink serializes headers as JSON, so cookies # also appear as ``"Cookie": ""`` — the bare regex above cannot see # that shape. A ``name=value`` pair is required so non-header ``"Cookie"`` values # are left untouched. JSON-shaped, so applied under the ``code_file`` gate. _COOKIE_HEADER_JSON_RE = re.compile( # Escape-aware single-group value match (see _SECRET_HEADER_JSON_RE): the # whole JSON string value of a "Cookie"/"Set-Cookie" key is masked. A single # unambiguous group (no internal "=" split) avoids superlinear backtracking # on malformed/unterminated JSON. A JSON key literally named Cookie is a # strong-enough header signal that the "name=value" requirement is dropped. r'("(?:Set-)?Cookie"\s*:\s*")((?:\\[^\r\n]|[^"\\\r\n])*)(")', re.IGNORECASE, ) _COOKIE_HEADER_JSON_UNTERMINATED_RE = re.compile( r'("(?:Set-)?Cookie"\s*:\s*")((?:\\[^\r\n]|[^"\\\r\n])*(?:\\)?)(?=\r?\n|\Z)', re.IGNORECASE | re.MULTILINE, ) # A Cookie/Set-Cookie header in the middle of a line (log records, prose dumps) that the # line/curl/JSON passes above did not see. Every ``name=value`` pair is masked, not only # the first; ``Set-Cookie:`` also masks a bare 16+ char token (``Set-Cookie: header`` is prose). # An apostrophe is a cookie-octet (``sid=pre'opaque``) unless it closes a shell word. _COOKIE_PAIR = r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+=(?:\"[^\"\r\n]*\"|(?:[^\s;\\\"'`]|'(?![\s'\"`;&|<>()]|$))*)" _COOKIE_HEADER_INLINE_RE = re.compile( rf"(\b(?PSet-)?Cookie:[ \t]*)" rf"(?:{_COOKIE_PAIR}(?:;[ \t]*{_COOKIE_PAIR})*|(?(set)[^\s\\\"';&|<>()$`*=]{{16,}}+(?![^\s;])|(?!)))", re.IGNORECASE, ) _MASKED_VALUE_RE = re.compile(r"(?:\*{3}|«redacted(?::[^»\s]*|-secret)»|\[REDACTED\])(?:\s|$)") def _redact_inline_secret_header(m: re.Match) -> str: """Mask an inline header token without stealing an empty curl header's URL.""" before = m.string[max(0, m.start() - 32):m.start()] if re.search(r"(?:-H|--header)(?:[ \t]*|=)[\"']?$", before, re.IGNORECASE): return m.group(0) return m.group(1) + "***" def _redact_header_line_value(m: re.Match) -> str: """Mask a header line unless it is the continued argument of curl -H.""" line_start = m.string.rfind("\n", 0, m.start()) + 1 if line_start: previous_start = m.string.rfind("\n", 0, line_start - 1) + 1 previous = m.string[previous_start:line_start - 1].rstrip("\r") if previous.rstrip().endswith("\\") and re.search( r"(?:^|[ \t])(?:-H|--header)(?:[ \t]+|=).*\\[ \t]*$", previous, re.IGNORECASE, ): return m.group(0) # An earlier pass already masked the value: what follows the marker is prose, and # re-masking it would make the redactor non-idempotent (``x-api-key: *** see docs``). if _MASKED_VALUE_RE.match(m.group(2)): return m.group(0) return m.group(1) + "***" def redact_header_credentials(text: str) -> str: """Mask secret-header and cookie-header values in raw, curl and inline forms; the JSON forms run with the assignment passes in agent/redact.py.""" text = _SECRET_HEADER_LINE_RE.sub( _redact_header_line_value, text, ) for pattern in ( _SECRET_HEADER_DOUBLE_QUOTED_RE, _SECRET_HEADER_SINGLE_QUOTED_RE, ): text = pattern.sub( lambda m: m.group(1) + m.group(2) + "***" + m.group(4), text, ) text = _SECRET_HEADER_UNQUOTED_CURL_RE.sub( lambda m: m.group(1) + "***", text, ) text = _SECRET_HEADER_INLINE_RE.sub(_redact_inline_secret_header, text) # Cookie: / Set-Cookie: header values (session/auth/CSRF tokens), whole payload masked. # Not gated on ``code_file``: terminal output from ``curl -i`` is scrubbed with # code_file=True, and a raw ``Cookie: name=value`` line is a credential wherever it # appears, like the secret headers above. The JSON-fixture exemption lives in the # assignment passes. IGNORECASE is the filter, so ``CoOkie:`` cannot slip past. text = _COOKIE_HEADER_LINE_RE.sub(_redact_header_line_value, text) for pattern in (_COOKIE_HEADER_DOUBLE_QUOTED_RE, _COOKIE_HEADER_SINGLE_QUOTED_RE): text = pattern.sub(lambda m: m.group(1) + m.group(2) + "***" + m.group(4), text) text = _COOKIE_HEADER_UNQUOTED_CURL_RE.sub(lambda m: m.group(1) + "***", text) text = _COOKIE_HEADER_INLINE_RE.sub(lambda m: m.group(1) + "***", text) return text # Cookie credentials are bearer credentials, but ``cookie`` stays out of the shared # secret-keyword set (``cookie_policy: strict``, ``cookies enabled`` are prose). The # assignment form needs a key that IS ``cookie`` or ends in ``_cookie``; ``mycookiejar=1``, # ``cookie_policy=x`` and URL query params (``?cookie=``) never match. _COOKIE_ASSIGN_RE = re.compile( r"(?[\"'])(?P(?:(?!(?P=q))[^\\\r\n]|\\.)*)(?P=q)" r"(?P(?:\\.|'[^'\r\n]*+'|\"[^\"\s\\,:{}\[\]]*+\"|[^\s\"'\\;&|<>(),]++)*+)" r"|(?P(?:[^\s\"'`\\;,&|<>()]|&(?![&\s]|$)|;(?=[!#$%'*+\-.^_|~0-9A-Za-z]+=))+))", re.IGNORECASE, ) def cookie_value_is_credential(value: str, *, qualified: bool = False) -> bool: """A bare ``cookie`` key is weak, so the value shape decides as it does for ``token=``: a ``name=value`` pair is a cookie payload; ``getCookie()``, ``None``, ``$(cat jar)`` and ``chocolate`` are code or prose, while an opaque token is masked. A qualified ``SESSION_COOKIE`` is as strong as ``SESSION_TOKEN`` and masks any literal value.""" from agent.redact import _should_redact_assignment if value.startswith("«redacted"): return False key = "session_token" if qualified else "cookie" return ("=" in value and not value.startswith("$")) or _should_redact_assignment(key, value, check_keyword=False) def json_header_value_is_credential(key: str, value: str) -> bool: """Gate for a JSON ``"
": ""`` field: cookie fields follow the cookie value shape; an API-key header masks every value but a whole ``${API_KEY}`` / env-lookup placeholder (no path exemption: ``/abc…`` is a valid key shape).""" from agent.redact import _ENV_LOOKUP_VALUE_RE, _SHELL_VAR_REF if "cookie" in key.lower(): return cookie_value_is_credential(value) # An earlier pass's marker keeps its vendor label (``«redacted:ghp_…»``), as main's field pass does. return not (value == "***" or value.startswith("«redacted") or _ENV_LOOKUP_VALUE_RE.match(value) or re.fullmatch(_SHELL_VAR_REF, value)) def _redact_cookie_assignment(m: "re.Match[str]", mask) -> str: """Mask a ``cookie=`` value; a lone quoted value keeps its quotes, and a shell word joining quoted and bare parts (``'sid='\\''opaque'``) is masked as one word.""" joined = m.group("q") is None or m.group("tail") value = m.group(0)[len(m.group(1)):] if joined else m.group("quoted") if not cookie_value_is_credential(value, qualified="_" in m.group(1)): return m.group(0) q = "" if joined else m.group("q") return f"{m.group(1)}{q}{mask(value)}{q}" def redact_cookie_assignments(text: str, mask) -> str: """``cookie=`` / ``*_COOKIE=`` assignment values (an ``=`` gated assignment pass in agent/redact.py).""" return _COOKIE_ASSIGN_RE.sub(lambda m: _redact_cookie_assignment(m, mask), text)