import path from 'node:path' import fastGlob from 'fast-glob' import ignore from 'ignore' import micromatch from 'micromatch' import { parse as yamlParse } from 'yaml' import { isDirSync, safeReadFile } from '@socketsecurity/registry/lib/fs' import { defaultIgnore } from '@socketsecurity/registry/lib/globs' import { readPackageJson } from '@socketsecurity/registry/lib/packages' import { normalizePath } from '@socketsecurity/registry/lib/path' import { transform } from '@socketsecurity/registry/lib/streams' import { isNonEmptyString } from '@socketsecurity/registry/lib/strings' import { NODE_MODULES, PNPM } from '../constants.mts' import type { Agent } from './package-environment.mts' import type { SocketYml } from '@socketsecurity/config' import type { SocketSdkSuccessResult } from '@socketsecurity/sdk' import type { Options as GlobOptions } from 'fast-glob' const DEFAULT_IGNORE_FOR_GIT_IGNORE = defaultIgnore.filter( p => !p.endsWith('.gitignore'), ) export const IGNORED_DIRS = [ // Taken from ignore-by-default: // https://github.com/novemberborn/ignore-by-default/blob/v2.1.0/index.js '.git', // Git repository files, see '.log', // Log files emitted by tools such as `tsserver`, see '.nyc_output', // Temporary directory where nyc stores coverage data, see '.sass-cache', // Cache folder for node-sass, see '.yarn', // Where node modules are installed when using Yarn, see 'bower_components', // Where Bower packages are installed, see 'coverage', // Standard output directory for code coverage reports, see NODE_MODULES, // Where Node modules are installed, see // Taken from globby: // https://github.com/sindresorhus/globby/blob/v14.0.2/ignore.js#L11-L16 'flow-typed', // Conventional Python virtual environment dir. Arbitrarily-named venvs are // detected via their pyvenv.cfg marker during the discovery walk below. '.venv', ] as const const IGNORED_DIR_PATTERNS = IGNORED_DIRS.map(i => `**/${i}`) // Marker file at the root of every Python virtual environment (stdlib `venv` // per PEP 405, and virtualenv >= 20). Lets us detect venvs that don't use a // conventional directory name. const PYVENV_CFG = 'pyvenv.cfg' async function getWorkspaceGlobs( agent: Agent, cwd = process.cwd(), ): Promise { let workspacePatterns if (agent === PNPM) { const workspacePath = path.join(cwd, 'pnpm-workspace.yaml') const yml = await safeReadFile(workspacePath) if (yml) { try { workspacePatterns = yamlParse(yml)?.packages } catch {} } } else { workspacePatterns = (await readPackageJson(cwd, { throws: false }))?.[ 'workspaces' ] } return Array.isArray(workspacePatterns) ? workspacePatterns .filter(isNonEmptyString) .map(workspacePatternToGlobPattern) : [] } function ignoreFileLinesToGlobPatterns( lines: string[] | readonly string[], filepath: string, cwd: string, ): string[] { const base = path.relative(cwd, path.dirname(filepath)).replace(/\\/g, '/') const patterns = [] for (let i = 0, { length } = lines; i < length; i += 1) { const pattern = lines[i]!.trim() if (pattern.length > 0 && pattern.charCodeAt(0) !== 35 /*'#'*/) { patterns.push( ignorePatternToMinimatch( pattern.length && pattern.charCodeAt(0) === 33 /*'!'*/ ? `!${path.posix.join(base, pattern.slice(1))}` : path.posix.join(base, pattern), ), ) } } return patterns } function ignoreFileToGlobPatterns( content: string, filepath: string, cwd: string, ): string[] { return ignoreFileLinesToGlobPatterns(content.split(/\r?\n/), filepath, cwd) } function escapeMinimatchGroupStarts(pattern: string): string { let escaped = '' let previousWasOddBackslash = false for (const char of pattern) { if ((char === '{' || char === '(') && !previousWasOddBackslash) { escaped += '\\' } escaped += char previousWasOddBackslash = char === '\\' ? !previousWasOddBackslash : false } return escaped } // Based on `@eslint/compat` convertIgnorePatternToMinimatch. // Apache v2.0 licensed // Copyright Nicholas C. Zakas // https://github.com/eslint/rewrite/blob/compat-v1.2.1/packages/compat/src/ignore-file.js#L28 function ignorePatternToMinimatch(pattern: string): string { const isNegated = pattern.startsWith('!') const negatedPrefix = isNegated ? '!' : '' const patternToTest = (isNegated ? pattern.slice(1) : pattern).trimEnd() // Special cases. if ( patternToTest === '' || patternToTest === '**' || patternToTest === '/**' || patternToTest === '**' ) { return `${negatedPrefix}${patternToTest}` } const firstIndexOfSlash = patternToTest.indexOf('/') const matchEverywherePrefix = firstIndexOfSlash === -1 || firstIndexOfSlash === patternToTest.length - 1 ? '**/' : '' const patternWithoutLeadingSlash = firstIndexOfSlash === 0 ? patternToTest.slice(1) : patternToTest // Escape `{` and `(` because in gitignore patterns they are just // literal characters without any specific syntactic meaning, // while in minimatch patterns they can form brace expansion or extglob syntax. // // For example, gitignore pattern `src/{a,b}.js` ignores file `src/{a,b}.js`. // But, the same minimatch pattern `src/{a,b}.js` ignores files `src/a.js` and `src/b.js`. // Minimatch pattern `src/\{a,b}.js` is equivalent to gitignore pattern `src/{a,b}.js`. const escapedPatternWithoutLeadingSlash = escapeMinimatchGroupStarts( patternWithoutLeadingSlash, ) const matchInsideSuffix = patternToTest.endsWith('/**') ? '/*' : '' return `${negatedPrefix}${matchEverywherePrefix}${escapedPatternWithoutLeadingSlash}${matchInsideSuffix}` } // fast-glob silently discards `ignore` entries that end in `/` (it // treats them as literal directory paths, not glob patterns). The // gitignore convention of writing directory entries as `dist/` lands // here as `**/dist/` after `ignorePatternToMinimatch`, which fast-glob // then drops — defeating the entire ignore. Strip the trailing slash // so fast-glob actually honors the pattern. export function stripTrailingSlash(pattern: string): string { if ( pattern.length > 1 && pattern.charCodeAt(pattern.length - 1) === 47 /*'/'*/ ) { return pattern.slice(0, -1) } return pattern } function workspacePatternToGlobPattern(workspace: string): string { const { length } = workspace if (!length) { return '' } // If the workspace ends with "/" if (workspace.charCodeAt(length - 1) === 47 /*'/'*/) { return `${workspace}/*/package.json` } // If the workspace ends with "/**" if ( workspace.charCodeAt(length - 1) === 42 /*'*'*/ && workspace.charCodeAt(length - 2) === 42 /*'*'*/ && workspace.charCodeAt(length - 3) === 47 /*'/'*/ ) { return `${workspace}/*/**/package.json` } // Things like "packages/a" or "packages/*" return `${workspace}/package.json` } export function filterBySupportedScanFiles( filepaths: string[] | readonly string[], supportedFiles: SocketSdkSuccessResult<'getSupportedFiles'>['data'], ): string[] { const patterns = getSupportedFilePatterns(supportedFiles) return filepaths.filter(p => micromatch.some(p, patterns, { dot: true, nocase: true }), ) } export function createSupportedFilesFilter( supportedFiles: SocketSdkSuccessResult<'getSupportedFiles'>['data'], ): (filepath: string) => boolean { const patterns = getSupportedFilePatterns(supportedFiles) return (filepath: string) => micromatch.some(filepath, patterns, { dot: true, nocase: true }) } export function getSupportedFilePatterns( supportedFiles: SocketSdkSuccessResult<'getSupportedFiles'>['data'], ): string[] { const patterns: string[] = [] for (const key of Object.keys(supportedFiles)) { const supported = supportedFiles[key] if (supported) { patterns.push( ...Object.values(supported as Record).map( p => `**/${p.pattern}`, ), ) } } return patterns } type GlobWithGitIgnoreOptions = GlobOptions & { // Already-anchored minimatch patterns merged into fast-glob's `ignore` // option in every code path. These bypass the gitignore translator and // the `ignore` package matcher entirely; use this channel for CLI flags // whose contract is anchored micromatch from `cwd` (e.g. --exclude-paths). // Patterns in `socketConfig.projectIgnorePaths` and discovered `.gitignore` // files take the other channel: they're gitignore-translated first. additionalIgnores?: readonly string[] | undefined // Optional filter function to apply during streaming. // When provided, only files passing this filter are accumulated. // This is critical for memory efficiency when scanning large monorepos. filter?: ((filepath: string) => boolean) | undefined socketConfig?: SocketYml | undefined } type IgnoreMatcher = ReturnType // Whether `targetPath` is ignored by the gitignore matchers on its ancestor // directories, root to leaf, last match wins. Each matcher tests the path // relative to its own directory (how git stores patterns); `isDir` appends a // trailing slash so a directory-only rule like `build/` matches the directory. // `targetPath` is POSIX, relative to cwd. function pathIgnoredByChain( targetPath: string, isDir: boolean, matchersByDir: Map, ): boolean { let ignored = false // Ancestor directories of targetPath: '', then each parent dir prefix. const segments = targetPath.split('/') segments.pop() const dirs = [''] let prefix = '' for (let i = 0, { length } = segments; i < length; i += 1) { prefix = prefix ? `${prefix}/${segments[i]}` : segments[i]! dirs.push(prefix) } for (let i = 0, { length } = dirs; i < length; i += 1) { const dir = dirs[i]! const matchers = matchersByDir.get(dir) if (!matchers) { continue } const relToDir = dir === '' ? targetPath : targetPath.slice(dir.length + 1) const probe = isDir ? `${relToDir}/` : relToDir for (let j = 0, len = matchers.length; j < len; j += 1) { const result = matchers[j]!.test(probe) if (result.ignored) { ignored = true } else if (result.unignored) { ignored = false } } } return ignored } // Whether `relPath` is ignored, honoring git's rule that an excluded directory // is never descended into: a file is ignored if any ancestor directory is, and a // deeper `!` cannot re-include a file under an excluded parent. Walks ancestors // top-down, short-circuiting on the first excluded one. POSIX, relative to cwd. function isIgnoredAlongChain( relPath: string, matchersByDir: Map, ): boolean { // Outside cwd (a `..` prefix) or empty: outside every gitignore's domain, and // the `ignore` package throws on such input, so report not-ignored. if (!relPath || relPath === '..' || relPath.startsWith('../')) { return false } const segments = relPath.split('/') const last = segments.length - 1 let prefix = '' for (let i = 0; i <= last; i += 1) { prefix = prefix ? `${prefix}/${segments[i]}` : segments[i]! if (pathIgnoredByChain(prefix, i < last, matchersByDir)) { return true } } return false } export async function globWithGitIgnore( patterns: string[] | readonly string[], options: GlobWithGitIgnoreOptions, ): Promise { const { additionalIgnores, cwd = process.cwd(), filter, socketConfig, ...additionalOptions } = { __proto__: null, ...options } as GlobWithGitIgnoreOptions // Anchored minimatch patterns for fast-glob: built-in ignored dirs, venv // markers, projectIgnorePaths, and every discovered `.gitignore`, translated // and anchored from cwd. When no pattern is negated, fast-glob does the whole // gitignore match from this set. const ignores = new Set(IGNORED_DIR_PATTERNS) // CLI-supplied `additionalIgnores` are already anchored minimatch from cwd, so // they go straight to fast-glob and never enter the gitignore matchers (whose // "match anywhere" semantics would re-interpret a bare `tests` as `**/tests`). const cliMinimatchIgnores = additionalIgnores ?? [] const projectIgnorePaths = socketConfig?.projectIgnorePaths const projectIgnoreLines = Array.isArray(projectIgnorePaths) ? projectIgnorePaths : [] const projectIgnoreGlobs = projectIgnoreLines.length ? ignoreFileLinesToGlobPatterns( projectIgnoreLines, path.join(cwd, '.gitignore'), cwd, ) : [] for (const pattern of projectIgnoreGlobs) { ignores.add(pattern) } // Raw per-directory `.gitignore` contents from discovery. Matchers are built // from these only when a pattern is negated (see below). const gitignoreFiles: Array<{ content: string; dir: string }> = [] // Directory excludes from discovered pyvenv.cfg markers, so virtualenvs with // non-conventional names are pruned. Fed to fast-glob's ignore on every path. const venvGlobs: string[] = [] // Discover `.gitignore` files and `pyvenv.cfg` venv markers in one walk, // honoring the same dir exclusions as the package walk so an unreadable subtree // (EACCES, foreign-uid pgdata, Docker mount) can't abort it. `suppressErrors` // skips dirs the user cannot read; negated patterns are dropped here because // fast-glob treats `!` ignore entries inconsistently. const discoveryStream = fastGlob.globStream( ['**/.gitignore', `**/${PYVENV_CFG}`], { absolute: true, cwd, dot: true, ignore: [ ...DEFAULT_IGNORE_FOR_GIT_IGNORE, ...projectIgnoreGlobs, ...cliMinimatchIgnores, ] .filter(p => p.charCodeAt(0) !== 33 /*'!'*/) .map(stripTrailingSlash), suppressErrors: true, }, ) for await (const found of transform( discoveryStream, async (filepath: string) => { const dirRel = normalizePath(path.relative(cwd, path.dirname(filepath))) if (path.basename(filepath) === PYVENV_CFG) { // A pyvenv.cfg sits at the venv root, so exclude the whole directory. An // empty dirRel means the scan target itself is a venv root; don't emit // `/**`, which would exclude everything the user explicitly targeted. return { content: '', dir: dirRel, patterns: dirRel ? [`${dirRel}/**`] : [], venv: true, } } const content = (await safeReadFile(filepath)) ?? '' return { content, dir: dirRel, patterns: ignoreFileToGlobPatterns(content, filepath, cwd), venv: false, } }, { concurrency: 8 }, )) { for (const p of found.patterns) { ignores.add(p) } if (found.venv) { venvGlobs.push(...found.patterns) } else if (found.content) { gitignoreFiles.push({ content: found.content, dir: found.dir }) } } let hasNegatedPattern = false for (const p of ignores) { if (p.charCodeAt(0) === 33 /*'!'*/) { hasNegatedPattern = true break } } const globOptions = { __proto__: null, absolute: true, cwd, dot: true, // With a negation, the per-dir matcher chain (below) covers only gitignore // and projectIgnore patterns, so fast-glob still prunes the built-in ignored // dirs and discovered venvs. Without one, the full anchored set goes to // fast-glob, which does the whole match. ignore: hasNegatedPattern ? [ ...defaultIgnore, ...IGNORED_DIR_PATTERNS, ...venvGlobs, ...cliMinimatchIgnores, ] : [...ignores, ...cliMinimatchIgnores].map(stripTrailingSlash), ...additionalOptions, // Skip dirs the running user cannot read instead of aborting on the first // `EACCES`. Pinned after `...additionalOptions` so a caller's options bag // cannot flip it off; it is a safety invariant, not a tunable. suppressErrors: true, } as GlobOptions // No negation and no filter: fast-glob's anchored ignore set is authoritative. if (!hasNegatedPattern && !filter) { return await fastGlob.glob(patterns as string[], globOptions) } // When a pattern is negated, match each candidate against the gitignore // ancestor chain. One matcher per `.gitignore` is built from its raw lines and // deduped by content via `igByContent`, keeping compiled-regex memory bounded // by the number of DISTINCT `.gitignore` contents, not by file count (a single // matcher over every anchored pattern can exhaust V8 code space on big repos). let matchersByDir: Map | undefined if (hasNegatedPattern) { const byDir = new Map() const igByContent = new Map() const addMatcher = (dirRel: string, content: string): void => { let ig = igByContent.get(content) if (!ig) { ig = ignore().add(content.split(/\r?\n/)) igByContent.set(content, ig) } const existing = byDir.get(dirRel) if (existing) { existing.push(ig) } else { byDir.set(dirRel, [ig]) } } // projectIgnorePaths act as a root-level gitignore. if (projectIgnoreLines.length) { addMatcher('', projectIgnoreLines.join('\n')) } for (let i = 0, { length } = gitignoreFiles; i < length; i += 1) { addMatcher(gitignoreFiles[i]!.dir, gitignoreFiles[i]!.content) } matchersByDir = byDir } // Stream so memory stays bounded on large monorepos with 100k+ files: the // optional caller filter drops non-matches before they accumulate. On the slow // path each surviving entry is also re-checked against its gitignore ancestor // chain, which carries the full negation support fast-glob lacks. const results: string[] = [] const stream = fastGlob.globStream( patterns as string[], globOptions, ) as AsyncIterable for await (const p of stream) { if (matchersByDir) { // Patterns are forward-slash anchored and tested relative to each // gitignore's directory; normalize so a Windows backslash path matches. const relPath = normalizePath( globOptions.absolute ? path.relative(cwd, p) : p, ) if (isIgnoredAlongChain(relPath, matchersByDir)) { continue } } if (filter && !filter(p)) { continue } results.push(p) } return results } export async function globWorkspace( agent: Agent, cwd = process.cwd(), ): Promise { const workspaceGlobs = await getWorkspaceGlobs(agent, cwd) return workspaceGlobs.length ? await fastGlob.glob(workspaceGlobs, { absolute: true, cwd, dot: true, ignore: defaultIgnore, }) : [] } export function isReportSupportedFile( filepath: string, supportedFiles: SocketSdkSuccessResult<'getSupportedFiles'>['data'], ) { const patterns = getSupportedFilePatterns(supportedFiles) return micromatch.some(filepath, patterns, { dot: true, nocase: true }) } export function pathsToGlobPatterns( paths: string[] | readonly string[], cwd?: string | undefined, ): string[] { // TODO: Does not support `~/` paths. return paths.map(p => { // Convert current directory references to glob patterns. if (p === '.' || p === './') { return '**/*' } const absolutePath = path.isAbsolute(p) ? p : path.resolve(cwd ?? process.cwd(), p) // If the path is a directory, scan it recursively for all files. if (isDirSync(absolutePath)) { return `${p}/**/*` } return p }) }