import { CandidatesCollection, CodeCompletionCore } from 'antlr4-c3'; import { ANTLRErrorListener, CharStream, CharStreams, CommonTokenStream, Lexer, Parser, ParserRuleContext, ParseTreeListener, ParseTreeWalker, PredictionMode, Token, } from 'antlr4ng'; import { SQLParserBase } from '../../lib/SQLParserBase'; import type { EntityCollector } from './entityCollector'; import { EntityContext } from './entityCollector'; import { ErrorStrategy } from './errorStrategy'; import { findCaretTokenIndex } from './findCaretTokenIndex'; import { ErrorListener, ParseError } from './parseErrorListener'; import SemanticContextCollector from './semanticContextCollector'; import type { SplitListener } from './splitListener'; import { ctxToText, TextSlice, tokenToWord, WordRange } from './textAndWord'; import { CaretPosition, LOCALE_TYPE, SemanticCollectOptions, SuggestionOptions, Suggestions, SyntaxSuggestion, } from './types'; export const SQL_SPLIT_SYMBOL_TEXT = ';'; /** * Basic SQL class, every sql needs extends it. */ export abstract class BasicSQL< L extends Lexer = Lexer, PRC extends ParserRuleContext = ParserRuleContext, P extends SQLParserBase = SQLParserBase, > { /** members for cache start */ protected _charStreams: CharStream; protected _lexer: L; protected _tokenStream: CommonTokenStream; protected _parser: P; protected _parseTree: PRC | null; protected _parsedInput: string; protected _parseErrors: ParseError[] = []; private _statementStartTokenTypes: Set | null = null; /** members for cache end */ private _errorListener: ErrorListener = (error) => { this._parseErrors.push(error); }; /** * PreferredRules for antlr4-c3 */ protected abstract preferredRules: Set; /** * Create a antlr4 Lexer instance. * @param input source string */ protected abstract createLexerFromCharStream(charStreams: CharStream): L; /** * Create Parser by CommonTokenStream * @param tokenStream CommonTokenStream */ protected abstract createParserFromTokenStream(tokenStream: CommonTokenStream): P; /** * Convert candidates to suggestions * @param candidates candidate list * @param allTokens slice all tokens from input by tokenIndexOffset * @param caretTokenIndex tokenIndex of caretPosition */ protected abstract processCandidates( candidates: CandidatesCollection, allTokens: Token[], caretTokenIndex: number ): Suggestions; protected getCandidateTokenRanges( candidates: CandidatesCollection, candidateStartTokenIndex: number, allTokens: Token[], caretTokenIndex: number ): Token[] { // antlr4-c3 may return both entity and alias candidates; use the nearest candidate's start index as boundary const endTokenIndex = Array.from(candidates.rules.values()).reduce( (nearestStartTokenIndex, candidateRule) => { if (candidateRule.startTokenIndex <= candidateStartTokenIndex) { return nearestStartTokenIndex; } return Math.min(nearestStartTokenIndex, candidateRule.startTokenIndex); }, caretTokenIndex + 1 ); const previousVisibleToken = allTokens .slice(candidateStartTokenIndex, endTokenIndex) .reverse() .find((token) => token.channel === Token.DEFAULT_CHANNEL); const visibleTokenIndexes = allTokens .slice(endTokenIndex, caretTokenIndex + 1) .reduce((indexes, token, offset) => { if (token.channel === Token.DEFAULT_CHANNEL) { indexes.push(endTokenIndex + offset); } return indexes; }, []); const firstVisibleToken = allTokens[visibleTokenIndexes[0]]; let rangeEndTokenIndex = endTokenIndex; // candidate boundary may fall inside a multi-level qualified name; extend along the identifier and dot chain if (previousVisibleToken?.text === '.' || firstVisibleToken?.text === '.') { let visibleTokenOffset = 0; if (previousVisibleToken?.text === '.' && firstVisibleToken) { rangeEndTokenIndex = visibleTokenIndexes[visibleTokenOffset] + 1; visibleTokenOffset += 1; } while (allTokens[visibleTokenIndexes[visibleTokenOffset]]?.text === '.') { rangeEndTokenIndex = visibleTokenIndexes[visibleTokenOffset] + 1; visibleTokenOffset += 1; if (visibleTokenOffset < visibleTokenIndexes.length) { rangeEndTokenIndex = visibleTokenIndexes[visibleTokenOffset] + 1; visibleTokenOffset += 1; } } } return allTokens .slice(candidateStartTokenIndex, rangeEndTokenIndex) .filter((token) => token.channel === Token.DEFAULT_CHANNEL); } /** * Get a new splitListener instance. */ protected abstract get splitListener(): SplitListener; /** * Get a new errorListener instance. */ protected abstract createErrorListener(errorListener: ErrorListener): ANTLRErrorListener; /** * Get a new entityCollector instance. */ protected abstract createEntityCollector( input: string, allTokens?: Token[], caretTokenIndex?: number ): EntityCollector; public locale: LOCALE_TYPE = 'en_US'; /** * Get a new semanticContextCollector instance. */ protected abstract createSemanticContextCollector( input: string, caretPosition: CaretPosition, allTokens: Token[], options?: SemanticCollectOptions ): SemanticContextCollector; /** * Create an antlr4 lexer from input. * @param input string */ public createLexer(input: string, errorListener?: ErrorListener) { const charStreams = CharStreams.fromString(input); const lexer = this.createLexerFromCharStream(charStreams); if (errorListener) { lexer.removeErrorListeners(); lexer.addErrorListener(this.createErrorListener(errorListener)); } return lexer; } /** * Create an antlr4 parser from input. * @param input string */ public createParser(input: string, errorListener?: ErrorListener) { const lexer = this.createLexer(input, errorListener); const tokenStream = new CommonTokenStream(lexer); tokenStream.fill(); const parser = this.createParserFromTokenStream(tokenStream); parser.interpreter.predictionMode = PredictionMode.SLL; if (errorListener) { parser.removeErrorListeners(); parser.addErrorListener(this.createErrorListener(errorListener)); } return parser; } /** * Parse input string and return parseTree. * @param input string * @param errorListener listen parse errors and lexer errors. * @returns parseTree */ public parse(input: string, errorListener?: ErrorListener) { const parser = this.createParser(input, errorListener); parser.buildParseTrees = true; parser.errorHandler = new ErrorStrategy(); return parser.program(); } /** * Create an antlr4 parser from input. * And the instances will be cache. * @param input string */ private createParserWithCache(input: string): P { this._parseTree = null; this._charStreams = CharStreams.fromString(input); this._lexer = this.createLexerFromCharStream(this._charStreams); this._lexer.removeErrorListeners(); this._lexer.addErrorListener(this.createErrorListener(this._errorListener)); this._tokenStream = new CommonTokenStream(this._lexer); /** * All tokens are generated in advance. * This can cause performance degradation, but it seems necessary for now. * Because the tokens will be used multiple times. */ this._tokenStream.fill(); this._parser = this.createParserFromTokenStream(this._tokenStream); this._parser.interpreter.predictionMode = PredictionMode.SLL; this._parser.buildParseTrees = true; this._parser.errorHandler = new ErrorStrategy(); return this._parser; } /** * If it is invoked multiple times in a row and the input parameters is the same, * this method returns the parsing result directly for the first time * unless the errorListener parameter is passed. * @param input source string * @param errorListener listen errors * @returns parseTree */ private parseWithCache(input: string, errorListener?: ErrorListener): PRC { // Avoid parsing the same input repeatedly. if (this._parsedInput === input && !errorListener && this._parseTree) { return this._parseTree; } this._parseErrors = []; const parser = this.createParserWithCache(input); this._parsedInput = input; parser.removeErrorListeners(); parser.addErrorListener(this.createErrorListener(this._errorListener)); this._parseTree = parser.program(); return this._parseTree; } /** * Reset the shared Lexer and Parser DFA caches for cold-start benchmarks. * This affects every parser instance of the same dialect. */ public resetDfaCache(): void { if (!this._parser || !this._lexer) { this.createParserWithCache(''); } this._parser.interpreter.clearDFA(); this._lexer.interpreter.clearDFA(); this._parseTree = null; this._parsedInput = ''; this._parseErrors = []; this._statementStartTokenTypes = null; } /** * Validate input string and return syntax errors if exists. * When input contains multiple statements separated by semicolons * and the initial parse doesn't capture all errors, each statement * is validated independently to collect all errors. * @param input source string * @returns syntax errors */ public validate(input: string): ParseError[] { this.parseWithCache(input); if (this._parseErrors.length > 0) { const statements = this.splitStatements(input); if (statements.length > 1) { const splitErrors = this.validateStatements(statements); if (splitErrors.length > this._parseErrors.length) { this._parseErrors = splitErrors; } } } return this._parseErrors; } /** * Validate each statement fragment independently and collect all errors. */ private validateStatements(statements: { text: string; start: number }[]): ParseError[] { const allErrors: ParseError[] = []; let lineOffset = 0; for (const statement of statements) { if (!statement.text.trim()) { const newlines = (statement.text.match(/\n/g) || []).length; lineOffset += newlines; continue; } const parser = this.createParser(statement.text); const errors: ParseError[] = []; parser.removeErrorListeners(); parser.addErrorListener( this.createErrorListener((error) => { errors.push({ startLine: error.startLine + lineOffset, endLine: error.endLine + lineOffset, startColumn: error.startColumn, endColumn: error.endColumn, message: error.message, }); }) ); parser.errorHandler = new ErrorStrategy(); parser.program(); allErrors.push(...errors); const newlines = (statement.text.match(/\n/g) || []).length; lineOffset += newlines; } return allErrors; } /** * Split input into individual statement strings by semicolons. * Handles semicolons inside quoted strings correctly. */ private splitStatements(input: string): { text: string; start: number }[] { const statements: { text: string; start: number }[] = []; let inSingleQuote = false; let inDoubleQuote = false; let lastSplit = 0; for (let i = 0; i < input.length; i++) { const ch = input[i]; if (ch === "'" && !inDoubleQuote) { inSingleQuote = !inSingleQuote; } else if (ch === '"' && !inSingleQuote) { inDoubleQuote = !inDoubleQuote; } else if (ch === ';' && !inSingleQuote && !inDoubleQuote) { statements.push({ text: input.slice(lastSplit, i + 1), start: lastSplit }); lastSplit = i + 1; } } if (lastSplit < input.length) { statements.push({ text: input.slice(lastSplit), start: lastSplit }); } return statements; } /** * Get the input string that has been parsed. */ public getParsedInput(): string { return this._parsedInput; } /** * Get all Tokens of input string,'' is not included. * @param input source string * @returns Token[] */ public getAllTokens(input: string): Token[] { this.parseWithCache(input); let allTokens = this._tokenStream.getTokens(); if (allTokens[allTokens.length - 1].text === '') { allTokens = allTokens.slice(0, -1); } return allTokens; } /** * @param listener Listener instance extends ParserListener * @param parseTree parser Tree */ public listen( listener: PTL, parseTree: ParserRuleContext ) { ParseTreeWalker.DEFAULT.walk(listener, parseTree); } /** * Split input into statements. * If exist syntax error it will return null. * @param input source string */ public splitSQLByStatement(input: string): TextSlice[] | null { const errors = this.validate(input); if (errors.length || !this._parseTree) { return null; } const splitListener = this.splitListener; this.listen(splitListener, this._parseTree); const res = splitListener.statementsContext .map((context) => { return ctxToText(context, this._parsedInput); }) .filter(Boolean) as TextSlice[]; return res; } /** * Get the smaller range of input * @param input string * @param allTokens all tokens from input * @param tokenIndexOffset offset of the tokenIndex in the range of input * @param caretTokenIndex tokenIndex of caretPosition * @returns inputSlice: string, caretTokenIndex: number */ private splitInputBySymbolText( input: string, allTokens: Token[], tokenIndexOffset: number, caretTokenIndex: number ): { inputSlice: string; allTokens: Token[]; caretTokenIndex: number } { const tokens = allTokens.slice(tokenIndexOffset); /** * Set startToken */ let startToken: Token | null = null; for (let tokenIndex = caretTokenIndex - tokenIndexOffset; tokenIndex >= 0; tokenIndex--) { const token = tokens[tokenIndex]; if (token?.text === SQL_SPLIT_SYMBOL_TEXT) { startToken = tokens[tokenIndex + 1]; break; } } if (startToken === null) { startToken = tokens[0]; } /** * Set stopToken */ let stopToken: Token | null = null; for ( let tokenIndex = caretTokenIndex - tokenIndexOffset; tokenIndex < tokens.length; tokenIndex++ ) { const token = tokens[tokenIndex]; if (token?.text === SQL_SPLIT_SYMBOL_TEXT) { stopToken = token; break; } } if (stopToken === null) { stopToken = tokens[tokens.length - 1]; } const indexOffset = tokens[0].start; let startIndex = startToken.start - indexOffset; let stopIndex = stopToken.stop + 1 - indexOffset; /** * Save offset of the tokenIndex in the range of input * compared to the tokenIndex in the whole input */ const _tokenIndexOffset = startToken.tokenIndex; const _caretTokenIndex = caretTokenIndex - _tokenIndexOffset; /** * Get the smaller range of _input */ const _input = input.slice(startIndex, stopIndex); return { inputSlice: _input, allTokens: allTokens.slice(_tokenIndexOffset), caretTokenIndex: _caretTokenIndex, }; } /** * Get the minimum input string that can be parsed successfully by c3. * @param input source string * @param caretTokenIndex tokenIndex of caretPosition * @param originParseTree origin parseTree * @returns MinimumInputInfo */ public getMinimumInputInfo( input: string, caretTokenIndex: number, originParseTree: ParserRuleContext | undefined ): { input: string; tokenIndexOffset: number; statementCount: number } | null { if (!originParseTree || !input?.length) return null; let inputSlice = input; /** * Split sql by statement. * Try to collect candidates in as small a range as possible. */ const splitListener = this.splitListener; this.listen(splitListener, originParseTree); const statementCount = splitListener.statementsContext?.length; const statementsContext = splitListener.statementsContext; let tokenIndexOffset = 0; // If there are multiple statements. if (statementCount > 1) { /** * Find a minimum valid range, reparse the fragment, and provide a new parse tree to C3. * The boundaries of this range must be statements with no syntax errors. * This can ensure the stable performance of the C3. */ let startStatement: ParserRuleContext | null = null; let stopStatement: ParserRuleContext | null = null; for (let index = 0; index < statementCount; index++) { const ctx = statementsContext[index]; const isCurrentCtxValid = !ctx.exception; if (!isCurrentCtxValid) continue; /** * Ensure that the statementContext before the left boundary * and the last statementContext on the right boundary are qualified SQL statements. */ const isPrevCtxValid = index === 0 || !statementsContext[index - 1]?.exception; const isNextCtxValid = index === statementCount - 1 || !statementsContext[index + 1]?.exception; if (ctx.stop && ctx.stop.tokenIndex < caretTokenIndex && isPrevCtxValid) { startStatement = ctx; } if ( ctx.start && !stopStatement && ctx.start.tokenIndex > caretTokenIndex && isNextCtxValid ) { stopStatement = ctx; break; } } // A boundary consisting of the index of the input. const startIndex = startStatement?.start?.start ?? 0; const stopIndex = stopStatement?.stop?.stop ?? inputSlice.length; /** * Save offset of the tokenIndex in the range of input * compared to the tokenIndex in the whole input */ tokenIndexOffset = startStatement?.start?.tokenIndex ?? 0; inputSlice = inputSlice.slice(startIndex, stopIndex); } return { input: inputSlice, tokenIndexOffset, statementCount, }; } /** * Get a minimum boundary parser near caretTokenIndex. * @param input source string. * @param caretTokenIndex start from which index to minimize the boundary. * @param originParseTree the parse tree need to be minimized, default value is the result of parsing `input`. * @returns minimum parser info */ public getMinimumParserInfo( input: string, caretTokenIndex: number, originParseTree: ParserRuleContext | undefined ): { parser: Parser; parseTree: ParserRuleContext; tokenIndexOffset: number; newTokenIndex: number; } | null { if (!originParseTree || !input?.length) return null; const inputInfo = this.getMinimumInputInfo(input, caretTokenIndex, originParseTree); if (!inputInfo) return null; const { input: inputSlice, tokenIndexOffset } = inputInfo; caretTokenIndex = caretTokenIndex - tokenIndexOffset; let sqlParserIns = this._parser; let parseTree = originParseTree; /** * Reparse the input fragment, * and c3 will collect candidates in the newly generated parseTree when input changed. */ if (inputSlice !== input) { sqlParserIns = this.createParser(inputSlice); parseTree = sqlParserIns.program(); } return { parser: sqlParserIns, parseTree, tokenIndexOffset, newTokenIndex: caretTokenIndex, }; } /** * Get the minimum statement tree for collecting completion candidates * Each supported grammar exposes one top-level statement per direct program child * Keep the lookup shallow to avoid selecting nested statements or subqueries */ private getSuggestionParseTree( parseTree: ParserRuleContext, caretTokenIndex: number ): ParserRuleContext { const children = parseTree.children; if (!children?.length) return parseTree; for (let index = children.length - 1; index >= 0; index--) { const child = children[index]; if (!(child instanceof ParserRuleContext)) continue; const startTokenIndex = child.start?.tokenIndex; const stopTokenIndex = child.stop?.tokenIndex; if ( startTokenIndex === undefined || stopTokenIndex === undefined || startTokenIndex > caretTokenIndex ) continue; // Use the current statement tree when the caret is inside it if (stopTokenIndex >= caretTokenIndex) return child; // Keep using the current statement until it ends with a semicolon return child.stop?.text === SQL_SPLIT_SYMBOL_TEXT ? parseTree : child; } return parseTree; } /** * Collect candidates for the current statement and remove new-statement-only keywords */ private collectSuggestionCandidates( parser: Parser, parseTree: ParserRuleContext, caretTokenIndex: number ): CandidatesCollection { const core = new CodeCompletionCore(parser); core.preferredRules = this.preferredRules; const candidates = core.collectCandidates(caretTokenIndex, parseTree); const suggestionParseTree = this.getSuggestionParseTree(parseTree, caretTokenIndex); if (suggestionParseTree === parseTree) return candidates; // Keep the program candidates for outer rule paths and use statement candidates for isolation const statementCore = new CodeCompletionCore(parser); statementCore.preferredRules = this.preferredRules; const statementCandidates = statementCore.collectCandidates( caretTokenIndex, suggestionParseTree ); if (this._statementStartTokenTypes === null) { const statementStartCore = new CodeCompletionCore(parser); statementStartCore.preferredRules = this.preferredRules; const statementStartCandidates = statementStartCore.collectCandidates(0, parseTree); this._statementStartTokenTypes = new Set(statementStartCandidates.tokens.keys()); } const tokens = new Map(candidates.tokens); for (const tokenType of this._statementStartTokenTypes) { if (!statementCandidates.tokens.has(tokenType)) { tokens.delete(tokenType); } else if (tokens.has(tokenType)) { // Use the current statement follow-list to preserve valid combined keywords tokens.set(tokenType, statementCandidates.tokens.get(tokenType)!); } } return { rules: candidates.rules, tokens, }; } /** * Get suggestions of syntax and token at caretPosition * @param input source string * @param caretPosition caret position, such as cursor position * @param options suggestion options * @returns suggestion */ public getSuggestionAtCaretPosition( input: string, caretPosition: CaretPosition, options?: SuggestionOptions ): Suggestions | null { this.parseWithCache(input); if (!this._parseTree) return null; let allTokens = this.getAllTokens(input); let caretTokenIndex = findCaretTokenIndex(caretPosition, allTokens); if (!caretTokenIndex && caretTokenIndex !== 0) return null; const inputInfo = this.getMinimumInputInfo(input, caretTokenIndex, this._parseTree); if (!inputInfo) return null; const { input: _input, tokenIndexOffset, statementCount } = inputInfo; let inputSlice = _input; /** * Split the inputSlice by separator to get the smaller range of inputSlice. */ if (inputSlice.includes(SQL_SPLIT_SYMBOL_TEXT)) { const { inputSlice: _inputSlice, allTokens: _allTokens, caretTokenIndex: _caretTokenIndex, } = this.splitInputBySymbolText( inputSlice, allTokens, tokenIndexOffset, caretTokenIndex ); allTokens = _allTokens; caretTokenIndex = _caretTokenIndex; inputSlice = _inputSlice; } else { if (statementCount > 1) { caretTokenIndex = caretTokenIndex - tokenIndexOffset; allTokens = allTokens.slice(tokenIndexOffset); } } let sqlParserIns = this._parser; let parseTree = this._parseTree; /** * Reparse the input fragment, * and c3 will collect candidates in the newly generated parseTree when input changed. */ if (inputSlice !== input) { sqlParserIns = this.createParser(inputSlice); parseTree = sqlParserIns.program(); } const candidates = this.collectSuggestionCandidates( sqlParserIns, parseTree, caretTokenIndex ); const originalSuggestions = this.processCandidates(candidates, allTokens, caretTokenIndex); const syntaxSuggestions: SyntaxSuggestion[] = originalSuggestions.syntax.map( (syntaxCtx) => { const wordRanges: WordRange[] = syntaxCtx.wordRanges.map((token) => { return tokenToWord(token, this._parsedInput); }); return { syntaxContextType: syntaxCtx.syntaxContextType, wordRanges, }; } ); const keywordFilter = options?.keywordFilter; const keywords = keywordFilter ? originalSuggestions.keywords.filter((keyword) => keywordFilter(keyword)) : originalSuggestions.keywords; return { syntax: syntaxSuggestions, keywords, }; } public getAllEntities(input: string, caretPosition?: CaretPosition): EntityContext[] | null { /** * Create a new parser to generate brand new parse tree. * And the new parse tree should not effect cached parse tree which is used by validate and getSuggestionAtCaretPosition method. */ const parser = this.createParser(input); const allTokens = (parser.tokenStream as CommonTokenStream).getTokens(); const caretTokenIndex = caretPosition ? findCaretTokenIndex(caretPosition, allTokens) : void 0; const collectListener = this.createEntityCollector(input, allTokens, caretTokenIndex); parser.entityCollecting = true; if (caretPosition && caretTokenIndex !== undefined) { parser.caretTokenIndex = caretTokenIndex; } const parseTree = parser.program(); this.listen(collectListener, parseTree); parser.caretTokenIndex = -1; parser.entityCollecting = false; return collectListener.getEntities(); } /** * Get semantic context infos * @param input source string * @param caretPosition caret position, such as cursor position * @param options semantic context options * @returns analyzed semantic context */ public getSemanticContextAtCaretPosition( input: string, caretPosition: CaretPosition, options?: SemanticCollectOptions ) { const allTokens = this.getAllTokens(input); const parseTree = this.parseWithCache(input); const statementContextListener = this.createSemanticContextCollector( input, caretPosition, allTokens, options ); this.listen(statementContextListener, parseTree); return statementContextListener.semanticContext; } }