import { EventEmitter } from 'events'; import { ChildProcess, spawn, SpawnOptions, exec, execSync } from 'child_process'; import { EOL as newline, tmpdir } from 'os'; import { join, sep } from 'path' import { Readable, Transform, TransformCallback, Writable } from 'stream' import { writeFile, writeFileSync } from 'fs'; import { promisify } from 'util'; function toArray(source?: T | T[]): T[] { if (typeof source === 'undefined' || source === null) { return []; } else if (!Array.isArray(source)) { return [source]; } return source; } /** * adds arguments as properties to obj */ function extend(obj: {}, ...args) { Array.prototype.slice.call(arguments, 1).forEach(function (source) { if (source) { for (let key in source) { obj[key] = source[key]; } } }); return obj; } /** * gets a random int from 0-10000000000 */ function getRandomInt() { return Math.floor(Math.random() * 10000000000); } const execPromise = promisify(exec) export interface Options extends SpawnOptions { /** * if binary is enabled message and stderr events will not be emitted */ mode?: 'text' | 'json' | 'binary' formatter?: string | ((param: string) => any) parser?: string | ((param: string) => any) stderrParser?: string | ((param: string) => any) encoding?: BufferEncoding pythonPath?: string /** * see https://docs.python.org/3.7/using/cmdline.html */ pythonOptions?: string[] /** * overrides scriptPath passed into PythonShell constructor */ scriptPath?: string /** * arguments to your program */ args?: string[] } export class PythonShellError extends Error { traceback: string | Buffer; exitCode?: number; } /** * Takes in a string stream and emits batches seperated by newlines */ export class NewlineTransformer extends Transform { // NewlineTransformer: Megatron's little known once-removed cousin private _lastLineData: string; _transform(chunk: any, encoding: string, callback: TransformCallback){ let data: string = chunk.toString() if (this._lastLineData) data = this._lastLineData + data const lines = data.split(newline) this._lastLineData = lines.pop() //@ts-ignore this works, node ignores the encoding if it's a number lines.forEach(this.push.bind(this)) callback() } _flush(done: TransformCallback){ if (this._lastLineData) this.push(this._lastLineData) this._lastLineData = null; done() } } /** * An interactive Python shell exchanging data through stdio * @param {string} script The python script to execute * @param {object} [options] The launch options (also passed to child_process.spawn) * @param [stdoutSplitter] Optional. Splits stdout into chunks, defaulting to splitting into newline-seperated lines * @param [stderrSplitter] Optional. splits stderr into chunks, defaulting to splitting into newline-seperated lines * @constructor */ export class PythonShell extends EventEmitter { scriptPath: string command: string[] mode: string formatter: (param: string | Object) => any parser: (param: string) => any stderrParser: (param: string) => any terminated: boolean childProcess: ChildProcess stdin: Writable; stdout: Readable; stderr: Readable; exitSignal: string; exitCode: number; private stderrHasEnded: boolean; private stdoutHasEnded: boolean; private _remaining: string private _endCallback: (err: PythonShellError, exitCode: number, exitSignal: string) => any // starting 2020 python2 is deprecated so we choose 3 as default static defaultPythonPath = process.platform != "win32" ? "python3" : "python"; static defaultOptions: Options = {}; //allow global overrides for options /** * spawns a python process * @param scriptPath path to script. Relative to current directory or options.scriptFolder if specified * @param options * @param stdoutSplitter Optional. Splits stdout into chunks, defaulting to splitting into newline-seperated lines * @param stderrSplitter Optional. splits stderr into chunks, defaulting to splitting into newline-seperated lines */ constructor(scriptPath: string, options?: Options, stdoutSplitter: Transform = null, stderrSplitter: Transform = null) { super(); /** * returns either pythonshell func (if val string) or custom func (if val Function) */ function resolve(type, val: string | Function) { if (typeof val === 'string') { // use a built-in function using its name return PythonShell[type][val]; } else if (typeof val === 'function') { // use a custom function return val; } } if (scriptPath.trim().length == 0) throw Error("scriptPath cannot be empty! You must give a script for python to run") let self = this; let errorData = ''; EventEmitter.call(this); options = extend({}, PythonShell.defaultOptions, options); let pythonPath: string; if (!options.pythonPath) { pythonPath = PythonShell.defaultPythonPath; } else pythonPath = options.pythonPath; let pythonOptions = toArray(options.pythonOptions); let scriptArgs = toArray(options.args); this.scriptPath = join(options.scriptPath || '', scriptPath); this.command = pythonOptions.concat(this.scriptPath, scriptArgs); this.mode = options.mode || 'text'; this.formatter = resolve('format', options.formatter || this.mode); this.parser = resolve('parse', options.parser || this.mode); // We don't expect users to ever format stderr as JSON so we default to text mode this.stderrParser = resolve('parse', options.stderrParser || 'text'); this.terminated = false; this.childProcess = spawn(pythonPath, this.command, options); ['stdout', 'stdin', 'stderr'].forEach(function (name) { self[name] = self.childProcess[name]; self.parser && self[name] && self[name].setEncoding(options.encoding || 'utf8'); }); // Node buffers stdout&stderr in batches regardless of newline placement // This is troublesome if you want to recieve distinct individual messages // for example JSON parsing breaks if it recieves partial JSON // so we use newlineTransformer to emit each batch seperated by newline if (this.parser && this.stdout) { if(!stdoutSplitter) stdoutSplitter = new NewlineTransformer() // note that setting the encoding turns the chunk into a string stdoutSplitter.setEncoding(options.encoding || 'utf8') this.stdout.pipe(stdoutSplitter).on('data', (chunk: string) => { this.emit('message', self.parser(chunk)); }); } // listen to stderr and emit errors for incoming data if (this.stderrParser && this.stderr) { if(!stderrSplitter) stderrSplitter = new NewlineTransformer() // note that setting the encoding turns the chunk into a string stderrSplitter.setEncoding(options.encoding || 'utf8') this.stderr.pipe(stderrSplitter).on('data', (chunk: string) => { this.emit('stderr', self.stderrParser(chunk)); }); } if (this.stderr) { this.stderr.on('data', function (data) { errorData += '' + data; }); this.stderr.on('end', function () { self.stderrHasEnded = true; terminateIfNeeded(); }); } else { self.stderrHasEnded = true; } if (this.stdout) { this.stdout.on('end', function () { self.stdoutHasEnded = true; terminateIfNeeded(); }); } else { self.stdoutHasEnded = true; } this.childProcess.on('error', function (err: NodeJS.ErrnoException) { self.emit('error', err); }) this.childProcess.on('exit', function (code, signal) { self.exitCode = code; self.exitSignal = signal; terminateIfNeeded(); }); function terminateIfNeeded() { if (!self.stderrHasEnded || !self.stdoutHasEnded || (self.exitCode == null && self.exitSignal == null)) return; let err: PythonShellError; if (self.exitCode && self.exitCode !== 0) { if (errorData) { err = self.parseError(errorData); } else { err = new PythonShellError('process exited with code ' + self.exitCode); } err = extend(err, { executable: pythonPath, options: pythonOptions.length ? pythonOptions : null, script: self.scriptPath, args: scriptArgs.length ? scriptArgs : null, exitCode: self.exitCode }); // do not emit error if only a callback is used if (self.listeners('pythonError').length || !self._endCallback) { self.emit('pythonError', err); } } self.terminated = true; self.emit('close'); self._endCallback && self._endCallback(err, self.exitCode, self.exitSignal); }; } // built-in formatters static format = { text: function toText(data): string { if (!data) return ''; else if (typeof data !== 'string') return data.toString(); return data; }, json: function toJson(data) { return JSON.stringify(data); } }; //built-in parsers static parse = { text: function asText(data): string { return data; }, json: function asJson(data: string) { return JSON.parse(data); } }; /** * checks syntax without executing code * @returns rejects promise w/ string error output if syntax failure */ static async checkSyntax(code: string) { const randomInt = getRandomInt(); const filePath = tmpdir() + sep + `pythonShellSyntaxCheck${randomInt}.py` const writeFilePromise = promisify(writeFile) return writeFilePromise(filePath, code).then(() => { return this.checkSyntaxFile(filePath) }) } static getPythonPath() { return this.defaultOptions.pythonPath ? this.defaultOptions.pythonPath : this.defaultPythonPath; } /** * checks syntax without executing code * @returns {Promise} rejects w/ stderr if syntax failure */ static async checkSyntaxFile(filePath: string) { const pythonPath = this.getPythonPath() let compileCommand = `${pythonPath} -m py_compile ${filePath}` return execPromise(compileCommand) } /** * Runs a Python script and returns collected messages * @param {string} scriptPath The path to the script to execute * @param {Options} options The execution options * @param {Function} callback The callback function to invoke with the script results * @return {PythonShell} The PythonShell instance */ static run(scriptPath: string, options?: Options, callback?: (err?: PythonShellError, output?: any[]) => any) { let pyshell = new PythonShell(scriptPath, options); let output = []; return pyshell.on('message', function (message) { output.push(message); }).end(function (err) { return callback(err ? err : null, output.length ? output : null); }); }; /** * Runs the inputted string of python code and returns collected messages. DO NOT ALLOW UNTRUSTED USER INPUT HERE! * @param {string} code The python code to execute * @param {Options} options The execution options * @param {Function} callback The callback function to invoke with the script results * @return {PythonShell} The PythonShell instance */ static runString(code: string, options?: Options, callback?: (err: PythonShellError, output?: any[]) => any) { // put code in temp file const randomInt = getRandomInt(); const filePath = tmpdir + sep + `pythonShellFile${randomInt}.py` writeFileSync(filePath, code); return PythonShell.run(filePath, options, callback); }; static getVersion(pythonPath?: string) { if (!pythonPath) pythonPath = this.getPythonPath() return execPromise(pythonPath + " --version"); } static getVersionSync(pythonPath?: string) { if (!pythonPath) pythonPath = this.getPythonPath() return execSync(pythonPath + " --version").toString() } /** * Parses an error thrown from the Python process through stderr * @param {string|Buffer} data The stderr contents to parse * @return {Error} The parsed error with extended stack trace when traceback is available */ private parseError(data: string | Buffer) { let text = '' + data; let error: PythonShellError; if (/^Traceback/.test(text)) { // traceback data is available let lines = text.trim().split(newline); let exception = lines.pop(); error = new PythonShellError(exception); error.traceback = data; // extend stack trace error.stack += newline + ' ----- Python Traceback -----' + newline + ' '; error.stack += lines.slice(1).join(newline + ' '); } else { // otherwise, create a simpler error with stderr contents error = new PythonShellError(text); } return error; }; /** * Sends a message to the Python shell through stdin * Override this method to format data to be sent to the Python process * @returns {PythonShell} The same instance for chaining calls */ send(message: string | Object) { if (!this.stdin) throw new Error("stdin not open for writing"); let data = this.formatter ? this.formatter(message) : message; if (this.mode !== 'binary') data += newline; this.stdin.write(data); return this; }; /** * Closes the stdin stream. Unless python is listening for stdin in a loop * this should cause the process to finish its work and close. * @returns {PythonShell} The same instance for chaining calls */ end(callback: (err: PythonShellError, exitCode: number, exitSignal: string) => any) { if (this.childProcess.stdin) { this.childProcess.stdin.end(); } this._endCallback = callback; return this; }; /** * Sends a kill signal to the process * @returns {PythonShell} The same instance for chaining calls */ kill(signal?: NodeJS.Signals) { this.terminated = this.childProcess.kill(signal); return this; }; /** * Alias for kill. * @deprecated */ terminate(signal?: NodeJS.Signals) { // todo: remove this next breaking release return this.kill(signal) } }; // This interface is merged in with the above class definition export interface PythonShell { addListener(event: string, listener: (...args: any[]) => void): this; emit(event: string | symbol, ...args: any[]): boolean; on(event: string, listener: (...args: any[]) => void): this; once(event: string, listener: (...args: any[]) => void): this; prependListener(event: string, listener: (...args: any[]) => void): this; prependOnceListener(event: string, listener: (...args: any[]) => void): this; addListener(event: "message", listener: (parsedChunk: any) => void): this; emit(event: "message", parsedChunk: any): boolean; on(event: "message", listener: (parsedChunk: any) => void): this; once(event: "message", listener: (parsedChunk: any) => void): this; prependListener(event: "message", listener: (parsedChunk: any) => void): this; prependOnceListener(event: "message", listener: (parsedChunk: any) => void): this; addListener(event: "stderr", listener: (parsedChunk: any) => void): this; emit(event: "stderr", parsedChunk: any): boolean; on(event: "stderr", listener: (parsedChunk: any) => void): this; once(event: "stderr", listener: (parsedChunk: any) => void): this; prependListener(event: "stderr", listener: (parsedChunk: any) => void): this; prependOnceListener(event: "stderr", listener: (parsedChunk: any) => void): this; addListener(event: "close", listener: () => void): this; emit(event: "close",): boolean; on(event: "close", listener: () => void): this; once(event: "close", listener: () => void): this; prependListener(event: "close", listener: () => void): this; prependOnceListener(event: "close", listener: () => void): this; addListener(event: "error", listener: (error: NodeJS.ErrnoException) => void): this; emit(event: "error", error: NodeJS.ErrnoException): boolean; on(event: "error", listener: (error: NodeJS.ErrnoException) => void): this; once(event: "error", listener: (error: NodeJS.ErrnoException) => void): this; prependListener(event: "error", listener: (error: NodeJS.ErrnoException) => void): this; prependOnceListener(event: "error", listener: (error: NodeJS.ErrnoException) => void): this; addListener(event: "pythonError", listener: (error: PythonShellError) => void): this; emit(event: "pythonError", error: PythonShellError): boolean; on(event: "pythonError", listener: (error: PythonShellError) => void): this; once(event: "pythonError", listener: (error: PythonShellError) => void): this; prependListener(event: "pythonError", listener: (error: PythonShellError) => void): this; prependOnceListener(event: "pythonError", listener: (error: PythonShellError) => void): this; }