import { EventEmitter } from 'events'; import { ChildProcess, spawn, SpawnOptions, execFile, execFileSync, } from 'child_process'; import { EOL as newline, tmpdir } from 'os'; import { join, sep } from 'path'; import { Readable, Transform, TransformCallback, Writable } from 'stream'; import { writeFile, writeFileSync } from 'fs'; import { promisify } from 'util'; function toArray(source?: T | T[]): T[] { if (typeof source === 'undefined' || source === null) { return []; } else if (!Array.isArray(source)) { return [source]; } return source; } /** * adds arguments as properties to obj */ function extend(obj: {}, ...args) { Array.prototype.slice.call(arguments, 1).forEach(function (source) { if (source) { for (let key in source) { obj[key] = source[key]; } } }); return obj; } /** * gets a random int from 0-10000000000 */ function getRandomInt() { return Math.floor(Math.random() * 10000000000); } const execFilePromise = promisify(execFile); export interface Options extends SpawnOptions { /** * if binary is enabled message and stderr events will not be emitted */ mode?: 'text' | 'json' | 'binary'; formatter?: string | ((param: string) => any); parser?: string | ((param: string) => any); stderrParser?: string | ((param: string) => any); encoding?: BufferEncoding; pythonPath?: string; /** * see https://docs.python.org/3.7/using/cmdline.html */ pythonOptions?: string[]; /** * overrides scriptPath passed into PythonShell constructor */ scriptPath?: string; /** * arguments to your program */ args?: string[]; } export class PythonShellError extends Error { traceback: string | Buffer; exitCode?: number; } export class PythonShellErrorWithLogs extends PythonShellError { logs: any[]; } /** * Takes in a string stream and emits batches seperated by newlines */ export class NewlineTransformer extends Transform { // NewlineTransformer: Megatron's little known once-removed cousin private _lastLineData: string; _transform(chunk: any, encoding: string, callback: TransformCallback) { let data: string = chunk.toString(); if (this._lastLineData) data = this._lastLineData + data; const lines = data.split(newline); this._lastLineData = lines.pop(); lines.forEach(this.push.bind(this)); callback(); } _flush(done: TransformCallback) { if (this._lastLineData) this.push(this._lastLineData); this._lastLineData = null; done(); } } /** * An interactive Python shell exchanging data through stdio * @param {string} script The python script to execute * @param {object} [options] The launch options (also passed to child_process.spawn) * @param [stdoutSplitter] Optional. Splits stdout into chunks, defaulting to splitting into newline-seperated lines * @param [stderrSplitter] Optional. splits stderr into chunks, defaulting to splitting into newline-seperated lines * @constructor */ export class PythonShell extends EventEmitter { scriptPath: string; command: string[]; mode: string; formatter: (param: string | Object) => any; parser: (param: string) => any; stderrParser: (param: string) => any; terminated: boolean; childProcess: ChildProcess; stdin: Writable; stdout: Readable; stderr: Readable; exitSignal: string; exitCode: number; private stderrHasEnded: boolean; private stdoutHasEnded: boolean; private _remaining: string; private _endCallback: ( err: PythonShellError, exitCode: number, exitSignal: string, ) => any; // starting 2020 python2 is deprecated so we choose 3 as default static defaultPythonPath = process.platform != 'win32' ? 'python3' : 'python'; static defaultOptions: Options = {}; //allow global overrides for options /** * spawns a python process * @param scriptPath path to script. Relative to current directory or options.scriptFolder if specified * @param options * @param stdoutSplitter Optional. Splits stdout into chunks, defaulting to splitting into newline-seperated lines * @param stderrSplitter Optional. splits stderr into chunks, defaulting to splitting into newline-seperated lines */ constructor( scriptPath: string, options?: Options, stdoutSplitter: Transform = null, stderrSplitter: Transform = null, ) { super(); /** * returns either pythonshell func (if val string) or custom func (if val Function) */ function resolve(type, val: string | Function) { if (typeof val === 'string') { // use a built-in function using its name return PythonShell[type][val]; } else if (typeof val === 'function') { // use a custom function return val; } } if (scriptPath.trim().length == 0) throw Error( 'scriptPath cannot be empty! You must give a script for python to run', ); let self = this; let errorData = ''; EventEmitter.call(this); options = extend({}, PythonShell.defaultOptions, options); let pythonPath: string; if (!options.pythonPath) { pythonPath = PythonShell.defaultPythonPath; } else pythonPath = options.pythonPath; let pythonOptions = toArray(options.pythonOptions); let scriptArgs = toArray(options.args); this.scriptPath = join(options.scriptPath || '', scriptPath); this.command = pythonOptions.concat(this.scriptPath, scriptArgs); this.mode = options.mode || 'text'; this.formatter = resolve('format', options.formatter || this.mode); this.parser = resolve('parse', options.parser || this.mode); // We don't expect users to ever format stderr as JSON so we default to text mode this.stderrParser = resolve('parse', options.stderrParser || 'text'); this.terminated = false; this.childProcess = spawn(pythonPath, this.command, options); ['stdout', 'stdin', 'stderr'].forEach(function (name) { self[name] = self.childProcess[name]; self.parser && self[name] && self[name].setEncoding(options.encoding || 'utf8'); }); // Node buffers stdout&stderr in batches regardless of newline placement // This is troublesome if you want to recieve distinct individual messages // for example JSON parsing breaks if it recieves partial JSON // so we use newlineTransformer to emit each batch seperated by newline if (this.parser && this.stdout) { if (!stdoutSplitter) stdoutSplitter = new NewlineTransformer(); // note that setting the encoding turns the chunk into a string stdoutSplitter.setEncoding(options.encoding || 'utf8'); this.stdout.pipe(stdoutSplitter).on('data', (chunk: string) => { this.emit('message', self.parser(chunk)); }); } // listen to stderr and emit errors for incoming data if (this.stderrParser && this.stderr) { if (!stderrSplitter) stderrSplitter = new NewlineTransformer(); // note that setting the encoding turns the chunk into a string stderrSplitter.setEncoding(options.encoding || 'utf8'); this.stderr.pipe(stderrSplitter).on('data', (chunk: string) => { this.emit('stderr', self.stderrParser(chunk)); }); } if (this.stderr) { this.stderr.on('data', function (data) { errorData += '' + data; }); this.stderr.on('end', function () { self.stderrHasEnded = true; terminateIfNeeded(); }); } else { self.stderrHasEnded = true; } if (this.stdout) { this.stdout.on('end', function () { self.stdoutHasEnded = true; terminateIfNeeded(); }); } else { self.stdoutHasEnded = true; } this.childProcess.on('error', function (err: NodeJS.ErrnoException) { self.emit('error', err); }); this.childProcess.on('exit', function (code, signal) { self.exitCode = code; self.exitSignal = signal; terminateIfNeeded(); }); function terminateIfNeeded() { if ( !self.stderrHasEnded || !self.stdoutHasEnded || (self.exitCode == null && self.exitSignal == null) ) return; let err: PythonShellError; if (self.exitCode && self.exitCode !== 0) { if (errorData) { err = self.parseError(errorData); } else { err = new PythonShellError( 'process exited with code ' + self.exitCode, ); } err = extend(err, { executable: pythonPath, options: pythonOptions.length ? pythonOptions : null, script: self.scriptPath, args: scriptArgs.length ? scriptArgs : null, exitCode: self.exitCode, }); // do not emit error if only a callback is used if (self.listeners('pythonError').length || !self._endCallback) { self.emit('pythonError', err); } } self.terminated = true; self.emit('close'); self._endCallback && self._endCallback(err, self.exitCode, self.exitSignal); } } // built-in formatters static format = { text: function toText(data): string { if (!data) return ''; else if (typeof data !== 'string') return data.toString(); return data; }, json: function toJson(data) { return JSON.stringify(data); }, }; //built-in parsers static parse = { text: function asText(data): string { return data; }, json: function asJson(data: string) { return JSON.parse(data); }, }; /** * checks syntax without executing code * @returns rejects promise w/ string error output if syntax failure */ static async checkSyntax(code: string) { const randomInt = getRandomInt(); const filePath = tmpdir() + sep + `pythonShellSyntaxCheck${randomInt}.py`; const writeFilePromise = promisify(writeFile); return writeFilePromise(filePath, code).then(() => { return this.checkSyntaxFile(filePath); }); } static getPythonPath() { return this.defaultOptions.pythonPath ? this.defaultOptions.pythonPath : this.defaultPythonPath; } /** * checks syntax without executing code * @returns {Promise} rejects w/ stderr if syntax failure */ static async checkSyntaxFile(filePath: string) { const pythonPath = this.getPythonPath(); return execFilePromise(pythonPath, ['-m', 'py_compile', filePath]); } /** * Runs a Python script and returns collected messages as a promise. * If the promise is rejected, the err will probably be of type PythonShellErrorWithLogs * @param scriptPath The path to the script to execute * @param options The execution options */ static run(scriptPath: string, options?: Options): Promise { return new Promise((resolve, reject) => { let pyshell = new PythonShell(scriptPath, options); let output = []; pyshell .on('message', function (message) { output.push(message); }) .end(function (err) { if (err) { (err as PythonShellErrorWithLogs).logs = output; reject(err); } else resolve(output); }); }); } /** * Runs the inputted string of python code and returns collected messages as a promise. DO NOT ALLOW UNTRUSTED USER INPUT HERE! * @param code The python code to execute * @param options The execution options * @return a promise with the output from the python script */ static runString(code: string, options?: Options) { // put code in temp file const randomInt = getRandomInt(); const filePath = tmpdir + sep + `pythonShellFile${randomInt}.py`; writeFileSync(filePath, code); return PythonShell.run(filePath, options); } static getVersion(pythonPath?: string) { if (!pythonPath) pythonPath = this.getPythonPath(); return execFilePromise(pythonPath, ['--version']); } static getVersionSync(pythonPath?: string) { if (!pythonPath) pythonPath = this.getPythonPath(); return execFileSync(pythonPath, ['--version']).toString(); } /** * Parses an error thrown from the Python process through stderr * @param {string|Buffer} data The stderr contents to parse * @return {Error} The parsed error with extended stack trace when traceback is available */ private parseError(data: string | Buffer) { let text = '' + data; let error: PythonShellError; if (/^Traceback/.test(text)) { // traceback data is available let lines = text.trim().split(newline); let exception = lines.pop(); error = new PythonShellError(exception); error.traceback = data; // extend stack trace error.stack += newline + ' ----- Python Traceback -----' + newline + ' '; error.stack += lines.slice(1).join(newline + ' '); } else { // otherwise, create a simpler error with stderr contents error = new PythonShellError(text); } return error; } /** * Sends a message to the Python shell through stdin * Override this method to format data to be sent to the Python process * @returns {PythonShell} The same instance for chaining calls */ send(message: string | Object) { if (!this.stdin) throw new Error('stdin not open for writing'); let data = this.formatter ? this.formatter(message) : message; if (this.mode !== 'binary') data += newline; this.stdin.write(data); return this; } /** * Closes the stdin stream. Unless python is listening for stdin in a loop * this should cause the process to finish its work and close. * @returns {PythonShell} The same instance for chaining calls */ end( callback: ( err: PythonShellError, exitCode: number, exitSignal: string, ) => any, ) { if (this.childProcess.stdin) { this.childProcess.stdin.end(); } this._endCallback = callback; return this; } /** * Sends a kill signal to the process * @returns {PythonShell} The same instance for chaining calls */ kill(signal?: NodeJS.Signals) { this.terminated = this.childProcess.kill(signal); return this; } /** * Alias for kill. * @deprecated */ terminate(signal?: NodeJS.Signals) { // todo: remove this next breaking release return this.kill(signal); } } // This interface is merged in with the above class definition export interface PythonShell { addListener(event: string, listener: (...args: any[]) => void): this; emit(event: string | symbol, ...args: any[]): boolean; on(event: string, listener: (...args: any[]) => void): this; once(event: string, listener: (...args: any[]) => void): this; prependListener(event: string, listener: (...args: any[]) => void): this; prependOnceListener(event: string, listener: (...args: any[]) => void): this; addListener(event: 'message', listener: (parsedChunk: any) => void): this; emit(event: 'message', parsedChunk: any): boolean; on(event: 'message', listener: (parsedChunk: any) => void): this; once(event: 'message', listener: (parsedChunk: any) => void): this; prependListener(event: 'message', listener: (parsedChunk: any) => void): this; prependOnceListener( event: 'message', listener: (parsedChunk: any) => void, ): this; addListener(event: 'stderr', listener: (parsedChunk: any) => void): this; emit(event: 'stderr', parsedChunk: any): boolean; on(event: 'stderr', listener: (parsedChunk: any) => void): this; once(event: 'stderr', listener: (parsedChunk: any) => void): this; prependListener(event: 'stderr', listener: (parsedChunk: any) => void): this; prependOnceListener( event: 'stderr', listener: (parsedChunk: any) => void, ): this; addListener(event: 'close', listener: () => void): this; emit(event: 'close'): boolean; on(event: 'close', listener: () => void): this; once(event: 'close', listener: () => void): this; prependListener(event: 'close', listener: () => void): this; prependOnceListener(event: 'close', listener: () => void): this; addListener( event: 'error', listener: (error: NodeJS.ErrnoException) => void, ): this; emit(event: 'error', error: NodeJS.ErrnoException): boolean; on(event: 'error', listener: (error: NodeJS.ErrnoException) => void): this; once(event: 'error', listener: (error: NodeJS.ErrnoException) => void): this; prependListener( event: 'error', listener: (error: NodeJS.ErrnoException) => void, ): this; prependOnceListener( event: 'error', listener: (error: NodeJS.ErrnoException) => void, ): this; addListener( event: 'pythonError', listener: (error: PythonShellError) => void, ): this; emit(event: 'pythonError', error: PythonShellError): boolean; on(event: 'pythonError', listener: (error: PythonShellError) => void): this; once(event: 'pythonError', listener: (error: PythonShellError) => void): this; prependListener( event: 'pythonError', listener: (error: PythonShellError) => void, ): this; prependOnceListener( event: 'pythonError', listener: (error: PythonShellError) => void, ): this; }