#!/bin/sh _=[[ exec lua "$0" "$@" ]]and nil --============================================================== --= --= LuaPreprocess command line program --= by Marcus 'ReFreezed' Thunström --= --= Requires preprocess.lua to be in the same folder! --= --= License: MIT (see the bottom of this file) --= Website: http://refreezed.com/luapreprocess/ --= Documentation: http://refreezed.com/luapreprocess/docs/command-line/ --= --= Tested with Lua 5.1, 5.2, 5.3, 5.4 and LuaJIT. --= --============================================================== local help = [[ Script usage: lua preprocess-cl.lua [options] [--] filepath1 [filepath2 ...] OR lua preprocess-cl.lua --outputpaths [options] [--] inputpath1 outputpath1 [inputpath2 outputpath2 ...] File paths can be "-" for usage of stdin/stdout. Examples: lua preprocess-cl.lua --saveinfo=logs/info.lua --silent src/main.lua2p src/network.lua2p lua preprocess-cl.lua --debug src/main.lua2p src/network.lua2p lua preprocess-cl.lua --outputpaths --linenumbers src/main.lua2p output/main.lua src/network.lua2p output/network.lua Options: --backtickstrings Enable the backtick (`) to be used as string literal delimiters. Backtick strings don't interpret any escape sequences and can't contain other backticks. --data|-d="Any data." A string with any data. If this option is present then the value will be available through the global 'dataFromCommandLine' in the processed files (and any message handler). Otherwise, 'dataFromCommandLine' is nil. --faststrings Force fast serialization of string values. (Non-ASCII characters will look ugly.) --handler|-h=pathToMessageHandler Path to a Lua file that's expected to return a function or a table of functions. If it returns a function then it will be called with various messages as it's first argument. If it's a table, the keys should be the message names and the values should be functions to handle the respective message. (See 'Handler messages' and tests/quickTestHandler*.lua) The file shares the same environment as the processed files. --help Show this help. --jitsyntax Allow LuaJIT-specific syntax, specifically literals for 64-bit integers, complex numbers and binary numbers. (https://luajit.org/ext_ffi_api.html#literals) --linenumbers Add comments with line numbers to the output. --loglevel=levelName Set maximum log level for the @@LOG() macro. Can be "off", "error", "warning", "info", "debug" or "trace". The default is "trace", which enables all logging. --macroprefix=prefix String to prepend to macro names. --macrosuffix=suffix String to append to macro names. --meta OR --meta=pathToSaveMetaprogramTo Output the metaprogram to a temporary file (*.meta.lua). Useful if an error happens when the metaprogram runs. This file is removed if there's no error and --debug isn't enabled. --nogc Stop the garbage collector. This may speed up the preprocessing. --nonil Disallow !(expression) and outputValue() from outputting nil. --nostrictmacroarguments Disable checks that macro arguments are valid Lua expressions. --novalidate Disable validation of outputted Lua. --outputextension=fileExtension Specify what file extension generated files should have. The default is "lua". If any input files end in .lua then you must specify another file extension with this option. (It's suggested that you use .lua2p (as in "Lua To Process") as extension for unprocessed files.) --outputpaths|-o This flag makes every other specified path be the output path for the previous path. --release Enable release mode. Currently only disables the @@ASSERT() macro. --saveinfo|-i=pathToSaveProcessingInfoTo Processing information includes what files had any preprocessor code in them, and things like that. The format of the file is a lua module that returns a table. Search this file for 'SavedInfo' to see what information is saved. --silent Only print errors to the console. (This flag is automatically enabled if an output path is stdout.) --version Print the version of LuaPreprocess to stdout and exit. --debug Enable some preprocessing debug features. Useful if you want to inspect the generated metaprogram (*.meta.lua). (This also enables the --meta option.) -- Stop options from being parsed further. Needed if you have paths starting with "-" (except for usage of stdin/stdout). Handler messages: "init" Sent before any other message. Arguments: inputPaths: Array of file paths to process. Paths can be added or removed freely. outputPaths: If the --outputpaths option is present this is an array of output paths for the respective path in inputPaths, otherwise it's nil. "insert" Sent for each @insert"name" statement. The handler is expected to return a Lua code string. Arguments: path: The file being processed. name: The name of the resource to be inserted (could be a file path or anything). "beforemeta" Sent before a file's metaprogram runs, if a metaprogram is generated. Arguments: path: The file being processed. luaString: The generated metaprogram. "aftermeta" Sent after a file's metaprogram has produced output (before the output is written to a file). Arguments: path: The file being processed. luaString: The produced Lua code. You can modify this and return the modified string. "filedone" Sent after a file has finished processing and the output written to file. Arguments: path: The file being processed. outputPath: Where the output of the metaprogram was written. info: Info about the processed file. (See 'ProcessInfo' in preprocess.lua) "fileerror" Sent if an error happens while processing a file (right before the program exits). Arguments: path: The file being processed. error: The error message. "alldone" Sent after all other messages (right before the program exits). Arguments: (none) ]] --============================================================== local startTime = os.time() local startClock = os.clock() local args = arg if not args[0] then error("Expected to run from the Lua interpreter.") end local pp = dofile((args[0]:gsub("[^/\\]+$", "preprocess.lua"))) -- From args: local addLineNumbers = false local allowBacktickStrings = false local allowJitSyntax = false local canOutputNil = true local customData = nil local fastStrings = false local hasOutputExtension = false local hasOutputPaths = false local isDebug = false local outputExtension = "lua" local outputMeta = false -- flag|path local processingInfoPath = "" local silent = false local validate = true local macroPrefix = "" local macroSuffix = "" local releaseMode = false local maxLogLevel = "trace" local strictMacroArguments = true --============================================================== --= Local functions ============================================ --============================================================== local F = string.format local function formatBytes(n) if n >= 1024*1024*1024 then return F("%.2f GiB", n/(1024*1024*1024)) elseif n >= 1024*1024 then return F("%.2f MiB", n/(1024*1024)) elseif n >= 1024 then return F("%.2f KiB", n/(1024)) elseif n == 1 then return F("1 byte", n) else return F("%d bytes", n) end end local function printfNoise(s, ...) print(s:format(...)) end local function printError(s) io.stderr:write(s, "\n") end local function printfError(s, ...) io.stderr:write(s:format(...), "\n") end local function errorLine(err) printError(pp.tryToFormatError(err)) os.exit(1) end local loadLuaFile = ( (_VERSION >= "Lua 5.2" or jit) and function(path, env) return loadfile(path, "bt", env) end or function(path, env) local chunk, err = loadfile(path) if not chunk then return nil, err end if env then setfenv(chunk, env) end return chunk end ) --============================================================== --= Preprocessor script ======================================== --============================================================== io.stdout:setvbuf("no") io.stderr:setvbuf("no") math.randomseed(os.time()) -- In case math.random() is used anywhere. math.random() -- Must kickstart... local processOptions = true local messageHandlerPath = "" local pathsIn = {} local pathsOut = {} for _, arg in ipairs(args) do if processOptions and (arg:find"^%-%-?help$" or arg == "/?" or arg:find"^/[Hh][Ee][Ll][Pp]$") then print("LuaPreprocess v"..pp.VERSION) print((help:gsub("\t", " "))) os.exit() elseif not (processOptions and arg:find"^%-.") then local paths = (hasOutputPaths and #pathsOut < #pathsIn) and pathsOut or pathsIn table.insert(paths, arg) if arg == "-" and (not hasOutputPaths or paths == pathsOut) then silent = true end elseif arg == "--" then processOptions = false elseif arg:find"^%-%-data=" or arg:find"^%-d=" then customData = arg:gsub("^.-=", "") elseif arg == "--backtickstrings" then allowBacktickStrings = true elseif arg == "--debug" then isDebug = true outputMeta = outputMeta or true elseif arg:find"^%-%-handler=" or arg:find"^%-h=" then messageHandlerPath = arg:gsub("^.-=", "") elseif arg == "--jitsyntax" then allowJitSyntax = true elseif arg == "--linenumbers" then addLineNumbers = true elseif arg == "--meta" then outputMeta = true elseif arg:find"^%-%-meta=" then outputMeta = arg:gsub("^.-=", "") elseif arg == "--nonil" then canOutputNil = false elseif arg == "--novalidate" then validate = false elseif arg:find"^%-%-outputextension=" then if hasOutputPaths then errorLine("Cannot specify both --outputextension and --outputpaths") end hasOutputExtension = true outputExtension = arg:gsub("^.-=", "") elseif arg == "--outputpaths" or arg == "-o" then if hasOutputExtension then errorLine("Cannot specify both --outputpaths and --outputextension") elseif pathsIn[1] then errorLine(arg.." must appear before any input path.") end hasOutputPaths = true elseif arg:find"^%-%-saveinfo=" or arg:find"^%-i=" then processingInfoPath = arg:gsub("^.-=", "") elseif arg == "--silent" then silent = true elseif arg == "--faststrings" then fastStrings = true elseif arg == "--nogc" then collectgarbage("stop") elseif arg:find"^%-%-macroprefix=" then macroPrefix = arg:gsub("^.-=", "") elseif arg:find"^%-%-macrosuffix=" then macroSuffix = arg:gsub("^.-=", "") elseif arg == "--release" then releaseMode = true elseif arg:find"^%-%-loglevel=" then maxLogLevel = arg:gsub("^.-=", "") elseif arg == "--version" then io.stdout:write(pp.VERSION) os.exit() elseif arg == "--nostrictmacroarguments" then strictMacroArguments = false else errorLine("Unknown option '"..arg:gsub("=.*", "").."'.") end end if silent then printfNoise = function()end end local header = "= LuaPreprocess v"..pp.VERSION..os.date(", %Y-%m-%d %H:%M:%S =", startTime) printfNoise(("="):rep(#header)) printfNoise("%s", header) printfNoise(("="):rep(#header)) if hasOutputPaths and #pathsOut < #pathsIn then errorLine("Missing output path for "..pathsIn[#pathsIn]) end -- Prepare metaEnvironment. pp.metaEnvironment.dataFromCommandLine = customData -- May be nil. -- Load message handler. local messageHandler = nil local function hasMessageHandler(message) if not messageHandler then return false elseif type(messageHandler) == "function" then return true elseif type(messageHandler) == "table" then return messageHandler[message] ~= nil else assert(false) end end local function sendMessage(message, ...) if not messageHandler then return elseif type(messageHandler) == "function" then local returnValues = pp.pack(messageHandler(message, ...)) return pp.unpack(returnValues, 1, returnValues.n) elseif type(messageHandler) == "table" then local _messageHandler = messageHandler[message] if not _messageHandler then return end local returnValues = pp.pack(_messageHandler(...)) return pp.unpack(returnValues, 1, returnValues.n) else assert(false) end end if messageHandlerPath ~= "" then -- Make the message handler and the metaprogram share the same environment. -- This way the message handler can easily define globals that the metaprogram uses. local mainChunk, err = loadLuaFile(messageHandlerPath, pp.metaEnvironment) if not mainChunk then errorLine("Could not load message handler...\n"..pp.tryToFormatError(err)) end messageHandler = mainChunk() if type(messageHandler) == "function" then -- void elseif type(messageHandler) == "table" then for message, _messageHandler in pairs(messageHandler) do if type(message) ~= "string" then errorLine(messageHandlerPath..": Table of handlers must only contain messages as keys.") elseif type(_messageHandler) ~= "function" then errorLine(messageHandlerPath..": Table of handlers must only contain functions as values.") end end else errorLine(messageHandlerPath..": File did not return a table or a function.") end end -- Init stuff. sendMessage("init", pathsIn, (hasOutputPaths and pathsOut or nil)) -- @Incomplete: Use pcall and format error message better? if not hasOutputPaths then for i, pathIn in ipairs(pathsIn) do pathsOut[i] = (pathIn == "-") and "-" or pathIn:gsub("%.%w+$", "").."."..outputExtension end end if not pathsIn[1] then errorLine("No path(s) specified.") elseif #pathsIn ~= #pathsOut then errorLine(F("Number of input and output paths differ. (%d in, %d out)", #pathsIn, #pathsOut)) end local pathsSetIn = {} local pathsSetOut = {} for i = 1, #pathsIn do if pathsSetIn [pathsIn [i]] then errorLine("Duplicate input path: " ..pathsIn [i]) end if pathsSetOut[pathsOut[i]] then errorLine("Duplicate output path: "..pathsOut[i]) end pathsSetIn [pathsIn [i]] = true pathsSetOut[pathsOut[i]] = true if pathsIn [i] ~= "-" and pathsSetOut[pathsIn [i]] then errorLine("Path is both input and output: "..pathsIn [i]) end if pathsOut[i] ~= "-" and pathsSetIn [pathsOut[i]] then errorLine("Path is both input and output: "..pathsOut[i]) end end -- Process files. -- :SavedInfo local processingInfo = { date = os.date("%Y-%m-%d %H:%M:%S", startTime), files = {}, } local byteCount = 0 local lineCount = 0 local lineCountCode = 0 local tokenCount = 0 for i, pathIn in ipairs(pathsIn) do local startClockForPath = os.clock() printfNoise("Processing '%s'...", pathIn) local pathOut = pathsOut[i] local pathMeta = (type(outputMeta) == "string") and outputMeta or pathOut:gsub("%.%w+$", "")..".meta.lua" if not outputMeta or pathOut == "-" then pathMeta = nil end local info, err = pp.processFile{ pathIn = pathIn, pathMeta = pathMeta, pathOut = pathOut, debug = isDebug, addLineNumbers = addLineNumbers, backtickStrings = allowBacktickStrings, jitSyntax = allowJitSyntax, canOutputNil = canOutputNil, fastStrings = fastStrings, validate = validate, strictMacroArguments = strictMacroArguments, macroPrefix = macroPrefix, macroSuffix = macroSuffix, release = releaseMode, logLevel = maxLogLevel, onInsert = (hasMessageHandler("insert") or nil) and function(name) local lua = sendMessage("insert", pathIn, name) -- onInsert() is expected to return a Lua code string and so is the message -- handler. However, if the handler is a single catch-all function we allow -- the message to not be handled and we fall back to the default behavior of -- treating 'name' as a path to a file to be inserted. If we didn't allow this -- then it would be required for the "insert" message to be handled. I think -- it's better if the user can choose whether to handle a message or not! -- if lua == nil and type(messageHandler) == "function" then return assert(pp.readFile(name)) end return lua end, onBeforeMeta = messageHandler and function(lua) sendMessage("beforemeta", pathIn, lua) end, onAfterMeta = messageHandler and function(lua) local luaModified = sendMessage("aftermeta", pathIn, lua) if type(luaModified) == "string" then lua = luaModified elseif luaModified ~= nil then error(F( "%s: Message handler did not return a string for 'aftermeta'. (Got %s)", messageHandlerPath, type(luaModified) )) end return lua end, onDone = messageHandler and function(info) sendMessage("filedone", pathIn, pathOut, info) end, onError = function(err) xpcall(function() sendMessage("fileerror", pathIn, err) end, function(err) printfError("Additional error in 'fileerror' message handler...\n%s", pp.tryToFormatError(err)) end) os.exit(1) end, } assert(info, err) -- The onError() handler above should have been called and we should have exited already. byteCount = byteCount + info.processedByteCount lineCount = lineCount + info.lineCount lineCountCode = lineCountCode + info.linesOfCode tokenCount = tokenCount + info.tokenCount if processingInfoPath ~= "" then -- :SavedInfo table.insert(processingInfo.files, info) -- See 'ProcessInfo' in preprocess.lua for what more 'info' contains. end printfNoise("Processing '%s' successful! (%.3fs)", pathIn, os.clock()-startClockForPath) printfNoise(("-"):rep(#header)) end -- Finalize stuff. if processingInfoPath ~= "" then printfNoise("Saving processing info to '%s'.", processingInfoPath) local luaParts = {"return"} assert(pp.serialize(luaParts, processingInfo)) local lua = table.concat(luaParts) local file = assert(io.open(processingInfoPath, "wb")) file:write(lua) file:close() end printfNoise( "All done! (%.3fs, %.0f file%s, %.0f LOC, %.0f line%s, %.0f token%s, %s)", os.clock()-startClock, #pathsIn, (#pathsIn == 1) and "" or "s", lineCountCode, lineCount, (lineCount == 1) and "" or "s", tokenCount, (tokenCount == 1) and "" or "s", formatBytes(byteCount) ) sendMessage("alldone") -- @Incomplete: Use pcall and format error message better? --[[!=========================================================== Copyright © 2018-2022 Marcus 'ReFreezed' Thunström Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ==============================================================]]