32 """pump v0.2.0 - Pretty Useful for Meta Programming. 34 A tool for preprocessor meta programming. Useful for generating 35 repetitive boilerplate code. Especially useful for writing C++ 36 classes, functions, macros, and templates that need to work with 37 various number of arguments. 44 Converts foo.cc.pump to foo.cc. 48 ATOMIC_CODE ::= $var ID = EXPRESSION 49 | $var ID = [[ CODE ]] 50 | $range ID EXPRESSION..EXPRESSION 51 | $for ID SEPARATOR [[ CODE ]] 55 | $if EXPRESSION [[ CODE ]] ELSE_BRANCH 58 SEPARATOR ::= RAW_CODE | EMPTY 59 ELSE_BRANCH ::= $else [[ CODE ]] 60 | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH 62 EXPRESSION has Python syntax. 65 __author__ =
'wan@google.com (Zhanyong Wan)' 73 (re.compile(
r'\$var\s+'),
'$var'),
74 (re.compile(
r'\$elif\s+'),
'$elif'),
75 (re.compile(
r'\$else\s+'),
'$else'),
76 (re.compile(
r'\$for\s+'),
'$for'),
77 (re.compile(
r'\$if\s+'),
'$if'),
78 (re.compile(
r'\$range\s+'),
'$range'),
79 (re.compile(
r'\$[_A-Za-z]\w*'),
'$id'),
80 (re.compile(
r'\$\(\$\)'),
'$($)'),
81 (re.compile(
r'\$'),
'$'),
82 (re.compile(
r'\[\[\n?'),
'[['),
83 (re.compile(
r'\]\]\n?'),
']]'),
88 """Represents a position (line and column) in a text file.""" 95 return self.
line == rhs.line
and self.
column == rhs.column
98 return not self == rhs
101 return self.
line < rhs.line
or (
102 self.
line == rhs.line
and self.
column < rhs.column)
105 return self < rhs
or self == rhs
117 return '%s(%s)' % (self.
line + 1, self.
column)
126 """Returns a copy of self.""" 133 """Returns the special cursor to denote the end-of-file.""" 138 """Represents a token in a Pump source file.""" 140 def __init__(self, start=None, end=None, value=None, token_type=None):
153 return 'Token @%s: \'%s\' type=%s' % (
157 """Returns a copy of self.""" 159 return Token(self.start.Clone(), self.end.Clone(), self.
value,
164 """Returns True iff the given position in lines starts with 'string'.""" 166 return lines[pos.line][pos.column:].startswith(string)
170 best_match_start = -1
171 for (regex, token_type)
in token_table:
172 m = regex.search(line)
175 if best_match_start < 0
or m.start() < best_match_start:
176 best_match_start = m.start()
177 best_match_length = m.end() - m.start()
178 best_match_token_type = token_type
180 if best_match_start < 0:
183 return (best_match_start, best_match_length, best_match_token_type)
186 def FindFirst(lines, token_table, cursor):
187 """Finds the first occurrence of any string in strings in lines.""" 189 start = cursor.Clone()
190 cur_line_number = cursor.line
191 for line
in lines[start.line:]:
192 if cur_line_number == start.line:
193 line = line[start.column:]
197 (start_column, length, token_type) = m
198 if cur_line_number == start.line:
199 start_column += start.column
200 found_start =
Cursor(cur_line_number, start_column)
201 found_end = found_start + length
202 return MakeToken(lines, found_start, found_end, token_type)
209 """Returns a substring in lines.""" 212 end =
Cursor(len(lines) - 1, len(lines[-1]))
217 if start.line == end.line:
218 return lines[start.line][start.column:end.column]
220 result_lines = ([lines[start.line][start.column:]] +
221 lines[start.line + 1:end.line] +
222 [lines[end.line][:end.column]])
223 return ''.
join(result_lines)
227 """Strip meta comments from each line in the given string.""" 231 str = re.sub(
r'^\s*\$\$.*\n',
'', str)
234 return re.sub(
r'\s*\$\$.*',
'', str)
237 def MakeToken(lines, start, end, token_type):
238 """Creates a new instance of Token.""" 240 return Token(start, end,
SubString(lines, start, end), token_type)
243 def ParseToken(lines, pos, regex, token_type):
244 line = lines[pos.line][pos.column:]
245 m = regex.search(line)
246 if m
and not m.start():
247 return MakeToken(lines, pos, pos + m.end(), token_type)
249 print 'ERROR: %s expected at %s.' % (token_type, pos)
253 ID_REGEX = re.compile(
r'[_A-Za-z]\w*')
254 EQ_REGEX = re.compile(
r'=')
255 REST_OF_LINE_REGEX = re.compile(
r'.*?(?=$|\$\$)')
256 OPTIONAL_WHITE_SPACES_REGEX = re.compile(
r'\s*')
257 WHITE_SPACE_REGEX = re.compile(
r'\s')
258 DOT_DOT_REGEX = re.compile(
r'\.\.')
261 def Skip(lines, pos, regex):
262 line = lines[pos.line][pos.column:]
263 m = re.search(regex, line)
264 if m
and not m.start():
270 def SkipUntil(lines, pos, regex, token_type):
271 line = lines[pos.line][pos.column:]
272 m = re.search(regex, line)
274 return pos + m.start()
276 print (
'ERROR: %s expected on line %s after column %s.' %
277 (token_type, pos.line + 1, pos.column))
282 def ParseInParens(pos):
283 pos =
Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX)
284 pos =
Skip(lines, pos,
r'\(')
286 pos =
Skip(lines, pos,
r'\)')
290 pos =
SkipUntil(lines, pos,
r'\(|\)',
')')
291 if SubString(lines, pos, pos + 1) ==
'(':
293 pos =
Skip(lines, pos,
r'\)')
299 pos = ParseInParens(pos)
300 return MakeToken(lines, start, pos,
'exp')
304 if token.value.endswith(
'\n'):
305 return Token(token.start, token.end, token.value[:-1], token.token_type)
312 found =
FindFirst(lines, TOKEN_TABLE, pos)
317 if found.start == pos:
319 prev_token_rstripped =
None 321 prev_token =
MakeToken(lines, pos, found.start,
'code')
324 if found.token_type ==
'$var':
325 if prev_token_rstripped:
326 yield prev_token_rstripped
328 id_token =
ParseToken(lines, found.end, ID_REGEX,
'id')
330 pos =
Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
332 eq_token =
ParseToken(lines, pos, EQ_REGEX,
'=')
334 pos =
Skip(lines, eq_token.end,
r'\s*')
336 if SubString(lines, pos, pos + 2) !=
'[[':
337 exp_token =
ParseToken(lines, pos, REST_OF_LINE_REGEX,
'exp')
339 pos =
Cursor(exp_token.end.line + 1, 0)
340 elif found.token_type ==
'$for':
341 if prev_token_rstripped:
342 yield prev_token_rstripped
344 id_token =
ParseToken(lines, found.end, ID_REGEX,
'id')
346 pos =
Skip(lines, id_token.end, WHITE_SPACE_REGEX)
347 elif found.token_type ==
'$range':
348 if prev_token_rstripped:
349 yield prev_token_rstripped
351 id_token =
ParseToken(lines, found.end, ID_REGEX,
'id')
353 pos =
Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
355 dots_pos =
SkipUntil(lines, pos, DOT_DOT_REGEX,
'..')
356 yield MakeToken(lines, pos, dots_pos,
'exp')
357 yield MakeToken(lines, dots_pos, dots_pos + 2,
'..')
359 new_pos =
Cursor(pos.line + 1, 0)
360 yield MakeToken(lines, pos, new_pos,
'exp')
362 elif found.token_type ==
'$':
369 elif (found.token_type ==
']]' or found.token_type ==
'$if' or 370 found.token_type ==
'$elif' or found.token_type ==
'$else'):
371 if prev_token_rstripped:
372 yield prev_token_rstripped
383 """A generator that yields the tokens in the given string.""" 385 lines = s.splitlines(
True)
396 def __init__(self, identifier=None, atomic_code=None):
402 def __init__(self, identifier=None, exp1=None, exp2=None):
409 def __init__(self, identifier=None, sep=None, code=None):
421 def __init__(self, exp=None, then_branch=None, else_branch=None):
453 def PopToken(a_list, token_type=None):
455 if token_type
is not None and token.token_type != token_type:
456 print 'ERROR: %s expected at %s' % (token_type, token.start)
457 print 'ERROR: %s found instead' % (token,)
471 python_exp = re.sub(
r'([_A-Za-z]\w*)',
r'self.GetValue("\1")', token.value)
472 return ExpNode(token, python_exp)
476 def Pop(token_type=None):
482 if next.token_type ==
'$else':
488 elif next.token_type ==
'$elif':
496 elif not next.value.strip():
504 def Pop(token_type=None):
515 if next.token_type ==
'exp':
521 return VarNode(id_token, code_node)
525 if next_token.token_type ==
'code':
526 sep_token = next_token
533 return ForNode(id_token, sep_token, code_node)
535 exp_token = Pop(
'code')
543 exp1_token = Pop(
'exp')
545 exp2_token = Pop(
'exp')
553 exp_token = Pop(
'exp')
565 atomic_code_list = []
571 atomic_code_list.append(atomic_code_node)
578 """Convert the given Pump source text into an AST.""" 592 clone.ranges = self.
ranges[:]
598 int_value = int(value)
599 if (
'%s' % int_value) == value:
609 self.
ranges[:0] = [(var, lower, upper)]
616 if identifier == var:
619 print 'ERROR: meta variable %s is undefined.' % (identifier,)
624 result = eval(exp.python_exp)
626 print 'ERROR: caught exception %s: %s' % (e.__class__.__name__, e)
627 print (
'ERROR: failed to evaluate meta expression %s at %s' %
628 (exp.python_exp, exp.token.start))
633 for (var, lower, upper)
in self.
ranges:
634 if identifier == var:
635 return (lower, upper)
637 print 'ERROR: range %s is undefined.' % (identifier,)
646 index = self.string.rfind(
'\n')
650 return self.
string[index + 1:]
657 if isinstance(node, VarNode):
658 identifier = node.identifier.value.strip()
661 value = result.string
662 env.PushVariable(identifier, value)
663 elif isinstance(node, RangeNode):
664 identifier = node.identifier.value.strip()
665 lower = int(env.EvalExp(node.exp1))
666 upper = int(env.EvalExp(node.exp2))
667 env.PushRange(identifier, lower, upper)
668 elif isinstance(node, ForNode):
669 identifier = node.identifier.value.strip()
674 (lower, upper) = env.GetRange(identifier)
675 for i
in range(lower, upper + 1):
676 new_env = env.Clone()
677 new_env.PushVariable(identifier, i)
678 RunCode(new_env, node.code, output)
681 elif isinstance(node, RawCodeNode):
682 output.Append(node.raw_code.value)
683 elif isinstance(node, IfNode):
684 cond = env.EvalExp(node.exp)
686 RunCode(env.Clone(), node.then_branch, output)
687 elif node.else_branch
is not None:
688 RunCode(env.Clone(), node.else_branch, output)
689 elif isinstance(node, ExpNode):
690 value = env.EvalExp(node)
691 output.Append(
'%s' % (value,))
692 elif isinstance(node, LiteralDollarNode):
694 elif isinstance(node, CodeNode):
695 RunCode(env.Clone(), node, output)
702 def RunCode(env, code_node, output):
703 for atomic_code
in code_node.atomic_code:
708 return '//' in cur_line
712 if cur_line.lstrip().startswith(
'#'):
714 return prev_lines
and prev_lines[-1].endswith(
'\\')
718 loc = line.find(
'//')
719 before_comment = line[:loc].rstrip()
720 if before_comment ==
'':
723 output.append(before_comment)
724 indent = len(before_comment) - len(before_comment.lstrip())
725 prefix = indent*
' ' +
'// ' 726 max_len = 80 - len(prefix)
727 comment = line[loc + 2:].
strip()
728 segs = [seg
for seg
in re.split(
r'(\w+\W*)', comment)
if seg !=
'']
731 if len((cur_line + seg).rstrip()) < max_len:
734 if cur_line.strip() !=
'':
735 output.append(prefix + cur_line.rstrip())
736 cur_line = seg.lstrip()
737 if cur_line.strip() !=
'':
738 output.append(prefix + cur_line.strip())
741 def WrapCode(line, line_concat, output):
742 indent = len(line) - len(line.lstrip())
744 max_len = 80 - indent - len(line_concat)
745 new_prefix = prefix + 4*
' ' 746 new_max_len = max_len - 4
748 segs = [seg
for seg
in re.split(
r'([^,;]+[,;]?)', line.strip())
if seg !=
'']
752 while cur_line ==
'' and len(seg.strip()) > max_len:
754 split_at = seg.rfind(
' ', 0, max_len)
755 output.append(prefix + seg[:split_at].
strip() + line_concat)
756 seg = seg[split_at + 1:]
758 max_len = new_max_len
760 if len((cur_line + seg).rstrip()) < max_len:
761 cur_line = (cur_line + seg).lstrip()
763 output.append(prefix + cur_line.rstrip() + line_concat)
765 max_len = new_max_len
766 cur_line = seg.lstrip()
767 if cur_line.strip() !=
'':
768 output.append(prefix + cur_line.strip())
780 return re.search(
r'/\* IWYU pragma: ', line)
784 return (re.match(
r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line)
or 785 re.match(
r'^#include\s', line)
or 787 re.search(
r'// IWYU pragma: ', line))
815 lines = string.splitlines()
819 output2 = [line.rstrip()
for line
in output]
820 return '\n'.
join(output2) +
'\n' 824 """Return the text generated from the given Pump source text.""" 838 if file_path.endswith(
'.pump'):
839 output_file_path = file_path[:-5]
841 output_file_path =
'-' 842 if output_file_path ==
'-':
845 output_file = file(output_file_path,
'w')
846 output_file.write(
'// This file was GENERATED by command:\n')
847 output_file.write(
'// %s %s\n' %
848 (os.path.basename(__file__), os.path.basename(file_path)))
849 output_file.write(
'// DO NOT EDIT BY HAND!!!\n\n')
850 output_file.write(output_str)
854 if __name__ ==
'__main__':
def RunAtomicCode(env, node, output)
def IsSingleLineComment(cur_line)
def WrapPreprocessorDirective(line, output)
def __init__(self, line=-1, column=-1)
def WrapComment(line, output)
def __init__(self, token=None)
def ParseToken(lines, pos, regex, token_type)
def PushRange(self, var, lower, upper)
def SubString(lines, start, end)
def SkipUntil(lines, pos, regex, token_type)
def ParseCodeNode(tokens)
def Skip(lines, pos, regex)
def WrapPlainCode(line, output)
def GetValue(self, identifier)
def __init__(self, identifier=None, sep=None, code=None)
def FindFirstInLine(line, token_table)
def IsHeaderGuardIncludeOrOneLineIWYUPragma(line)
def WrapCode(line, line_concat, output)
def StartsWith(lines, pos, string)
Gen range(Value begin, Value end)
def TokenizeLines(lines, pos)
std::uint64_t strip(std::chrono::nanoseconds t)
def RStripNewLineFromToken(token)
def __init__(self, token, python_exp)
def ParseAtomicCodeNode(tokens)
def IsInPreprocessorDirective(prev_lines, cur_line)
def __init__(self, exp=None, then_branch=None, else_branch=None)
def __init__(self, token)
Encoder::MutableCompressedList list
def __sub__(self, offset)
def __init__(self, else_branch=None)
def IsMultiLineIWYUPragma(line)
def FindFirst(lines, token_table, cursor)
def ParseExpTokenInParens(lines, pos)
def MakeToken(lines, start, end, token_type)
def PushVariable(self, var, value)
def PopToken(a_list, token_type=None)
def __init__(self, identifier=None, atomic_code=None)
def __init__(self, identifier=None, exp1=None, exp2=None)
def WrapLongLine(line, output)
def StripMetaComments(str)
def __init__(self, start=None, end=None, value=None, token_type=None)
def __init__(self, atomic_code_list=None)
def ParseToAST(pump_src_text)
def PushFront(a_list, elem)
def ParseElseNode(tokens)
def ConvertFromPumpSource(src_text)
def GetRange(self, identifier)
def __add__(self, offset)
def RunCode(env, code_node, output)