proxygen
pump.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # Copyright 2008, Google Inc.
4 # All rights reserved.
5 #
6 # Redistribution and use in source and binary forms, with or without
7 # modification, are permitted provided that the following conditions are
8 # met:
9 #
10 # * Redistributions of source code must retain the above copyright
11 # notice, this list of conditions and the following disclaimer.
12 # * Redistributions in binary form must reproduce the above
13 # copyright notice, this list of conditions and the following disclaimer
14 # in the documentation and/or other materials provided with the
15 # distribution.
16 # * Neither the name of Google Inc. nor the names of its
17 # contributors may be used to endorse or promote products derived from
18 # this software without specific prior written permission.
19 #
20 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 
32 """pump v0.2.0 - Pretty Useful for Meta Programming.
33 
34 A tool for preprocessor meta programming. Useful for generating
35 repetitive boilerplate code. Especially useful for writing C++
36 classes, functions, macros, and templates that need to work with
37 various number of arguments.
38 
39 USAGE:
40  pump.py SOURCE_FILE
41 
42 EXAMPLES:
43  pump.py foo.cc.pump
44  Converts foo.cc.pump to foo.cc.
45 
46 GRAMMAR:
47  CODE ::= ATOMIC_CODE*
48  ATOMIC_CODE ::= $var ID = EXPRESSION
49  | $var ID = [[ CODE ]]
50  | $range ID EXPRESSION..EXPRESSION
51  | $for ID SEPARATOR [[ CODE ]]
52  | $($)
53  | $ID
54  | $(EXPRESSION)
55  | $if EXPRESSION [[ CODE ]] ELSE_BRANCH
56  | [[ CODE ]]
57  | RAW_CODE
58  SEPARATOR ::= RAW_CODE | EMPTY
59  ELSE_BRANCH ::= $else [[ CODE ]]
60  | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH
61  | EMPTY
62  EXPRESSION has Python syntax.
63 """
64 
65 __author__ = 'wan@google.com (Zhanyong Wan)'
66 
67 import os
68 import re
69 import sys
70 
71 
72 TOKEN_TABLE = [
73  (re.compile(r'\$var\s+'), '$var'),
74  (re.compile(r'\$elif\s+'), '$elif'),
75  (re.compile(r'\$else\s+'), '$else'),
76  (re.compile(r'\$for\s+'), '$for'),
77  (re.compile(r'\$if\s+'), '$if'),
78  (re.compile(r'\$range\s+'), '$range'),
79  (re.compile(r'\$[_A-Za-z]\w*'), '$id'),
80  (re.compile(r'\$\(\$\)'), '$($)'),
81  (re.compile(r'\$'), '$'),
82  (re.compile(r'\[\[\n?'), '[['),
83  (re.compile(r'\]\]\n?'), ']]'),
84  ]
85 
86 
87 class Cursor:
88  """Represents a position (line and column) in a text file."""
89 
90  def __init__(self, line=-1, column=-1):
91  self.line = line
92  self.column = column
93 
94  def __eq__(self, rhs):
95  return self.line == rhs.line and self.column == rhs.column
96 
97  def __ne__(self, rhs):
98  return not self == rhs
99 
100  def __lt__(self, rhs):
101  return self.line < rhs.line or (
102  self.line == rhs.line and self.column < rhs.column)
103 
104  def __le__(self, rhs):
105  return self < rhs or self == rhs
106 
107  def __gt__(self, rhs):
108  return rhs < self
109 
110  def __ge__(self, rhs):
111  return rhs <= self
112 
113  def __str__(self):
114  if self == Eof():
115  return 'EOF'
116  else:
117  return '%s(%s)' % (self.line + 1, self.column)
118 
119  def __add__(self, offset):
120  return Cursor(self.line, self.column + offset)
121 
122  def __sub__(self, offset):
123  return Cursor(self.line, self.column - offset)
124 
125  def Clone(self):
126  """Returns a copy of self."""
127 
128  return Cursor(self.line, self.column)
129 
130 
131 # Special cursor to indicate the end-of-file.
132 def Eof():
133  """Returns the special cursor to denote the end-of-file."""
134  return Cursor(-1, -1)
135 
136 
137 class Token:
138  """Represents a token in a Pump source file."""
139 
140  def __init__(self, start=None, end=None, value=None, token_type=None):
141  if start is None:
142  self.start = Eof()
143  else:
144  self.start = start
145  if end is None:
146  self.end = Eof()
147  else:
148  self.end = end
149  self.value = value
150  self.token_type = token_type
151 
152  def __str__(self):
153  return 'Token @%s: \'%s\' type=%s' % (
154  self.start, self.value, self.token_type)
155 
156  def Clone(self):
157  """Returns a copy of self."""
158 
159  return Token(self.start.Clone(), self.end.Clone(), self.value,
160  self.token_type)
161 
162 
163 def StartsWith(lines, pos, string):
164  """Returns True iff the given position in lines starts with 'string'."""
165 
166  return lines[pos.line][pos.column:].startswith(string)
167 
168 
169 def FindFirstInLine(line, token_table):
170  best_match_start = -1
171  for (regex, token_type) in token_table:
172  m = regex.search(line)
173  if m:
174  # We found regex in lines
175  if best_match_start < 0 or m.start() < best_match_start:
176  best_match_start = m.start()
177  best_match_length = m.end() - m.start()
178  best_match_token_type = token_type
179 
180  if best_match_start < 0:
181  return None
182 
183  return (best_match_start, best_match_length, best_match_token_type)
184 
185 
186 def FindFirst(lines, token_table, cursor):
187  """Finds the first occurrence of any string in strings in lines."""
188 
189  start = cursor.Clone()
190  cur_line_number = cursor.line
191  for line in lines[start.line:]:
192  if cur_line_number == start.line:
193  line = line[start.column:]
194  m = FindFirstInLine(line, token_table)
195  if m:
196  # We found a regex in line.
197  (start_column, length, token_type) = m
198  if cur_line_number == start.line:
199  start_column += start.column
200  found_start = Cursor(cur_line_number, start_column)
201  found_end = found_start + length
202  return MakeToken(lines, found_start, found_end, token_type)
203  cur_line_number += 1
204  # We failed to find str in lines
205  return None
206 
207 
208 def SubString(lines, start, end):
209  """Returns a substring in lines."""
210 
211  if end == Eof():
212  end = Cursor(len(lines) - 1, len(lines[-1]))
213 
214  if start >= end:
215  return ''
216 
217  if start.line == end.line:
218  return lines[start.line][start.column:end.column]
219 
220  result_lines = ([lines[start.line][start.column:]] +
221  lines[start.line + 1:end.line] +
222  [lines[end.line][:end.column]])
223  return ''.join(result_lines)
224 
225 
227  """Strip meta comments from each line in the given string."""
228 
229  # First, completely remove lines containing nothing but a meta
230  # comment, including the trailing \n.
231  str = re.sub(r'^\s*\$\$.*\n', '', str)
232 
233  # Then, remove meta comments from contentful lines.
234  return re.sub(r'\s*\$\$.*', '', str)
235 
236 
237 def MakeToken(lines, start, end, token_type):
238  """Creates a new instance of Token."""
239 
240  return Token(start, end, SubString(lines, start, end), token_type)
241 
242 
243 def ParseToken(lines, pos, regex, token_type):
244  line = lines[pos.line][pos.column:]
245  m = regex.search(line)
246  if m and not m.start():
247  return MakeToken(lines, pos, pos + m.end(), token_type)
248  else:
249  print 'ERROR: %s expected at %s.' % (token_type, pos)
250  sys.exit(1)
251 
252 
253 ID_REGEX = re.compile(r'[_A-Za-z]\w*')
254 EQ_REGEX = re.compile(r'=')
255 REST_OF_LINE_REGEX = re.compile(r'.*?(?=$|\$\$)')
256 OPTIONAL_WHITE_SPACES_REGEX = re.compile(r'\s*')
257 WHITE_SPACE_REGEX = re.compile(r'\s')
258 DOT_DOT_REGEX = re.compile(r'\.\.')
259 
260 
261 def Skip(lines, pos, regex):
262  line = lines[pos.line][pos.column:]
263  m = re.search(regex, line)
264  if m and not m.start():
265  return pos + m.end()
266  else:
267  return pos
268 
269 
270 def SkipUntil(lines, pos, regex, token_type):
271  line = lines[pos.line][pos.column:]
272  m = re.search(regex, line)
273  if m:
274  return pos + m.start()
275  else:
276  print ('ERROR: %s expected on line %s after column %s.' %
277  (token_type, pos.line + 1, pos.column))
278  sys.exit(1)
279 
280 
281 def ParseExpTokenInParens(lines, pos):
282  def ParseInParens(pos):
283  pos = Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX)
284  pos = Skip(lines, pos, r'\(')
285  pos = Parse(pos)
286  pos = Skip(lines, pos, r'\)')
287  return pos
288 
289  def Parse(pos):
290  pos = SkipUntil(lines, pos, r'\(|\)', ')')
291  if SubString(lines, pos, pos + 1) == '(':
292  pos = Parse(pos + 1)
293  pos = Skip(lines, pos, r'\)')
294  return Parse(pos)
295  else:
296  return pos
297 
298  start = pos.Clone()
299  pos = ParseInParens(pos)
300  return MakeToken(lines, start, pos, 'exp')
301 
302 
304  if token.value.endswith('\n'):
305  return Token(token.start, token.end, token.value[:-1], token.token_type)
306  else:
307  return token
308 
309 
310 def TokenizeLines(lines, pos):
311  while True:
312  found = FindFirst(lines, TOKEN_TABLE, pos)
313  if not found:
314  yield MakeToken(lines, pos, Eof(), 'code')
315  return
316 
317  if found.start == pos:
318  prev_token = None
319  prev_token_rstripped = None
320  else:
321  prev_token = MakeToken(lines, pos, found.start, 'code')
322  prev_token_rstripped = RStripNewLineFromToken(prev_token)
323 
324  if found.token_type == '$var':
325  if prev_token_rstripped:
326  yield prev_token_rstripped
327  yield found
328  id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
329  yield id_token
330  pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
331 
332  eq_token = ParseToken(lines, pos, EQ_REGEX, '=')
333  yield eq_token
334  pos = Skip(lines, eq_token.end, r'\s*')
335 
336  if SubString(lines, pos, pos + 2) != '[[':
337  exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp')
338  yield exp_token
339  pos = Cursor(exp_token.end.line + 1, 0)
340  elif found.token_type == '$for':
341  if prev_token_rstripped:
342  yield prev_token_rstripped
343  yield found
344  id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
345  yield id_token
346  pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX)
347  elif found.token_type == '$range':
348  if prev_token_rstripped:
349  yield prev_token_rstripped
350  yield found
351  id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
352  yield id_token
353  pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
354 
355  dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..')
356  yield MakeToken(lines, pos, dots_pos, 'exp')
357  yield MakeToken(lines, dots_pos, dots_pos + 2, '..')
358  pos = dots_pos + 2
359  new_pos = Cursor(pos.line + 1, 0)
360  yield MakeToken(lines, pos, new_pos, 'exp')
361  pos = new_pos
362  elif found.token_type == '$':
363  if prev_token:
364  yield prev_token
365  yield found
366  exp_token = ParseExpTokenInParens(lines, found.end)
367  yield exp_token
368  pos = exp_token.end
369  elif (found.token_type == ']]' or found.token_type == '$if' or
370  found.token_type == '$elif' or found.token_type == '$else'):
371  if prev_token_rstripped:
372  yield prev_token_rstripped
373  yield found
374  pos = found.end
375  else:
376  if prev_token:
377  yield prev_token
378  yield found
379  pos = found.end
380 
381 
382 def Tokenize(s):
383  """A generator that yields the tokens in the given string."""
384  if s != '':
385  lines = s.splitlines(True)
386  for token in TokenizeLines(lines, Cursor(0, 0)):
387  yield token
388 
389 
390 class CodeNode:
391  def __init__(self, atomic_code_list=None):
392  self.atomic_code = atomic_code_list
393 
394 
395 class VarNode:
396  def __init__(self, identifier=None, atomic_code=None):
397  self.identifier = identifier
398  self.atomic_code = atomic_code
399 
400 
401 class RangeNode:
402  def __init__(self, identifier=None, exp1=None, exp2=None):
403  self.identifier = identifier
404  self.exp1 = exp1
405  self.exp2 = exp2
406 
407 
408 class ForNode:
409  def __init__(self, identifier=None, sep=None, code=None):
410  self.identifier = identifier
411  self.sep = sep
412  self.code = code
413 
414 
415 class ElseNode:
416  def __init__(self, else_branch=None):
417  self.else_branch = else_branch
418 
419 
420 class IfNode:
421  def __init__(self, exp=None, then_branch=None, else_branch=None):
422  self.exp = exp
423  self.then_branch = then_branch
424  self.else_branch = else_branch
425 
426 
428  def __init__(self, token=None):
429  self.raw_code = token
430 
431 
433  def __init__(self, token):
434  self.token = token
435 
436 
437 class ExpNode:
438  def __init__(self, token, python_exp):
439  self.token = token
440  self.python_exp = python_exp
441 
442 
443 def PopFront(a_list):
444  head = a_list[0]
445  a_list[:1] = []
446  return head
447 
448 
449 def PushFront(a_list, elem):
450  a_list[:0] = [elem]
451 
452 
453 def PopToken(a_list, token_type=None):
454  token = PopFront(a_list)
455  if token_type is not None and token.token_type != token_type:
456  print 'ERROR: %s expected at %s' % (token_type, token.start)
457  print 'ERROR: %s found instead' % (token,)
458  sys.exit(1)
459 
460  return token
461 
462 
463 def PeekToken(a_list):
464  if not a_list:
465  return None
466 
467  return a_list[0]
468 
469 
470 def ParseExpNode(token):
471  python_exp = re.sub(r'([_A-Za-z]\w*)', r'self.GetValue("\1")', token.value)
472  return ExpNode(token, python_exp)
473 
474 
475 def ParseElseNode(tokens):
476  def Pop(token_type=None):
477  return PopToken(tokens, token_type)
478 
479  next = PeekToken(tokens)
480  if not next:
481  return None
482  if next.token_type == '$else':
483  Pop('$else')
484  Pop('[[')
485  code_node = ParseCodeNode(tokens)
486  Pop(']]')
487  return code_node
488  elif next.token_type == '$elif':
489  Pop('$elif')
490  exp = Pop('code')
491  Pop('[[')
492  code_node = ParseCodeNode(tokens)
493  Pop(']]')
494  inner_else_node = ParseElseNode(tokens)
495  return CodeNode([IfNode(ParseExpNode(exp), code_node, inner_else_node)])
496  elif not next.value.strip():
497  Pop('code')
498  return ParseElseNode(tokens)
499  else:
500  return None
501 
502 
504  def Pop(token_type=None):
505  return PopToken(tokens, token_type)
506 
507  head = PopFront(tokens)
508  t = head.token_type
509  if t == 'code':
510  return RawCodeNode(head)
511  elif t == '$var':
512  id_token = Pop('id')
513  Pop('=')
514  next = PeekToken(tokens)
515  if next.token_type == 'exp':
516  exp_token = Pop()
517  return VarNode(id_token, ParseExpNode(exp_token))
518  Pop('[[')
519  code_node = ParseCodeNode(tokens)
520  Pop(']]')
521  return VarNode(id_token, code_node)
522  elif t == '$for':
523  id_token = Pop('id')
524  next_token = PeekToken(tokens)
525  if next_token.token_type == 'code':
526  sep_token = next_token
527  Pop('code')
528  else:
529  sep_token = None
530  Pop('[[')
531  code_node = ParseCodeNode(tokens)
532  Pop(']]')
533  return ForNode(id_token, sep_token, code_node)
534  elif t == '$if':
535  exp_token = Pop('code')
536  Pop('[[')
537  code_node = ParseCodeNode(tokens)
538  Pop(']]')
539  else_node = ParseElseNode(tokens)
540  return IfNode(ParseExpNode(exp_token), code_node, else_node)
541  elif t == '$range':
542  id_token = Pop('id')
543  exp1_token = Pop('exp')
544  Pop('..')
545  exp2_token = Pop('exp')
546  return RangeNode(id_token, ParseExpNode(exp1_token),
547  ParseExpNode(exp2_token))
548  elif t == '$id':
549  return ParseExpNode(Token(head.start + 1, head.end, head.value[1:], 'id'))
550  elif t == '$($)':
551  return LiteralDollarNode(head)
552  elif t == '$':
553  exp_token = Pop('exp')
554  return ParseExpNode(exp_token)
555  elif t == '[[':
556  code_node = ParseCodeNode(tokens)
557  Pop(']]')
558  return code_node
559  else:
560  PushFront(tokens, head)
561  return None
562 
563 
564 def ParseCodeNode(tokens):
565  atomic_code_list = []
566  while True:
567  if not tokens:
568  break
569  atomic_code_node = ParseAtomicCodeNode(tokens)
570  if atomic_code_node:
571  atomic_code_list.append(atomic_code_node)
572  else:
573  break
574  return CodeNode(atomic_code_list)
575 
576 
577 def ParseToAST(pump_src_text):
578  """Convert the given Pump source text into an AST."""
579  tokens = list(Tokenize(pump_src_text))
580  code_node = ParseCodeNode(tokens)
581  return code_node
582 
583 
584 class Env:
585  def __init__(self):
586  self.variables = []
587  self.ranges = []
588 
589  def Clone(self):
590  clone = Env()
591  clone.variables = self.variables[:]
592  clone.ranges = self.ranges[:]
593  return clone
594 
595  def PushVariable(self, var, value):
596  # If value looks like an int, store it as an int.
597  try:
598  int_value = int(value)
599  if ('%s' % int_value) == value:
600  value = int_value
601  except Exception:
602  pass
603  self.variables[:0] = [(var, value)]
604 
605  def PopVariable(self):
606  self.variables[:1] = []
607 
608  def PushRange(self, var, lower, upper):
609  self.ranges[:0] = [(var, lower, upper)]
610 
611  def PopRange(self):
612  self.ranges[:1] = []
613 
614  def GetValue(self, identifier):
615  for (var, value) in self.variables:
616  if identifier == var:
617  return value
618 
619  print 'ERROR: meta variable %s is undefined.' % (identifier,)
620  sys.exit(1)
621 
622  def EvalExp(self, exp):
623  try:
624  result = eval(exp.python_exp)
625  except Exception, e:
626  print 'ERROR: caught exception %s: %s' % (e.__class__.__name__, e)
627  print ('ERROR: failed to evaluate meta expression %s at %s' %
628  (exp.python_exp, exp.token.start))
629  sys.exit(1)
630  return result
631 
632  def GetRange(self, identifier):
633  for (var, lower, upper) in self.ranges:
634  if identifier == var:
635  return (lower, upper)
636 
637  print 'ERROR: range %s is undefined.' % (identifier,)
638  sys.exit(1)
639 
640 
641 class Output:
642  def __init__(self):
643  self.string = ''
644 
645  def GetLastLine(self):
646  index = self.string.rfind('\n')
647  if index < 0:
648  return ''
649 
650  return self.string[index + 1:]
651 
652  def Append(self, s):
653  self.string += s
654 
655 
656 def RunAtomicCode(env, node, output):
657  if isinstance(node, VarNode):
658  identifier = node.identifier.value.strip()
659  result = Output()
660  RunAtomicCode(env.Clone(), node.atomic_code, result)
661  value = result.string
662  env.PushVariable(identifier, value)
663  elif isinstance(node, RangeNode):
664  identifier = node.identifier.value.strip()
665  lower = int(env.EvalExp(node.exp1))
666  upper = int(env.EvalExp(node.exp2))
667  env.PushRange(identifier, lower, upper)
668  elif isinstance(node, ForNode):
669  identifier = node.identifier.value.strip()
670  if node.sep is None:
671  sep = ''
672  else:
673  sep = node.sep.value
674  (lower, upper) = env.GetRange(identifier)
675  for i in range(lower, upper + 1):
676  new_env = env.Clone()
677  new_env.PushVariable(identifier, i)
678  RunCode(new_env, node.code, output)
679  if i != upper:
680  output.Append(sep)
681  elif isinstance(node, RawCodeNode):
682  output.Append(node.raw_code.value)
683  elif isinstance(node, IfNode):
684  cond = env.EvalExp(node.exp)
685  if cond:
686  RunCode(env.Clone(), node.then_branch, output)
687  elif node.else_branch is not None:
688  RunCode(env.Clone(), node.else_branch, output)
689  elif isinstance(node, ExpNode):
690  value = env.EvalExp(node)
691  output.Append('%s' % (value,))
692  elif isinstance(node, LiteralDollarNode):
693  output.Append('$')
694  elif isinstance(node, CodeNode):
695  RunCode(env.Clone(), node, output)
696  else:
697  print 'BAD'
698  print node
699  sys.exit(1)
700 
701 
702 def RunCode(env, code_node, output):
703  for atomic_code in code_node.atomic_code:
704  RunAtomicCode(env, atomic_code, output)
705 
706 
707 def IsSingleLineComment(cur_line):
708  return '//' in cur_line
709 
710 
711 def IsInPreprocessorDirective(prev_lines, cur_line):
712  if cur_line.lstrip().startswith('#'):
713  return True
714  return prev_lines and prev_lines[-1].endswith('\\')
715 
716 
717 def WrapComment(line, output):
718  loc = line.find('//')
719  before_comment = line[:loc].rstrip()
720  if before_comment == '':
721  indent = loc
722  else:
723  output.append(before_comment)
724  indent = len(before_comment) - len(before_comment.lstrip())
725  prefix = indent*' ' + '// '
726  max_len = 80 - len(prefix)
727  comment = line[loc + 2:].strip()
728  segs = [seg for seg in re.split(r'(\w+\W*)', comment) if seg != '']
729  cur_line = ''
730  for seg in segs:
731  if len((cur_line + seg).rstrip()) < max_len:
732  cur_line += seg
733  else:
734  if cur_line.strip() != '':
735  output.append(prefix + cur_line.rstrip())
736  cur_line = seg.lstrip()
737  if cur_line.strip() != '':
738  output.append(prefix + cur_line.strip())
739 
740 
741 def WrapCode(line, line_concat, output):
742  indent = len(line) - len(line.lstrip())
743  prefix = indent*' ' # Prefix of the current line
744  max_len = 80 - indent - len(line_concat) # Maximum length of the current line
745  new_prefix = prefix + 4*' ' # Prefix of a continuation line
746  new_max_len = max_len - 4 # Maximum length of a continuation line
747  # Prefers to wrap a line after a ',' or ';'.
748  segs = [seg for seg in re.split(r'([^,;]+[,;]?)', line.strip()) if seg != '']
749  cur_line = '' # The current line without leading spaces.
750  for seg in segs:
751  # If the line is still too long, wrap at a space.
752  while cur_line == '' and len(seg.strip()) > max_len:
753  seg = seg.lstrip()
754  split_at = seg.rfind(' ', 0, max_len)
755  output.append(prefix + seg[:split_at].strip() + line_concat)
756  seg = seg[split_at + 1:]
757  prefix = new_prefix
758  max_len = new_max_len
759 
760  if len((cur_line + seg).rstrip()) < max_len:
761  cur_line = (cur_line + seg).lstrip()
762  else:
763  output.append(prefix + cur_line.rstrip() + line_concat)
764  prefix = new_prefix
765  max_len = new_max_len
766  cur_line = seg.lstrip()
767  if cur_line.strip() != '':
768  output.append(prefix + cur_line.strip())
769 
770 
771 def WrapPreprocessorDirective(line, output):
772  WrapCode(line, ' \\', output)
773 
774 
775 def WrapPlainCode(line, output):
776  WrapCode(line, '', output)
777 
778 
780  return re.search(r'/\* IWYU pragma: ', line)
781 
782 
784  return (re.match(r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line) or
785  re.match(r'^#include\s', line) or
786  # Don't break IWYU pragmas, either; that causes iwyu.py problems.
787  re.search(r'// IWYU pragma: ', line))
788 
789 
790 def WrapLongLine(line, output):
791  line = line.rstrip()
792  if len(line) <= 80:
793  output.append(line)
794  elif IsSingleLineComment(line):
796  # The style guide made an exception to allow long header guard lines,
797  # includes and IWYU pragmas.
798  output.append(line)
799  else:
800  WrapComment(line, output)
801  elif IsInPreprocessorDirective(output, line):
803  # The style guide made an exception to allow long header guard lines,
804  # includes and IWYU pragmas.
805  output.append(line)
806  else:
807  WrapPreprocessorDirective(line, output)
808  elif IsMultiLineIWYUPragma(line):
809  output.append(line)
810  else:
811  WrapPlainCode(line, output)
812 
813 
814 def BeautifyCode(string):
815  lines = string.splitlines()
816  output = []
817  for line in lines:
818  WrapLongLine(line, output)
819  output2 = [line.rstrip() for line in output]
820  return '\n'.join(output2) + '\n'
821 
822 
823 def ConvertFromPumpSource(src_text):
824  """Return the text generated from the given Pump source text."""
825  ast = ParseToAST(StripMetaComments(src_text))
826  output = Output()
827  RunCode(Env(), ast, output)
828  return BeautifyCode(output.string)
829 
830 
831 def main(argv):
832  if len(argv) == 1:
833  print __doc__
834  sys.exit(1)
835 
836  file_path = argv[-1]
837  output_str = ConvertFromPumpSource(file(file_path, 'r').read())
838  if file_path.endswith('.pump'):
839  output_file_path = file_path[:-5]
840  else:
841  output_file_path = '-'
842  if output_file_path == '-':
843  print output_str,
844  else:
845  output_file = file(output_file_path, 'w')
846  output_file.write('// This file was GENERATED by command:\n')
847  output_file.write('// %s %s\n' %
848  (os.path.basename(__file__), os.path.basename(file_path)))
849  output_file.write('// DO NOT EDIT BY HAND!!!\n\n')
850  output_file.write(output_str)
851  output_file.close()
852 
853 
854 if __name__ == '__main__':
855  main(sys.argv)
def RunAtomicCode(env, node, output)
Definition: pump.py:656
def IsSingleLineComment(cur_line)
Definition: pump.py:707
def WrapPreprocessorDirective(line, output)
Definition: pump.py:771
def __init__(self, line=-1, column=-1)
Definition: pump.py:90
def PopRange(self)
Definition: pump.py:611
def BeautifyCode(string)
Definition: pump.py:814
def WrapComment(line, output)
Definition: pump.py:717
def __init__(self, token=None)
Definition: pump.py:428
def PopFront(a_list)
Definition: pump.py:443
def ParseToken(lines, pos, regex, token_type)
Definition: pump.py:243
def PushRange(self, var, lower, upper)
Definition: pump.py:608
def SubString(lines, start, end)
Definition: pump.py:208
def main(argv)
Definition: pump.py:831
def SkipUntil(lines, pos, regex, token_type)
Definition: pump.py:270
def ParseCodeNode(tokens)
Definition: pump.py:564
def __init__(self)
Definition: pump.py:585
def Skip(lines, pos, regex)
Definition: pump.py:261
def WrapPlainCode(line, output)
Definition: pump.py:775
def GetValue(self, identifier)
Definition: pump.py:614
def ParseExpNode(token)
Definition: pump.py:470
def __init__(self, identifier=None, sep=None, code=None)
Definition: pump.py:409
def FindFirstInLine(line, token_table)
Definition: pump.py:169
def IsHeaderGuardIncludeOrOneLineIWYUPragma(line)
Definition: pump.py:783
def WrapCode(line, line_concat, output)
Definition: pump.py:741
def StartsWith(lines, pos, string)
Definition: pump.py:163
Gen range(Value begin, Value end)
Definition: Base.h:467
def TokenizeLines(lines, pos)
Definition: pump.py:310
std::uint64_t strip(std::chrono::nanoseconds t)
def RStripNewLineFromToken(token)
Definition: pump.py:303
def __ne__(self, rhs)
Definition: pump.py:97
def __ge__(self, rhs)
Definition: pump.py:110
def Clone(self)
Definition: pump.py:589
def __init__(self, token, python_exp)
Definition: pump.py:438
def __lt__(self, rhs)
Definition: pump.py:100
def ParseAtomicCodeNode(tokens)
Definition: pump.py:503
def __le__(self, rhs)
Definition: pump.py:104
def IsInPreprocessorDirective(prev_lines, cur_line)
Definition: pump.py:711
def __gt__(self, rhs)
Definition: pump.py:107
def __init__(self, exp=None, then_branch=None, else_branch=None)
Definition: pump.py:421
def __init__(self)
Definition: pump.py:642
def __init__(self, token)
Definition: pump.py:433
Encoder::MutableCompressedList list
def PeekToken(a_list)
Definition: pump.py:463
def __sub__(self, offset)
Definition: pump.py:122
def __init__(self, else_branch=None)
Definition: pump.py:416
then_branch
Definition: pump.py:423
def IsMultiLineIWYUPragma(line)
Definition: pump.py:779
def FindFirst(lines, token_table, cursor)
Definition: pump.py:186
def PopVariable(self)
Definition: pump.py:605
def ParseExpTokenInParens(lines, pos)
Definition: pump.py:281
def MakeToken(lines, start, end, token_type)
Definition: pump.py:237
def EvalExp(self, exp)
Definition: pump.py:622
def PushVariable(self, var, value)
Definition: pump.py:595
def PopToken(a_list, token_type=None)
Definition: pump.py:453
def __init__(self, identifier=None, atomic_code=None)
Definition: pump.py:396
def __init__(self, identifier=None, exp1=None, exp2=None)
Definition: pump.py:402
def WrapLongLine(line, output)
Definition: pump.py:790
def __eq__(self, rhs)
Definition: pump.py:94
token_type
Definition: pump.py:150
#define join
def StripMetaComments(str)
Definition: pump.py:226
def __init__(self, start=None, end=None, value=None, token_type=None)
Definition: pump.py:140
def Append(self, s)
Definition: pump.py:652
def __str__(self)
Definition: pump.py:152
def __init__(self, atomic_code_list=None)
Definition: pump.py:391
def GetLastLine(self)
Definition: pump.py:645
def ParseToAST(pump_src_text)
Definition: pump.py:577
def PushFront(a_list, elem)
Definition: pump.py:449
def ParseElseNode(tokens)
Definition: pump.py:475
def __str__(self)
Definition: pump.py:113
def ConvertFromPumpSource(src_text)
Definition: pump.py:823
def Eof()
Definition: pump.py:132
def GetRange(self, identifier)
Definition: pump.py:632
def __add__(self, offset)
Definition: pump.py:119
def Clone(self)
Definition: pump.py:125
def Tokenize(s)
Definition: pump.py:382
variables
Definition: pump.py:586
else_branch
Definition: pump.py:424
ranges
Definition: pump.py:587
def Clone(self)
Definition: pump.py:156
def RunCode(env, code_node, output)
Definition: pump.py:702