#!/usr/bin/env python3.4 # Copyright 2018 The go-python Authors. All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. """ Write symtable_data_test.go """ import sys import ast import subprocess import dis from symtable import symtable try: # run ./build.sh in the readsymtab directory to create this module from readsymtab import readsymtab use_readsymtab = True except ImportError: import ctypes use_readsymtab = False print("Using compiler dependent code to read bitfields - compile readsymtab module to be sure!") inp = [ ('''1''', "eval"), ('''a*b*c''', "eval"), # Functions ('''def fn(): pass''', "exec"), ('''def fn(a,b):\n e=1\n return a*b*c*d*e''', "exec"), ('''def fn(*args,a=2,b=3,**kwargs): return (args,a,b,kwargs)''', "exec"), ('''def fn(a,b):\n def nested(c,d):\n return a*b*c*d*e''', "exec"), ('''\ def fn(a:A,*arg:ARG,b:B=BB,c:C=CC,**kwargs:KW) -> RET: def fn(A,b): e=1 return a*arg*b*c*kwargs*A*e*glob''', "exec"), ('''\ def fn(a): global b b = a''', "exec"), ('''\ def fn(a): global b global b return b''', "exec"), ('''\ def inner(): print(x) global x ''', "exec"), ('''\ def fn(a): b = 6 global b b = a''', "exec"), ('''\ def fn(a=b,c=1): return a+b''', "exec"), ('''\ @sausage @potato(beans) def outer(): x = 1 def inner(): nonlocal x x = 2''', "exec"), ('''\ def fn(a): nonlocal b ''', "exec", SyntaxError), ('''\ def outer(): def inner(): nonlocal x x = 2''', "exec", SyntaxError), ('''\ def outer(): x = 1 def inner(): print(x) nonlocal x ''', "exec"), ('''\ def outer(): x = 1 def inner(): x = 2 nonlocal x''', "exec"), ('''\ def outer(): x = 1 def inner(x): nonlocal x''', "exec", SyntaxError), ('''\ def outer(): x = 1 def inner(x): global x''', "exec", SyntaxError), ('''\ def outer(): def inner(): global x nonlocal x ''', "exec", SyntaxError), ('''\ def outer(): x = 1 def inner(): y = 2 return x + y + z ''', "exec"), ('''\ def outer(): global x def inner(): return x ''', "exec"), ('''\ nonlocal x ''', "exec", SyntaxError), ('''def fn(a,a): pass''', "exec", SyntaxError), # List Comp ('''[ x for x in xs ]''', "exec"), ('''[ x+y for x in xs for y in ys ]''', "exec"), ('''[ x+y+z for x in xs if x if y if z if r for y in ys if x if y if z if p for z in zs if x if y if z if q]''', "exec"), ('''[ x+y for x in [ x for x in xs ] ]''', "exec"), ('''[ x for x in xs ]\n[ y for y in ys ]''', "exec"), # Generator expr ('''( x for x in xs )''', "exec"), ('''( x+y for x in xs for y in ys )''', "exec"), ('''( x+y+z for x in xs if x if y if z if r for y in ys if x if y if z if p for z in zs if x if y if z if q)''', "exec"), ('''( x+y for x in ( x for x in xs ) )''', "exec"), # Set comp ('''{ x for x in xs }''', "exec"), ('''{ x+y for x in xs for y in ys }''', "exec"), ('''{ x+y+z for x in xs if x if y if z if r for y in ys if x if y if z if p for z in zs if x if y if z if q}''', "exec"), ('''{ x+y for x in { x for x in xs } }''', "exec"), # Dict comp ('''{ x:1 for x in xs }''', "exec"), ('''{ x+y:1 for x in xs for y in ys }''', "exec"), ('''{ x+y+z:1 for x in xs if x if y if z if r for y in ys if x if y if z if p for z in zs if x if y if z if q}''', "exec"), ('''{ x+y:k for k, x in { x:1 for x in xs } }''', "exec"), # Class ('''\ @potato @sausage() class A(a,b,c="1",d="2",*args,**kwargs): VAR = x def method(self): super().method() return VAR ''', "exec"), # Lambda ('''lambda: x''', "exec"), ('''lambda y: x+y''', "exec"), ('''lambda a,*arg,b=BB,c=CC,**kwargs: POTATO+a+arg+b+c+kwargs''', "exec"), # With ('''\ with x() as y: y.floop() print(y) ''', "exec"), # try, except ('''\ try: something() except RandomError as e: print(e) print(e) ''', "exec"), # Import ('''import potato''', "exec"), ('''import potato.sausage''', "exec"), ('''from potato import sausage''', "exec"), ('''from potato import sausage as salami''', "exec"), ('''from potato import *''', "exec"), ('''\ def fn(): from potato import * ''', "exec", SyntaxError), # Yield ('''\ def f(): yield ''', "exec"), ('''\ def f(): yield from range(10) ''', "exec"), ] def dump_bool(b): return ("true" if b else "false") def dump_strings(ss): return "[]string{"+",".join([ '"%s"' % s for s in ss ])+"}" # Scope numbers to names (from symtable.h) SCOPES = { 1: "ScopeLocal", 2: "ScopeGlobalExplicit", 3: "ScopeGlobalImplicit", 4: "ScopeFree", 5: "ScopeCell", } #def-use flags to names (from symtable.h) DEF_FLAGS = ( ("DefGlobal", 1), # global stmt ("DefLocal", 2), # assignment in code block ("DefParam", 2<<1), # formal parameter ("DefNonlocal", 2<<2), # nonlocal stmt ("DefUse", 2<<3), # name is used ("DefFree", 2<<4), # name used but not defined in nested block ("DefFreeClass", 2<<5),# free variable from class's method ("DefImport", 2<<6), # assignment occurred via import ) #opt flags flags to names (from symtable.h) OPT_FLAGS = ( ("optImportStar", 1), ("optTopLevel", 2), ) BLOCK_TYPES = { "function": "FunctionBlock", "class": "ClassBlock", "module": "ModuleBlock", } def dump_flags(flag_bits, flags_dict): """Dump the bits in flag_bits using the flags_dict""" flags = [] for name, mask in flags_dict: if (flag_bits & mask) != 0: flags.append(name) if not flags: flags = ["0"] return "|".join(flags) def dump_symtable(st): """Dump the symtable""" out = "&SymTable{\n" out += 'Type:%s,\n' % BLOCK_TYPES[st.get_type()] # Return the type of the symbol table. Possible values are 'class', 'module', and 'function'. out += 'Name:"%s",\n' % st.get_name() # Return the table’s name. This is the name of the class if the table is for a class, the name of the function if the table is for a function, or 'top' if the table is global (get_type() returns 'module'). out += 'Lineno:%s,\n' % st.get_lineno() # Return the number of the first line in the block this table represents. out += 'Unoptimized:%s,\n' % dump_flags(st._table.optimized, OPT_FLAGS) # Return False if the locals in this table can be optimized. out += 'Nested:%s,\n' % dump_bool(st.is_nested()) # Return True if the block is a nested class or function. if use_readsymtab: # Use readsymtab modules to read the bitfields which aren't normally exported free, child_free, generator, varargs, varkeywords, returns_value, needs_class_closure = readsymtab(st._table) out += 'Free:%s,\n' % dump_bool(free) out += 'ChildFree:%s,\n' % dump_bool(child_free) out += 'Generator:%s,\n' % dump_bool(generator) out += 'Varargs:%s,\n' % dump_bool(varargs) out += 'Varkeywords:%s,\n' % dump_bool(varkeywords) out += 'ReturnsValue:%s,\n' % dump_bool(returns_value) out += 'NeedsClassClosure:%s,\n' % dump_bool(needs_class_closure) else: # Use ctypes to read the bitfields which aren't normally exported # FIXME compiler dependent! base_addr = id(st._table) + ctypes.sizeof(ctypes.c_long)*8+ ctypes.sizeof(ctypes.c_int)*3 flags = int.from_bytes(ctypes.c_int.from_address(base_addr), sys.byteorder) out += 'Free:%s,\n' % dump_bool(flags & (1 << 0)) out += 'ChildFree:%s,\n' % dump_bool(flags & (1 << 1)) out += 'Generator:%s,\n' % dump_bool(flags & (1 << 2)) out += 'Varargs:%s,\n' % dump_bool(flags & (1 << 3)) out += 'Varkeywords:%s,\n' % dump_bool(flags & (1 << 4)) out += 'ReturnsValue:%s,\n' % dump_bool(flags & (1 << 5)) out += 'NeedsClassClosure:%s,\n' % dump_bool(flags & (1 << 6)) #out += 'Exec:%s,\n' % dump_bool(st.has_exec()) # Return True if the block uses exec. #out += 'ImportStar:%s,\n' % dump_bool(st.has_import_star()) # Return True if the block uses a starred from-import. out += 'Varnames:%s,\n' % dump_strings(st._table.varnames) out += 'Symbols: Symbols{\n' for name in sorted(st.get_identifiers()): s = st.lookup(name) out += '"%s":%s,\n' % (name, dump_symbol(s)) out += '},\n' out += 'Children:Children{\n' for symtable in st.get_children(): out += '%s,\n' % dump_symtable(symtable) out += '},\n' out += "}" return out def dump_symbol(s): """Dump a symbol""" #class symtable.Symbol # An entry in a SymbolTable corresponding to an identifier in the source. The constructor is not public. out = "Symbol{\n" out += 'Flags:%s,\n' % dump_flags(s._Symbol__flags, DEF_FLAGS) scope = SCOPES.get(s._Symbol__scope, "scopeUnknown") out += 'Scope:%s,\n' % scope out += "}" return out def escape(x): """Encode strings with backslashes for python/go""" return x.replace('\\', "\\\\").replace('"', r'\"').replace("\n", r'\n').replace("\t", r'\t') def main(): """Write symtable_data_test.go""" path = "symtable_data_test.go" out = ["""// Test data generated by make_symtable_test.py - do not edit package symtable import ( "github.com/go-python/gpython/py" ) var symtableTestData = []struct { in string mode string // exec, eval or single out *SymTable exceptionType *py.Type errString string }{"""] for x in inp: source, mode = x[:2] if len(x) > 2: exc = x[2] try: table = symtable(source, "", mode) except exc as e: error = e.msg else: raise ValueError("Expecting exception %s" % exc) dumped_symtable = "nil" gostring = "nil" exc_name = "py.%s" % exc.__name__ else: table = symtable(source, "", mode) exc_name = "nil" error = "" dumped_symtable = dump_symtable(table) out.append('{"%s", "%s", %s, %s, "%s"},' % (escape(source), mode, dumped_symtable, exc_name, escape(error))) out.append("}\n") print("Writing %s" % path) with open(path, "w") as f: f.write("\n".join(out)) f.write("\n") subprocess.check_call(["gofmt", "-w", path]) if __name__ == "__main__": main()