pythonscopebuilt-indynamic-scope

Is there a way to declare that a function should use the scope of the caller?


is there a feautre similar to C macros which lets you reuse code in an inline manner, without creating a seperate scope for that piece of code?

for example:

a=3
def foo():
    a=4
foo()
print a

will print 3, however i want it to print 4.

i am aware of solutions involving objects like classes or a global dict, however i'm looking for a more primitive solution (like a function decorator for example) that would simply let me make changes inside the scope of the caller instead.

thank you very much

edit:any solution that requires declaring which variables i'm going to use OR declaring a "namespace" like mutabale objects beforehand is not a solution i'm looking for.

i had made an attempt on my own:

def pgame():
a=3
c=5
print locals()
game(a)
print locals()


class inline_func(object):
    def __init__(self, f):
        self.f = f

    def __call__(self, *args, **kwargs):
        return self.f(*args, **kwargs)


#to be @inline_func
def game(b, a=4):
    exec("inspect.stack()[3][0].f_locals.update(inspect.stack()[1] [0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))\ninspect.stack()[1][0].f_locals.update(inspect.stack()[3][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[1][0]),ctypes.c_int(0))")
try:
    print "your code here"
finally:
    exec("inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))")

@inline_func
def strip_game(b, a=4):
    print "your code here"

but i have ran into a serious problem with how to inject code into strip_game without ruining the debugability of the program, because i had only thought of creating a new code object or using exec, both suffering from some severe problems.

MAJOR EDIT:

ok, so i have something close to a working solution, however i encounter a very wierd problem:

import inspect
import ctypes
import struct
import dis
import types



def cgame():
    a=3
    c=5
    print locals()
    strip_game(a)
    print locals()


def pgame():
    a=3
    c=5
    print locals()
    game(a)
    print locals()


class empty_deco(object):
    def __init__(self, f):
        self.f = f

    def __call__(self, *args, **kwargs):
        return self.f(*args, **kwargs)

debug_func = None
class inline_func(object):
    def __init__(self, f):
        self.f = f

    def __call__(self, *args, **kwargs):

        init_exec_string = "inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\n" + \
                           "ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))\n" + \
                           "inspect.stack()[1][0].f_locals.update(inspect.stack()[3][0].f_locals)\n" + \
                           "ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[1][0]),ctypes.c_int(0))"
        fini_exec_string = "inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\n" + \
                           "ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))" 

        co_stacksize = max(6, self.f.func_code.co_stacksize) # make sure we have enough space on the stack for everything
        co_consts = self.f.func_code.co_consts +(init_exec_string, fini_exec_string)
        init = "d"  + struct.pack("H", len(strip_game.f.func_code.co_consts)) #LOAD_CONST init_exec_string
        init += "d\x00\x00\x04U" # LOAD_CONST None, DUP_TOP, EXEC_STMT
        init += "z" + struct.pack("H", len(self.f.func_code.co_code) + 4) #SETUP_FINALLY

        fini = "Wd\x00\x00" # POP_BLOCK, LOAD_CONST None
        fini += "d" + struct.pack("H", len(strip_game.f.func_code.co_consts) + 1) #LOAD_CONST fini_exec_string
        fini += "d\x00\x00\x04UXd\x00\x00S" # LOAD_CONST None, DUP_TOP, EXEC_STMT, END_FINALLY, LOAD_CONST None, RETURN
        co_code = init + self.f.func_code.co_code + fini
        co_lnotab = "\x00\x00\x0b" + self.f.func_code.co_lnotab[1:] # every error in init will be attributed to @inline_func, errors in the function will be treated as expected, errors in fini will be attributed to the last line probably.
        new_code = types.CodeType(
        self.f.func_code.co_argcount,
        self.f.func_code.co_nlocals,
        co_stacksize,
        self.f.func_code.co_flags & ~(1), # optimized functions are problematic for us
        co_code,
        co_consts,
        self.f.func_code.co_names,
        self.f.func_code.co_varnames,
        self.f.func_code.co_filename,
        self.f.func_code.co_name,
        self.f.func_code.co_firstlineno,
        co_lnotab,
        self.f.func_code.co_freevars,
        self.f.func_code.co_cellvars,)

        self.inline_f =  types.FunctionType(new_code, self.f.func_globals, self.f.func_name, self.f.func_defaults, self.f.func_closure)
        #dis.dis(self.inline_f)
        global debug_func
        debug_func = self.inline_f
        return self.inline_f(*args, **kwargs)


@empty_deco
def game(b, a=4):
    exec("inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))\ninspect.stack()[1][0].f_locals.update(inspect.stack()[3][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[1][0]),ctypes.c_int(0))")
    try:
        print "inner locals:"
        print locals()
        print c
        return None
    finally:
        exec("inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))")

@inline_func
def strip_game(b, a=4):
    print "inner locals:"
    print locals()
    print c
    return None



def stupid():
    exec("print 'hello'")
    try:
        a=1
        b=2
        c=3
        d=4
    finally:
        exec("print 'goodbye'")

now this seems to work however, i get the following:

>>>cgame()
{'a': 3, 'c': 5}
{'a': 4, 'c': 5, 'b': 3}
your code here

Traceback (most recent call last):
  File "<pyshell#43>", line 1, in <module>
    cgame()
  File "C:\Python27\somefile.py", line 14, in cgame
    strip_game(a)
  File "C:\Python27\somefile.py", line 78, in __call__
    return self.inline_f(*args, **kwargs)
  File "C:\Python27\somefile.py", line 94, in strip_game
    z = c
NameError: global name 'c' is not defined

now when i disassemble the functions, i get the following very wierd compilation difference between game and strip_game:

in game:

86          16 LOAD_NAME                0 (locals)
             19 CALL_FUNCTION            0
             22 PRINT_ITEM          
             23 PRINT_NEWLINE       

 87          24 **LOAD_NAME**                1 (c)
             27 PRINT_ITEM          
             28 PRINT_NEWLINE       

in strip game:

95          16 LOAD_GLOBAL              0 (locals)
             19 CALL_FUNCTION            0
             22 PRINT_ITEM          
             23 PRINT_NEWLINE       

 96          24 LOAD_GLOBAL              1 (c)
             27 PRINT_ITEM          
             28 PRINT_NEWLINE       

why is does this difference occur?


Solution

  • ok, so after several hours of sitting on this thing i've managed to write a solution, there are some major pitfalls when approaching this and i'll note them below

    import inspect
    import ctypes
    import struct
    import dis
    import types
    
    def dump(obj):
      for attr in dir(obj):
        print("obj.%s = %r" % (attr, getattr(obj, attr)))
    
    def cgame():
        a=3
        c=5
        print locals()
        strip_game(a)
        print locals()
    
    
    def pgame():
        a=3
        c=5
        print locals()
        game(a)
        print locals()
    
    
    class empty_deco(object):
        def __init__(self, f):
            self.f = f
    
        def __call__(self, *args, **kwargs):
            return self.f(*args, **kwargs)
    
    
    
    debug_func = None
    class inline_func(object):
        def __init__(self, f):
            self.f = f
        # this is the price we pay for using 2.7
        # also, there is a huge glraing issue here, which is what happens if the user TRIES to access a global variable?
        @staticmethod
        def replace_globals_with_name_lookups(co):
            res = ""
            code = list(co)
            n = len(code)
            i = 0
            while i < n:
                c = code[i]
                op = ord(c)
                if dis.opname[op] == "STORE_GLOBAL":
                    code[i] = chr(dis.opmap['STORE_NAME'])
                elif dis.opname[op] == "DELETE_GLOBAL":
                    code[i] = chr(dis.opmap['DELETE_NAME'])
                elif dis.opname[op] == "LOAD_GLOBAL":
                    code[i] = chr(dis.opmap['LOAD_NAME'])
                i = i+1
                if op >= dis.HAVE_ARGUMENT:
                    i = i+2
            return "".join(code)
    
        def __call__(self, *args, **kwargs):
    
            init_exec_string = "inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\n" + \
                               "ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))\n" + \
                               "inspect.stack()[1][0].f_locals.update(inspect.stack()[3][0].f_locals)\n" + \
                               "ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[1][0]),ctypes.c_int(0))"
            fini_exec_string = "inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\n" + \
                               "ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))" 
    
            co_stacksize = max(6, self.f.func_code.co_stacksize) # make sure we have enough space on the stack for everything
            co_consts = self.f.func_code.co_consts +(init_exec_string, fini_exec_string)
            init = "d"  + struct.pack("H", len(strip_game.f.func_code.co_consts)) #LOAD_CONST init_exec_string
            init += "d\x00\x00\x04U" # LOAD_CONST None, DUP_TOP, EXEC_STMT
            init += "z" + struct.pack("H", len(self.f.func_code.co_code) + 4) #SETUP_FINALLY
    
            fini = "Wd\x00\x00" # POP_BLOCK, LOAD_CONST None
            fini += "d" + struct.pack("H", len(strip_game.f.func_code.co_consts) + 1) #LOAD_CONST fini_exec_string
            fini += "d\x00\x00\x04UXd\x00\x00S" # LOAD_CONST None, DUP_TOP, EXEC_STMT, END_FINALLY, LOAD_CONST None, RETURN
            co_code = init + self.replace_globals_with_name_lookups(self.f.func_code.co_code) + fini
            co_lnotab = "\x00\x00\x0b" + self.f.func_code.co_lnotab[1:] # every error in init will be attributed to @inline_func, errors in the function will be treated as expected, errors in fini will be attributed to the last line probably.
            new_code = types.CodeType(
            self.f.func_code.co_argcount,
            self.f.func_code.co_nlocals,
            co_stacksize,
            self.f.func_code.co_flags & ~(1), # optimized functions are problematic for us
            co_code,
            co_consts,
            self.f.func_code.co_names,
            self.f.func_code.co_varnames,
            self.f.func_code.co_filename,
            self.f.func_code.co_name,
            self.f.func_code.co_firstlineno,
            co_lnotab,
            self.f.func_code.co_freevars,
            self.f.func_code.co_cellvars,)
    
            self.inline_f =  types.FunctionType(new_code, self.f.func_globals, self.f.func_name, self.f.func_defaults, self.f.func_closure)
            #dis.dis(self.inline_f)
            global debug_func
            debug_func = self.inline_f
            return self.inline_f(*args, **kwargs)
    
    
    @empty_deco
    def game(b, a=4):
        exec("inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))\ninspect.stack()[1][0].f_locals.update(inspect.stack()[3][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[1][0]),ctypes.c_int(0))")
        try:
            print "inner locals:"
            print locals()
            print c
            return None
        finally:
            exec("inspect.stack()[3][0].f_locals.update(inspect.stack()[1][0].f_locals)\nctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(inspect.stack()[3][0]),ctypes.c_int(0))")
    
    @inline_func
    def strip_game(b, a=4):
        print "inner locals:"
        print locals()
        print c
        return None
    

    where the acutal code needed lies in the class inline_func and some of the imports (maybe you can make them internal to the class? i'm really not sure)

    so what does this whole thing do? well, it makes it so the code for strip_game and game are (nearly) identical, namely:

    1. it inserts a function prologue which updates the locals of the caller, then adds to locals of the caller to the callee.
    2. insert a try finally block around the function
    3. changes every symbol lookup from a global lookup to a normal (name) lookup, after some thought i had realized that this doens't really have any effects
    4. upon entering the finally block, updates the caller locals.

    there are some major pitfalls making things like these, i'll list a few problems i've encountered:

    1. cpython compiler_nameop function optimizes namespace lookup based on the simplicity of the given function, that means that it will optimize name lookups to global lookups if it can
    2. changing the bytecode means affecting the debug-ability of the program, i had addressed this in the co_lnotab variable
    3. for large functions this solution won't work as some of the opcodes would have to use extended_args: namely, the loads of the variables and the try-finally block (this point is solvable by using extended_args anyways...)

    thank @jsbueno for putting in the time and pointing me to PyFrame_LocalsToFast.

    P.S. this solution works for python 2.7.6, python has some issues when it comes to stability of the API, so for newer versions this might need to be fixed.