Hello.
Recently, Generator Comprehensions were mentioned again on python-list.
I have written an implementation for the compiler module. To try it
out, however, you must be able to rebuild Python from source, because it
also requires a change to Grammar.
1. Edit Python-2.3/Grammar/Grammar and add an alternative to the
"listmaker" production:
-listmaker: test ( list_for | (',' test)* [','] )
+listmaker: test ( list_for | (',' test)* [','] ) | 'yield' test list_for
1.5. Now [yield None for x in []] parses, but crashes the written-in-C
compiler:[color=blue][color=green][color=darkred]
>>> [yield None for x in []][/color][/color][/color]
SystemError: com_node: unexpected node type
2. Apply the patch below to Lib/compiler
3. Use compiler.compil e to compile code with generator comprehensions:
from compiler import compile
import dis
code = compile("""
def f():
gg = [yield (x,y) for x in range(10) for y in range(10) if y > x]
print gg, type(gg), list(gg)
""", "<None>", "exec")
exec code
dis.dis(f.func_ code.co_consts[1])
f()
4. It's possible to write code so that __import__ uses compiler.compil e
instead of the written-in-C compiler, but I don't have this code handy.
Also, a test suite is needed, and presumably a written-in-C implementation
as well. (option 2: make the compiler.compil e interface the standard
compiler, and let the builtin compiler support a Python subset
sufficient to bootstrap the written-in-python compiler, or arrange
to ship .pyc of the compiler package and completely get rid of the
written-in-C compiler)
5. PEP remains rejected by BDFL anyway
diff -ur compiler.orig/ast.py compiler/ast.py
--- compiler.orig/ast.py 2002-02-23 16:35:33.000000 000 -0600
+++ compiler/ast.py 2003-08-26 06:55:51.000000 000 -0500
@@ -1191,6 +1191,53 @@
def __repr__(self):
return "If(%s, %s)" % (repr(self.test s), repr(self.else_ ))
+class GenCompInner(No de):
+ nodes["gencompinn er"] = "GenCompInn er"
+ def __init__(self, expr, quals):
+ self.expr = expr
+ self.quals = quals
+
+ def getChildren(sel f):
+ children = []
+ children.append (self.expr)
+ children.extend (flatten(self.q uals))
+ return tuple(children)
+
+ def getChildNodes(s elf):
+ nodelist = []
+ nodelist.append (self.expr)
+ nodelist.extend (flatten_nodes( self.quals))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "GenCompInner(% s, %s)" % (repr(self.expr ), repr(self.quals ))
+
+class GenComp(Node):
+ nodes["gencomp"] = "GenComp"
+ def __init__(self, inner):
+ self.argnames = ()
+ self.defaults = ()
+ self.flags = 0
+ self.code = inner
+ self.varargs = self.kwargs = None
+
+ def getChildren(sel f):
+ children = []
+ children.append (self.argnames)
+ children.extend (flatten(self.d efaults))
+ children.append (self.flags)
+ children.append (self.code)
+ return tuple(children)
+
+ def getChildNodes(s elf):
+ nodelist = []
+ nodelist.extend (flatten_nodes( self.defaults))
+ nodelist.append (self.code)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "GenComp(%s )" % (repr(self.code ),)
+
class ListComp(Node):
nodes["listcomp"] = "ListComp"
def __init__(self, expr, quals):
diff -ur compiler.orig/pycodegen.py compiler/pycodegen.py
--- compiler.orig/pycodegen.py 2002-12-31 12:26:17.000000 000 -0600
+++ compiler/pycodegen.py 2003-08-26 06:54:53.000000 000 -0500
@@ -563,6 +563,51 @@
# list comprehensions
__list_count = 0
+ def visitGenCompInn er(self, node):
+ self.set_lineno (node)
+ # setup list
+
+ stack = []
+ for i, for_ in zip(range(len(n ode.quals)), node.quals):
+ start, anchor = self.visit(for_ )
+ cont = None
+ for if_ in for_.ifs:
+ if cont is None:
+ cont = self.newBlock()
+ self.visit(if_, cont)
+ stack.insert(0, (start, cont, anchor))
+
+ self.visit(node .expr)
+ self.emit('YIEL D_VALUE')
+
+ for start, cont, anchor in stack:
+ if cont:
+ skip_one = self.newBlock()
+ self.emit('JUMP _FORWARD', skip_one)
+ self.startBlock (cont)
+ self.emit('POP_ TOP')
+ self.nextBlock( skip_one)
+ self.emit('JUMP _ABSOLUTE', start)
+ self.startBlock (anchor)
+ self.emit('LOAD _CONST', None)
+
+ def visitGenComp(se lf, node):
+ gen = GenCompCodeGene rator(node, self.scopes, self.class_name ,
+ self.get_module ())
+ walk(node.code, gen)
+ gen.finish()
+ self.set_lineno (node)
+ frees = gen.scope.get_f ree_vars()
+ if frees:
+ for name in frees:
+ self.emit('LOAD _CLOSURE', name)
+ self.emit('LOAD _CONST', gen)
+ self.emit('MAKE _CLOSURE', len(node.defaul ts))
+ else:
+ self.emit('LOAD _CONST', gen)
+ self.emit('MAKE _FUNCTION', len(node.defaul ts))
+ self.emit('CALL _FUNCTION', 0)
+
def visitListComp(s elf, node):
self.set_lineno (node)
# setup list
@@ -1245,6 +1290,20 @@
unpackTuple = unpackSequence
+class GenCompCodeGene rator(NestedSco peMixin, AbstractFunctio nCode,
+ CodeGenerator):
+ super_init = CodeGenerator._ _init__ # call be other init
+
+ __super_init = AbstractFunctio nCode.__init__
+
+ def __init__(self, comp, scopes, class_name, mod):
+ self.scopes = scopes
+ self.scope = scopes[comp]
+ self.__super_in it(comp, scopes, 1, class_name, mod)
+ self.graph.setF reeVars(self.sc ope.get_free_va rs())
+ self.graph.setC ellVars(self.sc ope.get_cell_va rs())
+ self.graph.setF lag(CO_GENERATO R)
+
class FunctionCodeGen erator(NestedSc opeMixin, AbstractFunctio nCode,
CodeGenerator):
super_init = CodeGenerator._ _init__ # call be other init
diff -ur compiler.orig/symbols.py compiler/symbols.py
--- compiler.orig/symbols.py 2002-12-31 12:17:42.000000 000 -0600
+++ compiler/symbols.py 2003-08-25 17:03:27.000000 000 -0500
@@ -231,6 +231,15 @@
self.visit(node .code, scope)
self.handle_fre e_vars(scope, parent)
+ def visitGenComp(se lf, node, parent):
+ scope = LambdaScope(sel f.module, self.klass);
+ if parent.nested or isinstance(pare nt, FunctionScope):
+ scope.nested = 1
+ self.scopes[node] = scope
+ self._do_args(s cope, node.argnames)
+ self.visit(node .code, scope)
+ self.handle_fre e_vars(scope, parent)
+
def _do_args(self, scope, args):
for name in args:
if type(name) == types.TupleType :
diff -ur compiler.orig/transformer.py compiler/transformer.py
--- compiler.orig/transformer.py 2003-04-06 04:00:45.000000 000 -0500
+++ compiler/transformer.py 2003-08-26 06:56:02.000000 000 -0500
@@ -1026,18 +1026,25 @@
if hasattr(symbol, 'list_for'):
def com_list_constr uctor(self, nodelist):
# listmaker: test ( list_for | (',' test)* [','] )
+ # | 'yield' list_for
values = []
+ yield_flag = 0
+ if nodelist[1][1] == 'yield':
+ yield_flag = 1
+ nodelist = nodelist[1:]
for i in range(1, len(nodelist)):
if nodelist[i][0] == symbol.list_for :
assert len(nodelist[i:]) == 1
return self.com_list_c omprehension(va lues[0],
- nodelist[i])
+ nodelist[i],
+ yield_flag)
elif nodelist[i][0] == token.COMMA:
continue
values.append(s elf.com_node(no delist[i]))
+ assert not yieldflag
return List(values)
- def com_list_compre hension(self, expr, node):
+ def com_list_compre hension(self, expr, node, yield_flag):
# list_iter: list_for | list_if
# list_for: 'for' exprlist 'in' testlist[list_iter]
# list_if: 'if' test[list_iter]
@@ -1071,7 +1078,10 @@
raise SyntaxError, \
("unexpected list comprehension element: %s %d"
% (node, lineno))
- n = ListComp(expr, fors)
+ if yield_flag:
+ n = GenComp(GenComp Inner(expr, fors))
+ else:
+ n = ListComp(expr, fors)
n.lineno = lineno
return n
Recently, Generator Comprehensions were mentioned again on python-list.
I have written an implementation for the compiler module. To try it
out, however, you must be able to rebuild Python from source, because it
also requires a change to Grammar.
1. Edit Python-2.3/Grammar/Grammar and add an alternative to the
"listmaker" production:
-listmaker: test ( list_for | (',' test)* [','] )
+listmaker: test ( list_for | (',' test)* [','] ) | 'yield' test list_for
1.5. Now [yield None for x in []] parses, but crashes the written-in-C
compiler:[color=blue][color=green][color=darkred]
>>> [yield None for x in []][/color][/color][/color]
SystemError: com_node: unexpected node type
2. Apply the patch below to Lib/compiler
3. Use compiler.compil e to compile code with generator comprehensions:
from compiler import compile
import dis
code = compile("""
def f():
gg = [yield (x,y) for x in range(10) for y in range(10) if y > x]
print gg, type(gg), list(gg)
""", "<None>", "exec")
exec code
dis.dis(f.func_ code.co_consts[1])
f()
4. It's possible to write code so that __import__ uses compiler.compil e
instead of the written-in-C compiler, but I don't have this code handy.
Also, a test suite is needed, and presumably a written-in-C implementation
as well. (option 2: make the compiler.compil e interface the standard
compiler, and let the builtin compiler support a Python subset
sufficient to bootstrap the written-in-python compiler, or arrange
to ship .pyc of the compiler package and completely get rid of the
written-in-C compiler)
5. PEP remains rejected by BDFL anyway
diff -ur compiler.orig/ast.py compiler/ast.py
--- compiler.orig/ast.py 2002-02-23 16:35:33.000000 000 -0600
+++ compiler/ast.py 2003-08-26 06:55:51.000000 000 -0500
@@ -1191,6 +1191,53 @@
def __repr__(self):
return "If(%s, %s)" % (repr(self.test s), repr(self.else_ ))
+class GenCompInner(No de):
+ nodes["gencompinn er"] = "GenCompInn er"
+ def __init__(self, expr, quals):
+ self.expr = expr
+ self.quals = quals
+
+ def getChildren(sel f):
+ children = []
+ children.append (self.expr)
+ children.extend (flatten(self.q uals))
+ return tuple(children)
+
+ def getChildNodes(s elf):
+ nodelist = []
+ nodelist.append (self.expr)
+ nodelist.extend (flatten_nodes( self.quals))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "GenCompInner(% s, %s)" % (repr(self.expr ), repr(self.quals ))
+
+class GenComp(Node):
+ nodes["gencomp"] = "GenComp"
+ def __init__(self, inner):
+ self.argnames = ()
+ self.defaults = ()
+ self.flags = 0
+ self.code = inner
+ self.varargs = self.kwargs = None
+
+ def getChildren(sel f):
+ children = []
+ children.append (self.argnames)
+ children.extend (flatten(self.d efaults))
+ children.append (self.flags)
+ children.append (self.code)
+ return tuple(children)
+
+ def getChildNodes(s elf):
+ nodelist = []
+ nodelist.extend (flatten_nodes( self.defaults))
+ nodelist.append (self.code)
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "GenComp(%s )" % (repr(self.code ),)
+
class ListComp(Node):
nodes["listcomp"] = "ListComp"
def __init__(self, expr, quals):
diff -ur compiler.orig/pycodegen.py compiler/pycodegen.py
--- compiler.orig/pycodegen.py 2002-12-31 12:26:17.000000 000 -0600
+++ compiler/pycodegen.py 2003-08-26 06:54:53.000000 000 -0500
@@ -563,6 +563,51 @@
# list comprehensions
__list_count = 0
+ def visitGenCompInn er(self, node):
+ self.set_lineno (node)
+ # setup list
+
+ stack = []
+ for i, for_ in zip(range(len(n ode.quals)), node.quals):
+ start, anchor = self.visit(for_ )
+ cont = None
+ for if_ in for_.ifs:
+ if cont is None:
+ cont = self.newBlock()
+ self.visit(if_, cont)
+ stack.insert(0, (start, cont, anchor))
+
+ self.visit(node .expr)
+ self.emit('YIEL D_VALUE')
+
+ for start, cont, anchor in stack:
+ if cont:
+ skip_one = self.newBlock()
+ self.emit('JUMP _FORWARD', skip_one)
+ self.startBlock (cont)
+ self.emit('POP_ TOP')
+ self.nextBlock( skip_one)
+ self.emit('JUMP _ABSOLUTE', start)
+ self.startBlock (anchor)
+ self.emit('LOAD _CONST', None)
+
+ def visitGenComp(se lf, node):
+ gen = GenCompCodeGene rator(node, self.scopes, self.class_name ,
+ self.get_module ())
+ walk(node.code, gen)
+ gen.finish()
+ self.set_lineno (node)
+ frees = gen.scope.get_f ree_vars()
+ if frees:
+ for name in frees:
+ self.emit('LOAD _CLOSURE', name)
+ self.emit('LOAD _CONST', gen)
+ self.emit('MAKE _CLOSURE', len(node.defaul ts))
+ else:
+ self.emit('LOAD _CONST', gen)
+ self.emit('MAKE _FUNCTION', len(node.defaul ts))
+ self.emit('CALL _FUNCTION', 0)
+
def visitListComp(s elf, node):
self.set_lineno (node)
# setup list
@@ -1245,6 +1290,20 @@
unpackTuple = unpackSequence
+class GenCompCodeGene rator(NestedSco peMixin, AbstractFunctio nCode,
+ CodeGenerator):
+ super_init = CodeGenerator._ _init__ # call be other init
+
+ __super_init = AbstractFunctio nCode.__init__
+
+ def __init__(self, comp, scopes, class_name, mod):
+ self.scopes = scopes
+ self.scope = scopes[comp]
+ self.__super_in it(comp, scopes, 1, class_name, mod)
+ self.graph.setF reeVars(self.sc ope.get_free_va rs())
+ self.graph.setC ellVars(self.sc ope.get_cell_va rs())
+ self.graph.setF lag(CO_GENERATO R)
+
class FunctionCodeGen erator(NestedSc opeMixin, AbstractFunctio nCode,
CodeGenerator):
super_init = CodeGenerator._ _init__ # call be other init
diff -ur compiler.orig/symbols.py compiler/symbols.py
--- compiler.orig/symbols.py 2002-12-31 12:17:42.000000 000 -0600
+++ compiler/symbols.py 2003-08-25 17:03:27.000000 000 -0500
@@ -231,6 +231,15 @@
self.visit(node .code, scope)
self.handle_fre e_vars(scope, parent)
+ def visitGenComp(se lf, node, parent):
+ scope = LambdaScope(sel f.module, self.klass);
+ if parent.nested or isinstance(pare nt, FunctionScope):
+ scope.nested = 1
+ self.scopes[node] = scope
+ self._do_args(s cope, node.argnames)
+ self.visit(node .code, scope)
+ self.handle_fre e_vars(scope, parent)
+
def _do_args(self, scope, args):
for name in args:
if type(name) == types.TupleType :
diff -ur compiler.orig/transformer.py compiler/transformer.py
--- compiler.orig/transformer.py 2003-04-06 04:00:45.000000 000 -0500
+++ compiler/transformer.py 2003-08-26 06:56:02.000000 000 -0500
@@ -1026,18 +1026,25 @@
if hasattr(symbol, 'list_for'):
def com_list_constr uctor(self, nodelist):
# listmaker: test ( list_for | (',' test)* [','] )
+ # | 'yield' list_for
values = []
+ yield_flag = 0
+ if nodelist[1][1] == 'yield':
+ yield_flag = 1
+ nodelist = nodelist[1:]
for i in range(1, len(nodelist)):
if nodelist[i][0] == symbol.list_for :
assert len(nodelist[i:]) == 1
return self.com_list_c omprehension(va lues[0],
- nodelist[i])
+ nodelist[i],
+ yield_flag)
elif nodelist[i][0] == token.COMMA:
continue
values.append(s elf.com_node(no delist[i]))
+ assert not yieldflag
return List(values)
- def com_list_compre hension(self, expr, node):
+ def com_list_compre hension(self, expr, node, yield_flag):
# list_iter: list_for | list_if
# list_for: 'for' exprlist 'in' testlist[list_iter]
# list_if: 'if' test[list_iter]
@@ -1071,7 +1078,10 @@
raise SyntaxError, \
("unexpected list comprehension element: %s %d"
% (node, lineno))
- n = ListComp(expr, fors)
+ if yield_flag:
+ n = GenComp(GenComp Inner(expr, fors))
+ else:
+ n = ListComp(expr, fors)
n.lineno = lineno
return n
Comment