Mailing List Archive

Update of /cvsroot/python/python/nondist/sandbox/ast
In directory usw-pr-cvs1:/tmp/cvs-serv13934

Modified Files:
astmodule.c test.py
Log Message:
New statements & expressions supported, plus some code cleanup.

New statements:

FunctionDef, ClassDef, TryExcept, TryFinally, While

argument handling is incomplete; don't handle fplist yet

New expressions:

Lambda, Attribute, ListComp, Dict, List

XXX I don't think List is right

Cleanups:

Add NEW_IDENTIFIER() macro, which saves a lot of typing & reading.
Move operator_ty lookup to get_operator() function.



Index: astmodule.c
===================================================================
RCS file: /cvsroot/python/python/nondist/sandbox/ast/astmodule.c,v
retrieving revision 1.6
retrieving revision 1.7
diff -C2 -d -r1.6 -r1.7
*** astmodule.c 21 Apr 2002 14:45:41 -0000 1.6
--- astmodule.c 22 Apr 2002 04:25:19 -0000 1.7
***************
*** 12,21 ****
--- 12,57 ----
#include "graminit.h"

+ #define NEW_IDENTIFIER(n) PyString_InternFromString(STR(n))
+
static asdl_seq *seq_for_testlist(node *);
static expr_ty ast_for_expr(node *);
static stmt_ty ast_for_stmt(node *);
+ static asdl_seq *ast_for_suite(node *);
+ static asdl_seq *ast_for_exprlist(node *, int);
+ static expr_ty ast_for_testlist(node *);

extern grammar _PyParser_Grammar; /* From graminit.c */

+ static operator_ty
+ get_operator(node *n)
+ {
+ switch (TYPE(n)) {
+ case VBAR:
+ return BitOr;
+ case CIRCUMFLEX:
+ return BitXor;
+ case AMPER:
+ return BitAnd;
+ case LEFTSHIFT:
+ return LShift;
+ case RIGHTSHIFT:
+ return RShift;
+ case PLUS:
+ return Add;
+ case MINUS:
+ return Sub;
+ case STAR:
+ return Mult;
+ case SLASH:
+ return Div;
+ case DOUBLESLASH:
+ return FloorDiv;
+ case PERCENT:
+ return Mod;
+ default:
+ return 0;
+ }
+ }
+
static int
set_context(expr_ty e, expr_context_ty ctx)
***************
*** 131,135 ****
int i;

- REQ(n, testlist);
seq = asdl_seq_new(NCH(n) / 2);
for (i = 0; i < NCH(n); i += 2) {
--- 167,170 ----
***************
*** 139,146 ****
}

static expr_ty
ast_for_lambdef(node *n)
{
! return NULL;
}

--- 174,372 ----
}

+ static arguments_ty
+ ast_for_arguments(node *n)
+ {
+
+ /* XXX TO DO
+ check for invalid argument lists like normal after default
+ handle nested tuple arguments
+ */
+
+ /* parameters: '(' [varargslist] ')'
+ varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME]
+ | '**' NAME) | fpdef ['=' test] (',' fpdef ['=' test])* [',']
+ */
+ int i, n_args = 0, n_defaults = 0;
+ asdl_seq *args, *defaults;
+ identifier vararg = NULL, kwarg = NULL;
+ node *ch;
+
+ if (TYPE(n) == parameters) {
+ if (NCH(n) == 2)
+ return arguments(NULL, NULL, NULL, NULL);
+ n = CHILD(n, 1);
+ }
+ REQ(n, varargslist);
+
+ /* first count the number of normal args & defaults */
+ for (i = 0; i < NCH(n); i++) {
+ ch = CHILD(n, i);
+ if (TYPE(ch) == fpdef)
+ n_args++;
+ if (TYPE(ch) == EQUAL)
+ n_defaults++;
+ }
+ args = n_args ? asdl_seq_new(n_args) : NULL;
+ defaults = n_defaults? asdl_seq_new(n_defaults) : NULL;
+
+ /* fpdef: NAME | '(' fplist ')'
+ fplist: fpdef (',' fpdef)* [',']
+ */
+ i = 0;
+ while (i < NCH(n)) {
+ ch = CHILD(n, i);
+ switch (TYPE(ch)) {
+ case fpdef:
+ if (NCH(ch) == 3) {
+ /* XXX don't handle fplist yet */
+ return NULL;
+ }
+ if (TYPE(CHILD(ch, 0)) == NAME)
+ asdl_seq_append(args, NEW_IDENTIFIER(CHILD(ch, 0)));
+ if (i + 1 < NCH(n) && TYPE(CHILD(n, i + 1)) == EQUAL) {
+ asdl_seq_append(defaults, ast_for_expr(CHILD(n, i + 2)));
+ i += 2;
+ }
+ i += 2; /* the name and the comma */
+ break;
+ case STAR:
+ vararg = NEW_IDENTIFIER(CHILD(n, i+1));
+ i += 3;
+ break;
+ case DOUBLESTAR:
+ kwarg = NEW_IDENTIFIER(CHILD(n, i+1));
+ i += 3;
+ break;
+ default:
+ fprintf(stderr, "unexpected node in varargslist: %d @ %d\n",
+ TYPE(ch), i);
+ }
+ }
+
+ return arguments(args, vararg, kwarg, defaults);
+ }
+
+ static stmt_ty
+ ast_for_funcdef(node *n)
+ {
+ /* funcdef: 'def' NAME parameters ':' suite */
+ identifier name = NEW_IDENTIFIER(CHILD(n, 1));
+ REQ(n, funcdef);
+ return FunctionDef(name, ast_for_arguments(CHILD(n, 2)),
+ ast_for_suite(CHILD(n, 4)));
+ }
+
static expr_ty
ast_for_lambdef(node *n)
{
! /* lambdef: 'lambda' [varargslist] ':' test */
! if (NCH(n) == 3)
! return Lambda(NULL, ast_for_expr(CHILD(n, 2)));
! else
! return Lambda(ast_for_arguments(CHILD(n, 1)),
! ast_for_expr(CHILD(n, 3)));
! }
!
! static int
! count_list_fors(node *n)
! {
! int n_fors = 0;
! node *ch = CHILD(n, 1);
! count_list_for:
! n_fors++;
! REQ(ch, list_for);
! if (NCH(ch) == 5)
! ch = CHILD(ch, 4);
! else
! return n_fors;
! count_list_iter:
! REQ(ch, list_iter);
! ch = CHILD(ch, 0);
! if (TYPE(ch) == list_for)
! goto count_list_for;
! else if (TYPE(ch) == list_if) {
! if (NCH(ch) == 3) {
! ch = CHILD(ch, 2);
! goto count_list_iter;
! } else
! return n_fors;
! }
! assert(0); /* can't get here */
! return -1;
! }
!
! static int
! count_list_ifs(node *n)
! {
! int n_ifs = 0;
! count_list_iter:
! REQ(n, list_iter);
! if (TYPE(CHILD(n, 0)) == list_for)
! return n_ifs;
! n = CHILD(n, 0);
! REQ(n, list_if);
! n_ifs++;
! if (NCH(n) == 2)
! return n_ifs;
! n = CHILD(n, 2);
! goto count_list_iter;
! }
!
! static expr_ty
! ast_for_listcomp(node *n)
! {
! /* listmaker: test ( list_for | (',' test)* [','] )
! list_for: 'for' exprlist 'in' testlist_safe [list_iter]
! list_iter: list_for | list_if
! list_if: 'if' test [list_iter]
! testlist_safe: test [(',' test)+ [',']]
! */
! expr_ty target;
! asdl_seq *listcomps;
! int i, n_fors;
! node *ch;
!
! REQ(n, listmaker);
! assert(NCH(n) > 1);
!
! target = ast_for_expr(CHILD(n, 0));
! set_context(target, Store);
!
! n_fors = count_list_fors(n);
! listcomps = asdl_seq_new(n_fors);
! ch = CHILD(n, 1);
! for (i = 0; i < n_fors; i++) {
! listcomp_ty c;
! asdl_seq *t;
! REQ(ch, list_for);
! t = ast_for_exprlist(CHILD(ch, 1), Store);
! if (asdl_seq_LEN(t) == 1)
! c = listcomp(asdl_seq_get(t, 0),
! ast_for_testlist(CHILD(ch, 3)), NULL);
! else
! c = listcomp(Tuple(t, Store),
! ast_for_testlist(CHILD(ch, 3)), NULL);
! if (NCH(ch) == 5) {
! int j, n_ifs;
! asdl_seq *ifs;
! ch = CHILD(ch, 4);
! n_ifs = count_list_ifs(ch);
! ifs = asdl_seq_new(n_ifs);
! for (j = 0; j < n_ifs; j++) {
! REQ(ch, list_iter);
! ch = CHILD(ch, 0);
! REQ(ch, list_if);
! asdl_seq_append(ifs, CHILD(ch, 1));
! if (NCH(ch) == 3)
! ch = CHILD(ch, 2);
! }
! /* on exit, must guarantee that ch is a list_for */
! if (TYPE(ch) == list_iter)
! ch = CHILD(ch, 0);
! }
! asdl_seq_append(listcomps, c);
! }
!
! return ListComp(target, listcomps);
}

***************
*** 158,162 ****
change later if needed.
*/
! return Name(PyString_InternFromString(STR(ch)), Load);
break;
case STRING:
--- 384,388 ----
change later if needed.
*/
! return Name(NEW_IDENTIFIER(ch), Load);
break;
case STRING:
***************
*** 171,177 ****
break;
case LSQB: /* list (or list comprehension) */
break;
! case LBRACE: /* dict */
break;
case BACKQUOTE: /* repr */
return Repr(ast_for_expr(CHILD(n, 1)));
--- 397,422 ----
break;
case LSQB: /* list (or list comprehension) */
+ ch = CHILD(n, 1);
+ REQ(ch, listmaker);
+ if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA)
+ return List(ast_for_testlist(ch), Load);
+ else
+ return ast_for_listcomp(ch);
break;
! case LBRACE: {
! /* dictmaker: test ':' test (',' test ':' test)* [','] */
! int i, size;
! asdl_seq *keys, *values;
! ch = CHILD(n, 1);
! size = (NCH(ch) + 1) / 4; /* plus one in case no trailing comma */
! keys = asdl_seq_new(size);
! values = asdl_seq_new(size);
! for (i = 0; i < NCH(ch); i += 4) {
! asdl_seq_append(keys, ast_for_expr(CHILD(ch, i)));
! asdl_seq_append(values, ast_for_expr(CHILD(ch, i + 2)));
! }
! return Dict(keys, values);
break;
+ }
case BACKQUOTE: /* repr */
return Repr(ast_for_expr(CHILD(n, 1)));
***************
*** 202,211 ****

asdl_seq *seq;
- operator_ty op = 0;
int i;

fprintf(stderr, "ast_for_expr(%d, %d)\n", TYPE(n), NCH(n));
loop:
- fprintf(stderr, "\texpr %d %d\n", TYPE(n), NCH(n));
switch (TYPE(n)) {
case test:
--- 447,454 ----
***************
*** 268,307 ****
goto loop;
}
! switch (TYPE(CHILD(n, 1))) {
! case VBAR:
! op = BitOr;
! break;
! case CIRCUMFLEX:
! op = BitXor;
! break;
! case AMPER:
! op = BitAnd;
! break;
! case LEFTSHIFT:
! op = LShift;
! break;
! case RIGHTSHIFT:
! op = RShift;
! break;
! case PLUS:
! op = Add;
! break;
! case MINUS:
! op = Sub;
! break;
! case STAR:
! op = Mult;
! break;
! case SLASH:
! op = Div;
! break;
! case DOUBLESLASH:
! op = FloorDiv;
! break;
! case PERCENT:
! op = Mod;
! break;
! }
! return BinOp(ast_for_expr(CHILD(n, 0)), op,
ast_for_expr(CHILD(n, 2)));
break;
--- 511,515 ----
goto loop;
}
! return BinOp(ast_for_expr(CHILD(n, 0)), get_operator(CHILD(n, 1)),
ast_for_expr(CHILD(n, 2)));
break;
***************
*** 326,330 ****
if (NCH(n) == 1)
return ast_for_atom(CHILD(n, 0));
! fprintf(stderr, "unhandled power\n");
return NULL;
break;
--- 534,550 ----
if (NCH(n) == 1)
return ast_for_atom(CHILD(n, 0));
! /* power: atom trailer* ('**' factor)*
! trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME */
! if (NCH(n) == 2 && TYPE(CHILD(n, 1)) == trailer) {
! node *ch = CHILD(n, 1);
! if (TYPE(CHILD(ch, 0)) == LPAR) {
! /* XXX a call */
! } else if (TYPE(CHILD(ch, 0)) == LSQB) {
! /* XXX a subscript */
! } else if (TYPE(CHILD(ch, 0)) == DOT) {
! return Attribute(ast_for_atom(CHILD(n, 0)),
! NEW_IDENTIFIER(CHILD(ch, 1)), Load);
! }
! }
return NULL;
break;
***************
*** 337,341 ****
ast_for_testlist(node *n)
{
! REQ(n, testlist);
if (NCH(n) == 1)
return ast_for_expr(CHILD(n, 0));
--- 557,561 ----
ast_for_testlist(node *n)
{
! /* could be a testlist or a listmaker with no list_for */
if (NCH(n) == 1)
return ast_for_expr(CHILD(n, 0));
***************
*** 411,414 ****
--- 631,635 ----
expr_ty e;

+ fprintf(stderr, "ast_for_exprlist(%d, %d)\n", TYPE(n), context);
REQ(n, exprlist);

***************
*** 490,497 ****
case import_as_name:
if (NCH(n) == 3)
! return alias(PyString_InternFromString(STR(CHILD(n, 0))),
! PyString_InternFromString(STR(CHILD(n, 2))));
else
! return alias(PyString_InternFromString(STR(CHILD(n, 0))),
NULL);
break;
--- 711,718 ----
case import_as_name:
if (NCH(n) == 3)
! return alias(NEW_IDENTIFIER(CHILD(n, 0)),
! NEW_IDENTIFIER(CHILD(n, 2)));
else
! return alias(NEW_IDENTIFIER(CHILD(n, 0)),
NULL);
break;
***************
*** 503,507 ****
alias_ty a = alias_for_import_name(CHILD(n, 0));
assert(!a->asname);
! a->asname = PyString_InternFromString(STR(CHILD(n, 2)));
return a;
}
--- 724,728 ----
alias_ty a = alias_for_import_name(CHILD(n, 0));
assert(!a->asname);
! a->asname = NEW_IDENTIFIER(CHILD(n, 2));
return a;
}
***************
*** 509,513 ****
case dotted_name:
if (NCH(n) == 1)
! return alias(PyString_InternFromString(STR(CHILD(n, 0))), NULL);
else {
/* Create a string of the form "a.b.c" */
--- 730,734 ----
case dotted_name:
if (NCH(n) == 1)
! return alias(NEW_IDENTIFIER(CHILD(n, 0)), NULL);
else {
/* Create a string of the form "a.b.c" */
***************
*** 581,585 ****
s = asdl_seq_new(NCH(n) / 2);
for (i = 1; i < NCH(n); i += 2) {
! name = PyString_InternFromString(STR(CHILD(n, i)));
if (!name)
return NULL;
--- 802,806 ----
s = asdl_seq_new(NCH(n) / 2);
for (i = 1; i < NCH(n); i += 2) {
! name = NEW_IDENTIFIER(CHILD(n, i));
if (!name)
return NULL;
***************
*** 715,719 ****
REQ(n, while_stmt);

!

return NULL;
--- 936,946 ----
REQ(n, while_stmt);

! if (NCH(n) == 4)
! return While(ast_for_expr(CHILD(n, 1)),
! ast_for_suite(CHILD(n, 3)), NULL);
! else
! return While(ast_for_expr(CHILD(n, 1)),
! ast_for_suite(CHILD(n, 3)),
! ast_for_suite(CHILD(n, 6)));

return NULL;
***************
*** 746,760 ****
}

! static stmt_ty
! ast_for_try_stmt(node *n)
{
! REQ(n, try_stmt);
! return NULL;
}

static stmt_ty
! ast_for_funcdef(node *n)
{
! REQ(n, funcdef);
return NULL;
}
--- 973,1022 ----
}

! static except_ty
! ast_for_except_clause(node *exc, node *body)
{
! /* except_clause: 'except' [test [',' test]] */
! REQ(exc, except_clause);
! REQ(body, suite);
!
! if (NCH(exc) == 1)
! return except(NULL, NULL, ast_for_suite(body));
! else if (NCH(exc) == 2)
! return except(ast_for_expr(CHILD(exc, 1)), NULL, ast_for_suite(body));
! else {
! expr_ty e = ast_for_expr(CHILD(exc, 3));
! set_context(e, Store);
! return except(ast_for_expr(CHILD(exc, 1)), e,
! ast_for_suite(body));
! }
}

static stmt_ty
! ast_for_try_stmt(node *n)
{
! REQ(n, try_stmt);
! if (TYPE(CHILD(n, 3)) == NAME) {/* must be 'finally' */
! /* try_stmt: 'try' ':' suite 'finally' ':' suite) */
! return TryFinally(ast_for_suite(CHILD(n, 2)),
! ast_for_suite(CHILD(n, 5)));
! } else {
! /* try_stmt: ('try' ':' suite (except_clause ':' suite)+
! ['else' ':' suite]
! */
! asdl_seq *handlers;
! int i, has_else = 0, n_except = NCH(n) - 3;
! if (TYPE(CHILD(n, NCH(n) - 3)) == NAME) {
! has_else = 1;
! n_except -= 3;
! }
! n_except /= 3;
! handlers = asdl_seq_new(n_except);
! for (i = 0; i < n_except; i++)
! asdl_seq_append(handlers,
! ast_for_except_clause(CHILD(n, 3 + i * 3),
! CHILD(n, 5 + i * 3)));
! return TryExcept(ast_for_suite(CHILD(n, 2)), handlers,
! has_else ? ast_for_suite(CHILD(n, NCH(n) - 1)): NULL);
! }
return NULL;
}
***************
*** 763,767 ****
--- 1025,1045 ----
ast_for_classdef(node *n)
{
+ /* classdef: 'class' NAME ['(' testlist ')'] ':' suite */
+ expr_ty _bases;
+ asdl_seq *bases;
REQ(n, classdef);
+ if (NCH(n) == 4)
+ return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), NULL,
+ ast_for_suite(CHILD(n, 3)));
+ /* else handle the base class list */
+ _bases = ast_for_testlist(CHILD(n, 3));
+ if (_bases->kind == Tuple_kind)
+ bases = _bases->v.Tuple.elts;
+ else {
+ bases = asdl_seq_new(1);
+ asdl_seq_append(bases, _bases);
+ }
+ return ClassDef(NEW_IDENTIFIER(CHILD(n, 1)), bases,
+ ast_for_suite(CHILD(n, 6)));
return NULL;
}
***************
*** 770,776 ****
ast_for_stmt(node *n)
{
! REQ(n, stmt);
! assert(NCH(n) == 1);
! n = CHILD(n, 0);
if (TYPE(n) == simple_stmt) {
/* I'm explicitly punting on multiple statements joined by a
--- 1048,1056 ----
ast_for_stmt(node *n)
{
! if (TYPE(n) == stmt) {
! assert(NCH(n) == 1);
! n = CHILD(n, 0);
! }
! fprintf(stderr, "stmt lineno %d\n", n->n_lineno);
if (TYPE(n) == simple_stmt) {
/* I'm explicitly punting on multiple statements joined by a

Index: test.py
===================================================================
RCS file: /cvsroot/python/python/nondist/sandbox/ast/test.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -C2 -d -r1.3 -r1.4
*** test.py 19 Apr 2002 22:15:12 -0000 1.3
--- test.py 22 Apr 2002 04:25:19 -0000 1.4
***************
*** 1,4 ****
import ast
! ast.transform("""global a, b, c
a = 1
d = c = 7
--- 1,20 ----
import ast
! ast.transform("""# let's start with a whole bunch of argument lists
! def f(x): pass
! def f(x, y): pass
! def f(x, y,): pass
! def f(x, y=1): pass
! def f(x=1, y=1): pass
! def f(*x): pass
! def f(**x): pass
! def f(*x, **y): pass
! lambda:1
! lambda x:1
! lambda x:x
! lambda x, y: 1
! lambda x, y, z,: y
! lambda x=lambda x:1 : 1
! def f(x, y, z=None, f=lambda x: lambda y: x + y): pass
! global a, b, c
a = 1
d = c = 7
***************
*** 60,62 ****
--- 76,125 ----
else:
pass
+ while 1:
+ 1
+ 2
+ while 1:
+ pass
+ while 1 and 2:
+ pass
+ else:
+ 1
+ try:
+ 1
+ finally:
+ 2
+ try:
+ a + b
+ except:
+ pass
+ try:
+ a + b
+ except E:
+ pass
+ except:
+ pass
+ try:
+ a + b
+ except E, b:
+ pass
+ except:
+ pass
+ try:
+ a + b
+ except (E, E2), b:
+ pass
+ except:
+ pass
+ class C: pass
+ class C(A): pass
+ class C(A, B): pass
+ class C(A, B, (C,)): pass
+ d = {}
+ d = {1:2}
+ d = {1:2,}
+ d = {1:2, 3:4,}
+ [1,2,3,4]
+ [.x for x in x if x if x for y in y if y]
+ [x for x in x]
+ obj.attr
""")