Skip to content

Commit 3e0055f

Browse files
jeremyhyltonjeremyhylton
committed
Merge ast-branch to head
This change implements a new bytecode compiler, based on a transformation of the parse tree to an abstract syntax defined in Parser/Python.asdl. The compiler implementation is not complete, but it is in stable enough shape to run the entire test suite excepting two disabled tests.
1 parent 2cb94ab commit 3e0055f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+13379
-6514
lines changed

‎Include/Python-ast.h

+418
Large diffs are not rendered by default.

‎Include/Python.h

+1-2
Original file line numberDiff line numberDiff line change
@@ -128,8 +128,7 @@
128128
#include "pystrtod.h"
129129

130130
/* _Py_Mangle is defined in compile.c */
131-
PyAPI_FUNC(int) _Py_Mangle(char *p, char *name, \
132-
char *buffer, size_t maxlen);
131+
PyAPI_FUNC(PyObject*) _Py_Mangle(PyObject *p, PyObject *name);
133132

134133
/* PyArg_GetInt is deprecated and should not be used, use PyArg_Parse(). */
135134
#define PyArg_GetInt(v, a) PyArg_Parse((v), "i", (a))

‎Include/asdl.h

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#ifndef Py_ASDL_H
2+
#define Py_ASDL_H
3+
4+
typedef PyObject * identifier;
5+
typedef PyObject * string;
6+
typedef PyObject * object;
7+
8+
typedef enum {false, true} bool;
9+
10+
/* It would be nice if the code generated by asdl_c.py was completely
11+
independent of Python, but it is a goal the requires too much work
12+
at this stage. So, for example, I'll represent identifiers as
13+
interned Python strings.
14+
*/
15+
16+
/* XXX A sequence should be typed so that its use can be typechecked. */
17+
18+
/* XXX We shouldn't pay for offset when we don't need APPEND. */
19+
20+
typedef struct {
21+
int size;
22+
int offset;
23+
void *elements[1];
24+
} asdl_seq;
25+
26+
asdl_seq *asdl_seq_new(int size);
27+
void asdl_seq_free(asdl_seq *);
28+
29+
#ifdef Py_DEBUG
30+
#define asdl_seq_GET(S, I) (S)->elements[(I)]
31+
#define asdl_seq_SET(S, I, V) { \
32+
int _asdl_i = (I); \
33+
assert((S) && _asdl_i < (S)->size); \
34+
(S)->elements[_asdl_i] = (V); \
35+
}
36+
#define asdl_seq_APPEND(S, V) { \
37+
assert((S) && (S)->offset < (S)->size); \
38+
(S)->elements[(S)->offset++] = (V); \
39+
}
40+
#else
41+
#define asdl_seq_GET(S, I) (S)->elements[(I)]
42+
#define asdl_seq_SET(S, I, V) (S)->elements[I] = (V)
43+
#define asdl_seq_APPEND(S, V) (S)->elements[(S)->offset++] = (V)
44+
#endif
45+
#define asdl_seq_LEN(S) ((S) == NULL ? 0 : (S)->size)
46+
47+
/* Routines to marshal the basic types. */
48+
int marshal_write_int(PyObject **, int *, int);
49+
int marshal_write_bool(PyObject **, int *, bool);
50+
int marshal_write_identifier(PyObject **, int *, identifier);
51+
int marshal_write_string(PyObject **, int *, string);
52+
int marshal_write_object(PyObject **, int *, object);
53+
54+
#endif /* !Py_ASDL_H */

‎Include/ast.h

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#ifndef Py_AST_H
2+
#define Py_AST_H
3+
#ifdef __cplusplus
4+
extern "C" {
5+
#endif
6+
7+
extern DL_IMPORT(mod_ty) PyAST_FromNode(const node *, PyCompilerFlags *flags,
8+
const char *);
9+
10+
#ifdef __cplusplus
11+
}
12+
#endif
13+
#endif /* !Py_AST_H */

‎Include/code.h

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/* Definitions for bytecode */
2+
3+
#ifndef Py_CODE_H
4+
#define Py_CODE_H
5+
#ifdef __cplusplus
6+
extern "C" {
7+
#endif
8+
9+
/* Bytecode object */
10+
typedef struct {
11+
PyObject_HEAD
12+
int co_argcount; /* #arguments, except *args */
13+
int co_nlocals; /* #local variables */
14+
int co_stacksize; /* #entries needed for evaluation stack */
15+
int co_flags; /* CO_..., see below */
16+
PyObject *co_code; /* instruction opcodes */
17+
PyObject *co_consts; /* list (constants used) */
18+
PyObject *co_names; /* list of strings (names used) */
19+
PyObject *co_varnames; /* tuple of strings (local variable names) */
20+
PyObject *co_freevars; /* tuple of strings (free variable names) */
21+
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
22+
/* The rest doesn't count for hash/cmp */
23+
PyObject *co_filename; /* string (where it was loaded from) */
24+
PyObject *co_name; /* string (name, for reference) */
25+
int co_firstlineno; /* first source line number */
26+
PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) */
27+
} PyCodeObject;
28+
29+
/* Masks for co_flags above */
30+
#define CO_OPTIMIZED 0x0001
31+
#define CO_NEWLOCALS 0x0002
32+
#define CO_VARARGS 0x0004
33+
#define CO_VARKEYWORDS 0x0008
34+
#define CO_NESTED 0x0010
35+
#define CO_GENERATOR 0x0020
36+
/* The CO_NOFREE flag is set if there are no free or cell variables.
37+
This information is redundant, but it allows a single flag test
38+
to determine whether there is any extra work to be done when the
39+
call frame it setup.
40+
*/
41+
#define CO_NOFREE 0x0040
42+
/* XXX Temporary hack. Until generators are a permanent part of the
43+
language, we need a way for a code object to record that generators
44+
were *possible* when it was compiled. This is so code dynamically
45+
compiled *by* a code object knows whether to allow yield stmts. In
46+
effect, this passes on the "from __future__ import generators" state
47+
in effect when the code block was compiled. */
48+
#define CO_GENERATOR_ALLOWED 0x1000 /* no longer used in an essential way */
49+
#define CO_FUTURE_DIVISION 0x2000
50+
51+
#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */
52+
53+
extern DL_IMPORT(PyTypeObject) PyCode_Type;
54+
55+
#define PyCode_Check(op) ((op)->ob_type == &PyCode_Type)
56+
#define PyCode_GetNumFree(op) (PyTuple_GET_SIZE((op)->co_freevars))
57+
58+
/* Public interface */
59+
DL_IMPORT(PyCodeObject *) PyCode_New(
60+
int, int, int, int, PyObject *, PyObject *, PyObject *, PyObject *,
61+
PyObject *, PyObject *, PyObject *, PyObject *, int, PyObject *);
62+
/* same as struct above */
63+
DL_IMPORT(int) PyCode_Addr2Line(PyCodeObject *, int);
64+
65+
/* for internal use only */
66+
#define _PyCode_GETCODEPTR(co, pp) \
67+
((*(co)->co_code->ob_type->tp_as_buffer->bf_getreadbuffer) \
68+
((co)->co_code, 0, (void **)(pp)))
69+
70+
#ifdef __cplusplus
71+
}
72+
#endif
73+
#endif /* !Py_CODE_H */

‎Include/compile.h

+13-58
Original file line numberDiff line numberDiff line change
@@ -1,61 +1,13 @@
1-
2-
/* Definitions for bytecode */
1+
#ifndef Py_CODE_H
2+
#include "code.h"
3+
#endif
34

45
#ifndef Py_COMPILE_H
56
#define Py_COMPILE_H
67
#ifdef __cplusplus
78
extern "C" {
89
#endif
910

10-
/* Bytecode object */
11-
typedef struct {
12-
PyObject_HEAD
13-
int co_argcount; /* #arguments, except *args */
14-
int co_nlocals; /* #local variables */
15-
int co_stacksize; /* #entries needed for evaluation stack */
16-
int co_flags; /* CO_..., see below */
17-
PyObject *co_code; /* instruction opcodes */
18-
PyObject *co_consts; /* list (constants used) */
19-
PyObject *co_names; /* list of strings (names used) */
20-
PyObject *co_varnames; /* tuple of strings (local variable names) */
21-
PyObject *co_freevars; /* tuple of strings (free variable names) */
22-
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
23-
/* The rest doesn't count for hash/cmp */
24-
PyObject *co_filename; /* string (where it was loaded from) */
25-
PyObject *co_name; /* string (name, for reference) */
26-
int co_firstlineno; /* first source line number */
27-
PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) */
28-
} PyCodeObject;
29-
30-
/* Masks for co_flags above */
31-
#define CO_OPTIMIZED 0x0001
32-
#define CO_NEWLOCALS 0x0002
33-
#define CO_VARARGS 0x0004
34-
#define CO_VARKEYWORDS 0x0008
35-
#define CO_NESTED 0x0010
36-
#define CO_GENERATOR 0x0020
37-
/* The CO_NOFREE flag is set if there are no free or cell variables.
38-
This information is redundant, but it allows a single flag test
39-
to determine whether there is any extra work to be done when the
40-
call frame it setup.
41-
*/
42-
#define CO_NOFREE 0x0040
43-
/* XXX Temporary hack. Until generators are a permanent part of the
44-
language, we need a way for a code object to record that generators
45-
were *possible* when it was compiled. This is so code dynamically
46-
compiled *by* a code object knows whether to allow yield stmts. In
47-
effect, this passes on the "from __future__ import generators" state
48-
in effect when the code block was compiled. */
49-
#define CO_GENERATOR_ALLOWED 0x1000 /* no longer used in an essential way */
50-
#define CO_FUTURE_DIVISION 0x2000
51-
52-
PyAPI_DATA(PyTypeObject) PyCode_Type;
53-
54-
#define PyCode_Check(op) ((op)->ob_type == &PyCode_Type)
55-
#define PyCode_GetNumFree(op) (PyTuple_GET_SIZE((op)->co_freevars))
56-
57-
#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */
58-
5911
/* Public interface */
6012
struct _node; /* Declare the existence of this type */
6113
PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
@@ -68,19 +20,22 @@ PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int);
6820
/* Future feature support */
6921

7022
typedef struct {
71-
int ff_found_docstring;
72-
int ff_last_lineno;
73-
int ff_features;
23+
int ff_features; /* flags set by future statements */
24+
int ff_lineno; /* line number of last future statement */
7425
} PyFutureFeatures;
7526

76-
PyAPI_FUNC(PyFutureFeatures *) PyNode_Future(struct _node *, const char *);
77-
PyAPI_FUNC(PyCodeObject *) PyNode_CompileFlags(struct _node *, const char *,
78-
PyCompilerFlags *);
79-
8027
#define FUTURE_NESTED_SCOPES "nested_scopes"
8128
#define FUTURE_GENERATORS "generators"
8229
#define FUTURE_DIVISION "division"
8330

31+
struct _mod; /* Declare the existence of this type */
32+
DL_IMPORT(PyCodeObject *) PyAST_Compile(struct _mod *, const char *,
33+
PyCompilerFlags *);
34+
DL_IMPORT(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
35+
36+
#define ERR_LATE_FUTURE \
37+
"from __future__ imports must occur at the beginning of the file"
38+
8439
#ifdef __cplusplus
8540
}
8641
#endif

‎Include/pyport.h

+1
Original file line numberDiff line numberDiff line change
@@ -583,6 +583,7 @@ typedef struct fd_set {
583583

584584
#ifndef INT_MAX
585585
#define INT_MAX 2147483647
586+
#define INT_MIN (-INT_MAX - 1)
586587
#endif
587588

588589
#ifndef LONG_MAX

‎Include/pythonrun.h

+41-31
Original file line numberDiff line numberDiff line change
@@ -29,46 +29,37 @@ PyAPI_FUNC(int) Py_IsInitialized(void);
2929
PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void);
3030
PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *);
3131

32-
PyAPI_FUNC(int) PyRun_AnyFile(FILE *, const char *);
33-
PyAPI_FUNC(int) PyRun_AnyFileEx(FILE *, const char *, int);
34-
35-
PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *);
36-
PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
37-
38-
PyAPI_FUNC(int) PyRun_SimpleString(const char *);
32+
PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, char *, PyCompilerFlags *);
33+
PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, char *, int, PyCompilerFlags *);
3934
PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *);
40-
PyAPI_FUNC(int) PyRun_SimpleFile(FILE *, const char *);
41-
PyAPI_FUNC(int) PyRun_SimpleFileEx(FILE *, const char *, int);
4235
PyAPI_FUNC(int) PyRun_SimpleFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
43-
PyAPI_FUNC(int) PyRun_InteractiveOne(FILE *, const char *);
4436
PyAPI_FUNC(int) PyRun_InteractiveOneFlags(FILE *, const char *, PyCompilerFlags *);
45-
PyAPI_FUNC(int) PyRun_InteractiveLoop(FILE *, const char *);
4637
PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags *);
4738

48-
PyAPI_FUNC(struct _node *) PyParser_SimpleParseString(const char *, int);
49-
PyAPI_FUNC(struct _node *) PyParser_SimpleParseFile(FILE *, const char *, int);
50-
PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int, int);
51-
PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlagsFilename(const char *,
52-
const char *,
53-
int,
54-
int);
39+
PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *,
40+
int, PyCompilerFlags *flags);
41+
PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *, int,
42+
char *, char *,
43+
PyCompilerFlags *, int *);
44+
#define PyParser_SimpleParseString(S, B) \
45+
PyParser_SimpleParseStringFlags(S, B, 0)
46+
#define PyParser_SimpleParseFile(FP, S, B) \
47+
PyParser_SimpleParseFileFlags(FP, S, B, 0)
48+
PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int,
49+
int);
5550
PyAPI_FUNC(struct _node *) PyParser_SimpleParseFileFlags(FILE *, const char *,
5651
int, int);
5752

58-
PyAPI_FUNC(PyObject *) PyRun_String(const char *, int, PyObject *, PyObject *);
59-
PyAPI_FUNC(PyObject *) PyRun_File(FILE *, const char *, int, PyObject *, PyObject *);
60-
PyAPI_FUNC(PyObject *) PyRun_FileEx(FILE *, const char *, int,
61-
PyObject *, PyObject *, int);
62-
PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, PyObject *,
63-
PyCompilerFlags *);
64-
PyAPI_FUNC(PyObject *) PyRun_FileFlags(FILE *, const char *, int, PyObject *,
65-
PyObject *, PyCompilerFlags *);
66-
PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int, PyObject *,
67-
PyObject *, int, PyCompilerFlags *);
68-
69-
PyAPI_FUNC(PyObject *) Py_CompileString(const char *, const char *, int);
53+
PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *,
54+
PyObject *, PyCompilerFlags *);
55+
56+
PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int,
57+
PyObject *, PyObject *, int,
58+
PyCompilerFlags *);
59+
60+
#define Py_CompileString(str, p, s) Py_CompileStringFlags(str, p, s, NULL)
7061
PyAPI_FUNC(PyObject *) Py_CompileStringFlags(const char *, const char *, int,
71-
PyCompilerFlags *);
62+
PyCompilerFlags *);
7263
PyAPI_FUNC(struct symtable *) Py_SymtableString(const char *, const char *, int);
7364

7465
PyAPI_FUNC(void) PyErr_Print(void);
@@ -84,6 +75,25 @@ PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *);
8475
/* Bootstrap */
8576
PyAPI_FUNC(int) Py_Main(int argc, char **argv);
8677

78+
/* Use macros for a bunch of old variants */
79+
#define PyRun_String(str, s, g, l) PyRun_StringFlags(str, s, g, l, NULL)
80+
#define PyRun_AnyFile(fp, name) PyRun_AnyFileExFlags(fp, name, 0, NULL)
81+
#define PyRun_AnyFileEx(fp, name, closeit) \
82+
PyRun_AnyFileExFlags(fp, name, closeit, NULL)
83+
#define PyRun_AnyFileFlags(fp, name, flags) \
84+
PyRun_AnyFileExFlags(fp, name, 0, flags)
85+
#define PyRun_SimpleString(s, f) PyRunSimpleStringFlags(s, f, NULL)
86+
#define PyRun_SimpleFile(f, p) PyRun_SimpleFileExFlags(f, p, 0, NULL)
87+
#define PyRun_SimpleFileEx(f, p, c) PyRun_SimpleFileExFlags(f, p, c, NULL)
88+
#define PyRun_InteractiveOne(f, p) PyRun_InteractiveOneFlags(f, p, NULL)
89+
#define PyRun_InteractiveLoop(f, p) PyRun_InteractiveLoopFlags(f, p, NULL)
90+
#define PyRun_File(fp, p, s, g, l) \
91+
PyRun_FileExFlags(fp, p, s, g, l, 0, NULL)
92+
#define PyRun_FileEx(fp, p, s, g, l, c) \
93+
PyRun_FileExFlags(fp, p, s, g, l, c, NULL)
94+
#define PyRun_FileFlags(fp, p, s, g, l, flags) \
95+
PyRun_FileExFlags(fp, p, s, g, l, 0, flags)
96+
8797
/* In getpath.c */
8898
PyAPI_FUNC(char *) Py_GetProgramFullPath(void);
8999
PyAPI_FUNC(char *) Py_GetPrefix(void);

0 commit comments

Comments
 (0)