Skip to content

Commit f8be47c

Browse files
pablogsalpablogsal
committed
gh-153569: group indentation and logical-line state
1 parent 334815b commit f8be47c

12 files changed

Lines changed: 242 additions & 178 deletions

File tree

‎Makefile.pre.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -395,6 +395,7 @@ PEGEN_OBJS= \
395395

396396
TOKENIZER_OBJS= \
397397
Parser/lexer/lexer.o \
398+
Parser/lexer/layout.o \
398399
Parser/lexer/number.o \
399400
Parser/lexer/state.o \
400401
Parser/lexer/string.o \

‎PCbuild/_freeze_module.vcxproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,7 @@
183183
<ClCompile Include="..\Parser\token.c" />
184184
<ClCompile Include="..\Parser\lexer\state.c" />
185185
<ClCompile Include="..\Parser\lexer\lexer.c" />
186+
<ClCompile Include="..\Parser\lexer\layout.c" />
186187
<ClCompile Include="..\Parser\lexer\number.c" />
187188
<ClCompile Include="..\Parser\lexer\string.c" />
188189
<ClCompile Include="..\Parser\tokenizer\decoder.c" />

‎PCbuild/_freeze_module.vcxproj.filters‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -463,6 +463,9 @@
463463
<ClCompile Include="..\Parser\lexer\lexer.c">
464464
<Filter>Source Files</Filter>
465465
</ClCompile>
466+
<ClCompile Include="..\Parser\lexer\layout.c">
467+
<Filter>Source Files</Filter>
468+
</ClCompile>
466469
<ClCompile Include="..\Parser\lexer\number.c">
467470
<Filter>Source Files</Filter>
468471
</ClCompile>

‎PCbuild/pythoncore.vcxproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -590,6 +590,7 @@
590590
<ClCompile Include="..\Parser\myreadline.c" />
591591
<ClCompile Include="..\Parser\lexer\state.c" />
592592
<ClCompile Include="..\Parser\lexer\lexer.c" />
593+
<ClCompile Include="..\Parser\lexer\layout.c" />
593594
<ClCompile Include="..\Parser\lexer\number.c" />
594595
<ClCompile Include="..\Parser\lexer\string.c" />
595596
<ClCompile Include="..\Parser\tokenizer\cursor.c" />

‎PCbuild/pythoncore.vcxproj.filters‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1349,6 +1349,9 @@
13491349
<ClCompile Include="..\Parser\lexer\lexer.c">
13501350
<Filter>Parser</Filter>
13511351
</ClCompile>
1352+
<ClCompile Include="..\Parser\lexer\layout.c">
1353+
<Filter>Parser</Filter>
1354+
</ClCompile>
13521355
<ClCompile Include="..\Parser\lexer\number.c">
13531356
<Filter>Parser</Filter>
13541357
</ClCompile>

‎Parser/lexer/layout.c‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
#include "Python.h"
2+
#include "errcode.h"
3+
#include "pycore_token.h"
4+
5+
#include "lexer_internal.h"
6+
#include "../tokenizer/helpers.h"
7+
#include "../tokenizer/reader.h"
8+
9+
#define TABSIZE 8
10+
#define ALTTABSIZE 1
11+
12+
int
13+
_PyLexer_ContinueLine(struct tok_state *tok)
14+
{
15+
int c = tok_nextc(tok);
16+
if (c == '\r') {
17+
c = tok_nextc(tok);
18+
}
19+
if (c != '\n') {
20+
tok->done = E_LINECONT;
21+
return -1;
22+
}
23+
c = tok_nextc(tok);
24+
if (c == EOF) {
25+
tok->done = E_EOF;
26+
tok->cur = tok->inp;
27+
return -1;
28+
} else {
29+
tok_backup(tok, c);
30+
}
31+
return c;
32+
}
33+
34+
35+
static int
36+
update_indentation(struct tok_state *tok, int col, int altcol)
37+
{
38+
lexer_layout_state *layout = &tok->layout;
39+
if (col == layout->stack[layout->depth].column) {
40+
if (altcol != layout->stack[layout->depth].alternate_column) {
41+
_PyTokenizer_indenterror(tok);
42+
return -1;
43+
}
44+
}
45+
else if (col > layout->stack[layout->depth].column) {
46+
if (layout->depth + 1 >= MAXINDENT) {
47+
tok->done = E_TOODEEP;
48+
tok->cur = tok->inp;
49+
return -1;
50+
}
51+
if (altcol <= layout->stack[layout->depth].alternate_column) {
52+
_PyTokenizer_indenterror(tok);
53+
return -1;
54+
}
55+
layout->pending++;
56+
layout->stack[++layout->depth] = (indentation_level){col, altcol};
57+
}
58+
else {
59+
while (layout->depth > 0 &&
60+
col < layout->stack[layout->depth].column) {
61+
layout->pending--;
62+
layout->depth--;
63+
}
64+
if (col != layout->stack[layout->depth].column) {
65+
tok->done = E_DEDENT;
66+
tok->cur = tok->inp;
67+
return -1;
68+
}
69+
if (altcol != layout->stack[layout->depth].alternate_column) {
70+
_PyTokenizer_indenterror(tok);
71+
return -1;
72+
}
73+
}
74+
return 0;
75+
}
76+
77+
int
78+
_PyLexer_BeginLine(struct tok_state *tok)
79+
{
80+
assert(tok->layout.at_bol);
81+
int c;
82+
int blankline = 0;
83+
int col = 0;
84+
int altcol = 0;
85+
tok->layout.at_bol = 0;
86+
int cont_line_col = 0;
87+
for (;;) {
88+
c = tok_nextc(tok);
89+
if (c == ' ') {
90+
col++, altcol++;
91+
}
92+
else if (c == '\t') {
93+
col = (col / TABSIZE + 1) * TABSIZE;
94+
altcol = (altcol / ALTTABSIZE + 1) * ALTTABSIZE;
95+
}
96+
else if (c == '\014') {/* Control-L (formfeed) */
97+
col = altcol = 0; /* For Emacs users */
98+
}
99+
else if (c == '\\') {
100+
// Indentation cannot be split over multiple physical lines
101+
// using backslashes. This means that if we found a backslash
102+
// preceded by whitespace, **the first one we find** determines
103+
// the level of indentation of whatever comes next.
104+
cont_line_col = cont_line_col ? cont_line_col : col;
105+
if ((c = _PyLexer_ContinueLine(tok)) == -1) {
106+
return -1;
107+
}
108+
}
109+
else if (c == EOF && PyErr_Occurred()) {
110+
return -1;
111+
}
112+
else {
113+
break;
114+
}
115+
}
116+
tok_backup(tok, c);
117+
if (c == '#' || c == '\n' || c == '\r') {
118+
int interactive = _PyTok_ReaderIsInteractive(tok);
119+
/* Lines with only whitespace and/or comments
120+
shouldn't affect the indentation and are
121+
not passed to the parser as NEWLINE tokens,
122+
except *totally* empty lines in interactive
123+
mode, which signal the end of a command group. */
124+
if (col == 0 && c == '\n' && interactive) {
125+
blankline = 0; /* Let it through */
126+
}
127+
else if (interactive && tok->lineno == 1) {
128+
/* In interactive mode, if the first line contains
129+
only spaces and/or a comment, let it through. */
130+
blankline = 0;
131+
col = altcol = 0;
132+
}
133+
else {
134+
blankline = 1; /* Ignore completely */
135+
}
136+
}
137+
if (!blankline && tok->level == 0) {
138+
col = cont_line_col ? cont_line_col : col;
139+
altcol = cont_line_col ? cont_line_col : altcol;
140+
if (update_indentation(tok, col, altcol) < 0) {
141+
return -1;
142+
}
143+
}
144+
return blankline;
145+
}
146+
147+
int
148+
_PyLexer_IndentationToken(struct tok_state *tok, struct token *token)
149+
{
150+
assert(tok->layout.pending != 0);
151+
const char *p_start = NULL;
152+
const char *p_end = NULL;
153+
if (tok->layout.pending < 0) {
154+
if (tok->tok_extra_tokens) {
155+
p_start = tok->cur;
156+
p_end = tok->cur;
157+
}
158+
tok->layout.pending++;
159+
return _PyLexer_token_setup(tok, token, DEDENT, p_start, p_end);
160+
}
161+
else {
162+
if (tok->tok_extra_tokens) {
163+
p_start = tok->buf;
164+
p_end = tok->cur;
165+
}
166+
tok->layout.pending--;
167+
return _PyLexer_token_setup(tok, token, INDENT, p_start, p_end);
168+
}
169+
}
170+
171+
int
172+
_PyLexer_Newline(struct tok_state *tok, struct token *token, int blankline)
173+
{
174+
tok->layout.at_bol = 1;
175+
if (blankline || tok->level > 0) {
176+
if (!tok->tok_extra_tokens) {
177+
return 0;
178+
}
179+
}
180+
else if (!tok->layout.comment_newline || !tok->tok_extra_tokens) {
181+
return _PyLexer_token_setup(tok, token, NEWLINE,
182+
tok->start, tok->cur - 1);
183+
}
184+
tok->layout.comment_newline = 0;
185+
return _PyLexer_token_setup(tok, token, NL, tok->start, tok->cur);
186+
}
187+
188+
void
189+
_PyLexer_ImplyDedents(struct tok_state *tok)
190+
{
191+
if (tok->layout.depth != 0) {
192+
tok->layout.pending = -tok->layout.depth;
193+
tok->layout.depth = 0;
194+
}
195+
}

0 commit comments

Comments
 (0)