Skip to content

Commit 7440eb4

Browse files
EarlopainEarlopain
committed
Fix %Q with newline delimiter and heredoc interpolation
The lexer did not jump to the `heredoc_end`, causing the heredoc end delimiter to be parsed twice. Normally the heredocs get flushed when a newline is encountered. But because the newline is part of the string delimiter, that codepath is not taken. Fixes [Bug #21758]
1 parent 81051c1 commit 7440eb4

File tree

6 files changed

+158
-0
lines changed

6 files changed

+158
-0
lines changed

‎snapshots/heredoc_percent_q_newline_delimiter.txt‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
@ ProgramNode (location: (1,0)-(22,0))
2+
├── flags: ∅
3+
├── locals: []
4+
└── statements:
5+
@ StatementsNode (location: (1,0)-(22,0))
6+
├── flags: ∅
7+
└── body: (length: 7)
8+
├── @ InterpolatedStringNode (location: (1,0)-(3,0))
9+
│ ├── flags: newline, static_literal, mutable
10+
│ ├── opening_loc: (1,0)-(2,0) = "%Q\n"
11+
│ ├── parts: (length: 1)
12+
│ │ └── @ EmbeddedStatementsNode (location: (2,0)-(2,6))
13+
│ │ ├── flags: ∅
14+
│ │ ├── opening_loc: (2,0)-(2,2) = "\#{"
15+
│ │ ├── statements:
16+
│ │ │ @ StatementsNode (location: (2,2)-(2,5))
17+
│ │ │ ├── flags: ∅
18+
│ │ │ └── body: (length: 1)
19+
│ │ │ └── @ StringNode (location: (2,2)-(2,5))
20+
│ │ │ ├── flags: static_literal, frozen
21+
│ │ │ ├── opening_loc: (2,2)-(2,5) = "<<B"
22+
│ │ │ ├── content_loc: (3,0)-(3,0) = ""
23+
│ │ │ ├── closing_loc: (3,0)-(4,0) = "B\n"
24+
│ │ │ └── unescaped: ""
25+
│ │ └── closing_loc: (2,5)-(2,6) = "}"
26+
│ └── closing_loc: (2,6)-(3,0) = "\n"
27+
├── @ InterpolatedStringNode (location: (5,0)-(7,0))
28+
│ ├── flags: newline, static_literal, mutable
29+
│ ├── opening_loc: (5,0)-(6,0) = "%\n"
30+
│ ├── parts: (length: 1)
31+
│ │ └── @ EmbeddedStatementsNode (location: (6,0)-(6,6))
32+
│ │ ├── flags: ∅
33+
│ │ ├── opening_loc: (6,0)-(6,2) = "\#{"
34+
│ │ ├── statements:
35+
│ │ │ @ StatementsNode (location: (6,2)-(6,5))
36+
│ │ │ ├── flags: ∅
37+
│ │ │ └── body: (length: 1)
38+
│ │ │ └── @ StringNode (location: (6,2)-(6,5))
39+
│ │ │ ├── flags: static_literal, frozen
40+
│ │ │ ├── opening_loc: (6,2)-(6,5) = "<<B"
41+
│ │ │ ├── content_loc: (7,0)-(7,0) = ""
42+
│ │ │ ├── closing_loc: (7,0)-(8,0) = "B\n"
43+
│ │ │ └── unescaped: ""
44+
│ │ └── closing_loc: (6,5)-(6,6) = "}"
45+
│ └── closing_loc: (6,6)-(7,0) = "\n"
46+
├── @ StringNode (location: (9,0)-(9,3))
47+
│ ├── flags: newline
48+
│ ├── opening_loc: (9,0)-(9,3) = "<<A"
49+
│ ├── content_loc: (10,0)-(10,0) = ""
50+
│ ├── closing_loc: (10,0)-(11,0) = "A\n"
51+
│ └── unescaped: ""
52+
├── @ InterpolatedStringNode (location: (9,5)-(12,0))
53+
│ ├── flags: newline, static_literal, mutable
54+
│ ├── opening_loc: (9,5)-(10,0) = "%Q\n"
55+
│ ├── parts: (length: 1)
56+
│ │ └── @ EmbeddedStatementsNode (location: (11,0)-(11,6))
57+
│ │ ├── flags: ∅
58+
│ │ ├── opening_loc: (11,0)-(11,2) = "\#{"
59+
│ │ ├── statements:
60+
│ │ │ @ StatementsNode (location: (11,2)-(11,5))
61+
│ │ │ ├── flags: ∅
62+
│ │ │ └── body: (length: 1)
63+
│ │ │ └── @ StringNode (location: (11,2)-(11,5))
64+
│ │ │ ├── flags: static_literal, frozen
65+
│ │ │ ├── opening_loc: (11,2)-(11,5) = "<<B"
66+
│ │ │ ├── content_loc: (12,0)-(12,0) = ""
67+
│ │ │ ├── closing_loc: (12,0)-(13,0) = "B\n"
68+
│ │ │ └── unescaped: ""
69+
│ │ └── closing_loc: (11,5)-(11,6) = "}"
70+
│ └── closing_loc: (11,6)-(12,0) = "\n"
71+
├── @ StringNode (location: (14,0)-(14,3))
72+
│ ├── flags: newline
73+
│ ├── opening_loc: (14,0)-(14,3) = "<<A"
74+
│ ├── content_loc: (15,0)-(15,0) = ""
75+
│ ├── closing_loc: (15,0)-(16,0) = "A\n"
76+
│ └── unescaped: ""
77+
├── @ InterpolatedStringNode (location: (14,5)-(17,0))
78+
│ ├── flags: newline, static_literal, mutable
79+
│ ├── opening_loc: (14,5)-(15,0) = "%\n"
80+
│ ├── parts: (length: 1)
81+
│ │ └── @ EmbeddedStatementsNode (location: (16,0)-(16,6))
82+
│ │ ├── flags: ∅
83+
│ │ ├── opening_loc: (16,0)-(16,2) = "\#{"
84+
│ │ ├── statements:
85+
│ │ │ @ StatementsNode (location: (16,2)-(16,5))
86+
│ │ │ ├── flags: ∅
87+
│ │ │ └── body: (length: 1)
88+
│ │ │ └── @ StringNode (location: (16,2)-(16,5))
89+
│ │ │ ├── flags: static_literal, frozen
90+
│ │ │ ├── opening_loc: (16,2)-(16,5) = "<<B"
91+
│ │ │ ├── content_loc: (17,0)-(17,0) = ""
92+
│ │ │ ├── closing_loc: (17,0)-(18,0) = "B\n"
93+
│ │ │ └── unescaped: ""
94+
│ │ └── closing_loc: (16,5)-(16,6) = "}"
95+
│ └── closing_loc: (16,6)-(17,0) = "\n"
96+
└── @ InterpolatedStringNode (location: (20,0)-(22,0))
97+
├── flags: newline, static_literal, mutable
98+
├── opening_loc: (20,0)-(21,0) = "%Q\r\n"
99+
├── parts: (length: 1)
100+
│ └── @ EmbeddedStatementsNode (location: (21,0)-(21,6))
101+
│ ├── flags: ∅
102+
│ ├── opening_loc: (21,0)-(21,2) = "\#{"
103+
│ ├── statements:
104+
│ │ @ StatementsNode (location: (21,2)-(21,5))
105+
│ │ ├── flags: ∅
106+
│ │ └── body: (length: 1)
107+
│ │ └── @ StringNode (location: (21,2)-(21,5))
108+
│ │ ├── flags: static_literal, frozen
109+
│ │ ├── opening_loc: (21,2)-(21,5) = "<<B"
110+
│ │ ├── content_loc: (22,0)-(22,0) = ""
111+
│ │ ├── closing_loc: (22,0)-(23,0) = "B\n"
112+
│ │ └── unescaped: ""
113+
│ └── closing_loc: (21,5)-(21,6) = "}"
114+
└── closing_loc: (21,6)-(22,0) = "\r\n"

‎src/prism.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11612,6 +11612,13 @@ parser_lex(pm_parser_t *parser) {
1161211612
LEX(PM_TOKEN_LABEL_END);
1161311613
}
1161411614

11615+
// When the delimiter itself is a newline, we won't
11616+
// get a chance to flush heredocs in the usual places since
11617+
// the newline is already consumed.
11618+
if (term == '\n' && parser->heredoc_end) {
11619+
parser_flush_heredoc_end(parser);
11620+
}
11621+
1161511622
lex_state_set(parser, PM_LEX_STATE_END);
1161611623
lex_mode_pop(parser);
1161711624
LEX(PM_TOKEN_STRING_END);

‎test/prism/errors/heredoc_percent_q_newline_delimiter.txt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
%q
2+
#{<<B}
3+
B
4+
^ unexpected constant, expecting end-of-input
5+
6+
<<A; %q
7+
A
8+
#{<<B}
9+
B
10+
^ unexpected constant, expecting end-of-input
11+

‎test/prism/fixtures/heredoc_percent_q_newline_delimiter.txt‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
%Q
2+
#{<<B}
3+
B
4+
5+
%
6+
#{<<B}
7+
B
8+
9+
<<A; %Q
10+
A
11+
#{<<B}
12+
B
13+
14+
<<A; %
15+
A
16+
#{<<B}
17+
B
18+
19+
# \r\n
20+
%Q
21+
#{<<B}
22+
B

‎test/prism/ruby/parser_test.rb‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ class ParserTest < TestCase
6262
"alias.txt",
6363
"seattlerb/bug_215.txt",
6464

65+
# %Q with newline delimiter and heredoc interpolation
66+
"heredoc_percent_q_newline_delimiter.txt",
67+
6568
# 1.. && 2
6669
"ranges.txt",
6770

‎test/prism/ruby/ruby_parser_test.rb‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ class RubyParserTest < TestCase
3737
"alias.txt",
3838
"dsym_str.txt",
3939
"dos_endings.txt",
40+
"heredoc_percent_q_newline_delimiter.txt",
4041
"heredocs_with_fake_newlines.txt",
4142
"heredocs_with_ignored_newlines.txt",
4243
"method_calls.txt",

0 commit comments

Comments
 (0)