Skip to content

Commit b3bcf48

Browse files
Martin521Martin521
authored
Refactor #line processing - keep the original positions in the AST (#18699)
1 parent ecd8e2a commit b3bcf48

30 files changed

+241
-261
lines changed

‎docs/release-notes/.FSharp.Compiler.Service/10.0.100.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,4 @@
2828
* `SynExprRecordField` now includes a `range` field ([PR #18617](https://github.com/dotnet/fsharp/pull/18617))
2929
* Mark `Range.Zero` as obsolete in favor of `Range.range0` ([PR #18664](https://github.com/dotnet/fsharp/pull/18664))
3030
* Use `Synbinding` to model `and!` ([PR #18805](https://github.com/dotnet/fsharp/pull/18805))
31+
* Redesign #line processing. The original positions (unaffected by #line directives) are now kept in the AST, and `__LINE__` and `__SOURCE_LINE__` show the original line numbers / file names. However, all diagnostics and debug information stays the same (shows the position transformed by the #line directives). ([Issue #18553](https://github.com/dotnet/fsharp/issues/18553), [PR #18699](https://github.com/dotnet/fsharp/pull/18699))

‎src/Compiler/Checking/PatternMatchCompilation.fs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1069,11 +1069,12 @@ let CompilePatternBasic
10691069
mkCompGenSequential mMatch e (mkDefault (mMatch, resultTy))
10701070

10711071
| ThrowIncompleteMatchException ->
1072+
let mmMatch = mMatch.ApplyLineDirectives()
10721073
mkThrow mMatch resultTy
10731074
(mkExnExpr(g.MatchFailureException_tcr,
1074-
[ mkString g mMatch mMatch.FileName
1075-
mkInt g mMatch mMatch.StartLine
1076-
mkInt g mMatch mMatch.StartColumn], mMatch))
1075+
[ mkString g mMatch mmMatch.FileName
1076+
mkInt g mMatch mmMatch.StartLine
1077+
mkInt g mMatch mmMatch.StartColumn], mMatch))
10771078

10781079
| IgnoreWithWarning ->
10791080
mkUnit g mMatch

‎src/Compiler/Checking/QuotationTranslator.fs

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -214,20 +214,21 @@ let (|ObjectInitializationCheck|_|) g expr =
214214
isUnitTy g resultTy -> ValueSome()
215215
| _ -> ValueNone
216216

217-
let rec EmitDebugInfoIfNecessary cenv env m astExpr : ExprData =
217+
let rec EmitDebugInfoIfNecessary cenv env (m: range) astExpr : ExprData =
218218
// do not emit debug info if emitDebugInfoInQuotations = false or it was already written for the given expression
219219
if cenv.emitDebugInfoInQuotations && not (QP.isAttributedExpression astExpr) then
220220
cenv.emitDebugInfoInQuotations <- false
221221
try
222222
let mk_tuple g m es = mkRefTupled g m es (List.map (tyOfExpr g) es)
223223

224224
let rangeExpr =
225+
let mm = m.ApplyLineDirectives() // LineDirectives: map ranges for the debugger
225226
mk_tuple cenv.g m
226-
[ mkString cenv.g m m.FileName
227-
mkInt cenv.g m m.StartLine
228-
mkInt cenv.g m m.StartColumn
229-
mkInt cenv.g m m.EndLine
230-
mkInt cenv.g m m.EndColumn ]
227+
[ mkString cenv.g m mm.FileName
228+
mkInt cenv.g m mm.StartLine
229+
mkInt cenv.g m mm.StartColumn
230+
mkInt cenv.g m mm.EndLine
231+
mkInt cenv.g m mm.EndColumn ]
231232
let attrExpr =
232233
mk_tuple cenv.g m
233234
[ mkString cenv.g m "DebugRange"

‎src/Compiler/CodeGen/IlxGen.fs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1849,6 +1849,8 @@ let AddIncrementalLocalAssemblyFragmentToIlxGenEnv (cenv: cenv, isIncrementalFra
18491849

18501850
/// Generate IL debugging information.
18511851
let GenILSourceMarker (g: TcGlobals) (m: range) =
1852+
let m = m.ApplyLineDirectives()
1853+
18521854
ILDebugPoint.Create(
18531855
document = g.memoize_file m.FileIndex,
18541856
line = m.StartLine,
@@ -2512,7 +2514,7 @@ type CodeGenBuffer(m: range, mgbuf: AssemblyBuilder, methodName, alreadyUsedArgs
25122514
// Add a nop to make way for the first debug point.
25132515
do
25142516
if mgbuf.cenv.options.generateDebugSymbols then
2515-
let doc = g.memoize_file m.FileIndex
2517+
let doc = g.memoize_file (m.ApplyLineDirectives().FileIndex)
25162518
let i = FeeFeeInstr mgbuf.cenv doc
25172519
codebuf.Add i // for the FeeFee or a better debug point
25182520

@@ -2599,7 +2601,7 @@ type CodeGenBuffer(m: range, mgbuf: AssemblyBuilder, methodName, alreadyUsedArgs
25992601
// Emit FeeFee breakpoints for hidden code, see https://blogs.msdn.microsoft.com/jmstall/2005/06/19/line-hidden-and-0xfeefee-sequence-points/
26002602
member cgbuf.EmitStartOfHiddenCode() =
26012603
if mgbuf.cenv.options.generateDebugSymbols then
2602-
let doc = g.memoize_file m.FileIndex
2604+
let doc = g.memoize_file (m.ApplyLineDirectives().FileIndex)
26032605
let i = FeeFeeInstr mgbuf.cenv doc
26042606
hasDebugPoints <- true
26052607

‎src/Compiler/Driver/CompilerDiagnostics.fs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2063,6 +2063,7 @@ let FormatDiagnosticLocation (tcConfig: TcConfig) (m: Range) : FormattedDiagnost
20632063
File = ""
20642064
}
20652065
else
2066+
let m = m.ApplyLineDirectives()
20662067
let file = m.FileName
20672068

20682069
let file =
@@ -2194,6 +2195,8 @@ let CollectFormattedDiagnostics (tcConfig: TcConfig, severity: FSharpDiagnosticS
21942195
| DiagnosticStyle.Rich ->
21952196
match diagnostic.Range with
21962197
| Some m ->
2198+
let m = m.ApplyLineDirectives()
2199+
21972200
let content =
21982201
m.FileName
21992202
|> FileSystem.GetFullFilePathInDirectoryShim tcConfig.implicitIncludeDir
@@ -2279,6 +2282,7 @@ type PhasedDiagnostic with
22792282
match diagnostic.Range with
22802283
| None -> ()
22812284
| Some m ->
2285+
let m = m.ApplyLineDirectives()
22822286
let fileName = m.FileName
22832287
let lineA = m.StartLine
22842288
let lineB = m.EndLine

‎src/Compiler/Driver/ParseAndCheckInputs.fs

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -215,24 +215,24 @@ let PostParseModuleSpec (_i, defaultNamespace, isLastCompiland, fileName, intf)
215215

216216
SynModuleOrNamespaceSig(lid, isRecursive, kind, decls, xmlDoc, attributes, None, range, trivia)
217217

218-
let private collectCodeComments (lexbuf: UnicodeLexing.Lexbuf) =
219-
let tripleSlashComments = XmlDocStore.ReportInvalidXmlDocPositions(lexbuf)
220-
221-
[
222-
yield! CommentStore.GetComments(lexbuf)
223-
yield! (List.map CommentTrivia.LineComment tripleSlashComments)
224-
]
225-
|> List.sortBy (function
226-
| CommentTrivia.LineComment r
227-
| CommentTrivia.BlockComment r -> r.StartLine, r.StartColumn)
228-
229-
let private collectParsedInputTrivia lexbuf diagnosticOptions isScript submoduleRanges =
218+
let private finishPreprocessing lexbuf diagnosticOptions isScript submoduleRanges =
230219
WarnScopes.MergeInto diagnosticOptions isScript submoduleRanges lexbuf
220+
LineDirectives.add lexbuf.StartPos.FileIndex (LineDirectiveStore.GetLineDirectives lexbuf)
231221

222+
let private collectParsedInputTrivia lexbuf =
232223
{
233224
ConditionalDirectives = IfdefStore.GetTrivia(lexbuf)
234225
WarnDirectives = WarnScopes.getDirectiveTrivia (lexbuf)
235-
CodeComments = collectCodeComments lexbuf
226+
CodeComments =
227+
let tripleSlashComments = XmlDocStore.ReportInvalidXmlDocPositions(lexbuf)
228+
229+
[
230+
yield! CommentStore.GetComments(lexbuf)
231+
yield! List.map CommentTrivia.LineComment tripleSlashComments
232+
]
233+
|> List.sortBy (function
234+
| CommentTrivia.LineComment r
235+
| CommentTrivia.BlockComment r -> r.StartLine, r.StartColumn)
236236
}
237237

238238
let private getImplSubmoduleRanges (impls: ParsedImplFileFragment list) =
@@ -276,8 +276,9 @@ let PostParseModuleImpls
276276

277277
let isScript = IsScript fileName
278278

279-
let trivia =
280-
collectParsedInputTrivia lexbuf diagnosticOptions isScript (getImplSubmoduleRanges impls)
279+
finishPreprocessing lexbuf diagnosticOptions isScript (getImplSubmoduleRanges impls)
280+
281+
let trivia = collectParsedInputTrivia lexbuf
281282

282283
let othersWithSameName =
283284
impls
@@ -309,8 +310,9 @@ let PostParseModuleSpecs
309310
identifiers: Set<string>
310311
) =
311312

312-
let trivia =
313-
collectParsedInputTrivia lexbuf diagnosticOptions false (getSpecSubmoduleRanges specs)
313+
finishPreprocessing lexbuf diagnosticOptions false (getSpecSubmoduleRanges specs)
314+
315+
let trivia = collectParsedInputTrivia lexbuf
314316

315317
let othersWithSameName =
316318
specs

‎src/Compiler/Facilities/prim-lexing.fs

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -211,40 +211,35 @@ open System.Collections.Generic
211211
type internal Position =
212212
val FileIndex: int
213213
val Line: int
214-
val OriginalLine: int
215214
val AbsoluteOffset: int
216215
val StartOfLineAbsoluteOffset: int
217216
member x.Column = x.AbsoluteOffset - x.StartOfLineAbsoluteOffset
218217

219-
new(fileIndex: int, line: int, originalLine: int, startOfLineAbsoluteOffset: int, absoluteOffset: int) =
218+
new(fileIndex: int, line: int, startOfLineAbsoluteOffset: int, absoluteOffset: int) =
220219
{
221220
FileIndex = fileIndex
222221
Line = line
223-
OriginalLine = originalLine
224222
AbsoluteOffset = absoluteOffset
225223
StartOfLineAbsoluteOffset = startOfLineAbsoluteOffset
226224
}
227225

228226
member x.NextLine =
229-
Position(x.FileIndex, x.Line + 1, x.OriginalLine + 1, x.AbsoluteOffset, x.AbsoluteOffset)
227+
Position(x.FileIndex, x.Line + 1, x.AbsoluteOffset, x.AbsoluteOffset)
230228

231229
member x.EndOfToken n =
232-
Position(x.FileIndex, x.Line, x.OriginalLine, x.StartOfLineAbsoluteOffset, x.AbsoluteOffset + n)
230+
Position(x.FileIndex, x.Line, x.StartOfLineAbsoluteOffset, x.AbsoluteOffset + n)
233231

234232
member x.ShiftColumnBy by =
235-
Position(x.FileIndex, x.Line, x.OriginalLine, x.StartOfLineAbsoluteOffset, x.AbsoluteOffset + by)
233+
Position(x.FileIndex, x.Line, x.StartOfLineAbsoluteOffset, x.AbsoluteOffset + by)
236234

237235
member x.ColumnMinusOne =
238-
Position(x.FileIndex, x.Line, x.OriginalLine, x.StartOfLineAbsoluteOffset, x.StartOfLineAbsoluteOffset - 1)
239-
240-
member x.ApplyLineDirective(fileIdx, line) =
241-
Position(fileIdx, line, x.OriginalLine + 1, x.AbsoluteOffset, x.AbsoluteOffset)
236+
Position(x.FileIndex, x.Line, x.StartOfLineAbsoluteOffset, x.StartOfLineAbsoluteOffset - 1)
242237

243238
override p.ToString() = $"({p.Line},{p.Column})"
244239

245240
static member Empty = Position()
246241

247-
static member FirstLine fileIdx = Position(fileIdx, 1, 1, 0, 0)
242+
static member FirstLine fileIdx = Position(fileIdx, 1, 0, 0)
248243

249244
type internal LexBufferFiller<'Char> = LexBuffer<'Char> -> unit
250245

‎src/Compiler/Facilities/prim-lexing.fsi

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,6 @@ type internal Position =
7676
/// for the new line by modifying the EndPos property of the LexBuffer.
7777
val Line: int
7878

79-
/// The line number for the position in the input stream, assuming fresh positions have been updated
80-
/// using for the new line.
81-
val OriginalLine: int
82-
8379
/// The character number in the input stream.
8480
val AbsoluteOffset: int
8581

@@ -102,9 +98,6 @@ type internal Position =
10298
/// Same line, column -1.
10399
member ColumnMinusOne: Position
104100

105-
/// Apply a #line directive.
106-
member ApplyLineDirective: fileIdx: int * line: int -> Position
107-
108101
/// Get an arbitrary position, with the empty string as file name.
109102
static member Empty: Position
110103

‎src/Compiler/Symbols/FSharpDiagnostic.fs

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ type FSharpDiagnostic(m: range, severity: FSharpDiagnosticSeverity, message: str
235235
| _ -> diagnostic.FormatCore(flatErrors, suggestNames)
236236

237237
let errorNum = diagnostic.Number
238-
let m = match diagnostic.Range with Some m -> m | None -> range0
238+
let m = match diagnostic.Range with Some m -> m.ApplyLineDirectives() | None -> range0
239239
FSharpDiagnostic(m, severity, msg, diagnostic.Subcategory(), errorNum, "FS", extendedData)
240240

241241
static member NewlineifyErrorString(message) = NewlineifyErrorString(message)
@@ -335,10 +335,13 @@ module DiagnosticHelpers =
335335
| FSharpDiagnosticSeverity.Hidden -> []
336336
| adjustedSeverity ->
337337

338-
let diagnostic = FSharpDiagnostic.CreateFromException (diagnostic, adjustedSeverity, suggestNames, flatErrors, symbolEnv)
339-
let fileName = diagnostic.Range.FileName
338+
let fileName =
339+
match diagnostic.Range with
340+
| Some r -> r.FileName
341+
| None -> TcGlobals.DummyFileNameForRangesWithoutASpecificLocation
342+
let fDiagnostic = FSharpDiagnostic.CreateFromException (diagnostic, adjustedSeverity, suggestNames, flatErrors, symbolEnv)
340343
if allErrors || fileName = mainInputFileName || fileName = TcGlobals.DummyFileNameForRangesWithoutASpecificLocation then
341-
[diagnostic]
344+
[fDiagnostic]
342345
else []
343346

344347
let CreateDiagnostics (options, allErrors, mainInputFileName, diagnostics, suggestNames, flatErrors, symbolEnv) =

‎src/Compiler/SyntaxTree/LexFilter.fs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ open FSharp.Compiler.UnicodeLexing
1919

2020
let forceDebug = false
2121

22-
let stringOfPos (pos: Position) = sprintf "(%d:%d)" pos.OriginalLine pos.Column
22+
let stringOfPos (pos: Position) = sprintf "(%d:%d)" pos.Line pos.Column
2323

24-
let outputPos os (pos: Position) = Printf.fprintf os "(%d:%d)" pos.OriginalLine pos.Column
24+
let outputPos os (pos: Position) = Printf.fprintf os "(%d:%d)" pos.Line pos.Column
2525

2626
/// Used for warning strings, which should display columns as 1-based and display
2727
/// the lines after taking '# line' directives into account (i.e. do not use
@@ -996,7 +996,7 @@ type LexFilterImpl (
996996
let warnF = if strictIndentation then error else warn
997997
warnF tokenTup
998998
(if debug then
999-
sprintf "possible incorrect indentation: this token is offside of context at position %s, newCtxt = %A, stack = %A, newCtxtPos = %s, c1 = %d, c2 = %d"
999+
sprintf "possible incorrect indentation: this token is offside of context at (original!) position %s, newCtxt = %A, stack = %A, newCtxtPos = %s, c1 = %d, c2 = %d"
10001000
(warningStringOfPosition p1.Position) newCtxt offsideStack (stringOfPos newCtxt.StartPos) p1.Column c2
10011001
else
10021002
FSComp.SR.lexfltTokenIsOffsideOfContextStartedEarlier(warningStringOfPosition p1.Position))
@@ -1305,7 +1305,7 @@ type LexFilterImpl (
13051305
let isSameLine() =
13061306
match token with
13071307
| EOF _ -> false
1308-
| _ -> (startPosOfTokenTup (peekNextTokenTup())).OriginalLine = tokenStartPos.OriginalLine
1308+
| _ -> (startPosOfTokenTup (peekNextTokenTup())).Line = tokenStartPos.Line
13091309

13101310
let isControlFlowOrNotSameLine() =
13111311
match token with
@@ -1817,7 +1817,7 @@ type LexFilterImpl (
18171817
elif isTypeCtxt then isTypeSeqBlockElementContinuator token
18181818
else isSeqBlockElementContinuator token)
18191819
&& (tokenStartCol = offsidePos.Column)
1820-
&& (tokenStartPos.OriginalLine <> offsidePos.OriginalLine) ->
1820+
&& (tokenStartPos.Line <> offsidePos.Line) ->
18211821
if debug then dprintf "offside at column %d matches start of block(%a)! delaying token, returning OBLOCKSEP\n" tokenStartCol outputPos offsidePos
18221822
replaceCtxt tokenTup (CtxtSeqBlock (FirstInSeqBlock, offsidePos, addBlockEnd))
18231823
// No change to offside stack: another statement block starts...
@@ -2320,7 +2320,7 @@ type LexFilterImpl (
23202320
// For attributes on properties:
23212321
// member x.PublicGetSetProperty
23222322
// with [<Foo>] get() = "Ralf"
2323-
if (match lookaheadTokenTup.Token with LBRACK_LESS -> true | _ -> false) && (lookaheadTokenStartPos.OriginalLine = tokenTup.StartPos.OriginalLine) then
2323+
if (match lookaheadTokenTup.Token with LBRACK_LESS -> true | _ -> false) && (lookaheadTokenStartPos.Line = tokenTup.StartPos.Line) then
23242324
let offsidePos = tokenStartPos
23252325
pushCtxt tokenTup (CtxtWithAsLet offsidePos)
23262326
returnToken tokenLexbufState OWITH

0 commit comments

Comments
 (0)