Merge pull request #405 from dsyme/vf-align-5

Alignment and make internal lexer/parser properly internal

Author: dsyme

docs/content/editor.fsx

@@ -125,7 +125,7 @@ identifier (the other option lets you get tooltip with full assembly location wh
 *)
 // Get tag of the IDENT token to be used as the last argument
 open Microsoft.FSharp.Compiler
-let identToken = Parser.tagOfToken(Parser.token.IDENT("")) 
+let identToken = FSharpTokenTag.Identifier
 
 // Get tool tip at the specified location
 let tip = checkFileResults.GetToolTipTextAlternate(4, 7, inputLines.[1], ["foo"], identToken)

samples/EditorService/Program.fs

@@ -28,10 +28,10 @@ let input =
 let inputLines = input.Split('\n')
 let file = "/home/user/Test.fsx"
 
-let identToken = Parser.tagOfToken (Parser.token.IDENT(""))
+let identTokenTag = FSharpTokenTag.Identifier
 let untyped, parsed = parseWithTypeInfo (file, input)
 // Get tool tip at the specified location
-let tip = parsed.GetToolTipTextAlternate(2, 7, inputLines.[1], [ "foo" ], identToken)
+let tip = parsed.GetToolTipTextAlternate(2, 7, inputLines.[1], [ "foo" ], identTokenTag)
 
 printfn "%A" tip
 

src/absil/ilreflect.fs

@@ -46,7 +46,7 @@ let logRefEmitCalls = false
 
 type System.AppDomain with 
     member x.DefineDynamicAssemblyAndLog(asmName,flags,asmDir:string)  =
-        let asmB = x.DefineDynamicAssembly(asmName,flags,asmDir) 
+        let asmB = x.DefineDynamicAssembly(asmName,flags,asmDir)
         if logRefEmitCalls then 
             printfn "open System"
             printfn "open System.Reflection"
@@ -56,7 +56,7 @@ type System.AppDomain with
 
 
 type System.Reflection.Emit.AssemblyBuilder with 
-    member asmB.DefineDynamicModuleAndLog(a,b,c) =
+    member asmB.DefineDynamicModuleAndLog(a,b,c) =  
         let modB = asmB.DefineDynamicModule(a,b,c)
         if logRefEmitCalls then printfn "let moduleBuilder%d = assemblyBuilder%d.DefineDynamicModule(%A,%A,%A)" (abs <| hash modB) (abs <| hash asmB) a b c
         modB
@@ -199,7 +199,7 @@ type System.Reflection.Emit.TypeBuilder with
 #if FX_NO_INVOKE_MEMBER
 #else
     member typB.InvokeMemberAndLog(nm,flags,args)        = 
-        if logRefEmitCalls then printfn "typeBuilder%d.InvokeMember(\"%s\",enum %d,null,null,%A,Globalization.CultureInfo.InvariantCulture)" (abs <| hash typB) nm (LanguagePrimitives.EnumToValue flags) args
+        if logRefEmitCalls then printfn "typeBuilder%d.InvokeMember(\"%s\",enum %d,null,null,%A,Globalization.CultureInfo.InvariantCulture)" (abs <| hash typB) nm (LanguagePrimitives.EnumToValue flags) args     
         typB.InvokeMember(nm,flags,null,null,args,Globalization.CultureInfo.InvariantCulture)
 #endif
 
@@ -339,9 +339,9 @@ let convTypeRefAux (cenv:cenv) (tref:ILTypeRef) =
     match tref.Scope with
     | ILScopeRef.Assembly asmref ->
         let assembly = 
-            match cenv.resolvePath asmref with
+            match cenv.resolvePath asmref with                     
             | Some (Choice1Of2 path) ->
-                FileSystem.AssemblyLoadFrom(path)
+                FileSystem.AssemblyLoadFrom(path)              
             | Some (Choice2Of2 assembly) ->
                 assembly
             | None ->
@@ -1068,7 +1068,7 @@ let rec emitInstr cenv (modB : ModuleBuilder) emEnv (ilG:ILGenerator) instr =
     | I_callvirt       (tail,mspec,varargs)   -> emitSilverlightCheck ilG
                                                  emitInstrCall cenv emEnv ilG OpCodes.Callvirt tail mspec varargs
     | I_callconstraint (tail,typ,mspec,varargs) -> ilG.Emit(OpCodes.Constrained,convType cenv emEnv typ); 
-                                                   emitInstrCall cenv emEnv ilG OpCodes.Callvirt tail mspec varargs   
+                                                   emitInstrCall cenv emEnv ilG OpCodes.Callvirt tail mspec varargs                                                     
 #if FX_NO_REFLECTION_EMIT_CALLI                                                   
     | I_calli (tail,_callsig,None)             -> emitInstrTail ilG tail (fun () -> ())
     | I_calli (tail,_callsig,Some _vartyps)     -> emitInstrTail ilG tail (fun () -> ())
@@ -1085,7 +1085,7 @@ let rec emitInstr cenv (modB : ModuleBuilder) emEnv (ilG:ILGenerator) instr =
                                                                  convCallConv callsig.CallingConv,
                                                                  convType cenv emEnv callsig.ReturnType,
                                                                  convTypesToArray cenv emEnv callsig.ArgTypes,
-                                                                 convTypesToArray cenv emEnv vartyps))
+                                                                 convTypesToArray cenv emEnv vartyps))                                                                
 #endif                                                                 
     | I_ldftn mspec                           -> ilG.EmitAndLog(OpCodes.Ldftn,convMethodSpec cenv emEnv mspec)
     | I_newobj (mspec,varargs)                -> emitInstrNewobj cenv emEnv ilG mspec varargs
@@ -1954,7 +1954,7 @@ let createTypeRef (visited : Dictionary<_,_>, created : Dictionary<_,_>) emEnv t
         if not (visited.ContainsKey(tref)) || visited.[tref] > priority then 
             visited.[tref] <- priority;
             let tdef = envGetTypeDef emEnv tref
-            if verbose2 then dprintf "- traversing type %s\n" typB.FullName;
+            if verbose2 then dprintf "- traversing type %s\n" typB.FullName;        
 #if FX_NO_TYPE_RESOLVE_EVENT
             traverseTypeDef priority tref tdef;
 #else            
@@ -1975,7 +1975,7 @@ let createTypeRef (visited : Dictionary<_,_>, created : Dictionary<_,_>) emEnv t
             try
                 traverseTypeDef priority tref tdef;
             finally
-               System.AppDomain.CurrentDomain.remove_TypeResolve typeCreationHandler
+               System.AppDomain.CurrentDomain.remove_TypeResolve typeCreationHandler           
 #endif               
             if not (created.ContainsKey(tref)) then 
                 created.[tref] <- true;   
@@ -2049,7 +2049,7 @@ let buildModuleFragment cenv emEnv (asmB : AssemblyBuilder) (modB : ModuleBuilde
 
 let mkDynamicAssemblyAndModule (assemblyName, optimize, debugInfo, collectible) =
     let filename = assemblyName ^ ".dll"
-    let currentDom  = System.AppDomain.CurrentDomain
+    let currentDom  = System.AppDomain.CurrentDomain   
     let asmName = new AssemblyName()
     asmName.Name <- assemblyName
 #if FX_NO_REFLECTION_EMIT_SAVE_ASSEMBLY
@@ -2085,7 +2085,7 @@ let emitModuleFragment (ilg, emEnv, asmB : AssemblyBuilder, modB : ModuleBuilder
        emitCustomAttrs cenv emEnv asmB.SetCustomAttributeAndLog mani.CustomAttrs;    
     // invoke entry point methods
     let execEntryPtFun ((typB : TypeBuilder),methodName) () =
-      try 
+      try        
 #if FX_NO_INVOKE_MEMBER
         let mi = typB.GetMethod(methodName, BindingFlags.InvokeMethod ||| BindingFlags.Public ||| BindingFlags.Static)
         System.Diagnostics.Debug.WriteLine("mi: {0}", string(mi.ToString()))
@@ -2096,7 +2096,7 @@ let emitModuleFragment (ilg, emEnv, asmB : AssemblyBuilder, modB : ModuleBuilder
         let invokedm = dm.CreateDelegate(typeof<EntryDelegate>)
         invokedm.DynamicInvoke(null) |> ignore
 #else          
-        ignore (typB.InvokeMemberAndLog(methodName,BindingFlags.InvokeMethod ||| BindingFlags.Public ||| BindingFlags.Static,[| |]));
+        ignore (typB.InvokeMemberAndLog(methodName,BindingFlags.InvokeMethod ||| BindingFlags.Public ||| BindingFlags.Static,[| |]));       
 #endif        
         None
       with 

src/absil/ilsupp.fs

@@ -1195,8 +1195,8 @@ let pdbReadOpen (moduleName:string) (path:string) :  PdbReader =
         with _ ->  
             { symReader = null } 
 #else 
-        let symbolBinder = new System.Diagnostics.SymbolStore.SymBinder() 
-        { symReader = symbolBinder.GetReader(importerPtr, moduleName, path) } 
+        let symbolBinder = new System.Diagnostics.SymbolStore.SymBinder()
+        { symReader = symbolBinder.GetReader(importerPtr, moduleName, path) }
 #endif
     finally
         // Marshal.GetComInterfaceForObject adds an extra ref for importerPtr

src/fsharp/CompileOps.fs

@@ -4008,19 +4008,22 @@ type TcImports(tcConfigP:TcConfigProvider, initialResolutions:TcAssemblyResoluti
                 let name = System.Reflection.AssemblyName.GetAssemblyName(resolution.resolvedPath)
                 name.Version
 
-            // The callback captured by the TypeProviderConfig. Disconnect when things are disposed
-            let systemRuntimeContainsType =
-                let systemRuntimeContainsTypeRef = ref tcImports.SystemRuntimeContainsType
-                tcImports.AttachDisposeAction(fun () -> systemRuntimeContainsTypeRef := (fun _ -> raise (System.ObjectDisposedException("The type provider has been disposed"))))
-                fun arg -> systemRuntimeContainsTypeRef.Value arg
-                
             let typeProviderEnvironment = 
                  { resolutionFolder       = tcConfig.implicitIncludeDir
                    outputFile             = tcConfig.outputFile
                    showResolutionMessages = tcConfig.showExtensionTypeMessages 
                    referencedAssemblies   = [| for r in resolutions.GetAssemblyResolutions() -> r.resolvedPath |]
                    temporaryFolder        = FileSystem.GetTempPathShim() }
 
+            // The type provider should not hold strong references to disposed
+            // TcImport objects.  So the callbacks provided in the type provider config
+            // dispatch via a thunk which gets set to a non-resource-capturing 
+            // failing function when the object is disposed. 
+            let systemRuntimeContainsType =  
+                let systemRuntimeContainsTypeRef = ref tcImports.SystemRuntimeContainsType  
+                tcImports.AttachDisposeAction(fun () -> systemRuntimeContainsTypeRef := (fun _ -> raise (System.ObjectDisposedException("The type provider has been disposed"))))  
+                fun arg -> systemRuntimeContainsTypeRef.Value arg  
+
             let providers = 
                 [ for assemblyName in providerAssemblies do
                       yield ExtensionTyping.GetTypeProvidersOfAssembly(displayPSTypeProviderSecurityDialogBlockingUI, tcConfig.validateTypeProviders, 
@@ -5280,3 +5283,5 @@ let TypeCheckClosedInputSet (checkForErrors, tcConfig, tcImports, tcGlobals, pre
     tcState, topAttrs, tassembly, tcEnvAtEndOfLastFile
 
 
+
+

src/fsharp/CompileOps.fsi

@@ -764,4 +764,4 @@ type LoadClosure =
     static member ComputeClosureOfSourceText : filename: string * source: string * implicitDefines:CodeContext * useMonoResolution: bool * useFsiAuxLib: bool * lexResourceManager: Lexhelp.LexResourceManager * applyCompilerOptions: (TcConfigBuilder -> unit) -> LoadClosure
 
     /// Used from fsi.fs and fsc.fs, for #load and command line. The resulting references are then added to a TcConfig.
-    static member ComputeClosureOfSourceFiles : tcConfig:TcConfig * (string * range) list * implicitDefines:CodeContext * useDefaultScriptingReferences: bool * lexResourceManager: Lexhelp.LexResourceManager -> LoadClosure
+    static member ComputeClosureOfSourceFiles : tcConfig:TcConfig * (string * range) list * implicitDefines:CodeContext * useDefaultScriptingReferences : bool * lexResourceManager : Lexhelp.LexResourceManager -> LoadClosure

src/fsharp/ErrorLogger.fs

@@ -108,13 +108,13 @@ type Exiter =
 
 let QuitProcessExiter = 
     { new Exiter with 
-        member x.Exit(n) = 
+        member x.Exit(n) =                    
 #if FX_NO_SYSTEM_ENVIRONMENT_EXIT
 #else                    
             try 
               System.Environment.Exit(n)
             with _ -> 
-              ()
+              ()            
 #endif              
             failwithf "%s" <| FSComp.SR.elSysEnvExitDidntExit() }
 

src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj

@@ -98,7 +98,7 @@
     <FsYacc Include="..\pars.fsy">
       <Module>Microsoft.FSharp.Compiler.Parser</Module>
       <Open>Microsoft.FSharp.Compiler</Open>
-      <OtherFlags>--lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing</OtherFlags>
+      <OtherFlags>--internal --lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing</OtherFlags>
       <Link>pars.fsy</Link>
     </FsYacc>
     <Compile Include="..\..\utils\sformat.fsi">
@@ -219,12 +219,6 @@
       <OtherFlags>--lexlib Internal.Utilities.Text.Lexing</OtherFlags>
       <Link>AbsIL/illex.fsl</Link>
     </FsLex>
-    <FsYacc Include="..\..\absil\ilpars.fsy">
-      <Module>Microsoft.FSharp.Compiler.AbstractIL.Internal.AsciiParser</Module>
-      <Open>Microsoft.FSharp.Compiler.AbstractIL</Open>
-      <OtherFlags>--internal --lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing</OtherFlags>
-      <Link>AbsIL/ilpars.fsy</Link>
-    </FsYacc>
     <Compile Include="..\..\absil\il.fsi">
       <Link>AbsIL/il.fsi</Link>
     </Compile>
@@ -317,16 +311,10 @@
       <OtherFlags>--lexlib Internal.Utilities.Text.Lexing</OtherFlags>
       <Link>ParserAndUntypedAST/pplex.fsl</Link>
     </FsLex>
-    <FsYacc Include="..\pars.fsy">
-      <Module>Microsoft.FSharp.Compiler.Parser</Module>
-      <Open>Microsoft.FSharp.Compiler</Open>
-      <OtherFlags>--lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing</OtherFlags>
-      <Link>ParserAndUntypedAST/pars.fsy</Link>
-    </FsYacc>
     <FsYacc Include="..\pppars.fsy">
       <Module>Microsoft.FSharp.Compiler.PPParser</Module>
       <Open>Microsoft.FSharp.Compiler</Open>
-      <OtherFlags>--lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing</OtherFlags>
+      <OtherFlags>--internal --lexlib Internal.Utilities.Text.Lexing --parslib Internal.Utilities.Text.Parsing</OtherFlags>
       <Link>ParserAndUntypedAST/pppars.fsy</Link>
     </FsYacc>
     <Compile Include="..\UnicodeLexing.fsi">

src/fsharp/LexFilter.fs

@@ -2,7 +2,7 @@
 
 /// LexFilter - process the token stream prior to parsing.
 /// Implements the offside rule and a copule of other lexical transformations.
-module (*internal*) Microsoft.FSharp.Compiler.LexFilter
+module internal Microsoft.FSharp.Compiler.LexFilter
 
 open Internal.Utilities
 open Internal.Utilities.Text.Lexing

src/fsharp/UnicodeLexing.fs

@@ -1,6 +1,6 @@
 // Copyright (c) Microsoft Open Technologies, Inc.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
-module (*internal*) Microsoft.FSharp.Compiler.UnicodeLexing
+module internal Microsoft.FSharp.Compiler.UnicodeLexing
 
 //------------------------------------------------------------------
 // Functions for Unicode char-based lexing (new code).

src/fsharp/UnicodeLexing.fsi

@@ -1,6 +1,6 @@
 // Copyright (c) Microsoft Open Technologies, Inc.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
 
-module (*internal*) Microsoft.FSharp.Compiler.UnicodeLexing
+module internal Microsoft.FSharp.Compiler.UnicodeLexing
 
 open Microsoft.FSharp.Text
 open Internal.Utilities.Text.Lexing

src/fsharp/ast.fs

@@ -94,7 +94,7 @@ type XmlDoc =
                 let lineAT = lineA.TrimStart([|' '|])
                 if lineAT = "" then processLines rest
                 else if String.hasPrefix lineAT "<" then lines
-                else ["<summary>"] @
+                else ["<summary>"] @     
 #if FX_NO_SECURITY_ELEMENT_ESCAPE
                      lines @
 #else        
@@ -2084,15 +2084,15 @@ and LexCont = LexerWhitespaceContinuation
 exception SyntaxError of obj (* ParseErrorContext<_> *) * range
 
 /// Get an F# compiler position from a lexer position
-let posOfLexPosition (p:Position) = 
+let internal posOfLexPosition (p:Position) = 
     mkPos p.Line p.Column
 
 /// Get an F# compiler range from a lexer range
-let mkSynRange (p1:Position) (p2: Position) = 
+let internal mkSynRange (p1:Position) (p2: Position) = 
     mkFileIndexRange p1.FileIndex (posOfLexPosition p1) (posOfLexPosition p2)
 
 type LexBuffer<'Char> with 
-    member lexbuf.LexemeRange  = mkSynRange lexbuf.StartPos lexbuf.EndPos
+    member internal lexbuf.LexemeRange  = mkSynRange lexbuf.StartPos lexbuf.EndPos
 
 /// Get the range corresponding to the result of a grammar rule while it is being reduced
 let internal lhs (parseState: IParseState) = 
@@ -2129,19 +2129,19 @@ module LexbufLocalXmlDocStore =
     // The key into the BufferLocalStore used to hold the current accumulated XmlDoc lines 
     let private xmlDocKey = "XmlDoc"
 
-    let ClearXmlDoc (lexbuf:Lexbuf) = 
+    let internal ClearXmlDoc (lexbuf:Lexbuf) = 
         lexbuf.BufferLocalStore.[xmlDocKey] <- box (XmlDocCollector())
 
     /// Called from the lexer to save a single line of XML doc comment.
-    let SaveXmlDocLine (lexbuf:Lexbuf, lineText, pos) = 
+    let internal SaveXmlDocLine (lexbuf:Lexbuf, lineText, pos) = 
         if not (lexbuf.BufferLocalStore.ContainsKey(xmlDocKey)) then 
             lexbuf.BufferLocalStore.[xmlDocKey] <- box (XmlDocCollector())
         let collector = unbox<XmlDocCollector>(lexbuf.BufferLocalStore.[xmlDocKey])
         collector.AddXmlDocLine (lineText, pos)
 
     /// Called from the parser each time we parse a construct that marks the end of an XML doc comment range,
     /// e.g. a 'type' declaration. The markerRange is the range of the keyword that delimits the construct.
-    let GrabXmlDocBeforeMarker (lexbuf:Lexbuf, markerRange:range)  = 
+    let internal GrabXmlDocBeforeMarker (lexbuf:Lexbuf, markerRange:range)  = 
         if lexbuf.BufferLocalStore.ContainsKey(xmlDocKey) then 
             PreXmlDoc.CreateFromGrabPoint(unbox<XmlDocCollector>(lexbuf.BufferLocalStore.[xmlDocKey]),markerRange.End)
         else