Move the WIP LSP extension to the common vsix

VSFSharpExtension.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.0.31903.59
+# Visual Studio Version 18
+VisualStudioVersion = 18.0.10806.39 main
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{B4A1E626-4A48-4977-B291-219882DB3413}"
 EndProject
@@ -17,6 +17,8 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.DependencyManager.Nu
 EndProject
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Editor", "vsintegration\src\FSharp.Editor\FSharp.Editor.fsproj", "{3D4A95CB-1563-CD9A-3949-FE01922CD5BD}"
 EndProject
+Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Compiler.LanguageServer.Tests", "tests\FSharp.Compiler.LanguageServer.Tests\FSharp.Compiler.LanguageServer.Tests.fsproj", "{ECB3A7E8-824B-6769-6A01-754555E89E27}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -61,6 +63,12 @@ Global
 		{3D4A95CB-1563-CD9A-3949-FE01922CD5BD}.Proto|Any CPU.Build.0 = Debug|Any CPU
 		{3D4A95CB-1563-CD9A-3949-FE01922CD5BD}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{3D4A95CB-1563-CD9A-3949-FE01922CD5BD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{ECB3A7E8-824B-6769-6A01-754555E89E27}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{ECB3A7E8-824B-6769-6A01-754555E89E27}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{ECB3A7E8-824B-6769-6A01-754555E89E27}.Proto|Any CPU.ActiveCfg = Debug|Any CPU
+		{ECB3A7E8-824B-6769-6A01-754555E89E27}.Proto|Any CPU.Build.0 = Debug|Any CPU
+		{ECB3A7E8-824B-6769-6A01-754555E89E27}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{ECB3A7E8-824B-6769-6A01-754555E89E27}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/FSharp.Compiler.LanguageServer/Common/CapabilitiesManager.fs

@@ -31,21 +31,9 @@ type CapabilitiesManager(config: FSharpLanguageServerConfig, scOverrides: IServe
                         InterFileDependencies = true,
                         Identifier = "potato",
                         WorkspaceDiagnostics = true
-                    )),
+                    ))
             //CompletionProvider = CompletionOptions(TriggerCharacters = [| "."; " " |], ResolveProvider = true, WorkDoneProgress = true),
             //HoverProvider = SumType<bool, HoverOptions>(HoverOptions(WorkDoneProgress = true))
-            SemanticTokensOptions =
-                addIf
-                    config.EnabledFeatures.SemanticHighlighting
-
-                    (SemanticTokensOptions(
-                        Legend =
-                            SemanticTokensLegend(
-                                TokenTypes = (SemanticTokenTypes.AllTypes |> Seq.toArray), // XXX should be extended
-                                TokenModifiers = (SemanticTokenModifiers.AllModifiers |> Seq.toArray)
-                            ),
-                        Range = false
-                    ))
         )
 
     interface IInitializeManager<InitializeParams, InitializeResult> with

src/FSharp.Compiler.LanguageServer/Common/FSharpRequestContext.fs

@@ -17,150 +17,11 @@ open FSharp.Compiler.CodeAnalysis.Workspace
 
 #nowarn "57"
 
-module TokenTypes =
-
-    [<return: Struct>]
-    let (|LexicalClassification|_|) (tok: FSharpToken) =
-        if tok.IsKeyword then
-            ValueSome SemanticTokenTypes.Keyword
-        elif tok.IsNumericLiteral then
-            ValueSome SemanticTokenTypes.Number
-        elif tok.IsCommentTrivia then
-            ValueSome SemanticTokenTypes.Comment
-        elif tok.IsStringLiteral then
-            ValueSome SemanticTokenTypes.String
-        else
-            ValueNone
-
-    // Tokenizes the source code and returns a list of token ranges and their SemanticTokenTypes
-    let GetSyntacticTokenTypes (source: FSharp.Compiler.Text.ISourceText) (fileName: string) =
-        let mutable tokRangesAndTypes = []
-
-        let tokenCallback =
-            fun (tok: FSharpToken) ->
-                match tok with
-                | LexicalClassification tokType -> tokRangesAndTypes <- (tok.Range, tokType) :: tokRangesAndTypes
-                | _ -> ()
-
-        FSharpLexer.Tokenize(
-            source,
-            tokenCallback,
-            flags =
-                (FSharpLexerFlags.Default
-                 &&& ~~~FSharpLexerFlags.Compiling
-                 &&& ~~~FSharpLexerFlags.UseLexFilter),
-            filePath = fileName
-        )
-
-        tokRangesAndTypes
-
-    let FSharpTokenTypeToLSP (fst: SemanticClassificationType) =
-        // XXX kinda arbitrary mapping
-        match fst with
-        | SemanticClassificationType.ReferenceType -> SemanticTokenTypes.Class
-        | SemanticClassificationType.ValueType -> SemanticTokenTypes.Struct
-        | SemanticClassificationType.UnionCase -> SemanticTokenTypes.Enum
-        | SemanticClassificationType.UnionCaseField -> SemanticTokenTypes.EnumMember
-        | SemanticClassificationType.Function -> SemanticTokenTypes.Function
-        | SemanticClassificationType.Property -> SemanticTokenTypes.Property
-        | SemanticClassificationType.Module -> SemanticTokenTypes.Type
-        | SemanticClassificationType.Namespace -> SemanticTokenTypes.Namespace
-        | SemanticClassificationType.Interface -> SemanticTokenTypes.Interface
-        | SemanticClassificationType.TypeArgument -> SemanticTokenTypes.TypeParameter
-        | SemanticClassificationType.Operator -> SemanticTokenTypes.Operator
-        | SemanticClassificationType.Method -> SemanticTokenTypes.Method
-        | SemanticClassificationType.ExtensionMethod -> SemanticTokenTypes.Method
-        | SemanticClassificationType.Field -> SemanticTokenTypes.Property
-        | SemanticClassificationType.Event -> SemanticTokenTypes.Event
-        | SemanticClassificationType.Delegate -> SemanticTokenTypes.Function
-        | SemanticClassificationType.NamedArgument -> SemanticTokenTypes.Parameter
-        | SemanticClassificationType.LocalValue -> SemanticTokenTypes.Variable
-        | SemanticClassificationType.Plaintext -> SemanticTokenTypes.String
-        | SemanticClassificationType.Type -> SemanticTokenTypes.Type
-        | SemanticClassificationType.Printf -> SemanticTokenTypes.Keyword
-        | _ -> SemanticTokenTypes.Comment
-
-    let toIndex (x: string) =
-        SemanticTokenTypes.AllTypes |> Seq.findIndex (fun y -> y = x)
-
 type FSharpRequestContext(lspServices: ILspServices, logger: ILspLogger, workspace: FSharpWorkspace) =
     member _.LspServices = lspServices
     member _.Logger = logger
     member _.Workspace = workspace
 
-    member this.GetSemanticTokensForFile(file) =
-        task {
-            let! scv = this.Workspace.Query.GetSemanticClassification file
-            let! source = this.Workspace.Query.GetSource file
-
-            match scv, source with
-            | Some view, Some source ->
-
-                let tokens = ResizeArray()
-
-                view.ForEach(fun item ->
-                    let range = item.Range
-                    let tokenType = item.Type |> TokenTypes.FSharpTokenTypeToLSP |> TokenTypes.toIndex
-                    tokens.Add(range, tokenType))
-
-                let syntacticClassifications =
-                    TokenTypes.GetSyntacticTokenTypes source file.LocalPath
-                    |> Seq.map (fun (r, t) -> (r, TokenTypes.toIndex t))
-
-                let allTokens = Seq.append tokens syntacticClassifications
-
-                let lspFormatTokens =
-                    allTokens
-                    |> Seq.map (fun (r, tokType) ->
-                        let length = r.EndColumn - r.StartColumn // XXX Does not deal with multiline tokens?
-
-                        {|
-                            startLine = r.StartLine - 1
-                            startCol = r.StartColumn
-                            length = length
-                            tokType = tokType
-                            tokMods = 0
-                        |})
-                    //(startLine, startCol, length, tokType, tokMods))
-                    |> Seq.sortWith (fun x1 x2 ->
-                        let c = x1.startLine.CompareTo(x2.startLine)
-                        if c <> 0 then c else x1.startCol.CompareTo(x2.startCol))
-
-                let tokensRelative =
-                    lspFormatTokens
-                    |> Seq.append
-                        [|
-                            {|
-                                startLine = 0
-                                startCol = 0
-                                length = 0
-                                tokType = 0
-                                tokMods = 0
-                            |}
-                        |]
-                    |> Seq.pairwise
-                    |> Seq.map (fun (prev, this) ->
-                        {|
-                            startLine = this.startLine - prev.startLine
-                            startCol =
-                                (if prev.startLine = this.startLine then
-                                     this.startCol - prev.startCol
-                                 else
-                                     this.startCol)
-                            length = this.length
-                            tokType = this.tokType
-                            tokMods = this.tokMods
-                        |})
-
-                return
-                    tokensRelative
-                    |> Seq.map (fun tok -> [| tok.startLine; tok.startCol; tok.length; tok.tokType; tok.tokMods |])
-                    |> Seq.concat
-                    |> Seq.toArray
-
-            | _ -> return [||]
-        }
-
 type ContextHolder(workspace, lspServices: ILspServices) =
 
     let logger = lspServices.GetRequiredService<ILspLogger>()

src/FSharp.Compiler.LanguageServer/FSharpLanguageServer.fs

@@ -1,18 +1,19 @@
 namespace FSharp.Compiler.LanguageServer
 
+open System
+open System.Diagnostics
 open System.Runtime.CompilerServices
+
 open FSharp.Compiler.LanguageServer.Common
 open FSharp.Compiler.LanguageServer.Handlers
 
-open System
 open Microsoft.CommonLanguageServerProtocol.Framework.Handlers
 open Microsoft.CommonLanguageServerProtocol.Framework
 open Microsoft.Extensions.DependencyInjection
 open Microsoft.VisualStudio.LanguageServer.Protocol
 
 open StreamJsonRpc
 open Nerdbank.Streams
-open System.Diagnostics
 open FSharp.Compiler.CodeAnalysis.Workspace
 
 #nowarn "57"

src/FSharp.Compiler.LanguageServer/FSharpLanguageServerConfig.fs

@@ -3,13 +3,11 @@ namespace FSharp.Compiler.LanguageServer
 type FSharpLanguageServerFeatures =
     {
         Diagnostics: bool
-        SemanticHighlighting: bool
     }
 
     static member Default =
         {
             Diagnostics = true
-            SemanticHighlighting = true
         }
 
 type FSharpLanguageServerConfig =

src/FSharp.Compiler.LanguageServer/Handlers/LanguageFeaturesHandler.fs

@@ -48,12 +48,3 @@ type LanguageFeaturesHandler() =
                     )
             }
             |> CancellableTask.start cancellationToken
-
-    interface IRequestHandler<SemanticTokensParams, SemanticTokens, FSharpRequestContext> with
-        [<LanguageServerEndpoint(Methods.TextDocumentSemanticTokensFullName, LanguageServerConstants.DefaultLanguageName)>]
-        member _.HandleRequestAsync(request: SemanticTokensParams, context: FSharpRequestContext, cancellationToken: CancellationToken) =
-            cancellableTask {
-                let! tokens = context.GetSemanticTokensForFile(request.TextDocument.Uri)
-                return SemanticTokens(Data = tokens)
-            }
-            |> CancellableTask.start cancellationToken

src/FSharp.VisualStudio.Extension/ExtensionEntrypoint.cs

@@ -5,6 +5,7 @@
 using System.Threading;
 using System;
 using Extension = Microsoft.VisualStudio.Extensibility.Extension;
+using System.Diagnostics;
 
 /// <summary>
 /// Extension entrypoint for the VisualStudio.Extensibility extension.
@@ -21,7 +22,11 @@ internal class ExtensionEntrypoint : Extension
                 version: this.ExtensionAssemblyVersion,
                 publisherName: "Publisher name",
                 displayName: "FSharp.VisualStudio.Extension",
-                description: "Extension description"),
+                description: "Extension description")
+        {
+            // TODO: Probably should be replaced by a range of versions
+            InstallationTargetVersion = "[17.14, 18.0]",
+        },
     };
 
     /// <inheritdoc />

src/FSharp.VisualStudio.Extension/FSharp.VisualStudio.Extension.csproj

@@ -4,16 +4,18 @@
 		<Nullable>enable</Nullable>
 		<LangVersion>12</LangVersion>
 		<NeutralLanguage>en-US</NeutralLanguage>
+		<CreateVsixContainer>false</CreateVsixContainer>
+		<AssemblyVSIXSubPath>LSPComponent</AssemblyVSIXSubPath>
 		<!-- The VisualStudio.Extensibility preview packages are available from the azure-public/vside/msft_consumption feed -->
 
 	</PropertyGroup>
 
 	<ItemGroup>
-		<PackageReference Include="Microsoft.VisualStudio.Extensibility.Sdk" Version="17.13.39620" />
-		<PackageReference Include="Microsoft.VisualStudio.Extensibility.Build" Version="17.13.39620" />
+		<PackageReference Include="Microsoft.VisualStudio.Extensibility.Sdk" Version="17.14.40313" />
+		<PackageReference Include="Microsoft.VisualStudio.Extensibility.Build" Version="17.14.40313" />
 		<PackageReference Include="Microsoft.VisualStudio.LanguageServer.Protocol.Internal" Version="17.13.9" />
-		<PackageReference Include="Microsoft.VisualStudio.ProjectSystem.Query" Version="17.13.66" />
-		<PackageReference Include="Microsoft.VisualStudio.Threading" Version="17.13.2" />
+		<PackageReference Include="Microsoft.VisualStudio.ProjectSystem.Query" Version="17.14.143" />
+		<PackageReference Include="Microsoft.VisualStudio.Threading" Version="$(MicrosoftVisualStudioThreadingVersion)" />
 		<!--<PackageReference Include="Microsoft.VisualStudio.OpenTelemetry.ClientExtensions" Version="0.1.718-beta" />
 		<PackageReference Include="Microsoft.VisualStudio.OpenTelemetry.Collector" Version="0.1.718-beta" />
 		<PackageReference Include="OpenTelemetry" Version="1.10.0" />

src/FSharp.VisualStudio.Extension/FSharpLanguageServerProvider.cs

@@ -72,20 +72,6 @@ public ServerCapabilities OverrideServerCapabilities(FSharpLanguageServerConfig
                         //new(PullDiagnosticCategories.DocumentAnalyzerSemantic),
                     ]
             } : null,
-            SemanticTokensOptions = config.EnabledFeatures.SemanticHighlighting ? new()
-            {
-                Legend = new()
-                {
-                    TokenTypes = [.. SemanticTokenTypes.AllTypes], // XXX should be extended
-                    TokenModifiers = [.. SemanticTokenModifiers.AllModifiers]
-                },
-                Full = new SemanticTokensFullOptions()
-                {
-                    Delta = false
-                },
-                Range = false
-            } : null,
-            //,
             //HoverProvider = new HoverOptions()
             //{
             //    WorkDoneProgress = true
@@ -291,8 +277,7 @@ await this.Extensibility.Settings().WriteAsync(batch =>
 
         var serverConfig = new FSharpLanguageServerConfig(
             new FSharpLanguageServerFeatures(
-                diagnostics: enabled.Contains(settingsReadResult.ValueOrDefault(FSharpExtensionSettings.GetDiagnosticsFrom, defaultValue: FSharpExtensionSettings.BOTH)),
-                semanticHighlighting: enabled.Contains(settingsReadResult.ValueOrDefault(FSharpExtensionSettings.GetSemanticHighlightingFrom, defaultValue: FSharpExtensionSettings.BOTH))
+                diagnostics: enabled.Contains(settingsReadResult.ValueOrDefault(FSharpExtensionSettings.GetDiagnosticsFrom, defaultValue: FSharpExtensionSettings.BOTH))
                 ));
 
         var disposeToEndSubscription =

tests/FSharp.Compiler.LanguageServer.Tests/Protocol.fs

@@ -154,34 +154,6 @@ let ``Basic server workflow`` () =
         )
     }
 
-[<Fact>]
-let ``Full semantic tokens`` () =
-
-    task {
-        let! client = initializeLanguageServer None
-        let workspace = client.Workspace
-        let contentOnDisk = "let x = 1"
-        let fileOnDisk = sourceFileOnDisk contentOnDisk
-        let _projectIdentifier =
-            workspace.Projects.AddOrUpdate(ProjectConfig.Create(), [ fileOnDisk.LocalPath ])
-        do!
-            client.JsonRpc.NotifyAsync(
-                Methods.TextDocumentDidOpenName,
-                DidOpenTextDocumentParams(
-                    TextDocument = TextDocumentItem(Uri = fileOnDisk, LanguageId = "F#", Version = 1, Text = contentOnDisk)
-                )
-            )
-        let! semanticTokensResponse =
-            client.JsonRpc.InvokeAsync<SemanticTokens>(
-                Methods.TextDocumentSemanticTokensFullName,
-                SemanticTokensParams(TextDocument = TextDocumentIdentifier(Uri = fileOnDisk))
-            )
-
-        let expected = [| 0; 0; 0; 1; 0; 0; 0; 3; 15; 0; 0; 4; 1; 17; 0; 0; 4; 1; 19; 0 |]
-
-        Assert.Equal<int array>(expected, semanticTokensResponse.Data)
-    }
-
 [<Fact>]
 let ``Shutdown and exit`` () =
     task {

vsintegration/Vsix/VisualFSharpFull/VisualFSharp.Core.targets

@@ -24,6 +24,10 @@
     </Content>
   </ItemGroup>
 
+  <ItemGroup>
+    <VSIXSourceItem Include="$(ArtifactsBinDir)\FSharp.VisualStudio.Extension\$(Configuration)\net8.0-windows\.vsextension\extension.json" VsixSubPath=".vsextension" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Build\FSharp.Build.fsproj">
       <Project>{702A7979-BCF9-4C41-853E-3ADFC9897890}</Project>
@@ -127,6 +131,15 @@
       <Private>True</Private>
     </ProjectReference>
 
+
+    <ProjectReference Include="$(FSharpSourcesRoot)\FSharp.VisualStudio.Extension\FSharp.VisualStudio.Extension.csproj">
+      <Name>FSharp.VisualStudio.Extension</Name>
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+      <SkipGetTargetFrameworkProperties>true</SkipGetTargetFrameworkProperties>
+      <IncludeInVSIX>true</IncludeInVSIX>
+      <IncludeOutputGroupsInVSIX>ExtensionFilesOutputGroup</IncludeOutputGroupsInVSIX>
+    </ProjectReference>
+
     <ProjectReference Include="$(FSharpSourcesRoot)\..\vsintegration\src\FSharp.UIResources\FSharp.UIResources.csproj">
       <Project>{c4586a06-1402-48bc-8e35-a1b8642f895b}</Project>
       <Name>FSharp.UIResources</Name>