Merge branch 'master' into remove-ekg

Author: lorenzo

.circleci/config.yml

@@ -18,7 +18,7 @@ defaults: &defaults
 
     - run:
         name: Create a composite cabal file for changes detection
-        command: find . -name "*.cabal" | grep -v "stack-work" | xargs cat > all-cabal.txt
+        command: find . -name "*.cabal" | grep -v -e "stack-work" -e "dist-newstyle" -e "submodules" -e "testdata" | sort | xargs cat > all-cabal.txt
 
     - restore_cache:
         keys:
@@ -52,7 +52,7 @@ defaults: &defaults
 
     - run:
         name: Generate Hoogle database
-        command: stack --stack-yaml=${STACK_FILE} exec hoogle generate
+        command: if [ ! -d ~/.hoogle ]; then stack --stack-yaml=${STACK_FILE} exec hoogle generate; fi
 
     - run:
         name: Clear cabal-helper cache
@@ -63,6 +63,7 @@ defaults: &defaults
         paths: &cache_paths
           - ~/.stack
           - ~/.cache
+          - ~/.hoogle
           - ~/build/.stack-work
           - ~/build/hie-plugin-api/.stack-work
           - ~/build/submodules/HaRe/.stack-work
@@ -78,6 +79,9 @@ defaults: &defaults
         command: stack -j 1 --stack-yaml=${STACK_FILE} test --dump-logs
         no_output_timeout: 120m
 
+    - store_test_results:
+        path: test-results
+
     - store_artifacts:
         path: test-logs
 

.gitignore

@@ -61,6 +61,7 @@ dist-newstyle/
 dist
 tags
 test-logs/
+test-results/
 .DS_Store
 
 .hspec-failures

Makefile

@@ -37,8 +37,8 @@ hie-%: submodules cabal
 
 ## Updates local git submodules
 submodules:
-	git submodule sync
-	git submodule update --init
+	git submodule sync --recursive
+	git submodule update --init --recursive
 .PHONY: submodules
 
 # ------------------------------------------------------

README.md

@@ -7,7 +7,7 @@
 
 [badge-license]: https://img.shields.io/badge/license-BSD3-green.svg?dummy
 [license]: https://github.com/haskell/haskell-ide-engine/blob/master/LICENSE
-[badge-circleci]: https://img.shields.io/circleci/project/github/haskell/haskell-ide-engine.svg
+[badge-circleci]: https://img.shields.io/circleci/project/github/haskell/haskell-ide-engine/master.svg
 [circleci]: https://circleci.com/gh/haskell/haskell-ide-engine/
 [badge-appveyor]: https://ci.appveyor.com/api/projects/status/6hit7mxvgdrao3q0?svg=true
 [appveyor]: https://ci.appveyor.com/project/Bubba/haskell-ide-engine-74xec

Shakefile.hs

@@ -29,7 +29,7 @@ type GhcPath = String
 -- |Defines all different hie versions that are buildable.
 -- If they are edited, 
 hieVersions :: [VersionNumber]
-hieVersions = ["8.2.2", "8.4.3", "8.4.4", "8.6.1", "8.6.2", "8.6.3"]
+hieVersions = ["8.2.1", "8.2.2", "8.4.2", "8.4.3", "8.4.4", "8.6.1", "8.6.2", "8.6.3"]
 
 -- |Most recent version of hie.
 -- Important for `dist`, the `hie-wrapper` of the most recent hie 
@@ -129,8 +129,8 @@ buildIcuMacosFix version = execStackWithYaml_
 
 updateSubmodules :: Action ()
 updateSubmodules = do
-  command_ [] "git" ["submodule", "sync"]
-  command_ [] "git" ["submodule", "update", "--init"]
+  command_ [] "git" ["submodule", "sync", "--recursive"]
+  command_ [] "git" ["submodule", "update", "--init", "--recursive"]
 
 installCabal :: GhcPath -> Action ()
 installCabal ghc = do

cabal.project

@@ -7,4 +7,3 @@ packages:
          ./submodules/cabal-helper/
          ./submodules/ghc-mod/
          ./submodules/ghc-mod/core/
-

docs/CircleCI.md

@@ -0,0 +1,41 @@
+# CircleCI
+
+## Jobs
+
+For each version of GHC supported, CircleCI will kick off a build that will build and test HIE for that version.
+Whenever a new stack-x.y.z is added, be sure to also add it to both the `workflow` and `jobs` sections of `config.yml`.
+
+In addition to the stack based jobs, there is also a job that builds HIE with `cabal new-build`. 
+Currently tests are not run as there is an issue with cabal-helper-wrapper that prevents any ghc-mod tests from passing.
+
+## Docker
+
+Each job is carried out in a Docker container built from https://github.com/alanz/haskell-dockerfiles/blob/master/haskell-hie-ci/Dockerfile
+
+## Caching
+
+Since HIE takes a long time to build, CI caches things very liberally.
+Most importantly, the `~/.stack-work` directories for HIE as well as every submodule are cached before and after testing (This is so that if the build passes but the tests fail there is still something cached for the next build). 
+
+### Change detection 
+
+Before restoring any cache, the CircleCI job creates three files and uses their checksum to detect any changes:
+
+1. `all-cabal.txt` detects any changes to `.cabal` files (not in submodules or test data)
+2. `stack-build.txt` detects any changes to the `stack-x.y.z` file
+3. `resolver.txt` detects any changes to the stack resolver
+
+Each job will tries to restore the most specific cache avaialble for it, and fallback to a more general cache if not available:
+
+
+```yaml
+- restore_cache:
+    keys:
+        - stack-cache-{{ .Environment.HIE_CACHE }}-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ checksum "stack-build.txt" }}-{{ checksum "all-cabal.txt" }}
+        - stack-cache-{{ .Environment.HIE_CACHE }}-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ checksum "stack-build.txt" }}
+        - stack-cache-{{ .Environment.HIE_CACHE }}-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ checksum "resolver.txt" }}
+```
+
+## Invalidating the cache
+
+If you need to clear the cache on CircleCI, bump the `$HIE_CACHE` environment variable via the CircleCI web interface and rerun the workflow. 

haskell-ide-engine.cabal

@@ -109,7 +109,6 @@ executable hie
                      , hie-plugin-api
                      , hslogger
                      , optparse-simple
-                     , stm
   ghc-options:         -threaded -rtsopts -with-rtsopts=-N -Wall -Wredundant-constraints
                        -with-rtsopts=-T
   if flag(pedantic)
@@ -136,11 +135,35 @@ executable hie-wrapper
      ghc-options:      -Werror
   default-language:    Haskell2010
 
+library hie-test-utils
+  hs-source-dirs:      test/utils
+  exposed-modules:     TestUtils
+  build-depends:       base
+                     , haskell-ide-engine
+                     , haskell-lsp
+                     , hie-plugin-api
+                     , aeson
+                     , blaze-markup
+                     , containers
+                     , data-default
+                     , directory
+                     , filepath
+                     , ghc-mod-core
+                     , hslogger
+                     , hspec
+                     , hspec-core
+                     , stm
+                     , text
+                     , unordered-containers
+                     , yaml
+  ghc-options:         -Wall -Wredundant-constraints
+  if flag(pedantic)
+     ghc-options:      -Werror
+  default-language:    Haskell2010
 
 test-suite unit-test
   type:                exitcode-stdio-1.0
   hs-source-dirs:      test/unit
-                     , test/utils
   main-is:             Main.hs
   other-modules:       ApplyRefactPluginSpec
                        BrittanySpec
@@ -153,8 +176,6 @@ test-suite unit-test
                        HooglePluginSpec
                        JsonSpec
                        Spec
-
-                       TestUtils
   build-depends:       QuickCheck
                      , aeson
                      , base
@@ -163,25 +184,16 @@ test-suite unit-test
                      , directory
                      , filepath
                      , free
-                     , haskell-lsp
                      , haskell-ide-engine
-                     -- , hie-test-utils
+                     , haskell-lsp-types >= 0.4
+                     , hie-test-utils
                      , hie-plugin-api
                      , hoogle > 5.0.11
                      , hspec
                      , quickcheck-instances
-                     , stm
                      , text
                      , unordered-containers
 
-                     -- remove these once hie-test-utils is reinstated
-                     , data-default
-                     , hie-plugin-api
-                     , ghc-mod-core
-                     , hslogger
-                     , yaml
-  -- don't multithread i.e. --with-rtsopts=-N
-  -- otherwise ghc-mod operations become super slow on CircleCI
   ghc-options:         -Wall -Wredundant-constraints
   if flag(pedantic)
      ghc-options:      -Werror
@@ -191,31 +203,22 @@ test-suite unit-test
 test-suite dispatcher-test
   type:                exitcode-stdio-1.0
   hs-source-dirs:      test/dispatcher
-                       test/utils
   main-is:             Main.hs
-  other-modules:       TestUtils
   build-depends:       base
                      , aeson
-                     , containers
                      , data-default
                      , directory
                      , filepath
                      , ghc
-                     , haskell-lsp
                      , haskell-ide-engine
-                     -- , hie-test-utils
+                     , haskell-lsp-types
+                     , hie-test-utils
                      , hie-plugin-api
                      , hspec
                      , stm
                      , text
                      , unordered-containers
 
-                     -- remove these once hie-test-utils is reinstated
-                     , hie-plugin-api
-                     , ghc-mod-core
-                     , hslogger
-                     , unordered-containers
-                     , yaml
   ghc-options:         -threaded -rtsopts -with-rtsopts=-N -Wall -Wredundant-constraints
   if flag(pedantic)
      ghc-options:      -Werror
@@ -225,63 +228,25 @@ test-suite dispatcher-test
 test-suite plugin-dispatcher-test
   type:                exitcode-stdio-1.0
   hs-source-dirs:      test/plugin-dispatcher
-                       test/utils
   main-is:             Main.hs
-  other-modules:       TestUtils
   build-depends:       base
-                     , aeson
-                     , containers
                      , data-default
-                     , directory
-                     , filepath
-                     , ghc
-                     , haskell-lsp
                      , haskell-ide-engine
-                     -- , hie-test-utils
+                     , haskell-lsp-types
                      , hie-plugin-api
+                     , hie-test-utils
                      , hspec
                      , stm
                      , text
-                     , unordered-containers
 
-                     -- remove these once hie-test-utils is reinstated
-                     , hie-plugin-api
-                     , ghc-mod-core
-                     , hslogger
-                     , unordered-containers
-                     , yaml
   ghc-options:         -threaded -rtsopts -with-rtsopts=-N -Wall -Wredundant-constraints
   if flag(pedantic)
      ghc-options:      -Werror
   default-language:    Haskell2010
 
--- library hie-test-utils
---   hs-source-dirs:      test/utils
---   exposed-modules:     TestUtils
---   build-depends:       base
---                      , haskell-ide-engine
---                      , haskell-lsp
---                      , hie-plugin-api
---                      , aeson
---                      , containers
---                      , data-default
---                      , directory
---                      , filepath
---                      , ghc-mod-core
---                      , hslogger
---                      , hspec
---                      , text
---                      , unordered-containers
---                      , yaml
---   ghc-options:         -Wall -Wredundant-constraints
---   if flag(pedantic)
---      ghc-options:      -Werror
---   default-language:    Haskell2010
-
 test-suite func-test
   type:                exitcode-stdio-1.0
   hs-source-dirs:      test/functional
-                     , test/utils
   main-is:             Main.hs
   other-modules:       CompletionSpec
                      , CommandSpec
@@ -300,33 +265,22 @@ test-suite func-test
                      , RenameSpec
                      , SymbolsSpec
                      , Utils
-
-                     , TestUtils
   -- This cannot currently be handled by hie (cabal-helper)
   -- build-tool-depends:  haskell-ide-engine:hie
   build-depends:       aeson
                      , base
-                     , containers
                      , data-default
                      , directory
                      , filepath
                      , lsp-test == 0.5.*
                      , haskell-ide-engine
-                     -- , hie-test-utils
+                     , haskell-lsp-types >= 0.4
+                     , hie-test-utils
                      , hie-plugin-api
                      , hspec
                      , lens
-                     , stm
                      , text
                      , unordered-containers
-
-                     -- remove these once hie-test-utils is reinstated
-                     , hie-plugin-api
-                     , ghc-mod-core
-                     , hslogger
-                     , unordered-containers
-                     , yaml
-                     , haskell-lsp
   ghc-options:         -threaded -rtsopts -with-rtsopts=-N -Wall -Wredundant-constraints
   if flag(pedantic)
      ghc-options:      -Werror

stack-8.6.1.yaml

@@ -31,7 +31,6 @@ extra-deps:
 - temporary-1.2.1.1
 - yaml-0.8.32
 
-
 flags:
   haskell-ide-engine:
     pedantic: true

stack-8.6.3.yaml

@@ -21,7 +21,6 @@ extra-deps:
 - temporary-1.2.1.1
 - yaml-0.8.32
 
-
 flags:
   haskell-ide-engine:
     pedantic: true

submodules/ghc-mod

@@ -23,6 +23,7 @@ import           System.Directory
 import           System.FilePath
 
 import           Test.Hspec
+import           Test.Hspec.Runner
 
 -- ---------------------------------------------------------------------
 -- plugins
@@ -40,8 +41,9 @@ import           Haskell.Ide.Engine.Plugin.HieExtras
 main :: IO ()
 main = do
   setupStackFiles
+  config <- getHspecFormattedConfig "dispatcher"
   withFileLogging "main-dispatcher.log" $ do
-    hspec funcSpec
+    hspecWith config funcSpec
 
 -- main :: IO ()
 -- main = do