Merge pull request #1031 from bubba/hspec-formatter

CI improvements

Author: lukel97

.circleci/config.yml

@@ -18,7 +18,7 @@ defaults: &defaults
 
     - run:
         name: Create a composite cabal file for changes detection
-        command: find . -name "*.cabal" | grep -v "stack-work" | xargs cat > all-cabal.txt
+        command: find . -name "*.cabal" | grep -v -e "stack-work" -e "dist-newstyle" -e "submodules" -e "testdata" | sort | xargs cat > all-cabal.txt
 
     - restore_cache:
         keys:
@@ -52,7 +52,7 @@ defaults: &defaults
 
     - run:
         name: Generate Hoogle database
-        command: stack --stack-yaml=${STACK_FILE} exec hoogle generate
+        command: if [ ! -d ~/.hoogle ]; then stack --stack-yaml=${STACK_FILE} exec hoogle generate; fi
 
     - run:
         name: Clear cabal-helper cache
@@ -63,6 +63,7 @@ defaults: &defaults
         paths: &cache_paths
           - ~/.stack
           - ~/.cache
+          - ~/.hoogle
           - ~/build/.stack-work
           - ~/build/hie-plugin-api/.stack-work
           - ~/build/submodules/HaRe/.stack-work
@@ -78,6 +79,9 @@ defaults: &defaults
         command: stack -j 1 --stack-yaml=${STACK_FILE} test --dump-logs
         no_output_timeout: 120m
 
+    - store_test_results:
+        path: test-results
+
     - store_artifacts:
         path: test-logs
 

.gitignore

@@ -61,6 +61,7 @@ dist-newstyle/
 dist
 tags
 test-logs/
+test-results/
 .DS_Store
 
 .hspec-failures

cabal.project

@@ -6,8 +6,4 @@ packages:
          ./submodules/brittany
          ./submodules/cabal-helper/
          ./submodules/ghc-mod/
-         ./submodules/ghc-mod/core/
-
-
--- allow-newer:ekg-core-0.1.1.4:base
--- allow-newer:ekg-json-0.1.0.6:base
+         ./submodules/ghc-mod/core/

docs/CircleCI.md

@@ -0,0 +1,41 @@
+# CircleCI
+
+## Jobs
+
+For each version of GHC supported, CircleCI will kick off a build that will build and test HIE for that version.
+Whenever a new stack-x.y.z is added, be sure to also add it to both the `workflow` and `jobs` sections of `config.yml`.
+
+In addition to the stack based jobs, there is also a job that builds HIE with `cabal new-build`. 
+Currently tests are not run as there is an issue with cabal-helper-wrapper that prevents any ghc-mod tests from passing.
+
+## Docker
+
+Each job is carried out in a Docker container built from https://github.com/alanz/haskell-dockerfiles/blob/master/haskell-hie-ci/Dockerfile
+
+## Caching
+
+Since HIE takes a long time to build, CI caches things very liberally.
+Most importantly, the `~/.stack-work` directories for HIE as well as every submodule are cached before and after testing (This is so that if the build passes but the tests fail there is still something cached for the next build). 
+
+### Change detection 
+
+Before restoring any cache, the CircleCI job creates three files and uses their checksum to detect any changes:
+
+1. `all-cabal.txt` detects any changes to `.cabal` files (not in submodules or test data)
+2. `stack-build.txt` detects any changes to the `stack-x.y.z` file
+3. `resolver.txt` detects any changes to the stack resolver
+
+Each job will tries to restore the most specific cache avaialble for it, and fallback to a more general cache if not available:
+
+
+```yaml
+- restore_cache:
+    keys:
+        - stack-cache-{{ .Environment.HIE_CACHE }}-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ checksum "stack-build.txt" }}-{{ checksum "all-cabal.txt" }}
+        - stack-cache-{{ .Environment.HIE_CACHE }}-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ checksum "stack-build.txt" }}
+        - stack-cache-{{ .Environment.HIE_CACHE }}-{{ arch }}-{{ .Environment.CIRCLE_JOB }}-{{ checksum "resolver.txt" }}
+```
+
+## Invalidating the cache
+
+If you need to clear the cache on CircleCI, bump the `$HIE_CACHE` environment variable via the CircleCI web interface and rerun the workflow. 

haskell-ide-engine.cabal

@@ -144,13 +144,15 @@ library hie-test-utils
                      , haskell-lsp
                      , hie-plugin-api
                      , aeson
+                     , blaze-markup
                      , containers
                      , data-default
                      , directory
                      , filepath
                      , ghc-mod-core
                      , hslogger
                      , hspec
+                     , hspec-core
                      , stm
                      , text
                      , unordered-containers

stack-8.6.1.yaml

@@ -34,7 +34,6 @@ extra-deps:
 - temporary-1.2.1.1
 - yaml-0.8.32
 
-
 flags:
   haskell-ide-engine:
     pedantic: true

stack-8.6.3.yaml

@@ -23,7 +23,6 @@ extra-deps:
 - temporary-1.2.1.1
 - yaml-0.8.32
 
-
 flags:
   haskell-ide-engine:
     pedantic: true

test/dispatcher/Main.hs

@@ -23,6 +23,7 @@ import           System.Directory
 import           System.FilePath
 
 import           Test.Hspec
+import           Test.Hspec.Runner
 
 -- ---------------------------------------------------------------------
 -- plugins
@@ -40,8 +41,9 @@ import           Haskell.Ide.Engine.Plugin.HieExtras
 main :: IO ()
 main = do
   setupStackFiles
+  config <- getHspecFormattedConfig "dispatcher"
   withFileLogging "main-dispatcher.log" $ do
-    hspec funcSpec
+    hspecWith config funcSpec
 
 -- main :: IO ()
 -- main = do

test/functional/Main.hs

@@ -3,7 +3,7 @@ module Main where
 import           Control.Monad.IO.Class
 import           Language.Haskell.LSP.Test
 import qualified FunctionalSpec
-import           Test.Hspec
+import           Test.Hspec.Runner (hspecWith)
 import           TestUtils
 
 main :: IO ()
@@ -13,5 +13,6 @@ main = do
   putStrLn "Warming up HIE cache..."
   runSessionWithConfig (defaultConfig { messageTimeout = 120 }) hieCommand fullCaps "test/testdata" $
     liftIO $ putStrLn "HIE cache is warmed up"
-  -- withFileLogging "functional.log" $ hspec FunctionalSpec.spec
-  withFileLogging logFilePath $ hspec FunctionalSpec.spec
+
+  config <- getHspecFormattedConfig "functional"
+  withFileLogging logFilePath $ hspecWith config $ FunctionalSpec.spec

test/plugin-dispatcher/Main.hs

@@ -14,14 +14,15 @@ import           Language.Haskell.LSP.Types
 import           TestUtils
 
 import           Test.Hspec
+import           Test.Hspec.Runner
 
 -- ---------------------------------------------------------------------
 
 main :: IO ()
 main = do
   setupStackFiles
-  withFileLogging "plugin-dispatcher.log" $ do
-    hspec newPluginSpec
+  config <- getHspecFormattedConfig "plugin-dispatcher"
+  withFileLogging "plugin-dispatcher.log" $ hspecWith config newPluginSpec
 
 -- ---------------------------------------------------------------------
 

test/unit/ApplyRefactPluginSpec.hs

@@ -134,7 +134,7 @@ applyRefactSpec = do
       filePath  <- filePathToUri <$> makeAbsolute "./test/testdata/HlintPragma.hs"
 
       let req = lintCmd' filePath
-      r <- cdAndDo "./test/testdata" $ runIGM testPlugins req
+      r <- withCurrentDirectory "./test/testdata" $ runIGM testPlugins req
       r `shouldBe`
         (IdeResultOk
            (PublishDiagnosticsParams

test/unit/GhcModPluginSpec.hs

@@ -39,7 +39,7 @@ testPlugins = pluginDescToIdePlugins [ghcmodDescriptor "ghcmod"]
 ghcmodSpec :: Spec
 ghcmodSpec =
   describe "ghc-mod plugin commands(old plugin api)" $ do
-    it "runs the check command" $ cdAndDo "./test/testdata" $ do
+    it "runs the check command" $ withCurrentDirectory "./test/testdata" $ do
       fp <- makeAbsolute "./FileWithWarning.hs"
       let act = setTypecheckedModule arg
           arg = filePathToUri fp
@@ -57,7 +57,7 @@ ghcmodSpec =
 
     -- ---------------------------------
 
-    it "runs the lint command" $ cdAndDo "./test/testdata" $ do
+    it "runs the lint command" $ withCurrentDirectory "./test/testdata" $ do
       fp <- makeAbsolute "FileWithWarning.hs"
       let uri = filePathToUri fp
           act = lintCmd' uri
@@ -71,7 +71,7 @@ ghcmodSpec =
 
     -- ---------------------------------
 
-    it "runs the info command" $ cdAndDo "./test/testdata" $ do
+    it "runs the info command" $ withCurrentDirectory "./test/testdata" $ do
       fp <- makeAbsolute "HaReRename.hs"
       let uri = filePathToUri fp
           act = infoCmd' uri "main"
@@ -82,7 +82,7 @@ ghcmodSpec =
 
     -- ---------------------------------
 
-    it "runs the type command" $ cdAndDo "./test/testdata" $ do
+    it "runs the type command" $ withCurrentDirectory "./test/testdata" $ do
       fp <- makeAbsolute "HaReRename.hs"
       let uri = filePathToUri fp
           act = do
@@ -117,7 +117,7 @@ ghcmodSpec =
 
     -- ---------------------------------
 
-    it "runs the casesplit command" $ cdAndDo "./test/testdata" $ do
+    it "runs the casesplit command" $ withCurrentDirectory "./test/testdata" $ do
       fp <- makeAbsolute "GhcModCaseSplit.hs"
       let uri = filePathToUri fp
           act = do

test/unit/HaRePluginSpec.hs

@@ -43,7 +43,7 @@ testPlugins = pluginDescToIdePlugins [hareDescriptor "hare"]
 
 dispatchRequestPGoto :: IdeGhcM a -> IO a
 dispatchRequestPGoto =
-  cdAndDo "./test/testdata/gototest"
+  withCurrentDirectory "./test/testdata/gototest"
     . runIGM testPlugins
 
 -- ---------------------------------------------------------------------
@@ -54,7 +54,7 @@ hareSpec = do
     cwd <- runIO getCurrentDirectory
     -- ---------------------------------
 
-    it "renames" $ cdAndDo "test/testdata" $ do
+    it "renames" $ withCurrentDirectory "test/testdata" $ do
 
       let uri = filePathToUri $ cwd </> "test/testdata/HaReRename.hs"
           act = renameCmd' uri (toPos (5,1)) "foolong"
@@ -67,7 +67,7 @@ hareSpec = do
 
     -- ---------------------------------
 
-    it "returns an error for invalid rename" $ cdAndDo "test/testdata" $ do
+    it "returns an error for invalid rename" $ withCurrentDirectory "test/testdata" $ do
       let uri = filePathToUri $ cwd </> "test/testdata/HaReRename.hs"
           act = renameCmd' uri (toPos (15,1)) "foolong"
           arg = HPT uri (toPos (15,1)) "foolong"
@@ -78,7 +78,7 @@ hareSpec = do
 
     -- ---------------------------------
 
-    it "demotes" $ cdAndDo "test/testdata" $ do
+    it "demotes" $ withCurrentDirectory "test/testdata" $ do
       let uri = filePathToUri $ cwd </> "test/testdata/HaReDemote.hs"
           act = demoteCmd' uri (toPos (6,1))
           arg = HP uri (toPos (6,1))
@@ -90,7 +90,7 @@ hareSpec = do
 
     -- ---------------------------------
 
-    it "duplicates a definition" $ cdAndDo "test/testdata" $ do
+    it "duplicates a definition" $ withCurrentDirectory "test/testdata" $ do
       let uri = filePathToUri $ cwd </> "test/testdata/HaReRename.hs"
           act = dupdefCmd' uri (toPos (5,1)) "foonew"
           arg = HPT uri (toPos (5,1)) "foonew"
@@ -102,7 +102,7 @@ hareSpec = do
 
     -- ---------------------------------
 
-    it "converts if to case" $ cdAndDo "test/testdata" $ do
+    it "converts if to case" $ withCurrentDirectory "test/testdata" $ do
 
       let uri = filePathToUri $ cwd </> "test/testdata/HaReCase.hs"
           act = iftocaseCmd' uri (Range (toPos (5,9))
@@ -117,7 +117,7 @@ hareSpec = do
 
     -- ---------------------------------
 
-    it "lifts one level" $ cdAndDo "test/testdata" $ do
+    it "lifts one level" $ withCurrentDirectory "test/testdata" $ do
 
       let uri = filePathToUri $ cwd </> "test/testdata/HaReMoveDef.hs"
           act = liftonelevelCmd' uri (toPos (6,5))
@@ -131,7 +131,7 @@ hareSpec = do
 
     -- ---------------------------------
 
-    it "lifts to top level" $ cdAndDo "test/testdata" $ do
+    it "lifts to top level" $ withCurrentDirectory "test/testdata" $ do
 
       let uri = filePathToUri $ cwd </> "test/testdata/HaReMoveDef.hs"
           act = lifttotoplevelCmd' uri (toPos (12,9))
@@ -147,7 +147,7 @@ hareSpec = do
 
     -- ---------------------------------
 
-    it "deletes a definition" $ cdAndDo "test/testdata" $ do
+    it "deletes a definition" $ withCurrentDirectory "test/testdata" $ do
       let uri = filePathToUri $ cwd </> "test/testdata/FuncTest.hs"
           act = deleteDefCmd' uri (toPos (6,1))
           arg = HP uri (toPos (6,1))
@@ -159,7 +159,7 @@ hareSpec = do
 
     -- ---------------------------------
 
-    it "generalises an applicative" $ cdAndDo "test/testdata" $ do
+    it "generalises an applicative" $ withCurrentDirectory "test/testdata" $ do
       let uri = filePathToUri $ cwd </> "test/testdata/HaReGA1.hs"
           act = genApplicativeCommand' uri (toPos (4,1))
           arg = HP uri (toPos (4,1))

test/unit/Main.hs

@@ -9,7 +9,8 @@ import qualified Spec
 main :: IO ()
 main = do
   setupStackFiles
-  withFileLogging "main.log" $ hspec Spec.spec
+  config <- getHspecFormattedConfig "unit"
+  withFileLogging "main.log" $ hspecWith config $ Spec.spec
 
 -- main :: IO ()
 -- main = do