chore: support live reload in dev (#4586)

* chore: support live reload in dev

* fix: test due to injected live reload script

* chore: fix docs

* chore: add/update watching individual projects

README.md

@@ -88,10 +88,58 @@ nx build apps-uesio-studio
 
 While developing you may want the entire monorepo to rebuild changed files. You can do this with:
 
-```
+```bash
 npm run watch-all
 ```
 
+Or, if you want to watch just a single project, you can do the following which will watch the project for changes and ensure that any of the projects its dependent on are (re)built (if needed) first:
+
+`npx nx watch -p <projectname> --includeDependentProjects -- nx run-many -t build -p \$NX_PROJECT_NAME`
+
+For example, to monitor the `apps-uesio-studio` project:
+
+```bash
+npx nx watch -p apps-uesio-studio --includeDependentProjects -- nx run-many -t build -p \$NX_PROJECT_NAME
+```
+
+## Live Reload (for development)
+
+> [!NOTE]
+> Live reload is not currently supported when running against the docker image (e.g., `npm run in-docker`).
+
+When running the platform, in addition to [watching](#watch-mode-for-development) and rebuilding when files change, you likely want to have the platform itself and/or the browser to automatically reload after assets have been rebuilt.
+
+See [Live Reload](./docs/development/live-reload.md) for more details on how live reload works.
+
+Depending on your preference, there are two options:
+
+### Platform & Browser
+
+Whenever any changes are made to code within the platform go module itself or any of the libraries that are a part of the platform itself (e.g., `libs/apps/uesio/studio`, `libs/apps/uesio/appkit`, `libs/ui`, `libs/vendor`, etc.) the module/package will be rebuilt and if necessary, the platform automatically restarted. Additionally, the browser will automatically refresh.
+
+In order to accomplish, the [Air](https://github.com/air-verse/air) package must be installed on your machine. This is a one-time installation:
+
+```bash
+go install github.com/air-verse/air@latest
+```
+
+Once `Air` is installed, you can simply run the following:
+
+```bash
+npm run watch-dev
+```
+
+### Browser Only
+
+Whenever any changes are made to code within the libraries that the platform uses (e.g., `libs/apps/uesio/studio`, `libs/apps/uesio/appkit`, `libs/ui`, `libs/vendor`, etc.), the module/package will be rebuilt and the browser reloaded. This is very similar to [Platform & Browser](#platform--browser) but changes to the platform itself (e.g., `*.go` files) will not trigger a rebuild or a restart of the platform go module.
+
+It is recommended to use [Platform & Browser](#platform--browser) for live reload, however `Browser Only` can be used if you do not want to install `Air` or if you know you will not be making any changes to the platform module directly.
+
+```bash
+npm run watch-platform-deps # In a separate terminal
+npm run start # In a separate terminal
+```
+
 ## (Optional) Using Uesio CLI globally
 
 If you'd like to use the Uesio CLI that you have built elsewhere on your machine without having to explicitly reference the binary in `dist/cli`:
@@ -449,7 +497,7 @@ bash apps/platform/migrations_test/test_migrations.sh
 
 ## Testing (Unit, Integration & E2E)
 
-> ![IMPORTANT]
+> [!IMPORTANT]
 > The default behavior for all tests is to run against `https://studio.uesio-dev.com:3000` so you must ensure that [SSL](#set-up-ssl) and [local DNS](#set-up-your-local-dns) have been configured.
 
 To run the various test suites, there are a number of commands available:

apps/platform-integration-tests/hurl_specs/workspace_context.hurl

@@ -98,7 +98,7 @@ Accept: application/json
 HTTP 200
 [Asserts]
 header "Location" == "/login?r=/workspace/uesio/tests/dev/configvalues&expired=true"
-body == "<a href=\"/login?r=/workspace/uesio/tests/dev/configvalues&amp;expired=true\">OK</a>.\n\n"
+body startsWith "<a href=\"/login?r=/workspace/uesio/tests/dev/configvalues&amp;expired=true\">OK</a>.\n\n"
 
 # Logout
 POST https://{{host}}:{{port}}/site/auth/logout

apps/platform/.air.toml

@@ -0,0 +1,52 @@
+root = "."
+testdata_dir = "testdata"
+tmp_dir = ".watch"
+
+[build]
+  args_bin = []
+  bin = "./.watch/main"
+  cmd = "go build -o ./.watch/main ."
+  delay = 1000
+  exclude_dir = ["localbundlestore", "migrations", "migrations_test", "scripts", "seed", "ssl", "userfiles"]
+  exclude_file = []
+  exclude_regex = ["_test.go"]
+  exclude_unchanged = false
+  follow_symlink = false
+  full_bin = ""
+  include_dir = []
+  include_ext = ["go", "gohtml"]
+  include_file = []
+  kill_delay = "0s"
+  log = "build-errors.log"
+  poll = false
+  poll_interval = 0
+  post_cmd = []
+  pre_cmd = []
+  rerun = false
+  rerun_delay = 500
+  send_interrupt = false
+  stop_on_error = false
+
+[color]
+  app = ""
+  build = "yellow"
+  main = "magenta"
+  runner = "green"
+  watcher = "cyan"
+
+[log]
+  main_only = false
+  silent = false
+  time = false
+
+[misc]
+  clean_on_exit = false
+
+[proxy]
+  app_port = 3000
+  enabled = false
+  proxy_port = 8000
+
+[screen]
+  clear_on_rebuild = false
+  keep_scroll = true

apps/platform/.gitignore

@@ -19,9 +19,9 @@ pgdata
 /temp/
 /ssl/certificate.crt
 /ssl/private.key
-sessions/*
-userfiles/*
-localbundlestore/*
+sessions/
+userfiles/
+localbundlestore/
 .DS_Store
 .idea
 .vscode/launch.json
@@ -34,6 +34,7 @@ firebase-debug.log
 __debug_bin*
 uesiofiles-dev/
 uesiobundlestore-dev/
+.watch/
 
 # static js files, compilated by webpack, which are served by gorilla/mux
 /platform/*.js

apps/platform/go.mod

@@ -5,6 +5,7 @@ go 1.23.5
 require (
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/PaesslerAG/gval v1.2.4
+	github.com/aarol/reload v1.2.0
 	github.com/aws/aws-sdk-go-v2 v1.36.0
 	github.com/aws/aws-sdk-go-v2/config v1.29.4
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.57
@@ -61,9 +62,11 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 // indirect
 	github.com/beevik/etree v1.5.0 // indirect
+	github.com/bep/debounce v1.2.1 // indirect
 	github.com/crewjam/httperr v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dlclark/regexp2 v1.11.4 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
@@ -74,6 +77,7 @@ require (
 	github.com/google/s2a-go v0.1.9 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
 	github.com/googleapis/gax-go/v2 v2.14.1 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect

apps/platform/go.sum

@@ -26,6 +26,8 @@ github.com/PaesslerAG/gval v1.2.4 h1:rhX7MpjJlcxYwL2eTTYIOBUyEKZ+A96T9vQySWkVUiU
 github.com/PaesslerAG/gval v1.2.4/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac=
 github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi0jPI=
 github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
+github.com/aarol/reload v1.2.0 h1:zJZuWREX+rrB5V8o/PfQn1y9bG3rG8vmE0NeJl8aep0=
+github.com/aarol/reload v1.2.0/go.mod h1:3XL4gixCRx2VRXr4l3/qemjo2oWq8WxDRetd+EUkBcg=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/aws/aws-sdk-go-v2 v1.36.0 h1:b1wM5CcE65Ujwn565qcwgtOTT1aT4ADOHHgglKjG7fk=
 github.com/aws/aws-sdk-go-v2 v1.36.0/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM=
@@ -71,6 +73,8 @@ github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62
 github.com/beevik/etree v1.5.0 h1:iaQZFSDS+3kYZiGoc9uKeOkUY3nYMXOKLl6KIJxiJWs=
 github.com/beevik/etree v1.5.0/go.mod h1:gPNJNaBGVZ9AwsidazFZyygnd+0pAU38N4D+WemwKNs=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY=
+github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0=
 github.com/bigkevmcd/go-cachewrapper v0.0.0-20240507155736-346a72d92df1 h1:Y26bsH9MpuAIa0xGk4TplHKxgKWOBp40r9I9p9gm044=
 github.com/bigkevmcd/go-cachewrapper v0.0.0-20240507155736-346a72d92df1/go.mod h1:X9bv2LO4wvNDIGtHmvY7p9rITSOeNp4X28Dy35xwtD8=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
@@ -113,6 +117,8 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI
 github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
+github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
@@ -169,6 +175,8 @@ github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEP
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=

apps/platform/pkg/auth/cache.go

@@ -29,6 +29,10 @@ func DeleteUserCacheEntries(userKeys ...string) error {
 	return userCache.Del(userKeys...)
 }
 
+func InvalidateUserCache() error {
+	return userCache.DeleteAll()
+}
+
 func setUserCache(userUniqueKey string, site *meta.Site, user *meta.User) error {
 	// Shallow clone the user, so the caller doesn't have
 	// a reference to the one in the cache.
@@ -71,10 +75,26 @@ func ClearHostCacheForDomains(ids []string) error {
 	return hostCache.Del(keys...)
 }
 
+func InvalidateHostCache() error {
+	return hostCache.DeleteAll()
+}
+
 func getHostKeyFromDomainId(id string) (string, error) {
 	idParts := strings.Split(id, ":")
 	if len(idParts) != 2 {
 		return "", errors.New("Bad Domain ID: " + id)
 	}
 	return getHostKey(idParts[1], idParts[0]), nil
 }
+
+func InvalidateCache() error {
+	if err := InvalidateUserCache(); err != nil {
+		return err
+	}
+
+	if err := InvalidateHostCache(); err != nil {
+		return err
+	}
+
+	return nil
+}

apps/platform/pkg/bundle/cache.go

@@ -63,3 +63,23 @@ func (bsc *BundleStoreCache) AddItemToCache(namespace, version, groupName, itemK
 func (bsc *BundleStoreCache) InvalidateCacheItem(namespace, version, groupName, itemKey string) error {
 	return bsc.bundleEntryCache.Del(bsc.getItemCacheKey(namespace, version, groupName, itemKey))
 }
+
+func (bsc *BundleStoreCache) InvalidateCache() error {
+	if err := bsc.invalidateBundleEntryCache(); err != nil {
+		return err
+	}
+
+	if err := bsc.invalidateFileListCache(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (bsc *BundleStoreCache) invalidateBundleEntryCache() error {
+	return bsc.bundleEntryCache.DeleteAll()
+}
+
+func (bsc *BundleStoreCache) invalidateFileListCache() error {
+	return bsc.fileListCache.DeleteAll()
+}

apps/platform/pkg/bundlestore/platformbundlestore/platformbundlestore.go

@@ -44,6 +44,14 @@ func (b *PlatformBundleStore) GetConnection(options bundlestore.ConnectionOption
 	}, nil
 }
 
+func InvalidateCache() error {
+	if bundleStoreCache != nil {
+		return bundleStoreCache.InvalidateCache()
+	}
+
+	return nil
+}
+
 func getBasePath(namespace, version string) string {
 	return filepath.Join(namespace, version, "bundle")
 }

apps/platform/pkg/bundlestore/systembundlestore/systembundlestore.go

@@ -39,3 +39,11 @@ func (b *SystemBundleStore) GetConnection(options bundlestore.ConnectionOptions)
 		ReadOnly:          false,
 	}, nil
 }
+
+func InvalidateCache() error {
+	if bundleStoreCache != nil {
+		return bundleStoreCache.InvalidateCache()
+	}
+
+	return nil
+}

apps/platform/pkg/cache/cache.go

@@ -6,6 +6,7 @@ type Cache[T any] interface {
 	Get(key string) (T, error)
 	Set(key string, value T) error
 	Del(keys ...string) error
+	DeleteAll() error
 }
 
 type CacheOptions[T any] struct {

apps/platform/pkg/cache/memory.go

@@ -50,6 +50,7 @@ func (mc *MemoryCache[T]) GetAll() map[string]T {
 	return result
 }
 
-func (mc *MemoryCache[T]) DeleteAll() {
+func (mc *MemoryCache[T]) DeleteAll() error {
 	mc.c.Flush()
+	return nil
 }

apps/platform/pkg/cache/redis.go

@@ -71,6 +71,11 @@ func (r RedisCache[T]) Del(keys ...string) error {
 	return deleteKeys(namespacedKeys)
 }
 
+func (r RedisCache[T]) DeleteAll() error {
+	// TODO: This may need to change to flushDB if/when we partition in to different databases in redis
+	return flushAll()
+}
+
 func (r RedisCache[T]) WithExpiration(expiration time.Duration) RedisCache[T] {
 	r.options.Expiration = expiration
 	return r
@@ -175,6 +180,16 @@ func deleteKeys(keys []string) error {
 	return nil
 }
 
+func flushAll() error {
+	conn := GetRedisConn()
+	defer conn.Close()
+	_, err := conn.Do("FLUSHALL")
+	if err != nil {
+		return fmt.Errorf("Error flushing cache from bot: %w", err)
+	}
+	return nil
+}
+
 func setString(key string, data string, ttlSeconds int64) error {
 	conn := GetRedisConn()
 	defer conn.Close()

apps/platform/pkg/cmd/serve.go

@@ -7,6 +7,9 @@ import (
 	"os"
 	"time"
 
+	"github.com/aarol/reload"
+	"github.com/thecloudmasters/uesio/pkg/bundlestore/platformbundlestore"
+	"github.com/thecloudmasters/uesio/pkg/bundlestore/systembundlestore"
 	"github.com/thecloudmasters/uesio/pkg/controller/oauth"
 	"github.com/thecloudmasters/uesio/pkg/env"
 	"github.com/thecloudmasters/uesio/pkg/tls"
@@ -419,9 +422,24 @@ func serve(cmd *cobra.Command, args []string) {
 	serveAddr := host + ":" + port
 
 	// Universal middlewares
-	r.Use(middleware.GZip())
+	// In order for reload package to inject its script (and avoid manually having to have it in our code)
+	// gzip compression must be disabled.  Additionally, when running in dev mode, gzip isn't really needed.
+	if !env.InDevMode() {
+		r.Use(middleware.GZip())
+	}
+
+	var handler http.Handler = r
+	if env.InDevMode() {
+		reloader := reload.New(".watch")
+		reloader.OnReload = func() {
+			// invalidate any caches
+			platformbundlestore.InvalidateCache()
+			systembundlestore.InvalidateCache()
+		}
+		handler = reloader.Handle(r)
+	}
 
-	server := controller.NewServer(serveAddr, r)
+	server := controller.NewServer(serveAddr, handler)
 	var serveErr error
 
 	done := make(chan bool)