Redesign REPL I/O and add racket-hash-lang-mode

This commit is a squash of nearly 250 commits from the long-running
branch, `hash-lang`.

Major themes:

1. Change REPL I/O. We no longer use a TCP connection to do I/O for
each REPL. Instead use commands (input) and notifications (output).
Furthermore send various kinds of output as distinct notifications.

2. Support use of hash-lang colors, indent, navigation when editing
and in REPL.

Add racket-hash-lang-mode, an alternative to racket-mode for editing
source files, which uses coloring, indent and navigation supplied by a
lang.

Any number of racket-mode or racket-hash-lang-mode buffers may take
turns using the same racket-repl-mode. The last-run edit buffer's
settings are used in the REPL.

Needs Racket 6.12+ for interval-map-ref/bounds.

Use syntax-color/color-textoid when available (with new-enough
versions of Racket and/or syntax-color-lib) but not required.

3. racket-xp-mode: Do "semantic" highlighting of binding sites.
Intended for use by racket-hash-lang-mode to get more than just lexer
colors.

---

Fixes greghendershott#482.
Fixes greghendershott#619.
Fixes greghendershott#642.
Fixes greghendershott#663.
Fixes greghendershott#667.
Fixes greghendershott#671.
Fixes greghendershott#672.
Fixes greghendershott#673.

Closes greghendershott#64.
Closes greghendershott#633.

Closes PR greghendershott#661.

Author: greghendershott

.github/workflows/test.yml

@@ -13,9 +13,9 @@ jobs:
         emacs_version:
           - '25.1'              # our minimum supported version
           - '26.3'
-          - '28.1'              # most recent release
+          - '29.1'              # most recent release
         racket_version:
-          - '6.9'               # our minimum supported version
+          - '6.12'              # our minimum supported version
           - 'stable'            # most recent release
         # Also include bleeding edge snapshots of both Emacs and
         # Racket. Note that "allow_failure: true" doesn't seem to
@@ -46,18 +46,20 @@ jobs:
         run: make show-versions
       - name: Install Emacs Packages
         run: make deps
-      - name: Compile Elisp
+      - name: Compile Emacs Lisp
         run: make compile
-      - name: Run Tests
-        run: make test
+      - name: Run Emacs Lisp Tests
+        run: make test-elisp
+      - name: Run Racket Tests
+        run: xvfb-run make test-racket
 
   windows:
     runs-on: windows-latest
     strategy:
       fail-fast: false
       matrix:
         emacs_version:
-          - '28.1'              # most recent release
+          - '29.1'              # most recent release
         racket_version:
           - 'stable'            # most recent release
     name: Windows Emacs ${{ matrix.emacs_version }} and Racket ${{ matrix.racket_version }}
@@ -78,6 +80,7 @@ jobs:
         run: make deps
       - name: Compile Elisp
         run: make compile
-      - name: Run Tests
-        run: make test
-
+      - name: Run Emacs Lisp Tests
+        run: make test-elisp
+      - name: Run Racket Tests
+        run: make test-racket

Makefile

@@ -7,14 +7,19 @@ help:
 # default on PATH. e.g. `EMACS=/path/to/emacs make`.
 EMACS ?= emacs
 RACKET ?= racket
+# Allow another locations for Emacs packages.
+EMACS_PACKAGES ?= ~/.emacs.d/elpa
 
 show-versions:
 	@echo `which $(RACKET)`
 	@$(RACKET) --version
 	@echo `which $(EMACS)`
 	@$(EMACS) --version
 
-batch-emacs := $(EMACS) --batch -Q -L . --eval '(package-initialize)'
+batch-emacs := \
+  $(EMACS) --batch -Q -L . \
+  --eval '(setq package-user-dir "$(EMACS_PACKAGES)")' \
+  --eval '(package-initialize)'
 
 byte-compile := \
   $(batch-emacs) \
@@ -61,10 +66,16 @@ test-elisp:
       --eval '(setq racket-program "$(RACKET)")' \
       -f ert-run-tests-batch-and-exit
 
+# Files to test using `raco test -x`.
+test-x-rkt-files := $(wildcard ./racket/*.rkt) $(wildcard ./racket/commands/*.rkt)
+# Exclude hash-lang.rkt because it will fail to eval on older Rackets;
+# normally we only dynamic-require it. Furthermore its tests are in
+# ./test/racket/hash-lang-test.rkt.
+test-x-rkt-files := $(filter-out ./racket/hash-lang.rkt, $(test-x-rkt-files))
+
 test-racket:
+	$(RACKET) -l raco test -x $(test-x-rkt-files)
 	$(RACKET) -l raco test ./test/racket/
-	$(RACKET) -l raco test -x ./racket/*.rkt
-	$(RACKET) -l raco test -x ./racket/commands/*.rkt
 
 test-slow:
 	$(RACKET) -l raco test --submodule slow-test ./racket/imports.rkt

README.org

@@ -9,7 +9,7 @@ A variety of Emacs major and minor modes for [[https://www.racket-lang.org/][Rac
 check-syntax, debug, profile, logging, and more. The edit/run
 experience is similar to [[https://docs.racket-lang.org/drracket/][DrRacket]].
 
-Compatible with *Emacs 25.1+* and *Racket 6.9+*.
+Compatible with *Emacs 25.1+* and *Racket 6.12+*.
 
 ** Documentation
 

doc/arch-pict.rkt

@@ -9,7 +9,6 @@
 
 (define pipe-color "blue")
 (define ssh-color "purple")
-(define tcp-color "brown")
 
 (define host-color      (make-color 0 0 0 0.0))
 (define front-end-color (make-color #xF0 #xF7 #xF0 1.0))
@@ -48,17 +47,14 @@
    (vl-append
     (text "Emacs front end" '(bold))
     (hc-append
-     (text "Command requests/responses via ")
+     (text "Command requests/responses and notifications via ")
      (colorize (text "pipe" '(bold)) pipe-color)
      (text " or ")
      (colorize (text "ssh" '(bold)) ssh-color)
-     (text "."))
-    (hc-append
-     (text "REPL I/O via one ")
-     (colorize (text "TCP" '(bold)) tcp-color)
-     (text " connection per REPL buffer.")))))
+     (text ".")))))
 
 (define (backend path)
+  (define i/o-color (if (regexp-match? #rx"^/[^:]+:" path) ssh-color pipe-color))
   (box
    #:inset 5
    #:color (light "black")
@@ -73,17 +69,17 @@
      10
      (colorize
       (box #:inset 5 (text "Commands"))
-      (if (regexp-match? #rx"^/[^:]+:" path) ssh-color pipe-color))
+      i/o-color)
      (vl-append
       4
       (colorize (box #:inset 2 (text "REPL 1"))
-                tcp-color)
+                i/o-color)
       (colorize (box #:inset 2 (text "REPL 2"))
-                tcp-color)
+                i/o-color)
       (colorize (box #:inset 2
                      #:segment 2
                      (text "REPL n" '(italic)))
-                tcp-color))))))
+                i/o-color))))))
 
 (define (back-end-source-files)
   (box

doc/generate.el

@@ -19,6 +19,7 @@
 (require 'racket-unicode-input-method)
 (require 'racket-smart-open)
 (require 'racket-repl-buffer-name)
+(require 'racket-hash-lang)
 (require 'seq)
 
 (defun racket-generate-reference.org ()
@@ -51,6 +52,13 @@
     racket-align
     racket-unalign
     racket-complete-at-point
+    "Hash Langs"
+    racket-hash-lang-mode
+    (racket-hash-lang-backward ,racket-hash-lang-mode-map)
+    (racket-hash-lang-forward ,racket-hash-lang-mode-map)
+    (racket-hash-lang-up ,racket-hash-lang-mode-map)
+    (racket-hash-lang-down ,racket-hash-lang-mode-map)
+    (racket-hash-lang-C-M-q-dwim ,racket-hash-lang-mode-map)
     "Explore"
     racket-xp-mode
     (racket-xp-describe ,racket-xp-mode-map)
@@ -180,7 +188,11 @@
     racket-browse-url-function
     racket-xp-after-change-refresh-delay
     racket-xp-highlight-unused-regexp
+    racket-xp-binding-font-lock-face-modes
     racket-documentation-search-location
+    "Hash lang variables"
+    racket-hash-lang-token-face-alist
+    racket-hash-lang-module-language-hook
     "REPL variables"
     racket-repl-buffer-name-function
     racket-submodules-to-run
@@ -241,6 +253,12 @@
     racket-xp-unused-face
     racket-xp-tail-target-face
     racket-xp-tail-position-face
+    racket-xp-binding-lang-face
+    racket-xp-binding-lang-use-face
+    racket-xp-binding-import-face
+    racket-xp-binding-import-use-face
+    racket-xp-binding-local-face
+    racket-xp-binding-local-use-face
     racket-logger-config-face
     racket-logger-topic-face
     racket-logger-fatal-face
@@ -251,7 +269,14 @@
     racket-doc-link-face
     racket-ext-link-face
     racket-doc-output-face
-    racket-doc-litchar-face)
+    racket-doc-litchar-face
+    racket-repl-message
+    racket-repl-prompt
+    racket-repl-value
+    racket-repl-error-message
+    racket-repl-error-location
+    racket-repl-stdout
+    racket-repl-stderr)
   "Faces to include in the Reference.")
 
 (defun racket-generate--faces ()

doc/racket-mode.org

@@ -1,7 +1,7 @@
 #+OPTIONS: ':t toc:t author:t email:t H:4
 
 #+MACRO: kbd @@texinfo:@kbd{$1}@@ @@html:<kbd>$1</kbd>@@
-#+MACRO: img @@texinfo:@image{$1,,,$2. Command I/O via pipe (local) or ssh (remote). REPL I/O via one TCP connection per REPL buffer (local/remote). Each back end provides zero or more REPLs.,.svg}@@
+#+MACRO: img @@texinfo:@image{$1,,,$2. Command I/O via pipe (local) or ssh (remote). Each back end provides zero or more REPLs.,.svg}@@
 #+MACRO: ref @@texinfo:@ref{$1}@@
 #+MACRO: see @@texinfo:@xref{$1}@@
 
@@ -32,11 +32,13 @@ SPDX-License-Identifier: GPL-3.0-or-later
 
 The [[https://www.racket-mode.com/][Racket Mode]] package consists of a variety of Emacs major and minor modes, including:
 
-- ~racket-mode~: A major mode for editing ~.rkt~ files.
+- ~racket-mode~: A major mode to edit ~.rkt~ files. Generally assumes ~#lang racket~.
 
-- {{{ref(racket-xp-mode)}}}: An optional minor mode that enhances ~racket-mode~ to explain and explore code.
+- {{{ref(racket-hash-lang-mode)}}}: An alternative to ~racket-mode~ using behavior specified by a ~#lang~ for colors, indent, expression navigation, etc. /Experimental/.
 
-- ~racket-repl-mode~: A major mode for running programs providing a REPL.
+- {{{ref(racket-xp-mode)}}}: A minor mode to enhance either edit mode. Explain and explore code, similar to background check-syntax in Dr Racket.
+
+- ~racket-repl-mode~: A major mode to run programs and use a REPL.
 
 - Various other modes to support specific features:
   - {{{ref(racket-logger-mode)}}}
@@ -45,7 +47,7 @@ The [[https://www.racket-mode.com/][Racket Mode]] package consists of a variety
 
 For code, issues, and pull requests, see the [[https://github.com/greghendershott/racket-mode][Git repo]].
 
-To fund this work, see [[https://github.com/users/greghendershott/sponsorship][GitHub Sponsors]] or [[https://www.paypal.me/greghendershott][PayPal]].
+To sponsor this work, see [[https://github.com/users/greghendershott/sponsorship][GitHub Sponsors]] or [[https://www.paypal.me/greghendershott][PayPal]].
 
 * Install, Update, and Uninstall
 
@@ -163,6 +165,29 @@ On macOS, downloading Racket doesn't add its ~bin~ directory to your ~PATH~. Eve
 
 You can ~setq~ this directly in your Emacs init file (=~/.emacs= or =~/.emacs.d/init.el=), or, use {{{kbd(M-x)}}} ~customize~, as you prefer.
 
+** Which major mode to use
+
+Racket is a programming language.
+
+Racket is also a "language-oriented programming language". Most Racket source files contain a `#lang` line. The lang may be an s-expression lang like ~racket~, or an at-expression lang like ~scribble/manual~, or something completely different like ~datalog~ or ~rhombus~.
+
+The Racket Mode package offers a choice of two major modes to use in buffers for viewing and editing source code. Each has pros and cons.
+
+Whereas ~racket-mode~ is in the tradition of Emacs ~lisp-mode~ and ~scheme-mode~ and assumes s-expression langs, ~racket-hash-lang-mode~ takes the approach of DrRacket to work for all langs.
+
+- ~racket-mode~ is the original, "classic" mode for ~#lang racket~ and related s-expression languages. It is implemented entirely in Emacs and does /not/ need Racket Mode's back end racket process running. Font-lock (coloring) uses rules for a fixed set of identifiers from ~racket~ lang and popular modules like ~racket/match~. Indentation uses rules for a fixed set of forms, and may be customized (see below).
+
+- ~racket-hash-lang-mode~ uses font-lock (colors) and indentation determined by the lang; to get this information it /does/ need the Racket Mode's back end racket process running. Although basic editing should feel fast, you might notice some delay when indenting. You might see colors appear after a small delay (but it will not block editing). Speaking of colors, they will be "plainer" than ~racket-mode~ -- mostly just for different kinds of tokens like numbers, comments, strings, and keywords. This looks similar to DrRacket. However if you /also/ enable the minor mode ~racket-xp-mode~, it will eventually add more colors at definition and use sites, and vary the colors depending on whether the identifier is local, imported, or from the module language. So you may see the "syntax" highlighting appear fairly quickly from ~racket-hash-lang-mode~, and later see more "semantic" highlighting contributed by ~racket-xp-mode~. The end result will be about as rich, although not exactly the same, as ~racket-mode~.
+
+You can use different major modes for different kinds of files:
+
+- For editing ~.rkt~ files and s-expression langs, which mode to use is personal preference.
+
+- For ~.scrbl~ and at-expression langs like ~scribble/manual~, ~racket-hash-lang-mode~ is probably better than ~racket-mode~. (Note there is also an unrelated ~scribble-mode~ package.)
+
+- For non-s-expression langs like ~datalog~ or ~rhombus~ (~.rhm~), ~racket-hash-lang-mode~ is definitely better than ~racket-mode~. (Note there is also an unrelated ~rhombus-mode~ package.)
+
+You can use ~auto-mode-alist~ to tell Emacs which major mode to use initially for certain file extensions. Also, in a buffer you can use ~M-x racket-mode~ and ~M-x racket-hash-lang-mode~ to switch between them.
 ** Key bindings
 
 To customize things like key bindings, you can use ~racket-mode-hook~ in your Emacs init file to modify ~racket-mode-map~. For example, although {{{kbd(C-c C-c)}}} is bound by default to the ~racket-run~ command, let's say you wanted {{{kbd(F5)}}} to be an additional binding:
@@ -171,13 +196,16 @@ To customize things like key bindings, you can use ~racket-mode-hook~ in your Em
 (add-hook 'racket-mode-hook
           (lambda ()
             (define-key racket-mode-map (kbd "<f5>") 'racket-run)))
-
 #+END_SRC
 
 Likewise for ~racket-repl-mode-hook~ and ~racket-repl-mode-map~.
 
 ** Font-lock (syntax highlighting)
 
+#+BEGIN_QUOTE
+Note: The alternative major mode {{{ref(racket-hash-lang-mode)}}} disables all of the following behavior and uses colors determined by the #lang.
+#+END_QUOTE
+
 Font-lock (as Emacs calls syntax highlighting) can be controlled using the variable ~font-lock-maximum-decoration~, which defaults to ~t~ (maximum). You can set it to a number, where ~0~ is the lowest level. You can even supply an association list to specify different values for different major modes.
 
 Historically you might choose a lower level for speed. These days you might do so because you prefer a simpler appearance.
@@ -249,12 +277,20 @@ In any case, using the Emacs xref API allows for consistent command names, short
 
 ** Indent
 
+#+BEGIN_QUOTE
+Note: The alternative major mode {{{ref(racket-hash-lang-mode)}}} disables all of the following behavior and uses indentation determined by the #lang.
+#+END_QUOTE
+
 Indentation can be customized in a way similar to lisp-mode and scheme-mode: {{{ref(racket-indent-line)}}}.
 
 (Indentation preserves your line breaks. If you want to use an auto-reformatter --- an expressive pretty printer that chooses line breaks while computing an optimal layout --- the Racket package [[https://docs.racket-lang.org/fmt/][fmt]] is supported by the Emacs package [[https://github.com/lassik/emacs-format-all-the-code][emacs-format-all-the-code]].)
 
 ** paredit
 
+#+BEGIN_QUOTE
+Note: If you use {{{ref(racket-hash-lang-mode)}}}, you can use ~racket-hash-lang-mode-hook~ to enable/disable paredit based on the specific #lang.
+#+END_QUOTE
+
 If you use [[https://melpa.org/#/paredit][paredit]], you might want to add keybindings to ~paredit-mode-map~:
 
 - Bind the curly brace keys to ~paredit-open-curly~ and ~paredit-close-curly~.