Fallback to Talon actions when focus is not on the text editor (#2235)

Edit operations supported by community will now work in vscode outside
of the text editor. eg the search widget
`take line`
`chuck token`

Everything appears to be working when I have tested it. With that said I
have not tested on community and we should probably have a discussion
about some of the finer details of this.


## Checklist

- [x] I have added
[tests](https://www.cursorless.org/docs/contributing/test-case-recorder/)
- [ ] I have updated the
[docs](https://github.com/cursorless-dev/cursorless/tree/main/docs) and
[cheatsheet](https://github.com/cursorless-dev/cursorless/tree/main/cursorless-talon/src/cheatsheet)
- [x] I have not broken the cheatsheet

---------

Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>
Co-authored-by: Pokey Rule <[email protected]>

Author: 3 people

changelog/2024-03-fallBackToTalonActions.md

@@ -0,0 +1,6 @@
+---
+tags: [enhancement]
+pullRequest: 2235
+---
+
+- Fall back to text-based Talon actions when editor is not focused. This allows you to say things like "take token", "bring air", etc, when in the terminal, search bar, etc.

cursorless-talon/src/command.py

@@ -3,10 +3,13 @@
 
 from talon import Module, actions, speech_system
 
+from .fallback import perform_fallback
+from .versions import COMMAND_VERSION
+
 
 @dataclasses.dataclass
 class CursorlessCommand:
-    version = 6
+    version = COMMAND_VERSION
     spokenForm: str
     usePrePhraseSnapshot: bool
     action: dict
@@ -30,10 +33,12 @@ def on_phrase(d):
 class Actions:
     def private_cursorless_command_and_wait(action: dict):
         """Execute cursorless command and wait for it to finish"""
-        actions.user.private_cursorless_run_rpc_command_and_wait(
+        response = actions.user.private_cursorless_run_rpc_command_get(
             CURSORLESS_COMMAND_ID,
             construct_cursorless_command(action),
         )
+        if "fallback" in response:
+            perform_fallback(response["fallback"])
 
     def private_cursorless_command_no_wait(action: dict):
         """Execute cursorless command without waiting"""
@@ -44,10 +49,15 @@ def private_cursorless_command_no_wait(action: dict):
 
     def private_cursorless_command_get(action: dict):
         """Execute cursorless command and return result"""
-        return actions.user.private_cursorless_run_rpc_command_get(
+        response = actions.user.private_cursorless_run_rpc_command_get(
             CURSORLESS_COMMAND_ID,
             construct_cursorless_command(action),
         )
+        if "fallback" in response:
+            return perform_fallback(response["fallback"])
+        if "returnValue" in response:
+            return response["returnValue"]
+        return None
 
 
 def construct_cursorless_command(action: dict) -> dict:

cursorless-talon/src/fallback.py

@@ -0,0 +1,107 @@
+from typing import Callable
+
+from talon import actions
+
+from .versions import COMMAND_VERSION
+
+# This ensures that we remember to update fallback if the response payload changes
+assert COMMAND_VERSION == 7
+
+action_callbacks = {
+    "getText": lambda: [actions.edit.selected_text()],
+    "setSelection": actions.skip,
+    "setSelectionBefore": actions.edit.left,
+    "setSelectionAfter": actions.edit.right,
+    "copyToClipboard": actions.edit.copy,
+    "cutToClipboard": actions.edit.cut,
+    "pasteFromClipboard": actions.edit.paste,
+    "clearAndSetSelection": actions.edit.delete,
+    "remove": actions.edit.delete,
+    "editNewLineBefore": actions.edit.line_insert_up,
+    "editNewLineAfter": actions.edit.line_insert_down,
+}
+
+modifier_callbacks = {
+    "extendThroughStartOf.line": actions.user.select_line_start,
+    "extendThroughEndOf.line": actions.user.select_line_end,
+    "containingScope.document": actions.edit.select_all,
+    "containingScope.paragraph": actions.edit.select_paragraph,
+    "containingScope.line": actions.edit.select_line,
+    "containingScope.token": actions.edit.select_word,
+}
+
+
+def call_as_function(callee: str):
+    wrap_with_paired_delimiter(f"{callee}(", ")")
+
+
+def wrap_with_paired_delimiter(left: str, right: str):
+    selected = actions.edit.selected_text()
+    actions.insert(f"{left}{selected}{right}")
+    for _ in right:
+        actions.edit.left()
+
+
+def containing_token_if_empty():
+    if actions.edit.selected_text() == "":
+        actions.edit.select_word()
+
+
+def perform_fallback(fallback: dict):
+    try:
+        modifier_callbacks = get_modifier_callbacks(fallback)
+        action_callback = get_action_callback(fallback)
+        for callback in reversed(modifier_callbacks):
+            callback()
+        return action_callback()
+    except ValueError as ex:
+        actions.app.notify(str(ex))
+
+
+def get_action_callback(fallback: dict) -> Callable:
+    action = fallback["action"]
+
+    if action in action_callbacks:
+        return action_callbacks[action]
+
+    match action:
+        case "insert":
+            return lambda: actions.insert(fallback["text"])
+        case "callAsFunction":
+            return lambda: call_as_function(fallback["callee"])
+        case "wrapWithPairedDelimiter":
+            return lambda: wrap_with_paired_delimiter(
+                fallback["left"], fallback["right"]
+            )
+
+    raise ValueError(f"Unknown Cursorless fallback action: {action}")
+
+
+def get_modifier_callbacks(fallback: dict) -> list[Callable]:
+    return [get_modifier_callback(modifier) for modifier in fallback["modifiers"]]
+
+
+def get_modifier_callback(modifier: dict) -> Callable:
+    modifier_type = modifier["type"]
+
+    match modifier_type:
+        case "containingTokenIfEmpty":
+            return containing_token_if_empty
+        case "containingScope":
+            scope_type_type = modifier["scopeType"]["type"]
+            return get_simple_modifier_callback(f"{modifier_type}.{scope_type_type}")
+        case "extendThroughStartOf":
+            if "modifiers" not in modifier:
+                return get_simple_modifier_callback(f"{modifier_type}.line")
+        case "extendThroughEndOf":
+            if "modifiers" not in modifier:
+                return get_simple_modifier_callback(f"{modifier_type}.line")
+
+    raise ValueError(f"Unknown Cursorless fallback modifier: {modifier_type}")
+
+
+def get_simple_modifier_callback(key: str) -> Callable:
+    try:
+        return modifier_callbacks[key]
+    except KeyError:
+        raise ValueError(f"Unknown Cursorless fallback modifier: {key}")

cursorless-talon/src/versions.py

@@ -0,0 +1 @@
+COMMAND_VERSION = 7

packages/common/src/FakeCommandServerApi.ts

@@ -0,0 +1,25 @@
+import {
+  CommandServerApi,
+  FocusedElementType,
+  InboundSignal,
+} from "./types/CommandServerApi";
+
+export class FakeCommandServerApi implements CommandServerApi {
+  private focusedElementType: FocusedElementType | undefined;
+  signals: { prePhrase: InboundSignal };
+
+  constructor() {
+    this.signals = { prePhrase: { getVersion: async () => null } };
+    this.focusedElementType = "textEditor";
+  }
+
+  getFocusedElementType(): FocusedElementType | undefined {
+    return this.focusedElementType;
+  }
+
+  setFocusedElementType(
+    focusedElementType: FocusedElementType | undefined,
+  ): void {
+    this.focusedElementType = focusedElementType;
+  }
+}

packages/common/src/getFakeCommandServerApi.ts

@@ -85,16 +85,18 @@ export * from "./types/command/legacy/ActionCommandV5";
 export * from "./types/command/legacy/CommandV5.types";
 export * from "./types/command/legacy/PartialTargetDescriptorV5.types";
 export * from "./types/command/CommandV6.types";
+export * from "./types/command/CommandV7.types";
 export * from "./types/command/legacy/PartialTargetDescriptorV3.types";
 export * from "./types/command/legacy/PartialTargetDescriptorV4.types";
 export * from "./types/CommandServerApi";
 export * from "./util/itertools";
 export * from "./extensionDependencies";
-export * from "./getFakeCommandServerApi";
+export * from "./FakeCommandServerApi";
 export * from "./types/TestCaseFixture";
 export * from "./util/getEnvironmentVariableStrict";
 export * from "./util/CompositeKeyDefaultMap";
 export * from "./util/toPlainObject";
+export * from "./util/clientSupportsFallback";
 export * from "./scopeSupportFacets/scopeSupportFacets.types";
 export * from "./scopeSupportFacets/scopeSupportFacetInfos";
 export * from "./scopeSupportFacets/textualScopeSupportFacetInfos";

packages/common/src/index.ts

@@ -7,6 +7,7 @@ function reorderFields(
 ): EnforceUndefined<TestCaseFixtureLegacy> {
   return {
     languageId: fixture.languageId,
+    focusedElementType: fixture.focusedElementType,
     postEditorOpenSleepTimeMs: fixture.postEditorOpenSleepTimeMs,
     postCommandSleepTimeMs: fixture.postCommandSleepTimeMs,
     command: fixture.command,
@@ -15,6 +16,7 @@ function reorderFields(
     initialState: fixture.initialState,
     finalState: fixture.finalState,
     returnValue: fixture.returnValue,
+    fallback: fixture.fallback,
     thrownError: fixture.thrownError,
     ide: fixture.ide,
   };

packages/common/src/testUtil/serializeTestFixture.ts

@@ -2,11 +2,15 @@
  * API object for interacting with the command server
  */
 export interface CommandServerApi {
+  getFocusedElementType: () => FocusedElementType | undefined;
+
   signals: {
     prePhrase: InboundSignal;
   };
 }
 
+export type FocusedElementType = "textEditor" | "terminal";
+
 export interface InboundSignal {
   getVersion(): Promise<string | null>;
 }

packages/common/src/types/CommandServerApi.ts

@@ -1,6 +1,6 @@
-import { Command, CommandLatest } from "..";
-import { TestCaseSnapshot } from "../testUtil/TestCaseSnapshot";
-import { PlainSpyIDERecordedValues } from "../testUtil/spyToPlainObject";
+import type { Command, CommandLatest, Fallback, FocusedElementType } from "..";
+import type { TestCaseSnapshot } from "../testUtil/TestCaseSnapshot";
+import type { PlainSpyIDERecordedValues } from "../testUtil/spyToPlainObject";
 
 export type ThrownError = {
   name: string;
@@ -12,6 +12,11 @@ interface TestCaseFixtureBase {
   postCommandSleepTimeMs?: number;
   spokenFormError?: string;
 
+  /**
+   * The type of element that is focused before the command is executed. If undefined default to text editor.
+   */
+  focusedElementType?: FocusedElementType | "other";
+
   /**
    * A list of marks to check in the case of navigation map test otherwise undefined
    */
@@ -30,6 +35,11 @@ interface TestCaseFixtureBase {
    * error test case.
    */
   returnValue?: unknown;
+
+  /**
+   * The fallback of the command. Will be undefined if the command was executed by the extension.
+   */
+  fallback?: Fallback;
 }
 
 export interface TestCaseFixture extends TestCaseFixtureBase {