Use different URI schemes to inform the LLM about interactivity

Source/Chatbook/Prompting.wl

@@ -35,6 +35,10 @@ $basePromptOrder = {
     "Checkboxes",
     "CheckboxesIndeterminate",
     "ConversionFormatting",
+    "SpecialURI",
+    "SpecialURIAudio",
+    "SpecialURIVideo",
+    "SpecialURIDynamic",
     "VisibleUserInput",
     "TrivialCode",
     "Packages",
@@ -51,6 +55,7 @@ $basePromptClasses = <|
     "Math"              -> { "MathExpressions" },
     "Formatting"        -> { "CodeBlocks", "MathExpressions", "EscapedCharacters" },
     "MessageConversion" -> { "ConversionLargeOutputs", "ConversionGraphics", "ConversionFormatting" },
+    "SpecialURIs"       -> { "SpecialURIAudio", "SpecialURIVideo", "SpecialURIDynamic" },
     "All"               -> $basePromptOrder
 |>;
 
@@ -71,6 +76,10 @@ $basePromptDependencies = Append[ "GeneralInstructionsHeader" ] /@ <|
     "ConversionGraphics"           -> { "MessageConversionHeader" },
     "MarkdownImageBox"             -> { "MessageConversionHeader" },
     "ConversionFormatting"         -> { "MessageConversionHeader" },
+    "SpecialURI"                   -> { },
+    "SpecialURIAudio"              -> { "SpecialURI" },
+    "SpecialURIVideo"              -> { "SpecialURI" },
+    "SpecialURIDynamic"            -> { "SpecialURI" },
     "VisibleUserInput"             -> { },
     "TrivialCode"                  -> { },
     "WolframSymbolCapitalization"  -> { },
@@ -159,6 +168,19 @@ $basePromptComponents[ "ConversionFormatting" ] = "\
 ``Cell[TextData[{StyleBox[\"Styled\", FontSlant -> \"Italic\"], \" message\"}], \"ChatInput\"]`` \
 becomes ``Styled message``.";
 
+$basePromptComponents[ "SpecialURI" ] = "\
+* You will occasionally see markdown links with special URI schemes, e.g. ![label](scheme://content-id) that represent \
+interactive interface elements. You can use these in your responses to display the same elements to the user.";
+
+$basePromptComponents[ "SpecialURIAudio" ] = "\
+    * ![label](audio://content-id) represents an interactive audio player.";
+
+$basePromptComponents[ "SpecialURIVideo" ] = "\
+    * ![label](video://content-id) represents an interactive video player.";
+
+$basePromptComponents[ "SpecialURIDynamic" ] = "\
+    * ![label](dynamic://content-id) represents an embedded dynamic UI.";
+
 $basePromptComponents[ "VisibleUserInput" ] = "\
 * The user can still see their input, so there's no need to repeat it in your response";
 

Source/Chatbook/Sandbox.wl

@@ -32,9 +32,14 @@ $cloudLineNumber           = 1;
 
 (* Tests for expressions that lose their initialized status when sending over a link: *)
 $initializationTests = HoldComplete[
+    AudioQ,
+    BoundaryMeshRegionQ,
     DateObjectQ,
     GraphQ,
-    SparseArrayQ
+    MeshRegionQ,
+    SparseArrayQ,
+    TreeQ,
+    VideoQ
 ];
 
 
@@ -670,6 +675,14 @@ sandboxResultString[ HoldComplete[ ] ] := "Null";
 
 sandboxResultString // endDefinition;
 
+(* ::**************************************************************************************************************:: *)
+(* ::Subsubsection::Closed:: *)
+(*initializedQ*)
+initializedQ // beginDefinition;
+initializedQ // Attributes = { HoldAllComplete };
+initializedQ[ expr_ ] := $initializationTest @ Unevaluated @ expr;
+initializedQ // endDefinition;
+
 (* ::**************************************************************************************************************:: *)
 (* ::Subsubsection::Closed:: *)
 (*simpleResultQ*)
@@ -683,7 +696,7 @@ simpleResultQ // endDefinition;
 (*fancyResultQ*)
 fancyResultQ // beginDefinition;
 fancyResultQ // Attributes = { HoldAllComplete };
-fancyResultQ[ _Manipulate|_DynamicModule ] := True;
+fancyResultQ[ _Manipulate|_DynamicModule|_Video|_Audio|_Tree ] := True;
 fancyResultQ[ gfx_ ] := graphicsQ @ Unevaluated @ gfx;
 fancyResultQ // endDefinition;
 

Source/Chatbook/Serialization.wl

@@ -79,7 +79,7 @@ $defaultWindowWidth = 625;
 $maxMarkdownBoxes = 5;
 
 (* Whether to generate a transcript and preview images for Video[...] expressions: *)
-$serializeVideo = False;
+$generateVideoPrompt = False;
 
 (* Whether to collect data that can help discover missing definitions *)
 $CellToStringDebug = False;
@@ -666,17 +666,27 @@ rasterizeGraphics // endDefinition;
 (* ::**************************************************************************************************************:: *)
 (* ::Subsubsubsection::Closed:: *)
 (*Video*)
-fasterCellToString0[ box: TemplateBox[ _, "VideoBox2", ___ ] ] /; $multimodalMessages && $serializeVideo :=
-    With[ { video = ToExpression[ box, StandardForm ] },
-        serializeVideo @ video /; VideoQ @ video
-    ];
+fasterCellToString0[ box: TemplateBox[ _, "VideoBox2", ___ ] ] /; $multimodalMessages && $generateVideoPrompt :=
+    generateVideoPrompt @ box;
+
+fasterCellToString0[ box: TemplateBox[ _, "VideoBox2", ___ ] ] :=
+    serializeVideo @ box;
 
 (* ::**************************************************************************************************************:: *)
 (* ::Subsubsubsubsection::Closed:: *)
-(*serializeVideo*)
-serializeVideo // beginDefinition;
+(*generateVideoPrompt*)
+generateVideoPrompt // beginDefinition;
+
+generateVideoPrompt[ box: TemplateBox[ _, "VideoBox2", ___ ] ] := generateVideoPrompt[ box ] =
+    With[ { video = Quiet @ ToExpression[ box, StandardForm ] },
+        If[ VideoQ @ video,
+            generateVideoPrompt @ video,
+            "\\!\\(\\*VideoBox[...]\\)"
+        ]
+    ];
 
-serializeVideo[ video_? VideoQ ] := Enclose[
+
+generateVideoPrompt[ video_? VideoQ ] := Enclose[
     Module[ { small, audio, transcript, w, h, t, d, frames, preview },
 
         small      = ConfirmBy[ ImageResize[ video, { UpTo[ 150 ], UpTo[ 150 ] } ], VideoQ, "Resize" ];
@@ -690,18 +700,59 @@ serializeVideo[ video_? VideoQ ] := Enclose[
         frames     = ConfirmMatch[ VideoExtractFrames[ small, t ], { __Image }, "Frames" ];
         preview    = ToBoxes @ ConfirmBy[ ImageAssemble[ Partition[ frames, w ], Spacings -> 3 ], ImageQ, "Assemble" ];
 
-        serializeVideo[ video ] = StringJoin[
+        StringJoin[
             "VIDEO TRANSCRIPT\n-----\n",
             transcript,
             "\n\nVIDEO PREVIEW\n-----\n",
             ConfirmBy[ toMarkdownImageBox @ preview, StringQ, "Preview" ]
         ]
     ],
-    throwInternalFailure[ serializeVideo @ video, ##1 ] &
+    throwInternalFailure
+];
+
+generateVideoPrompt // endDefinition;
+
+(* ::**************************************************************************************************************:: *)
+(* ::Subsubsubsubsection::Closed:: *)
+(*serializeVideo*)
+serializeVideo // beginDefinition;
+
+serializeVideo[ box: TemplateBox[ _, "VideoBox2", ___ ] ] := serializeVideo[ box ] =
+    serializeVideo[ box, Quiet @ ToExpression[ box, StandardForm ] ];
+
+serializeVideo[ box_, video_ ] := Enclose[
+    If[ VideoQ @ video,
+        "\\!\\(\\*VideoBox[\"" <> ConfirmBy[ MakeExpressionURI @ video, StringQ, "URI" ] <> "\"]\\)",
+        "\\!\\(\\*VideoBox[...]\\)"
+    ],
+    throwInternalFailure
 ];
 
 serializeVideo // endDefinition;
 
+(* ::**************************************************************************************************************:: *)
+(* ::Subsubsubsection::Closed:: *)
+(*Audio*)
+fasterCellToString0[ box: TagBox[ _, _Audio`AudioBox, ___ ] ] := serializeAudio @ box;
+
+(* ::**************************************************************************************************************:: *)
+(* ::Subsubsubsubsection::Closed:: *)
+(*serializeAudio*)
+serializeAudio // beginDefinition;
+
+serializeAudio[ box: TagBox[ content_, _Audio`AudioBox, ___ ] ] := serializeAudio[ box ] =
+    serializeAudio[ content, Quiet @ ToExpression[ box, StandardForm ] ];
+
+serializeAudio[ content_, audio_ ] := Enclose[
+    If[ AudioQ @ audio,
+        "\\!\\(\\*AudioBox[\"" <> ConfirmBy[ MakeExpressionURI @ audio, StringQ, "URI" ] <> "\"]\\)",
+        "\\!\\(\\*AudioBox[...]\\)"
+    ],
+    throwInternalFailure
+];
+
+serializeAudio // endDefinition;
+
 (* ::**************************************************************************************************************:: *)
 (* ::Subsubsubsection::Closed:: *)
 (*Template Boxes*)
@@ -1022,11 +1073,13 @@ fasterCellToString0[ cell: Cell[ a_, ___ ] ] :=
         fasterCellToString0 @ a
     ];
 
-fasterCellToString0[ InterpretationBox[ _, expr_, ___ ] ] :=
+fasterCellToString0[ InterpretationBox[ _, expr_, ___ ] ] := Quiet[
     With[ { held = replaceCellContext @ HoldComplete @ expr },
         needsBasePrompt[ "WolframLanguage" ];
         Replace[ held, HoldComplete[ e_ ] :> inputFormString @ Unevaluated @ e ]
-    ];
+    ],
+    Rule::rhs
+];
 
 fasterCellToString0[ Cell[ TextData @ { _, _, text_String, _, Cell[ _, "ExampleCount", ___ ] }, ___ ] ] :=
     fasterCellToString0 @ text;

Source/Chatbook/Tools.wl

@@ -1591,11 +1591,11 @@ Basic Examples
 [system]
 Out[n]= Piecewise[...]
 
-![Formatted Result](expression://result-{id})
+![Formatted Result](expression://content-{id})
 
 [assistant]
 The half-order fractional derivative of $x^n$ with respect to $x$ is given by:
-![Fractional Derivative](expression://result-{id})
+![Fractional Derivative](expression://content-{id})
 ";
 
 (* ::**************************************************************************************************************:: *)
@@ -1612,11 +1612,11 @@ Plot sin(x) from -5 to 5
 ], "
 
 [system]
-Out[n]= ![image](attachment://result-{id})
+Out[n]= ![image](attachment://content-{id})
 
 [assistant]
 Here's the plot of $\\sin{x}$ from -5 to 5:
-![Plot](attachment://result-{id})"
+![Plot](attachment://content-{id})"
 ];
 
 (* ::**************************************************************************************************************:: *)
@@ -1645,7 +1645,8 @@ The temporary directory is located at C:\\Users\\UserName\\AppData\\Local\\Temp.
 (* ::**************************************************************************************************************:: *)
 (* ::Section::Closed:: *)
 (*Expression URIs*)
-$$expressionScheme = "attachment"|"expression";
+$expressionSchemes = { "attachment", "audio", "dynamic", "expression", "video" };
+$$expressionScheme = Alternatives @@ $expressionSchemes;
 
 Chatbook::URIUnavailable = "The expression URI `1` is no longer available.";
 
@@ -1813,7 +1814,7 @@ makeExpressionURI[ scheme_, Automatic, expr_ ] :=
     makeExpressionURI[ scheme, expressionURILabel @ expr, Unevaluated @ expr ];
 
 makeExpressionURI[ scheme_, label_, expr_ ] :=
-    With[ { id = "result-" <> Hash[ Unevaluated @ expr, Automatic, "HexString" ] },
+    With[ { id = "content-" <> Hash[ Unevaluated @ expr, Automatic, "HexString" ] },
         $attachments[ id ] = HoldComplete @ expr;
         "![" <> TextString @ label <> "](" <> TextString @ scheme <> "://" <> id <> ")"
     ];
@@ -1825,17 +1826,42 @@ makeExpressionURI // endDefinition;
 (*expressionURILabel*)
 expressionURILabel // beginDefinition;
 expressionURILabel // Attributes = { HoldAllComplete };
-expressionURILabel[ _Graphics|_Graphics3D|_Image|_Image3D|_Legended|_RawBoxes ] := "image";
-expressionURILabel[ _List|_Association ] := "data";
-expressionURILabel[ _ ] := "result";
+
+(* Audio *)
+expressionURILabel[ Audio[ path_String, ___ ] ] := "Audio Player: " <> path;
+expressionURILabel[ Audio[ File[ path_String ], ___ ] ] := "Audio Player: " <> path;
+expressionURILabel[ _Audio ] := "Embedded Audio Player";
+
+(* Video *)
+expressionURILabel[ Video[ path_String, ___ ] ] := "Video Player: " <> path;
+expressionURILabel[ Video[ File[ path_String ], ___ ] ] := "Video Player: " <> path;
+expressionURILabel[ _Video ] := "Embedded Video Player";
+
+(* Dynamic *)
+expressionURILabel[ _Manipulate ] := "Embedded Interactive Content";
+
+(* Graphics *)
+expressionURILabel[ _Graph|_Graph3D ] := "Graph";
+expressionURILabel[ _Tree ] := "Tree";
+expressionURILabel[ _Graphics|_Graphics3D|_Image|_Image3D|_Legended|_RawBoxes ] := "Image";
+
+(* Data *)
+expressionURILabel[ _List|_Association ] := "Data";
+
+(* Other *)
+expressionURILabel[ _ ] := "Content";
+
 expressionURILabel // endDefinition;
 
 (* ::**************************************************************************************************************:: *)
 (* ::Subsubsubsection::Closed:: *)
 (*expressionURIScheme*)
 expressionURIScheme // beginDefinition;
 expressionURIScheme // Attributes = { HoldAllComplete };
-expressionURIScheme[ _Graphics|_Graphics3D|_Image|_Image3D|_Legended|_RawBoxes ] := "attachment";
+expressionURIScheme[ _Video ] := (needsBasePrompt[ "SpecialURIVideo" ]; "video");
+expressionURIScheme[ _Audio ] := (needsBasePrompt[ "SpecialURIAudio" ]; "audio");
+expressionURIScheme[ _Manipulate|_DynamicModule|_Dynamic ] := (needsBasePrompt[ "SpecialURIDynamic" ]; "dynamic");
+expressionURIScheme[ _Graph|_Graph3D|_Graphics|_Graphics3D|_Image|_Image3D|_Legended|_Tree|_RawBoxes ] := "attachment";
 expressionURIScheme[ _ ] := "expression";
 expressionURIScheme // endDefinition;