Add syntax to let the AI insert images inline in the evaluator and markdown code blocks

Source/Chatbook/Common.wl

@@ -41,6 +41,7 @@ BeginPackage[ "Wolfram`Chatbook`Common`" ];
 `catchTop;
 `catchTopAs;
 `endDefinition;
+`endExportedDefinition;
 `importResourceFunction;
 `messageFailure;
 `messagePrint;
@@ -137,6 +138,7 @@ KeyValueMap[ Function[ MessageName[ Chatbook, #1 ] = #2 ], <|
     "Internal"                        -> "An unexpected error occurred. `1`",
     "InvalidAPIKey"                   -> "Invalid value for API key: `1`",
     "InvalidArguments"                -> "Invalid arguments given for `1` in `2`.",
+    "InvalidExpressionURI"            -> "The string \"`1`\" is not a valid expression URI.",
     "InvalidFrontEndScope"            -> "The value `1` is not a valid scope for `2`.",
     "InvalidFunctions"                -> "Invalid setting for ProcessingFunctions: `1`; using defaults instead.",
     "InvalidHandlerArguments"         -> "Invalid value for $ChatHandlerData: `1`; resetting to default value.",
@@ -305,6 +307,30 @@ appendFallthroughError0[ s_Symbol, OwnValues  ] := e: HoldPattern @ s
 appendFallthroughError0[ s_Symbol, DownValues ] := e: HoldPattern @ s[ ___ ]        := throwInternalFailure @ e;
 appendFallthroughError0[ s_Symbol, UpValues   ] := e: HoldPattern @ s[ ___ ][ ___ ] := throwInternalFailure @ e;
 
+(* ::**************************************************************************************************************:: *)
+(* ::Subsubsection::Closed:: *)
+(*appendExportedFallthroughError*)
+appendExportedFallthroughError // ClearAll;
+appendExportedFallthroughError // Attributes = { HoldFirst };
+
+appendExportedFallthroughError[ s_Symbol ] :=
+    Module[ { block = Internal`InheritedBlock, before, after },
+        block[ { s },
+            before = DownValues @ s;
+            appendExportedFallthroughError0 @ s;
+            after = DownValues @ s;
+        ];
+
+        If[ TrueQ[ Length @ after > Length @ before ],
+            DownValues[ s ] = after,
+            DownValues[ s ]
+        ]
+    ];
+
+appendExportedFallthroughError0 // ClearAll;
+appendExportedFallthroughError0[ f_Symbol ] := f[ a___ ] :=
+    catchTop[ throwFailure[ "InvalidArguments", f, HoldForm @ f @ a ], f ];
+
 (* ::**************************************************************************************************************:: *)
 (* ::Subsection::Closed:: *)
 (*endDefinition*)
@@ -326,6 +352,21 @@ endDefinition[ s_Symbol, list_List ] := (endDefinition[ s, #1 ] &) /@ list;
 
 endDefinition // endDefinition;
 
+(* ::**************************************************************************************************************:: *)
+(* ::Subsection::Closed:: *)
+(*endExportedDefinition*)
+endExportedDefinition // beginDefinition;
+endExportedDefinition // Attributes = { HoldFirst };
+
+endExportedDefinition[ s_Symbol ] :=
+    WithCleanup[
+        optimizeEnclosures @ s;
+        appendExportedFallthroughError @ s,
+        $inDef = False
+    ];
+
+endExportedDefinition // endDefinition;
+
 (* ::**************************************************************************************************************:: *)
 (* ::Section::Closed:: *)
 (*Resource Functions*)

Source/Chatbook/Formatting.wl

@@ -26,6 +26,7 @@ Begin[ "`Private`" ];
 Needs[ "Wolfram`Chatbook`"          ];
 Needs[ "Wolfram`Chatbook`Common`"   ];
 Needs[ "Wolfram`Chatbook`FrontEnd`" ];
+Needs[ "Wolfram`Chatbook`Sandbox`"  ];
 Needs[ "Wolfram`Chatbook`Tools`"    ];
 
 (* FIXME: Use ParagraphSpacing to squeeze text closer together *)
@@ -1387,7 +1388,7 @@ makeInteractiveCodeCell[ language_, code_String ] /; $dynamicText :=
 makeInteractiveCodeCell[ lang_String? wolframLanguageQ, code_ ] :=
     Module[ { display, handler },
         display = RawBoxes @ Cell[
-            BoxData @ If[ StringQ @ code, stringToBoxes @ code, code ],
+            BoxData @ If[ StringQ @ code, wlStringToBoxes @ code, code ],
             "ChatCode",
             "Input",
             Background -> GrayLevel[ 1 ]
@@ -1427,6 +1428,13 @@ makeInteractiveCodeCell[ language_String, code_String ] :=
 
 makeInteractiveCodeCell // endDefinition;
 
+(* ::**************************************************************************************************************:: *)
+(* ::Subsubsection::Closed:: *)
+(*wlStringToBoxes*)
+wlStringToBoxes // beginDefinition;
+wlStringToBoxes[ string_String ] := inlineExpressionURIs @ stringToBoxes @ preprocessSandboxString @ string;
+wlStringToBoxes // endDefinition;
+
 (* ::**************************************************************************************************************:: *)
 (* ::Subsubsection::Closed:: *)
 (*wolframLanguageQ*)

Source/Chatbook/Prompting.wl

@@ -36,10 +36,12 @@ $basePromptOrder = {
     "ConversionLargeOutputs",
     "ConversionGraphics",
     "MarkdownImageBox",
+    "MarkdownImageBoxImporting",
     "Checkboxes",
     "CheckboxesIndeterminate",
     "ConversionFormatting",
     "SpecialURI",
+    "SpecialURIImporting",
     "SpecialURIAudio",
     "SpecialURIVideo",
     "SpecialURIDynamic",
@@ -81,8 +83,10 @@ $basePromptDependencies = Append[ "GeneralInstructionsHeader" ] /@ <|
     "ConversionLargeOutputs"       -> { "MessageConversionHeader" },
     "ConversionGraphics"           -> { "MessageConversionHeader" },
     "MarkdownImageBox"             -> { "MessageConversionHeader" },
+    "MarkdownImageBoxImporting"    -> { "MarkdownImageBox" },
     "ConversionFormatting"         -> { "MessageConversionHeader" },
     "SpecialURI"                   -> { },
+    "SpecialURIImporting"          -> { "SpecialURI" },
     "SpecialURIAudio"              -> { "SpecialURI" },
     "SpecialURIVideo"              -> { "SpecialURI" },
     "SpecialURIDynamic"            -> { "SpecialURI" },
@@ -162,7 +166,7 @@ $$chatIndicatorSymbol$$. Cells appearing above the chat input are included to pr
 but chat inputs represent the actual message from the user to you.";
 
 $basePromptComponents[ "WolframAlphaInputIndicator" ] = "\
-    * Inputs denoted with \[FreeformPrompt] are Wolfram Alpha inputs. When available, the parsed Wolfram \
+	* Inputs denoted with \[FreeformPrompt] are Wolfram Alpha inputs. When available, the parsed Wolfram \
 Language code will be included on the next line.";
 
 $basePromptComponents[ "ConversionLargeOutputs" ] = "\
@@ -172,17 +176,23 @@ $basePromptComponents[ "ConversionGraphics" ] = "\
 	* Rendered graphics will typically be replaced with a shortened box representation: \\!\\(\\*GraphicsBox[<<>>]\\)";
 
 $basePromptComponents[ "MarkdownImageBox" ] = "\
-	* If there are images embedded in the notebook, they will be replaced by a box representation in the \
-form ``MarkdownImageBox[\"![label](uri)\"]``. You will also receive the original image immediately after this. \
-You can use the markdown from this box ``![label](uri)`` in your responses if you want to display the original image.";
+	* If there are images embedded in the notebook, they will be replaced by a box representation in the form \
+``MarkdownImageBox[\"![label](attachment://content-id)\"]``. You will also receive the original image immediately \
+after this. You can use the markdown from this box ``![label](attachment://content-id)`` in your responses if you \
+want to display the original image.";
+
+$basePromptComponents[ "MarkdownImageBoxImporting" ] = "\
+		* Use the syntax <!attachment://content-id!> to inline one of these images in code you write for the evaluator \
+tool. For example, ``ColorNegate[<!attachment://content-id!>]``. The expression will be inserted in place. Do not \
+include the MarkdownImageBox wrapper. You can also use this syntax to inline images into WL code blocks.";
 
 $basePromptComponents[ "Checkboxes" ] = "\
-    * Checkboxes in the UI will be replaced with one of the following text representations:
-        * A Checkbox that's selected becomes ``[\[Checkmark]]``
-        * A Checkbox that's not selected becomes ``[ ]``";
+	* Checkboxes in the UI will be replaced with one of the following text representations:
+		* A Checkbox that's selected becomes ``[\[Checkmark]]``
+		* A Checkbox that's not selected becomes ``[ ]``";
 
 $basePromptComponents[ "CheckboxesIndeterminate" ] = "\
-        * An indeterminate Checkbox becomes ``[-]``";
+    	* An indeterminate Checkbox becomes ``[-]``";
 
 $basePromptComponents[ "ConversionFormatting" ] = "\
 	* Cell formatting is converted to markdown where possible, so \
@@ -194,14 +204,18 @@ $basePromptComponents[ "SpecialURI" ] = "\
 interactive interface elements. You can use these in your responses to display the same elements to the user, but they \
 must be formatted as image links (include the '!' at the beginning). If you do not include the '!', the link will fail.";
 
+$basePromptComponents[ "SpecialURIImporting" ] = "\
+	* Use the syntax <!scheme://content-id!> to inline one of these expressions in code you write for the evaluator \
+tool.";
+
 $basePromptComponents[ "SpecialURIAudio" ] = "\
-    * ![label](audio://content-id) represents an interactive audio player.";
+	* ![label](audio://content-id) represents an interactive audio player.";
 
 $basePromptComponents[ "SpecialURIVideo" ] = "\
-    * ![label](video://content-id) represents an interactive video player.";
+	* ![label](video://content-id) represents an interactive video player.";
 
 $basePromptComponents[ "SpecialURIDynamic" ] = "\
-    * ![label](dynamic://content-id) represents an embedded dynamic UI.";
+	* ![label](dynamic://content-id) represents an embedded dynamic UI.";
 
 $basePromptComponents[ "VisibleUserInput" ] = "\
 * The user can still see their input, so there's no need to repeat it in your response";