chore: improve json fixing

yiwen-ai · Nov 27, 2023 · 2a06607 · 2a06607
1 parent acf5292
commit 2a06607
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 13 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "jarvis"
-version = "1.1.4"
+version = "1.1.5"
 edition = "2021"
 rust-version = "1.64"
 description = ""

diff --git a/src/json_util.rs b/src/json_util.rs
@@ -58,6 +58,7 @@ impl RawJSONArray {
         }
     }
 
+    // return error message if failed
     fn array(&mut self) -> Option<String> {
         self.result.push('[');
         self.offset += 1;
@@ -102,7 +103,9 @@ impl RawJSONArray {
 
             self.skip_space();
             if self.offset >= self.chars.len() {
-                return Some("no token to scan in array".to_string());
+                self.result.push(']');
+                self.offset += 1;
+                return None;
             }
 
             match self.chars[self.offset] {
@@ -313,6 +316,19 @@ mod tests {
                 output: r#"[[],["] Stream: ["],["Internet Engineering Task Force \\(IETF)"]]"#.to_string(),
                 err: None,
             },
+            Case {
+                input: r#"[
+                    [],
+                    [
+                        ""] Stream: ["
+                    ],
+                    [
+                        "Internet Engineering Task Force \(IETF)"
+                ]"#
+                .to_string(),
+                output: r#"[[],["] Stream: ["],["Internet Engineering Task Force \\(IETF)"]]"#.to_string(),
+                err: None,
+            },
             Case {
                 input: r#"[["作为UTF-8 [","RFC3629",""]编码的文本字符串（","第2节", "）。字符串中的字节数等于参数。包含无效UTF-8序列的字符串是格式良好但无效的（","第1.2节", "）。此类型适用于需要解释或显示人类可读文本的系统，并允许区分结构化字节和具有指定曲目（Unicode）和编码（UTF-8）的文本。与JSON等格式不同，此类型中的Unicode字符永远不会被转义。因此，换行符（U+000A）始终表示为字符串中的字节0x0a，而不是字符0x5c6e（字符“\”和“n”）或0x5c7530303061（字符“\”，“u”，“0”，“0”，“0”和“a”）。","¶"]]"#
                 .to_string(),
@@ -333,7 +349,7 @@ mod tests {
                     assert_eq!(val, case.output);
                 }
                 Err(err) => {
-                    // println!("FIX_ERR:  `{}` => `{}`", case.input, err);
+                    println!("FIX_ERR:  `{}` => `{}`", case.input, err);
                     assert!(case.err.is_some());
                     assert!(err.contains::<&str>(case.err.unwrap().as_ref()));
                 }

diff --git a/src/openai.rs b/src/openai.rs
@@ -311,9 +311,7 @@ impl OpenAI {
                 Err(er) => {
                     ctx.set_kvs(vec![
                         ("json_fixed", false.into()),
-                        ("json_input", text.clone().into()),
-                        ("json_output", oc.clone().into()),
-                        ("json_error", er.into()),
+                        ("json_fix_error", er.into()),
                     ])
                     .await;
                 }
@@ -492,8 +490,8 @@ impl OpenAI {
         let mut req_body = CreateChatCompletionRequestArgs::default()
             .max_tokens(model.max_tokens() as u16)
             .model(&model_name)
-            .temperature(0.2f32)
-            .top_p(0.9f32)
+            .temperature(0.1f32)
+            .top_p(0.618f32)
             .messages(messages)
             .build()
             .map_err(HTTPError::with_500)?;
@@ -586,8 +584,8 @@ impl OpenAI {
 
         let mut req_body = CreateChatCompletionRequestArgs::default()
             .max_tokens(800u16)
-            .temperature(0.3f32)
-            .top_p(0.95f32)
+            .temperature(0.382f32)
+            .top_p(0.618f32)
             .model(&model_name)
             .messages(messages)
             .build()
@@ -723,8 +721,8 @@ impl OpenAI {
 
         let mut req_body = CreateChatCompletionRequestArgs::default()
             .max_tokens(256u16)
-            .temperature(0.7f32)
-            .top_p(1f32)
+            .temperature(0.1f32)
+            .top_p(0.618f32)
             .model(&model_name)
             .messages(messages)
             .build()