We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 6d68252 commit 0fad66cCopy full SHA for 0fad66c
crates/bpe-openai/src/lib.rs
@@ -78,7 +78,7 @@ mod tests {
78
.map(|i| i as u32)
79
.collect();
80
81
- let without_splitting = BPE_CL100K.encode_via_backtracking(&input);
+ let without_splitting = BPE_CL100K.encode_via_backtracking(input);
82
assert_ne!(without_splitting, expected);
83
84
let pat = "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+";
0 commit comments