fix: OPENAI_API_KEY Required Even When Not Using OpenAI Provider with undefined apiKey #135

Author: RPate97

.changeset/cold-cherries-serve.md

@@ -0,0 +1,5 @@
+---
+'token.js': patch
+---
+
+Default to hardcoded empty string for API key in the OpenAI compatible provider if no API key is provided

scripts/example.ts

@@ -1,7 +1,7 @@
 import * as dotenv from 'dotenv'
 import { OpenAI } from 'openai'
 
-import { TokenJS } from '../dist/index.cjs'
+import { TokenJS } from '../src'
 dotenv.config()
 
 const messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = [
@@ -12,10 +12,12 @@ const messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = [
 ]
 
 const callLLM = async () => {
-  const tokenjs = new TokenJS()
+  const tokenjs = new TokenJS({
+    baseURL: 'http://localhost:3000/api/',
+  })
   const result = await tokenjs.chat.completions.create({
     // stream: true,
-    provider: 'gemini',
+    provider: 'openai-compatible',
     model: 'gemini-1.5-pro',
     messages,
   })

src/handlers/openai-compatible.ts

@@ -34,6 +34,30 @@ export class OpenAICompatibleHandler extends BaseHandler<OpenAICompatibleModel>
     }
   }
 
+  determineAPIKey = () => {
+    if (this.opts.apiKey) {
+      return this.opts.apiKey
+    } else if (process.env.OPENAI_COMPATIBLE_API_KEY) {
+      return process.env.OPENAI_COMPATIBLE_API_KEY
+    } else {
+      /**
+       * We hardcode an empty API key if none is defined because the OpenAI SDK throws an error if we do not do this.
+       * There are plenty of reasonable cases where an API is not required by an openai compartible model provider (locally
+       * hosted models for example), so we want to avoid runtime errors in those situations.
+       * See this issue for an example: https://github.com/twinnydotdev/twinny/issues/440
+       *
+       * However, the tradeoff with this is that if the underlying provider requires an API key and the user does not provide one,
+       * they may get an unpredictable error. We deem this tradeoff acceptible in this case because using an unvetted openai-compatible
+       * model provider is inherently less safe than using a provider officially integrated and supported by Token.js. If users often,
+       * report errors related to this, we should consider officially supporting the behavior of the underlying provider that is causing issues.
+       *
+       * For example, we may want to officially support ollama local models if users often report problems related to using that provider via this
+       * generic implementation.
+       */
+      return ''
+    }
+  }
+
   async create(
     body: ProviderCompletionParams<'openai'>
   ): Promise<CompletionResponse | StreamCompletionResponse> {
@@ -42,7 +66,7 @@ export class OpenAICompatibleHandler extends BaseHandler<OpenAICompatibleModel>
     // Uses the OPENAI_API_KEY environment variable, if the apiKey is not provided.
     // This makes the UX better for switching between providers because you can just
     // define all the environment variables and then change the model field without doing anything else.
-    const apiKey = this.opts.apiKey ?? process.env.OPENAI_COMPATIBLE_API_KEY
+    const apiKey = this.determineAPIKey()
     const openai = new OpenAI({
       ...this.opts,
       apiKey,