[MLOB-1561] LLM Observability SDK API

docs/package.json

@@ -11,6 +11,6 @@
   "private": true,
   "devDependencies": {
     "typedoc": "^0.25.8",
-    "typescript": "^4.6"
+    "typescript": "^5.0"
   }
 }

docs/test.ts

@@ -536,3 +536,105 @@ const otelTraceId: string = spanContext.traceId
 const otelSpanId: string = spanContext.spanId
 const otelTraceFlags: number = spanContext.traceFlags
 const otelTraceState: opentelemetry.TraceState = spanContext.traceState!
+
+// -- LLM Observability --
+const llmobsEnableOptions = {
+  apiKey: 'dd-api-key',
+  mlApp: 'mlApp',
+  agentlessEnabled: true
+}
+tracer.init({
+  llmobs: llmobsEnableOptions,
+  experimental: {
+    llmobs: llmobsEnableOptions
+  }
+})
+const llmobs = tracer.llmobs
+const enabled = llmobs.enabled
+
+// manually enable
+llmobs.enable({
+  apiKey: 'dd-api-key',
+  mlApp: 'mlApp',
+  agentlessEnabled: true
+})
+
+// manually disable
+llmobs.disable()
+
+// start span in current context
+llmobs.startSpan({ name: 'name', kind: 'llm' })
+llmobs.startSpan({ name: 'name', kind: 'embedding'})
+llmobs.startSpan({ name: 'name', kind: 'agent'})
+llmobs.startSpan({ name: 'name', kind: 'retrieval'})
+llmobs.startSpan({ name: 'name', kind: 'task'})
+llmobs.startSpan({ name: 'name', kind: 'tool'})
+llmobs.startSpan({ name: 'name', kind: 'workflow'})
+
+// trace block of code
+llmobs.trace({ name: 'name', kind: 'llm' }, () => {})
+llmobs.trace({ kind: 'llm', name: 'myLLM', modelName: 'myModel', modelProvider: 'myProvider' }, () => {})
+llmobs.trace({ name: 'name', kind: 'llm' }, (span, cb) => {
+  llmobs.annotate(span, {})
+  span.setTag('foo', 'bar')
+  cb(new Error('boom'))
+})
+
+// wrap a function
+llmobs.wrap({ kind: 'llm' }, function myLLM () {
+  const s = llmobs.active()
+  llmobs.annotate(s, {})
+})()
+llmobs.wrap({ kind: 'llm', name: 'myLLM', modelName: 'myModel', modelProvider: 'myProvider' }, function myFunction () {})()
+
+// decorate a function
+class MyClass {
+  @llmobs.decorate({ kind: 'llm' })
+  myLLM () {}
+
+  @llmobs.decorate({ kind: 'llm', name: 'myOtherLLM', modelName: 'myModel', modelProvider: 'myProvider' })
+  myOtherLLM () {}
+}
+
+const cls = new MyClass()
+cls.myLLM()
+cls.myOtherLLM()
+
+// export a span
+llmobs.enable({ mlApp: 'myApp' })
+llmobs.exportSpan()
+const llmobsSpanCtx = llmobs.exportSpan(llmobs.startSpan({ name: 'name', kind: 'llm' }))
+llmobsSpanCtx.traceId;
+llmobsSpanCtx.spanId;
+
+// annotate a span
+llmobs.annotate({
+  inputData: 'input',
+  outputData: 'output',
+  metadata: {},
+  metrics: {},
+  tags: {}
+})
+llmobs.annotate(span, {
+  inputData: 'input',
+  outputData: 'output',
+  metadata: {},
+  metrics: {},
+  tags: {}
+})
+
+// submit evaluation
+llmobs.disable()
+llmobs.submitEvaluation(llmobsSpanCtx, {
+  label: 'my-eval-metric',
+  metricType: 'categorical',
+  value: 'good',
+  mlApp: 'myApp',
+  tags: {},
+  timestampMs: Date.now()
+})
+
+// flush
+llmobs.flush()
+
+const llmobsSpan: Span | undefined = llmobs.active()

docs/tsconfig.json

@@ -4,7 +4,7 @@
     "moduleResolution": "node",
     "module": "commonjs",
     "baseUrl": ".",
-    "strict": true
+    "strict": true,
   },
   "files": [
     "../index.d.ts"

docs/yarn.lock

@@ -61,10 +61,10 @@ typedoc@^0.25.8:
     minimatch "^9.0.3"
     shiki "^0.14.7"
 
-typescript@^4.6:
-  version "4.9.5"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a"
-  integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==
+typescript@^5.0:
+  version "5.6.3"
+  resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.6.3.tgz#5f3449e31c9d94febb17de03cc081dd56d81db5b"
+  integrity sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==
 
 vscode-oniguruma@^1.7.0:
   version "1.7.0"