improvement(search): require half the query's terms in Jira partial matches (#8987)

* improvement(search): keep a Jira partial match only when it holds half the query's terms

The any-term retry ORs pairs of terms, so on a large Jira site it returns tickets sharing only a couple of common words, which then ranked alongside every other source's best results. Jira partial matches must now contain at least half the query's distinct terms in their title or description, a word and its regular plural counting as one as Jira's stemmed search matches them (minimum-should-match over the relaxed tier). Confluence's relevance-ranked site search is unchanged. When no partial match qualifies, the all-terms page is returned as it was before the retry existed.

* improvement(search): one provider setting for the any-term retry

Replace the parallel requiresEveryTerm and partialMatchesNeedHalfTheTerms flags and their two lookups with a single anyTermRetry setting that carries the fallback's options.

* fix(search): compare query and ticket words by the same singular key in the Jira floor

Rebuilding plural forms from a word's key missed singular words whose plural adds -es (release, database, license): their key drops the final e, so neither the word nor its plural was found and the floor dropped valid partial matches. Both sides now go through the same key, as an analyzer does for query and document, with a term containing punctuation matched as written.

* refactor(search): apply the Jira half-the-terms floor in the Jira adapter

The floor tightens Jira's own any-term query, so it now sits beside that query in the Jira branch of searchAtlassian. The shared any-term fallback and the provider registry are back to their previous shape: no option, flag, or exported type. The term test is built once per search from the query, splits words with the shared WORD_CHARACTER definition, and is covered directly in text-match tests, with the adapter tests covering the wiring and Confluence staying untouched.

* chore(search): terse floor comment, guard the query-side plural fold

* fix(search): measure Jira partial-match coverage on the summary and description only

The issue mapper appends a Status line, so a query containing 'status' gained a term from the label alone. The floor now reads the raw summary and description fields before mapping.

Author: waleedlatif1

apps/sim/lib/knowledge/search/snippet.ts

@@ -16,6 +16,12 @@ const HEADER_LINE = /^[A-Z][A-Za-z-]{1,15}: .*$/
  */
 const WORD_CHARACTER =
   /(?![\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Hangul}\p{sc=Thai}])[\p{L}\p{N}_]/u
+const WORD_RUN = new RegExp(`(?:${WORD_CHARACTER.source})+`, 'gu')
+
+/** The words of a text, split where a run of `WORD_CHARACTER`s ends. */
+export function textWords(text: string): string[] {
+  return text.match(WORD_RUN) ?? []
+}
 
 /**
  * The document text without the header block some connectors prefix (the

apps/sim/lib/sim-search/live/atlassian.test.ts

@@ -197,3 +197,51 @@ describe('Confluence live documents', () => {
     expect(await previewConfluence(page('<p>Rollout steps</p>'), document)).toBe('Rollout steps')
   })
 })
+
+describe('Jira any-term search', () => {
+  const issues = {
+    '/ex/jira/cloud/rest/api/3/search/jql': {
+      issues: [
+        { key: 'ENG-1', fields: { summary: 'Token creator role request' } },
+        { key: 'ENG-2', fields: { summary: 'Role request form' } },
+      ],
+    },
+  }
+  const query = { query: 'glean token creator role request', limit: 25, scopes: [] }
+
+  it('keeps only issues holding at least half the query terms', async () => {
+    const page = await searchAtlassian(client(issues), 'jira', { ...query, textMatch: 'any' })
+    expect(page.documents.map((document) => document.id)).toEqual(['ENG-1'])
+  })
+
+  it('counts only the summary and description, not the added status line', async () => {
+    const api = client({
+      '/ex/jira/cloud/rest/api/3/search/jql': {
+        issues: [{ key: 'ENG-3', fields: { summary: 'Release update', status: { name: 'Open' } } }],
+      },
+    })
+    const page = await searchAtlassian(api, 'jira', {
+      query: 'release status update owner timeline',
+      limit: 25,
+      scopes: [],
+      textMatch: 'any',
+    })
+    expect(page.documents).toEqual([])
+  })
+
+  it('keeps every issue an all-terms search returns', async () => {
+    const page = await searchAtlassian(client(issues), 'jira', query)
+    expect(page.documents.map((document) => document.id)).toEqual(['ENG-1', 'ENG-2'])
+  })
+
+  it('leaves a Confluence any-term search to its relevance ranking', async () => {
+    const api = client({
+      '/ex/confluence/cloud/wiki/rest/api/search': {
+        results: [{ content: { id: '5', type: 'page', title: 'Role request form' } }],
+        _links: {},
+      },
+    })
+    const page = await searchAtlassian(api, 'confluence', { ...query, textMatch: 'any' })
+    expect(page.documents.map((document) => document.id)).toEqual(['5'])
+  })
+})

apps/sim/lib/sim-search/live/atlassian.ts

@@ -9,7 +9,11 @@ import {
 } from '@/lib/sim-search/live/http'
 import { interleaveByRank } from '@/lib/sim-search/live/pages'
 import { providerText } from '@/lib/sim-search/live/text'
-import { confluenceTextClause, jiraTextClause } from '@/lib/sim-search/live/text-match'
+import {
+  confluenceTextClause,
+  holdsHalfTheTerms,
+  jiraTextClause,
+} from '@/lib/sim-search/live/text-match'
 import type {
   NativeClient,
   NativeDocument,
@@ -152,6 +156,9 @@ export async function searchAtlassian(
   const direction = dateSortDirection(input.filters)
   const order = direction ? `${field} ${direction.toUpperCase()}` : undefined
   const text = nativeText(input)
+  // Jira's any-term JQL matches issues sharing two common words; like minimum-should-match,
+  // keep only issues holding at least half the terms.
+  const relevant = input.textMatch === 'any' && text ? holdsHalfTheTerms(text) : undefined
   const pages = await Promise.all(
     selected.map(async (site) => {
       const cloudId = string(site.id)
@@ -173,7 +180,15 @@ export async function searchAtlassian(
           })
         )
         return {
-          documents: array(data.issues).map((row) => issue(row, cloudId, origin)),
+          documents: array(data.issues)
+            .filter((row) => {
+              const fields = object(row.fields)
+              return (
+                !relevant ||
+                relevant(`${string(fields.summary)}\n${textContent(fields.description)}`)
+              )
+            })
+            .map((row) => issue(row, cloudId, origin)),
           excluded: false,
           next: string(data.nextPageToken) || undefined,
         }

apps/sim/lib/sim-search/live/text-match.test.ts

@@ -1,5 +1,9 @@
 import { describe, expect, it } from 'vitest'
-import { confluenceTextClause, jiraTextClause } from '@/lib/sim-search/live/text-match'
+import {
+  confluenceTextClause,
+  holdsHalfTheTerms,
+  jiraTextClause,
+} from '@/lib/sim-search/live/text-match'
 
 describe('text match clauses', () => {
   it('matches either term of a two-term query, leaving out stopwords', () => {
@@ -57,4 +61,30 @@ describe('text match clauses', () => {
   it('falls back to the all-terms clause when fewer than two terms are distinctive', () => {
     expect(jiraTextClause('how does it work', 'any')).toBe('text ~ "how does it work"')
   })
+
+  describe('holdsHalfTheTerms', () => {
+    it('needs at least half the distinct terms, rounding up', () => {
+      const holds = holdsHalfTheTerms('glean token creator role request')
+      expect(holds('Role request form')).toBe(false)
+      expect(holds('Token creator role')).toBe(true)
+    })
+
+    it('counts a word and its plural as one term in either direction', () => {
+      expect(holdsHalfTheTerms('glean token creator role request')('Glean tokens, creators')).toBe(
+        true
+      )
+      expect(
+        holdsHalfTheTerms('database release checklist owner')('Databases and weekly releases')
+      ).toBe(true)
+      expect(holdsHalfTheTerms('databases releases checklist owner')('Database release')).toBe(true)
+    })
+
+    it('matches a term with punctuation inside it as written', () => {
+      expect(holdsHalfTheTerms('ci/cd pipeline failure owner')('CI/CD pipeline')).toBe(true)
+    })
+
+    it('accepts one matching term for a two-term query', () => {
+      expect(holdsHalfTheTerms('contractor offboarding')('Contractor directory')).toBe(true)
+    })
+  })
 })

apps/sim/lib/sim-search/live/text-match.ts

@@ -1,4 +1,4 @@
-import { findTermMatches, queryTerms } from '@/lib/knowledge/search/snippet'
+import { findTermMatches, queryTerms, textWords } from '@/lib/knowledge/search/snippet'
 
 /**
  * How a plain query's words must match a document. Confluence and Jira text search require every
@@ -115,3 +115,21 @@ export function confluenceTextClause(text: string, match: TextMatch = 'all'): st
 export function termCoverage(text: string, terms: readonly string[]): number {
   return terms.filter((term) => findTermMatches(text, [term]).length > 0).length
 }
+
+/**
+ * A test for whether a text holds at least half the query's distinct terms. Query and text words
+ * are compared by their singular key, so a word and its regular plural count as one term as a
+ * stemmed search matches them; a term with punctuation inside it is matched as written.
+ */
+export function holdsHalfTheTerms(query: string): (text: string) => boolean {
+  const terms = distinctTerms(query, singularKey)
+  const keys = terms.map(singularKey)
+  const required = Math.ceil(terms.length / 2)
+  return (text) => {
+    const words = new Set(textWords(text).map(singularKey))
+    const present = terms.filter(
+      (term, index) => words.has(keys[index]) || findTermMatches(text, [term]).length > 0
+    )
+    return present.length >= required
+  }
+}