Skip to content

Commit 8d65aa3

Browse files
Sg312Sg312
committed
grep: resolve --in against listings, fetch only the named resources
A bare --in selector (e.g. --in agent, --in file_v5) materialized every searched world to find one block: 65 block details plus every workflow state per call. On dev that took 18-34s per grep and tripped the per-user rate limit. Each world now has a cheap index (its listing) and a per-resource fetch; a --in search reads the indexes and fetches only the matches. Whole-world searches are unchanged.
1 parent ca4f245 commit 8d65aa3

2 files changed

Lines changed: 204 additions & 109 deletions

File tree

‎apps/sim/lib/mothership/agent-cli/engines/universal-grep.test.ts‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,16 @@ const SLACK_V2 = {
1212
operations: { send_message: { toolId: 'slack_send' } },
1313
}
1414

15-
function runtimeWith(responses: Record<string, unknown>): AgentCliRuntime {
15+
function runtimeWith(
16+
responses: Record<string, unknown>,
17+
requested: string[] = []
18+
): AgentCliRuntime {
1619
return {
1720
workspaceId: `ws-${Math.random().toString(36).slice(2)}`,
1821
userId: 'user-1',
1922
client: {
2023
request: async <T>(path: string): Promise<T> => {
24+
requested.push(path)
2125
const hit = responses[path]
2226
if (hit === undefined) throw new Error(`Unexpected request: ${path}`)
2327
return hit as T
@@ -58,6 +62,32 @@ describe('universal grep', () => {
5862
expect(count.stdout).toMatch(/^\d+ \(blocks=\d+\)$/)
5963
})
6064

65+
it('resolves a bare --in against the listings and fetches only what it names', async () => {
66+
// `grep x --in agent` used to materialize every searched world to find one block —
67+
// 65 block details plus every workflow state; on dev that took 18-34s per call and
68+
// tripped the per-user rate limit. Now the listings resolve the selector and only
69+
// the matching resources are fetched.
70+
const requested: string[] = []
71+
const runtime = runtimeWith(
72+
{
73+
...CATALOG,
74+
'/api/v2/workflows': { data: [{ id: 'wf-1', name: 'Agent runner' }], nextCursor: null },
75+
'/api/v2/workflows/wf-1/state': { data: { blocks: { b1: { type: 'agent' } } } },
76+
},
77+
requested
78+
)
79+
const result = await runEngine('grep', ['id'], runtime, {
80+
scope: 'blocks,workflows',
81+
in: 'agent',
82+
})
83+
expect(result.exitCode).toBe(0)
84+
expect(result.stdout).toContain('blocks/agent:')
85+
expect(requested).toContain('/api/v2/blocks/agent')
86+
expect(requested).not.toContain('/api/v2/blocks/slack_v2')
87+
// A workflow whose NAME contains the selector is a match too, fetched by the same rule.
88+
expect(requested).toContain('/api/v2/workflows/wf-1/state')
89+
})
90+
6191
it('accepts the world/resource path a match line prints as --in', async () => {
6292
const byPath = await runEngine('grep', ['id'], runtimeWith(CATALOG), { in: 'blocks/agent' })
6393
expect(byPath.exitCode).toBe(0)

‎apps/sim/lib/mothership/agent-cli/engines/universal-grep.ts‎

Lines changed: 173 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -109,95 +109,52 @@ function render(scope: Scope, id: string, label: string, value: unknown): Materi
109109
return { scope, id, label, text: `${header}${JSON.stringify(value, null, 2)}` }
110110
}
111111

112-
/** One materializer per scope: the list, then each resource as its `get` returns it. */
113-
const MATERIALIZERS: Record<Scope, (runtime: AgentCliRuntime) => Promise<Materialized[]>> = {
114-
workflows: async (runtime) => {
115-
const list = await listAll(runtime, '/api/v2/workflows')
116-
return mapConcurrent(list, FETCH_CONCURRENCY, async (w) => {
117-
const id = str(w.id) ?? ''
118-
// The draft state, not the export: export is sanitized for sharing and nulls
119-
// workspace-specific fields (a Table block's `tableId`), so a grep for a table id
120-
// inside a workflow would miss it. The state route scopes by workflow id alone
121-
// (`query: noInputSchema`); a workspaceId here is an "Unrecognized key".
122-
const state = await runtime.client.request<{ data: unknown }>(`/api/v2/workflows/${id}/state`)
123-
return render('workflows', id, str(w.name) ?? id, state.data)
124-
})
125-
},
126-
blocks: async (runtime) => {
127-
const key = runtime.workspaceId
128-
const cached = catalogCache.get(key)
129-
if (cached !== undefined) return cached
130-
const list = await listAll(runtime, '/api/v2/blocks')
131-
const materialized = await mapConcurrent(list, FETCH_CONCURRENCY, async (b) => {
132-
const id = str(b.id) ?? ''
133-
const detail = await runtime.client.request<{ data: unknown }>(`/api/v2/blocks/${id}`, {
134-
query: { workspaceId: runtime.workspaceId },
135-
})
136-
return render('blocks', id, id, detail.data)
137-
})
138-
catalogCache.set(key, materialized)
139-
return materialized
140-
},
141-
tools: async (runtime) => {
142-
const list = await listAll(runtime, '/api/v2/tools')
143-
return list.map((t) => render('tools', str(t.id) ?? '', str(t.id) ?? '', t))
144-
},
145-
tables: async (runtime) => {
146-
const list = await listAll(runtime, '/api/v2/tables')
147-
return mapConcurrent(list, FETCH_CONCURRENCY, async (t) => {
148-
const id = str(t.id) ?? ''
149-
const detail = await runtime.client.request<{ data: unknown }>(`/api/v2/tables/${id}`, {
150-
query: { workspaceId: runtime.workspaceId },
151-
})
152-
return render('tables', id, str(t.name) ?? id, detail.data)
153-
})
154-
},
155-
skills: async (runtime) => {
156-
const list = await listAll(runtime, '/api/v2/skills')
157-
return mapConcurrent(list, FETCH_CONCURRENCY, async (s) => {
158-
const id = str(s.id) ?? ''
159-
const detail = await runtime.client.request<{ data: unknown }>(`/api/v2/skills/${id}`, {
160-
query: { workspaceId: runtime.workspaceId },
161-
})
162-
return render('skills', id, str(s.name) ?? id, detail.data)
163-
})
164-
},
165-
'custom-tools': async (runtime) => {
166-
const list = await listAll(runtime, '/api/v2/custom-tools')
167-
return mapConcurrent(list, FETCH_CONCURRENCY, async (t) => {
168-
const id = str(t.id) ?? ''
169-
const detail = await runtime.client.request<{ data: unknown }>(`/api/v2/custom-tools/${id}`, {
170-
query: { workspaceId: runtime.workspaceId },
171-
})
172-
return render('custom-tools', id, str(t.title) ?? str(t.name) ?? id, detail.data)
173-
})
174-
},
175-
files: async (runtime) => {
176-
// File contents, through the v2 read-text endpoint (binary/degraded files are
177-
// honestly skipped there); the label is the path the model sees in `files ls`.
178-
const list = (await listAll(runtime, '/api/v2/files')).slice(0, MAX_FILES)
179-
const texts = await mapConcurrent(list, FILE_READ_CONCURRENCY, async (file) => {
180-
const id = str(file.id) ?? ''
181-
try {
182-
const response = await runtime.client.request<ReadFileTextResponse>(
183-
`/api/v2/files/${encodeURIComponent(id)}/text`,
184-
{ query: { workspaceId: runtime.workspaceId, maxBytes: String(MAX_BYTES_PER_FILE) } }
185-
)
186-
return { file, text: response.data.degraded ? null : response.data.text }
187-
} catch {
188-
return { file, text: null }
189-
}
190-
})
191-
return texts.flatMap(({ file, text }) => {
192-
if (text === null) return []
193-
// `folderPath` is `/` at the root and `/Ops` below it; the match header already
194-
// supplies the `files/` prefix, so the label carries no slash of its own.
195-
const folder = (str(file.folderPath) ?? '').replace(/^\/+|\/+$/g, '')
196-
const name = str(file.name) ?? str(file.id) ?? ''
197-
const label = folder ? `${folder}/${name}` : name
198-
return [{ scope: 'files' as const, id: str(file.id) ?? '', label, text }]
199-
})
200-
},
112+
interface IndexEntry {
113+
id: string
114+
/** Display identity: the resource's name when it has one, else its id. */
115+
label: string
116+
raw: unknown
117+
}
118+
119+
function entry(id: string, label: string, raw: unknown): IndexEntry {
120+
return { id, label, raw }
121+
}
122+
123+
function fileLabel(file: Record<string, unknown>): string {
124+
// `folderPath` is `/` at the root and `/Ops` below it; the match header already
125+
// supplies the `files/` prefix, so the label carries no slash of its own.
126+
const folder = (str(file.folderPath) ?? '').replace(/^\/+|\/+$/g, '')
127+
const name = str(file.name) ?? str(file.id) ?? ''
128+
return folder ? `${folder}/${name}` : name
129+
}
130+
131+
/**
132+
* One cheap index per scope: the listing, which carries ids and names. A `--in` selector
133+
* resolves against these and fetches only what it names — materializing every world to
134+
* find one block cost 65 detail calls per `--in agent` (18-34s and the per-user rate
135+
* limit on dev, 2026-09-03).
136+
*/
137+
const INDEXERS: Record<Scope, (runtime: AgentCliRuntime) => Promise<IndexEntry[]>> = {
138+
workflows: async (runtime) =>
139+
(await listAll(runtime, '/api/v2/workflows')).map((w) =>
140+
entry(str(w.id) ?? '', str(w.name) ?? str(w.id) ?? '', w)
141+
),
142+
blocks: async (runtime) =>
143+
(await listAll(runtime, '/api/v2/blocks')).map((b) =>
144+
entry(str(b.id) ?? '', str(b.id) ?? '', b)
145+
),
146+
tools: async (runtime) =>
147+
(await listAll(runtime, '/api/v2/tools')).map((t) =>
148+
entry(str(t.id) ?? '', str(t.id) ?? '', t)
149+
),
150+
tables: async (runtime) =>
151+
(await listAll(runtime, '/api/v2/tables')).map((t) =>
152+
entry(str(t.id) ?? '', str(t.name) ?? str(t.id) ?? '', t)
153+
),
154+
files: async (runtime) =>
155+
(await listAll(runtime, '/api/v2/files'))
156+
.slice(0, MAX_FILES)
157+
.map((f) => entry(str(f.id) ?? '', fileLabel(f), f)),
201158
integrations: async (runtime) => {
202159
// The viewer's callable connected-service operations — the same projection the
203160
// chat request carries, so `integrations list` and this world never disagree.
@@ -207,26 +164,140 @@ const MATERIALIZERS: Record<Scope, (runtime: AgentCliRuntime) => Promise<Materia
207164
{ schemaSurface: 'copilot' },
208165
runtime.workspaceId
209166
)
210-
return tools.map((tool) => render('integrations', tool.name, tool.name, tool))
167+
return tools.map((tool) => entry(tool.name, tool.name, tool))
211168
},
212-
secrets: async (runtime) => {
169+
skills: async (runtime) =>
170+
(await listAll(runtime, '/api/v2/skills')).map((s) =>
171+
entry(str(s.id) ?? '', str(s.name) ?? str(s.id) ?? '', s)
172+
),
173+
'custom-tools': async (runtime) =>
174+
(await listAll(runtime, '/api/v2/custom-tools')).map((t) =>
175+
entry(str(t.id) ?? '', str(t.title) ?? str(t.name) ?? str(t.id) ?? '', t)
176+
),
177+
secrets: async (runtime) =>
213178
// Names only, by construction: a secret's value never enters the model window.
214-
const list = await listAll(runtime, '/api/v2/secrets')
215-
return list.map((s) =>
216-
render('secrets', str(s.name) ?? '', str(s.name) ?? '', { name: s.name })
217-
)
218-
},
219-
credentials: async (runtime) => {
220-
const list = await listAll(runtime, '/api/v2/credentials')
221-
return list.map((c) =>
222-
render('credentials', str(c.id) ?? '', str(c.name) ?? str(c.id) ?? '', {
179+
(await listAll(runtime, '/api/v2/secrets')).map((s) =>
180+
entry(str(s.name) ?? '', str(s.name) ?? '', { name: s.name })
181+
),
182+
credentials: async (runtime) =>
183+
(await listAll(runtime, '/api/v2/credentials')).map((c) =>
184+
entry(str(c.id) ?? '', str(c.name) ?? str(c.id) ?? '', {
223185
id: c.id,
224186
name: c.name,
225187
provider: c.provider ?? c.providerId,
226188
type: c.type,
227189
})
190+
),
191+
}
192+
193+
/** One fetch per scope: the resource as its `get` returns it, or null when unreadable. */
194+
const FETCHERS: Record<
195+
Scope,
196+
(runtime: AgentCliRuntime, item: IndexEntry) => Promise<Materialized | null>
197+
> = {
198+
workflows: async (runtime, item) => {
199+
// The draft state, not the export: export is sanitized for sharing and nulls
200+
// workspace-specific fields (a Table block's `tableId`), so a grep for a table id
201+
// inside a workflow would miss it. The state route scopes by workflow id alone
202+
// (`query: noInputSchema`); a workspaceId here is an "Unrecognized key".
203+
const state = await runtime.client.request<{ data: unknown }>(
204+
`/api/v2/workflows/${item.id}/state`
228205
)
206+
return render('workflows', item.id, item.label, state.data)
207+
},
208+
blocks: async (runtime, item) => {
209+
const detail = await runtime.client.request<{ data: unknown }>(`/api/v2/blocks/${item.id}`, {
210+
query: { workspaceId: runtime.workspaceId },
211+
})
212+
return render('blocks', item.id, item.label, detail.data)
229213
},
214+
tools: async (_runtime, item) => render('tools', item.id, item.label, item.raw),
215+
tables: async (runtime, item) => {
216+
const detail = await runtime.client.request<{ data: unknown }>(`/api/v2/tables/${item.id}`, {
217+
query: { workspaceId: runtime.workspaceId },
218+
})
219+
return render('tables', item.id, item.label, detail.data)
220+
},
221+
files: async (runtime, item) => {
222+
// File contents, through the v2 read-text endpoint (binary/degraded files are
223+
// honestly skipped there); the label is the path the model sees in `files ls`.
224+
try {
225+
const response = await runtime.client.request<ReadFileTextResponse>(
226+
`/api/v2/files/${encodeURIComponent(item.id)}/text`,
227+
{ query: { workspaceId: runtime.workspaceId, maxBytes: String(MAX_BYTES_PER_FILE) } }
228+
)
229+
if (response.data.degraded) return null
230+
return { scope: 'files', id: item.id, label: item.label, text: response.data.text }
231+
} catch {
232+
return null
233+
}
234+
},
235+
integrations: async (_runtime, item) => render('integrations', item.id, item.label, item.raw),
236+
skills: async (runtime, item) => {
237+
const detail = await runtime.client.request<{ data: unknown }>(`/api/v2/skills/${item.id}`, {
238+
query: { workspaceId: runtime.workspaceId },
239+
})
240+
return render('skills', item.id, item.label, detail.data)
241+
},
242+
'custom-tools': async (runtime, item) => {
243+
const detail = await runtime.client.request<{ data: unknown }>(
244+
`/api/v2/custom-tools/${item.id}`,
245+
{ query: { workspaceId: runtime.workspaceId } }
246+
)
247+
return render('custom-tools', item.id, item.label, detail.data)
248+
},
249+
secrets: async (_runtime, item) => render('secrets', item.id, item.label, item.raw),
250+
credentials: async (_runtime, item) => render('credentials', item.id, item.label, item.raw),
251+
}
252+
253+
function concurrencyFor(scope: Scope): number {
254+
return scope === 'files' ? FILE_READ_CONCURRENCY : FETCH_CONCURRENCY
255+
}
256+
257+
async function fetchAll(
258+
runtime: AgentCliRuntime,
259+
scope: Scope,
260+
entries: IndexEntry[]
261+
): Promise<Materialized[]> {
262+
const items = await mapConcurrent(entries, concurrencyFor(scope), (item) =>
263+
FETCHERS[scope](runtime, item)
264+
)
265+
return items.flatMap((m) => (m ? [m] : []))
266+
}
267+
268+
/** A whole world, for a search with no `--in`: every resource the index lists. */
269+
async function materializeScope(runtime: AgentCliRuntime, scope: Scope): Promise<Materialized[]> {
270+
if (scope === 'blocks') {
271+
const cached = catalogCache.get(runtime.workspaceId)
272+
if (cached !== undefined) return cached
273+
}
274+
const materialized = await fetchAll(runtime, scope, await INDEXERS[scope](runtime))
275+
if (scope === 'blocks') catalogCache.set(runtime.workspaceId, materialized)
276+
return materialized
277+
}
278+
279+
function selects(nameFilter: string): (id: string, label: string) => boolean {
280+
return (id, label) => id.toLowerCase() === nameFilter || label.toLowerCase().includes(nameFilter)
281+
}
282+
283+
/** Only the resources a `--in` selector names: the indexes are read, the matches fetched. */
284+
async function materializeWithin(
285+
runtime: AgentCliRuntime,
286+
scopes: Scope[],
287+
nameFilter: string
288+
): Promise<Materialized[]> {
289+
const wanted = selects(nameFilter)
290+
const perScope = await Promise.all(
291+
scopes.map(async (scope) => {
292+
if (scope === 'blocks') {
293+
const cached = catalogCache.get(runtime.workspaceId)
294+
if (cached !== undefined) return cached.filter((m) => wanted(m.id, m.label))
295+
}
296+
const entries = (await INDEXERS[scope](runtime)).filter((e) => wanted(e.id, e.label))
297+
return fetchAll(runtime, scope, entries)
298+
})
299+
)
300+
return perScope.flat()
230301
}
231302

232303
function compilePattern(raw: string, ignoreCase: boolean): (line: string) => boolean {
@@ -332,15 +403,9 @@ export const universalGrepCommand: AgentCliEngine = {
332403
return agentCliFail(unknownWithin(within))
333404
}
334405
const matches = compilePattern(pattern, ignoreCase)
335-
336-
const materialized = (
337-
await Promise.all(searched.map((scope) => MATERIALIZERS[scope](runtime)))
338-
).flat()
339406
const candidates = nameFilter
340-
? materialized.filter(
341-
(m) => m.id.toLowerCase() === nameFilter || m.label.toLowerCase().includes(nameFilter)
342-
)
343-
: materialized
407+
? await materializeWithin(runtime, searched, nameFilter)
408+
: (await Promise.all(searched.map((scope) => materializeScope(runtime, scope)))).flat()
344409
/**
345410
* A resource nothing in the searched worlds answers to is a wrong selector, not a
346411
* search with no hits — a silent "No matches" would hide the misspelling.

0 commit comments

Comments
 (0)