Skip to content

Commit 5d42f19

Browse files
thedadamsthedadams
authored
Merge pull request #306 from ibuildthecloud/main
feat: add parse and fmt CLI commands
2 parents d8d10c5 + b07aec1 commit 5d42f19

File tree

7 files changed

+272
-55
lines changed

7 files changed

+272
-55
lines changed

‎pkg/cli/fmt.go

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
package cli
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"os"
7+
"strings"
8+
9+
"github.com/gptscript-ai/gptscript/pkg/input"
10+
"github.com/gptscript-ai/gptscript/pkg/parser"
11+
"github.com/spf13/cobra"
12+
)
13+
14+
type Fmt struct {
15+
Write bool `usage:"Write output to file instead of stdout" short:"w"`
16+
}
17+
18+
func (e *Fmt) Customize(cmd *cobra.Command) {
19+
cmd.Args = cobra.ExactArgs(1)
20+
}
21+
22+
func (e *Fmt) Run(_ *cobra.Command, args []string) error {
23+
input, err := input.FromFile(args[0])
24+
if err != nil {
25+
return err
26+
}
27+
28+
var (
29+
doc parser.Document
30+
loc = locationName(args[0])
31+
)
32+
if strings.HasPrefix(input, "{") {
33+
if err := json.Unmarshal([]byte(input), &doc); err != nil {
34+
return err
35+
}
36+
} else {
37+
doc, err = parser.Parse(strings.NewReader(input), parser.Options{
38+
Location: locationName(args[0]),
39+
})
40+
if err != nil {
41+
return err
42+
}
43+
}
44+
45+
if e.Write && loc != "" {
46+
return os.WriteFile(loc, []byte(doc.String()), 0644)
47+
}
48+
49+
fmt.Print(doc.String())
50+
return nil
51+
}

‎pkg/cli/gptscript.go

+9-3
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,15 @@ type GPTScript struct {
6868

6969
func New() *cobra.Command {
7070
root := &GPTScript{}
71-
command := cmd.Command(root, &Eval{
72-
gptscript: root,
73-
}, &Credential{root: root})
71+
command := cmd.Command(
72+
root,
73+
&Eval{
74+
gptscript: root,
75+
},
76+
&Credential{root: root},
77+
&Parse{},
78+
&Fmt{},
79+
)
7480

7581
// Hide all the global flags for the credential subcommand.
7682
for _, child := range command.Commands() {

‎pkg/cli/parse.go

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package cli
2+
3+
import (
4+
"encoding/json"
5+
"os"
6+
"strings"
7+
8+
"github.com/gptscript-ai/gptscript/pkg/input"
9+
"github.com/gptscript-ai/gptscript/pkg/parser"
10+
"github.com/spf13/cobra"
11+
)
12+
13+
type Parse struct {
14+
PrettyPrint bool `usage:"Indent the json output" short:"p"`
15+
}
16+
17+
func (e *Parse) Customize(cmd *cobra.Command) {
18+
cmd.Args = cobra.ExactArgs(1)
19+
}
20+
21+
func locationName(l string) string {
22+
if l == "-" {
23+
return ""
24+
}
25+
return l
26+
}
27+
28+
func (e *Parse) Run(_ *cobra.Command, args []string) error {
29+
input, err := input.FromFile(args[0])
30+
if err != nil {
31+
return err
32+
}
33+
34+
docs, err := parser.Parse(strings.NewReader(input), parser.Options{
35+
Location: locationName(args[0]),
36+
})
37+
if err != nil {
38+
return err
39+
}
40+
41+
enc := json.NewEncoder(os.Stdout)
42+
if e.PrettyPrint {
43+
enc.SetIndent("", " ")
44+
}
45+
46+
return enc.Encode(docs)
47+
}

‎pkg/loader/loader.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ func readTool(ctx context.Context, prg *types.Program, base *source, targetToolN
147147

148148
// If we didn't get any tools from trying to parse it as OpenAPI, try to parse it as a GPTScript
149149
if len(tools) == 0 {
150-
tools, err = parser.Parse(bytes.NewReader(data), parser.Options{
150+
tools, err = parser.ParseTools(bytes.NewReader(data), parser.Options{
151151
AssignGlobals: true,
152152
})
153153
if err != nil {

‎pkg/parser/parser.go

+108-16
Original file line numberDiff line numberDiff line change
@@ -174,42 +174,124 @@ type context struct {
174174
instructions []string
175175
inBody bool
176176
skipNode bool
177+
skipLines []string
177178
seenParam bool
178179
}
179180

180-
func (c *context) finish(tools *[]types.Tool) {
181+
func (c *context) finish(tools *[]Node) {
181182
c.tool.Instructions = strings.TrimSpace(strings.Join(c.instructions, ""))
182183
if c.tool.Instructions != "" || c.tool.Parameters.Name != "" ||
183184
len(c.tool.Export) > 0 || len(c.tool.Tools) > 0 ||
184185
c.tool.GlobalModelName != "" ||
185186
len(c.tool.GlobalTools) > 0 ||
186187
c.tool.Chat {
187-
*tools = append(*tools, c.tool)
188+
*tools = append(*tools, Node{
189+
ToolNode: &ToolNode{
190+
Tool: c.tool,
191+
},
192+
})
193+
}
194+
if c.skipNode && len(c.skipLines) > 0 {
195+
*tools = append(*tools, Node{
196+
TextNode: &TextNode{
197+
Text: strings.Join(c.skipLines, ""),
198+
},
199+
})
188200
}
189201
*c = context{}
190202
}
191203

192204
type Options struct {
193205
AssignGlobals bool
206+
Location string
194207
}
195208

196209
func complete(opts ...Options) (result Options) {
197210
for _, opt := range opts {
198211
result.AssignGlobals = types.FirstSet(opt.AssignGlobals, result.AssignGlobals)
212+
result.Location = types.FirstSet(opt.Location, result.Location)
199213
}
200214
return
201215
}
202216

203-
func Parse(input io.Reader, opts ...Options) ([]types.Tool, error) {
204-
tools, err := parse(input)
217+
type Document struct {
218+
Nodes []Node `json:"nodes,omitempty"`
219+
}
220+
221+
func writeSep(buf *strings.Builder, lastText bool) {
222+
if buf.Len() > 0 {
223+
if !lastText {
224+
buf.WriteString("\n")
225+
}
226+
buf.WriteString("---\n")
227+
}
228+
}
229+
230+
func (d Document) String() string {
231+
buf := strings.Builder{}
232+
lastText := false
233+
for _, node := range d.Nodes {
234+
if node.TextNode != nil {
235+
writeSep(&buf, lastText)
236+
buf.WriteString(node.TextNode.Text)
237+
lastText = true
238+
}
239+
if node.ToolNode != nil {
240+
writeSep(&buf, lastText)
241+
buf.WriteString(node.ToolNode.Tool.String())
242+
lastText = false
243+
}
244+
}
245+
return buf.String()
246+
}
247+
248+
type Node struct {
249+
TextNode *TextNode `json:"textNode,omitempty"`
250+
ToolNode *ToolNode `json:"toolNode,omitempty"`
251+
}
252+
253+
type TextNode struct {
254+
Text string `json:"text,omitempty"`
255+
}
256+
257+
type ToolNode struct {
258+
Tool types.Tool `json:"tool,omitempty"`
259+
}
260+
261+
func ParseTools(input io.Reader, opts ...Options) (result []types.Tool, _ error) {
262+
doc, err := Parse(input, opts...)
205263
if err != nil {
206264
return nil, err
207265
}
266+
for _, node := range doc.Nodes {
267+
if node.ToolNode != nil {
268+
result = append(result, node.ToolNode.Tool)
269+
}
270+
}
271+
272+
return
273+
}
274+
275+
func Parse(input io.Reader, opts ...Options) (Document, error) {
276+
nodes, err := parse(input)
277+
if err != nil {
278+
return Document{}, err
279+
}
208280

209281
opt := complete(opts...)
210282

283+
if opt.Location != "" {
284+
for _, node := range nodes {
285+
if node.ToolNode != nil && node.ToolNode.Tool.Source.Location == "" {
286+
node.ToolNode.Tool.Source.Location = opt.Location
287+
}
288+
}
289+
}
290+
211291
if !opt.AssignGlobals {
212-
return tools, nil
292+
return Document{
293+
Nodes: nodes,
294+
}, nil
213295
}
214296

215297
var (
@@ -218,10 +300,14 @@ func Parse(input io.Reader, opts ...Options) ([]types.Tool, error) {
218300
globalTools []string
219301
)
220302

221-
for _, tool := range tools {
303+
for _, node := range nodes {
304+
if node.ToolNode == nil {
305+
continue
306+
}
307+
tool := node.ToolNode.Tool
222308
if tool.GlobalModelName != "" {
223309
if globalModel != "" {
224-
return nil, fmt.Errorf("global model name defined multiple times")
310+
return Document{}, fmt.Errorf("global model name defined multiple times")
225311
}
226312
globalModel = tool.GlobalModelName
227313
}
@@ -234,26 +320,30 @@ func Parse(input io.Reader, opts ...Options) ([]types.Tool, error) {
234320
}
235321
}
236322

237-
for i, tool := range tools {
238-
if globalModel != "" && tool.ModelName == "" {
239-
tool.ModelName = globalModel
323+
for _, node := range nodes {
324+
if node.ToolNode == nil {
325+
continue
326+
}
327+
if globalModel != "" && node.ToolNode.Tool.ModelName == "" {
328+
node.ToolNode.Tool.ModelName = globalModel
240329
}
241330
for _, globalTool := range globalTools {
242-
if !slices.Contains(tool.Tools, globalTool) {
243-
tool.Tools = append(tool.Tools, globalTool)
331+
if !slices.Contains(node.ToolNode.Tool.Tools, globalTool) {
332+
node.ToolNode.Tool.Tools = append(node.ToolNode.Tool.Tools, globalTool)
244333
}
245334
}
246-
tools[i] = tool
247335
}
248336

249-
return tools, nil
337+
return Document{
338+
Nodes: nodes,
339+
}, nil
250340
}
251341

252-
func parse(input io.Reader) ([]types.Tool, error) {
342+
func parse(input io.Reader) ([]Node, error) {
253343
scan := bufio.NewScanner(input)
254344

255345
var (
256-
tools []types.Tool
346+
tools []Node
257347
context context
258348
lineNo int
259349
)
@@ -277,6 +367,7 @@ func parse(input io.Reader) ([]types.Tool, error) {
277367
}
278368

279369
if context.skipNode {
370+
context.skipLines = append(context.skipLines, line)
280371
continue
281372
}
282373

@@ -292,6 +383,7 @@ func parse(input io.Reader) ([]types.Tool, error) {
292383
}
293384

294385
if !context.seenParam && skipRegex.MatchString(line) {
386+
context.skipLines = append(context.skipLines, line)
295387
context.skipNode = true
296388
continue
297389
}

0 commit comments

Comments
 (0)