✨ Upload files instead of multi-part form. (#743)

Post a _manifest_ file instead of separate multi-part form files.
Much simpler and more easily supports the addon staging the issues and
deps files on disk rather than streaming. The more atomic approach will
prevent transaction deadlock which can more easily occur when the
addon-analyzer builder reported an error (which it should never do).

The uploaded file contains markers used to delimited the documents.
`^]` = `\x1D` = GS (group separator).

```
^]BEGIN-MAIN^]
---
commit: 1234
^]END-MAIN^]
^]BEGIN-ISSUES^]
---
ruleset: ruleset-1
rule: rule-1
incidents:
...
^]END-ISSUES^]
^]BEGIN-DEPS^]
---
name: github.com/jboss
version: 4.0
labels:
- konveyor.io/language=java
- konveyor.io/otherA=dog
^]END-DEPS^]
```

Flow:
1. post (upload) manifest.yaml file.
2. post `ref` to the manifest file.
3. delete manifest file.

Orphaned files will be reaped.

---

The binding client needed to be updated to handle different file
encoding (MIME).

---------

Signed-off-by: Jeff Ortel <[email protected]>

Author: jortel

api/analysis.go

@@ -1,12 +1,15 @@
 package api
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"os"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -53,9 +56,15 @@ const (
 	AppAnalysisIssuesRoot = AppAnalysisRoot + "/issues"
 )
 
+// Manifest markers.
+// The GS=\x1D (group separator).
 const (
-	IssueField = "issues"
-	DepField   = "dependencies"
+	BeginMainMarker   = "\x1DBEGIN-MAIN\x1D"
+	EndMainMarker     = "\x1DEND-MAIN\x1D"
+	BeginIssuesMarker = "\x1DBEGIN-ISSUES\x1D"
+	EndIssuesMarker   = "\x1DEND-ISSUES\x1D"
+	BeginDepsMarker   = "\x1DBEGIN-DEPS\x1D"
+	EndDepsMarker     = "\x1DEND-DEPS\x1D"
 )
 
 // AnalysisHandler handles analysis resource routes.
@@ -315,9 +324,20 @@ func (h AnalysisHandler) AppList(ctx *gin.Context) {
 // @summary Create an analysis.
 // @description Create an analysis.
 // @description Form fields:
-// @description   - file: file that contains the api.Analysis resource.
-// @description   - issues: file that multiple api.Issue resources.
-// @description   - dependencies: file that multiple api.TechDependency resources.
+// @description file: A manifest file that contains 3 sections
+// @description containing documents delimited by markers.
+// @description The manifest must contain ALL markers even when sections are empty.
+// @description Note: `^]` = `\x1D` = GS (group separator).
+// @description Section markers:
+// @description	  ^]BEGIN-MAIN^]
+// @description	  ^]END-MAIN^]
+// @description	  ^]BEGIN-ISSUES^]
+// @description	  ^]END-ISSUES^]
+// @description	  ^]BEGIN-DEPS^]
+// @description	  ^]END-DEPS^]
+// @description The encoding must be:
+// @description - application/json
+// @description - application/x-yaml
 // @tags analyses
 // @produce json
 // @success 201 {object} api.Analysis
@@ -337,32 +357,40 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) {
 			return
 		}
 	}
+	db := h.DB(ctx)
 	//
-	// Analysis
-	input, err := ctx.FormFile(FileField)
+	// Manifest
+	fh := FileHandler{}
+	name := fmt.Sprintf("app.%d.manifest", id)
+	file, err := fh.create(ctx, name)
 	if err != nil {
-		err = &BadRequestError{err.Error()}
 		_ = ctx.Error(err)
 		return
 	}
-	reader, err := input.Open()
+	defer func() {
+		err = fh.delete(ctx, file)
+		if err != nil {
+			_ = ctx.Error(err)
+		}
+	}()
+	reader := &ManifestReader{}
+	f, err := reader.open(file.Path, BeginMainMarker, EndMainMarker)
 	if err != nil {
 		err = &BadRequestError{err.Error()}
 		_ = ctx.Error(err)
 		return
 	}
 	defer func() {
-		_ = reader.Close()
+		_ = f.Close()
 	}()
-	encoding := input.Header.Get(ContentType)
-	d, err := h.Decoder(ctx, encoding, reader)
+	d, err := h.Decoder(ctx, file.Encoding, reader)
 	if err != nil {
 		err = &BadRequestError{err.Error()}
 		_ = ctx.Error(err)
 		return
 	}
-	r := Analysis{}
-	err = d.Decode(&r)
+	r := &Analysis{}
+	err = d.Decode(r)
 	if err != nil {
 		err = &BadRequestError{err.Error()}
 		_ = ctx.Error(err)
@@ -371,7 +399,6 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) {
 	analysis := r.Model()
 	analysis.ApplicationID = id
 	analysis.CreateUser = h.BaseHandler.CurrentUser(ctx)
-	db := h.DB(ctx)
 	db.Logger = db.Logger.LogMode(logger.Error)
 	err = db.Create(analysis).Error
 	if err != nil {
@@ -380,23 +407,17 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) {
 	}
 	//
 	// Issues
-	input, err = ctx.FormFile(IssueField)
-	if err != nil {
-		err = &BadRequestError{err.Error()}
-		_ = ctx.Error(err)
-		return
-	}
-	reader, err = input.Open()
+	reader = &ManifestReader{}
+	f, err = reader.open(file.Path, BeginIssuesMarker, EndIssuesMarker)
 	if err != nil {
 		err = &BadRequestError{err.Error()}
 		_ = ctx.Error(err)
 		return
 	}
 	defer func() {
-		_ = reader.Close()
+		_ = f.Close()
 	}()
-	encoding = input.Header.Get(ContentType)
-	d, err = h.Decoder(ctx, encoding, reader)
+	d, err = h.Decoder(ctx, file.Encoding, reader)
 	if err != nil {
 		err = &BadRequestError{err.Error()}
 		_ = ctx.Error(err)
@@ -425,23 +446,17 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) {
 	}
 	//
 	// Dependencies
-	input, err = ctx.FormFile(DepField)
-	if err != nil {
-		err = &BadRequestError{err.Error()}
-		_ = ctx.Error(err)
-		return
-	}
-	reader, err = input.Open()
+	reader = &ManifestReader{}
+	f, err = reader.open(file.Path, BeginDepsMarker, EndDepsMarker)
 	if err != nil {
 		err = &BadRequestError{err.Error()}
 		_ = ctx.Error(err)
 		return
 	}
 	defer func() {
-		_ = reader.Close()
+		_ = f.Close()
 	}()
-	encoding = input.Header.Get(ContentType)
-	d, err = h.Decoder(ctx, encoding, reader)
+	d, err = h.Decoder(ctx, file.Encoding, reader)
 	if err != nil {
 		err = &BadRequestError{err.Error()}
 		_ = ctx.Error(err)
@@ -2860,3 +2875,116 @@ func (r *yamlEncoder) embed(object any) encoder {
 	r.write(s)
 	return r
 }
+
+// ManifestReader analysis manifest reader.
+// The manifest contains 3 sections containing documents delimited by markers.
+// The manifest must contain ALL markers even when sections are empty.
+// Note: `^]` = `\x1D` = GS (group separator).
+// Section markers:
+//
+//	^]BEGIN-MAIN^]
+//	^]END-MAIN^]
+//	^]BEGIN-ISSUES^]
+//	^]END-ISSUES^]
+//	^]BEGIN-DEPS^]
+//	^]END-DEPS^]
+type ManifestReader struct {
+	file   *os.File
+	marker map[string]int64
+	begin  int64
+	end    int64
+	read   int64
+}
+
+// scan manifest and catalog position of markers.
+func (r *ManifestReader) scan(path string) (err error) {
+	if r.marker != nil {
+		return
+	}
+	r.file, err = os.Open(path)
+	if err != nil {
+		return
+	}
+	defer func() {
+		_ = r.file.Close()
+	}()
+	pattern, err := regexp.Compile(`^\x1D[A-Z-]+\x1D$`)
+	if err != nil {
+		return
+	}
+	p := int64(0)
+	r.marker = make(map[string]int64)
+	scanner := bufio.NewScanner(r.file)
+	for scanner.Scan() {
+		content := scanner.Text()
+		matched := strings.TrimSpace(content)
+		if pattern.Match([]byte(matched)) {
+			r.marker[matched] = p
+		}
+		p += int64(len(content))
+		p++
+	}
+
+	return
+}
+
+// open returns a read delimited by the specified markers.
+func (r *ManifestReader) open(path, begin, end string) (reader io.ReadCloser, err error) {
+	found := false
+	err = r.scan(path)
+	if err != nil {
+		return
+	}
+	r.begin, found = r.marker[begin]
+	if !found {
+		err = &BadRequestError{
+			Reason: fmt.Sprintf("marker: %s not found.", begin),
+		}
+		return
+	}
+	r.end, found = r.marker[end]
+	if !found {
+		err = &BadRequestError{
+			Reason: fmt.Sprintf("marker: %s not found.", end),
+		}
+		return
+	}
+	if r.begin >= r.end {
+		err = &BadRequestError{
+			Reason: fmt.Sprintf("marker: %s must preceed %s.", begin, end),
+		}
+		return
+	}
+	r.begin += int64(len(begin))
+	r.begin++
+	r.read = r.end - r.begin
+	r.file, err = os.Open(path)
+	if err != nil {
+		return
+	}
+	_, err = r.file.Seek(r.begin, io.SeekStart)
+	reader = r
+	return
+}
+
+// Read bytes.
+func (r *ManifestReader) Read(b []byte) (n int, err error) {
+	n, err = r.file.Read(b)
+	if n == 0 || err != nil {
+		return
+	}
+	if int64(n) > r.read {
+		n = int(r.read)
+	}
+	r.read -= int64(n)
+	if n < 1 {
+		err = io.EOF
+	}
+	return
+}
+
+// Close the reader.
+func (r *ManifestReader) Close() (err error) {
+	err = r.file.Close()
+	return
+}