magika: add ml-based content type detection

detector/magika/_cmd/ortgen/go.mod

@@ -0,0 +1,14 @@
+module ortgen
+
+go 1.24.3
+
+require modernc.org/cc/v4 v4.27.1
+
+require (
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/opt v0.1.4 // indirect
+	modernc.org/sortutil v1.2.1 // indirect
+	modernc.org/strutil v1.2.1 // indirect
+	modernc.org/token v1.1.0 // indirect
+)

detector/magika/_cmd/ortgen/go.sum

@@ -0,0 +1,22 @@
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis=
+modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0=
+modernc.org/ccorpus2 v1.5.2 h1:Ui+4tc58mf/W+2arcYCJR903y3zl3ecsI7Fpaaqozyw=
+modernc.org/ccorpus2 v1.5.2/go.mod h1:Wifvo4Q/qS/h1aRoC2TffcHsnxwTikmi1AuLANuucJQ=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
+modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
+modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
+modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
+modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
+modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

detector/magika/_cmd/ortgen/ortgen.go

@@ -0,0 +1,246 @@
+// Ortgen is a helper program to generate type information from the ONNX Runtime
+// release tarball.
+package main
+
+import (
+	"archive/tar"
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"io/fs"
+	"log/slog"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"text/template"
+
+	"modernc.org/cc/v4"
+)
+
+const urlTmpl = "https://github.com/microsoft/onnxruntime/releases/download/v{{.}}/onnxruntime-linux-x64-{{.}}.tgz"
+
+func main() {
+	var code int
+	defer func() {
+		if code != 0 {
+			os.Exit(code)
+		}
+	}()
+	var out *os.File
+	tmpl := template.Must(template.New(`url`).Parse(urlTmpl))
+	var loglevel slog.LevelVar
+	loglevel.Set(slog.LevelError)
+	slog.SetDefault(slog.New(slog.NewTextHandler(
+		os.Stderr,
+		&slog.HandlerOptions{
+			Level: &loglevel,
+		})))
+	ctx := context.Background()
+
+	vers := flag.String("version", `1.15.1`, "use `version` as the argument to the URL template")
+	flag.Func("url", "fetch ONNX Runtime release tarball from templated `URL`", func(text string) error {
+		if _, err := tmpl.Parse(text); err != nil {
+			return err
+		}
+		return nil
+	})
+	pkg := flag.String("package", "magika", "generated package `name`")
+	flag.Func("o", "output to `file`", func(p string) error {
+		if out != nil {
+			if err := out.Close(); err != nil {
+				return err
+			}
+		}
+		f, err := os.Create(p)
+		if err != nil {
+			return err
+		}
+		out = f
+		return nil
+	})
+	flag.BoolFunc("D", "debug log output", func(v string) error {
+		if ok, err := strconv.ParseBool(v); err == nil && ok {
+			loglevel.Set(slog.LevelDebug)
+		}
+		return nil
+	})
+	flag.Parse()
+
+	var sb strings.Builder
+	if err := tmpl.Execute(&sb, *vers); err != nil {
+		slog.ErrorContext(ctx, "unexpected template error", "reason", err)
+		code = 1
+		return
+	}
+	slog.InfoContext(ctx, "templated URL", "url", &sb)
+
+	if out == nil {
+		out = os.Stdout
+	}
+	defer out.Close()
+
+	if err := Main(ctx, out, sb.String(), *pkg); err != nil {
+		slog.ErrorContext(ctx, "unexpected error", "reason", err)
+		code = 1
+	}
+}
+
+const genHeader = `// Code generated by ortgen. DO NOT EDIT.`
+
+func Main(ctx context.Context, out io.Writer, in, pkg string) error {
+	const header = `onnxruntime_c_api.h`
+	dir, err := fetchTarball(ctx, in)
+	if err != nil {
+		return fmt.Errorf("fetching: %w", err)
+	}
+	defer os.RemoveAll(dir.Name())
+	defer dir.Close()
+
+	cfg, err := cc.NewConfig("linux", "amd64")
+	if err != nil {
+		return fmt.Errorf("unable to create config: %w", err)
+	}
+	ms, _ := fs.Glob(dir.FS(), `*/include/`+header)
+	f, err := dir.Open(ms[0])
+	if err != nil {
+		return fmt.Errorf("unable to open header: %q: %w", ms[0], err)
+	}
+	src := []cc.Source{
+		{Name: "<predefined>", Value: cfg.Predefined},
+		{Name: "<builtin>", Value: cc.Builtin},
+		{Name: "<bool>", Value: "#define bool _Bool\n"},
+		{Name: header, Value: f},
+	}
+	ast, err := cc.Translate(cfg, src)
+	if err != nil {
+		return fmt.Errorf("parse error: %w", err)
+	}
+
+	fmt.Fprintf(out, "%s\n\npackage %s\n\nimport \"structs\"\n", genHeader, pkg)
+	// Walk the list, looking for the bits we want:
+	var buf bytes.Buffer
+	for cur := ast.TranslationUnit; cur != nil; cur = cur.TranslationUnit {
+		buf.Reset()
+		if cur.ExternalDeclaration.Case != cc.ExternalDeclarationDecl {
+			continue
+		}
+		decl := cur.ExternalDeclaration.Declaration
+		if decl.Position().Filename != header {
+			continue
+		}
+		spec := decl.DeclarationSpecifiers
+		if spec == nil || spec.Case != cc.DeclarationSpecifiersTypeSpec {
+			continue
+		}
+		ty := spec.TypeSpecifier
+		if ty == nil || ty.Case != cc.TypeSpecifierStructOrUnion {
+			continue
+		}
+		structSpec := ty.StructOrUnionSpecifier
+		if structSpec == nil ||
+			structSpec.Case != cc.StructOrUnionSpecifierDef ||
+			structSpec.StructOrUnion.Case != cc.StructOrUnionStruct {
+			continue
+		}
+
+		n := structSpec.Token.SrcStr()
+		switch n {
+		case "OrtApi":
+		case "OrtApiBase":
+		default:
+			continue
+		}
+		fmt.Fprintf(&buf, "\ntype %s struct {\n\t_ structs.HostLayout\n\n", strings.Replace(n, "O", "o", 1))
+
+		for cur := structSpec.StructDeclarationList; cur != nil; cur = cur.StructDeclarationList {
+			buf.WriteString("\t// ")
+			buf.WriteString(cc.NodeSource(cur.StructDeclaration))
+			buf.WriteByte('\n')
+
+			// pull out the function pointer ident:
+			decl := cur.StructDeclaration.StructDeclaratorList.StructDeclarator.Declarator.DirectDeclarator.DirectDeclarator.Declarator.DirectDeclarator
+			buf.WriteByte('\t')
+			buf.Write(decl.Token.Src())
+			buf.WriteString(" uintptr\n")
+		}
+
+		buf.WriteString("}\n")
+		if _, err := io.Copy(out, &buf); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func fetchTarball(ctx context.Context, in string) (*os.Root, error) {
+	slog.DebugContext(ctx, "fetching tarball", "url", in)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, in, nil)
+	if err != nil {
+		return nil, err
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status: %v", res.Status)
+	}
+	defer res.Body.Close()
+	z, err := gzip.NewReader(res.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	d, err := os.MkdirTemp("", "ortgen.")
+	if err != nil {
+		return nil, err
+	}
+	slog.DebugContext(ctx, "untarring to disk", "path", d)
+	root, err := os.OpenRoot(d)
+	if err != nil {
+		return nil, err
+	}
+
+	ok := false
+	defer func() {
+		if !ok {
+			os.RemoveAll(d)
+			root.Close()
+		}
+	}()
+
+	rd := tar.NewReader(bufio.NewReader(z))
+	h, err := rd.Next()
+	for ; err == nil; h, err = rd.Next() {
+		err := func() error {
+			p := filepath.Join(".", h.Name)
+			fi := h.FileInfo()
+			if fi.IsDir() {
+				return root.Mkdir(p, 0o755)
+			}
+			f, err := root.Create(p)
+			if err != nil {
+				return err
+			}
+			defer f.Close()
+
+			if _, err := io.Copy(f, rd); err != nil {
+				return err
+			}
+			return nil
+		}()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	ok = true
+	return root, nil
+}