Add Protobuf generation to the codegen utility

apps/.codegen.yaml

@@ -0,0 +1,2 @@
+protobuf:
+  disabled: false

authorization/.codegen.yaml

@@ -0,0 +1,2 @@
+protobuf:
+  disabled: false

build/.codegen.yaml

@@ -0,0 +1,2 @@
+protobuf:
+  disabled: false

cloudnetwork/.codegen.yaml

@@ -0,0 +1,2 @@
+protobuf:
+  disabled: false

hack/update-protobuf.sh

@@ -2,50 +2,10 @@
 
 source "$(dirname "${BASH_SOURCE}")/lib/init.sh"
 
-[[ -n "${PROTO_OPTIONAL:-}" ]] && exit 0
-
-if [ -z "${GOPATH:-}" ]; then
-  echo "Generating protobuf requires GOPATH to be set. Please set GOPATH.
-To skip protobuf generation, set \$PROTO_OPTIONAL."
-  exit 1
-fi
-
-if [[ "${GOPATH}/src/github.com/openshift/api" != "${SCRIPT_ROOT}" ]]; then
-  echo "Generating protobuf requires the repository to be checked out within the GOPATH."
-  echo "The repository must be checked out at ${GOPATH}/src/github.com/openshift/api."
-  exit 1
-fi
-
-if [[ "$(protoc --version)" != "libprotoc 23."* ]]; then
-  echo "Generating protobuf requires protoc 23.x. Please download and
-install the platform appropriate Protobuf package for your OS:
-
-  https://github.com/google/protobuf/releases
-
-To skip protobuf generation, set \$PROTO_OPTIONAL."
-  exit 1
-fi
-
-# Build go-to-protobuf when it's not present and not overriden for a specific file.
-if [ -z "${GO_TO_PROTOBUF:-}" ]; then
-  ${TOOLS_MAKE} go-to-protobuf
-  GO_TO_PROTOBUF="${TOOLS_OUTPUT}/go-to-protobuf"
-fi
-
 # Build protoc-gen-gogo when it's not present and not overriden for a specific file.
 if [ -z "${PROTOC_GEN_GOGO:-}" ]; then
   ${TOOLS_MAKE} protoc-gen-gogo
   PROTOC_GEN_GOGO="${TOOLS_OUTPUT}/protoc-gen-gogo"
 fi
 
-# We need to make sure that protoc-gen-gogo is on the path for go-to-protobuf to run correctly.
-protoc_bin_dir=$(dirname "${PROTOC_GEN_GOGO}")
-
-PATH="$PATH:${protoc_bin_dir}" "${GO_TO_PROTOBUF}" \
-  --output-dir="${GOPATH}/src" \
-  --apimachinery-packages='-k8s.io/apimachinery/pkg/util/intstr,-k8s.io/apimachinery/pkg/api/resource,-k8s.io/apimachinery/pkg/runtime/schema,-k8s.io/apimachinery/pkg/runtime,-k8s.io/apimachinery/pkg/apis/meta/v1,-k8s.io/apimachinery/pkg/apis/meta/v1beta1,-k8s.io/api/core/v1,-k8s.io/api/rbac/v1' \
-  --go-header-file=${SCRIPT_ROOT}/hack/empty.txt \
-  --proto-import=${SCRIPT_ROOT}/third_party/protobuf \
-  --proto-import=${SCRIPT_ROOT}/vendor \
-  --proto-import=${SCRIPT_ROOT}/tools/vendor \
-  --packages="${API_PACKAGES}"
+GENERATOR=go-to-protobuf ${SCRIPT_ROOT}/hack/update-codegen.sh

hack/verify-protobuf.sh

@@ -2,29 +2,10 @@
 
 source "$(dirname "${BASH_SOURCE}")/lib/init.sh"
 
-TMP_ROOT="${SCRIPT_ROOT}/_tmp"
+# Build protoc-gen-gogo when it's not present and not overriden for a specific file.
+if [ -z "${PROTOC_GEN_GOGO:-}" ]; then
+  ${TOOLS_MAKE} protoc-gen-gogo
+  PROTOC_GEN_GOGO="${TOOLS_OUTPUT}/protoc-gen-gogo"
+fi
 
-cleanup() {
-  rm -rf "${TMP_ROOT}"
-}
-trap "cleanup" EXIT SIGINT
-
-cleanup
-
-for gv in ${API_GROUP_VERSIONS}; do
-  mkdir -p "${TMP_ROOT}/${gv}"
-  cp -a "${SCRIPT_ROOT}/${gv}"/* "${TMP_ROOT}/${gv}"
-done
-
-"${SCRIPT_ROOT}/hack/update-protobuf.sh"
-echo "Checking against freshly generated protobuf..."
-for gv in ${API_GROUP_VERSIONS}; do
-  ret=0
-  diff -Naupr -I 'Auto generated by' -x 'zz_generated.*' -x 'types_swagger*' "${SCRIPT_ROOT}/${gv}" "${TMP_ROOT}/${gv}" || ret=$?
-  if [[ $ret -ne 0 ]]; then
-    cp -a "${TMP_ROOT}"/* "${SCRIPT_ROOT}/"
-    echo "Protobuf is out of date. Please run hack/update-protobuf.sh"
-    exit 1
-  fi
-done
-echo "Protobuf up to date."
+GENERATOR=go-to-protobuf EXTRA_ARGS=--verify ${SCRIPT_ROOT}/hack/update-codegen.sh

image/.codegen.yaml

@@ -1,2 +1,7 @@
 swaggerdocs:
   commentPolicy: Warn
+protobuf:
+  disabled: false
+  disabledVersions:
+  - "1.0"
+  - pre012

network/.codegen.yaml

@@ -1 +1,4 @@
-schemapatch:
+protobuf:
+  disabled: false
+  disabledVersions:
+  - v1alpha1

networkoperator/.codegen.yaml

@@ -1,2 +1,4 @@
 swaggerdocs:
   commentPolicy: Warn
+protobuf:
+  disabled: false

oauth/.codegen.yaml

@@ -1,2 +1,4 @@
 swaggerdocs:
   commentPolicy: Warn
+protobuf:
+  disabled: false

project/.codegen.yaml

@@ -0,0 +1,2 @@
+protobuf:
+  disabled: false

quota/.codegen.yaml

@@ -0,0 +1,2 @@
+protobuf:
+  disabled: false

route/.codegen.yaml

@@ -1,3 +1,5 @@
 schemapatch:
 swaggerdocs:
   commentPolicy: Warn
+protobuf:
+  disabled: false

samples/.codegen.yaml

@@ -1,2 +1,4 @@
 swaggerdocs:
   commentPolicy: Warn
+protobuf:
+  disabled: false

security/.codegen.yaml

@@ -0,0 +1,2 @@
+protobuf:
+  disabled: false

template/.codegen.yaml

@@ -0,0 +1,2 @@
+protobuf:
+  disabled: false

tools.go

@@ -0,0 +1,8 @@
+//go:build tools
+// +build tools
+
+package tools
+
+import (
+	_ "github.com/gogo/protobuf/gogoproto"
+)

tools/codegen/cmd/go_to_protobuf.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/openshift/api/tools/codegen/pkg/generation"
+	"github.com/openshift/api/tools/codegen/pkg/protobuf"
+	"github.com/spf13/cobra"
+)
+
+var (
+	goToProtobufHeaderFilePath string
+)
+
+// goToProtobufCmd represents the go-to-protobuf command
+var goToProtobufCmd = &cobra.Command{
+	Use:   "go-to-protobuf",
+	Short: "go-to-protobuf generates protofbuf definitions for API types",
+	RunE: func(cmd *cobra.Command, args []string) error {
+		genCtx, err := generation.NewContext(generation.Options{
+			BaseDir:          baseDir,
+			APIGroupVersions: apiGroupVersions,
+		})
+		if err != nil {
+			return fmt.Errorf("could not build generation context: %w", err)
+		}
+
+		gen := newGoToProtobufGenerator()
+
+		return executeGenerators(genCtx, gen)
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(goToProtobufCmd)
+
+	rootCmd.PersistentFlags().StringVar(&goToProtobufHeaderFilePath, "go-to-protobuf:header-file-path", "", "Path to file containing boilerplate header text. The string YEAR will be replaced with the current 4-digit year. When omitted, no header is added to the generated files.")
+}
+
+// newGoToProtobufGenerator builds a new go-to-protobuf generator.
+func newGoToProtobufGenerator() generation.Generator {
+	return protobuf.NewGenerator(protobuf.Options{
+		HeaderFilePath: goToProtobufHeaderFilePath,
+		Verify:         verify,
+	})
+}

tools/codegen/cmd/root.go

@@ -135,6 +135,7 @@ func allGenerators() []generation.Generator {
 	return []generation.Generator{
 		newCompatibilityGenerator(),
 		newDeepcopyGenerator(),
+		newGoToProtobufGenerator(),
 		newSwaggerDocsGenerator(),
 		newSchemaPatchGenerator(),
 	}

tools/codegen/pkg/generation/types.go

@@ -15,6 +15,10 @@ type Config struct {
 	// When omitted, the default configuration will be used.
 	OpenAPI *OpenAPIConfig `json:"openapi,omitempty"`
 
+	// Prootbuf represents the configuration of the go-to-protobuf generator.
+	// When omitted, the default configuration will be used.
+	Protobuf *ProtobufConfig `json:"protobuf,omitempty"`
+
 	// SchemaCheck represents the configuration for the schemacheck generator.
 	// When omitted, the default configuration will be used.
 	// When provided, any equivalent flag provided values are ignored.
@@ -72,6 +76,23 @@ type OpenAPIConfig struct {
 	Disabled bool `json:"disabled,omitempty"`
 }
 
+// ProtobufConfig is the configuration for the go-to-protobuf generator.
+type ProtobufConfig struct {
+	// Disabled determines whether the go-to-protobuf generator should be run or not.
+	// This generator is disabled by default so this field defaults to true.
+	Disabled *bool `json:"disabled,omitempty"`
+
+	// DisabledVersions allows you to explicitly disable the generation of protobuf for
+	// specific versions of an API.
+	// This is a list of version names.
+	// When omitted, no versions are disabled.
+	DisabledVersions []string `json:"disabledVersions,omitempty"`
+
+	// HeaderFilePath is the path to the file containing the boilerplate header text.
+	// When omitted, no header is added to the generated files.
+	HeaderFilePath string `json:"headerFilePath,omitempty"`
+}
+
 // SchemaCheckConfig is the configuration for the schemacheck generator.
 type SchemaCheckConfig struct {
 	// Disabled determines whether the schemacheck generator should be run or not.

tools/codegen/pkg/protobuf/generator.go

@@ -0,0 +1,121 @@
+package protobuf
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/openshift/api/tools/codegen/pkg/generation"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/klog/v2"
+)
+
+const (
+	// DefaultOutputFileBaseName is the default output file base name for the generated protobuf functions.
+	DefaultOutputFileBaseName = "generated.pb"
+)
+
+// Options contains the configuration required for the protobuf generator.
+type Options struct {
+	// Disabled indicates whether the deepcopy generator is enabled or not.
+	// This defaults to true as the protobuf generator is disabled by default.
+	Disabled *bool
+
+	// DisabledVersions allows you to explicitly disable the generation of protobuf for
+	// specific versions of an API.
+	// This is a list of version names.
+	// When omitted, no versions are disabled.
+	DisabledVersions []string
+
+	// HeaderFilePath is the path to the file containing the boilerplate header text.
+	// When omitted, no header is added to the generated files.
+	HeaderFilePath string
+
+	// Verify determines whether the generator should verify the content instead
+	// of updating the generated file.
+	Verify bool
+}
+
+// generator implements the generation.Generator interface.
+// It is designed to generate deepcopy function for a particular API group.
+type generator struct {
+	disabled         bool
+	disabledVersions sets.String
+	headerFilePath   string
+	verify           bool
+}
+
+// NewGenerator builds a new protobuf generator.
+func NewGenerator(opts Options) generation.Generator {
+	disabled := true
+	if opts.Disabled != nil {
+		disabled = *opts.Disabled
+	}
+
+	return &generator{
+		disabled:         disabled,
+		disabledVersions: sets.NewString(opts.DisabledVersions...),
+		headerFilePath:   opts.HeaderFilePath,
+		verify:           opts.Verify,
+	}
+}
+
+// ApplyConfig creates returns a new generator based on the configuration passed.
+// If the deepcopy configuration is empty, the existing generation is returned.
+func (g *generator) ApplyConfig(config *generation.Config) generation.Generator {
+	if config == nil || config.Protobuf == nil {
+		return g
+	}
+
+	return NewGenerator(Options{
+		Disabled:         config.Protobuf.Disabled,
+		DisabledVersions: config.Protobuf.DisabledVersions,
+		HeaderFilePath:   config.Protobuf.HeaderFilePath,
+		Verify:           g.verify,
+	})
+}
+
+// Name returns the name of the generator.
+func (g *generator) Name() string {
+	return "go-to-protobuf"
+}
+
+// GenGroup runs the go-to-protobuf generator against the given group context.
+func (g *generator) GenGroup(groupCtx generation.APIGroupContext) ([]generation.Result, error) {
+	if g.disabled {
+		return nil, nil
+	}
+
+	// If there is no header file, create an empty file and pass that through.
+	headerFilePath := g.headerFilePath
+	if headerFilePath == "" {
+		tmpFile, err := os.CreateTemp("", "protobuf-header-*.txt")
+		if err != nil {
+			return nil, fmt.Errorf("failed to create temporary file: %w", err)
+		}
+		tmpFile.Close()
+
+		defer os.Remove(tmpFile.Name())
+
+		headerFilePath = tmpFile.Name()
+	}
+
+	for _, version := range groupCtx.Versions {
+		if g.disabledVersions.Has(version.Name) {
+			klog.V(1).Infof("Skipping generation of protobuf functions for %s/%s", groupCtx.Name, version.Name)
+			continue
+		}
+
+		action := "Generating"
+		if g.verify {
+			action = "Verifying"
+		}
+
+		klog.V(1).Infof("%s protobuf functions for for %s/%s", action, groupCtx.Name, version.Name)
+
+		if err := generateProtobufFunctions(version.Path, version.PackagePath, headerFilePath, g.verify); err != nil {
+			return nil, fmt.Errorf("could not generate protobuf functions for %s/%s: %w", groupCtx.Name, version.Name, err)
+		}
+	}
+
+	return nil, nil
+}