init paddleflow

Author: alexqdh

Makefile

@@ -0,0 +1,80 @@
+# init project path
+HOMEDIR := $(shell pwd)
+OUTDIR  := $(HOMEDIR)/output
+
+# init command params
+GO      := go
+GOPATH  := $(shell $(GO) env GOPATH)
+GOMOD   := $(GO) mod
+GOBUILD := $(GO) build
+GOTEST  := $(GO) test -gcflags="-N -l"
+GOPKGS  := $$($(GO) list ./...| grep -vE "vendor" | grep -vE "paddleflow/pkg/fs/fuse/ufs")
+export PATH := $(GOPATH)/bin/:$(PATH)
+
+# test cover files
+COVPROF := $(HOMEDIR)/covprof.out  # coverage profile
+COVFUNC := $(HOMEDIR)/covfunc.txt  # coverage profile information for each function
+COVHTML := $(HOMEDIR)/covhtml.html # HTML representation of coverage profile
+
+GIT_COMMIT  = `git rev-parse HEAD`
+GIT_DATE    = `date "+%Y-%m-%d %H:%M:%S"`
+GIT_VERSION = `git --version`
+
+LD_FLAGS    = " \
+    -X 'paddleflow/pkg/version.GitVersion=${GIT_VERSION}' \
+    -X 'paddleflow/pkg/version.GitCommit=${GIT_COMMIT}' \
+    -X 'paddleflow/pkg/version.BuildDate=${GIT_DATE}' \
+    '-extldflags=-static' \
+    -w -s"
+
+# make, make all
+all: prepare compile package
+
+# make prepare, download dependencies
+prepare: gomod
+
+gomod:
+	$(GO) env -w GO111MODULE=on
+	$(GO) env -w GOPROXY=https://goproxy.io,direct
+	$(GO) env -w CGO_ENABLED=1
+	$(GOMOD) download
+
+# make compile
+compile: build
+
+build:
+	$(GOBUILD) -ldflags ${LD_FLAGS} -trimpath -o $(HOMEDIR)/paddleflow $(HOMEDIR)/cmd/server/main.go
+	$(GOBUILD) -ldflags ${LD_FLAGS} -trimpath -o $(HOMEDIR)/pfs-fuse   $(HOMEDIR)/cmd/fs/fuse/main.go
+	$(GOBUILD) -ldflags ${LD_FLAGS} -trimpath -o $(HOMEDIR)/csi-plugin $(HOMEDIR)/cmd/fs/csi-plugin/main.go
+
+# make doc
+doc:
+	$(GO) get -u github.com/swaggo/swag/cmd/[email protected]
+	swag init -g router.go --parseDependency --parseInternal  -o $(HOMEDIR)/docs/api -d $(HOMEDIR)/pkg/apiserver/router/v1/
+	mkdir -p $(OUTDIR)/docs
+	mv $(HOMEDIR)/docs/api $(OUTDIR)/docs
+
+# make test, test your code
+test: prepare mock-gen test-case
+mock-gen:
+	$(GO) get golang.org/x/tools/go/packages
+	$(GO) get github.com/golang/mock/[email protected]
+	mockgen -destination=pkg/pipeline/mock_job.go -source=pkg/pipeline/job.go -package=pipeline
+test-case:
+	$(GOTEST) -v -cover $(GOPKGS)
+
+# make package
+package:
+	mkdir -p $(OUTDIR)/bin
+	mv $(HOMEDIR)/paddleflow $(OUTDIR)/bin
+	mv $(HOMEDIR)/pfs-fuse   $(OUTDIR)/bin
+	mv $(HOMEDIR)/csi-plugin $(OUTDIR)/bin
+	cp $(HOMEDIR)/pkg/fs/csiplugin/client/mount.sh $(OUTDIR)/bin
+
+# make clean
+clean:
+	$(GO) clean
+	rm -rf $(OUTDIR)
+
+# avoid filename conflict and speed up build 
+.PHONY: all prepare compile test package clean build

bin/start.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+workdir=`cd $(dirname $0); pwd`
+root=$workdir/..
+supervise_dir=$root/status/paddleflow
+
+pushd $root
+mkdir -p $supervise_dir
+./bin/supervise -p $supervise_dir -f "./bin/paddleflow"
+popd

cmd/fs/csi-plugin/app/csiplugin.go

@@ -0,0 +1,37 @@
+/*
+Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"github.com/spf13/pflag"
+
+	"paddleflow/cmd/fs/csi-plugin/app/options"
+	"paddleflow/pkg/common/config"
+	"paddleflow/pkg/common/logger"
+)
+
+func initConfig() {
+	// init from yaml config
+	config.InitCSIPluginConfig()
+	f := options.NewCSIPluginOption()
+	f.InitFlag(pflag.CommandLine)
+}
+
+func Init() error {
+	initConfig()
+	return logger.Init(&config.CSIPluginConf.Log)
+}

cmd/fs/csi-plugin/app/options/options.go

@@ -0,0 +1,53 @@
+/*
+Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	goflag "flag"
+	"github.com/spf13/pflag"
+
+	"paddleflow/pkg/common/config"
+	"paddleflow/pkg/common/logger"
+)
+
+// CSIPluginOption is the main context object for the csi plugin.
+type CSIPluginOption struct{}
+
+func NewCSIPluginOption() *CSIPluginOption {
+	return &CSIPluginOption{}
+}
+
+func (csi *CSIPluginOption) AddFlagSet(fs *pflag.FlagSet) {
+	if fs == nil {
+		fs = pflag.CommandLine
+	}
+	csiPluginConf := &config.CSIPluginConf.CSIPlugin
+	fs.StringVar(&csiPluginConf.UnixEndpoint, "unix-endpoint", csiPluginConf.UnixEndpoint, "CSI endpoint")
+	fs.StringVar(&csiPluginConf.NodeID, "node-id", csiPluginConf.NodeID, "node id")
+	fs.BoolVar(&csiPluginConf.PprofEnable, "pprof-enable", csiPluginConf.PprofEnable, "pprof debug tool")
+	fs.IntVar(&csiPluginConf.PprofPort, "pprof-port", csiPluginConf.PprofPort, "pprof port")
+}
+
+func (csi *CSIPluginOption) InitFlag(fs *pflag.FlagSet) {
+	if fs == nil {
+		fs = pflag.CommandLine
+	}
+	csi.AddFlagSet(fs)
+	logger.AddFlagSet(fs, &config.CSIPluginConf.Log)
+	pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
+	pflag.Parse()
+}

cmd/fs/csi-plugin/main.go

@@ -0,0 +1,82 @@
+/*
+Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/gin-contrib/pprof"
+	"github.com/gin-gonic/gin"
+	log "github.com/sirupsen/logrus"
+
+	"paddleflow/cmd/fs/csi-plugin/app"
+	"paddleflow/pkg/common/config"
+	"paddleflow/pkg/fs/csiplugin/controller"
+	"paddleflow/pkg/fs/csiplugin/csidriver"
+)
+
+func exitWithError() {
+	exit(true)
+}
+
+func exitNormal() {
+	exit(false)
+}
+
+// exit the function that stop program with return value
+func exit(hasError bool) {
+	// it is required, to work around bug of log4go
+	// 在程序退出前，需要先sleep一段时间，否则有可能日志打印不全
+	time.Sleep(100 * time.Millisecond)
+	if hasError {
+		os.Exit(-1)
+	}
+	os.Exit(0)
+}
+
+// main the function where execution of the program begins
+func main() {
+	if err := app.Init(); err != nil {
+		log.Errorf("init csi plugin failed: %v", err)
+		exitWithError()
+	}
+
+	if config.CSIPluginConf.CSIPlugin.PprofEnable {
+		go func() {
+			router := gin.Default()
+			pprof.Register(router, "debug/pprof")
+			if err := router.Run(fmt.Sprintf(":%d", config.CSIPluginConf.CSIPlugin.PprofPort)); err != nil {
+				log.Errorf("run pprof failed: %s, skip this error", err.Error())
+			} else {
+				log.Infof("pprof started")
+			}
+		}()
+	}
+
+	stopChan := make(chan struct{})
+	defer close(stopChan)
+	controller := controller.GetMountPointController(config.CSIPluginConf.CSIPlugin.NodeID)
+	go controller.Start(stopChan)
+	defer controller.Stop()
+
+	d := csidriver.NewDriver(config.CSIPluginConf.CSIPlugin.NodeID, config.CSIPluginConf.CSIPlugin.UnixEndpoint)
+	d.Run()
+
+	exitNormal()
+}