feat: add trace logger, and insert buried point to run api server (#412)

* feat(trace_logger): add trace logger package

* feat(trace_log): add trace logger buried point to creat run function

* feat(api_server): insert buried point to run api
server #412

* feat(api_server): add trace log query result to log api server

Author: kiritoxkiriko

.gitignore

@@ -20,3 +20,6 @@ __pycache__/
 
 # IDE
 .idea/
+
+# go vendor
+vendor

AUTHORS.md

@@ -1,19 +1,20 @@
 * This is the list of people who have contributed code/doc to the PaddleFlow repository.
 * Please keep the list sorted by **Github account**.
 
-| Github account | Name              |
-|----------------|-------------------|
-| aiheshan       | Shan He           |
-| alexqdh        | Duohao Qin        |
-| badger         | Haoqiang Wang     |
-| D0m021ng       | Zezhao Dong       |
-| ElsieFan       | Jiangxun Fan      |
-| HaozhengAN     | Haozheng An       |
-| loleek         | Kai Deng          |
-| luoyuedong     | YueDong Luo       |
-| mikemoto       | Xianyuan Mo       |
-| mkavim         | Xiaoping Xu       |
-| qiaoshuangshuang      | Shuangshuang Qiao |
-| stonekim       | Kuan Shi          |
-| tornado404     | Zichao Zhong      |
-| xiangyuelin    | xiangyuelin       |
+| Github account   | Name              |
+|------------------|-------------------|
+| aiheshan         | Shan He           |
+| alexqdh          | Duohao Qin        |
+| badger           | Haoqiang Wang     |
+| D0m021ng         | Zezhao Dong       |
+| ElsieFan         | Jiangxun Fan      |
+| HaozhengAN       | Haozheng An       |
+| kiritoxkiriko    | Xinmeng Wang      |
+| loleek           | Kai Deng          |
+| luoyuedong       | YueDong Luo       |
+| mikemoto         | Xianyuan Mo       |
+| mkavim           | Xiaoping Xu       |
+| qiaoshuangshuang | Shuangshuang Qiao |
+| stonekim         | Kuan Shi          |
+| tornado404       | Zichao Zhong      |
+| xiangyuelin      | xiangyuelin       |

cmd/server/main.go

@@ -26,6 +26,7 @@ import (
 	"github.com/PaddlePaddle/PaddleFlow/pkg/job"
 	"github.com/PaddlePaddle/PaddleFlow/pkg/storage"
 	"github.com/PaddlePaddle/PaddleFlow/pkg/storage/driver"
+	"github.com/PaddlePaddle/PaddleFlow/pkg/trace_logger"
 	"github.com/PaddlePaddle/PaddleFlow/pkg/version"
 )
 
@@ -104,6 +105,13 @@ func start() error {
 	go jobCtrl.WSManager.SendGroupData()
 	go jobCtrl.WSManager.GetGroupData()
 
+	err = trace_logger.Start(ServerConf.TraceLog)
+	if err != nil {
+		errMsg := fmt.Errorf("start trace logger failed. error: %w", err)
+		log.Errorf(errMsg.Error())
+		return errMsg
+	}
+
 	go func() {
 		if err := HttpSvr.ListenAndServe(); err != nil && errors.Is(err, http.ErrServerClosed) {
 			log.Infof("listen: %s", err)
@@ -167,11 +175,19 @@ func initClusterAndQueue(serverConf *config.ServerConfig) error {
 }
 
 func setup() {
+	var err error
 	if err := logger.InitStandardFileLogger(&ServerConf.Log); err != nil {
 		log.Errorf("InitStandardFileLogger err: %v", err)
 		gracefullyExit(err)
 	}
 
+	// init trace logger config
+	err = trace_logger.Init(ServerConf.TraceLog)
+	if err != nil {
+		log.Errorf("InitTraceLoggerManager err: %v", err)
+		gracefullyExit(err)
+	}
+
 	log.Infof("The final server config is: %s ", config.PrettyFormat(ServerConf))
 
 	dbConf := &ServerConf.Storage

config/server/default/paddleserver.yaml

@@ -7,6 +7,21 @@ log:
   maxFileSizeInMB: 100
   isCompress: true
 
+# trace log config
+traceLog:
+  dir: ./
+  filePrefix: trace-log
+  level: INFO
+  maxKeepDays: 7
+  maxFileNum: 7
+  maxFileSizeInMB: 100
+  isCompress: true
+  # unit is s/m/h/d
+  timeout: 2h
+  maxCacheSize: 10000
+  syncInterval: 30s
+  deleteInterval: 10s
+
 apiServer:
   host: 0.0.0.0
   port: 8083

docs/zh_cn/deployment/how_to_install_paddleflow.md

@@ -9,7 +9,7 @@ PaddleFlow部署分为客户端和服务端两个部分。
 **安装**PaddleFlow客户端有两种方式：
 
 1. 从[release](https://github.com/PaddlePaddle/PaddleFlow/releases)下载最新版, 执行`pip3 install PaddleFlow-1.4.2-py3-none-any.whl`
-2. 通过编译包安装，编译包有两处来源，分别是分支的最新产出、执行编译命令`cd client && python3 setup.py bdist_wheel`。得到编译包后执行`pip3 install PaddleFlow-1.4.2-py3-none-any.whl`
+2. 通过编译包安装，编译包有两处来源，分别是分支的最新产出、执行编译命令`cd client && python3 setup.py bdist_wheel`。得到编译包后执行`cd dist && pip3 install PaddleFlow-1.4.2-py3-none-any.whl`
 
 **卸载**PaddleFlow
 

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/docker v17.12.1-ce+incompatible
 	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/emirpasic/gods v1.18.1
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-chi/chi v1.5.4
 	github.com/go-openapi/spec v0.20.4 // indirect
@@ -35,6 +36,7 @@ require (
 	github.com/mitchellh/mapstructure v1.4.1
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/orcaman/concurrent-map v1.0.0
 	github.com/paddleflow/paddle-operator v0.3.1
 	github.com/panjf2000/ants/v2 v2.4.8
 	github.com/pkg/errors v0.9.1
@@ -51,6 +53,7 @@ require (
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/urfave/cli/v2 v2.4.0
 	github.com/vbauerster/mpb/v7 v7.4.1
+	github.com/viney-shih/go-lock v1.1.2
 	github.com/xujiajun/nutsdb v0.8.0
 	go.uber.org/automaxprocs v1.4.0
 	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97

go.sum

@@ -175,6 +175,8 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb
 github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
+github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -625,6 +627,8 @@ github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3I
 github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY=
+github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
 github.com/paddleflow/paddle-operator v0.3.1 h1:0i0EOGIXM3qL94XBhskNwk6vY5/EhxQhdhIM4GSKvWs=
 github.com/paddleflow/paddle-operator v0.3.1/go.mod h1:sMEu7Rx1E1DwyolierbVp1zxFTK8iI73e2OiWaS9oqg=
 github.com/panjf2000/ants/v2 v2.4.8 h1:JgTbolX6K6RreZ4+bfctI0Ifs+3mrE5BIHudQxUDQ9k=
@@ -772,6 +776,8 @@ github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV
 github.com/vbauerster/mpb/v7 v7.4.1 h1:NhLMWQ3gNg2KJR8oeA9lO8Xvq+eNPmixDmB6JEQOUdA=
 github.com/vbauerster/mpb/v7 v7.4.1/go.mod h1:Ygg2mV9Vj9sQBWqsK2m2pidcf9H3s6bNKtqd3/M4gBo=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
+github.com/viney-shih/go-lock v1.1.2 h1:3TdGTiHZCPqBdTvFbQZQN/TRZzKF3KWw2rFEyKz3YqA=
+github.com/viney-shih/go-lock v1.1.2/go.mod h1:Yijm78Ljteb3kRiJrbLAxVntkUukGu5uzSxq/xV7OO8=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=

pkg/apiserver/controller/log/log.go

@@ -26,6 +26,7 @@ import (
 	"github.com/PaddlePaddle/PaddleFlow/pkg/common/logger"
 	"github.com/PaddlePaddle/PaddleFlow/pkg/common/schema"
 	"github.com/PaddlePaddle/PaddleFlow/pkg/job/runtime"
+	"github.com/PaddlePaddle/PaddleFlow/pkg/trace_logger"
 )
 
 type GetRunLogRequest struct {
@@ -36,7 +37,7 @@ type GetRunLogRequest struct {
 }
 
 type GetRunLogResponse struct {
-	SubmitLog schema.LogInfo      `json:"submitLog"`
+	SubmitLog string              `json:"submitLog"`
 	RunLog    []schema.JobLogInfo `json:"runLog"`
 	RunID     string              `json:"runID"`
 }
@@ -100,6 +101,13 @@ func GetRunLog(ctx *logger.RequestContext, runID string, request GetRunLogReques
 		}
 		response.RunLog = append(response.RunLog, jobLogInfo)
 	}
+	trace, ok := trace_logger.GetTraceFromCache(runID)
+	if !ok {
+		ctx.Logging().Warnf("get trace log failed. runID[%s]", runID)
+	} else {
+		response.SubmitLog = trace.String()
+	}
+
 	return response, nil
 }