[Feature] Export metrics in OpenMetrics text format

Author: kaushikmadhu

pom.xml

@@ -99,6 +99,7 @@
         <swagger-annotations.version>1.5.10</swagger-annotations.version>
         <hibernate.validator.version>6.2.2.Final</hibernate.validator.version>
         <javax.annotation-api.version>1.3.2</javax.annotation-api.version>
+        <prometheus.simpleclient.version>0.16.0</prometheus.simpleclient.version>
         <jsoup.version>1.14.3</jsoup.version>
         <jwt.version>0.10.7</jwt.version>
         <cron-utils.version>9.1.6</cron-utils.version>
@@ -352,6 +353,12 @@
                 <artifactId>spring-boot-starter-jdbc</artifactId>
                 <version>${spring-boot.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-starter-aop</artifactId>
+                <version>${spring-boot.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>org.springframework.security</groupId>
                 <artifactId>spring-security-core</artifactId>
@@ -378,7 +385,29 @@
                 <artifactId>druid-spring-boot-starter</artifactId>
                 <version>${druid-spring-boot-starter.version}</version>
             </dependency>
-
+            <!-- Metric Dependencies -->
+            <dependency>
+                <groupId>io.prometheus</groupId>
+                <artifactId>simpleclient</artifactId>
+                <version>${prometheus.simpleclient.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>io.prometheus</groupId>
+                <artifactId>simpleclient_common</artifactId>
+                <version>${prometheus.simpleclient.version}</version>
+            </dependency>
+            <!-- Hotspot JVM metrics-->
+            <dependency>
+                <groupId>io.prometheus</groupId>
+                <artifactId>simpleclient_hotspot</artifactId>
+                <version>${prometheus.simpleclient.version}</version>
+            </dependency>
+            <!-- Exposition HTTPServer-->
+            <dependency>
+                <groupId>io.prometheus</groupId>
+                <artifactId>simpleclient_httpserver</artifactId>
+                <version>${prometheus.simpleclient.version}</version>
+            </dependency>
             <!-- ORM -->
             <dependency>
                 <groupId>com.baomidou</groupId>

seatunnel-server/seatunnel-app/pom.xml

@@ -325,6 +325,11 @@
             <artifactId>spring-security-ldap</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-aop</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.baomidou</groupId>
             <artifactId>mybatis-plus-boot-starter</artifactId>
@@ -386,6 +391,24 @@
             <artifactId>h2</artifactId>
         </dependency>
 
+        <!-- Metric Dependencies -->
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient_common</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient_hotspot</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient_httpserver</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>

seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/adapter/SeatunnelWebAdapter.java

@@ -44,6 +44,7 @@ public class SeatunnelWebAdapter implements WebMvcConfigurer {
     public static final String REGISTER_PATH_PATTERN = "/users/register";
     private static final String RESOURCE_NAME_PATH_PATTERN =
             "/seatunnel/api/v1/resources/workspace";
+    private static final String METRIC_PATH_PATTERN = "/metrics";
 
     @Bean
     public AuthenticationInterceptor authenticationInterceptor() {
@@ -77,6 +78,7 @@ public void addInterceptors(InterceptorRegistry registry) {
                         LOGIN_PATH_PATTERN,
                         REGISTER_PATH_PATTERN,
                         RESOURCE_NAME_PATH_PATTERN,
+                        METRIC_PATH_PATTERN,
                         "/swagger-resources/**",
                         "/webjars/**",
                         "/v2/**",

seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobDefinitionController.java

@@ -22,6 +22,8 @@
 import org.apache.seatunnel.app.domain.request.job.JobReq;
 import org.apache.seatunnel.app.domain.response.PageInfo;
 import org.apache.seatunnel.app.domain.response.job.JobDefinitionRes;
+import org.apache.seatunnel.app.metrics.annotations.Counted;
+import org.apache.seatunnel.app.metrics.annotations.Timed;
 import org.apache.seatunnel.app.service.IJobDefinitionService;
 import org.apache.seatunnel.app.service.IJobTaskService;
 import org.apache.seatunnel.server.common.CodeGenerateUtils;
@@ -54,6 +56,12 @@ public class JobDefinitionController {
      *
      * @return created job id
      */
+    @Counted(
+            name = "create_job_definition_request",
+            help = "Total number of create job definition requests")
+    @Timed(
+            name = "create_job_definition_request_latency",
+            help = "Latency of create job definition API in seconds")
     @PostMapping
     @ApiOperation(value = "create job definition", httpMethod = "POST")
     Result<Long> createJobDefinition(@RequestBody JobReq jobReq)
@@ -65,6 +73,9 @@ Result<Long> createJobDefinition(@RequestBody JobReq jobReq)
         }
     }
 
+    @Counted(
+            name = "get_job_definitions_request",
+            help = "Total number of get job definitions request")
     @GetMapping
     @ApiOperation(value = "get job definition", httpMethod = "GET")
     Result<PageInfo<JobDefinitionRes>> getJobDefinition(
@@ -75,12 +86,21 @@ Result<PageInfo<JobDefinitionRes>> getJobDefinition(
         return Result.success(jobService.getJob(searchName, pageNo, pageSize, jobMode));
     }
 
+    @Counted(
+            name = "get_job_definition_by_id_request",
+            help = "Total number of get job definition by id requests")
     @GetMapping("/{jobId}")
     @ApiOperation(value = "get job definition", httpMethod = "GET")
     Result<JobDefinition> getJobDefinition(@PathVariable long jobId) {
         return Result.success(jobService.getJobDefinitionByJobId(jobId));
     }
 
+    @Counted(
+            name = "delete_job_definition_request",
+            help = "Total number of delete job definition requests")
+    @Timed(
+            name = "delete_job_definition_request_latency",
+            help = "Latency of delete job definition API in seconds")
     @DeleteMapping
     @ApiOperation(value = "delete job definition", httpMethod = "DELETE")
     Result<Void> deleteJobDefinition(

seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobExecutorController.java

@@ -22,6 +22,7 @@
 import org.apache.seatunnel.app.domain.request.job.JobExecParam;
 import org.apache.seatunnel.app.domain.response.executor.JobExecutionStatus;
 import org.apache.seatunnel.app.domain.response.executor.JobExecutorRes;
+import org.apache.seatunnel.app.metrics.annotations.Counted;
 import org.apache.seatunnel.app.service.IJobExecutorService;
 import org.apache.seatunnel.app.service.IJobInstanceService;
 import org.apache.seatunnel.app.service.ITaskInstanceService;
@@ -53,6 +54,7 @@ public class JobExecutorController {
     @Resource private IJobInstanceService jobInstanceService;
     @Resource private ITaskInstanceService<SeaTunnelJobInstanceDto> taskInstanceService;
 
+    @Counted(name = "execute_job_request", help = "Total number of execute job requests")
     @PostMapping("/execute")
     @ApiOperation(value = "Execute synchronization tasks", httpMethod = "POST")
     public Result<Long> jobExecutor(
@@ -77,6 +79,7 @@ public Result<JobExecutorRes> resource(
         }
     }
 
+    @Counted(name = "pause_job_request", help = "Total number of pause job requests")
     @GetMapping("/pause")
     public Result<Void> jobPause(
             @ApiParam(value = "jobInstanceId", required = true) @RequestParam Long jobInstanceId) {

seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/JobTaskController.java

@@ -22,6 +22,8 @@
 import org.apache.seatunnel.app.domain.request.job.JobTaskInfo;
 import org.apache.seatunnel.app.domain.request.job.PluginConfig;
 import org.apache.seatunnel.app.domain.response.job.JobTaskCheckRes;
+import org.apache.seatunnel.app.metrics.annotations.Counted;
+import org.apache.seatunnel.app.metrics.annotations.Timed;
 import org.apache.seatunnel.app.service.IJobTaskService;
 
 import org.springframework.web.bind.annotation.DeleteMapping;
@@ -44,6 +46,10 @@ public class JobTaskController {
 
     @Resource private IJobTaskService jobTaskService;
 
+    @Counted(name = "save_job_dag_request", help = "Total number of save job dag requests")
+    @Timed(
+            name = "save_job_dag_request_latency",
+            help = "Latency of save job dag request in seconds")
     @PostMapping("/dag/{jobVersionId}")
     @ApiOperation(value = "save job dag", httpMethod = "POST")
     Result<JobTaskCheckRes> saveJobDAG(
@@ -59,6 +65,10 @@ Result<JobTaskInfo> getJob(
         return Result.success(jobTaskService.getTaskConfig(jobVersionId));
     }
 
+    @Counted(name = "save_job_task_request", help = "Total number of save job task requests")
+    @Timed(
+            name = "save_job_task_request_latency",
+            help = "Latency of save job task request in seconds")
     @PostMapping("/task/{jobVersionId}")
     @ApiOperation(value = "save or update single task", httpMethod = "POST")
     Result<Void> saveSingleTask(
@@ -77,6 +87,10 @@ Result<PluginConfig> getSingleTask(
         return Result.success(jobTaskService.getSingleTask(jobVersionId, pluginId));
     }
 
+    @Counted(name = "delete_job_task_request", help = "Total number of delete job task requests")
+    @Timed(
+            name = "delete_job_task_request_latency",
+            help = "Latency of delete job task request in seconds")
     @DeleteMapping("/task/{jobVersionId}")
     @ApiOperation(value = "delete single task", httpMethod = "DELETE")
     Result<Void> deleteSingleTask(

seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/SeatunnelDatasourceController.java

@@ -29,6 +29,8 @@
 import org.apache.seatunnel.app.domain.response.PageInfo;
 import org.apache.seatunnel.app.domain.response.datasource.DatasourceDetailRes;
 import org.apache.seatunnel.app.domain.response.datasource.DatasourceRes;
+import org.apache.seatunnel.app.metrics.annotations.Counted;
+import org.apache.seatunnel.app.metrics.annotations.Timed;
 import org.apache.seatunnel.app.service.IDatasourceService;
 import org.apache.seatunnel.app.utils.CartesianProductUtils;
 import org.apache.seatunnel.app.utils.PropertyUtils;
@@ -80,6 +82,8 @@ public class SeatunnelDatasourceController extends BaseController {
     private static final List<String> wsSupportDatasources =
             PropertyUtils.getList(Constants.WS_SUPPORT_DATASOURCES, Constants.COMMA);
 
+    @Counted(name = "create_datasource_requests", help = "Number of create datasource requests")
+    @Timed(name = "create_datasource_latency", help = "Latency of create datasource API")
     @ApiOperation(value = "create datasource", notes = "create datasource")
     @ApiImplicitParams({
         @ApiImplicitParam(
@@ -120,6 +124,10 @@ Result<String> createDatasource(@RequestBody DatasourceReq req) {
                         stringStringMap));
     }
 
+    @Counted(
+            name = "test_connection_requests",
+            help = "Number of test datasource connection requests")
+    @Timed(name = "test_connection_latency", help = "Latency of check datasource connection API")
     @ApiOperation(value = "test datasource connection", notes = "test datasource connection")
     @ApiImplicitParams({
         @ApiImplicitParam(
@@ -142,6 +150,8 @@ Result<Boolean> testConnect(@RequestBody DatasourceCheckReq req) {
                         req.getPluginName(), DEFAULT_PLUGIN_VERSION, req.getDatasourceConfig()));
     }
 
+    @Counted(name = "update_datasource_requests", help = "Number of update datasource requests")
+    @Timed(name = "update_datasource_latency", help = "Latency of update datasource API")
     @ApiOperation(value = "update datasource", notes = "update datasource")
     @ApiImplicitParams({
         @ApiImplicitParam(
@@ -176,6 +186,8 @@ Result<Boolean> updateDatasource(
                         stringStringMap));
     }
 
+    @Counted(name = "delete_datasource_requests", help = "Number of delete datasource requests")
+    @Timed(name = "delete_datasource_latency", help = "Latency of delete datasource API")
     @ApiOperation(value = "delete datasource by id", notes = "delete datasource by id")
     @DeleteMapping("/{id}")
     Result<Boolean> deleteDatasource(@PathVariable("id") String id) {

seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/TaskInstanceController.java

@@ -19,6 +19,8 @@
 
 import org.apache.seatunnel.app.common.Result;
 import org.apache.seatunnel.app.domain.dto.job.SeaTunnelJobInstanceDto;
+import org.apache.seatunnel.app.metrics.annotations.Counted;
+import org.apache.seatunnel.app.metrics.annotations.Timed;
 import org.apache.seatunnel.app.service.ITaskInstanceService;
 import org.apache.seatunnel.app.utils.PageInfo;
 import org.apache.seatunnel.common.constants.JobMode;
@@ -37,6 +39,8 @@ public class TaskInstanceController {
 
     @Autowired ITaskInstanceService<SeaTunnelJobInstanceDto> taskInstanceService;
 
+    @Counted(name = "number_of_job_metric_request", help = "total number of job metrics request")
+    @Timed(name = "job_metric_request_latency", help = "Latency of job metric api  in seconds")
     @GetMapping("/jobMetrics")
     @ApiOperation(value = "get the jobMetrics list ", httpMethod = "GET")
     public Result<PageInfo<SeaTunnelJobInstanceDto>> getTaskInstanceList(

seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/controller/UserController.java

@@ -25,6 +25,8 @@
 import org.apache.seatunnel.app.domain.response.PageInfo;
 import org.apache.seatunnel.app.domain.response.user.AddUserRes;
 import org.apache.seatunnel.app.domain.response.user.UserSimpleInfoRes;
+import org.apache.seatunnel.app.metrics.annotations.Counted;
+import org.apache.seatunnel.app.metrics.annotations.Timed;
 import org.apache.seatunnel.app.service.IUserService;
 
 import org.springframework.web.bind.annotation.DeleteMapping;
@@ -110,6 +112,8 @@ public Result<Void> disable(
         return Result.success();
     }
 
+    @Counted(name = "user_login", help = "total number of user login request")
+    @Timed(name = "user_login_request_latency", help = "Latency of login request in seconds")
     @PostMapping("/login")
     public Result<UserSimpleInfoRes> login(
             @RequestBody UserLoginReq req,

seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/metrics/annotations/Counted.java

@@ -0,0 +1,14 @@
+package org.apache.seatunnel.app.metrics.annotations;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface Counted {
+    String name();
+
+    String help() default "";
+}