Dont write output if they havent changed. Distinct retry timer on err…

…or and success. Re-Read on api write if the target doesnt respond with data. json_query possible on read
sebt3 · Jan 27, 2025 · 69b7237 · 69b7237
1 parent 7e60025
commit 69b7237
Show file tree

Hide file tree

Showing 10 changed files with 766 additions and 198 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "kuberest"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Sébastien Huss <[email protected]>"]
 edition = "2021"
 default-run = "controller"
@@ -26,7 +26,7 @@ operator = { cmd=[
 ]}
 precommit = { cmd=[
     "cargo update",
-    "cargo clippy --fix --allow-dirty",
+    "cargo clippy --fix --allow-dirty --allow-staged",
     "cargo cmd generate",
     "cargo +nightly fmt"
 ]}
@@ -81,6 +81,7 @@ base64 = "0.22.1"
 rand = "0.8.5"
 argon2 = { version = "0.5.3", features = ["std"] }
 bcrypt = "0.16.0"
+serde_json_path = "0.7.1"
 
 [dev-dependencies]
 assert-json-diff = "2.0.2"

diff --git a/charts/kuberest/Chart.yaml b/charts/kuberest/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: kuberest
 description: Allow to Control remote REST api endpoints from the confort of your cluster
 type: application
-version: "1.2.0"
-appVersion: "1.2.0"
+version: "1.3.0"
+appVersion: "1.3.0"
diff --git a/deploy/crd/crd.yaml b/deploy/crd/crd.yaml
@@ -36,7 +36,7 @@ spec:
             description: Describe the specification of a RestEndPoint
             properties:
               checkFrequency:
-                description: 'checkFrequency define the pooling interval (in seconds, default: 300)'
+                description: 'checkFrequency define the pooling interval (in seconds, default: 3600 aka 1h)'
                 format: uint64
                 minimum: 0.0
                 nullable: true
@@ -258,12 +258,20 @@ spec:
                       items:
                         description: readGroupItem describe an object to read with the client
                         properties:
+                          json_query:
+                            description: Get the result from a json-query
+                            nullable: true
+                            type: string
                           key:
                             description: configuration of this object
                             type: string
                           name:
                             description: name of the item (used for handlebars renders)
                             type: string
+                          optional:
+                            description: Allow missing object (default false)
+                            nullable: true
+                            type: boolean
                         required:
                         - key
                         - name
@@ -288,6 +296,12 @@ spec:
                   type: object
                 nullable: true
                 type: array
+              retryFrequency:
+                description: 'retryFrequency define the pooling interval if previous try have failed (in seconds, default: 300 aka 5mn)'
+                format: uint64
+                minimum: 0.0
+                nullable: true
+                type: integer
               teardown:
                 description: A rhai teardown-script for a final cleanup on RestEndPoint deletion
                 nullable: true
@@ -315,7 +329,7 @@ spec:
                   description: writeGroup describe a rest endpoint within the client sub-paths,
                   properties:
                     createMethod:
-                      description: 'Method to use when creating an object (default: Get)'
+                      description: 'Method to use when creating an object (default: Post)'
                       enum:
                       - Post
                       nullable: true
@@ -334,6 +348,14 @@ spec:
                           name:
                             description: 'name of the item (used for handlebars renders: write.<group>.<name>)'
                             type: string
+                          readJsonQuery:
+                            description: If writes doesnt return values, (only used when readPath is specified too)
+                            nullable: true
+                            type: string
+                          readPath:
+                            description: If writes doesnt return values, use this read query to re-read
+                            nullable: true
+                            type: string
                           teardown:
                             description: 'Delete the Object on RestEndPoint deletion (default: true, inability to do so will block RestEndPoint)'
                             nullable: true
@@ -361,7 +383,7 @@ spec:
                       description: path appended to the client's baseurl for this group of objects
                       type: string
                     readMethod:
-                      description: 'Method to use when reading an object (default: Post)'
+                      description: 'Method to use when reading an object (default: Get)'
                       enum:
                       - Get
                       nullable: true
@@ -378,6 +400,10 @@ spec:
                       - Post
                       nullable: true
                       type: string
+                    updatePath:
+                      description: Path to use to update/delete this write_group
+                      nullable: true
+                      type: string
                   required:
                   - items
                   - name
@@ -422,6 +448,8 @@ spec:
                       - PostScriptFailed
                       - TeardownScriptFailed
                       - ReadFailed
+                      - ReadMissing
+                      - ReReadFailed
                       - WriteFailed
                       - WriteDeleteFailed
                       - WriteAlreadyExist

diff --git a/deploy/operator/deployment.yaml b/deploy/operator/deployment.yaml
@@ -8,7 +8,7 @@ metadata:
   labels:
     app: kuberest
     app.kubernetes.io/name: kuberest
-    app.kubernetes.io/version: "1.2.0"
+    app.kubernetes.io/version: "1.3.0"
   namespace: default
 automountServiceAccountToken: true
 ---
@@ -102,7 +102,7 @@ metadata:
   labels:
     app: kuberest
     app.kubernetes.io/name: kuberest
-    app.kubernetes.io/version: "1.2.0"
+    app.kubernetes.io/version: "1.3.0"
 spec:
   type: ClusterIP
   ports:
@@ -122,7 +122,7 @@ metadata:
   labels:
     app: kuberest
     app.kubernetes.io/name: kuberest
-    app.kubernetes.io/version: "1.2.0"
+    app.kubernetes.io/version: "1.3.0"
 spec:
   replicas: 1
   selector:
@@ -140,7 +140,7 @@ spec:
         {}
       containers:
       - name: kuberest
-        image: sebt3/kuberest:1.2.0
+        image: sebt3/kuberest:1.3.0
         imagePullPolicy: IfNotPresent
         securityContext:
           {}

diff --git a/examples/gitea/organisations-team.yaml b/examples/gitea/organisations-team.yaml
@@ -0,0 +1,111 @@
+apiVersion: kuberest.solidite.fr/v1
+kind: RestEndPoint
+metadata:
+  name: gitea-org-images
+spec:
+  inputs:
+  - name: admin
+    secretRef:
+      name: gitea-admin-user
+  client:
+    baseurl: https://gitea.your-company.com/api/v1
+    headers:
+      Authorization: '{{ header_basic (base64_decode input.admin.data.username) (base64_decode input.admin.data.password) }}'
+  writes:
+  - name: orgs
+    path: orgs
+    keyName: username
+    items:
+    - name: org
+      values: |
+        username: images
+        visibility: public
+        repo_admin_change_team_access: true
+  - name: teams
+    path: orgs/images/teams
+    updatePath: teams
+    items:
+    - name: dev
+      values: |-
+        name: dev
+        includes_all_repositories: true
+        can_create_org_repo: false
+        permission: read
+        units:
+        - repo.issues
+        - repo.ext_issues
+        - repo.releases
+        - repo.ext_wiki
+        - repo.projects
+        - repo.actions
+        - repo.code
+        - repo.pulls
+        - repo.wiki
+        - repo.packages
+        units_map:
+          repo.actions: write
+          repo.code: write
+          repo.ext_issues: read
+          repo.ext_wiki: read
+          repo.issues: write
+          repo.packages: write
+          repo.projects: write
+          repo.pulls: write
+          repo.releases: read
+          repo.wiki: write
+    - name: qa
+      values: |-
+        name: qa
+        includes_all_repositories: true
+        can_create_org_repo: false
+        permission: read
+        units:
+        - repo.issues
+        - repo.ext_issues
+        - repo.releases
+        - repo.ext_wiki
+        - repo.projects
+        - repo.actions
+        - repo.code
+        - repo.pulls
+        - repo.wiki
+        - repo.packages
+        units_map:
+          repo.actions: read
+          repo.code: read
+          repo.ext_issues: read
+          repo.ext_wiki: read
+          repo.issues: write
+          repo.packages: read
+          repo.projects: write
+          repo.pulls: read
+          repo.releases: read
+          repo.wiki: write
+    - name: read
+      values: |-
+        name: read
+        includes_all_repositories: true
+        can_create_org_repo: false
+        permission: read
+        units:
+        - repo.issues
+        - repo.ext_issues
+        - repo.releases
+        - repo.ext_wiki
+        - repo.projects
+        - repo.actions
+        - repo.code
+        - repo.pulls
+        - repo.wiki
+        - repo.packages
+        units_map:
+          repo.actions: read
+          repo.code: read
+          repo.ext_issues: read
+          repo.ext_wiki: read
+          repo.issues: read
+          repo.packages: read
+          repo.projects: read
+          repo.pulls: read
+          repo.releases: read
+          repo.wiki: read
diff --git a/examples/harbor/mirror-docker.yaml b/examples/harbor/mirror-docker.yaml
@@ -0,0 +1,37 @@
+apiVersion: kuberest.solidite.fr/v1
+kind: RestEndPoint
+metadata:
+  name: harbor-mirror-docker
+spec:
+  inputs:
+  - name: admin
+    secretRef:
+      name: harbor-basic
+  client:
+    baseurl: https://harbor.your-company.com/api/v2.0
+    headers:
+      Authorization: '{{ header_basic "admin" (base64_decode input.admin.data.HARBOR_ADMIN_PASSWORD) }}'
+  writes:
+  - name: registries
+    path: registries
+    updateMethod: Put
+    items:
+    - name: mirror
+      readPath: registries?name=docker
+      readJsonQuery: $[0]
+      values: |
+        name: docker
+        url: https://hub.docker.com
+        type: docker-hub
+  - name: projects
+    path: projects
+    updateMethod: Put
+    keyName: project_id
+    items:
+    - name: mirror
+      readPath: projects?name=docker
+      readJsonQuery: $[0]
+      values: |-
+        project_name: docker
+        public: true
+        registry_id: {{ write.registries.mirror.id }}
diff --git a/src/httphandler.rs b/src/httphandler.rs
@@ -106,9 +106,9 @@ impl RestClient {
     }
 
     pub fn add_header_json_accept(&mut self) -> &mut RestClient {
-        for (key, val) in self.headers.clone() {
+        /*for (key, val) in self.headers.clone() {
             debug!("RestClient.header: {:} {:}", key, val);
-        }
+        }*/
         if self.headers.clone().into_iter().any(|(c, _)| c == *"Accept") {
             self
         } else {