secrets (!10)

Branch auto created by JayporeCI

<details>
    <summary>JayporeCi: 🟢 03fe80b8b2</summary>

```mermaid
flowchart TB

            subgraph Pipeline
                direction TB

            end

            subgraph Docker
                direction TB

                s_Docker(( )) -.-> Docker_0(JciEnv):::passed
                s_Docker(( )) -.-> Docker_1(Jci):::passed
            end

            subgraph Jobs
                direction TB

                s_Jobs(( )) -.-> Jobs_0(black):::passed
                s_Jobs(( )) -.-> Jobs_1(pylint):::passed
                s_Jobs(( )) -.-> Jobs_2(pytest):::passed
            end

            subgraph Publish
                direction TB

                s_Publish(( )) -.-> Publish_0(DockerHubJci):::passed
                s_Publish(( )) -.-> Publish_1(PublishDocs):::passed
                s_Publish(( )) -.-> Publish_2(DockerHubJcienv):::passed
            end

            Pipeline ---> Docker

            Docker ---> Jobs

            Jobs ---> Publish

            classDef pending fill:#aaa, color:black, stroke:black,stroke-width:2px,stroke-dasharray: 5 5;
            classDef skipped fill:#aaa, color:black, stroke:black,stroke-width:2px;
            classDef assigned fill:#ddd, color:black, stroke:black,stroke-width:2px;
            classDef running fill:#bae1ff,color:black,stroke:black,stroke-width:2px,stroke-dasharray: 5 5;
            classDef passed fill:#88d8b0, color:black, stroke:black;
            classDef failed fill:#ff6f69, color:black, stroke:black;
            classDef timeout fill:#ffda9e, color:black, stroke:black;
```

Co-authored-by: arjoonn sharma <[email protected]>
Reviewed-on: https://gitea.midpathsoftware.com/midpath/jaypore_ci/pulls/10

.gitignore

@@ -1 +1,3 @@
 *.pyc
+*.env
+*.age

Dockerfile

@@ -1,7 +1,7 @@
 from    python:3.11 as jcibase
 workdir /app
 run     apt-get update
-run     apt-get install ca-certificates curl gnupg lsb-release -y
+run     apt-get install ca-certificates curl zip gnupg lsb-release vim -y
 run     mkdir -p /etc/apt/keyrings
 run     curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
 run     echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
@@ -15,7 +15,12 @@ add     pyproject.toml .
 add     poetry.lock .
 run     poetry export --with dev > req.txt
 run     python3 -m pip install -r req.txt
-env     PYTHONPATH=/jaypore_ci/run/
+env     PYTHONPATH=/jaypore_ci/run/:/app
+env     PATH=/jaypore_ci/run/:/app:$PATH
+run     wget -O /bin/sops https://github.com/mozilla/sops/releases/download/v3.7.3/sops-v3.7.3.linux
+run     wget -O ./age.tar.gz https://github.com/FiloSottile/age/releases/download/v1.0.0/age-v1.0.0-linux-amd64.tar.gz
+run     tar xf ./age.tar.gz && mv ./age/age /bin && mv ./age/age-keygen /bin && rm -rf ./age
+run     chmod u+x /bin/sops /bin/age /bin/age-keygen
 
 from jcienv as jci
 add     jaypore_ci/ /app/jaypore_ci

README.md

@@ -1,5 +1,6 @@
 # Jaypore CI
 
+
 > A CI system that sounds ancient and powerful.
 > Like the city of Jaypore.
 

cicd/build_and_publish_docs.sh

@@ -0,0 +1,27 @@
+#! /bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+build() {
+    echo "Building docs"
+    sphinx-build docs/source/ docs/build
+    (cd docs/build && zip -r ../../website.zip ./)
+}
+publish() {
+    echo "Publishing docs"
+    source cicd/set_env.sh
+    curl -H "Content-Type: application/zip" \
+         -H "Authorization: Bearer $NETLIFY_TOKEN" \
+         --data-binary "@website.zip" \
+         https://api.netlify.com/api/v1/sites/$NETLIFY_SITEID/deploys
+}
+
+(build)
+if [ $1 == "main" ]
+then
+    (publish)
+else
+    echo "Not publishing since branch is: $1"
+fi

cicd/build_and_push_docker.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source cicd/set_env.sh
+docker login -u arjoonn -p=$DOCKER_PWD
+docker build -t $1:latest .
+docker tag  $1:latest arjoonn/$1:latest
+docker push arjoonn/$1:latest

cicd/cicd.py

@@ -1,3 +1,4 @@
+import requests
 from jaypore_ci import jci
 
 
@@ -11,29 +12,6 @@
         p.job("pylint", "python3 -m pylint jaypore_ci/ tests/")
         p.job("pytest", "python3 -m pytest tests/")
     with p.stage("Publish", image=jcienv):
-        # docs
-        p.job("BuildDocs", "sphinx-build -b html docs/source/ docs/build/html")
-        p.job("PublishDocs", "bash -c 'echo hi'", depends_on=["BuildDocs"])
-        # pypi
-        p.job("PoetryBuild", "poetry build")
-        p.job("PoetryPublish", "poetry publish", depends_on=["PoetryBuild"])
-        # jcienv
-        p.job(
-            "DockerTagJcienv",
-            "docker tag -t jcienv:{p.remote.sha} arjoonn/jcienv:{p.remote.sha}",
-        )
-        p.job(
-            "DockerPublishJcienv",
-            "docker push arjoonn/jcienv:{p.remote.sha}",
-            depends_on=["DockerTagJcienv"],
-        )
-        # jci
-        p.job(
-            "DockerTagJci",
-            "docker tag -t jci:{p.remote.sha} arjoonn/jci:{p.remote.sha}",
-        )
-        p.job(
-            "DockerPublishJci",
-            "docker push arjoonn/jci:{p.remote.sha}",
-            depends_on=["DockerTagJci"],
-        )
+        p.job("PublishDocs", f"bash cicd/build_and_publish_docs.sh {p.remote.branch}")
+        p.job("DockerHubJcienv", "bash cicd/build_and_push_docker.sh jcienv")
+        p.job("DockerHubJci", "bash cicd/build_and_push_docker.sh jci")

cicd/edit_secrets.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+
+main (){
+    NAME=$1
+    SOPS_AGE_KEY_FILE=secrets/$NAME.age sops --decrypt --input-type dotenv --output-type dotenv secrets/$NAME.enc > secrets/$NAME.env
+    vim secrets/$NAME.env
+    sops --encrypt --age $(age-keygen -y secrets/$NAME.age) secrets/$NAME.env > secrets/$NAME.enc
+    rm secrets/$NAME.env
+}
+
+help_message (){
+
+    echo "
+    Easily edit env files.
+    Make sure you have age keys available in
+
+        secrets/<envname>.age
+
+    If that is available you can run the following to edit env files.
+
+        edit_secrets.sh <envname>
+
+    Upon exiting the editor the file will be re-encrypted.
+    "
+}
+
+if [[ $1 == "--help" || $1 == "-h" ]]; then
+    help_message
+    exit 0
+fi
+if [ -z $1 ]; then
+    help_message
+    exit 0
+fi
+(main $1)

cicd/pre-push.githook

@@ -26,7 +26,12 @@ main() {
         -v /tmp/jaypore_$SHA:/jaypore_ci/run \
         --workdir /jaypore_ci/run \
         jcienv \
-        bash -c 'cp -r /jaypore_ci/repo/. /jaypore_ci/run && cd /jaypore_ci/run/ && git clean -fdx && python cicd/cicd.py'
+        bash -c 'cp -r /jaypore_ci/repo/. /jaypore_ci/run \
+              && cd /jaypore_ci/run/ \
+              && git clean -fdx \
+              && rm -r /jaypore_ci/run/secrets \
+              && cp -r /jaypore_ci/repo/secrets/ /jaypore_ci/run \
+              && python cicd/cicd.py'
     echo '----------------------------------------------'
 }
 (main)

cicd/set_env.sh

@@ -0,0 +1,8 @@
+#! /bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+
+export $(SOPS_AGE_KEY_FILE=secrets/jaypore_ci.age  sops --decrypt --input-type dotenv --output-type dotenv secrets/jaypore_ci.enc | xargs)

docs/source/conf.py

@@ -22,5 +22,17 @@
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
+html_sidebars = {
+    "**": [
+        "about.html",
+        "navigation.html",
+        "relations.html",
+        "searchbox.html",
+        "donate.html",
+    ]
+}
 html_theme = "alabaster"
 html_static_path = ["_static"]
+html_theme_options = {
+    "logo": "logo.png",
+}

docs/source/examples.rst

@@ -24,7 +24,6 @@ You can cache your environment dependencies in docker easily.
 Complex dependencies between jobs
 ---------------------------------
 
-
 - A pipeline can have stages.
 - Stages are executed one after the other.
 - Jobs inside a stage are all run in parallel

docs/source/getting_started.rst

@@ -5,43 +5,41 @@ Installation
 ------------
 
 
-To use Jaypore CI, first install it using the bash script for installation:
+To use **Jaypore CI**, first install it using a bash script.
 
 .. code-block:: console
 
-   $ curl https://raw.githubusercontent.com/theSage21/jaypore_ci/main/setup.sh | bash
+   $ curl \
+     https://raw.githubusercontent.com/theSage21/jaypore_ci/main/setup.sh \
+     | bash
 
 
-This will
+Doing this will:
 
 1. Create a directory called `cicd` in the root of your repo.
 2. Create a file `cicd/pre-push.githook`
 3. Create a file `cicd/cicd.py`
 4. Update your repo's pre-push git hook so that it runs the `cicd/pre-push.githook` file when you push.
 
 
-How it works
+Basic config
 ------------
 
-1. Every time you run `git push`, the `cicd/pre-push.githook` is run.
-    1. This hook runs your `cicd/cicd.py` file using the `arjoonn/jaypore_ci:latest` docker container.
-    2. The run has access to your local docker instance and so can launch other containers.
-
-
-A Basic Config
---------------
-
-Your entire config is inside `cicd/cicd.py`. Edit it to whatever you like! A simple config would look like this:
+Your entire config is inside `cicd/cicd.py`. Edit it to whatever you like! A basic config would look like this:
 
 .. code-block:: python
 
     from jaypore_ci import jci
 
     with jci.Pipeline(image='mydocker/image') as p:
-        p.job("Black", "python3 -m black --check .")
-        p.job("Pylint", "python3 -m pylint mycode/ tests/")
-        p.job("PyTest", "python3 -m pytest tests/")
+        p.job("Black", "black --check .")
+        p.job("Pylint", "pylint mycode/ tests/")
+        p.job("PyTest", "pytest tests/")
+
+
+After you make these changes you can `git add -Av` and `git commit -m 'added Jaypore CI'`.
 
+When you do a `git push origin`, that's when the CI system will get triggered and will run the CI.
 
 This config will run three jobs in parallel, using the `mydocker/image` docker image.
 

docs/source/ideas.rst

@@ -6,5 +6,25 @@ I'm tired of
 
 - Spending hours figuring out how to do stuff in YAML configs.
 - Shift to something else when pricing changes.
-- Debugging why stuff is not working in a CI system but works otherwise.
+- Debugging why stuff is not working only in the CI system.
 - Not being able to run CI without internet.
+- In the case of self CI runners hosted on laptops, I don't want my laptop bogged down by jobs that other people pushed.
+
+
+What I like about existing systems
+----------------------------------
+
+- Use docker to run things.
+- Stateless job runs.
+- Job graphs showing status of the run.
+- We cannot merge PRs unless the pipeline passes.
+- Able to handle multiple languages easily in the same project.
+
+
+Concepts used
+-------------
+
+- We use a git hook as a trigger mechanism. There is no possibility that some "CI server" is down.
+- Jobs are run on the machine that pushed the job by default. If you write bad code, your machine suffers first.
+- CI run status is posted directly in the PR description. You don't have to click and reach another website to see what your job is doing.
+- All jobs run inside docker containers.

docs/source/index.rst

@@ -6,19 +6,29 @@
 Welcome to Jaypore CI's documentation!
 ======================================
 
-**Jaypore CI** is a Python library for continuous integration / testing / delivery.
+**Jaypore CI** is a small system for continuous integration / testing / delivery.
 
-It is slightly different from things like github actions, gitlab CI, drone CI and so on.
+It is different from the usual suspects like github actions, gitlab CI, drone CI and so on.
 
 - The configuration language is python.
 - CI runs on your local machine by default. 
-- There is no "server". CI reports are added to your pull request / merge request description.
+- There is no "server". You can run offline.
 
 
 For example, here's a CI pipeline for a project.
 
-.. image:: example.png
-  :alt: Example pipeline graph
+.. code-block:: python
+
+    from jaypore_ci import jci
+
+    with jci.Pipeline(image='mydockerhub/env_image') as p:
+        p.job("Black", "black --check .")
+        p.job("Pylint", "pylint mycode/ tests/")
+        p.job("PyTest", "pytest tests/")
+
+
+Go through the :doc:`getting_started` doc to set up your first instance.
+
 
 Contents
 ---------------

jaypore_ci/gitea.py

@@ -122,9 +122,9 @@ def publish(self, report: str, status: str):
         """
         Will publish the report to the remote.
 
-            report: Report to write to remote.
-            status: One of ["pending", "success", "error", "failure", "warning"]
-                    This is the dot next to each commit in gitea.
+        :param report: Report to write to remote.
+        :param status: One of ["pending", "success", "error", "failure",
+            "warning"] This is the dot next to each commit in gitea.
         """
         assert status in ("pending", "success", "error", "failure", "warning")
         issue_id = self.get_pr_id()

jaypore_ci/interfaces.py

@@ -8,7 +8,6 @@
 
 class TriggerFailed(Exception):
     "Failure to trigger a job"
-    ...
 
 
 class Executor:

jaypore_ci/jci.py

@@ -107,11 +107,9 @@ def logging(self):
 
     def update_report(self):
         """
-        Update the status report.
-        Usually called when a job changes some of it's internal state like:
-            - logs are updated
-            - status has changed
-            - last_check is called
+        Update the status report. Usually called when a job changes some of
+        it's internal state like when logs are updated or when status has
+        changed.
         """
         self.logging().debug("Update report")
         status = {
@@ -304,7 +302,7 @@ def render_report(self):
 
 </details>"""
 
-    def __render_graph__(self) -> str:
+    def __render_graph__(self) -> str:  # pylint: disable=too-many-locals
         """
         Render a mermaid graph given the jobs in the pipeline.
         """

secrets/jaypore_ci.enc

@@ -0,0 +1,10 @@
+DOCKER_USER=ENC[AES256_GCM,data:kodwp9BekQ==,iv:4kxwikp9m8obe3Hb1MCLdDOGga1VUnqxLPU4E6s25D4=,tag:XaRuNYq8k9QNlQ/Ftq4Atg==,type:str]
+DOCKER_PWD=ENC[AES256_GCM,data:GViemE/cvlOfMdmeDdqaM83J5E5fs590KIftj5Ul3QBafTQ+,iv:bIizRqtgXS4IY7Ml01etoQ01UvV5Efkicz2epTTlcNI=,tag:DHkC4pOimYL3siMgAbkyRg==,type:str]
+NETLIFY_TOKEN=ENC[AES256_GCM,data:h1yMdRv4ps+hstSCiLN388g/vq6VwYIsXT8C0Uxq7n5bA04syWosivoj1g==,iv:+m20LBp2NqzQ/EOdTXRLTe76N5kdMTqgGcuZ9QsCiL4=,tag:b7Kew1KQ6PfCIWZLvVF/9Q==,type:str]
+NETLIFY_SITEID=ENC[AES256_GCM,data:lr2RcnMdHKsXTWfwhx5VM0n9hiCF2MaezajRJRjYtmD8ILFY,iv:cHyNXTYZ7nW+JAvp29jncf6GWmCHN0obJvihoYQYHic=,tag:1618mQjs7CMFfwhLvaVQ0Q==,type:str]
+sops_mac=ENC[AES256_GCM,data:FIMFiy2ZQHmi/8WS+4etoiQhEFgXoGlPIUY14icexxxj2GdkGth3suDLFdQ5x92Jnq5quBkz52qQmE56fO/coIYoBCR2rZzaEmu68gDRlsfUv48vMTIHueAvN/DaEiyFwpGcg4HWoZNirbciCDyI3HBK8/C4/IpWb/arwrrNIKY=,iv:InPgDMxTdZigJN/T3TsqvQoUuBMYalAovP2dT5a7Fx8=,tag:3agYFIaLQTtr3QOPzDNdww==,type:str]
+sops_version=3.7.3
+sops_age__list_0__map_recipient=age1u0zd477nnqqausg4vtxl3laxz73t5tgt9qw8enmpeeadcmes3eusw3v3m9
+sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAvSzl4T2NhRFl3d2xDZkxE\nUk5lMlBtemV5UXVGMkxCbE1ma3JBbDJWcWdnCmlRejRqbUFJTnVPWVF2dUh1TThk\nZnVvWjBjTXNHTUpRd2NxU2pXVkVqQzgKLS0tIFcyNmhkbE5SazQvZ01nZ2dlcklX\ndVh5NkFuZkx3enV2Y3hFMUxmMzJhUVkKy6VXe+51BaDRN8I0KsyaviseaHSCInMY\nxMxiKM8zcs39OAv7VAkQZ6O6lhBi0digoqYMOYxPykzE7U9MfTE2aA==\n-----END AGE ENCRYPTED FILE-----\n
+sops_lastmodified=2022-12-23T06:01:23Z
+sops_unencrypted_suffix=_unencrypted