openvehicles · llange · Feb 8, 2023 · Jan 29, 2023 · Jan 29, 2023 · May 1, 2023
diff --git a/.github/workflows/build-ovms.yml b/.github/workflows/build-ovms.yml
@@ -0,0 +1,253 @@
+---
+# This is an ESP-IDF workflow to build OVMS v3
+
+name: 'Build and store artifacts for OVMS v3'
+
+# Controls when the action will run.
+on:
+  # # Triggers the workflow on tag create like v1.0, v2.0.0 and so on
+  push:
+  #   tags:
+  #    - 'v*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called 'build_ovms'
+  build_ovms:
+
+    strategy:
+      matrix:
+        # idf_ver: ["latest", "release-v5.1", "release-v5.0", "v5.0.1", "v5.0"]
+        # idf_ver: ["v5.0.1", "v5.0", "v3.3.4"]
+        # idf_ver: ["release-v5.0", "v5.0.1", "v5.0", "release-v4.4", "v4.4.4", "v3.3.4"]
+        # idf_ver: ["latest", "release-v5.1", "release-v5.0", "v5.0.1", "v5.0", "release-v4.4", "v3.3.4"]
+        # idf_ver: ["release-v5.2", "release-v5.1", "release-v5.0", "v5.0.2", "v5.0.1", "v5.0", "release-v4.4", "v3.3.4"] # v3.3.4 is our own fork / special handling
+        idf_ver: ["v3.3.4"] # v3.3.4 is our own fork / special handling
+        idf_target: ["esp32"]
+        # mongoose_ver: ["7.9", "7.10", "6.11"] # 6.11 is our own fork / special handling
+        mongoose_ver: ["6.11"] # 6.11 is our own fork / special handling
+        include:
+
+          # All the following items are 'default' values for ALL the matrix entries
+          - patch_mongoose_6_11: true
+            idf_component_file: "idf_component.yml.esp5"
+            build_command: "idf.py build"
+            sdkconfig: "sdkconfig.defaults.esp5.0.1"
+            patch_esp_idf_ovms: false
+            force_wolfssl_470: false
+            patch_esp_idf_whole_archive: false
+            node20_compatible_container: false
+
+          # All the following items are updates of specific entries in the matrix
+          # - idf_ver: "v5.0"
+          #   sdkconfig: "sdkconfig.defaults.esp5"
+
+          # - idf_ver: "release-v4.4"
+          #   sdkconfig: "sdkconfig.defaults.esp4"
+          #   patch_esp_idf_whole_archive: true
+
+          # - idf_ver: "v4.4.4"
+          #   sdkconfig: "sdkconfig.defaults.esp4"
+          #   idf_component_file: "idf_component.yml.esp4"
+          #   patch_mongoose_6_11: false
+          #   patch_esp_idf_whole_archive: true
+
+          # - idf_ver: "v3.3.6"
+          #   build_command: "make -j all"
+          #   patch_mongoose_6_11: false
+          #   force_wolfssl_470: true
+
+          - idf_ver: "v3.3.4"
+            build_command: "TERM=vt100 make defconfig all"
+            sdkconfig: "sdkconfig.defaults.esp3"
+            patch_mongoose_6_11: false
+            patch_esp_idf_ovms: true
+            force_wolfssl_470: true
+            # force_wolfssl_470: false
+
+    # The type of runner that the job will run on
+    runs-on: 'ubuntu-22.04'
+    container:
+      image: 'espressif/idf:${{ matrix.idf_ver }}'
+
+    # For the moment, only on this branch
+    if: github.repository == 'llange/Open-Vehicle-Monitoring-System-3'
+    steps:
+
+      # For older builds (3.3.x), a few tweaks are needed:
+      # - the git version bundled in the official ESP-IDF docker image is too old
+      #   for GitHub's `actions/checkout` below. We upgrade it
+      # - We also replace the official ESP-IDF repo with OVMSv3's customized ESP-IDF repo
+      - name: 'Use ESP-IDF v3.3 with OVMS tweaks'
+        if: matrix.patch_esp_idf_ovms
+        run: |
+          # Git 2.17.x is not enough for the `actions/checkout` below - upgrade to a more recent version
+          apt-get update
+          apt-get install -y software-properties-common
+          add-apt-repository ppa:git-core/ppa
+          apt-get update
+          apt-get upgrade -y git
+          git --version
+          cd "${IDF_PATH}"
+          git remote set-url origin "https://github.com/openvehicles/esp-idf.git"
+          git fetch --all
+          git reset --hard
+          git clean -fxd
+          git pull origin master
+          git submodule update --init --recursive
+          ./install.sh esp32
+          cd -
+
+      # For older builds (4.4.x), a patch is needed:
+      # - the WHOLE_ARCHIVE statement in CMake components has only been introduced
+      #   in ESP-IDF v5.0. We patch it.
+      - name: 'Use ESP-IDF v4.4.x with WHOLE_ARCHIVE support'
+        if: matrix.patch_esp_idf_whole_archive
+        run: |
+          cd "${IDF_PATH}"
+          git config user.email "[email protected]"
+          git config user.name "Github CI"
+          git config merge.renameLimit 999999
+          git fetch --all
+          git reset --hard
+          git clean -fxd
+          git cherry-pick 273633ee310fbc18b17edfaeae3f3121508e3b8d
+          cd -
+
+      # We're now able to fetch our OVMSv3 firmware repo
+      - name: 'Checkout repo (non node20 compatible container)'
+        #  checkout@v4 action causes some problems with the espressif/idf container (see actions/checkout#1474),
+        if: ${{ ! matrix.node20_compatible_container }}
+        uses: 'actions/checkout@v3'
+        with:
+          submodules: 'recursive'
+
+      # We're now able to fetch our OVMSv3 firmware repo
+      - name: 'Checkout repo (node20 compatible container)'
+        if: matrix.node20_compatible_container
+        uses: 'actions/checkout@v4'
+        with:
+          submodules: 'recursive'
+
+      - name: 'fix error message'
+        run: 'git config --global --add safe.directory "$GITHUB_WORKSPACE"'
+
+      # For ESP-IDF v5+ builds, we need to patch our "old" version of mongoose
+      - name: 'Patch mongoose (ESP-IDF v5+)'
+        if: |
+          matrix.patch_mongoose_6_11 &&
+          (matrix.mongoose_ver == '6.11')
+        run: 'git apply --directory="vehicle/OVMS.V3/components/mongoose/mongoose" "vehicle/OVMS.V3/support/mongoose-espv5.patch"'
+
+      # For Mongose >= 7, we checkout this specific version
+      - name: 'Switch mongoose dir'
+        if: |
+          (matrix.mongoose_ver != '6.11')
+        run: |
+          cd "vehicle/OVMS.V3/components/mongoose/mongoose"
+          git remote set-url origin "https://github.com/cesanta/mongoose.git"
+          git fetch --all
+          git reset --hard
+          git clean -fxd
+          git checkout "${{ matrix.mongoose_ver }}"
+          cd -
+
+      # For older builds (3.3.x), we need to switch back to the (official) wolfssl version that
+      # was used in those builds + patch it and update the build system
+      - name: 'Patch WolfSSL (ESP-IDF v3) with OVMS tweaks'
+        if: matrix.force_wolfssl_470
+        run: |
+          cd "vehicle/OVMS.V3/components/wolfssl/wolfssl"
+          git fetch --unshallow
+          git checkout "v4.7.0-stable"
+          cd -
+          cp "vehicle/OVMS.V3/support/wolfssl-4.7.0-esp3/user_settings.h" "vehicle/OVMS.V3/components/wolfssl/port/"
+          cp "vehicle/OVMS.V3/support/wolfssl-4.7.0-esp3/component.mk" "vehicle/OVMS.V3/components/wolfssl/"
+          git apply -p5 --directory="vehicle/OVMS.V3/components/wolfssl/wolfssl" "vehicle/OVMS.V3/support/wolfssl-4.7.0-esp3/wolfssl-4.7.0.patch"
+
+      # For all builds, we have a customized idf_components.yml file
+      - name: 'Setup IDF component file'
+        run: 'cp "vehicle/OVMS.V3/support/${{ matrix.idf_component_file }}" "vehicle/OVMS.V3/main/idf_component.yml"'
+
+      # For all builds, we have a default sdkconfig file
+      - name: 'Setup configuration'
+        run: 'cp "vehicle/OVMS.V3/support/${{ matrix.sdkconfig }}" "vehicle/OVMS.V3/sdkconfig.defaults"'
+
+      - name: 'Patch configuration for Mongoose 6.11 (SSL cannot compile)'
+        if: |
+          matrix.patch_mongoose_6_11 &&
+          (matrix.mongoose_ver == '6.11')
+        run: 'sed -i -e "s/CONFIG_MG_ENABLE_SSL=y/#CONFIG_MG_ENABLE_SSL=/g" "vehicle/OVMS.V3/sdkconfig.defaults"'
+
+      - name: 'Cache build dependencies (non node20 compatible container)'
+        #  cache@v4 action causes some problems with the espressif/idf container (see actions/checkout#1474),
+        if: ${{ ! matrix.node20_compatible_container }}
+        uses: actions/cache@v3
+        with:
+          path: |
+            vehicle/OVMS.V3/build
+            vehicle/OVMS.V3/sdkconfig
+          key: build-${{ runner.os }}-${{ matrix.idf_target }}_esp-idf-${{ matrix.idf_ver }}_mg-${{ matrix.mongoose_ver }}
+
+      - name: 'Cache build dependencies (node20 compatible container)'
+        if: matrix.node20_compatible_container
+        uses: actions/cache@v4
+        with:
+          path: |
+            vehicle/OVMS.V3/build
+            vehicle/OVMS.V3/sdkconfig
+          key: build-${{ runner.os }}-${{ matrix.idf_target }}_esp-idf-${{ matrix.idf_ver }}_mg-${{ matrix.mongoose_ver }}
+
+      # Now, we can build it. Let's not forget to install `dos2unix` first as it is needed.
+      - name: 'Build project with IDF-${{ matrix.idf_ver }} for ${{ matrix.idf_target }}'
+        env:
+          IDF_TARGET: ${{ matrix.idf_target }}
+        shell: bash
+        working-directory: vehicle/OVMS.V3
+        run: |
+          apt-get update
+          apt-get install -y dos2unix
+          . ${IDF_PATH}/export.sh
+          # https://github.com/espressif/idf-component-manager/issues/31#issuecomment-1535984388
+          pip install -U "urllib3<2"
+          git status --untracked-files
+          git ls-files -o --exclude-standard
+          ${{ matrix.build_command }}
+
+      - name: 'Archive build output artifacts (non node20 compatible container)'
+        #  upload-artifact@v4 action causes some problems with the espressif/idf container (see actions/checkout#1474),
+        if: ${{ ! matrix.node20_compatible_container }}
+        uses: 'actions/upload-artifact@v3'
+        with:
+          name: '${{ matrix.idf_target }}_esp-idf-${{ matrix.idf_ver }}_mongoose-${{ matrix.mongoose_ver }}'
+          path: |
+            vehicle/OVMS.V3/build/bootloader/bootloader.bin
+            vehicle/OVMS.V3/build/partition_table/partition-table.bin
+            vehicle/OVMS.V3/build/*.bin
+            vehicle/OVMS.V3/build/*.elf
+            vehicle/OVMS.V3/build/log/*
+            vehicle/OVMS.V3/build/flasher_args.json
+            vehicle/OVMS.V3/build/config/sdkconfig.h
+            vehicle/OVMS.V3/build/config/sdkconfig.json
+            vehicle/OVMS.V3/sdkconfig
+          if-no-files-found: error
+
+      - name: 'Archive build output artifacts (node20 compatible container)'
+        if: matrix.node20_compatible_container
+        uses: 'actions/upload-artifact@v4'
+        with:
+          name: '${{ matrix.idf_target }}_esp-idf-${{ matrix.idf_ver }}_mongoose-${{ matrix.mongoose_ver }}'
+          path: |
+            vehicle/OVMS.V3/build/bootloader/bootloader.bin
+            vehicle/OVMS.V3/build/partition_table/partition-table.bin
+            vehicle/OVMS.V3/build/*.bin
+            vehicle/OVMS.V3/build/*.elf
+            vehicle/OVMS.V3/build/log/*
+            vehicle/OVMS.V3/build/flasher_args.json
+            vehicle/OVMS.V3/build/config/sdkconfig.h
+            vehicle/OVMS.V3/build/config/sdkconfig.json
+            vehicle/OVMS.V3/sdkconfig
+          if-no-files-found: error
diff --git a/.github/workflows/ci-fallback.yaml b/.github/workflows/ci-fallback.yaml
@@ -0,0 +1,17 @@
+---
+name: 'CI Fallback'
+
+on:
+  push:
+    paths-ignore:
+      - 'docs/**'
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+
+jobs:
+  documentation:
+    runs-on: 'ubuntu-22.04'
+
+    steps:
+      - run: 'echo "No build required"'
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -0,0 +1,56 @@
+---
+name: 'CI'
+
+on:
+  push:
+    paths-ignore:
+      - 'docs/**'
+  pull_request:
+    paths:
+      - 'docs/**'
+
+jobs:
+  documentation:
+    runs-on: 'ubuntu-22.04'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        id: setup-python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3
+          cache: 'pip'
+          cache-dependency-path: 'docs/source/requirements.txt'
+
+      - name: Upgrade pip
+        run: |
+          sudo apt update
+          sudo apt install -y graphviz
+          python -m pip install --upgrade pip setuptools
+
+      - name: Cache doc dependencies
+        id: sphinx
+        uses: actions/cache@v4
+        with:
+          path: |
+            docs/build
+          key: sphinx-docs-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('docs/source/conf.py','docs/source/Makefile') }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade -r docs/source/requirements.txt sphinx sphinx_rtd_theme
+
+      - name: Render the documentation
+        run: |
+          cd docs
+          make SPHINXOPTS=-W html
+
+      - name: 'Archive build output artifacts'
+        uses: 'actions/upload-artifact@v4'
+        with:
+          name: 'documentation'
+          path: |
+            docs/build/html
+          if-no-files-found: error
diff --git a/.gitignore b/.gitignore
@@ -65,3 +65,8 @@ docs/build/
 .vscode/
 vehicle/OVMS.V3/components/ovms_webserver/assets/charts.js.gz
 *.gz
+vehicle/OVMS.V3/sdkconfig.defaults
+
+# Funny, but the component manager seems to always run during builds,
+# thus overwriting this file. No reason to manage it ourselves...
+vehicle/OVMS.V3/dependencies.lock
diff --git a/vehicle/OVMS.V3/components/wolfssl/CMakeLists.txt b/vehicle/OVMS.V3/components/wolfssl/CMakeLists.txt
@@ -1,22 +1,48 @@
 set(srcs)
+set(srcdirs)
+set(srcexclude)
 set(include_dirs)
 set(priv_include_dirs)
 
 if (CONFIG_OVMS_SC_GPL_WOLF)
-  list(APPEND srcs "wolfssl/src/crl.c" "wolfssl/src/internal.c" "wolfssl/src/keys.c" "wolfssl/src/ocsp.c" "wolfssl/src/sniffer.c" "wolfssl/src/ssl.c" "wolfssl/src/tls.c" "wolfssl/src/tls13.c" "wolfssl/src/wolfio.c" "wolfssl/wolfcrypt/src/aes.c" "wolfssl/wolfcrypt/src/arc4.c" "wolfssl/wolfcrypt/src/asm.c" "wolfssl/wolfcrypt/src/asn.c" "wolfssl/wolfcrypt/src/blake2b.c" "wolfssl/wolfcrypt/src/camellia.c" "wolfssl/wolfcrypt/src/chacha.c" "wolfssl/wolfcrypt/src/chacha20_poly1305.c" "wolfssl/wolfcrypt/src/cmac.c" "wolfssl/wolfcrypt/src/coding.c" "wolfssl/wolfcrypt/src/compress.c" "wolfssl/wolfcrypt/src/cpuid.c" "wolfssl/wolfcrypt/src/curve25519.c" "wolfssl/wolfcrypt/src/des3.c" "wolfssl/wolfcrypt/src/dh.c" "wolfssl/wolfcrypt/src/dsa.c" "wolfssl/wolfcrypt/src/ecc.c" "wolfssl/wolfcrypt/src/ecc_fp.c" "wolfssl/wolfcrypt/src/ed25519.c" "wolfssl/wolfcrypt/src/error.c" "wolfssl/wolfcrypt/src/fe_low_mem.c" "wolfssl/wolfcrypt/src/fe_operations.c" "wolfssl/wolfcrypt/src/ge_low_mem.c" "wolfssl/wolfcrypt/src/ge_operations.c" "wolfssl/wolfcrypt/src/hash.c" "wolfssl/wolfcrypt/src/hc128.c" "wolfssl/wolfcrypt/src/hmac.c" "wolfssl/wolfcrypt/src/idea.c" "wolfssl/wolfcrypt/src/integer.c" "wolfssl/wolfcrypt/src/logging.c" "wolfssl/wolfcrypt/src/md2.c" "wolfssl/wolfcrypt/src/md4.c" "wolfssl/wolfcrypt/src/md5.c" "wolfssl/wolfcrypt/src/memory.c" "wolfssl/wolfcrypt/src/pkcs12.c" "wolfssl/wolfcrypt/src/pkcs7.c" "wolfssl/wolfcrypt/src/poly1305.c" "wolfssl/wolfcrypt/src/port/Espressif/esp32_aes.c" "wolfssl/wolfcrypt/src/port/Espressif/esp32_mp.c" "wolfssl/wolfcrypt/src/port/Espressif/esp32_sha.c" "wolfssl/wolfcrypt/src/port/Espressif/esp32_util.c" "wolfssl/wolfcrypt/src/pwdbased.c" "wolfssl/wolfcrypt/src/rabbit.c" "wolfssl/wolfcrypt/src/random.c" "wolfssl/wolfcrypt/src/ripemd.c" "wolfssl/wolfcrypt/src/rsa.c" "wolfssl/wolfcrypt/src/sha.c" "wolfssl/wolfcrypt/src/sha256.c" "wolfssl/wolfcrypt/src/sha3.c" "wolfssl/wolfcrypt/src/sha512.c" "wolfssl/wolfcrypt/src/signature.c" "wolfssl/wolfcrypt/src/srp.c" "wolfssl/wolfcrypt/src/wc_encrypt.c" "wolfssl/wolfcrypt/src/wc_port.c" "wolfssl/wolfcrypt/src/wolfevent.c" "wolfssl/wolfcrypt/src/wolfmath.c")
+  # Cf https://github.com/gojimmypi/wolfssl/blob/Espressif_No_Install/IDE/Espressif/ESP-IDF/examples/wolfssl_server/components/wolfssl/CMakeLists.txt
+  # get a list of all wolfcrypt assembly files; we'll exclude them as they don't target Xtensa
+  FILE(GLOB EXCLUDE_ASM *.S)
+  file(GLOB_RECURSE EXCLUDE_ASM ${CMAKE_SOURCE_DIR} "wolfssl/wolfcrypt/src/*.S")
+  message(STATUS "wolfssl : EXCLUDE_ASM = ${EXCLUDE_ASM}")
+
   list(APPEND include_dirs "port" "wolfssl")
+  list(APPEND srcdirs "wolfssl/src/"
+                      "wolfssl/wolfcrypt/src"
+                      "wolfssl/wolfcrypt/src/port/Espressif/"
+                      "wolfssl/wolfcrypt/src/port/atmel/"
+                      )
+  list(APPEND srcexclude
+      "wolfssl/src/bio.c"
+      "wolfssl/src/conf.c"
+      "wolfssl/src/misc.c"
+      "wolfssl/src/pk.c"
+      "wolfssl/src/ssl_misc.c" # included by ssl.c
+      "wolfssl/src/x509.c"
+      "wolfssl/src/x509_str.c"
+      "wolfssl/wolfcrypt/src/evp.c"
+      "wolfssl/wolfcrypt/src/misc.c"
+      "${EXCLUDE_ASM}"
+      )
 endif ()
 
 # requirements can't depend on config
 idf_component_register(SRCS ${srcs}
+                       SRC_DIRS ${srcdirs}
                        INCLUDE_DIRS ${include_dirs}
                        PRIV_INCLUDE_DIRS ${priv_include_dirs}
-                       REQUIRES "freertos"
+                       REQUIRES "freertos" "lwip"
+                       EXCLUDE_SRCS ${srcexclude}
                        # PRIV_REQUIRES "freertos"
                        WHOLE_ARCHIVE)
 
 if (CONFIG_OVMS_SC_GPL_WOLF)
-  component_compile_definitions(WOLFSSL_USER_SETTINGS)
+  component_compile_definitions("WOLFSSL_USER_SETTINGS")
   component_compile_options("-Wno-cpp" "-Wno-char-subscripts")
   set_source_files_properties(wolfssl/src/ssl.c PROPERTIES COMPILE_FLAGS "-Wno-format-truncation -Wno-char-subscripts")
   set_source_files_properties(wolfssl/wolfcrypt/src/random.c PROPERTIES COMPILE_FLAGS "-Wno-implicit-function-declaration")