Merge branch 'onaio:main' into main

Author: skngetich

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ concurrency:
 jobs:
   static-analysis:
     name: Prospector Static Analysis
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test
     strategy:
@@ -29,11 +29,11 @@ jobs:
             requirements/dev.pip
             requirements/azure.pip
 
+      - name: Update apt sources
+        run: sudo apt-get update
+
       - name: Install APT requirements
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev
-          sudo rm -rf /var/lib/apt/lists/*
+        run: sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev
 
       - name: Install Pip requirements
         run: |
@@ -50,7 +50,7 @@ jobs:
         run: prospector -X -s veryhigh onadata
   unit-tests-1:
     name: Django Unit Tests (Libraries, Main, RestServices, SMS Support, Viewer, Messaging)
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs: static-analysis
     env:
       DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test
@@ -90,11 +90,11 @@ jobs:
             requirements/dev.pip
             requirements/azure.pip
 
+      - name: Update apt sources
+        run: sudo apt-get update
+
       - name: Install APT requirements
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev
-          sudo rm -rf /var/lib/apt/lists/*
+        run: sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev
 
       - name: Install Pip requirements
         run: |
@@ -107,7 +107,7 @@ jobs:
           python manage.py test onadata/libs onadata/apps/main onadata/apps/restservice onadata/apps/sms_support onadata/apps/viewer onadata/apps/messaging --noinput --timing --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4
   unit-tests-2:
     name: Django Unit Tests (API, Logger)
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs: static-analysis
     env:
       DJANGO_SETTINGS_MODULE: onadata.settings.github_actions_test
@@ -147,28 +147,32 @@ jobs:
             requirements/dev.pip
             requirements/azure.pip
 
+      - name: Update apt sources
+        run: sudo apt-get update
+
       - name: Install APT requirements
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev
-          sudo rm -rf /var/lib/apt/lists/*
+        run: sudo apt-get install -y --no-install-recommends libjpeg-dev zlib1g-dev software-properties-common ghostscript libxslt1-dev binutils libproj-dev gdal-bin memcached libmemcached-dev libxml2-dev libxslt-dev
 
       - name: Install Pip requirements
         run: |
           pip install -U pip
           pip install -r requirements/base.pip
           pip install -r requirements/dev.pip
+          pip install -r requirements/azure.pip
 
       - name: Run tests
         run: |
           python manage.py test onadata/apps/api onadata/apps/logger --noinput --timing --settings=onadata.settings.github_actions_test --verbosity=2 --parallel=4
   security-check:
     name: Trivy Security Checks
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
+      - name: Update apt sources
+        run: sudo apt-get update
+
       - name: Get the branch name
         id: get-branch-name
         if: github.event_name == 'push'

.gitignore

@@ -45,6 +45,7 @@ celerybeat.pid
 .virtualenv
 *.ropeproject
 .noseids
+.python-version
 
 gunicorn_cfg.py
 run_gunicorn.sh

CHANGES.rst

@@ -3,6 +3,88 @@ Changelog for Onadata
 
 ``* represents releases that introduce new migrations``
 
+v3.8.6(2023-03-07)
+------------------
+- Handle cases of duplicate metadata fields within exports
+  `PR #2385 <https://github.com/onaio/onadata/pull/2385>`_
+  [@DavisRayM]
+- Update dependencies
+  `PR #2387 <https://github.com/onaio/onadata/pull/2387>`_
+  [@DavisRayM]
+- Add backward compatibility for existing .xls form downloads
+  `PR #2383 <https://github.com/onaio/onadata/pull/2383>`_
+  [@KipSigei]
+
+v3.8.5(2023-02-22)
+------------------
+- Fix: FieldError: Cannot resolve keyword 'json' into field from Attachment model
+  `PR #2380 <https://github.com/onaio/onadata/pull/2380>`_
+  [@FrankApiyo]
+
+v3.8.4(2023-02-20)
+------------------
+- Fix choice labels bug on filtered dataset exports
+  `PR #2372 <https://github.com/onaio/onadata/pull/2372>`_
+  [@KipSigei]
+- Fix: Media endpoint currently returns an empty list for filtered and merged datasets
+  `PR #2371 <https://github.com/onaio/onadata/pull/2371>`_
+  [@FrankApiyo]
+
+v3.8.3(2023-02-14)
+------------------
+- Filter out deleted submissiions from linked GeoJSON
+  `PR #2371 <https://github.com/onaio/onadata/pull/2371>`_
+  [@KipSigei]
+
+v3.8.2(2023-02-07)
+------------------
+- Remove spaces from user-agent cached key
+  `PR #2369 <https://github.com/onaio/onadata/pull/2369>`_
+  [@KipSigei]
+- Correctly remove group name for GPS field headers for xlsx exports
+  `PR #2364 <https://github.com/onaio/onadata/pull/2364>`_
+  [@KipSigei]
+
+v3.8.1(2023-02-03)
+------------------
+- Add custom throttling class
+  `PR #2365 <https://github.com/onaio/onadata/pull/2365>`_
+  [@DavisRayM]
+
+v3.8.0(2023-02-01)
+------------------
+- Ensure that the user row is selected alongside the Token
+  `PR #2362 <https://github.com/onaio/onadata/pull/2362>`_
+  [@FrankApiyo]
+- Render filtered datasets and merged datasets as geojson
+  `PR #2360 <https://github.com/onaio/onadata/pull/2360>`_
+  [@FrankApiyo]
+- Fix an issue where GPS Data within a group is not correctly extracted when group name is removed
+  `PR #2355 <https://github.com/onaio/onadata/pull/2355>`_
+  [@DavisRayM]
+- Update setuptools & futures packages
+  `PR #2353 <https://github.com/onaio/onadata/pull/2353>`_
+  [@DavisRayM]
+- Ensure external select to csv conversion works as expected
+  `PR #2349 <https://github.com/onaio/onadata/pull/2349>`_
+  [@DavisRayM]
+- Return correct geojson for polygons and geotrace data
+  `PR #2348 <https://github.com/onaio/onadata/pull/2348>`_
+  [@FrankApiyo]
+- Ensure Excel exports are in .xlsx format
+  `PR #2346 <https://github.com/onaio/onadata/pull/2346>`_
+  [@KipSigei]
+- Ensure user profiles are created before building user permissions object
+  `PR #2344 <https://github.com/onaio/onadata/pull/2344>`_
+  [@KipSigei]
+
+v3.7.1(2022-12-21)
+------------------
+
+- Formbuilder permission changes
+  `PR #2345 <https://github.com/onaio/onadata/pull/2345>`_
+  [@DavisRayM]
+
 3.7.0(2022-12-07)
 ------------------
 - Update GeoJSON endpoint to filter by instances with geoms

Dockerfile

@@ -0,0 +1,79 @@
+FROM ubuntu:20.04
+
+# Silence configuration prompts
+ENV DEBIAN_FRONTEND noninteractive
+ENV PYTHONUNBUFFERED 1
+
+ENV DJANGO_SETTINGS_MODULE onadata.settings.docker
+
+# Install service dependencies
+# hadolint ignore=DL3008
+RUN apt-get update -q &&\
+    apt-get install -y --no-install-recommends software-properties-common \
+        binutils \
+        libproj-dev \
+        gdal-bin \
+        memcached \
+        libmemcached-dev \
+        build-essential \
+        supervisor \
+        python3.9 \
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        git \
+        libssl-dev \
+        libpq-dev \
+        gfortran \
+        libatlas-base-dev \
+        libjpeg-dev \
+        libxml2-dev \
+        libxslt1-dev \
+        libpython3.9-dev \
+        zlib1g-dev \
+        ghostscript \
+        python3-celery \
+        python3-sphinx \
+        pkg-config \
+        gcc \
+        automake \
+        libtool \
+        openjdk-11-jre-headless \
+        libpcre3 \
+        libpcre3-dev \
+        locales \
+        netcat && \
+    apt-get -y -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' dist-upgrade &&\
+    rm -rf /var/lib/apt/lists/*
+
+# Generate and set en_US.UTF-8 locale
+RUN locale-gen en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+ENV LC_CTYPE en_US.UTF-8
+RUN dpkg-reconfigure locales
+
+# Create OnaData user and add to tty group
+RUN useradd -G tty -m onadata
+
+# Make app directory
+RUN mkdir -p /srv/onadata && chown -R onadata:onadata /srv
+
+# Copy local codebase
+COPY . /srv/onadata
+
+# Install service requirements
+# hadolint ignore=DL3013
+RUN python3.9 -m pip install --no-cache-dir -U pip && \
+    python3.9 -m pip install --no-cache-dir -r /srv/onadata/requirements/base.pip && \
+    python3.9 -m pip install --no-cache-dir -r /srv/onadata/requirements/s3.pip && \
+    python3.9 -m pip install --no-cache-dir -r /srv/onadata/requirements/ses.pip && \
+    python3.9 -m pip install --no-cache-dir -r /srv/onadata/requirements/azure.pip && \
+    python3.9 -m pip install --no-cache-dir uwsgitop django-silk
+
+WORKDIR /srv/onadata
+
+EXPOSE 8000
+
+USER onadata
+
+CMD ["/usr/local/bin/uwsgi", "--ini", "/srv/onadata/uwsgi.ini"]

docker-compose.yml

@@ -0,0 +1,44 @@
+services:
+  api:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      - database
+      - cache
+    stdin_open: true
+    tty: true
+    user: "onadata"
+    volumes:
+      - ./:/srv/onadata
+    ports:
+      - 8000:8000
+    command: /usr/local/bin/uwsgi --ini /srv/onadata/uwsgi.ini
+
+  celery:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    depends_on:
+      - database
+      - cache
+      - api
+    volumes:
+      - ./:/srv/onadata
+    user: "onadata"
+    command: celery -A onadata.celeryapp worker -B -l INFO -E
+
+  database:
+    image: postgis/postgis:13-3.3-alpine
+    environment:
+      POSTGRES_PASSWORD: onadata
+      POSTGRES_USER: onadata
+      POSTGRES_DB: onadata
+    volumes:
+      - dbdata:/var/lib/postgresql/data
+
+  cache:
+    image: redis:alpine
+
+volumes:
+  dbdata:

docker/onadata-uwsgi/Dockerfile

@@ -13,10 +13,10 @@ RUN mkdir -m 0600 ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
 # hadolint ignore=DL3013
 RUN --mount=type=ssh if [ -n "$optional_packages" ]; then pip install ${optional_packages} ; fi
 
-FROM ubuntu:20.04
+FROM ubuntu:focal-20221130
 COPY --from=intermediate /usr/local/lib/python3.9/site-packages/ /usr/local/lib/python3.9/dist-packages/
 
-ARG release_version=v2.4.1
+ARG release_version=v3.7.1
 
 # Silence configuration prompts
 ENV DEBIAN_FRONTEND noninteractive
@@ -61,7 +61,6 @@ RUN apt-get update -q &&\
         libpcre3-dev \
         locales \
         netcat && \
-    apt-get -y -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' dist-upgrade &&\
     rm -rf /var/lib/apt/lists/*
 
 # Generate and set en_US.UTF-8 locale
@@ -87,7 +86,8 @@ RUN python3.9 -m pip install --no-cache-dir -U pip && \
     python3.9 -m pip install --no-cache-dir -r requirements/s3.pip && \
     python3.9 -m pip install --no-cache-dir -r requirements/ses.pip && \
     python3.9 -m pip install --no-cache-dir -r requirements/azure.pip && \
-    python3.9 -m pip install --no-cache-dir pyyaml uwsgitop django-prometheus==v2.2.0
+    python3.9 -m pip install setuptools==65.5.1 && \
+    python3.9 -m pip install --no-cache-dir pyyaml uwsgitop
 
 # Compile API Docs
 RUN make -C docs html