Merge branch 'develop' into etienne/sec-197-use-nonewpriviliges-for-postgres

Author: staaldraad

.github/CODEOWNERS

@@ -1,4 +1,4 @@
 * @supabase/backend @supabase/postgres
-migrations/ @supabase/cli @supabase/backend
+migrations/ @supabase/dev-workflows @supabase/postgres @supabase/backend
 docker/orioledb @supabase/postgres @supabase/backend
 common.vars.pkr.hcl @supabase/postgres @supabase/backend

.github/workflows/qemu-image-build.yml

@@ -23,10 +23,10 @@ jobs:
 
       - uses: DeterminateSystems/nix-installer-action@main
 
-      - name: Set PostgreSQL versions - only builds pg15 atm
+      - name: Set PostgreSQL versions - only builds pg17 atm
         id: set-versions
         run: |
-          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[0]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
+          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[1]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
           echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
 
   build:

README.md

@@ -81,7 +81,7 @@ Unmodified Postgres with some useful plugins. Our goal with this repo is not to
 
 | Goodie | Version | Description |
 | ------------- | :-------------: | ------------- |
-| [PgBouncer](https://www.pgbouncer.org/) | [1.16.1](http://www.pgbouncer.org/changelog.html#pgbouncer-116x) | Set up Connection Pooling. |
+| [PgBouncer](https://www.pgbouncer.org/) | [1.19.0](http://www.pgbouncer.org/changelog.html#pgbouncer-119x) | Set up Connection Pooling. |
 | [PostgREST](https://postgrest.org/en/stable/) | [v12.2.3](https://github.com/PostgREST/postgrest/releases/tag/v12.2.3) | Instantly transform your database into an RESTful API. |
 | [WAL-G](https://github.com/wal-g/wal-g#wal-g) | [v2.0.1](https://github.com/wal-g/wal-g/releases/tag/v2.0.1) | Tool for physical database backup and recovery. | -->
 
@@ -126,4 +126,4 @@ TODO: find way to automate this
 
 We are building the features of Firebase using enterprise-grade, open source products. We support existing communities wherever possible, and if the products don’t exist we build them and open source them ourselves.
 
-[![New Sponsor](https://user-images.githubusercontent.com/10214025/90518111-e74bbb00-e198-11ea-8f88-c9e3c1aa4b5b.png)](https://github.com/sponsors/supabase)
+[![New Sponsor](https://user-images.githubusercontent.com/10214025/90518111-e74bbb00-e198-11ea-8f88-c9e3c1aa4b5b.png)](https://github.com/sponsors/supabase)

amazon-arm64-nix.pkr.hcl

@@ -228,11 +228,6 @@ build {
     destination = "/tmp"
   }
 
-  provisioner "file" {
-    source = "ebssurrogate/files/unit-tests"
-    destination = "/tmp"
-  }
-
   # Copy ansible playbook
   provisioner "shell" {
     inline = ["mkdir /tmp/ansible-playbook"]

ansible/files/gotrue-optimizations.service.j2

@@ -5,6 +5,7 @@ Description=GoTrue (Auth) optimizations
 Type=oneshot
 # we don't want failures from this command to cause PG startup to fail
 ExecStart=/bin/bash -c "/opt/supabase-admin-api optimize auth --destination-config-file-path /etc/gotrue/gotrue.generated.env ; exit 0"
+ExecStartPost=/bin/bash -c "cp -a /etc/gotrue/gotrue.generated.env /etc/auth.d/20_generated.env ; exit 0"
 User=postgrest
 
 [Install]

ansible/files/gotrue.service.j2

@@ -4,7 +4,7 @@ Description=Gotrue
 [Service]
 Type=simple
 WorkingDirectory=/opt/gotrue
-ExecStart=/opt/gotrue/gotrue
+ExecStart=/opt/gotrue/gotrue --config-dir /etc/auth.d
 User=gotrue
 Restart=always
 RestartSec=3

ansible/tasks/setup-gotrue.yml

@@ -30,6 +30,13 @@
     owner: gotrue
     mode: 0775
 
+- name: gotrue - create /etc/auth.d
+  file:
+    path: /etc/auth.d
+    state: directory
+    owner: gotrue
+    mode: 0755
+
 - name: gotrue - unpack archive in /opt/gotrue
   unarchive:
     remote_src: yes

ansible/tasks/setup-wal-g.yml

@@ -44,7 +44,7 @@
 
 - name: Create symlink to make wal-g-v2 the default wal-g
   ansible.builtin.file:
-    src: /usr/local/bin/wal-g-v2
+    src: /home/wal-g/.nix-profile/bin/wal-g-2
     dest: /usr/local/bin/wal-g
     state: link
     force: yes

ansible/tasks/test-image.yml

@@ -1,9 +1,3 @@
-- name: install pg_prove
-  apt:
-    pkg:
-      - libtap-parser-sourcehandler-pgtap-perl
-  when: debpkg_mode
-
 # - name: Temporarily disable PG Sodium references in config
 #   become: yes
 #   become_user: postgres
@@ -16,9 +10,9 @@
   become_user: postgres
   shell:
     cmd: >
-      sed -i.bak 
-      -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/' 
-      -e 's/\(shared_preload_libraries = '\''.*\)supabase_vault,\(.*'\''\)/\1\2/' 
+      sed -i.bak
+      -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/'
+      -e 's/\(shared_preload_libraries = '\''.*\)supabase_vault,\(.*'\''\)/\1\2/'
       -e 's/\(shared_preload_libraries = '\''.*\), *supabase_vault'\''/\1'\''/'
       -e 's/pgsodium.getkey_script=/#pgsodium.getkey_script=/'
       /etc/postgresql/postgresql.conf
@@ -74,53 +68,6 @@
     LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
   when: stage2_nix
 
-
-- name: Check psql_version and modify migrations if oriole-xx
-  block:
-    - name: Check if psql_version is psql_orioledb-xx
-      set_fact:
-        is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}"
-
-    - name: Remove specified extensions from SQL file
-      ansible.builtin.command:
-        cmd: >
-          sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql
-      when: is_psql_oriole
-      become: yes
-
-    - name: Remove specified extension files from extensions directory
-      ansible.builtin.find:
-        paths: /tmp/migrations/tests/extensions
-        patterns: 
-          - '*timescaledb*.sql'
-          - '*plv8*.sql'
-          - '*postgis*.sql'
-          - '*pgrouting*.sql'
-      register: files_to_remove
-      when: is_psql_oriole
-
-    - name: Delete matched extension files
-      ansible.builtin.file:
-        path: "{{ item.path }}"
-        state: absent
-      loop: "{{ files_to_remove.files }}"
-      when: is_psql_oriole
-      become: yes
-
-- name: Run Unit tests (with filename unit-test-*) on Postgres Database
-  shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql
-  register: retval
-  failed_when: retval.rc != 0
-  when: debpkg_mode or stage2_nix
-
-- name: Run migrations tests
-  shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql
-  register: retval
-  failed_when: retval.rc != 0
-  when: debpkg_mode or stage2_nix
-  args:
-    chdir: /tmp/migrations
-
 - name: Re-enable PG Sodium references in config
   become: yes
   become_user: postgres
@@ -132,14 +79,6 @@
   shell: /usr/lib/postgresql/bin/psql --no-password --no-psqlrc -d postgres -h localhost -U supabase_admin -c 'SELECT pg_stat_statements_reset(); SELECT pg_stat_reset();'
   when: debpkg_mode or stage2_nix
 
-- name: remove pg_prove
-  apt:
-    pkg:
-      - libtap-parser-sourcehandler-pgtap-perl
-    state: absent
-    autoremove: yes
-  when: debpkg_mode
-
 - name: Stop Postgres Database
   become: yes
   become_user: postgres

ansible/vars.yml

@@ -8,8 +8,8 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.0.1.052-orioledb.etcro"
-  postgres15: "15.8.1.059-rc.etcro"
+  postgresorioledb-17: "17.0.1.054-orioledb"
+  postgres15: "15.8.1.061"
 
 # Non Postgres Extensions
 pgbouncer_release: "1.19.0"
@@ -51,7 +51,7 @@ postgres_exporter_release_checksum:
   arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0
   amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84
 
-adminapi_release: 0.75.0
+adminapi_release: 0.76.0
 adminmgr_release: 0.24.1
 
 vector_x86_deb: "https://packages.timber.io/vector/0.22.3/vector_0.22.3-1_amd64.deb"