Revert "Update Posgres to psycopg3 (#19322)" (#19499) (#19504)

Co-authored-by: Enrico Donnici <[email protected]>
Co-authored-by: Seth Samuel <[email protected]>

Author: 3 people

.builders/images/helpers.ps1

@@ -38,11 +38,3 @@ function Add-ToPath() {
     $target="$oldPath;$Append"
     [Environment]::SetEnvironmentVariable("Path", $target, [System.EnvironmentVariableTarget]::User)
 }
-
-function RunOnVSConsole() {
-    param(
-        [Parameter(Mandatory = $true)][string] $Command
-    )
-    Write-Host "Running $Command"
-    Start-Process -Wait -NoNewWindow "cmd.exe" -ArgumentList "/c ""$Env:VCVARSALL_BAT"" $Env:DD_TARGET_ARCH && $Command"
-}

.builders/images/linux-aarch64/Dockerfile

@@ -91,7 +91,7 @@ RUN \
  --without-debugger \
  --disable-static
 
-# libpq and pg_config as needed by psycopg
+# libpq and pg_config as needed by psycopg2
 RUN \
  DOWNLOAD_URL="https://ftp.postgresql.org/pub/source/v{{version}}/postgresql-{{version}}.tar.bz2" \
  VERSION="16.0" \

.builders/images/linux-x86_64/Dockerfile

@@ -95,7 +95,7 @@ RUN \
  --without-debugger \
  --disable-static
 
-# libpq and pg_config as needed by psycopg
+# libpq and pg_config as needed by psycopg2
 RUN \
  DOWNLOAD_URL="https://ftp.postgresql.org/pub/source/v{{version}}/postgresql-{{version}}.tar.bz2" \
  VERSION="16.0" \

.builders/images/macos-x86_64/builder_setup.sh

@@ -94,13 +94,6 @@ RELATIVE_PATH="curl-{{version}}" \
 # Remove the binary installed so that we consistenly use the same original `curl` binary
 rm "${DD_PREFIX_PATH}/bin/curl"
 
-# libpq and pg_config as needed by psycopg
-DOWNLOAD_URL="https://ftp.postgresql.org/pub/source/v{{version}}/postgresql-{{version}}.tar.bz2" \
-VERSION="16.0" \
-SHA256="df9e823eb22330444e1d48e52cc65135a652a6fdb3ce325e3f08549339f51b99" \
-RELATIVE_PATH="postgresql-{{version}}" \
-  install-from-source --without-readline --with-openssl --without-icu
-
 # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
 DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
 VERSION="0.9.29" \

.builders/images/windows-x86_64/Dockerfile

@@ -25,7 +25,6 @@ RUN curl -SL --output vs_buildtools.exe https://aka.ms/vs/17/release/vs_buildtoo
      --add Microsoft.VisualStudio.Workload.VCTools `
      || IF "%ERRORLEVEL%"=="3010" EXIT 0) `
  && del /q vs_buildtools.exe
-ENV VCVARSALL_BAT="C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvarsall.bat"
 
 # Upgrade PowerShell
 ENV POWERSHELL_VERSION="7.4.0"
@@ -115,42 +114,7 @@ RUN Get-RemoteFile `
     Add-ToPath -Append "C:\perl\perl\bin" && `
     Remove-Item "strawberry-perl-$Env:PERL_VERSION-64bit.zip"
 
-# Nasm
-ENV NASM_VERSION="2.16.03"
-RUN Get-RemoteFile `
-      -Uri https://www.nasm.us/pub/nasm/releasebuilds/$Env:NASM_VERSION/win64/nasm-$Env:NASM_VERSION-win64.zip `
-      -Path "nasm-$Env:NASM_VERSION-win64.zip" `
-    -Hash '3ee4782247bcb874378d02f7eab4e294a84d3d15f3f6ee2de2f47a46aa7226e6' && `
-    7z x "nasm-$Env:NASM_VERSION-win64.zip" -o"C:\nasm" && `
-    Add-ToPath -Append "C:\nasm\nasm-$Env:NASM_VERSION" && `
-    Remove-Item "nasm-$Env:NASM_VERSION-win64.zip"
-# openssl
 ENV OPENSSL_VERSION="3.3.2"
-RUN Get-RemoteFile `
-      -Uri https://www.openssl.org/source/openssl-$Env:OPENSSL_VERSION.tar.gz `
-      -Path openssl-$Env:OPENSSL_VERSION.tar.gz `
-      -Hash '2e8a40b01979afe8be0bbfb3de5dc1c6709fedb46d6c89c10da114ab5fc3d281'; `
-    7z x openssl-$Env:OPENSSL_VERSION.tar.gz -r -y && `
-    7z x openssl-$Env:OPENSSL_VERSION.tar -oC:\openssl_3 && `
-    cd C:\openssl_3\openssl-$Env:OPENSSL_VERSION && `
-    RunOnVSConsole -Command `
-    'C:\perl\perl\bin\perl.exe Configure && `
-    nmake && `
-    nmake install_sw'
-# libpq and pg_config as needed by psycopg
-ENV PG_VERSION="16.0"
-COPY pg_config.pl C:\pg_config.pl
-RUN Get-RemoteFile `
-      -Uri https://ftp.postgresql.org/pub/source/v$Env:PG_VERSION/postgresql-$Env:PG_VERSION.tar.bz2 `
-      -Path postgresql-$Env:PG_VERSION.tar.bz2 `
-      -Hash 'df9e823eb22330444e1d48e52cc65135a652a6fdb3ce325e3f08549339f51b99'; `
-    7z x postgresql-$Env:PG_VERSION.tar.bz2 -r -y && `
-    7z x postgresql-$Env:PG_VERSION.tar -oC:\postgresql_src && `
-    cd C:\postgresql_src\postgresql-$Env:PG_VERSION\src\tools\msvc && `
-    Copy-Item C:\pg_config.pl -Destination .\config.pl && `
-    RunOnVSConsole -Command 'C:\perl\perl\bin\perl.exe build.pl' && `
-    RunOnVSConsole -Command 'C:\perl\perl\bin\perl.exe install.pl C:\postgresql' && `
-    Add-ToPath -Append "C:\postgresql\bin"
 
 ENV CURL_VERSION="8.11.1"
 

.builders/images/windows-x86_64/pg_config.pl

@@ -83,8 +83,6 @@ oauthlib = ['BSD-3-Clause']
 mmh3 = ['CC0-1.0']
 # https://github.com/paramiko/paramiko/blob/master/LICENSE
 paramiko = ['LGPL-2.1-only']
-# https://github.com/psycopg/psycopg/blob/master/LICENSE.txt
-psycopg = ['LGPL-3.0-only']
 # https://github.com/psycopg/psycopg2/blob/master/LICENSE
 # https://github.com/psycopg/psycopg2/blob/master/doc/COPYING.LESSER
 psycopg2-binary = ['LGPL-3.0-only', 'BSD-3-Clause']

.ddev/ci/scripts/postgres/linux/50_install_postgres.sh

@@ -45,7 +45,6 @@ ply,PyPI,BSD-3-Clause,Copyright (C) 2001-2018
 prometheus-client,PyPI,Apache-2.0,Copyright 2015 The Prometheus Authors
 protobuf,PyPI,BSD-3-Clause,Copyright 2008 Google Inc.  All rights reserved.
 psutil,PyPI,BSD-3-Clause,"Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola"
-psycopg,PyPI,LGPL-3.0-only,Copyright (C) 2020 The Psycopg Team
 psycopg2-binary,PyPI,BSD-3-Clause,Copyright 2013 Federico Di Gregorio
 psycopg2-binary,PyPI,LGPL-3.0-only,Copyright (C) 2013 Federico Di Gregorio
 pyOpenSSL,PyPI,Apache-2.0,Copyright The pyOpenSSL developers

.ddev/config.toml

@@ -32,7 +32,6 @@ prometheus-client==0.21.1
 protobuf==5.29.3
 psutil==6.0.0
 psycopg2-binary==2.9.9
-psycopg[c]==3.2.3
 pyasn1==0.4.8
 pycryptodomex==3.21.0
 pydantic==2.10.5

LICENSE-3rdparty.csv

@@ -0,0 +1 @@
+Revert "Upgrade postgres to psycopg3" due to instability in testing

agent_requirements.in

@@ -8,7 +8,7 @@
 import time
 from typing import Callable, Dict
 
-import psycopg
+import psycopg2
 
 
 class ConnectionPoolFullError(Exception):
@@ -23,7 +23,7 @@ def __str__(self):
 class ConnectionInfo:
     def __init__(
         self,
-        connection: psycopg.Connection,
+        connection: psycopg2.extensions.connection,
         deadline: int,
         active: bool,
         last_accessed: int,
@@ -86,9 +86,9 @@ def _get_connection_raw(
         dbname: str,
         ttl_ms: int,
         timeout: int = None,
-        startup_fn: Callable[[psycopg.Connection], None] = None,
+        startup_fn: Callable[[psycopg2.extensions.connection], None] = None,
         persistent: bool = False,
-    ) -> psycopg.Connection:
+    ) -> psycopg2.extensions.connection:
         """
         Return a connection from the pool.
         Pass a function to startup_func if there is an action needed with the connection
@@ -117,7 +117,7 @@ def _get_connection_raw(
                 # if already in pool, retain persistence status
                 persistent = conn.persistent
 
-            if db.info.status != psycopg.pq.ConnStatus.OK:
+            if db.status != psycopg2.extensions.STATUS_READY:
                 # Some transaction went wrong and the connection is in an unhealthy state. Let's fix that
                 db.rollback()
 
@@ -138,7 +138,7 @@ def get_connection(
         dbname: str,
         ttl_ms: int,
         timeout: int = None,
-        startup_fn: Callable[[psycopg.Connection], None] = None,
+        startup_fn: Callable[[psycopg2.extensions.connection], None] = None,
         persistent: bool = False,
     ):
         """
@@ -147,14 +147,12 @@ def get_connection(
         make a new connection if the max_conn limit hasn't been reached.
         Blocks until a connection can be added to the pool,
         and optionally takes a timeout in seconds.
-        Note that leaving a connection context here does NOT close the connection in psycopg;
+        Note that leaving a connection context here does NOT close the connection in psycopg2;
         connections must be manually closed by `close_all_connections()`.
         """
         try:
             with self._mu:
-                db = self._get_connection_raw(
-                    dbname=dbname, ttl_ms=ttl_ms, timeout=timeout, startup_fn=startup_fn, persistent=persistent
-                )
+                db = self._get_connection_raw(dbname, ttl_ms, timeout, startup_fn, persistent)
             yield db
         finally:
             with self._mu:

postgres/changelog.d/19499.fixed

@@ -2,7 +2,8 @@
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
 
-import psycopg
+import psycopg2.extensions
+import psycopg2.extras
 
 from datadog_checks.base.utils.db.sql_commenter import add_sql_comment
 
@@ -27,9 +28,9 @@ def execute(self, query, vars=None, ignore_query_metric=False):
         return super().execute(query, vars)
 
 
-class CommenterCursor(BaseCommenterCursor, psycopg.ClientCursor):
+class CommenterCursor(BaseCommenterCursor, psycopg2.extensions.cursor):
     pass
 
 
-class CommenterDictCursor(BaseCommenterCursor, psycopg.ClientCursor):
+class CommenterDictCursor(BaseCommenterCursor, psycopg2.extras.DictCursor):
     pass

postgres/datadog_checks/postgres/connections.py

@@ -6,6 +6,7 @@
 
 from datadog_checks.base import AgentCheck
 from datadog_checks.base.utils.discovery import Discovery
+from datadog_checks.postgres.cursor import CommenterCursor
 from datadog_checks.postgres.util import DatabaseConfigurationError, warning_with_tags
 
 AUTODISCOVERY_QUERY: str = """select datname from pg_catalog.pg_database where datistemplate = false;"""
@@ -71,7 +72,7 @@ def get_items(self) -> List[str]:
 
     def _get_databases(self) -> List[str]:
         with self.db_pool.get_connection(self._db, self._default_ttl) as conn:
-            with conn.cursor() as cursor:
+            with conn.cursor(cursor_factory=CommenterCursor) as cursor:
                 cursor.execute(AUTODISCOVERY_QUERY)
                 databases = list(cursor.fetchall())
                 databases = [

postgres/datadog_checks/postgres/cursor.py

@@ -5,10 +5,11 @@
 import logging
 import re
 
-import psycopg
+import psycopg2
 
 from datadog_checks.base.utils.db.sql import compute_sql_signature
 from datadog_checks.base.utils.tracking import tracked_method
+from datadog_checks.postgres.cursor import CommenterDictCursor
 
 from .util import DBExplainError
 from .version_utils import V12
@@ -76,7 +77,7 @@ def explain_statement(self, dbname, statement, obfuscated_statement):
         if self._check.version < V12:
             # if pg version < 12, skip explaining parameterized queries because
             # plan_cache_mode is not supported
-            e = psycopg.errors.UndefinedParameter("Unable to explain parameterized query")
+            e = psycopg2.errors.UndefinedParameter("Unable to explain parameterized query")
             logger.debug(
                 "Unable to explain parameterized query. Postgres version %s does not support plan_cache_mode",
                 self._check.version,
@@ -179,14 +180,16 @@ def _deallocate_prepared_statement(self, dbname, query_signature):
             )
 
     def _execute_query(self, dbname, query):
+        # Psycopg2 connections do not get closed when context ends;
+        # leaving context will just mark the connection as inactive in MultiDatabaseConnectionPool
         with self._check.db_pool.get_connection(dbname, self._check._config.idle_connection_timeout) as conn:
-            with conn.cursor() as cursor:
+            with conn.cursor(cursor_factory=CommenterDictCursor) as cursor:
                 logger.debug('Executing query=[%s]', query)
                 cursor.execute(query, ignore_query_metric=True)
 
     def _execute_query_and_fetch_rows(self, dbname, query):
         with self._check.db_pool.get_connection(dbname, self._check._config.idle_connection_timeout) as conn:
-            with conn.cursor() as cursor:
+            with conn.cursor(cursor_factory=CommenterDictCursor) as cursor:
                 cursor.execute(query, ignore_query_metric=True)
                 return cursor.fetchall()