Skip to content

Commit 48ae6d7

Browse files
maxipavlovicmaxipavlovic
authored
Merge pull request #138 from cloudblue/feature/LITE-27792
LITE-27792 Support for logging of timed out PG and MySQL queries
2 parents b993d76 + 7fffd70 commit 48ae6d7

File tree

7 files changed

+268
-21
lines changed

7 files changed

+268
-21
lines changed

‎dj_cqrs/constants.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,10 @@ class SignalType:
2929
DEFAULT_REPLICA_MAX_RETRIES = 30
3030
DEFAULT_REPLICA_RETRY_DELAY = 2 # seconds
3131
DEFAULT_REPLICA_DELAY_QUEUE_MAX_SIZE = 1000
32+
33+
DB_VENDOR_PG = 'postgresql'
34+
DB_VENDOR_MYSQL = 'mysql'
35+
SUPPORTED_TIMEOUT_DB_VENDORS = {DB_VENDOR_MYSQL, DB_VENDOR_PG}
36+
37+
PG_TIMEOUT_FLAG = 'statement timeout'
38+
MYSQL_TIMEOUT_ERROR_CODE = 3024

‎dj_cqrs/controller/consumer.py‎

Lines changed: 4 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@
88
from django.db import Error, close_old_connections, transaction
99

1010
from dj_cqrs.constants import SignalType
11+
from dj_cqrs.logger import log_timed_out_queries
1112
from dj_cqrs.registries import ReplicaRegistry
13+
from dj_cqrs.utils import apply_query_timeouts
1214

1315

1416
logger = logging.getLogger('django-cqrs')
@@ -66,7 +68,7 @@ def route_signal_to_replica_model(
6668
is_meta_supported = model_cls.CQRS_META
6769
try:
6870
if db_is_needed:
69-
_apply_query_timeouts(model_cls)
71+
apply_query_timeouts(model_cls)
7072

7173
with transaction.atomic(savepoint=False) if db_is_needed else ExitStack():
7274
if signal_type == SignalType.DELETE:
@@ -101,23 +103,4 @@ def route_signal_to_replica_model(
101103
),
102104
)
103105

104-
105-
def _apply_query_timeouts(model_cls): # pragma: no cover
106-
query_timeout = int(settings.CQRS['replica'].get('CQRS_QUERY_TIMEOUT', 0))
107-
if query_timeout <= 0:
108-
return
109-
110-
model_db = model_cls._default_manager.db
111-
conn = transaction.get_connection(using=model_db)
112-
conn_vendor = getattr(conn, 'vendor', '')
113-
114-
if conn_vendor not in {'postgresql', 'mysql'}:
115-
return
116-
117-
if conn_vendor == 'postgresql':
118-
statement = 'SET statement_timeout TO %s'
119-
else:
120-
statement = 'SET SESSION MAX_EXECUTION_TIME=%s'
121-
122-
with conn.cursor() as cursor:
123-
cursor.execute(statement, params=(query_timeout,))
106+
log_timed_out_queries(e, model_cls)

‎dj_cqrs/logger.py‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import logging
2+
3+
from django.conf import settings
4+
from django.db import OperationalError, transaction
5+
6+
from dj_cqrs.constants import (
7+
DB_VENDOR_MYSQL,
8+
DB_VENDOR_PG,
9+
MYSQL_TIMEOUT_ERROR_CODE,
10+
PG_TIMEOUT_FLAG,
11+
SUPPORTED_TIMEOUT_DB_VENDORS,
12+
)
13+
14+
15+
def install_last_query_capturer(model_cls):
16+
conn = _connection(model_cls)
17+
if not _get_last_query_capturer(conn):
18+
conn.execute_wrappers.append(_LastQueryCaptureWrapper())
19+
20+
21+
def log_timed_out_queries(error, model_cls): # pragma: no cover
22+
log_q = bool(settings.CQRS['replica'].get('CQRS_LOG_TIMED_OUT_QUERIES', False))
23+
if not (log_q and isinstance(error, OperationalError) and error.args):
24+
return
25+
26+
conn = _connection(model_cls)
27+
conn_vendor = getattr(conn, 'vendor', '')
28+
if conn_vendor not in SUPPORTED_TIMEOUT_DB_VENDORS:
29+
return
30+
31+
e_arg = error.args[0]
32+
is_timeout_error = bool(
33+
(conn_vendor == DB_VENDOR_MYSQL and e_arg == MYSQL_TIMEOUT_ERROR_CODE)
34+
or (conn_vendor == DB_VENDOR_PG and isinstance(e_arg, str) and PG_TIMEOUT_FLAG in e_arg)
35+
)
36+
if is_timeout_error:
37+
query = getattr(_get_last_query_capturer(conn), 'query', None)
38+
if query:
39+
logger_name = settings.CQRS['replica'].get('CQRS_QUERY_LOGGER', '') or 'django-cqrs'
40+
logger = logging.getLogger(logger_name)
41+
logger.error('Timed out query:\n%s', query)
42+
43+
44+
class _LastQueryCaptureWrapper:
45+
def __init__(self):
46+
self.query = None
47+
48+
def __call__(self, execute, sql, params, many, context):
49+
try:
50+
execute(sql, params, many, context)
51+
finally:
52+
self.query = sql
53+
54+
55+
def _get_last_query_capturer(conn):
56+
return next((w for w in conn.execute_wrappers if isinstance(w, _LastQueryCaptureWrapper)), None)
57+
58+
59+
def _connection(model_cls):
60+
return transaction.get_connection(using=model_cls._default_manager.db)

‎dj_cqrs/utils.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,12 @@
55
from uuid import UUID
66

77
from django.conf import settings
8+
from django.db import transaction
89
from django.utils import timezone
910

11+
from dj_cqrs.constants import DB_VENDOR_PG, SUPPORTED_TIMEOUT_DB_VENDORS
12+
from dj_cqrs.logger import install_last_query_capturer
13+
1014

1115
logger = logging.getLogger('django-cqrs')
1216

@@ -54,3 +58,25 @@ def get_messages_prefetch_count_per_worker():
5458

5559
def get_json_valid_value(value):
5660
return str(value) if isinstance(value, (date, datetime, UUID)) else value
61+
62+
63+
def apply_query_timeouts(model_cls): # pragma: no cover
64+
query_timeout = int(settings.CQRS['replica'].get('CQRS_QUERY_TIMEOUT', 0))
65+
if query_timeout <= 0:
66+
return
67+
68+
model_db = model_cls._default_manager.db
69+
conn = transaction.get_connection(using=model_db)
70+
conn_vendor = getattr(conn, 'vendor', '')
71+
if conn_vendor not in SUPPORTED_TIMEOUT_DB_VENDORS:
72+
return
73+
74+
if conn_vendor == DB_VENDOR_PG:
75+
statement = 'SET statement_timeout TO %s'
76+
else:
77+
statement = 'SET SESSION MAX_EXECUTION_TIME=%s'
78+
79+
with conn.cursor() as cursor:
80+
cursor.execute(statement, params=(query_timeout,))
81+
82+
install_last_query_capturer(model_cls)

‎integration_tests/replica_settings.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@
6666
'delay_queue_max_size': 50,
6767
'dead_letter_queue': 'dead_letter_replica',
6868
'dead_message_ttl': 5,
69+
'CQRS_QUERY_TIMEOUT': 10000,
70+
'CQRS_LOG_TIMED_OUT_QUERIES': 1,
6971
},
7072
}
7173

‎tests/test_logger.py‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
import logging
2+
3+
import pytest
4+
from django.db import (
5+
DatabaseError,
6+
IntegrityError,
7+
OperationalError,
8+
connection,
9+
)
10+
11+
from dj_cqrs.logger import (
12+
_LastQueryCaptureWrapper,
13+
install_last_query_capturer,
14+
log_timed_out_queries,
15+
)
16+
from tests.dj_replica import models
17+
18+
19+
@pytest.mark.django_db(transaction=True)
20+
def test_install_last_query_capturer():
21+
for _ in range(2):
22+
install_last_query_capturer(models.AuthorRef)
23+
24+
assert len(connection.execute_wrappers) == 1
25+
assert isinstance(connection.execute_wrappers[0], _LastQueryCaptureWrapper)
26+
27+
with connection.cursor() as c:
28+
c.execute('SELECT 1')
29+
30+
assert connection.execute_wrappers[0].query == 'SELECT 1'
31+
32+
connection.execute_wrappers.pop()
33+
34+
35+
def test_log_timed_out_queries_not_supported(caplog):
36+
assert log_timed_out_queries(None, None) is None
37+
assert not caplog.record_tuples
38+
39+
40+
@pytest.mark.parametrize(
41+
'error',
42+
[
43+
IntegrityError('some error'),
44+
DatabaseError(),
45+
OperationalError(),
46+
],
47+
)
48+
def test_log_timed_out_queries_other_error(error, settings, caplog):
49+
settings.CQRS_LOG_TIMED_OUT_QUERIES = 1
50+
51+
assert log_timed_out_queries(error, None) is None
52+
assert not caplog.record_tuples
53+
54+
55+
@pytest.mark.django_db(transaction=True)
56+
@pytest.mark.parametrize(
57+
'engine, error, l_name, records',
58+
[
59+
('sqlite', None, None, []),
60+
(
61+
'postgres',
62+
OperationalError('canceling statement due to statement timeout'),
63+
None,
64+
[
65+
(
66+
'django-cqrs',
67+
logging.ERROR,
68+
'Timed out query:\nSELECT 1',
69+
)
70+
],
71+
),
72+
(
73+
'postgres',
74+
OperationalError('canceling statement due to statement timeout'),
75+
'long-query',
76+
[
77+
(
78+
'long-query',
79+
logging.ERROR,
80+
'Timed out query:\nSELECT 1',
81+
)
82+
],
83+
),
84+
(
85+
'postgres',
86+
OperationalError('could not connect to server'),
87+
None,
88+
[],
89+
),
90+
(
91+
'postgres',
92+
OperationalError(125, 'Some error'),
93+
None,
94+
[],
95+
),
96+
(
97+
'mysql',
98+
OperationalError(3024),
99+
None,
100+
[
101+
(
102+
'django-cqrs',
103+
logging.ERROR,
104+
'Timed out query:\nSELECT 1',
105+
)
106+
],
107+
),
108+
(
109+
'mysql',
110+
OperationalError(
111+
3024, 'Query exec was interrupted, max statement execution time exceeded'
112+
),
113+
'long-query-1',
114+
[
115+
(
116+
'long-query-1',
117+
logging.ERROR,
118+
'Timed out query:\nSELECT 1',
119+
)
120+
],
121+
),
122+
(
123+
'mysql',
124+
OperationalError(1040, 'Too many connections'),
125+
None,
126+
[],
127+
),
128+
],
129+
)
130+
def test_apply_query_timeouts(settings, engine, l_name, error, records, caplog):
131+
if settings.DB_ENGINE != engine:
132+
return
133+
134+
settings.CQRS['replica']['CQRS_LOG_TIMED_OUT_QUERIES'] = True
135+
settings.CQRS['replica']['CQRS_QUERY_LOGGER'] = l_name
136+
137+
model_cls = models.BasicFieldsModelRef
138+
install_last_query_capturer(model_cls)
139+
140+
with connection.cursor() as c:
141+
c.execute('SELECT 1')
142+
143+
assert log_timed_out_queries(error, model_cls) is None
144+
assert caplog.record_tuples == records
145+
146+
connection.execute_wrappers.pop()

‎tests/test_utils.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,19 @@
66
timedelta,
77
timezone,
88
)
9+
from unittest.mock import patch
910
from uuid import UUID
1011

1112
import pytest
1213

1314
from dj_cqrs.utils import (
15+
apply_query_timeouts,
1416
get_delay_queue_max_size,
1517
get_json_valid_value,
1618
get_message_expiration_dt,
1719
get_messages_prefetch_count_per_worker,
1820
)
21+
from tests.dj_replica import models
1922

2023

2124
def test_get_message_expiration_dt_fixed(mocker, settings):
@@ -86,3 +89,23 @@ def test_get_messaged_prefetch_count_per_worker_with_delay_queue(settings):
8689
)
8790
def test_get_json_valid_value(value, result):
8891
assert get_json_valid_value(value) == result
92+
93+
94+
@pytest.mark.django_db
95+
@pytest.mark.parametrize(
96+
'engine, p_count',
97+
[
98+
('sqlite', 0),
99+
('postgres', 1),
100+
('mysql', 1),
101+
],
102+
)
103+
def test_apply_query_timeouts(settings, engine, p_count):
104+
if settings.DB_ENGINE != engine:
105+
return
106+
107+
settings.CQRS['replica']['CQRS_QUERY_TIMEOUT'] = 1
108+
with patch('dj_cqrs.utils.install_last_query_capturer') as p:
109+
assert apply_query_timeouts(models.BasicFieldsModelRef) is None
110+
111+
assert p.call_count == p_count

0 commit comments

Comments
 (0)