Skip to content

Commit 18706d9

Browse files
nbraun-amazonnbraun-amazon
committed
Add support for exponential backoff on retry
1 parent 7c0ae87 commit 18706d9

File tree

10 files changed

+271
-46
lines changed

10 files changed

+271
-46
lines changed

‎backoff.c‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#include "common.h"
2+
3+
#include <ext/standard/php_rand.h>
4+
5+
#if PHP_VERSION_ID >= 70100
6+
#include <ext/standard/php_mt_rand.h>
7+
#else
8+
static zend_long php_mt_rand_range(zend_long min, zend_long max) {
9+
zend_long number = php_rand();
10+
RAND_RANGE(number, min, max, PHP_RAND_MAX);
11+
return number;
12+
}
13+
#endif
14+
15+
#include "backoff.h"
16+
17+
static zend_ulong random_range(zend_ulong min, zend_ulong max) {
18+
if (max < min) {
19+
return php_mt_rand_range(max, min);
20+
}
21+
22+
return php_mt_rand_range(min, max);
23+
}
24+
25+
static zend_ulong redis_default_backoff(struct RedisBackoff *self, unsigned int retry_index) {
26+
zend_ulong backoff = retry_index ? self->base : random_range(0, self->base);
27+
return MIN(self->cap, backoff);
28+
}
29+
30+
static zend_ulong redis_constant_backoff(struct RedisBackoff *self, unsigned int retry_index) {
31+
zend_ulong backoff = self->base;
32+
return MIN(self->cap, backoff);
33+
}
34+
35+
static zend_ulong redis_uniform_backoff(struct RedisBackoff *self, unsigned int retry_index) {
36+
zend_ulong backoff = random_range(0, self->base);
37+
return MIN(self->cap, backoff);
38+
}
39+
40+
static zend_ulong redis_exponential_backoff(struct RedisBackoff *self, unsigned int retry_index) {
41+
zend_ulong pow = MIN(retry_index, 10);
42+
zend_ulong backoff = self->base * (1 << pow);
43+
return MIN(self->cap, backoff);
44+
}
45+
46+
static zend_ulong redis_full_jitter_backoff(struct RedisBackoff *self, unsigned int retry_index) {
47+
zend_ulong pow = MIN(retry_index, 10);
48+
zend_ulong backoff = self->base * (1 << pow);
49+
zend_ulong cap = MIN(self->cap, backoff);
50+
return random_range(0, self->cap);
51+
}
52+
53+
static zend_ulong redis_equal_jitter_backoff(struct RedisBackoff *self, unsigned int retry_index) {
54+
zend_ulong pow = MIN(retry_index, 10);
55+
zend_ulong backoff = self->base * (1 << pow);
56+
zend_ulong temp = MIN(self->cap, backoff);
57+
return temp / 2 + random_range(0, temp) / 2;
58+
}
59+
60+
static zend_ulong redis_decorrelated_jitter_backoff(struct RedisBackoff *self, unsigned int retry_index) {
61+
self->previous_backoff = random_range(self->base, self->previous_backoff * 3);
62+
return MIN(self->cap, self->previous_backoff);
63+
}
64+
65+
typedef zend_ulong (*redis_backoff_algorithm)(struct RedisBackoff *self, unsigned int retry_index);
66+
67+
static redis_backoff_algorithm redis_backoff_algorithms[REDIS_BACKOFF_ALGORITHMS] = {
68+
redis_default_backoff,
69+
redis_decorrelated_jitter_backoff,
70+
redis_full_jitter_backoff,
71+
redis_equal_jitter_backoff,
72+
redis_exponential_backoff,
73+
redis_uniform_backoff,
74+
redis_constant_backoff,
75+
};
76+
77+
void redis_initialize_backoff(struct RedisBackoff *self, unsigned long retry_interval) {
78+
self->algorithm = 0; // default backoff
79+
self->base = retry_interval;
80+
self->cap = retry_interval;
81+
self->previous_backoff = 0;
82+
}
83+
84+
void redis_backoff_reset(struct RedisBackoff *self) {
85+
self->previous_backoff = 0;
86+
}
87+
88+
zend_ulong redis_backoff_compute(struct RedisBackoff *self, unsigned int retry_index) {
89+
return redis_backoff_algorithms[self->algorithm](self, retry_index);
90+
}

‎backoff.h‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#ifndef REDIS_BACKOFF_H
2+
#define REDIS_BACKOFF_H
3+
4+
/* {{{ struct RedisBackoff */
5+
struct RedisBackoff {
6+
unsigned int algorithm; /* index of algorithm function, returns backoff in microseconds*/
7+
zend_ulong base; /* base backoff in microseconds */
8+
zend_ulong cap; /* max backoff in microseconds */
9+
zend_ulong previous_backoff; /* previous backoff in microseconds */
10+
};
11+
/* }}} */
12+
13+
void redis_initialize_backoff(struct RedisBackoff *self, unsigned long retry_interval);
14+
void redis_backoff_reset(struct RedisBackoff *self);
15+
zend_ulong redis_backoff_compute(struct RedisBackoff *self, unsigned int retry_index);
16+
17+
#endif

‎common.h‎

Lines changed: 53 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
#define NULL ((void *) 0)
2222
#endif
2323

24+
#include "backoff.h"
25+
2426
typedef enum {
2527
REDIS_SOCK_STATUS_FAILED = -1,
2628
REDIS_SOCK_STATUS_DISCONNECTED,
@@ -83,6 +85,10 @@ typedef enum _PUBSUB_TYPE {
8385
#define REDIS_OPT_REPLY_LITERAL 8
8486
#define REDIS_OPT_COMPRESSION_LEVEL 9
8587
#define REDIS_OPT_NULL_MBULK_AS_NULL 10
88+
#define REDIS_OPT_MAX_RETRIES 11
89+
#define REDIS_OPT_BACKOFF_ALGORITHM 12
90+
#define REDIS_OPT_BACKOFF_BASE 13
91+
#define REDIS_OPT_BACKOFF_CAP 14
8692

8793
/* cluster options */
8894
#define REDIS_FAILOVER_NONE 0
@@ -109,6 +115,16 @@ typedef enum {
109115
#define REDIS_SCAN_PREFIX 2
110116
#define REDIS_SCAN_NOPREFIX 3
111117

118+
/* BACKOFF_ALGORITHM options */
119+
#define REDIS_BACKOFF_ALGORITHMS 7
120+
#define REDIS_BACKOFF_ALGORITHM_DEFAULT 0
121+
#define REDIS_BACKOFF_ALGORITHM_DECORRELATED_JITTER 1
122+
#define REDIS_BACKOFF_ALGORITHM_FULL_JITTER 2
123+
#define REDIS_BACKOFF_ALGORITHM_EQUAL_JITTER 3
124+
#define REDIS_BACKOFF_ALGORITHM_EXPONENTIAL 4
125+
#define REDIS_BACKOFF_ALGORITHM_UNIFORM 5
126+
#define REDIS_BACKOFF_ALGORITHM_CONSTANT 6
127+
112128
/* GETBIT/SETBIT offset range limits */
113129
#define BITOP_MIN_OFFSET 0
114130
#define BITOP_MAX_OFFSET 4294967295U
@@ -258,41 +274,43 @@ typedef enum {
258274

259275
/* {{{ struct RedisSock */
260276
typedef struct {
261-
php_stream *stream;
262-
php_stream_context *stream_ctx;
263-
zend_string *host;
264-
int port;
265-
zend_string *user;
266-
zend_string *pass;
267-
double timeout;
268-
double read_timeout;
269-
long retry_interval;
270-
redis_sock_status status;
271-
int persistent;
272-
int watching;
273-
zend_string *persistent_id;
274-
275-
redis_serializer serializer;
276-
int compression;
277-
int compression_level;
278-
long dbNumber;
279-
280-
zend_string *prefix;
281-
282-
short mode;
283-
struct fold_item *head;
284-
struct fold_item *current;
285-
286-
zend_string *pipeline_cmd;
287-
288-
zend_string *err;
289-
290-
int scan;
291-
292-
int readonly;
293-
int reply_literal;
294-
int null_mbulk_as_null;
295-
int tcp_keepalive;
277+
php_stream *stream;
278+
php_stream_context *stream_ctx;
279+
zend_string *host;
280+
int port;
281+
zend_string *user;
282+
zend_string *pass;
283+
double timeout;
284+
double read_timeout;
285+
long retry_interval;
286+
int max_retries;
287+
struct RedisBackoff backoff;
288+
redis_sock_status status;
289+
int persistent;
290+
int watching;
291+
zend_string *persistent_id;
292+
293+
redis_serializer serializer;
294+
int compression;
295+
int compression_level;
296+
long dbNumber;
297+
298+
zend_string *prefix;
299+
300+
short mode;
301+
struct fold_item *head;
302+
struct fold_item *current;
303+
304+
zend_string *pipeline_cmd;
305+
306+
zend_string *err;
307+
308+
int scan;
309+
310+
int readonly;
311+
int reply_literal;
312+
int null_mbulk_as_null;
313+
int tcp_keepalive;
296314
} RedisSock;
297315
/* }}} */
298316

‎config.m4‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -323,5 +323,5 @@ if test "$PHP_REDIS" != "no"; then
323323
fi
324324

325325
PHP_SUBST(REDIS_SHARED_LIBADD)
326-
PHP_NEW_EXTENSION(redis, redis.c redis_commands.c library.c redis_session.c redis_array.c redis_array_impl.c redis_cluster.c cluster_library.c redis_sentinel.c sentinel_library.c $lzf_sources, $ext_shared)
326+
PHP_NEW_EXTENSION(redis, redis.c redis_commands.c library.c redis_session.c redis_array.c redis_array_impl.c redis_cluster.c cluster_library.c redis_sentinel.c sentinel_library.c backoff.c $lzf_sources, $ext_shared)
327327
fi

‎config.w32‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ ARG_ENABLE("redis-session", "whether to enable sessions", "yes");
55
ARG_ENABLE("redis-igbinary", "whether to enable igbinary serializer support", "no");
66

77
if (PHP_REDIS != "no") {
8-
var sources = "redis.c redis_commands.c library.c redis_session.c redis_array.c redis_array_impl.c redis_cluster.c cluster_library.c redis_sentinel.c sentinel_library.c";
8+
var sources = "redis.c redis_commands.c library.c redis_session.c redis_array.c redis_array_impl.c redis_cluster.c cluster_library.c redis_sentinel.c sentinel_library.c backoff.c";
99
if (PHP_REDIS_SESSION != "no") {
1010
ADD_EXTENSION_DEP("redis", "session");
1111
ADD_FLAG("CFLAGS_REDIS", ' /D PHP_SESSION=1 ');

‎library.c‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,7 @@ redis_error_throw(RedisSock *redis_sock)
301301
PHP_REDIS_API int
302302
redis_check_eof(RedisSock *redis_sock, int no_throw)
303303
{
304-
int count;
304+
unsigned int retry_index;
305305
char *errmsg;
306306

307307
if (!redis_sock || !redis_sock->stream || redis_sock->status == REDIS_SOCK_STATUS_FAILED) {
@@ -333,18 +333,17 @@ redis_check_eof(RedisSock *redis_sock, int no_throw)
333333
errmsg = "Connection lost and socket is in MULTI/watching mode";
334334
} else {
335335
errmsg = "Connection lost";
336-
/* TODO: configurable max retry count */
337-
for (count = 0; count < 10; ++count) {
336+
redis_backoff_reset(&redis_sock->backoff);
337+
for (retry_index = 0; retry_index < redis_sock->max_retries; ++retry_index) {
338338
/* close existing stream before reconnecting */
339339
if (redis_sock->stream) {
340340
redis_sock_disconnect(redis_sock, 1);
341341
}
342-
// Wait for a while before trying to reconnect
343-
if (redis_sock->retry_interval) {
344-
// Random factor to avoid having several (or many) concurrent connections trying to reconnect at the same time
345-
long retry_interval = (count ? redis_sock->retry_interval : (php_rand() % redis_sock->retry_interval));
346-
usleep(retry_interval);
347-
}
342+
/* Sleep based on our backoff algorithm */
343+
zend_ulong delay = redis_backoff_compute(&redis_sock->backoff, retry_index);
344+
if (delay != 0)
345+
usleep(delay);
346+
348347
/* reconnect */
349348
if (redis_sock_connect(redis_sock) == 0) {
350349
/* check for EOF again. */
@@ -2150,6 +2149,8 @@ redis_sock_create(char *host, int host_len, int port,
21502149
redis_sock->host = zend_string_init(host, host_len, 0);
21512150
redis_sock->status = REDIS_SOCK_STATUS_DISCONNECTED;
21522151
redis_sock->retry_interval = retry_interval * 1000;
2152+
redis_sock->max_retries = 10;
2153+
redis_initialize_backoff(&redis_sock->backoff, retry_interval);
21532154
redis_sock->persistent = persistent;
21542155

21552156
if (persistent && persistent_id != NULL) {

‎package.xml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
8888
<file role='doc' name='arrays.markdown'/>
8989
<file role='doc' name='cluster.markdown'/>
9090
<file role='doc' name='sentinel.markdown'/>
91+
<file role='src' name='backoff.c'/>
92+
<file role='src' name='backoff.h'/>
9193
<file role='src' name='cluster_library.c'/>
9294
<file role='src' name='cluster_library.h'/>
9395
<file role='src' name='common.h'/>

‎redis.c‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -775,6 +775,22 @@ static void add_class_constants(zend_class_entry *ce, int is_cluster) {
775775
zend_declare_class_constant_stringl(ce, "LEFT", 4, "left", 4);
776776
zend_declare_class_constant_stringl(ce, "RIGHT", 5, "right", 5);
777777
}
778+
779+
/* retry/backoff options*/
780+
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_MAX_RETRIES"), REDIS_OPT_MAX_RETRIES);
781+
782+
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_BACKOFF_ALGORITHM"), REDIS_OPT_BACKOFF_ALGORITHM);
783+
zend_declare_class_constant_long(ce, ZEND_STRL("BACKOFF_ALGORITHM_DEFAULT"), REDIS_BACKOFF_ALGORITHM_DEFAULT);
784+
zend_declare_class_constant_long(ce, ZEND_STRL("BACKOFF_ALGORITHM_CONSTANT"), REDIS_BACKOFF_ALGORITHM_CONSTANT);
785+
zend_declare_class_constant_long(ce, ZEND_STRL("BACKOFF_ALGORITHM_UNIFORM"), REDIS_BACKOFF_ALGORITHM_UNIFORM);
786+
zend_declare_class_constant_long(ce, ZEND_STRL("BACKOFF_ALGORITHM_EXPONENTIAL"), REDIS_BACKOFF_ALGORITHM_EXPONENTIAL);
787+
zend_declare_class_constant_long(ce, ZEND_STRL("BACKOFF_ALGORITHM_FULL_JITTER"), REDIS_BACKOFF_ALGORITHM_FULL_JITTER);
788+
zend_declare_class_constant_long(ce, ZEND_STRL("BACKOFF_ALGORITHM_EQUAL_JITTER"), REDIS_BACKOFF_ALGORITHM_EQUAL_JITTER);
789+
zend_declare_class_constant_long(ce, ZEND_STRL("BACKOFF_ALGORITHM_DECORRELATED_JITTER"), REDIS_BACKOFF_ALGORITHM_DECORRELATED_JITTER);
790+
791+
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_BACKOFF_BASE"), REDIS_OPT_BACKOFF_BASE);
792+
793+
zend_declare_class_constant_long(ce, ZEND_STRL("OPT_BACKOFF_CAP"), REDIS_OPT_BACKOFF_CAP);
778794
}
779795

780796
static ZEND_RSRC_DTOR_FUNC(redis_connections_pool_dtor)

‎redis_commands.c‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4308,6 +4308,14 @@ void redis_getoption_handler(INTERNAL_FUNCTION_PARAMETERS,
43084308
RETURN_LONG(redis_sock->null_mbulk_as_null);
43094309
case REDIS_OPT_FAILOVER:
43104310
RETURN_LONG(c->failover);
4311+
case REDIS_OPT_MAX_RETRIES:
4312+
RETURN_LONG(redis_sock->max_retries);
4313+
case REDIS_OPT_BACKOFF_ALGORITHM:
4314+
RETURN_LONG(redis_sock->backoff.algorithm);
4315+
case REDIS_OPT_BACKOFF_BASE:
4316+
RETURN_LONG(redis_sock->backoff.base / 1000);
4317+
case REDIS_OPT_BACKOFF_CAP:
4318+
RETURN_LONG(redis_sock->backoff.cap / 1000);
43114319
default:
43124320
RETURN_FALSE;
43134321
}
@@ -4441,6 +4449,35 @@ void redis_setoption_handler(INTERNAL_FUNCTION_PARAMETERS,
44414449
RETURN_TRUE;
44424450
}
44434451
break;
4452+
case REDIS_OPT_MAX_RETRIES:
4453+
val_long = zval_get_long(val);
4454+
if(val_long >= 0) {
4455+
redis_sock->max_retries = val_long;
4456+
RETURN_TRUE;
4457+
}
4458+
break;
4459+
case REDIS_OPT_BACKOFF_ALGORITHM:
4460+
val_long = zval_get_long(val);
4461+
if(val_long >= 0 &&
4462+
val_long < REDIS_BACKOFF_ALGORITHMS) {
4463+
redis_sock->backoff.algorithm = val_long;
4464+
RETURN_TRUE;
4465+
}
4466+
break;
4467+
case REDIS_OPT_BACKOFF_BASE:
4468+
val_long = zval_get_long(val);
4469+
if(val_long >= 0) {
4470+
redis_sock->backoff.base = val_long * 1000;
4471+
RETURN_TRUE;
4472+
}
4473+
break;
4474+
case REDIS_OPT_BACKOFF_CAP:
4475+
val_long = zval_get_long(val);
4476+
if(val_long >= 0) {
4477+
redis_sock->backoff.cap = val_long * 1000;
4478+
RETURN_TRUE;
4479+
}
4480+
break;
44444481
EMPTY_SWITCH_DEFAULT_CASE()
44454482
}
44464483
RETURN_FALSE;

0 commit comments

Comments
 (0)