Initial commit of formalized "redirection" timeout logic

When phpredis is communicating with a cluster, there are two
different kinds of timeout events.

The first, is your standard read or write timeout where the socket is
blocked either because of network issues, or because Redis is taking
longer than the timeout to complete the request.

The second is unique to cluster.  Because Redis Cluster attempts to
automatically failover (in the case of replicas), phpredis cluster
will attempt to get data from a node where it thinks the key would
live, and upon a failure to connect, try a different node (at random).

This is because Redis could be resharding the connection and may point
the client to a new (now good node).  However, if it's not yet detected
a failure, it will just bounce us back to the prior node (which could
be actually down or have just sputtered due to various issues).

So in this case, phpredis uses a second timeout mechanism where we keep
track (in milleseconds) when we entered the query/response loop.  Once
we've been unsuccessful up to this timeout, phpredis will abort with
a different (catchable) exception.

TODO:  It may be a good idea to implement some small delay, so we don't
hit the cluster with lots of requests over and over until the cluster
comes back.

3 files changed, 45 insertions, 8 deletions

diff --git a/cluster_library.c b/cluster_library.c
index 09b34fc3..ee2a61aa 100644
--- a/cluster_library.c
+++ b/cluster_library.c
@@ -430,6 +430,18 @@ unsigned short cluster_hash_key(const char *key, int len) {
     return crc16((char*)key+s+1,e-s-1) & REDIS_CLUSTER_MOD;
 }
 
+/* Grab the current time in milliseconds */
+long long mstime(void) {
+    struct timeval tv;
+    long long mst;
+
+    gettimeofday(&tv, NULL);
+    mst = ((long long)tv.tv_sec)*1000;
+    mst += tv.tv_usec/1000;
+
+    return mst;
+}
+
 /* Hash a key from a ZVAL */
 unsigned short cluster_hash_key_zval(zval *z_key) {
     const char *kptr;
@@ -1260,9 +1272,15 @@ PHPAPI int cluster_send_slot(redisCluster *c, short slot, char *cmd,
 PHPAPI short cluster_send_command(redisCluster *c, short slot, const char *cmd, 
                                   int cmd_len TSRMLS_DC)
 {
-    int resp;
+    int resp, timedout=0;
+    long msstart;
 
-    // Issue commands until we find the right node or fail
+    /* Grab the current time in milliseconds */
+    msstart = mstime();
+
+    /* Our main cluster request/reply loop.  This loop runs until we're able
+     * to get a valid reply from a node, hit our "request" timeout, or encounter
+     * a CLUSTERDOWN state from Redis cluster. */
     do {
         // Send MULTI to the node if we haven't yet.
         if(c->flags->mode == MULTI && SLOT_SOCK(c,slot)->mode != MULTI) {
@@ -1309,13 +1327,19 @@ PHPAPI short cluster_send_command(redisCluster *c, short slot, const char *cmd,
             }
             slot = c->redir_slot;
         }
-    } while(resp != 0 && !c->clusterdown);
+
+        /* If we didn't get a valid response and we do have a timeout check it */
+        timedout = resp && c->waitms ? mstime() - msstart >= c->waitms : 0;
+    } while(resp != 0 && !c->clusterdown && !timedout);
 
     // If we've detected the cluster is down, throw an exception
     if(c->clusterdown) {
         zend_throw_exception(redis_cluster_exception_ce,
             "The Redis Cluster is down (CLUSTERDOWN)", 0 TSRMLS_CC);
         return -1;
+    } else if (timedout) {
+        zend_throw_exception(redis_cluster_exception_ce,
+            "Timed out attempting to find data in the correct node!", 0 TSRMLS_CC);
     }
 
     // Inform the cluster where to read the rest of our response,
diff --git a/cluster_library.h b/cluster_library.h
index 98ee305c..162b0d30 100644
--- a/cluster_library.h
+++ b/cluster_library.h
@@ -178,10 +178,13 @@ typedef struct redisCluster {
     /* Object reference for Zend */
     zend_object std;
 
-    /* Timeout and read timeout */
+    /* Timeout and read timeout (for normal operations) */
     double timeout;
     double read_timeout;
 
+    /* How long in milliseconds should we wait when being bounced around */
+    long waitms;
+
     /* Hash table of seed host/ports */
     HashTable *seeds;
 
@@ -214,9 +217,6 @@ typedef struct redisCluster {
     /* One RedisSock* struct for serialization and prefix information */
     RedisSock *flags;
 
-    /* Cluster distribution mode (speed, vs. maintaining order of execution) */
-    short dist_mode;
-
     /* The first line of our last reply, not including our reply type byte 
      * or the trailing \r\n */
     char line_reply[1024];
@@ -330,6 +330,9 @@ void cluster_multi_fini(clusterMultiCmd *mc);
 unsigned short cluster_hash_key_zval(zval *key);
 unsigned short cluster_hash_key(const char *key, int len);
 
+/* Get the current time in miliseconds */
+long long mstime(void);
+
 PHPAPI short cluster_send_command(redisCluster *c, short slot, const char *cmd, 
     int cmd_len TSRMLS_DC);
 
diff --git a/redis_cluster.c b/redis_cluster.c
index 47229370..7d077d96 100644
--- a/redis_cluster.c
+++ b/redis_cluster.c
@@ -333,7 +333,7 @@ void free_cluster_context(void *object TSRMLS_DC) {
 PHP_METHOD(RedisCluster, __construct) {
     zval *object, *z_seeds=NULL;
     char *name;
-    long name_len;
+    long name_len, tmsec;
     double timeout = 0.0, read_timeout = 0.0;
     redisCluster *context = GET_CONTEXT();
 
@@ -374,6 +374,16 @@ PHP_METHOD(RedisCluster, __construct) {
         RETURN_FALSE;
     }
 
+    /* Set our timeout and read_timeout which we'll pass through to the
+     * socket type operations */
+    context->timeout = timeout;
+    context->read_timeout = read_timeout;
+
+    /* Calculate the number of miliseconds we will wait when bouncing around,
+     * (e.g. a node goes down), which is not the same as a standard timeout. */
+    tmsec = (long)timeout * 1000;
+    context->waitms = tmsec + ((timeout-(long)timeout) * 1000);
+
     // Initialize our RedisSock "seed" objects
     cluster_init_seeds(context, Z_ARRVAL_P(z_seeds));