[PATCH] RPC: parametrize various transport connect timeouts

 Each transport implementation can now set unique bind, connect,
 reestablishment, and idle timeout values.  These are variables,
 allowing the values to be modified dynamically.  This permits
 exponential backoff of any of these values, for instance.

 As an example, we implement exponential backoff for the connection
 reestablishment timeout.

 Test-plan:
 Destructive testing (unplugging the network temporarily).  Connectathon
 with UDP and TCP.

 Signed-off-by: Chuck Lever <cel@netapp.com>
 Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>

diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index b6a1ca508e6068c7e8367d968e0bc2e3a2daeb3e..062911e7ceb57b55ccf3cb7cd1141b82b825b10d 100644 (file)
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -369,8 +369,8 @@ static void nfs_init_timeout_values(struct rpc_timeout *to, int proto, unsigned
        case IPPROTO_TCP:
                if (!to->to_initval)
                        to->to_initval = 60 * HZ;
-               if (to->to_initval > RPC_MAX_TCP_TIMEOUT)
-                       to->to_initval = RPC_MAX_TCP_TIMEOUT;
+               if (to->to_initval > NFS_MAX_TCP_TIMEOUT)
+                       to->to_initval = NFS_MAX_TCP_TIMEOUT;
                to->to_increment = to->to_initval;
                to->to_maxval = to->to_initval + (to->to_increment * to->to_retries);
                to->to_exponential = 0;
@@ -379,9 +379,9 @@ static void nfs_init_timeout_values(struct rpc_timeout *to, int proto, unsigned
        default:
                if (!to->to_initval)
                        to->to_initval = 11 * HZ / 10;
-               if (to->to_initval > RPC_MAX_UDP_TIMEOUT)
-                       to->to_initval = RPC_MAX_UDP_TIMEOUT;
-               to->to_maxval = RPC_MAX_UDP_TIMEOUT;
+               if (to->to_initval > NFS_MAX_UDP_TIMEOUT)
+                       to->to_initval = NFS_MAX_UDP_TIMEOUT;
+               to->to_maxval = NFS_MAX_UDP_TIMEOUT;
                to->to_exponential = 1;
                break;
        }

diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h
index 9a6047ff1b25b198a869678463ed0561cbf417cc..7bac2785c6e4f5a9f67b68540a2084e5092ebf94 100644 (file)
--- a/include/linux/nfs_fs.h
+++ b/include/linux/nfs_fs.h
@@ -41,6 +41,10 @@
 #define NFS_MAX_FILE_IO_BUFFER_SIZE    32768
 #define NFS_DEF_FILE_IO_BUFFER_SIZE    4096
 
+/* Default timeout values */
+#define NFS_MAX_UDP_TIMEOUT    (60*HZ)
+#define NFS_MAX_TCP_TIMEOUT    (600*HZ)
+
 /*
  * superblock magic number for NFS
  */

diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h
index 9d9266cf8a36f96059f90cf7ac6a19ee8e45a8be..2543adf1855121f47f2a3aead4e0d7324ae64541 100644 (file)
--- a/include/linux/sunrpc/xprt.h
+++ b/include/linux/sunrpc/xprt.h
@@ -22,28 +22,6 @@ extern unsigned int xprt_tcp_slot_table_entries;
 #define RPC_DEF_SLOT_TABLE     (16U)
 #define RPC_MAX_SLOT_TABLE     (128U)
 
-/* Default timeout values */
-#define RPC_MAX_UDP_TIMEOUT    (60*HZ)
-#define RPC_MAX_TCP_TIMEOUT    (600*HZ)
-
-/*
- * Wait duration for an RPC TCP connection to be established.  Solaris
- * NFS over TCP uses 60 seconds, for example, which is in line with how
- * long a server takes to reboot.
- */
-#define RPC_CONNECT_TIMEOUT    (60*HZ)
-
-/*
- * Delay an arbitrary number of seconds before attempting to reconnect
- * after an error.
- */
-#define RPC_REESTABLISH_TIMEOUT        (15*HZ)
-
-/*
- * RPC transport idle timeout.
- */
-#define RPC_IDLE_DISCONNECT_TIMEOUT    (5*60*HZ)
-
 /*
  * RPC call and reply header size as number of 32bit words (verifier
  * size computed separately)
@@ -182,14 +160,19 @@ struct rpc_xprt {
        /*
         * Connection of transports
         */
+       unsigned long           connect_timeout,
+                               bind_timeout,
+                               reestablish_timeout;
        struct work_struct      connect_worker;
        unsigned short          port;
+
        /*
         * Disconnection of idle transports
         */
        struct work_struct      task_cleanup;
        struct timer_list       timer;
-       unsigned long           last_used;
+       unsigned long           last_used,
+                               idle_timeout;
 
        /*
         * Send stuff

diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c
index cc1b773a79d3da537f1b540d8337d4bb66609d8c..24b44e73f39119abf43d8e7ce951978292cbd0c7 100644 (file)
--- a/net/sunrpc/clnt.c
+++ b/net/sunrpc/clnt.c
@@ -740,7 +740,7 @@ call_bind(struct rpc_task *task)
        task->tk_action = call_connect;
        if (!clnt->cl_port) {
                task->tk_action = call_bind_status;
-               task->tk_timeout = RPC_CONNECT_TIMEOUT;
+               task->tk_timeout = task->tk_xprt->bind_timeout;
                rpc_getport(task, clnt);
        }
 }

diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c
index 0458319a1bddfb371a72acf1e9f94a3c874b939a..215be0d0ef6b163515da48866c87a59777a6bd17 100644 (file)
--- a/net/sunrpc/xprt.c
+++ b/net/sunrpc/xprt.c
@@ -551,7 +551,7 @@ void xprt_connect(struct rpc_task *task)
                if (task->tk_rqstp)
                        task->tk_rqstp->rq_bytes_sent = 0;
 
-               task->tk_timeout = RPC_CONNECT_TIMEOUT;
+               task->tk_timeout = xprt->connect_timeout;
                rpc_sleep_on(&xprt->pending, task, xprt_connect_status, NULL);
                xprt->ops->connect(task);
        }
@@ -763,7 +763,6 @@ void xprt_transmit(struct rpc_task *task)
 
        switch (status) {
        case -ECONNREFUSED:
-               task->tk_timeout = RPC_REESTABLISH_TIMEOUT;
                rpc_sleep_on(&xprt->sending, task, NULL, NULL);
        case -EAGAIN:
        case -ENOTCONN:
@@ -857,7 +856,7 @@ void xprt_release(struct rpc_task *task)
        xprt->last_used = jiffies;
        if (list_empty(&xprt->recv) && !xprt->shutdown)
                mod_timer(&xprt->timer,
-                               xprt->last_used + RPC_IDLE_DISCONNECT_TIMEOUT);
+                               xprt->last_used + xprt->idle_timeout);
        spin_unlock_bh(&xprt->transport_lock);
        task->tk_rqstp = NULL;
        memset(req, 0, sizeof(*req));   /* mark unused */

diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c
index 88ac71fcd335bdcd8717cc04fd22802f8b635cc3..06c2d95484e032a45b3fec9a3bc8916e2c24a57f 100644 (file)
--- a/net/sunrpc/xprtsock.c
+++ b/net/sunrpc/xprtsock.c
@@ -41,6 +41,50 @@
  */
 #define XS_SENDMSG_RETRY       (10U)
 
+/*
+ * Time out for an RPC UDP socket connect.  UDP socket connects are
+ * synchronous, but we set a timeout anyway in case of resource
+ * exhaustion on the local host.
+ */
+#define XS_UDP_CONN_TO         (5U * HZ)
+
+/*
+ * Wait duration for an RPC TCP connection to be established.  Solaris
+ * NFS over TCP uses 60 seconds, for example, which is in line with how
+ * long a server takes to reboot.
+ */
+#define XS_TCP_CONN_TO         (60U * HZ)
+
+/*
+ * Wait duration for a reply from the RPC portmapper.
+ */
+#define XS_BIND_TO             (60U * HZ)
+
+/*
+ * Delay if a UDP socket connect error occurs.  This is most likely some
+ * kind of resource problem on the local host.
+ */
+#define XS_UDP_REEST_TO                (2U * HZ)
+
+/*
+ * The reestablish timeout allows clients to delay for a bit before attempting
+ * to reconnect to a server that just dropped our connection.
+ *
+ * We implement an exponential backoff when trying to reestablish a TCP
+ * transport connection with the server.  Some servers like to drop a TCP
+ * connection when they are overworked, so we start with a short timeout and
+ * increase over time if the server is down or not responding.
+ */
+#define XS_TCP_INIT_REEST_TO   (3U * HZ)
+#define XS_TCP_MAX_REEST_TO    (5U * 60 * HZ)
+
+/*
+ * TCP idle timeout; client drops the transport socket if it is idle
+ * for this long.  Note that we also timeout UDP sockets to prevent
+ * holding port numbers when there is no RPC traffic.
+ */
+#define XS_IDLE_DISC_TO                (5U * 60 * HZ)
+
 #ifdef RPC_DEBUG
 # undef  RPC_DEBUG_DATA
 # define RPCDBG_FACILITY       RPCDBG_TRANS
@@ -739,6 +783,7 @@ static void xs_tcp_state_change(struct sock *sk)
                        xprt->tcp_reclen = 0;
                        xprt->tcp_copied = 0;
                        xprt->tcp_flags = XPRT_COPY_RECM | XPRT_COPY_XID;
+                       xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
                        xprt_wake_pending_tasks(xprt, 0);
                }
                spin_unlock_bh(&xprt->transport_lock);
@@ -1066,6 +1111,13 @@ out_clear:
  * @task: address of RPC task that manages state of connect request
  *
  * TCP: If the remote end dropped the connection, delay reconnecting.
+ *
+ * UDP socket connects are synchronous, but we use a work queue anyway
+ * to guarantee that even unprivileged user processes can set up a
+ * socket on a privileged port.
+ *
+ * If a UDP socket connect fails, the delay behavior here prevents
+ * retry floods (hard mounts).
  */
 static void xs_connect(struct rpc_task *task)
 {
@@ -1075,9 +1127,13 @@ static void xs_connect(struct rpc_task *task)
                return;
 
        if (xprt->sock != NULL) {
-               dprintk("RPC:      xs_connect delayed xprt %p\n", xprt);
+               dprintk("RPC:      xs_connect delayed xprt %p for %lu seconds\n",
+                               xprt, xprt->reestablish_timeout / HZ);
                schedule_delayed_work(&xprt->connect_worker,
-                                       RPC_REESTABLISH_TIMEOUT);
+                                       xprt->reestablish_timeout);
+               xprt->reestablish_timeout <<= 1;
+               if (xprt->reestablish_timeout > XS_TCP_MAX_REEST_TO)
+                       xprt->reestablish_timeout = XS_TCP_MAX_REEST_TO;
        } else {
                dprintk("RPC:      xs_connect scheduled xprt %p\n", xprt);
                schedule_work(&xprt->connect_worker);
@@ -1139,6 +1195,10 @@ int xs_setup_udp(struct rpc_xprt *xprt, struct rpc_timeout *to)
        xprt->max_payload = (1U << 16) - (MAX_HEADER << 3);
 
        INIT_WORK(&xprt->connect_worker, xs_udp_connect_worker, xprt);
+       xprt->bind_timeout = XS_BIND_TO;
+       xprt->connect_timeout = XS_UDP_CONN_TO;
+       xprt->reestablish_timeout = XS_UDP_REEST_TO;
+       xprt->idle_timeout = XS_IDLE_DISC_TO;
 
        xprt->ops = &xs_udp_ops;
 
@@ -1176,6 +1236,10 @@ int xs_setup_tcp(struct rpc_xprt *xprt, struct rpc_timeout *to)
        xprt->max_payload = RPC_MAX_FRAGMENT_SIZE;
 
        INIT_WORK(&xprt->connect_worker, xs_tcp_connect_worker, xprt);
+       xprt->bind_timeout = XS_BIND_TO;
+       xprt->connect_timeout = XS_TCP_CONN_TO;
+       xprt->reestablish_timeout = XS_TCP_INIT_REEST_TO;
+       xprt->idle_timeout = XS_IDLE_DISC_TO;
 
        xprt->ops = &xs_tcp_ops;