iscsi: per-session timeouts and rapid teardown of session on reconnect

Add per-Session configurable ping (SCSI NOP) and login timeouts.

Remove the torn down, old iSCSI session quickly, when performing a reconnect.

Reviewed By: trasz
Sponsored by:        NetApp, Inc.
Differential Revision: https://reviews.freebsd.org/D34198

diff --git a/lib/libiscsiutil/libiscsiutil.h b/lib/libiscsiutil/libiscsiutil.h
index 97d699c6c4b94000781d9b0fcbb0a46071a1233f..c47e33ff90aa7bdcf096f5af40a348bb576d0379 100644 (file)
--- a/lib/libiscsiutil/libiscsiutil.h
+++ b/lib/libiscsiutil/libiscsiutil.h
@@ -54,6 +54,8 @@ struct connection {
        int             conn_max_send_data_segment_length;
        int             conn_max_burst_length;
        int             conn_first_burst_length;
+       int             conn_ping_timeout;
+       int             conn_login_timeout;
 };
 
 struct pdu {

diff --git a/sys/dev/iscsi/iscsi.c b/sys/dev/iscsi/iscsi.c
index 1621e31576cf1c6d4ed4601c169864194dee15f9..c97bfaf0e6c5827c7876c56aba96046bf02486cb 100644 (file)
--- a/sys/dev/iscsi/iscsi.c
+++ b/sys/dev/iscsi/iscsi.c
@@ -42,9 +42,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/kthread.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
+#include <sys/mbuf.h>
 #include <sys/mutex.h>
 #include <sys/module.h>
 #include <sys/socket.h>
+#include <sys/sockopt.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
 #include <sys/sx.h>
@@ -86,6 +88,7 @@ SYSCTL_NODE(_kern, OID_AUTO, iscsi, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
 static int debug = 1;
 SYSCTL_INT(_kern_iscsi, OID_AUTO, debug, CTLFLAG_RWTUN,
     &debug, 0, "Enable debug messages");
+
 static int ping_timeout = 5;
 SYSCTL_INT(_kern_iscsi, OID_AUTO, ping_timeout, CTLFLAG_RWTUN, &ping_timeout,
     0, "Timeout for ping (NOP-Out) requests, in seconds");
@@ -380,6 +383,26 @@ iscsi_session_cleanup(struct iscsi_session *is, bool destroy_sim)
 static void
 iscsi_maintenance_thread_reconnect(struct iscsi_session *is)
 {
+       /*
+        * As we will be reconnecting shortly,
+        * discard outstanding data immediately on
+        * close(), also notify peer via RST if
+        * any packets come in.
+        */
+       struct socket *so;
+       so = is->is_conn->ic_socket;
+       if (so != NULL) {
+               struct sockopt sopt;
+               struct linger sl;
+               sopt.sopt_dir     = SOPT_SET;
+               sopt.sopt_level   = SOL_SOCKET;
+               sopt.sopt_name    = SO_LINGER;
+               sopt.sopt_val     = &sl;
+               sopt.sopt_valsize = sizeof(sl);
+               sl.l_onoff        = 1;  /* non-zero value enables linger option in kernel */
+               sl.l_linger       = 0;  /* timeout interval in seconds */
+               sosetopt(is->is_conn->ic_socket, &sopt);
+       }
 
        icl_conn_close(is->is_conn);
 
@@ -576,7 +599,7 @@ iscsi_callout(void *context)
        }
 
        if (is->is_login_phase) {
-               if (login_timeout > 0 && is->is_timeout > login_timeout) {
+               if (is->is_login_timeout > 0 && is->is_timeout > is->is_login_timeout) {
                        ISCSI_SESSION_WARN(is, "login timed out after %d seconds; "
                            "reconnecting", is->is_timeout);
                        reconnect_needed = true;
@@ -584,7 +607,7 @@ iscsi_callout(void *context)
                goto out;
        }
 
-       if (ping_timeout <= 0) {
+       if (is->is_ping_timeout <= 0) {
                /*
                 * Pings are disabled.  Don't send NOP-Out in this case.
                 * Reset the timeout, to avoid triggering reconnection,
@@ -594,9 +617,9 @@ iscsi_callout(void *context)
                goto out;
        }
 
-       if (is->is_timeout >= ping_timeout) {
+       if (is->is_timeout >= is->is_ping_timeout) {
                ISCSI_SESSION_WARN(is, "no ping reply (NOP-In) after %d seconds; "
-                   "reconnecting", ping_timeout);
+                   "reconnecting", is->is_ping_timeout);
                reconnect_needed = true;
                goto out;
        }
@@ -1509,6 +1532,12 @@ iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
        is->is_waiting_for_iscsid = false;
        is->is_login_phase = false;
        is->is_timeout = 0;
+       is->is_ping_timeout = is->is_conf.isc_ping_timeout;
+       if (is->is_ping_timeout < 0)
+               is->is_ping_timeout = ping_timeout;
+       is->is_login_timeout = is->is_conf.isc_login_timeout;
+       if (is->is_login_timeout < 0)
+               is->is_login_timeout = login_timeout;
        is->is_connected = true;
        is->is_reason[0] = '\0';
 
@@ -1915,6 +1944,12 @@ iscsi_ioctl_session_add(struct iscsi_softc *sc, struct iscsi_session_add *isa)
                sx_xunlock(&sc->sc_lock);
                return (error);
        }
+       is->is_ping_timeout = is->is_conf.isc_ping_timeout;
+       if (is->is_ping_timeout < 0)
+               is->is_ping_timeout = ping_timeout;
+       is->is_login_timeout = is->is_conf.isc_login_timeout;
+       if (is->is_login_timeout < 0)
+               is->is_login_timeout = login_timeout;
 
        sbt = mstosbt(995);
        pr = mstosbt(10);

diff --git a/sys/dev/iscsi/iscsi.h b/sys/dev/iscsi/iscsi.h
index 871fc6fc90e9af9833d0661ca8493a9105af7112..06a1ecc56890a87d5fdffac8b2ba7731f8267465 100644 (file)
--- a/sys/dev/iscsi/iscsi.h
+++ b/sys/dev/iscsi/iscsi.h
@@ -75,6 +75,8 @@ struct iscsi_session {
 
        struct callout                  is_callout;
        unsigned int                    is_timeout;
+       int                             is_ping_timeout;
+       int                             is_login_timeout;
 
        /*
         * XXX: This could be rewritten using a single variable,

diff --git a/sys/dev/iscsi/iscsi_ioctl.h b/sys/dev/iscsi/iscsi_ioctl.h
index 81e49d8d9a338e254fd0d4d481bdf32c0918abb6..c1de089c9d3f2efe463ab913cec197c059f45121 100644 (file)
--- a/sys/dev/iscsi/iscsi_ioctl.h
+++ b/sys/dev/iscsi/iscsi_ioctl.h
@@ -71,7 +71,8 @@ struct iscsi_session_conf {
        int             isc_enable;
        int             isc_dscp;
        int             isc_pcp;
-       int             isc_spare[2];
+       int             isc_ping_timeout;
+       int             isc_login_timeout;
 };
 
 /*

diff --git a/usr.bin/iscsictl/iscsi.conf.5 b/usr.bin/iscsictl/iscsi.conf.5
index b4adf29878284872325e6973445c4540ff6ac57d..878a2fbb3cecbf8413f6ecfb84f8196c6bca919f 100644 (file)
--- a/usr.bin/iscsictl/iscsi.conf.5
+++ b/usr.bin/iscsictl/iscsi.conf.5
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 6, 2016
+.Dd February 25, 2022
 .Dt ISCSI.CONF 5
 .Os
 .Sh NAME
@@ -160,6 +160,30 @@ The PCP can be set to a value in the range between
 to
 .Qq Ar 7 .
 When omitted, the default for the outgoing interface is used.
+.It Cm PingTimeout
+Specify the time in seconds to wait between pings (SCSI NOP), and
+for a ping response before declaring the session as dead and
+attempting a re-establishment.
+If this entry is not present in the conf file, the default value
+configured using
+.Qq Ar kern.iscsi.ping_timeout
+(default at
+.Qq Ar 5
+seconds) is taken by the driver.
+If present, the PingTimeout can be set to any positive value
+starting with
+.Qq Ar 1 .
+.It Cm LoginTimeout
+Specify the time in seconds to wait for a login PDU to be sent or
+received after trying to establish a new session.
+When no login PDU is received within this time, the login on a
+particular connection fails and a new reconnection attempt is made.
+If this entry is not present in the conf file, the default value of
+.Qq Ar 60
+seconds is used, as configured by
+.Qq Ar kern.iscsi.login_timeout .
+The LoginTimeout can be set to any positive value starting with
+.Qq Ar 1 .
 .El
 .Sh FILES
 .Bl -tag -width indent

diff --git a/usr.bin/iscsictl/iscsictl.c b/usr.bin/iscsictl/iscsictl.c
index e4c861b6c6a99ad62aa6f6bdef67207a9bfe84d7..258b4f7c28ac5063817344998187b62db37383b6 100644 (file)
--- a/usr.bin/iscsictl/iscsictl.c
+++ b/usr.bin/iscsictl/iscsictl.c
@@ -88,6 +88,8 @@ target_new(struct conf *conf)
        targ->t_conf = conf;
        targ->t_dscp = -1;
        targ->t_pcp = -1;
+       targ->t_pingtimeout = -1;
+       targ->t_logintimeout = -1;
        TAILQ_INSERT_TAIL(&conf->conf_targets, targ, t_next);
 
        return (targ);
@@ -361,6 +363,8 @@ conf_from_target(struct iscsi_session_conf *conf,
                conf->isc_data_digest = ISCSI_DIGEST_NONE;
        conf->isc_dscp = targ->t_dscp;
        conf->isc_pcp = targ->t_pcp;
+       conf->isc_ping_timeout = targ->t_pingtimeout;
+       conf->isc_login_timeout = targ->t_logintimeout;
 }
 
 static int
@@ -544,6 +548,14 @@ kernel_list(int iscsi_fd, const struct target *targ __unused,
                        if (conf->isc_pcp != -1)
                                xo_emit("{L:/%-26s}{V:pcp/0x%02x}\n",
                                    "Target PCP:", conf->isc_pcp);
+                       if (conf->isc_ping_timeout != -1)
+                               xo_emit("{L:/%-26s}{V:PingTimeout/%d}\n",
+                                   "Target PingTimeout:",
+                                   conf->isc_ping_timeout);
+                       if (conf->isc_login_timeout != -1)
+                               xo_emit("{L:/%-26s}{V:LoginTimeout/%d}\n",
+                                   "Target LoginTimeout:",
+                                   conf->isc_login_timeout);
                        xo_close_container("target");
 
                        xo_open_container("auth");

diff --git a/usr.bin/iscsictl/iscsictl.h b/usr.bin/iscsictl/iscsictl.h
index 4360a6eee26d35c9e3a818cf7bf82d9857658bbb..17fc27e9b083ed20fc342383ec5f705068163d92 100644 (file)
--- a/usr.bin/iscsictl/iscsictl.h
+++ b/usr.bin/iscsictl/iscsictl.h
@@ -79,6 +79,8 @@ struct target {
        int                     t_protocol;
        int                     t_dscp;
        int                     t_pcp;
+       int                     t_pingtimeout;
+       int                     t_logintimeout;
        char                    *t_offload;
        char                    *t_user;
        char                    *t_secret;

diff --git a/usr.bin/iscsictl/parse.y b/usr.bin/iscsictl/parse.y
index 333a512b5905f56f7a93b17350b574177a46d561..0fd33d7bd49b76c90de4252fa1357fc5003d718f 100644 (file)
--- a/usr.bin/iscsictl/parse.y
+++ b/usr.bin/iscsictl/parse.y
@@ -61,7 +61,7 @@ extern void   yyrestart(FILE *);
 %token AUTH_METHOD ENABLE HEADER_DIGEST DATA_DIGEST TARGET_NAME TARGET_ADDRESS
 %token INITIATOR_NAME INITIATOR_ADDRESS INITIATOR_ALIAS USER SECRET
 %token MUTUAL_USER MUTUAL_SECRET SEMICOLON SESSION_TYPE PROTOCOL OFFLOAD
-%token IGNORED EQUALS OPENING_BRACKET CLOSING_BRACKET DSCP
+%token IGNORED EQUALS OPENING_BRACKET CLOSING_BRACKET DSCP PINGTIMEOUT LOGINTIMEOUT
 %token AF11 AF12 AF13 AF21 AF22 AF23 AF31 AF32 AF33 AF41 AF42 AF43
 %token BE EF CS0 CS1 CS2 CS3 CS4 CS5 CS6 CS7
 
@@ -133,6 +133,10 @@ target_entry:
        dscp
        |
        pcp
+       |
+       ping_timeout
+       |
+       login_timeout
        ;
 
 target_name:   TARGET_NAME EQUALS STR
@@ -367,6 +371,38 @@ pcp:       PCP EQUALS STR
        }
        ;
 
+ping_timeout:  PINGTIMEOUT EQUALS STR
+       {
+               uint64_t tmp;
+
+               if (target->t_pingtimeout != -1)
+                       xo_errx(1, "duplicated PingTimeout at line %d", lineno);
+
+               if (expand_number($3, &tmp) != 0) {
+                       yyerror("invalid numeric value");
+                       free($3);
+                       return(1);
+               }
+               target->t_pingtimeout = tmp;
+       }
+       ;
+
+login_timeout: LOGINTIMEOUT EQUALS STR
+       {
+               uint64_t tmp;
+
+               if (target->t_logintimeout != -1)
+                       xo_errx(1, "duplicated LoginTimeout at line %d", lineno);
+
+               if (expand_number($3, &tmp) != 0) {
+                       yyerror("invalid numeric value");
+                       free($3);
+                       return(1);
+               }
+               target->t_logintimeout = tmp;
+       }
+       ;
+
 %%
 
 void

diff --git a/usr.bin/iscsictl/token.l b/usr.bin/iscsictl/token.l
index 0d517e5a89ad92af74bd1007903dfc6636ed9a0c..aa0defc67676e2882d8f7db9dd4d4753f16afa5f 100644 (file)
--- a/usr.bin/iscsictl/token.l
+++ b/usr.bin/iscsictl/token.l
@@ -69,6 +69,8 @@ offload                       { return OFFLOAD; }
 port                   { return IGNORED; }
 dscp                   { return DSCP; }
 pcp                    { return PCP; }
+PingTimeout            { return PINGTIMEOUT; }
+LoginTimeout           { return LOGINTIMEOUT; }
 MaxConnections         { return IGNORED; }
 TargetAlias            { return IGNORED; }
 TargetPortalGroupTag   { return IGNORED; }

diff --git a/usr.sbin/iscsid/iscsid.c b/usr.sbin/iscsid/iscsid.c
index 2689c4a2b455ec7d7536d8ce38b534ef1abe62a9..ebfcfa34dd6df2fd8819f15af54fff2c1f64be3b 100644 (file)
--- a/usr.sbin/iscsid/iscsid.c
+++ b/usr.sbin/iscsid/iscsid.c
@@ -38,9 +38,11 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/linker.h>
 #include <sys/socket.h>
+#include <sys/sysctl.h>
 #include <sys/capsicum.h>
 #include <sys/wait.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <assert.h>
 #include <capsicum_helpers.h>
 #include <errno.h>
@@ -383,6 +385,32 @@ connection_new(int iscsi_fd, const struct iscsi_daemon_request *request)
                                    from_addr);
                }
        }
+       /*
+        * Reduce TCP SYN_SENT timeout while
+        * no connectivity exists, to allow
+        * rapid reuse of the available slots.
+        */
+       int keepinit = 0;
+       if (conn->conn_conf.isc_login_timeout > 0) {
+               keepinit = conn->conn_conf.isc_login_timeout;
+               log_debugx("session specific LoginTimeout at %d sec",
+                       keepinit);
+       }
+       if (conn->conn_conf.isc_login_timeout == -1) {
+               char value[8];
+               size_t size = sizeof(value);
+               sysctlbyname("kern.iscsi.login_timeout", &value, &size,
+                       NULL, 0);
+               keepinit = strtol(value, NULL, 10);
+               log_debugx("global login_timeout at %d sec", keepinit);
+       }
+       if (keepinit > 0) {
+               if (setsockopt(conn->conn.conn_socket,
+                   IPPROTO_TCP, TCP_KEEPINIT,
+                   &keepinit, sizeof(keepinit)) == -1)
+                       log_warnx("setsockopt(TCP_KEEPINIT) "
+                           "failed for %s", to_addr);
+       }
        if (from_ai != NULL) {
                error = bind(conn->conn.conn_socket, from_ai->ai_addr,
                    from_ai->ai_addrlen);