|
Hi,
Although after one end terminates the connection, the spliced
connection will be removed automatically in 2 minutes, I think we should have a
way to terminate it if the two ends are always transferring something and don't
stop, or they are out of control.
That cookie is not based on time(NULL), it's only a example. The cookie
is application specific. When you call setsockopt to do tcp splicing, if the
cookie given already existed in the kernel, it will return -EINVAL to indicate
duplicated cookie. The sock fds are closed by calling close() for saving socket
resource's sake, so they may be reused by the kernel, or even the whole
forwarder process may exit, then I think we can't use the two sock fds to
identify a spliced connection. A cookie may be helpful, as we can see it from
/proc and it doesn't vary.
As you say, this is not true unsplicing, it simply delete that spliced
connection. I saw in whitepaper of Cisco CSM(content switching module), it has
a function of unsplice splicied HTTP/1.1 connection in order to rebalance the
traffic to another server. I'd like to continue working on it.
Regards,
Runhua Yang
-----Original Message-----
From: lvs-users-bounces+yangrunhua=njupt.edu.cn@xxxxxxxxxxxxxxxxxxxxxx
[mailto:lvs-users-bounces+yangrunhua=njupt.edu.cn@xxxxxxxxxxxxxxxxxxxxxx] On
Behalf Of Wensong Zhang
Sent: Saturday, November 01, 2003 11:58 PM
To: LinuxVirtualServer.org users mailing list.
Subject: Re: [PATCH]simple unsplicing to tcpsp-0.0.3 with the help of unique
cookie
Hi,
Your change is to delete the whole spliced connection, not to unsplice the
connection. In fact, after one end terminates the connection, the spliced
connection will be removed automatically in 2 minutes, there is probably
no need to delete it on purpose.
The unsplicing feature is important, after the connection is unspliced,
the two sockets can read/write data separately.
As for using the cookie to identify the spliced connection, it might be
not very good because the cookie based on time(NULL) is not unique. We can
still use the two socket fd to identify the spliced connection.
Regards,
Wensong
On Thu, 30 Oct 2003, yangrunhua wrote:
> Hi all,
> Compared to my last patch to tcpsp-0.0.3, it refuses to call
> tcpsp_conn_new for duplicated cookies, and return -EINVAL, so it will
> make that cookie unique.
> I recommend this new patch rather than the one I posted yesterday.
> From my last letter: "Current tcpsp doesn't seem to have a way to
> stop forwarding for already spliced TCP connections. I wrote this small
> patch to archive this with the help of a cookie(a __u32 number). It is
> much useful in practice when we want to stop particular spliced TCP
> connections, rather than to rmmod the whole tcpsp module."
> Any feedback is welcome!
>
> Runhua Yang(yangrunhua@xxxxxxxxxxxx)
>
>
> diff -urN tcpsp/example/forwarder.c tcpspnew/example/forwarder.c
> --- tcpsp/example/forwarder.c 2002-04-24 19:42:07.000000000 +0800
> +++ tcpspnew/example/forwarder.c 2003-10-29 00:33:51.000000000 +0800
> @@ -19,6 +19,7 @@
> #include <sys/poll.h>
> #include <netdb.h>
> #include <popt.h>
> +#include <time.h>
>
> #include "tcpsp.h"
>
> @@ -40,10 +41,10 @@
> }
>
>
> -int tcpsplicing(int fd1, int fd2, int n)
> +int tcpsplicing(int fd1, int fd2, int n, __u32 cookie)
> {
> int sockfd;
> - splice_conn_t sp = {fd1, fd2, n};
> + splice_conn_t sp = {fd1, fd2, n, cookie};
>
> if ((sockfd = socket(AF_INET, SOCK_RAW, IPPROTO_RAW)) == -1)
> return -1;
> @@ -109,7 +110,7 @@
> exit(1);
> }
>
> - if (tcpsplicing(fd1, fd2, n))
> + if (tcpsplicing(fd1, fd2, n, (__u32)time(NULL)/*cookie*/))
> printf("tcpsplicing failed\n");
> else
> printf("tcpsplicing succeeded\n");
> diff -urN tcpsp/tcpsp_conn.c tcpspnew/tcpsp_conn.c
> --- tcpsp/tcpsp_conn.c 2003-10-27 00:26:19.000000000 +0800
> +++ tcpspnew/tcpsp_conn.c 2003-10-30 15:02:12.000000000 +0800
> @@ -46,9 +46,10 @@
> static kmem_cache_t *tcpsp_conn_cachep;
>
> /* tcpsp onnection hash tables */
> -#define TCPSP_NTABLES 2
> +#define TCPSP_NTABLES 3
> static struct list_head *tcpsp_conn_tab1;
> static struct list_head *tcpsp_conn_tab2;
> +static struct list_head *tcpsp_conn_tab3;
>
> static rwlock_t tcpsp_conn_lock = RW_LOCK_UNLOCKED;
>
> @@ -68,6 +69,16 @@
> & TCPSP_CONN_TAB_MASK;
> }
>
> +/*
> + * Returns hash value for tcpsp connection cookie
> + */
> +static inline unsigned
> +tcpsp_conn_hash_cookie(__u32 cookie)
> +{
> + return (cookie^(cookie>>TCPSP_CONN_TAB_BITS))
> + & TCPSP_CONN_TAB_MASK;
> +}
> +
>
> /*
> * Hashes tcpsp_conn in tcpsp_conn_tabs by <addr,port>.
> @@ -89,6 +100,9 @@
>
> hash = tcpsp_conn_hash_key(cp->conn[1].laddr, cp->conn[1].lport);
> list_add(&cp->s_list, &tcpsp_conn_tab2[hash]);
> +
> + hash = tcpsp_conn_hash_cookie(cp->cookie);
> + list_add(&cp->t_list, &tcpsp_conn_tab3[hash]);
>
> cp->flags |= TCPSP_CONN_F_HASHED;
> atomic_add(TCPSP_NTABLES, &cp->refcnt);
> @@ -111,6 +125,7 @@
> write_lock(&tcpsp_conn_lock);
> list_del(&cp->f_list);
> list_del(&cp->s_list);
> + list_del(&cp->t_list);
> cp->flags &= ~TCPSP_CONN_F_HASHED;
> atomic_sub(TCPSP_NTABLES, &cp->refcnt);
> write_unlock(&tcpsp_conn_lock);
> @@ -474,7 +489,7 @@
> * Create and hash a new tcpsp into the tcpsp_conn_tabs.
> */
> struct tcpsp_conn *
> -tcpsp_conn_new(struct socket *sock1, struct socket *sock2, int n)
> +tcpsp_conn_new(struct socket *sock1, struct socket *sock2, int n, __u32
> cookie)
> {
> struct tcp_opt *tp1, *tp2;
> struct tcpsp_conn *cp;
> @@ -500,6 +515,8 @@
> fill_conn_tuple(&cp->conn[1], sock2->sk);
>
> cp->conn[1].splice_iss += n;
> +
> + cp->cookie = cookie;
>
> atomic_inc(&tcpsp_conn_count);
>
> @@ -546,7 +563,7 @@
> pos = 128;
> if (pos > offset) {
> len += sprintf(buffer+len, "%-127s\n",
> - "FromIP FPrt ToIP TPrt LocalIP LPrt
> DestIP DPrt State Expires");
> + "FromIP FPrt ToIP TPrt LocalIP LPrt
> DestIP DPrt State Expires Cookie ");
> }
>
> for(idx = 0; idx < TCPSP_CONN_TAB_SIZE; idx++) {
> @@ -563,7 +580,7 @@
> if (pos <= offset)
> continue;
> sprintf(temp,
> - "%08X %04X %08X %04X %08X %04X %08X %04X %-11s
> %7lu",
> + "%08X %04X %08X %04X %08X %04X %08X %04X %-11s
> %7lu %08X",
> ntohl(cp->conn[0].raddr),
> ntohs(cp->conn[0].rport),
> ntohl(cp->conn[0].laddr),
> @@ -573,7 +590,8 @@
> ntohl(cp->conn[1].raddr),
> ntohs(cp->conn[1].rport),
> tcpsp_state_name(cp->state),
> - cp->timer.expires-jiffies);
> + cp->timer.expires-jiffies,
> + cp->cookie);
> len += sprintf(buffer+len, "%-127s\n", temp);
> if (pos >= offset+length) {
> read_unlock_bh(&tcpsp_conn_lock);
> @@ -630,6 +648,53 @@
> }
> }
>
> +/*
> + * Find the connection entry matches the cookie in the tcpsp_conn_tab,
> + * and return it without locking
> + */
> +struct tcpsp_conn * tcpsp_conn_find(__u32 cookie)
> +{
> + int idx;
> + struct tcpsp_conn *cp;
> + struct list_head *e, *nxt;
> +
> + idx = tcpsp_conn_hash_cookie(cookie);
> + /*
> + * Lock is actually needed in this loop.
> + */
> + read_lock(&tcpsp_conn_lock);
> +
> + list_for_each_safe (e, nxt, &tcpsp_conn_tab3[idx]) {
> + cp = list_entry(e, struct tcpsp_conn, t_list);
> + read_unlock(&tcpsp_conn_lock);
> + if (cp->cookie == cookie){
> + return cp;
> + }
> +
> + read_lock(&tcpsp_conn_lock);
> + }
> + read_unlock(&tcpsp_conn_lock);
> +
> + return NULL;
> +}
> +
> +/*
> + * Delete the connection entry matches the cookie in the tcpsp_conn_tab
> + */
> +int tcpsp_conn_del(__u32 cookie)
> +{
> + struct tcpsp_conn *cp = tcpsp_conn_find(cookie);
> +
> + if (cp == NULL) return -EINVAL;
> +
> + TCPSP_DBG(4, "del splicing connection\n");
> +
> + if (del_timer(&cp->timer)){
> + tcpsp_conn_expire_now(cp);
> + }
> +
> + return 0;
> +}
>
> int tcpsp_conn_init(void)
> {
> @@ -647,6 +712,13 @@
> vfree(tcpsp_conn_tab1);
> return -ENOMEM;
> }
> +
> + if (!(tcpsp_conn_tab3 =
> + vmalloc(TCPSP_CONN_TAB_SIZE * sizeof(struct list_head)))) {
> + vfree(tcpsp_conn_tab2);
> + vfree(tcpsp_conn_tab1);
> + return -ENOMEM;
> + }
>
> /* Allocate tcpsp_conn slab cache */
> tcpsp_conn_cachep = kmem_cache_create("tcpsp_conn",
> @@ -655,6 +727,7 @@
> if (!tcpsp_conn_cachep) {
> vfree(tcpsp_conn_tab1);
> vfree(tcpsp_conn_tab2);
> + vfree(tcpsp_conn_tab3);
> return -ENOMEM;
> }
>
> @@ -668,6 +741,7 @@
> for (idx = 0; idx < TCPSP_CONN_TAB_SIZE; idx++) {
> INIT_LIST_HEAD(&tcpsp_conn_tab1[idx]);
> INIT_LIST_HEAD(&tcpsp_conn_tab2[idx]);
> + INIT_LIST_HEAD(&tcpsp_conn_tab3[idx]);
> }
>
> proc_net_create("tcpsp_conn", 0, tcpsp_conn_getinfo);
> @@ -685,4 +759,5 @@
> proc_net_remove("tcpsp_conn");
> vfree(tcpsp_conn_tab1);
> vfree(tcpsp_conn_tab2);
> + vfree(tcpsp_conn_tab3);
> }
> diff -urN tcpsp/tcpsp_ctl.c tcpspnew/tcpsp_ctl.c
> --- tcpsp/tcpsp_ctl.c 2003-10-27 00:26:19.000000000 +0800
> +++ tcpspnew/tcpsp_ctl.c 2003-10-30 14:14:40.000000000 +0800
> @@ -169,7 +169,7 @@
> }
>
>
> -int sys_tcpsplicing(int fd1, int fd2, int n)
> +int sys_tcpsplicing(int fd1, int fd2, int n, __u32 cookie)
> {
> struct socket *sock1, *sock2;
> struct tcpsp_conn *cp;
> @@ -179,12 +179,18 @@
> goto out;
> if (!(sock2 = sockfd_lookup(fd2, &err)))
> goto out_put_sock1;
> +
> + if ((cp = tcpsp_conn_find(cookie))){
> + err = -EINVAL;
> + goto out_put_sock2;
> + }
>
> - if (!(cp = tcpsp_conn_new(sock1, sock2, n)))
> + if (!(cp = tcpsp_conn_new(sock1, sock2, n, cookie)))
> err = -ENOMEM;
> else
> tcpsp_conn_put(cp);
>
> + out_put_sock2:
> sockfd_put(sock2);
> out_put_sock1:
> sockfd_put(sock1);
> @@ -192,6 +198,11 @@
> return err;
> }
>
> +int sys_tcpunsplicing(__u32 cookie)
> +{
> + return tcpsp_conn_del(cookie);
> +}
> +
>
> /* this interface is ugly, will be changed to system call */
> static int
> @@ -217,7 +228,10 @@
>
> switch (cmd) {
> case TCPSP_SO_SET_ADD:
> - ret = sys_tcpsplicing(sp.s1, sp.s2, sp.n);
> + ret = sys_tcpsplicing(sp.s1, sp.s2, sp.n, sp.cookie);
> + break;
> + case TCPSP_SO_SET_DEL:
> + ret = sys_tcpunsplicing(sp.cookie);
> break;
> default:
> ret = -EINVAL;
> diff -urN tcpsp/tcpsp.h tcpspnew/tcpsp.h
> --- tcpsp/tcpsp.h 2002-05-20 11:22:42.000000000 +0800
> +++ tcpspnew/tcpsp.h 2003-10-30 14:16:16.000000000 +0800
> @@ -24,7 +24,8 @@
>
> #define TCPSP_SO_SET_NONE TCPSP_BASE_CTL /* just peek */
> #define TCPSP_SO_SET_ADD (TCPSP_BASE_CTL+1)
> -#define TCPSP_SO_SET_MAX TCPSP_SO_SET_ADD
> +#define TCPSP_SO_SET_DEL (TCPSP_BASE_CTL+2)
> +#define TCPSP_SO_SET_MAX TCPSP_SO_SET_DEL
>
> #define TCPSP_SO_GET_VERSION TCPSP_BASE_CTL
> #define TCPSP_SO_GET_INFO (TCPSP_BASE_CTL+1)
> @@ -39,6 +40,8 @@
>
> /* the number of bytes written to s2 */
> int n;
> +
> + __u32 cookie;
> } splice_conn_t;
>
>
> @@ -177,6 +180,7 @@
> struct tcpsp_conn {
> struct list_head f_list; /* first hash table */
> struct list_head s_list; /* second hash table */
> + struct list_head t_list; /* third hash table */
>
> atomic_t refcnt; /* reference count */
>
> @@ -197,6 +201,9 @@
>
> /* packet transmitter */
> int (*packet_xmit)(struct sk_buff *skb);
> +
> + /* cookie */
> + __u32 cookie; /* cookie */
> };
>
>
> @@ -232,7 +239,9 @@
> extern void tcpsp_conn_put(struct tcpsp_conn *cp);
>
> extern struct tcpsp_conn *
> -tcpsp_conn_new(struct socket *sock1, struct socket *sock2, int n);
> +tcpsp_conn_new(struct socket *sock1, struct socket *sock2, int n, __u32
> cookie);
> +extern int tcpsp_conn_del(__u32 cookie);
> +extern struct tcpsp_conn * tcpsp_conn_find(__u32 cookie);
> #define tcpsp_conn_expire_now(cp) cp->timer.function((unsigned long)cp)
>
> extern const char * tcpsp_state_name(int state);
>
> _______________________________________________
> LinuxVirtualServer.org mailing list - lvs-users@xxxxxxxxxxxxxxxxxxxxxx
> Send requests to lvs-users-request@xxxxxxxxxxxxxxxxxxxxxx
> or go to http://www.in-addr.de/mailman/listinfo/lvs-users
>
_______________________________________________
LinuxVirtualServer.org mailing list - lvs-users@xxxxxxxxxxxxxxxxxxxxxx
Send requests to lvs-users-request@xxxxxxxxxxxxxxxxxxxxxx
or go to http://www.in-addr.de/mailman/listinfo/lvs-users
|
