net/ipv4/tcp.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              Implementation of the Transmission Control Protocol(TCP).
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Mark Evans, <evansmp@uhura.aston.ac.uk>
  11  *              Corey Minyard <wf-rch!minyard@relay.EU.net>
  12  *              Florian La Roche, <flla@stud.uni-sb.de>
  13  *              Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
  14  *              Linus Torvalds, <torvalds@cs.helsinki.fi>
  15  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  16  *              Matthew Dillon, <dillon@apollo.west.oic.com>
  17  *              Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  18  *              Jorge Cwik, <jorge@laser.satlink.net>
  19  *
  20  * Fixes:
  21  *              Alan Cox        :       Numerous verify_area() calls
  22  *              Alan Cox        :       Set the ACK bit on a reset
  23  *              Alan Cox        :       Stopped it crashing if it closed while
  24  *                                      sk->inuse=1 and was trying to connect
  25  *                                      (tcp_err()).
  26  *              Alan Cox        :       All icmp error handling was broken
  27  *                                      pointers passed where wrong and the
  28  *                                      socket was looked up backwards. Nobody
  29  *                                      tested any icmp error code obviously.
  30  *              Alan Cox        :       tcp_err() now handled properly. It
  31  *                                      wakes people on errors. poll
  32  *                                      behaves and the icmp error race
  33  *                                      has gone by moving it into sock.c
  34  *              Alan Cox        :       tcp_send_reset() fixed to work for
  35  *                                      everything not just packets for
  36  *                                      unknown sockets.
  37  *              Alan Cox        :       tcp option processing.
  38  *              Alan Cox        :       Reset tweaked (still not 100%) [Had
  39  *                                      syn rule wrong]
  40  *              Herp Rosmanith  :       More reset fixes
  41  *              Alan Cox        :       No longer acks invalid rst frames.
  42  *                                      Acking any kind of RST is right out.
  43  *              Alan Cox        :       Sets an ignore me flag on an rst
  44  *                                      receive otherwise odd bits of prattle
  45  *                                      escape still
  46  *              Alan Cox        :       Fixed another acking RST frame bug.
  47  *                                      Should stop LAN workplace lockups.
  48  *              Alan Cox        :       Some tidyups using the new skb list
  49  *                                      facilities
  50  *              Alan Cox        :       sk->keepopen now seems to work
  51  *              Alan Cox        :       Pulls options out correctly on accepts
  52  *              Alan Cox        :       Fixed assorted sk->rqueue->next errors
  53  *              Alan Cox        :       PSH doesn't end a TCP read. Switched a
  54  *                                      bit to skb ops.
  55  *              Alan Cox        :       Tidied tcp_data to avoid a potential
  56  *                                      nasty.
  57  *              Alan Cox        :       Added some better commenting, as the
  58  *                                      tcp is hard to follow
  59  *              Alan Cox        :       Removed incorrect check for 20 * psh
  60  *      Michael O'Reilly        :       ack < copied bug fix.
  61  *      Johannes Stille         :       Misc tcp fixes (not all in yet).
  62  *              Alan Cox        :       FIN with no memory -> CRASH
  63  *              Alan Cox        :       Added socket option proto entries.
  64  *                                      Also added awareness of them to accept.
  65  *              Alan Cox        :       Added TCP options (SOL_TCP)
  66  *              Alan Cox        :       Switched wakeup calls to callbacks,
  67  *                                      so the kernel can layer network
  68  *                                      sockets.
  69  *              Alan Cox        :       Use ip_tos/ip_ttl settings.
  70  *              Alan Cox        :       Handle FIN (more) properly (we hope).
  71  *              Alan Cox        :       RST frames sent on unsynchronised
  72  *                                      state ack error.
  73  *              Alan Cox        :       Put in missing check for SYN bit.
  74  *              Alan Cox        :       Added tcp_select_window() aka NET2E
  75  *                                      window non shrink trick.
  76  *              Alan Cox        :       Added a couple of small NET2E timer
  77  *                                      fixes
  78  *              Charles Hedrick :       TCP fixes
  79  *              Toomas Tamm     :       TCP window fixes
  80  *              Alan Cox        :       Small URG fix to rlogin ^C ack fight
  81  *              Charles Hedrick :       Rewrote most of it to actually work
  82  *              Linus           :       Rewrote tcp_read() and URG handling
  83  *                                      completely
  84  *              Gerhard Koerting:       Fixed some missing timer handling
  85  *              Matthew Dillon  :       Reworked TCP machine states as per RFC
  86  *              Gerhard Koerting:       PC/TCP workarounds
  87  *              Adam Caldwell   :       Assorted timer/timing errors
  88  *              Matthew Dillon  :       Fixed another RST bug
  89  *              Alan Cox        :       Move to kernel side addressing changes.
  90  *              Alan Cox        :       Beginning work on TCP fastpathing
  91  *                                      (not yet usable)
  92  *              Arnt Gulbrandsen:       Turbocharged tcp_check() routine.
  93  *              Alan Cox        :       TCP fast path debugging
  94  *              Alan Cox        :       Window clamping
  95  *              Michael Riepe   :       Bug in tcp_check()
  96  *              Matt Dillon     :       More TCP improvements and RST bug fixes
  97  *              Matt Dillon     :       Yet more small nasties remove from the
  98  *                                      TCP code (Be very nice to this man if
  99  *                                      tcp finally works 100%) 8)
 100  *              Alan Cox        :       BSD accept semantics.
 101  *              Alan Cox        :       Reset on closedown bug.
 102  *      Peter De Schrijver      :       ENOTCONN check missing in tcp_sendto().
 103  *              Michael Pall    :       Handle poll() after URG properly in
 104  *                                      all cases.
 105  *              Michael Pall    :       Undo the last fix in tcp_read_urg()
 106  *                                      (multi URG PUSH broke rlogin).
 107  *              Michael Pall    :       Fix the multi URG PUSH problem in
 108  *                                      tcp_readable(), poll() after URG
 109  *                                      works now.
 110  *              Michael Pall    :       recv(...,MSG_OOB) never blocks in the
 111  *                                      BSD api.
 112  *              Alan Cox        :       Changed the semantics of sk->socket to
 113  *                                      fix a race and a signal problem with
 114  *                                      accept() and async I/O.
 115  *              Alan Cox        :       Relaxed the rules on tcp_sendto().
 116  *              Yury Shevchuk   :       Really fixed accept() blocking problem.
 117  *              Craig I. Hagan  :       Allow for BSD compatible TIME_WAIT for
 118  *                                      clients/servers which listen in on
 119  *                                      fixed ports.
 120  *              Alan Cox        :       Cleaned the above up and shrank it to
 121  *                                      a sensible code size.
 122  *              Alan Cox        :       Self connect lockup fix.
 123  *              Alan Cox        :       No connect to multicast.
 124  *              Ross Biro       :       Close unaccepted children on master
 125  *                                      socket close.
 126  *              Alan Cox        :       Reset tracing code.
 127  *              Alan Cox        :       Spurious resets on shutdown.
 128  *              Alan Cox        :       Giant 15 minute/60 second timer error
 129  *              Alan Cox        :       Small whoops in polling before an
 130  *                                      accept.
 131  *              Alan Cox        :       Kept the state trace facility since
 132  *                                      it's handy for debugging.
 133  *              Alan Cox        :       More reset handler fixes.
 134  *              Alan Cox        :       Started rewriting the code based on
 135  *                                      the RFC's for other useful protocol
 136  *                                      references see: Comer, KA9Q NOS, and
 137  *                                      for a reference on the difference
 138  *                                      between specifications and how BSD
 139  *                                      works see the 4.4lite source.
 140  *              A.N.Kuznetsov   :       Don't time wait on completion of tidy
 141  *                                      close.
 142  *              Linus Torvalds  :       Fin/Shutdown & copied_seq changes.
 143  *              Linus Torvalds  :       Fixed BSD port reuse to work first syn
 144  *              Alan Cox        :       Reimplemented timers as per the RFC
 145  *                                      and using multiple timers for sanity.
 146  *              Alan Cox        :       Small bug fixes, and a lot of new
 147  *                                      comments.
 148  *              Alan Cox        :       Fixed dual reader crash by locking
 149  *                                      the buffers (much like datagram.c)
 150  *              Alan Cox        :       Fixed stuck sockets in probe. A probe
 151  *                                      now gets fed up of retrying without
 152  *                                      (even a no space) answer.
 153  *              Alan Cox        :       Extracted closing code better
 154  *              Alan Cox        :       Fixed the closing state machine to
 155  *                                      resemble the RFC.
 156  *              Alan Cox        :       More 'per spec' fixes.
 157  *              Jorge Cwik      :       Even faster checksumming.
 158  *              Alan Cox        :       tcp_data() doesn't ack illegal PSH
 159  *                                      only frames. At least one pc tcp stack
 160  *                                      generates them.
 161  *              Alan Cox        :       Cache last socket.
 162  *              Alan Cox        :       Per route irtt.
 163  *              Matt Day        :       poll()->select() match BSD precisely on error
 164  *              Alan Cox        :       New buffers
 165  *              Marc Tamsky     :       Various sk->prot->retransmits and
 166  *                                      sk->retransmits misupdating fixed.
 167  *                                      Fixed tcp_write_timeout: stuck close,
 168  *                                      and TCP syn retries gets used now.
 169  *              Mark Yarvis     :       In tcp_read_wakeup(), don't send an
 170  *                                      ack if state is TCP_CLOSED.
 171  *              Alan Cox        :       Look up device on a retransmit - routes may
 172  *                                      change. Doesn't yet cope with MSS shrink right
 173  *                                      but it's a start!
 174  *              Marc Tamsky     :       Closing in closing fixes.
 175  *              Mike Shaver     :       RFC1122 verifications.
 176  *              Alan Cox        :       rcv_saddr errors.
 177  *              Alan Cox        :       Block double connect().
 178  *              Alan Cox        :       Small hooks for enSKIP.
 179  *              Alexey Kuznetsov:       Path MTU discovery.
 180  *              Alan Cox        :       Support soft errors.
 181  *              Alan Cox        :       Fix MTU discovery pathological case
 182  *                                      when the remote claims no mtu!
 183  *              Marc Tamsky     :       TCP_CLOSE fix.
 184  *              Colin (G3TNE)   :       Send a reset on syn ack replies in
 185  *                                      window but wrong (fixes NT lpd problems)
 186  *              Pedro Roque     :       Better TCP window handling, delayed ack.
 187  *              Joerg Reuter    :       No modification of locked buffers in
 188  *                                      tcp_do_retransmit()
 189  *              Eric Schenk     :       Changed receiver side silly window
 190  *                                      avoidance algorithm to BSD style
 191  *                                      algorithm. This doubles throughput
 192  *                                      against machines running Solaris,
 193  *                                      and seems to result in general
 194  *                                      improvement.
 195  *      Stefan Magdalinski      :       adjusted tcp_readable() to fix FIONREAD
 196  *      Willy Konynenberg       :       Transparent proxying support.
 197  *      Mike McLagan            :       Routing by source
 198  *              Keith Owens     :       Do proper merging with partial SKB's in
 199  *                                      tcp_do_sendmsg to avoid burstiness.
 200  *              Eric Schenk     :       Fix fast close down bug with
 201  *                                      shutdown() followed by close().
 202  *              Andi Kleen      :       Make poll agree with SIGIO
 203  *      Salvatore Sanfilippo    :       Support SO_LINGER with linger == 1 and
 204  *                                      lingertime == 0 (RFC 793 ABORT Call)
 205  *      Hirokazu Takahashi      :       Use copy_from_user() instead of
 206  *                                      csum_and_copy_from_user() if possible.
 207  *
 208  *              This program is free software; you can redistribute it and/or
 209  *              modify it under the terms of the GNU General Public License
 210  *              as published by the Free Software Foundation; either version
 211  *              2 of the License, or(at your option) any later version.
 212  *
 213  * Description of States:
 214  *
 215  *      TCP_SYN_SENT            sent a connection request, waiting for ack
 216  *
 217  *      TCP_SYN_RECV            received a connection request, sent ack,
 218  *                              waiting for final ack in three-way handshake.
 219  *
 220  *      TCP_ESTABLISHED         connection established
 221  *
 222  *      TCP_FIN_WAIT1           our side has shutdown, waiting to complete
 223  *                              transmission of remaining buffered data
 224  *
 225  *      TCP_FIN_WAIT2           all buffered data sent, waiting for remote
 226  *                              to shutdown
 227  *
 228  *      TCP_CLOSING             both sides have shutdown but we still have
 229  *                              data we have to finish sending
 230  *
 231  *      TCP_TIME_WAIT           timeout to catch resent junk before entering
 232  *                              closed, can only be entered from FIN_WAIT2
 233  *                              or CLOSING.  Required because the other end
 234  *                              may not have gotten our last ACK causing it
 235  *                              to retransmit the data packet (which we ignore)
 236  *
 237  *      TCP_CLOSE_WAIT          remote side has shutdown and is waiting for
 238  *                              us to finish writing our data and to shutdown
 239  *                              (we have to close() to move on to LAST_ACK)
 240  *
 241  *      TCP_LAST_ACK            out side has shutdown after remote has
 242  *                              shutdown.  There may still be data in our
 243  *                              buffer that we have to finish sending
 244  *
 245  *      TCP_CLOSE               socket is finished
 246  */
 247
 248 #include <linux/kernel.h>
 249 #include <linux/module.h>
 250 #include <linux/types.h>
 251 #include <linux/fcntl.h>
 252 #include <linux/poll.h>
 253 #include <linux/init.h>
 254 #include <linux/fs.h>
 255 #include <linux/skbuff.h>
 256 #include <linux/scatterlist.h>
 257 #include <linux/splice.h>
 258 #include <linux/net.h>
 259 #include <linux/socket.h>
 260 #include <linux/random.h>
 261 #include <linux/bootmem.h>
 262 #include <linux/highmem.h>
 263 #include <linux/swap.h>
 264 #include <linux/cache.h>
 265 #include <linux/err.h>
 266 #include <linux/crypto.h>
 267
 268 #include <net/icmp.h>
 269 #include <net/tcp.h>
 270 #include <net/xfrm.h>
 271 #include <net/ip.h>
 272 #include <net/netdma.h>
 273 #include <net/sock.h>
 274
 275 #include <asm/uaccess.h>
 276 #include <asm/ioctls.h>
 277
 278 int sysctl_tcp_fin_timeout __read_mostly = TCP_FIN_TIMEOUT;
 279
 280 DEFINE_SNMP_STAT(struct tcp_mib, tcp_statistics) __read_mostly;
 281
 282 atomic_t tcp_orphan_count = ATOMIC_INIT(0);
 283
 284 EXPORT_SYMBOL_GPL(tcp_orphan_count);
 285
 286 int sysctl_tcp_mem[3] __read_mostly;
 287 int sysctl_tcp_wmem[3] __read_mostly;
 288 int sysctl_tcp_rmem[3] __read_mostly;
 289
 290 EXPORT_SYMBOL(sysctl_tcp_mem);
 291 EXPORT_SYMBOL(sysctl_tcp_rmem);
 292 EXPORT_SYMBOL(sysctl_tcp_wmem);
 293
 294 atomic_t tcp_memory_allocated;  /* Current allocated memory. */
 295 atomic_t tcp_sockets_allocated; /* Current number of TCP sockets. */
 296
 297 EXPORT_SYMBOL(tcp_memory_allocated);
 298 EXPORT_SYMBOL(tcp_sockets_allocated);
 299
 300 /*
 301  * TCP splice context
 302  */
 303 struct tcp_splice_state {
 304         struct pipe_inode_info *pipe;
 305         size_t len;
 306         unsigned int flags;
 307 };
 308
 309 /*
 310  * Pressure flag: try to collapse.
 311  * Technical note: it is used by multiple contexts non atomically.
 312  * All the __sk_mem_schedule() is of this nature: accounting
 313  * is strict, actions are advisory and have some latency.
 314  */
 315 int tcp_memory_pressure __read_mostly;
 316
 317 EXPORT_SYMBOL(tcp_memory_pressure);
 318
 319 void tcp_enter_memory_pressure(void)
 320 {
 321         if (!tcp_memory_pressure) {
 322                 NET_INC_STATS(LINUX_MIB_TCPMEMORYPRESSURES);
 323                 tcp_memory_pressure = 1;
 324         }
 325 }
 326
 327 EXPORT_SYMBOL(tcp_enter_memory_pressure);
 328
 329 /*
 330  *      Wait for a TCP event.
 331  *
 332  *      Note that we don't need to lock the socket, as the upper poll layers
 333  *      take care of normal races (between the test and the event) and we don't
 334  *      go look at any of the socket buffers directly.
 335  */
 336 unsigned int tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
 337 {
 338         unsigned int mask;
 339         struct sock *sk = sock->sk;
 340         struct tcp_sock *tp = tcp_sk(sk);
 341
 342         poll_wait(file, sk->sk_sleep, wait);
 343         if (sk->sk_state == TCP_LISTEN)
 344                 return inet_csk_listen_poll(sk);
 345
 346         /* Socket is not locked. We are protected from async events
 347            by poll logic and correct handling of state changes
 348            made by another threads is impossible in any case.
 349          */
 350
 351         mask = 0;
 352         if (sk->sk_err)
 353                 mask = POLLERR;
 354
 355         /*
 356          * POLLHUP is certainly not done right. But poll() doesn't
 357          * have a notion of HUP in just one direction, and for a
 358          * socket the read side is more interesting.
 359          *
 360          * Some poll() documentation says that POLLHUP is incompatible
 361          * with the POLLOUT/POLLWR flags, so somebody should check this
 362          * all. But careful, it tends to be safer to return too many
 363          * bits than too few, and you can easily break real applications
 364          * if you don't tell them that something has hung up!
 365          *
 366          * Check-me.
 367          *
 368          * Check number 1. POLLHUP is _UNMASKABLE_ event (see UNIX98 and
 369          * our fs/select.c). It means that after we received EOF,
 370          * poll always returns immediately, making impossible poll() on write()
 371          * in state CLOSE_WAIT. One solution is evident --- to set POLLHUP
 372          * if and only if shutdown has been made in both directions.
 373          * Actually, it is interesting to look how Solaris and DUX
 374          * solve this dilemma. I would prefer, if PULLHUP were maskable,
 375          * then we could set it on SND_SHUTDOWN. BTW examples given
 376          * in Stevens' books assume exactly this behaviour, it explains
 377          * why PULLHUP is incompatible with POLLOUT.    --ANK
 378          *
 379          * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
 380          * blocking on fresh not-connected or disconnected socket. --ANK
 381          */
 382         if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE)
 383                 mask |= POLLHUP;
 384         if (sk->sk_shutdown & RCV_SHUTDOWN)
 385                 mask |= POLLIN | POLLRDNORM | POLLRDHUP;
 386
 387         /* Connected? */
 388         if ((1 << sk->sk_state) & ~(TCPF_SYN_SENT | TCPF_SYN_RECV)) {
 389                 /* Potential race condition. If read of tp below will
 390                  * escape above sk->sk_state, we can be illegally awaken
 391                  * in SYN_* states. */
 392                 if ((tp->rcv_nxt != tp->copied_seq) &&
 393                     (tp->urg_seq != tp->copied_seq ||
 394                      tp->rcv_nxt != tp->copied_seq + 1 ||
 395                      sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data))
 396                         mask |= POLLIN | POLLRDNORM;
 397
 398                 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
 399                         if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk)) {
 400                                 mask |= POLLOUT | POLLWRNORM;
 401                         } else {  /* send SIGIO later */
 402                                 set_bit(SOCK_ASYNC_NOSPACE,
 403                                         &sk->sk_socket->flags);
 404                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 405
 406                                 /* Race breaker. If space is freed after
 407                                  * wspace test but before the flags are set,
 408                                  * IO signal will be lost.
 409                                  */
 410                                 if (sk_stream_wspace(sk) >= sk_stream_min_wspace(sk))
 411                                         mask |= POLLOUT | POLLWRNORM;
 412                         }
 413                 }
 414
 415                 if (tp->urg_data & TCP_URG_VALID)
 416                         mask |= POLLPRI;
 417         }
 418         return mask;
 419 }
 420
 421 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
 422 {
 423         struct tcp_sock *tp = tcp_sk(sk);
 424         int answ;
 425
 426         switch (cmd) {
 427         case SIOCINQ:
 428                 if (sk->sk_state == TCP_LISTEN)
 429                         return -EINVAL;
 430
 431                 lock_sock(sk);
 432                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 433                         answ = 0;
 434                 else if (sock_flag(sk, SOCK_URGINLINE) ||
 435                          !tp->urg_data ||
 436                          before(tp->urg_seq, tp->copied_seq) ||
 437                          !before(tp->urg_seq, tp->rcv_nxt)) {
 438                         answ = tp->rcv_nxt - tp->copied_seq;
 439
 440                         /* Subtract 1, if FIN is in queue. */
 441                         if (answ && !skb_queue_empty(&sk->sk_receive_queue))
 442                                 answ -=
 443                        tcp_hdr((struct sk_buff *)sk->sk_receive_queue.prev)->fin;
 444                 } else
 445                         answ = tp->urg_seq - tp->copied_seq;
 446                 release_sock(sk);
 447                 break;
 448         case SIOCATMARK:
 449                 answ = tp->urg_data && tp->urg_seq == tp->copied_seq;
 450                 break;
 451         case SIOCOUTQ:
 452                 if (sk->sk_state == TCP_LISTEN)
 453                         return -EINVAL;
 454
 455                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 456                         answ = 0;
 457                 else
 458                         answ = tp->write_seq - tp->snd_una;
 459                 break;
 460         default:
 461                 return -ENOIOCTLCMD;
 462         }
 463
 464         return put_user(answ, (int __user *)arg);
 465 }
 466
 467 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
 468 {
 469         TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH;
 470         tp->pushed_seq = tp->write_seq;
 471 }
 472
 473 static inline int forced_push(struct tcp_sock *tp)
 474 {
 475         return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
 476 }
 477
 478 static inline void skb_entail(struct sock *sk, struct sk_buff *skb)
 479 {
 480         struct tcp_sock *tp = tcp_sk(sk);
 481         struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 482
 483         skb->csum    = 0;
 484         tcb->seq     = tcb->end_seq = tp->write_seq;
 485         tcb->flags   = TCPCB_FLAG_ACK;
 486         tcb->sacked  = 0;
 487         skb_header_release(skb);
 488         tcp_add_write_queue_tail(sk, skb);
 489         sk->sk_wmem_queued += skb->truesize;
 490         sk_mem_charge(sk, skb->truesize);
 491         if (tp->nonagle & TCP_NAGLE_PUSH)
 492                 tp->nonagle &= ~TCP_NAGLE_PUSH;
 493 }
 494
 495 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags,
 496                                 struct sk_buff *skb)
 497 {
 498         if (flags & MSG_OOB) {
 499                 tp->urg_mode = 1;
 500                 tp->snd_up = tp->write_seq;
 501         }
 502 }
 503
 504 static inline void tcp_push(struct sock *sk, int flags, int mss_now,
 505                             int nonagle)
 506 {
 507         struct tcp_sock *tp = tcp_sk(sk);
 508
 509         if (tcp_send_head(sk)) {
 510                 struct sk_buff *skb = tcp_write_queue_tail(sk);
 511                 if (!(flags & MSG_MORE) || forced_push(tp))
 512                         tcp_mark_push(tp, skb);
 513                 tcp_mark_urg(tp, flags, skb);
 514                 __tcp_push_pending_frames(sk, mss_now,
 515                                           (flags & MSG_MORE) ? TCP_NAGLE_CORK : nonagle);
 516         }
 517 }
 518
 519 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
 520                                 unsigned int offset, size_t len)
 521 {
 522         struct tcp_splice_state *tss = rd_desc->arg.data;
 523
 524         return skb_splice_bits(skb, offset, tss->pipe, tss->len, tss->flags);
 525 }
 526
 527 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
 528 {
 529         /* Store TCP splice context information in read_descriptor_t. */
 530         read_descriptor_t rd_desc = {
 531                 .arg.data = tss,
 532         };
 533
 534         return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
 535 }
 536
 537 /**
 538  *  tcp_splice_read - splice data from TCP socket to a pipe
 539  * @sock:       socket to splice from
 540  * @ppos:       position (not valid)
 541  * @pipe:       pipe to splice to
 542  * @len:        number of bytes to splice
 543  * @flags:      splice modifier flags
 544  *
 545  * Description:
 546  *    Will read pages from given socket and fill them into a pipe.
 547  *
 548  **/
 549 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
 550                         struct pipe_inode_info *pipe, size_t len,
 551                         unsigned int flags)
 552 {
 553         struct sock *sk = sock->sk;
 554         struct tcp_splice_state tss = {
 555                 .pipe = pipe,
 556                 .len = len,
 557                 .flags = flags,
 558         };
 559         long timeo;
 560         ssize_t spliced;
 561         int ret;
 562
 563         /*
 564          * We can't seek on a socket input
 565          */
 566         if (unlikely(*ppos))
 567                 return -ESPIPE;
 568
 569         ret = spliced = 0;
 570
 571         lock_sock(sk);
 572
 573         timeo = sock_rcvtimeo(sk, flags & SPLICE_F_NONBLOCK);
 574         while (tss.len) {
 575                 ret = __tcp_splice_read(sk, &tss);
 576                 if (ret < 0)
 577                         break;
 578                 else if (!ret) {
 579                         if (spliced)
 580                                 break;
 581                         if (flags & SPLICE_F_NONBLOCK) {
 582                                 ret = -EAGAIN;
 583                                 break;
 584                         }
 585                         if (sock_flag(sk, SOCK_DONE))
 586                                 break;
 587                         if (sk->sk_err) {
 588                                 ret = sock_error(sk);
 589                                 break;
 590                         }
 591                         if (sk->sk_shutdown & RCV_SHUTDOWN)
 592                                 break;
 593                         if (sk->sk_state == TCP_CLOSE) {
 594                                 /*
 595                                  * This occurs when user tries to read
 596                                  * from never connected socket.
 597                                  */
 598                                 if (!sock_flag(sk, SOCK_DONE))
 599                                         ret = -ENOTCONN;
 600                                 break;
 601                         }
 602                         if (!timeo) {
 603                                 ret = -EAGAIN;
 604                                 break;
 605                         }
 606                         sk_wait_data(sk, &timeo);
 607                         if (signal_pending(current)) {
 608                                 ret = sock_intr_errno(timeo);
 609                                 break;
 610                         }
 611                         continue;
 612                 }
 613                 tss.len -= ret;
 614                 spliced += ret;
 615
 616                 release_sock(sk);
 617                 lock_sock(sk);
 618
 619                 if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
 620                     (sk->sk_shutdown & RCV_SHUTDOWN) || !timeo ||
 621                     signal_pending(current))
 622                         break;
 623         }
 624
 625         release_sock(sk);
 626
 627         if (spliced)
 628                 return spliced;
 629
 630         return ret;
 631 }
 632
 633 struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp)
 634 {
 635         struct sk_buff *skb;
 636
 637         /* The TCP header must be at least 32-bit aligned.  */
 638         size = ALIGN(size, 4);
 639
 640         skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp);
 641         if (skb) {
 642                 if (sk_wmem_schedule(sk, skb->truesize)) {
 643                         /*
 644                          * Make sure that we have exactly size bytes
 645                          * available to the caller, no more, no less.
 646                          */
 647                         skb_reserve(skb, skb_tailroom(skb) - size);
 648                         return skb;
 649                 }
 650                 __kfree_skb(skb);
 651         } else {
 652                 sk->sk_prot->enter_memory_pressure();
 653                 sk_stream_moderate_sndbuf(sk);
 654         }
 655         return NULL;
 656 }
 657
 658 static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffset,
 659                          size_t psize, int flags)
 660 {
 661         struct tcp_sock *tp = tcp_sk(sk);
 662         int mss_now, size_goal;
 663         int err;
 664         ssize_t copied;
 665         long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 666
 667         /* Wait for a connection to finish. */
 668         if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
 669                 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
 670                         goto out_err;
 671
 672         clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 673
 674         mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 675         size_goal = tp->xmit_size_goal;
 676         copied = 0;
 677
 678         err = -EPIPE;
 679         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 680                 goto do_error;
 681
 682         while (psize > 0) {
 683                 struct sk_buff *skb = tcp_write_queue_tail(sk);
 684                 struct page *page = pages[poffset / PAGE_SIZE];
 685                 int copy, i, can_coalesce;
 686                 int offset = poffset % PAGE_SIZE;
 687                 int size = min_t(size_t, psize, PAGE_SIZE - offset);
 688
 689                 if (!tcp_send_head(sk) || (copy = size_goal - skb->len) <= 0) {
 690 new_segment:
 691                         if (!sk_stream_memory_free(sk))
 692                                 goto wait_for_sndbuf;
 693
 694                         skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation);
 695                         if (!skb)
 696                                 goto wait_for_memory;
 697
 698                         skb_entail(sk, skb);
 699                         copy = size_goal;
 700                 }
 701
 702                 if (copy > size)
 703                         copy = size;
 704
 705                 i = skb_shinfo(skb)->nr_frags;
 706                 can_coalesce = skb_can_coalesce(skb, i, page, offset);
 707                 if (!can_coalesce && i >= MAX_SKB_FRAGS) {
 708                         tcp_mark_push(tp, skb);
 709                         goto new_segment;
 710                 }
 711                 if (!sk_wmem_schedule(sk, copy))
 712                         goto wait_for_memory;
 713
 714                 if (can_coalesce) {
 715                         skb_shinfo(skb)->frags[i - 1].size += copy;
 716                 } else {
 717                         get_page(page);
 718                         skb_fill_page_desc(skb, i, page, offset, copy);
 719                 }
 720
 721                 skb->len += copy;
 722                 skb->data_len += copy;
 723                 skb->truesize += copy;
 724                 sk->sk_wmem_queued += copy;
 725                 sk_mem_charge(sk, copy);
 726                 skb->ip_summed = CHECKSUM_PARTIAL;
 727                 tp->write_seq += copy;
 728                 TCP_SKB_CB(skb)->end_seq += copy;
 729                 skb_shinfo(skb)->gso_segs = 0;
 730
 731                 if (!copied)
 732                         TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH;
 733
 734                 copied += copy;
 735                 poffset += copy;
 736                 if (!(psize -= copy))
 737                         goto out;
 738
 739                 if (skb->len < size_goal || (flags & MSG_OOB))
 740                         continue;
 741
 742                 if (forced_push(tp)) {
 743                         tcp_mark_push(tp, skb);
 744                         __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
 745                 } else if (skb == tcp_send_head(sk))
 746                         tcp_push_one(sk, mss_now);
 747                 continue;
 748
 749 wait_for_sndbuf:
 750                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 751 wait_for_memory:
 752                 if (copied)
 753                         tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
 754
 755                 if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
 756                         goto do_error;
 757
 758                 mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 759                 size_goal = tp->xmit_size_goal;
 760         }
 761
 762 out:
 763         if (copied)
 764                 tcp_push(sk, flags, mss_now, tp->nonagle);
 765         return copied;
 766
 767 do_error:
 768         if (copied)
 769                 goto out;
 770 out_err:
 771         return sk_stream_error(sk, flags, err);
 772 }
 773
 774 ssize_t tcp_sendpage(struct socket *sock, struct page *page, int offset,
 775                      size_t size, int flags)
 776 {
 777         ssize_t res;
 778         struct sock *sk = sock->sk;
 779
 780         if (!(sk->sk_route_caps & NETIF_F_SG) ||
 781             !(sk->sk_route_caps & NETIF_F_ALL_CSUM))
 782                 return sock_no_sendpage(sock, page, offset, size, flags);
 783
 784         lock_sock(sk);
 785         TCP_CHECK_TIMER(sk);
 786         res = do_tcp_sendpages(sk, &page, offset, size, flags);
 787         TCP_CHECK_TIMER(sk);
 788         release_sock(sk);
 789         return res;
 790 }
 791
 792 #define TCP_PAGE(sk)    (sk->sk_sndmsg_page)
 793 #define TCP_OFF(sk)     (sk->sk_sndmsg_off)
 794
 795 static inline int select_size(struct sock *sk)
 796 {
 797         struct tcp_sock *tp = tcp_sk(sk);
 798         int tmp = tp->mss_cache;
 799
 800         if (sk->sk_route_caps & NETIF_F_SG) {
 801                 if (sk_can_gso(sk))
 802                         tmp = 0;
 803                 else {
 804                         int pgbreak = SKB_MAX_HEAD(MAX_TCP_HEADER);
 805
 806                         if (tmp >= pgbreak &&
 807                             tmp <= pgbreak + (MAX_SKB_FRAGS - 1) * PAGE_SIZE)
 808                                 tmp = pgbreak;
 809                 }
 810         }
 811
 812         return tmp;
 813 }
 814
 815 int tcp_sendmsg(struct kiocb *iocb, struct socket *sock, struct msghdr *msg,
 816                 size_t size)
 817 {
 818         struct sock *sk = sock->sk;
 819         struct iovec *iov;
 820         struct tcp_sock *tp = tcp_sk(sk);
 821         struct sk_buff *skb;
 822         int iovlen, flags;
 823         int mss_now, size_goal;
 824         int err, copied;
 825         long timeo;
 826
 827         lock_sock(sk);
 828         TCP_CHECK_TIMER(sk);
 829
 830         flags = msg->msg_flags;
 831         timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 832
 833         /* Wait for a connection to finish. */
 834         if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
 835                 if ((err = sk_stream_wait_connect(sk, &timeo)) != 0)
 836                         goto out_err;
 837
 838         /* This should be in poll */
 839         clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 840
 841         mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 842         size_goal = tp->xmit_size_goal;
 843
 844         /* Ok commence sending. */
 845         iovlen = msg->msg_iovlen;
 846         iov = msg->msg_iov;
 847         copied = 0;
 848
 849         err = -EPIPE;
 850         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 851                 goto do_error;
 852
 853         while (--iovlen >= 0) {
 854                 int seglen = iov->iov_len;
 855                 unsigned char __user *from = iov->iov_base;
 856
 857                 iov++;
 858
 859                 while (seglen > 0) {
 860                         int copy;
 861
 862                         skb = tcp_write_queue_tail(sk);
 863
 864                         if (!tcp_send_head(sk) ||
 865                             (copy = size_goal - skb->len) <= 0) {
 866
 867 new_segment:
 868                                 /* Allocate new segment. If the interface is SG,
 869                                  * allocate skb fitting to single page.
 870                                  */
 871                                 if (!sk_stream_memory_free(sk))
 872                                         goto wait_for_sndbuf;
 873
 874                                 skb = sk_stream_alloc_skb(sk, select_size(sk),
 875                                                 sk->sk_allocation);
 876                                 if (!skb)
 877                                         goto wait_for_memory;
 878
 879                                 /*
 880                                  * Check whether we can use HW checksum.
 881                                  */
 882                                 if (sk->sk_route_caps & NETIF_F_ALL_CSUM)
 883                                         skb->ip_summed = CHECKSUM_PARTIAL;
 884
 885                                 skb_entail(sk, skb);
 886                                 copy = size_goal;
 887                         }
 888
 889                         /* Try to append data to the end of skb. */
 890                         if (copy > seglen)
 891                                 copy = seglen;
 892
 893                         /* Where to copy to? */
 894                         if (skb_tailroom(skb) > 0) {
 895                                 /* We have some space in skb head. Superb! */
 896                                 if (copy > skb_tailroom(skb))
 897                                         copy = skb_tailroom(skb);
 898                                 if ((err = skb_add_data(skb, from, copy)) != 0)
 899                                         goto do_fault;
 900                         } else {
 901                                 int merge = 0;
 902                                 int i = skb_shinfo(skb)->nr_frags;
 903                                 struct page *page = TCP_PAGE(sk);
 904                                 int off = TCP_OFF(sk);
 905
 906                                 if (skb_can_coalesce(skb, i, page, off) &&
 907                                     off != PAGE_SIZE) {
 908                                         /* We can extend the last page
 909                                          * fragment. */
 910                                         merge = 1;
 911                                 } else if (i == MAX_SKB_FRAGS ||
 912                                            (!i &&
 913                                            !(sk->sk_route_caps & NETIF_F_SG))) {
 914                                         /* Need to add new fragment and cannot
 915                                          * do this because interface is non-SG,
 916                                          * or because all the page slots are
 917                                          * busy. */
 918                                         tcp_mark_push(tp, skb);
 919                                         goto new_segment;
 920                                 } else if (page) {
 921                                         if (off == PAGE_SIZE) {
 922                                                 put_page(page);
 923                                                 TCP_PAGE(sk) = page = NULL;
 924                                                 off = 0;
 925                                         }
 926                                 } else
 927                                         off = 0;
 928
 929                                 if (copy > PAGE_SIZE - off)
 930                                         copy = PAGE_SIZE - off;
 931
 932                                 if (!sk_wmem_schedule(sk, copy))
 933                                         goto wait_for_memory;
 934
 935                                 if (!page) {
 936                                         /* Allocate new cache page. */
 937                                         if (!(page = sk_stream_alloc_page(sk)))
 938                                                 goto wait_for_memory;
 939                                 }
 940
 941                                 /* Time to copy data. We are close to
 942                                  * the end! */
 943                                 err = skb_copy_to_page(sk, from, skb, page,
 944                                                        off, copy);
 945                                 if (err) {
 946                                         /* If this page was new, give it to the
 947                                          * socket so it does not get leaked.
 948                                          */
 949                                         if (!TCP_PAGE(sk)) {
 950                                                 TCP_PAGE(sk) = page;
 951                                                 TCP_OFF(sk) = 0;
 952                                         }
 953                                         goto do_error;
 954                                 }
 955
 956                                 /* Update the skb. */
 957                                 if (merge) {
 958                                         skb_shinfo(skb)->frags[i - 1].size +=
 959                                                                         copy;
 960                                 } else {
 961                                         skb_fill_page_desc(skb, i, page, off, copy);
 962                                         if (TCP_PAGE(sk)) {
 963                                                 get_page(page);
 964                                         } else if (off + copy < PAGE_SIZE) {
 965                                                 get_page(page);
 966                                                 TCP_PAGE(sk) = page;
 967                                         }
 968                                 }
 969
 970                                 TCP_OFF(sk) = off + copy;
 971                         }
 972
 973                         if (!copied)
 974                                 TCP_SKB_CB(skb)->flags &= ~TCPCB_FLAG_PSH;
 975
 976                         tp->write_seq += copy;
 977                         TCP_SKB_CB(skb)->end_seq += copy;
 978                         skb_shinfo(skb)->gso_segs = 0;
 979
 980                         from += copy;
 981                         copied += copy;
 982                         if ((seglen -= copy) == 0 && iovlen == 0)
 983                                 goto out;
 984
 985                         if (skb->len < size_goal || (flags & MSG_OOB))
 986                                 continue;
 987
 988                         if (forced_push(tp)) {
 989                                 tcp_mark_push(tp, skb);
 990                                 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
 991                         } else if (skb == tcp_send_head(sk))
 992                                 tcp_push_one(sk, mss_now);
 993                         continue;
 994
 995 wait_for_sndbuf:
 996                         set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 997 wait_for_memory:
 998                         if (copied)
 999                                 tcp_push(sk, flags & ~MSG_MORE, mss_now, TCP_NAGLE_PUSH);
1000
1001                         if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
1002                                 goto do_error;
1003
1004                         mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
1005                         size_goal = tp->xmit_size_goal;
1006                 }
1007         }
1008
1009 out:
1010         if (copied)
1011                 tcp_push(sk, flags, mss_now, tp->nonagle);
1012         TCP_CHECK_TIMER(sk);
1013         release_sock(sk);
1014         return copied;
1015
1016 do_fault:
1017         if (!skb->len) {
1018                 tcp_unlink_write_queue(skb, sk);
1019                 /* It is the one place in all of TCP, except connection
1020                  * reset, where we can be unlinking the send_head.
1021                  */
1022                 tcp_check_send_head(sk, skb);
1023                 sk_wmem_free_skb(sk, skb);
1024         }
1025
1026 do_error:
1027         if (copied)
1028                 goto out;
1029 out_err:
1030         err = sk_stream_error(sk, flags, err);
1031         TCP_CHECK_TIMER(sk);
1032         release_sock(sk);
1033         return err;
1034 }
1035
1036 /*
1037  *      Handle reading urgent data. BSD has very simple semantics for
1038  *      this, no blocking and very strange errors 8)
1039  */
1040
1041 static int tcp_recv_urg(struct sock *sk, long timeo,
1042                         struct msghdr *msg, int len, int flags,
1043                         int *addr_len)
1044 {
1045         struct tcp_sock *tp = tcp_sk(sk);
1046
1047         /* No URG data to read. */
1048         if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1049             tp->urg_data == TCP_URG_READ)
1050                 return -EINVAL; /* Yes this is right ! */
1051
1052         if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1053                 return -ENOTCONN;
1054
1055         if (tp->urg_data & TCP_URG_VALID) {
1056                 int err = 0;
1057                 char c = tp->urg_data;
1058
1059                 if (!(flags & MSG_PEEK))
1060                         tp->urg_data = TCP_URG_READ;
1061
1062                 /* Read urgent data. */
1063                 msg->msg_flags |= MSG_OOB;
1064
1065                 if (len > 0) {
1066                         if (!(flags & MSG_TRUNC))
1067                                 err = memcpy_toiovec(msg->msg_iov, &c, 1);
1068                         len = 1;
1069                 } else
1070                         msg->msg_flags |= MSG_TRUNC;
1071
1072                 return err ? -EFAULT : len;
1073         }
1074
1075         if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1076                 return 0;
1077
1078         /* Fixed the recv(..., MSG_OOB) behaviour.  BSD docs and
1079          * the available implementations agree in this case:
1080          * this call should never block, independent of the
1081          * blocking state of the socket.
1082          * Mike <pall@rz.uni-karlsruhe.de>
1083          */
1084         return -EAGAIN;
1085 }
1086
1087 /* Clean up the receive buffer for full frames taken by the user,
1088  * then send an ACK if necessary.  COPIED is the number of bytes
1089  * tcp_recvmsg has given to the user so far, it speeds up the
1090  * calculation of whether or not we must ACK for the sake of
1091  * a window update.
1092  */
1093 void tcp_cleanup_rbuf(struct sock *sk, int copied)
1094 {
1095         struct tcp_sock *tp = tcp_sk(sk);
1096         int time_to_ack = 0;
1097
1098 #if TCP_DEBUG
1099         struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1100
1101         BUG_TRAP(!skb || before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq));
1102 #endif
1103
1104         if (inet_csk_ack_scheduled(sk)) {
1105                 const struct inet_connection_sock *icsk = inet_csk(sk);
1106                    /* Delayed ACKs frequently hit locked sockets during bulk
1107                     * receive. */
1108                 if (icsk->icsk_ack.blocked ||
1109                     /* Once-per-two-segments ACK was not sent by tcp_input.c */
1110                     tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1111                     /*
1112                      * If this read emptied read buffer, we send ACK, if
1113                      * connection is not bidirectional, user drained
1114                      * receive buffer and there was a small segment
1115                      * in queue.
1116                      */
1117                     (copied > 0 &&
1118                      ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1119                       ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1120                        !icsk->icsk_ack.pingpong)) &&
1121                       !atomic_read(&sk->sk_rmem_alloc)))
1122                         time_to_ack = 1;
1123         }
1124
1125         /* We send an ACK if we can now advertise a non-zero window
1126          * which has been raised "significantly".
1127          *
1128          * Even if window raised up to infinity, do not send window open ACK
1129          * in states, where we will not receive more. It is useless.
1130          */
1131         if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1132                 __u32 rcv_window_now = tcp_receive_window(tp);
1133
1134                 /* Optimize, __tcp_select_window() is not cheap. */
1135                 if (2*rcv_window_now <= tp->window_clamp) {
1136                         __u32 new_window = __tcp_select_window(sk);
1137
1138                         /* Send ACK now, if this read freed lots of space
1139                          * in our buffer. Certainly, new_window is new window.
1140                          * We can advertise it now, if it is not less than current one.
1141                          * "Lots" means "at least twice" here.
1142                          */
1143                         if (new_window && new_window >= 2 * rcv_window_now)
1144                                 time_to_ack = 1;
1145                 }
1146         }
1147         if (time_to_ack)
1148                 tcp_send_ack(sk);
1149 }
1150
1151 static void tcp_prequeue_process(struct sock *sk)
1152 {
1153         struct sk_buff *skb;
1154         struct tcp_sock *tp = tcp_sk(sk);
1155
1156         NET_INC_STATS_USER(LINUX_MIB_TCPPREQUEUED);
1157
1158         /* RX process wants to run with disabled BHs, though it is not
1159          * necessary */
1160         local_bh_disable();
1161         while ((skb = __skb_dequeue(&tp->ucopy.prequeue)) != NULL)
1162                 sk->sk_backlog_rcv(sk, skb);
1163         local_bh_enable();
1164
1165         /* Clear memory counter. */
1166         tp->ucopy.memory = 0;
1167 }
1168
1169 static inline struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1170 {
1171         struct sk_buff *skb;
1172         u32 offset;
1173
1174         skb_queue_walk(&sk->sk_receive_queue, skb) {
1175                 offset = seq - TCP_SKB_CB(skb)->seq;
1176                 if (tcp_hdr(skb)->syn)
1177                         offset--;
1178                 if (offset < skb->len || tcp_hdr(skb)->fin) {
1179                         *off = offset;
1180                         return skb;
1181                 }
1182         }
1183         return NULL;
1184 }
1185
1186 /*
1187  * This routine provides an alternative to tcp_recvmsg() for routines
1188  * that would like to handle copying from skbuffs directly in 'sendfile'
1189  * fashion.
1190  * Note:
1191  *      - It is assumed that the socket was locked by the caller.
1192  *      - The routine does not block.
1193  *      - At present, there is no support for reading OOB data
1194  *        or for 'peeking' the socket using this routine
1195  *        (although both would be easy to implement).
1196  */
1197 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1198                   sk_read_actor_t recv_actor)
1199 {
1200         struct sk_buff *skb;
1201         struct tcp_sock *tp = tcp_sk(sk);
1202         u32 seq = tp->copied_seq;
1203         u32 offset;
1204         int copied = 0;
1205
1206         if (sk->sk_state == TCP_LISTEN)
1207                 return -ENOTCONN;
1208         while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1209                 if (offset < skb->len) {
1210                         int used;
1211                         size_t len;
1212
1213                         len = skb->len - offset;
1214                         /* Stop reading if we hit a patch of urgent data */
1215                         if (tp->urg_data) {
1216                                 u32 urg_offset = tp->urg_seq - seq;
1217                                 if (urg_offset < len)
1218                                         len = urg_offset;
1219                                 if (!len)
1220                                         break;
1221                         }
1222                         used = recv_actor(desc, skb, offset, len);
1223                         if (used < 0) {
1224                                 if (!copied)
1225                                         copied = used;
1226                                 break;
1227                         } else if (used <= len) {
1228                                 seq += used;
1229                                 copied += used;
1230                                 offset += used;
1231                         }
1232                         /*
1233                          * If recv_actor drops the lock (e.g. TCP splice
1234                          * receive) the skb pointer might be invalid when
1235                          * getting here: tcp_collapse might have deleted it
1236                          * while aggregating skbs from the socket queue.
1237                          */
1238                         skb = tcp_recv_skb(sk, seq-1, &offset);
1239                         if (!skb || (offset+1 != skb->len))
1240                                 break;
1241                 }
1242                 if (tcp_hdr(skb)->fin) {
1243                         sk_eat_skb(sk, skb, 0);
1244                         ++seq;
1245                         break;
1246                 }
1247                 sk_eat_skb(sk, skb, 0);
1248                 if (!desc->count)
1249                         break;
1250         }
1251         tp->copied_seq = seq;
1252
1253         tcp_rcv_space_adjust(sk);
1254
1255         /* Clean up data we have read: This will do ACK frames. */
1256         if (copied > 0)
1257                 tcp_cleanup_rbuf(sk, copied);
1258         return copied;
1259 }
1260
1261 /*
1262  *      This routine copies from a sock struct into the user buffer.
1263  *
1264  *      Technical note: in 2.3 we work on _locked_ socket, so that
1265  *      tricks with *seq access order and skb->users are not required.
1266  *      Probably, code can be easily improved even more.
1267  */
1268
1269 int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
1270                 size_t len, int nonblock, int flags, int *addr_len)
1271 {
1272         struct tcp_sock *tp = tcp_sk(sk);
1273         int copied = 0;
1274         u32 peek_seq;
1275         u32 *seq;
1276         unsigned long used;
1277         int err;
1278         int target;             /* Read at least this many bytes */
1279         long timeo;
1280         struct task_struct *user_recv = NULL;
1281         int copied_early = 0;
1282         struct sk_buff *skb;
1283
1284         lock_sock(sk);
1285
1286         TCP_CHECK_TIMER(sk);
1287
1288         err = -ENOTCONN;
1289         if (sk->sk_state == TCP_LISTEN)
1290                 goto out;
1291
1292         timeo = sock_rcvtimeo(sk, nonblock);
1293
1294         /* Urgent data needs to be handled specially. */
1295         if (flags & MSG_OOB)
1296                 goto recv_urg;
1297
1298         seq = &tp->copied_seq;
1299         if (flags & MSG_PEEK) {
1300                 peek_seq = tp->copied_seq;
1301                 seq = &peek_seq;
1302         }
1303
1304         target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1305
1306 #ifdef CONFIG_NET_DMA
1307         tp->ucopy.dma_chan = NULL;
1308         preempt_disable();
1309         skb = skb_peek_tail(&sk->sk_receive_queue);
1310         {
1311                 int available = 0;
1312
1313                 if (skb)
1314                         available = TCP_SKB_CB(skb)->seq + skb->len - (*seq);
1315                 if ((available < target) &&
1316                     (len > sysctl_tcp_dma_copybreak) && !(flags & MSG_PEEK) &&
1317                     !sysctl_tcp_low_latency &&
1318                     __get_cpu_var(softnet_data).net_dma) {
1319                         preempt_enable_no_resched();
1320                         tp->ucopy.pinned_list =
1321                                         dma_pin_iovec_pages(msg->msg_iov, len);
1322                 } else {
1323                         preempt_enable_no_resched();
1324                 }
1325         }
1326 #endif
1327
1328         do {
1329                 u32 offset;
1330
1331                 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
1332                 if (tp->urg_data && tp->urg_seq == *seq) {
1333                         if (copied)
1334                                 break;
1335                         if (signal_pending(current)) {
1336                                 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
1337                                 break;
1338                         }
1339                 }
1340
1341                 /* Next get a buffer. */
1342
1343                 skb = skb_peek(&sk->sk_receive_queue);
1344                 do {
1345                         if (!skb)
1346                                 break;
1347
1348                         /* Now that we have two receive queues this
1349                          * shouldn't happen.
1350                          */
1351                         if (before(*seq, TCP_SKB_CB(skb)->seq)) {
1352                                 printk(KERN_INFO "recvmsg bug: copied %X "
1353                                        "seq %X\n", *seq, TCP_SKB_CB(skb)->seq);
1354                                 break;
1355                         }
1356                         offset = *seq - TCP_SKB_CB(skb)->seq;
1357                         if (tcp_hdr(skb)->syn)
1358                                 offset--;
1359                         if (offset < skb->len)
1360                                 goto found_ok_skb;
1361                         if (tcp_hdr(skb)->fin)
1362                                 goto found_fin_ok;
1363                         BUG_TRAP(flags & MSG_PEEK);
1364                         skb = skb->next;
1365                 } while (skb != (struct sk_buff *)&sk->sk_receive_queue);
1366
1367                 /* Well, if we have backlog, try to process it now yet. */
1368
1369                 if (copied >= target && !sk->sk_backlog.tail)
1370                         break;
1371
1372                 if (copied) {
1373                         if (sk->sk_err ||
1374                             sk->sk_state == TCP_CLOSE ||
1375                             (sk->sk_shutdown & RCV_SHUTDOWN) ||
1376                             !timeo ||
1377                             signal_pending(current) ||
1378                             (flags & MSG_PEEK))
1379                                 break;
1380                 } else {
1381                         if (sock_flag(sk, SOCK_DONE))
1382                                 break;
1383
1384                         if (sk->sk_err) {
1385                                 copied = sock_error(sk);
1386                                 break;
1387                         }
1388
1389                         if (sk->sk_shutdown & RCV_SHUTDOWN)
1390                                 break;
1391
1392                         if (sk->sk_state == TCP_CLOSE) {
1393                                 if (!sock_flag(sk, SOCK_DONE)) {
1394                                         /* This occurs when user tries to read
1395                                          * from never connected socket.
1396                                          */
1397                                         copied = -ENOTCONN;
1398                                         break;
1399                                 }
1400                                 break;
1401                         }
1402
1403                         if (!timeo) {
1404                                 copied = -EAGAIN;
1405                                 break;
1406                         }
1407
1408                         if (signal_pending(current)) {
1409                                 copied = sock_intr_errno(timeo);
1410                                 break;
1411                         }
1412                 }
1413
1414                 tcp_cleanup_rbuf(sk, copied);
1415
1416                 if (!sysctl_tcp_low_latency && tp->ucopy.task == user_recv) {
1417                         /* Install new reader */
1418                         if (!user_recv && !(flags & (MSG_TRUNC | MSG_PEEK))) {
1419                                 user_recv = current;
1420                                 tp->ucopy.task = user_recv;
1421                                 tp->ucopy.iov = msg->msg_iov;
1422                         }
1423
1424                         tp->ucopy.len = len;
1425
1426                         BUG_TRAP(tp->copied_seq == tp->rcv_nxt ||
1427                                  (flags & (MSG_PEEK | MSG_TRUNC)));
1428
1429                         /* Ugly... If prequeue is not empty, we have to
1430                          * process it before releasing socket, otherwise
1431                          * order will be broken at second iteration.
1432                          * More elegant solution is required!!!
1433                          *
1434                          * Look: we have the following (pseudo)queues:
1435                          *
1436                          * 1. packets in flight
1437                          * 2. backlog
1438                          * 3. prequeue
1439                          * 4. receive_queue
1440                          *
1441                          * Each queue can be processed only if the next ones
1442                          * are empty. At this point we have empty receive_queue.
1443                          * But prequeue _can_ be not empty after 2nd iteration,
1444                          * when we jumped to start of loop because backlog
1445                          * processing added something to receive_queue.
1446                          * We cannot release_sock(), because backlog contains
1447                          * packets arrived _after_ prequeued ones.
1448                          *
1449                          * Shortly, algorithm is clear --- to process all
1450                          * the queues in order. We could make it more directly,
1451                          * requeueing packets from backlog to prequeue, if
1452                          * is not empty. It is more elegant, but eats cycles,
1453                          * unfortunately.
1454                          */
1455                         if (!skb_queue_empty(&tp->ucopy.prequeue))
1456                                 goto do_prequeue;
1457
1458                         /* __ Set realtime policy in scheduler __ */
1459                 }
1460
1461                 if (copied >= target) {
1462                         /* Do not sleep, just process backlog. */
1463                         release_sock(sk);
1464                         lock_sock(sk);
1465                 } else
1466                         sk_wait_data(sk, &timeo);
1467
1468 #ifdef CONFIG_NET_DMA
1469                 tp->ucopy.wakeup = 0;
1470 #endif
1471
1472                 if (user_recv) {
1473                         int chunk;
1474
1475                         /* __ Restore normal policy in scheduler __ */
1476
1477                         if ((chunk = len - tp->ucopy.len) != 0) {
1478                                 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMBACKLOG, chunk);
1479                                 len -= chunk;
1480                                 copied += chunk;
1481                         }
1482
1483                         if (tp->rcv_nxt == tp->copied_seq &&
1484                             !skb_queue_empty(&tp->ucopy.prequeue)) {
1485 do_prequeue:
1486                                 tcp_prequeue_process(sk);
1487
1488                                 if ((chunk = len - tp->ucopy.len) != 0) {
1489                                         NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1490                                         len -= chunk;
1491                                         copied += chunk;
1492                                 }
1493                         }
1494                 }
1495                 if ((flags & MSG_PEEK) && peek_seq != tp->copied_seq) {
1496                         if (net_ratelimit())
1497                                 printk(KERN_DEBUG "TCP(%s:%d): Application bug, race in MSG_PEEK.\n",
1498                                        current->comm, task_pid_nr(current));
1499                         peek_seq = tp->copied_seq;
1500                 }
1501                 continue;
1502
1503         found_ok_skb:
1504                 /* Ok so how much can we use? */
1505                 used = skb->len - offset;
1506                 if (len < used)
1507                         used = len;
1508
1509                 /* Do we have urgent data here? */
1510                 if (tp->urg_data) {
1511                         u32 urg_offset = tp->urg_seq - *seq;
1512                         if (urg_offset < used) {
1513                                 if (!urg_offset) {
1514                                         if (!sock_flag(sk, SOCK_URGINLINE)) {
1515                                                 ++*seq;
1516                                                 offset++;
1517                                                 used--;
1518                                                 if (!used)
1519                                                         goto skip_copy;
1520                                         }
1521                                 } else
1522                                         used = urg_offset;
1523                         }
1524                 }
1525
1526                 if (!(flags & MSG_TRUNC)) {
1527 #ifdef CONFIG_NET_DMA
1528                         if (!tp->ucopy.dma_chan && tp->ucopy.pinned_list)
1529                                 tp->ucopy.dma_chan = get_softnet_dma();
1530
1531                         if (tp->ucopy.dma_chan) {
1532                                 tp->ucopy.dma_cookie = dma_skb_copy_datagram_iovec(
1533                                         tp->ucopy.dma_chan, skb, offset,
1534                                         msg->msg_iov, used,
1535                                         tp->ucopy.pinned_list);
1536
1537                                 if (tp->ucopy.dma_cookie < 0) {
1538
1539                                         printk(KERN_ALERT "dma_cookie < 0\n");
1540
1541                                         /* Exception. Bailout! */
1542                                         if (!copied)
1543                                                 copied = -EFAULT;
1544                                         break;
1545                                 }
1546                                 if ((offset + used) == skb->len)
1547                                         copied_early = 1;
1548
1549                         } else
1550 #endif
1551                         {
1552                                 err = skb_copy_datagram_iovec(skb, offset,
1553                                                 msg->msg_iov, used);
1554                                 if (err) {
1555                                         /* Exception. Bailout! */
1556                                         if (!copied)
1557                                                 copied = -EFAULT;
1558                                         break;
1559                                 }
1560                         }
1561                 }
1562
1563                 *seq += used;
1564                 copied += used;
1565                 len -= used;
1566
1567                 tcp_rcv_space_adjust(sk);
1568
1569 skip_copy:
1570                 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
1571                         tp->urg_data = 0;
1572                         tcp_fast_path_check(sk);
1573                 }
1574                 if (used + offset < skb->len)
1575                         continue;
1576
1577                 if (tcp_hdr(skb)->fin)
1578                         goto found_fin_ok;
1579                 if (!(flags & MSG_PEEK)) {
1580                         sk_eat_skb(sk, skb, copied_early);
1581                         copied_early = 0;
1582                 }
1583                 continue;
1584
1585         found_fin_ok:
1586                 /* Process the FIN. */
1587                 ++*seq;
1588                 if (!(flags & MSG_PEEK)) {
1589                         sk_eat_skb(sk, skb, copied_early);
1590                         copied_early = 0;
1591                 }
1592                 break;
1593         } while (len > 0);
1594
1595         if (user_recv) {
1596                 if (!skb_queue_empty(&tp->ucopy.prequeue)) {
1597                         int chunk;
1598
1599                         tp->ucopy.len = copied > 0 ? len : 0;
1600
1601                         tcp_prequeue_process(sk);
1602
1603                         if (copied > 0 && (chunk = len - tp->ucopy.len) != 0) {
1604                                 NET_ADD_STATS_USER(LINUX_MIB_TCPDIRECTCOPYFROMPREQUEUE, chunk);
1605                                 len -= chunk;
1606                                 copied += chunk;
1607                         }
1608                 }
1609
1610                 tp->ucopy.task = NULL;
1611                 tp->ucopy.len = 0;
1612         }
1613
1614 #ifdef CONFIG_NET_DMA
1615         if (tp->ucopy.dma_chan) {
1616                 dma_cookie_t done, used;
1617
1618                 dma_async_memcpy_issue_pending(tp->ucopy.dma_chan);
1619
1620                 while (dma_async_memcpy_complete(tp->ucopy.dma_chan,
1621                                                  tp->ucopy.dma_cookie, &done,
1622                                                  &used) == DMA_IN_PROGRESS) {
1623                         /* do partial cleanup of sk_async_wait_queue */
1624                         while ((skb = skb_peek(&sk->sk_async_wait_queue)) &&
1625                                (dma_async_is_complete(skb->dma_cookie, done,
1626                                                       used) == DMA_SUCCESS)) {
1627                                 __skb_dequeue(&sk->sk_async_wait_queue);
1628                                 kfree_skb(skb);
1629                         }
1630                 }
1631
1632                 /* Safe to free early-copied skbs now */
1633                 __skb_queue_purge(&sk->sk_async_wait_queue);
1634                 dma_chan_put(tp->ucopy.dma_chan);
1635                 tp->ucopy.dma_chan = NULL;
1636         }
1637         if (tp->ucopy.pinned_list) {
1638                 dma_unpin_iovec_pages(tp->ucopy.pinned_list);
1639                 tp->ucopy.pinned_list = NULL;
1640         }
1641 #endif
1642
1643         /* According to UNIX98, msg_name/msg_namelen are ignored
1644          * on connected socket. I was just happy when found this 8) --ANK
1645          */
1646
1647         /* Clean up data we have read: This will do ACK frames. */
1648         tcp_cleanup_rbuf(sk, copied);
1649
1650         TCP_CHECK_TIMER(sk);
1651         release_sock(sk);
1652         return copied;
1653
1654 out:
1655         TCP_CHECK_TIMER(sk);
1656         release_sock(sk);
1657         return err;
1658
1659 recv_urg:
1660         err = tcp_recv_urg(sk, timeo, msg, len, flags, addr_len);
1661         goto out;
1662 }
1663
1664 void tcp_set_state(struct sock *sk, int state)
1665 {
1666         int oldstate = sk->sk_state;
1667
1668         switch (state) {
1669         case TCP_ESTABLISHED:
1670                 if (oldstate != TCP_ESTABLISHED)
1671                         TCP_INC_STATS(TCP_MIB_CURRESTAB);
1672                 break;
1673
1674         case TCP_CLOSE:
1675                 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
1676                         TCP_INC_STATS(TCP_MIB_ESTABRESETS);
1677
1678                 sk->sk_prot->unhash(sk);
1679                 if (inet_csk(sk)->icsk_bind_hash &&
1680                     !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
1681                         inet_put_port(sk);
1682                 /* fall through */
1683         default:
1684                 if (oldstate==TCP_ESTABLISHED)
1685                         TCP_DEC_STATS(TCP_MIB_CURRESTAB);
1686         }
1687
1688         /* Change state AFTER socket is unhashed to avoid closed
1689          * socket sitting in hash tables.
1690          */
1691         sk->sk_state = state;
1692
1693 #ifdef STATE_TRACE
1694         SOCK_DEBUG(sk, "TCP sk=%p, State %s -> %s\n",sk, statename[oldstate],statename[state]);
1695 #endif
1696 }
1697 EXPORT_SYMBOL_GPL(tcp_set_state);
1698
1699 /*
1700  *      State processing on a close. This implements the state shift for
1701  *      sending our FIN frame. Note that we only send a FIN for some
1702  *      states. A shutdown() may have already sent the FIN, or we may be
1703  *      closed.
1704  */
1705
1706 static const unsigned char new_state[16] = {
1707   /* current state:        new state:      action:      */
1708   /* (Invalid)          */ TCP_CLOSE,
1709   /* TCP_ESTABLISHED    */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1710   /* TCP_SYN_SENT       */ TCP_CLOSE,
1711   /* TCP_SYN_RECV       */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
1712   /* TCP_FIN_WAIT1      */ TCP_FIN_WAIT1,
1713   /* TCP_FIN_WAIT2      */ TCP_FIN_WAIT2,
1714   /* TCP_TIME_WAIT      */ TCP_CLOSE,
1715   /* TCP_CLOSE          */ TCP_CLOSE,
1716   /* TCP_CLOSE_WAIT     */ TCP_LAST_ACK  | TCP_ACTION_FIN,
1717   /* TCP_LAST_ACK       */ TCP_LAST_ACK,
1718   /* TCP_LISTEN         */ TCP_CLOSE,
1719   /* TCP_CLOSING        */ TCP_CLOSING,
1720 };
1721
1722 static int tcp_close_state(struct sock *sk)
1723 {
1724         int next = (int)new_state[sk->sk_state];
1725         int ns = next & TCP_STATE_MASK;
1726
1727         tcp_set_state(sk, ns);
1728
1729         return next & TCP_ACTION_FIN;
1730 }
1731
1732 /*
1733  *      Shutdown the sending side of a connection. Much like close except
1734  *      that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
1735  */
1736
1737 void tcp_shutdown(struct sock *sk, int how)
1738 {
1739         /*      We need to grab some memory, and put together a FIN,
1740          *      and then put it into the queue to be sent.
1741          *              Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
1742          */
1743         if (!(how & SEND_SHUTDOWN))
1744                 return;
1745
1746         /* If we've already sent a FIN, or it's a closed state, skip this. */
1747         if ((1 << sk->sk_state) &
1748             (TCPF_ESTABLISHED | TCPF_SYN_SENT |
1749              TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) {
1750                 /* Clear out any half completed packets.  FIN if needed. */
1751                 if (tcp_close_state(sk))
1752                         tcp_send_fin(sk);
1753         }
1754 }
1755
1756 void tcp_close(struct sock *sk, long timeout)
1757 {
1758         struct sk_buff *skb;
1759         int data_was_unread = 0;
1760         int state;
1761
1762         lock_sock(sk);
1763         sk->sk_shutdown = SHUTDOWN_MASK;
1764
1765         if (sk->sk_state == TCP_LISTEN) {
1766                 tcp_set_state(sk, TCP_CLOSE);
1767
1768                 /* Special case. */
1769                 inet_csk_listen_stop(sk);
1770
1771                 goto adjudge_to_death;
1772         }
1773
1774         /*  We need to flush the recv. buffs.  We do this only on the
1775          *  descriptor close, not protocol-sourced closes, because the
1776          *  reader process may not have drained the data yet!
1777          */
1778         while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
1779                 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq -
1780                           tcp_hdr(skb)->fin;
1781                 data_was_unread += len;
1782                 __kfree_skb(skb);
1783         }
1784
1785         sk_mem_reclaim(sk);
1786
1787         /* As outlined in RFC 2525, section 2.17, we send a RST here because
1788          * data was lost. To witness the awful effects of the old behavior of
1789          * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
1790          * GET in an FTP client, suspend the process, wait for the client to
1791          * advertise a zero window, then kill -9 the FTP client, wheee...
1792          * Note: timeout is always zero in such a case.
1793          */
1794         if (data_was_unread) {
1795                 /* Unread data was tossed, zap the connection. */
1796                 NET_INC_STATS_USER(LINUX_MIB_TCPABORTONCLOSE);
1797                 tcp_set_state(sk, TCP_CLOSE);
1798                 tcp_send_active_reset(sk, GFP_KERNEL);
1799         } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
1800                 /* Check zero linger _after_ checking for unread data. */
1801                 sk->sk_prot->disconnect(sk, 0);
1802                 NET_INC_STATS_USER(LINUX_MIB_TCPABORTONDATA);
1803         } else if (tcp_close_state(sk)) {
1804                 /* We FIN if the application ate all the data before
1805                  * zapping the connection.
1806                  */
1807
1808                 /* RED-PEN. Formally speaking, we have broken TCP state
1809                  * machine. State transitions:
1810                  *
1811                  * TCP_ESTABLISHED -> TCP_FIN_WAIT1
1812                  * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible)
1813                  * TCP_CLOSE_WAIT -> TCP_LAST_ACK
1814                  *
1815                  * are legal only when FIN has been sent (i.e. in window),
1816                  * rather than queued out of window. Purists blame.
1817                  *
1818                  * F.e. "RFC state" is ESTABLISHED,
1819                  * if Linux state is FIN-WAIT-1, but FIN is still not sent.
1820                  *
1821                  * The visible declinations are that sometimes
1822                  * we enter time-wait state, when it is not required really
1823                  * (harmless), do not send active resets, when they are
1824                  * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
1825                  * they look as CLOSING or LAST_ACK for Linux)
1826                  * Probably, I missed some more holelets.
1827                  *                                              --ANK
1828                  */
1829                 tcp_send_fin(sk);
1830         }
1831
1832         sk_stream_wait_close(sk, timeout);
1833
1834 adjudge_to_death:
1835         state = sk->sk_state;
1836         sock_hold(sk);
1837         sock_orphan(sk);
1838         atomic_inc(sk->sk_prot->orphan_count);
1839
1840         /* It is the last release_sock in its life. It will remove backlog. */
1841         release_sock(sk);
1842
1843
1844         /* Now socket is owned by kernel and we acquire BH lock
1845            to finish close. No need to check for user refs.
1846          */
1847         local_bh_disable();
1848         bh_lock_sock(sk);
1849         BUG_TRAP(!sock_owned_by_user(sk));
1850
1851         /* Have we already been destroyed by a softirq or backlog? */
1852         if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
1853                 goto out;
1854
1855         /*      This is a (useful) BSD violating of the RFC. There is a
1856          *      problem with TCP as specified in that the other end could
1857          *      keep a socket open forever with no application left this end.
1858          *      We use a 3 minute timeout (about the same as BSD) then kill
1859          *      our end. If they send after that then tough - BUT: long enough
1860          *      that we won't make the old 4*rto = almost no time - whoops
1861          *      reset mistake.
1862          *
1863          *      Nope, it was not mistake. It is really desired behaviour
1864          *      f.e. on http servers, when such sockets are useless, but
1865          *      consume significant resources. Let's do it with special
1866          *      linger2 option.                                 --ANK
1867          */
1868
1869         if (sk->sk_state == TCP_FIN_WAIT2) {
1870                 struct tcp_sock *tp = tcp_sk(sk);
1871                 if (tp->linger2 < 0) {
1872                         tcp_set_state(sk, TCP_CLOSE);
1873                         tcp_send_active_reset(sk, GFP_ATOMIC);
1874                         NET_INC_STATS_BH(LINUX_MIB_TCPABORTONLINGER);
1875                 } else {
1876                         const int tmo = tcp_fin_time(sk);
1877
1878                         if (tmo > TCP_TIMEWAIT_LEN) {
1879                                 inet_csk_reset_keepalive_timer(sk,
1880                                                 tmo - TCP_TIMEWAIT_LEN);
1881                         } else {
1882                                 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
1883                                 goto out;
1884                         }
1885                 }
1886         }
1887         if (sk->sk_state != TCP_CLOSE) {
1888                 sk_mem_reclaim(sk);
1889                 if (tcp_too_many_orphans(sk,
1890                                 atomic_read(sk->sk_prot->orphan_count))) {
1891                         if (net_ratelimit())
1892                                 printk(KERN_INFO "TCP: too many of orphaned "
1893                                        "sockets\n");
1894                         tcp_set_state(sk, TCP_CLOSE);
1895                         tcp_send_active_reset(sk, GFP_ATOMIC);
1896                         NET_INC_STATS_BH(LINUX_MIB_TCPABORTONMEMORY);
1897                 }
1898         }
1899
1900         if (sk->sk_state == TCP_CLOSE)
1901                 inet_csk_destroy_sock(sk);
1902         /* Otherwise, socket is reprieved until protocol close. */
1903
1904 out:
1905         bh_unlock_sock(sk);
1906         local_bh_enable();
1907         sock_put(sk);
1908 }
1909
1910 /* These states need RST on ABORT according to RFC793 */
1911
1912 static inline int tcp_need_reset(int state)
1913 {
1914         return (1 << state) &
1915                (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
1916                 TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
1917 }
1918
1919 int tcp_disconnect(struct sock *sk, int flags)
1920 {
1921         struct inet_sock *inet = inet_sk(sk);
1922         struct inet_connection_sock *icsk = inet_csk(sk);
1923         struct tcp_sock *tp = tcp_sk(sk);
1924         int err = 0;
1925         int old_state = sk->sk_state;
1926
1927         if (old_state != TCP_CLOSE)
1928                 tcp_set_state(sk, TCP_CLOSE);
1929
1930         /* ABORT function of RFC793 */
1931         if (old_state == TCP_LISTEN) {
1932                 inet_csk_listen_stop(sk);
1933         } else if (tcp_need_reset(old_state) ||
1934                    (tp->snd_nxt != tp->write_seq &&
1935                     (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
1936                 /* The last check adjusts for discrepancy of Linux wrt. RFC
1937                  * states
1938                  */
1939                 tcp_send_active_reset(sk, gfp_any());
1940                 sk->sk_err = ECONNRESET;
1941         } else if (old_state == TCP_SYN_SENT)
1942                 sk->sk_err = ECONNRESET;
1943
1944         tcp_clear_xmit_timers(sk);
1945         __skb_queue_purge(&sk->sk_receive_queue);
1946         tcp_write_queue_purge(sk);
1947         __skb_queue_purge(&tp->out_of_order_queue);
1948 #ifdef CONFIG_NET_DMA
1949         __skb_queue_purge(&sk->sk_async_wait_queue);
1950 #endif
1951
1952         inet->dport = 0;
1953
1954         if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
1955                 inet_reset_saddr(sk);
1956
1957         sk->sk_shutdown = 0;
1958         sock_reset_flag(sk, SOCK_DONE);
1959         tp->srtt = 0;
1960         if ((tp->write_seq += tp->max_window + 2) == 0)
1961                 tp->write_seq = 1;
1962         icsk->icsk_backoff = 0;
1963         tp->snd_cwnd = 2;
1964         icsk->icsk_probes_out = 0;
1965         tp->packets_out = 0;
1966         tp->snd_ssthresh = 0x7fffffff;
1967         tp->snd_cwnd_cnt = 0;
1968         tp->bytes_acked = 0;
1969         tcp_set_ca_state(sk, TCP_CA_Open);
1970         tcp_clear_retrans(tp);
1971         inet_csk_delack_init(sk);
1972         tcp_init_send_head(sk);
1973         memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
1974         __sk_dst_reset(sk);
1975
1976         BUG_TRAP(!inet->num || icsk->icsk_bind_hash);
1977
1978         sk->sk_error_report(sk);
1979         return err;
1980 }
1981
1982 /*
1983  *      Socket option code for TCP.
1984  */
1985 static int do_tcp_setsockopt(struct sock *sk, int level,
1986                 int optname, char __user *optval, int optlen)
1987 {
1988         struct tcp_sock *tp = tcp_sk(sk);
1989         struct inet_connection_sock *icsk = inet_csk(sk);
1990         int val;
1991         int err = 0;
1992
1993         /* This is a string value all the others are int's */
1994         if (optname == TCP_CONGESTION) {
1995                 char name[TCP_CA_NAME_MAX];
1996
1997                 if (optlen < 1)
1998                         return -EINVAL;
1999
2000                 val = strncpy_from_user(name, optval,
2001                                         min(TCP_CA_NAME_MAX-1, optlen));
2002                 if (val < 0)
2003                         return -EFAULT;
2004                 name[val] = 0;
2005
2006                 lock_sock(sk);
2007                 err = tcp_set_congestion_control(sk, name);
2008                 release_sock(sk);
2009                 return err;
2010         }
2011
2012         if (optlen < sizeof(int))
2013                 return -EINVAL;
2014
2015         if (get_user(val, (int __user *)optval))
2016                 return -EFAULT;
2017
2018         lock_sock(sk);
2019
2020         switch (optname) {
2021         case TCP_MAXSEG:
2022                 /* Values greater than interface MTU won't take effect. However
2023                  * at the point when this call is done we typically don't yet
2024                  * know which interface is going to be used */
2025                 if (val < 8 || val > MAX_TCP_WINDOW) {
2026                         err = -EINVAL;
2027                         break;
2028                 }
2029                 tp->rx_opt.user_mss = val;
2030                 break;
2031
2032         case TCP_NODELAY:
2033                 if (val) {
2034                         /* TCP_NODELAY is weaker than TCP_CORK, so that
2035                          * this option on corked socket is remembered, but
2036                          * it is not activated until cork is cleared.
2037                          *
2038                          * However, when TCP_NODELAY is set we make
2039                          * an explicit push, which overrides even TCP_CORK
2040                          * for currently queued segments.
2041                          */
2042                         tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
2043                         tcp_push_pending_frames(sk);
2044                 } else {
2045                         tp->nonagle &= ~TCP_NAGLE_OFF;
2046                 }
2047                 break;
2048
2049         case TCP_CORK:
2050                 /* When set indicates to always queue non-full frames.
2051                  * Later the user clears this option and we transmit
2052                  * any pending partial frames in the queue.  This is
2053                  * meant to be used alongside sendfile() to get properly
2054                  * filled frames when the user (for example) must write
2055                  * out headers with a write() call first and then use
2056                  * sendfile to send out the data parts.
2057                  *
2058                  * TCP_CORK can be set together with TCP_NODELAY and it is
2059                  * stronger than TCP_NODELAY.
2060                  */
2061                 if (val) {
2062                         tp->nonagle |= TCP_NAGLE_CORK;
2063                 } else {
2064                         tp->nonagle &= ~TCP_NAGLE_CORK;
2065                         if (tp->nonagle&TCP_NAGLE_OFF)
2066                                 tp->nonagle |= TCP_NAGLE_PUSH;
2067                         tcp_push_pending_frames(sk);
2068                 }
2069                 break;
2070
2071         case TCP_KEEPIDLE:
2072                 if (val < 1 || val > MAX_TCP_KEEPIDLE)
2073                         err = -EINVAL;
2074                 else {
2075                         tp->keepalive_time = val * HZ;
2076                         if (sock_flag(sk, SOCK_KEEPOPEN) &&
2077                             !((1 << sk->sk_state) &
2078                               (TCPF_CLOSE | TCPF_LISTEN))) {
2079                                 __u32 elapsed = tcp_time_stamp - tp->rcv_tstamp;
2080                                 if (tp->keepalive_time > elapsed)
2081                                         elapsed = tp->keepalive_time - elapsed;
2082                                 else
2083                                         elapsed = 0;
2084                                 inet_csk_reset_keepalive_timer(sk, elapsed);
2085                         }
2086                 }
2087                 break;
2088         case TCP_KEEPINTVL:
2089                 if (val < 1 || val > MAX_TCP_KEEPINTVL)
2090                         err = -EINVAL;
2091                 else
2092                         tp->keepalive_intvl = val * HZ;
2093                 break;
2094         case TCP_KEEPCNT:
2095                 if (val < 1 || val > MAX_TCP_KEEPCNT)
2096                         err = -EINVAL;
2097                 else
2098                         tp->keepalive_probes = val;
2099                 break;
2100         case TCP_SYNCNT:
2101                 if (val < 1 || val > MAX_TCP_SYNCNT)
2102                         err = -EINVAL;
2103                 else
2104                         icsk->icsk_syn_retries = val;
2105                 break;
2106
2107         case TCP_LINGER2:
2108                 if (val < 0)
2109                         tp->linger2 = -1;
2110                 else if (val > sysctl_tcp_fin_timeout / HZ)
2111                         tp->linger2 = 0;
2112                 else
2113                         tp->linger2 = val * HZ;
2114                 break;
2115
2116         case TCP_DEFER_ACCEPT:
2117                 icsk->icsk_accept_queue.rskq_defer_accept = 0;
2118                 if (val > 0) {
2119                         /* Translate value in seconds to number of
2120                          * retransmits */
2121                         while (icsk->icsk_accept_queue.rskq_defer_accept < 32 &&
2122                                val > ((TCP_TIMEOUT_INIT / HZ) <<
2123                                        icsk->icsk_accept_queue.rskq_defer_accept))
2124                                 icsk->icsk_accept_queue.rskq_defer_accept++;
2125                         icsk->icsk_accept_queue.rskq_defer_accept++;
2126                 }
2127                 break;
2128
2129         case TCP_WINDOW_CLAMP:
2130                 if (!val) {
2131                         if (sk->sk_state != TCP_CLOSE) {
2132                                 err = -EINVAL;
2133                                 break;
2134                         }
2135                         tp->window_clamp = 0;
2136                 } else
2137                         tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
2138                                                 SOCK_MIN_RCVBUF / 2 : val;
2139                 break;
2140
2141         case TCP_QUICKACK:
2142                 if (!val) {
2143                         icsk->icsk_ack.pingpong = 1;
2144                 } else {
2145                         icsk->icsk_ack.pingpong = 0;
2146                         if ((1 << sk->sk_state) &
2147                             (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
2148                             inet_csk_ack_scheduled(sk)) {
2149                                 icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
2150                                 tcp_cleanup_rbuf(sk, 1);
2151                                 if (!(val & 1))
2152                                         icsk->icsk_ack.pingpong = 1;
2153                         }
2154                 }
2155                 break;
2156
2157 #ifdef CONFIG_TCP_MD5SIG
2158         case TCP_MD5SIG:
2159                 /* Read the IP->Key mappings from userspace */
2160                 err = tp->af_specific->md5_parse(sk, optval, optlen);
2161                 break;
2162 #endif
2163
2164         default:
2165                 err = -ENOPROTOOPT;
2166                 break;
2167         }
2168
2169         release_sock(sk);
2170         return err;
2171 }
2172
2173 int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval,
2174                    int optlen)
2175 {
2176         struct inet_connection_sock *icsk = inet_csk(sk);
2177
2178         if (level != SOL_TCP)
2179                 return icsk->icsk_af_ops->setsockopt(sk, level, optname,
2180                                                      optval, optlen);
2181         return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2182 }
2183
2184 #ifdef CONFIG_COMPAT
2185 int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
2186                           char __user *optval, int optlen)
2187 {
2188         if (level != SOL_TCP)
2189                 return inet_csk_compat_setsockopt(sk, level, optname,
2190                                                   optval, optlen);
2191         return do_tcp_setsockopt(sk, level, optname, optval, optlen);
2192 }
2193
2194 EXPORT_SYMBOL(compat_tcp_setsockopt);
2195 #endif
2196
2197 /* Return information about state of tcp endpoint in API format. */
2198 void tcp_get_info(struct sock *sk, struct tcp_info *info)
2199 {
2200         struct tcp_sock *tp = tcp_sk(sk);
2201         const struct inet_connection_sock *icsk = inet_csk(sk);
2202         u32 now = tcp_time_stamp;
2203
2204         memset(info, 0, sizeof(*info));
2205
2206         info->tcpi_state = sk->sk_state;
2207         info->tcpi_ca_state = icsk->icsk_ca_state;
2208         info->tcpi_retransmits = icsk->icsk_retransmits;
2209         info->tcpi_probes = icsk->icsk_probes_out;
2210         info->tcpi_backoff = icsk->icsk_backoff;
2211
2212         if (tp->rx_opt.tstamp_ok)
2213                 info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
2214         if (tcp_is_sack(tp))
2215                 info->tcpi_options |= TCPI_OPT_SACK;
2216         if (tp->rx_opt.wscale_ok) {
2217                 info->tcpi_options |= TCPI_OPT_WSCALE;
2218                 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
2219                 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
2220         }
2221
2222         if (tp->ecn_flags&TCP_ECN_OK)
2223                 info->tcpi_options |= TCPI_OPT_ECN;
2224
2225         info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
2226         info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato);
2227         info->tcpi_snd_mss = tp->mss_cache;
2228         info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
2229
2230         if (sk->sk_state == TCP_LISTEN) {
2231                 info->tcpi_unacked = sk->sk_ack_backlog;
2232                 info->tcpi_sacked = sk->sk_max_ack_backlog;
2233         } else {
2234                 info->tcpi_unacked = tp->packets_out;
2235                 info->tcpi_sacked = tp->sacked_out;
2236         }
2237         info->tcpi_lost = tp->lost_out;
2238         info->tcpi_retrans = tp->retrans_out;
2239         info->tcpi_fackets = tp->fackets_out;
2240
2241         info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
2242         info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
2243         info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
2244
2245         info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
2246         info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
2247         info->tcpi_rtt = jiffies_to_usecs(tp->srtt)>>3;
2248         info->tcpi_rttvar = jiffies_to_usecs(tp->mdev)>>2;
2249         info->tcpi_snd_ssthresh = tp->snd_ssthresh;
2250         info->tcpi_snd_cwnd = tp->snd_cwnd;
2251         info->tcpi_advmss = tp->advmss;
2252         info->tcpi_reordering = tp->reordering;
2253
2254         info->tcpi_rcv_rtt = jiffies_to_usecs(tp->rcv_rtt_est.rtt)>>3;
2255         info->tcpi_rcv_space = tp->rcvq_space.space;
2256
2257         info->tcpi_total_retrans = tp->total_retrans;
2258 }
2259
2260 EXPORT_SYMBOL_GPL(tcp_get_info);
2261
2262 static int do_tcp_getsockopt(struct sock *sk, int level,
2263                 int optname, char __user *optval, int __user *optlen)
2264 {
2265         struct inet_connection_sock *icsk = inet_csk(sk);
2266         struct tcp_sock *tp = tcp_sk(sk);
2267         int val, len;
2268
2269         if (get_user(len, optlen))
2270                 return -EFAULT;
2271
2272         len = min_t(unsigned int, len, sizeof(int));
2273
2274         if (len < 0)
2275                 return -EINVAL;
2276
2277         switch (optname) {
2278         case TCP_MAXSEG:
2279                 val = tp->mss_cache;
2280                 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
2281                         val = tp->rx_opt.user_mss;
2282                 break;
2283         case TCP_NODELAY:
2284                 val = !!(tp->nonagle&TCP_NAGLE_OFF);
2285                 break;
2286         case TCP_CORK:
2287                 val = !!(tp->nonagle&TCP_NAGLE_CORK);
2288                 break;
2289         case TCP_KEEPIDLE:
2290                 val = (tp->keepalive_time ? : sysctl_tcp_keepalive_time) / HZ;
2291                 break;
2292         case TCP_KEEPINTVL:
2293                 val = (tp->keepalive_intvl ? : sysctl_tcp_keepalive_intvl) / HZ;
2294                 break;
2295         case TCP_KEEPCNT:
2296                 val = tp->keepalive_probes ? : sysctl_tcp_keepalive_probes;
2297                 break;
2298         case TCP_SYNCNT:
2299                 val = icsk->icsk_syn_retries ? : sysctl_tcp_syn_retries;
2300                 break;
2301         case TCP_LINGER2:
2302                 val = tp->linger2;
2303                 if (val >= 0)
2304                         val = (val ? : sysctl_tcp_fin_timeout) / HZ;
2305                 break;
2306         case TCP_DEFER_ACCEPT:
2307                 val = !icsk->icsk_accept_queue.rskq_defer_accept ? 0 :
2308                         ((TCP_TIMEOUT_INIT / HZ) << (icsk->icsk_accept_queue.rskq_defer_accept - 1));
2309                 break;
2310         case TCP_WINDOW_CLAMP:
2311                 val = tp->window_clamp;
2312                 break;
2313         case TCP_INFO: {
2314                 struct tcp_info info;
2315
2316                 if (get_user(len, optlen))
2317                         return -EFAULT;
2318
2319                 tcp_get_info(sk, &info);
2320
2321                 len = min_t(unsigned int, len, sizeof(info));
2322                 if (put_user(len, optlen))
2323                         return -EFAULT;
2324                 if (copy_to_user(optval, &info, len))
2325                         return -EFAULT;
2326                 return 0;
2327         }
2328         case TCP_QUICKACK:
2329                 val = !icsk->icsk_ack.pingpong;
2330                 break;
2331
2332         case TCP_CONGESTION:
2333                 if (get_user(len, optlen))
2334                         return -EFAULT;
2335                 len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
2336                 if (put_user(len, optlen))
2337                         return -EFAULT;
2338                 if (copy_to_user(optval, icsk->icsk_ca_ops->name, len))
2339                         return -EFAULT;
2340                 return 0;
2341         default:
2342                 return -ENOPROTOOPT;
2343         }
2344
2345         if (put_user(len, optlen))
2346                 return -EFAULT;
2347         if (copy_to_user(optval, &val, len))
2348                 return -EFAULT;
2349         return 0;
2350 }
2351
2352 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
2353                    int __user *optlen)
2354 {
2355         struct inet_connection_sock *icsk = inet_csk(sk);
2356
2357         if (level != SOL_TCP)
2358                 return icsk->icsk_af_ops->getsockopt(sk, level, optname,
2359                                                      optval, optlen);
2360         return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2361 }
2362
2363 #ifdef CONFIG_COMPAT
2364 int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
2365                           char __user *optval, int __user *optlen)
2366 {
2367         if (level != SOL_TCP)
2368                 return inet_csk_compat_getsockopt(sk, level, optname,
2369                                                   optval, optlen);
2370         return do_tcp_getsockopt(sk, level, optname, optval, optlen);
2371 }
2372
2373 EXPORT_SYMBOL(compat_tcp_getsockopt);
2374 #endif
2375
2376 struct sk_buff *tcp_tso_segment(struct sk_buff *skb, int features)
2377 {
2378         struct sk_buff *segs = ERR_PTR(-EINVAL);
2379         struct tcphdr *th;
2380         unsigned thlen;
2381         unsigned int seq;
2382         __be32 delta;
2383         unsigned int oldlen;
2384         unsigned int len;
2385
2386         if (!pskb_may_pull(skb, sizeof(*th)))
2387                 goto out;
2388
2389         th = tcp_hdr(skb);
2390         thlen = th->doff * 4;
2391         if (thlen < sizeof(*th))
2392                 goto out;
2393
2394         if (!pskb_may_pull(skb, thlen))
2395                 goto out;
2396
2397         oldlen = (u16)~skb->len;
2398         __skb_pull(skb, thlen);
2399
2400         if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) {
2401                 /* Packet is from an untrusted source, reset gso_segs. */
2402                 int type = skb_shinfo(skb)->gso_type;
2403                 int mss;
2404
2405                 if (unlikely(type &
2406                              ~(SKB_GSO_TCPV4 |
2407                                SKB_GSO_DODGY |
2408                                SKB_GSO_TCP_ECN |
2409                                SKB_GSO_TCPV6 |
2410                                0) ||
2411                              !(type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))))
2412                         goto out;
2413
2414                 mss = skb_shinfo(skb)->gso_size;
2415                 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss);
2416
2417                 segs = NULL;
2418                 goto out;
2419         }
2420
2421         segs = skb_segment(skb, features);
2422         if (IS_ERR(segs))
2423                 goto out;
2424
2425         len = skb_shinfo(skb)->gso_size;
2426         delta = htonl(oldlen + (thlen + len));
2427
2428         skb = segs;
2429         th = tcp_hdr(skb);
2430         seq = ntohl(th->seq);
2431
2432         do {
2433                 th->fin = th->psh = 0;
2434
2435                 th->check = ~csum_fold((__force __wsum)((__force u32)th->check +
2436                                        (__force u32)delta));
2437                 if (skb->ip_summed != CHECKSUM_PARTIAL)
2438                         th->check =
2439                              csum_fold(csum_partial(skb_transport_header(skb),
2440                                                     thlen, skb->csum));
2441
2442                 seq += len;
2443                 skb = skb->next;
2444                 th = tcp_hdr(skb);
2445
2446                 th->seq = htonl(seq);
2447                 th->cwr = 0;
2448         } while (skb->next);
2449
2450         delta = htonl(oldlen + (skb->tail - skb->transport_header) +
2451                       skb->data_len);
2452         th->check = ~csum_fold((__force __wsum)((__force u32)th->check +
2453                                 (__force u32)delta));
2454         if (skb->ip_summed != CHECKSUM_PARTIAL)
2455                 th->check = csum_fold(csum_partial(skb_transport_header(skb),
2456                                                    thlen, skb->csum));
2457
2458 out:
2459         return segs;
2460 }
2461 EXPORT_SYMBOL(tcp_tso_segment);
2462
2463 #ifdef CONFIG_TCP_MD5SIG
2464 static unsigned long tcp_md5sig_users;
2465 static struct tcp_md5sig_pool **tcp_md5sig_pool;
2466 static DEFINE_SPINLOCK(tcp_md5sig_pool_lock);
2467
2468 int tcp_calc_md5_hash(char *md5_hash, struct tcp_md5sig_key *key,
2469                       int bplen,
2470                       struct tcphdr *th, unsigned int tcplen,
2471                       struct tcp_md5sig_pool *hp)
2472 {
2473         struct scatterlist sg[4];
2474         __u16 data_len;
2475         int block = 0;
2476         __sum16 cksum;
2477         struct hash_desc *desc = &hp->md5_desc;
2478         int err;
2479         unsigned int nbytes = 0;
2480
2481         sg_init_table(sg, 4);
2482
2483         /* 1. The TCP pseudo-header */
2484         sg_set_buf(&sg[block++], &hp->md5_blk, bplen);
2485         nbytes += bplen;
2486
2487         /* 2. The TCP header, excluding options, and assuming a
2488          * checksum of zero
2489          */
2490         cksum = th->check;
2491         th->check = 0;
2492         sg_set_buf(&sg[block++], th, sizeof(*th));
2493         nbytes += sizeof(*th);
2494
2495         /* 3. The TCP segment data (if any) */
2496         data_len = tcplen - (th->doff << 2);
2497         if (data_len > 0) {
2498                 u8 *data = (u8 *)th + (th->doff << 2);
2499                 sg_set_buf(&sg[block++], data, data_len);
2500                 nbytes += data_len;
2501         }
2502
2503         /* 4. an independently-specified key or password, known to both
2504          * TCPs and presumably connection-specific
2505          */
2506         sg_set_buf(&sg[block++], key->key, key->keylen);
2507         nbytes += key->keylen;
2508
2509         sg_mark_end(&sg[block - 1]);
2510
2511         /* Now store the hash into the packet */
2512         err = crypto_hash_init(desc);
2513         if (err) {
2514                 if (net_ratelimit())
2515                         printk(KERN_WARNING "%s(): hash_init failed\n", __func__);
2516                 return -1;
2517         }
2518         err = crypto_hash_update(desc, sg, nbytes);
2519         if (err) {
2520                 if (net_ratelimit())
2521                         printk(KERN_WARNING "%s(): hash_update failed\n", __func__);
2522                 return -1;
2523         }
2524         err = crypto_hash_final(desc, md5_hash);
2525         if (err) {
2526                 if (net_ratelimit())
2527                         printk(KERN_WARNING "%s(): hash_final failed\n", __func__);
2528                 return -1;
2529         }
2530
2531         /* Reset header */
2532         th->check = cksum;
2533
2534         return 0;
2535 }
2536 EXPORT_SYMBOL(tcp_calc_md5_hash);
2537
2538 static void __tcp_free_md5sig_pool(struct tcp_md5sig_pool **pool)
2539 {
2540         int cpu;
2541         for_each_possible_cpu(cpu) {
2542                 struct tcp_md5sig_pool *p = *per_cpu_ptr(pool, cpu);
2543                 if (p) {
2544                         if (p->md5_desc.tfm)
2545                                 crypto_free_hash(p->md5_desc.tfm);
2546                         kfree(p);
2547                         p = NULL;
2548                 }
2549         }
2550         free_percpu(pool);
2551 }
2552
2553 void tcp_free_md5sig_pool(void)
2554 {
2555         struct tcp_md5sig_pool **pool = NULL;
2556
2557         spin_lock_bh(&tcp_md5sig_pool_lock);
2558         if (--tcp_md5sig_users == 0) {
2559                 pool = tcp_md5sig_pool;
2560                 tcp_md5sig_pool = NULL;
2561         }
2562         spin_unlock_bh(&tcp_md5sig_pool_lock);
2563         if (pool)
2564                 __tcp_free_md5sig_pool(pool);
2565 }
2566
2567 EXPORT_SYMBOL(tcp_free_md5sig_pool);
2568
2569 static struct tcp_md5sig_pool **__tcp_alloc_md5sig_pool(void)
2570 {
2571         int cpu;
2572         struct tcp_md5sig_pool **pool;
2573
2574         pool = alloc_percpu(struct tcp_md5sig_pool *);
2575         if (!pool)
2576                 return NULL;
2577
2578         for_each_possible_cpu(cpu) {
2579                 struct tcp_md5sig_pool *p;
2580                 struct crypto_hash *hash;
2581
2582                 p = kzalloc(sizeof(*p), GFP_KERNEL);
2583                 if (!p)
2584                         goto out_free;
2585                 *per_cpu_ptr(pool, cpu) = p;
2586
2587                 hash = crypto_alloc_hash("md5", 0, CRYPTO_ALG_ASYNC);
2588                 if (!hash || IS_ERR(hash))
2589                         goto out_free;
2590
2591                 p->md5_desc.tfm = hash;
2592         }
2593         return pool;
2594 out_free:
2595         __tcp_free_md5sig_pool(pool);
2596         return NULL;
2597 }
2598
2599 struct tcp_md5sig_pool **tcp_alloc_md5sig_pool(void)
2600 {
2601         struct tcp_md5sig_pool **pool;
2602         int alloc = 0;
2603
2604 retry:
2605         spin_lock_bh(&tcp_md5sig_pool_lock);
2606         pool = tcp_md5sig_pool;
2607         if (tcp_md5sig_users++ == 0) {
2608                 alloc = 1;
2609                 spin_unlock_bh(&tcp_md5sig_pool_lock);
2610         } else if (!pool) {
2611                 tcp_md5sig_users--;
2612                 spin_unlock_bh(&tcp_md5sig_pool_lock);
2613                 cpu_relax();
2614                 goto retry;
2615         } else
2616                 spin_unlock_bh(&tcp_md5sig_pool_lock);
2617
2618         if (alloc) {
2619                 /* we cannot hold spinlock here because this may sleep. */
2620                 struct tcp_md5sig_pool **p = __tcp_alloc_md5sig_pool();
2621                 spin_lock_bh(&tcp_md5sig_pool_lock);
2622                 if (!p) {
2623                         tcp_md5sig_users--;
2624                         spin_unlock_bh(&tcp_md5sig_pool_lock);
2625                         return NULL;
2626                 }
2627                 pool = tcp_md5sig_pool;
2628                 if (pool) {
2629                         /* oops, it has already been assigned. */
2630                         spin_unlock_bh(&tcp_md5sig_pool_lock);
2631                         __tcp_free_md5sig_pool(p);
2632                 } else {
2633                         tcp_md5sig_pool = pool = p;
2634                         spin_unlock_bh(&tcp_md5sig_pool_lock);
2635                 }
2636         }
2637         return pool;
2638 }
2639
2640 EXPORT_SYMBOL(tcp_alloc_md5sig_pool);
2641
2642 struct tcp_md5sig_pool *__tcp_get_md5sig_pool(int cpu)
2643 {
2644         struct tcp_md5sig_pool **p;
2645         spin_lock_bh(&tcp_md5sig_pool_lock);
2646         p = tcp_md5sig_pool;
2647         if (p)
2648                 tcp_md5sig_users++;
2649         spin_unlock_bh(&tcp_md5sig_pool_lock);
2650         return (p ? *per_cpu_ptr(p, cpu) : NULL);
2651 }
2652
2653 EXPORT_SYMBOL(__tcp_get_md5sig_pool);
2654
2655 void __tcp_put_md5sig_pool(void)
2656 {
2657         tcp_free_md5sig_pool();
2658 }
2659
2660 EXPORT_SYMBOL(__tcp_put_md5sig_pool);
2661 #endif
2662
2663 void tcp_done(struct sock *sk)
2664 {
2665         if(sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
2666                 TCP_INC_STATS_BH(TCP_MIB_ATTEMPTFAILS);
2667
2668         tcp_set_state(sk, TCP_CLOSE);
2669         tcp_clear_xmit_timers(sk);
2670
2671         sk->sk_shutdown = SHUTDOWN_MASK;
2672
2673         if (!sock_flag(sk, SOCK_DEAD))
2674                 sk->sk_state_change(sk);
2675         else
2676                 inet_csk_destroy_sock(sk);
2677 }
2678 EXPORT_SYMBOL_GPL(tcp_done);
2679
2680 extern struct tcp_congestion_ops tcp_reno;
2681
2682 static __initdata unsigned long thash_entries;
2683 static int __init set_thash_entries(char *str)
2684 {
2685         if (!str)
2686                 return 0;
2687         thash_entries = simple_strtoul(str, &str, 0);
2688         return 1;
2689 }
2690 __setup("thash_entries=", set_thash_entries);
2691
2692 void __init tcp_init(void)
2693 {
2694         struct sk_buff *skb = NULL;
2695         unsigned long nr_pages, limit;
2696         int order, i, max_share;
2697
2698         BUILD_BUG_ON(sizeof(struct tcp_skb_cb) > sizeof(skb->cb));
2699
2700         tcp_hashinfo.bind_bucket_cachep =
2701                 kmem_cache_create("tcp_bind_bucket",
2702                                   sizeof(struct inet_bind_bucket), 0,
2703                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2704
2705         /* Size and allocate the main established and bind bucket
2706          * hash tables.
2707          *
2708          * The methodology is similar to that of the buffer cache.
2709          */
2710         tcp_hashinfo.ehash =
2711                 alloc_large_system_hash("TCP established",
2712                                         sizeof(struct inet_ehash_bucket),
2713                                         thash_entries,
2714                                         (num_physpages >= 128 * 1024) ?
2715                                         13 : 15,
2716                                         0,
2717                                         &tcp_hashinfo.ehash_size,
2718                                         NULL,
2719                                         thash_entries ? 0 : 512 * 1024);
2720         tcp_hashinfo.ehash_size = 1 << tcp_hashinfo.ehash_size;
2721         for (i = 0; i < tcp_hashinfo.ehash_size; i++) {
2722                 INIT_HLIST_HEAD(&tcp_hashinfo.ehash[i].chain);
2723                 INIT_HLIST_HEAD(&tcp_hashinfo.ehash[i].twchain);
2724         }
2725         if (inet_ehash_locks_alloc(&tcp_hashinfo))
2726                 panic("TCP: failed to alloc ehash_locks");
2727         tcp_hashinfo.bhash =
2728                 alloc_large_system_hash("TCP bind",
2729                                         sizeof(struct inet_bind_hashbucket),
2730                                         tcp_hashinfo.ehash_size,
2731                                         (num_physpages >= 128 * 1024) ?
2732                                         13 : 15,
2733                                         0,
2734                                         &tcp_hashinfo.bhash_size,
2735                                         NULL,
2736                                         64 * 1024);
2737         tcp_hashinfo.bhash_size = 1 << tcp_hashinfo.bhash_size;
2738         for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
2739                 spin_lock_init(&tcp_hashinfo.bhash[i].lock);
2740                 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
2741         }
2742
2743         /* Try to be a bit smarter and adjust defaults depending
2744          * on available memory.
2745          */
2746         for (order = 0; ((1 << order) << PAGE_SHIFT) <
2747                         (tcp_hashinfo.bhash_size * sizeof(struct inet_bind_hashbucket));
2748                         order++)
2749                 ;
2750         if (order >= 4) {
2751                 tcp_death_row.sysctl_max_tw_buckets = 180000;
2752                 sysctl_tcp_max_orphans = 4096 << (order - 4);
2753                 sysctl_max_syn_backlog = 1024;
2754         } else if (order < 3) {
2755                 tcp_death_row.sysctl_max_tw_buckets >>= (3 - order);
2756                 sysctl_tcp_max_orphans >>= (3 - order);
2757                 sysctl_max_syn_backlog = 128;
2758         }
2759
2760         /* Set the pressure threshold to be a fraction of global memory that
2761          * is up to 1/2 at 256 MB, decreasing toward zero with the amount of
2762          * memory, with a floor of 128 pages.
2763          */
2764         nr_pages = totalram_pages - totalhigh_pages;
2765         limit = min(nr_pages, 1UL<<(28-PAGE_SHIFT)) >> (20-PAGE_SHIFT);
2766         limit = (limit * (nr_pages >> (20-PAGE_SHIFT))) >> (PAGE_SHIFT-11);
2767         limit = max(limit, 128UL);
2768         sysctl_tcp_mem[0] = limit / 4 * 3;
2769         sysctl_tcp_mem[1] = limit;
2770         sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2;
2771
2772         /* Set per-socket limits to no more than 1/128 the pressure threshold */
2773         limit = ((unsigned long)sysctl_tcp_mem[1]) << (PAGE_SHIFT - 7);
2774         max_share = min(4UL*1024*1024, limit);
2775
2776         sysctl_tcp_wmem[0] = SK_MEM_QUANTUM;
2777         sysctl_tcp_wmem[1] = 16*1024;
2778         sysctl_tcp_wmem[2] = max(64*1024, max_share);
2779
2780         sysctl_tcp_rmem[0] = SK_MEM_QUANTUM;
2781         sysctl_tcp_rmem[1] = 87380;
2782         sysctl_tcp_rmem[2] = max(87380, max_share);
2783
2784         printk(KERN_INFO "TCP: Hash tables configured "
2785                "(established %d bind %d)\n",
2786                tcp_hashinfo.ehash_size, tcp_hashinfo.bhash_size);
2787
2788         tcp_register_congestion_control(&tcp_reno);
2789 }
2790
2791 EXPORT_SYMBOL(tcp_close);
2792 EXPORT_SYMBOL(tcp_disconnect);
2793 EXPORT_SYMBOL(tcp_getsockopt);
2794 EXPORT_SYMBOL(tcp_ioctl);
2795 EXPORT_SYMBOL(tcp_poll);
2796 EXPORT_SYMBOL(tcp_read_sock);
2797 EXPORT_SYMBOL(tcp_recvmsg);
2798 EXPORT_SYMBOL(tcp_sendmsg);
2799 EXPORT_SYMBOL(tcp_splice_read);
2800 EXPORT_SYMBOL(tcp_sendpage);
2801 EXPORT_SYMBOL(tcp_setsockopt);
2802 EXPORT_SYMBOL(tcp_shutdown);
2803 EXPORT_SYMBOL(tcp_statistics);