source: src/linux/universal/linux-3.18/net/ipv4/tcp_output.c @ 31869

Last change on this file since 31869 was 31869, checked in by brainslayer, 6 weeks ago

update

File size: 97.5 KB
Line 
1/*
2 * INET         An implementation of the TCP/IP protocol suite for the LINUX
3 *              operating system.  INET is implemented using the  BSD Socket
4 *              interface as the means of communication with the user level.
5 *
6 *              Implementation of the Transmission Control Protocol(TCP).
7 *
8 * Authors:     Ross Biro
9 *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 *              Mark Evans, <evansmp@uhura.aston.ac.uk>
11 *              Corey Minyard <wf-rch!minyard@relay.EU.net>
12 *              Florian La Roche, <flla@stud.uni-sb.de>
13 *              Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
14 *              Linus Torvalds, <torvalds@cs.helsinki.fi>
15 *              Alan Cox, <gw4pts@gw4pts.ampr.org>
16 *              Matthew Dillon, <dillon@apollo.west.oic.com>
17 *              Arnt Gulbrandsen, <agulbra@nvg.unit.no>
18 *              Jorge Cwik, <jorge@laser.satlink.net>
19 */
20
21/*
22 * Changes:     Pedro Roque     :       Retransmit queue handled by TCP.
23 *                              :       Fragmentation on mtu decrease
24 *                              :       Segment collapse on retransmit
25 *                              :       AF independence
26 *
27 *              Linus Torvalds  :       send_delayed_ack
28 *              David S. Miller :       Charge memory using the right skb
29 *                                      during syn/ack processing.
30 *              David S. Miller :       Output engine completely rewritten.
31 *              Andrea Arcangeli:       SYNACK carry ts_recent in tsecr.
32 *              Cacophonix Gaul :       draft-minshall-nagle-01
33 *              J Hadi Salim    :       ECN support
34 *
35 */
36
37#define pr_fmt(fmt) "TCP: " fmt
38
39#include <net/tcp.h>
40
41#include <linux/compiler.h>
42#include <linux/gfp.h>
43#include <linux/module.h>
44
45/* People can turn this off for buggy TCP's found in printers etc. */
46int sysctl_tcp_retrans_collapse __read_mostly = 1;
47
48/* People can turn this on to work with those rare, broken TCPs that
49 * interpret the window field as a signed quantity.
50 */
51int sysctl_tcp_workaround_signed_windows __read_mostly = 0;
52
53/* Default TSQ limit of two TSO segments */
54int sysctl_tcp_limit_output_bytes __read_mostly = 131072;
55
56/* This limits the percentage of the congestion window which we
57 * will allow a single TSO frame to consume.  Building TSO frames
58 * which are too large can cause TCP streams to be bursty.
59 */
60int sysctl_tcp_tso_win_divisor __read_mostly = 3;
61
62int sysctl_tcp_mtu_probing __read_mostly = 0;
63int sysctl_tcp_base_mss __read_mostly = TCP_BASE_MSS;
64
65/* By default, RFC2861 behavior.  */
66int sysctl_tcp_slow_start_after_idle __read_mostly = 1;
67
68unsigned int sysctl_tcp_notsent_lowat __read_mostly = UINT_MAX;
69EXPORT_SYMBOL(sysctl_tcp_notsent_lowat);
70
71static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
72                           int push_one, gfp_t gfp);
73
74/* Account for new data that has been sent to the network. */
75static void tcp_event_new_data_sent(struct sock *sk, const struct sk_buff *skb)
76{
77        struct inet_connection_sock *icsk = inet_csk(sk);
78        struct tcp_sock *tp = tcp_sk(sk);
79        unsigned int prior_packets = tp->packets_out;
80
81        tcp_advance_send_head(sk, skb);
82        tp->snd_nxt = TCP_SKB_CB(skb)->end_seq;
83
84        tp->packets_out += tcp_skb_pcount(skb);
85        if (!prior_packets || icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS ||
86            icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
87                tcp_rearm_rto(sk);
88        }
89
90        NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT,
91                      tcp_skb_pcount(skb));
92}
93
94/* SND.NXT, if window was not shrunk.
95 * If window has been shrunk, what should we make? It is not clear at all.
96 * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-(
97 * Anything in between SND.UNA...SND.UNA+SND.WND also can be already
98 * invalid. OK, let's make this for now:
99 */
100static inline __u32 tcp_acceptable_seq(const struct sock *sk)
101{
102        const struct tcp_sock *tp = tcp_sk(sk);
103
104        if (!before(tcp_wnd_end(tp), tp->snd_nxt))
105                return tp->snd_nxt;
106        else
107                return tcp_wnd_end(tp);
108}
109
110/* Calculate mss to advertise in SYN segment.
111 * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that:
112 *
113 * 1. It is independent of path mtu.
114 * 2. Ideally, it is maximal possible segment size i.e. 65535-40.
115 * 3. For IPv4 it is reasonable to calculate it from maximal MTU of
116 *    attached devices, because some buggy hosts are confused by
117 *    large MSS.
118 * 4. We do not make 3, we advertise MSS, calculated from first
119 *    hop device mtu, but allow to raise it to ip_rt_min_advmss.
120 *    This may be overridden via information stored in routing table.
121 * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible,
122 *    probably even Jumbo".
123 */
124static __u16 tcp_advertise_mss(struct sock *sk)
125{
126        struct tcp_sock *tp = tcp_sk(sk);
127        const struct dst_entry *dst = __sk_dst_get(sk);
128        int mss = tp->advmss;
129
130        if (dst) {
131                unsigned int metric = dst_metric_advmss(dst);
132
133                if (metric < mss) {
134                        mss = metric;
135                        tp->advmss = mss;
136                }
137        }
138
139        return (__u16)mss;
140}
141
142/* RFC2861. Reset CWND after idle period longer RTO to "restart window".
143 * This is the first part of cwnd validation mechanism. */
144static void tcp_cwnd_restart(struct sock *sk, const struct dst_entry *dst)
145{
146        struct tcp_sock *tp = tcp_sk(sk);
147        s32 delta = tcp_time_stamp - tp->lsndtime;
148        u32 restart_cwnd = tcp_init_cwnd(tp, dst);
149        u32 cwnd = tp->snd_cwnd;
150
151        tcp_ca_event(sk, CA_EVENT_CWND_RESTART);
152
153        tp->snd_ssthresh = tcp_current_ssthresh(sk);
154        restart_cwnd = min(restart_cwnd, cwnd);
155
156        while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd)
157                cwnd >>= 1;
158        tp->snd_cwnd = max(cwnd, restart_cwnd);
159        tp->snd_cwnd_stamp = tcp_time_stamp;
160        tp->snd_cwnd_used = 0;
161}
162
163/* Congestion state accounting after a packet has been sent. */
164static void tcp_event_data_sent(struct tcp_sock *tp,
165                                struct sock *sk)
166{
167        struct inet_connection_sock *icsk = inet_csk(sk);
168        const u32 now = tcp_time_stamp;
169        const struct dst_entry *dst = __sk_dst_get(sk);
170
171        if (sysctl_tcp_slow_start_after_idle &&
172            (!tp->packets_out && (s32)(now - tp->lsndtime) > icsk->icsk_rto))
173                tcp_cwnd_restart(sk, __sk_dst_get(sk));
174
175        tp->lsndtime = now;
176
177        /* If it is a reply for ato after last received
178         * packet, enter pingpong mode.
179         */
180        if ((u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato &&
181            (!dst || !dst_metric(dst, RTAX_QUICKACK)))
182                        icsk->icsk_ack.pingpong = 1;
183}
184
185/* Account for an ACK we sent. */
186static inline void tcp_event_ack_sent(struct sock *sk, unsigned int pkts)
187{
188        tcp_dec_quickack_mode(sk, pkts);
189        inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
190}
191
192
193u32 tcp_default_init_rwnd(u32 mss)
194{
195        /* Initial receive window should be twice of TCP_INIT_CWND to
196         * enable proper sending of new unsent data during fast recovery
197         * (RFC 3517, Section 4, NextSeg() rule (2)). Further place a
198         * limit when mss is larger than 1460.
199         */
200        u32 init_rwnd = TCP_INIT_CWND * 2;
201
202        if (mss > 1460)
203                init_rwnd = max((1460 * init_rwnd) / mss, 2U);
204        return init_rwnd;
205}
206
207/* Determine a window scaling and initial window to offer.
208 * Based on the assumption that the given amount of space
209 * will be offered. Store the results in the tp structure.
210 * NOTE: for smooth operation initial space offering should
211 * be a multiple of mss if possible. We assume here that mss >= 1.
212 * This MUST be enforced by all callers.
213 */
214void tcp_select_initial_window(int __space, __u32 mss,
215                               __u32 *rcv_wnd, __u32 *window_clamp,
216                               int wscale_ok, __u8 *rcv_wscale,
217                               __u32 init_rcv_wnd)
218{
219        unsigned int space = (__space < 0 ? 0 : __space);
220
221        /* If no clamp set the clamp to the max possible scaled window */
222        if (*window_clamp == 0)
223                (*window_clamp) = (65535 << 14);
224        space = min(*window_clamp, space);
225
226        /* Quantize space offering to a multiple of mss if possible. */
227        if (space > mss)
228                space = (space / mss) * mss;
229
230        /* NOTE: offering an initial window larger than 32767
231         * will break some buggy TCP stacks. If the admin tells us
232         * it is likely we could be speaking with such a buggy stack
233         * we will truncate our initial window offering to 32K-1
234         * unless the remote has sent us a window scaling option,
235         * which we interpret as a sign the remote TCP is not
236         * misinterpreting the window field as a signed quantity.
237         */
238        if (sysctl_tcp_workaround_signed_windows)
239                (*rcv_wnd) = min(space, MAX_TCP_WINDOW);
240        else
241                (*rcv_wnd) = space;
242
243        (*rcv_wscale) = 0;
244        if (wscale_ok) {
245                /* Set window scaling on max possible window
246                 * See RFC1323 for an explanation of the limit to 14
247                 */
248                space = max_t(u32, sysctl_tcp_rmem[2], sysctl_rmem_max);
249                space = min_t(u32, space, *window_clamp);
250                while (space > 65535 && (*rcv_wscale) < 14) {
251                        space >>= 1;
252                        (*rcv_wscale)++;
253                }
254        }
255
256        if (mss > (1 << *rcv_wscale)) {
257                if (!init_rcv_wnd) /* Use default unless specified otherwise */
258                        init_rcv_wnd = tcp_default_init_rwnd(mss);
259                *rcv_wnd = min(*rcv_wnd, init_rcv_wnd * mss);
260        }
261
262        /* Set the clamp no higher than max representable value */
263        (*window_clamp) = min(65535U << (*rcv_wscale), *window_clamp);
264}
265EXPORT_SYMBOL(tcp_select_initial_window);
266
267/* Chose a new window to advertise, update state in tcp_sock for the
268 * socket, and return result with RFC1323 scaling applied.  The return
269 * value can be stuffed directly into th->window for an outgoing
270 * frame.
271 */
272static u16 tcp_select_window(struct sock *sk)
273{
274        struct tcp_sock *tp = tcp_sk(sk);
275        u32 old_win = tp->rcv_wnd;
276        u32 cur_win = tcp_receive_window(tp);
277        u32 new_win = __tcp_select_window(sk);
278
279        /* Never shrink the offered window */
280        if (new_win < cur_win) {
281                /* Danger Will Robinson!
282                 * Don't update rcv_wup/rcv_wnd here or else
283                 * we will not be able to advertise a zero
284                 * window in time.  --DaveM
285                 *
286                 * Relax Will Robinson.
287                 */
288                if (new_win == 0)
289                        NET_INC_STATS(sock_net(sk),
290                                      LINUX_MIB_TCPWANTZEROWINDOWADV);
291                new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale);
292        }
293        tp->rcv_wnd = new_win;
294        tp->rcv_wup = tp->rcv_nxt;
295
296        /* Make sure we do not exceed the maximum possible
297         * scaled window.
298         */
299        if (!tp->rx_opt.rcv_wscale && sysctl_tcp_workaround_signed_windows)
300                new_win = min(new_win, MAX_TCP_WINDOW);
301        else
302                new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));
303
304        /* RFC1323 scaling applied */
305        new_win >>= tp->rx_opt.rcv_wscale;
306
307        /* If we advertise zero window, disable fast path. */
308        if (new_win == 0) {
309                tp->pred_flags = 0;
310                if (old_win)
311                        NET_INC_STATS(sock_net(sk),
312                                      LINUX_MIB_TCPTOZEROWINDOWADV);
313        } else if (old_win == 0) {
314                NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPFROMZEROWINDOWADV);
315        }
316
317        return new_win;
318}
319
320/* Packet ECN state for a SYN-ACK */
321static void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb)
322{
323        const struct tcp_sock *tp = tcp_sk(sk);
324
325        TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR;
326        if (!(tp->ecn_flags & TCP_ECN_OK))
327                TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE;
328        else if (tcp_ca_needs_ecn(sk))
329                INET_ECN_xmit(sk);
330}
331
332/* Packet ECN state for a SYN.  */
333static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb)
334{
335        struct tcp_sock *tp = tcp_sk(sk);
336
337        tp->ecn_flags = 0;
338        if (sock_net(sk)->ipv4.sysctl_tcp_ecn == 1 ||
339            tcp_ca_needs_ecn(sk)) {
340                TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR;
341                tp->ecn_flags = TCP_ECN_OK;
342                if (tcp_ca_needs_ecn(sk))
343                        INET_ECN_xmit(sk);
344        }
345}
346
347static void
348tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th,
349                    struct sock *sk)
350{
351        if (inet_rsk(req)->ecn_ok) {
352                th->ece = 1;
353                if (tcp_ca_needs_ecn(sk))
354                        INET_ECN_xmit(sk);
355        }
356}
357
358/* Set up ECN state for a packet on a ESTABLISHED socket that is about to
359 * be sent.
360 */
361static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb,
362                                int tcp_header_len)
363{
364        struct tcp_sock *tp = tcp_sk(sk);
365
366        if (tp->ecn_flags & TCP_ECN_OK) {
367                /* Not-retransmitted data segment: set ECT and inject CWR. */
368                if (skb->len != tcp_header_len &&
369                    !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) {
370                        INET_ECN_xmit(sk);
371                        if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) {
372                                tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
373                                tcp_hdr(skb)->cwr = 1;
374                                skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
375                        }
376                } else if (!tcp_ca_needs_ecn(sk)) {
377                        /* ACK or retransmitted segment: clear ECT|CE */
378                        INET_ECN_dontxmit(sk);
379                }
380                if (tp->ecn_flags & TCP_ECN_DEMAND_CWR)
381                        tcp_hdr(skb)->ece = 1;
382        }
383}
384
385/* Constructs common control bits of non-data skb. If SYN/FIN is present,
386 * auto increment end seqno.
387 */
388static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags)
389{
390        struct skb_shared_info *shinfo = skb_shinfo(skb);
391
392        skb->ip_summed = CHECKSUM_PARTIAL;
393        skb->csum = 0;
394
395        TCP_SKB_CB(skb)->tcp_flags = flags;
396        TCP_SKB_CB(skb)->sacked = 0;
397
398        tcp_skb_pcount_set(skb, 1);
399        shinfo->gso_size = 0;
400        shinfo->gso_type = 0;
401
402        TCP_SKB_CB(skb)->seq = seq;
403        if (flags & (TCPHDR_SYN | TCPHDR_FIN))
404                seq++;
405        TCP_SKB_CB(skb)->end_seq = seq;
406}
407
408static inline bool tcp_urg_mode(const struct tcp_sock *tp)
409{
410        return tp->snd_una != tp->snd_up;
411}
412
413#define OPTION_SACK_ADVERTISE   (1 << 0)
414#define OPTION_TS               (1 << 1)
415#define OPTION_MD5              (1 << 2)
416#define OPTION_WSCALE           (1 << 3)
417#define OPTION_FAST_OPEN_COOKIE (1 << 8)
418
419struct tcp_out_options {
420        u16 options;            /* bit field of OPTION_* */
421        u16 mss;                /* 0 to disable */
422        u8 ws;                  /* window scale, 0 to disable */
423        u8 num_sack_blocks;     /* number of SACK blocks to include */
424        u8 hash_size;           /* bytes in hash_location */
425        __u8 *hash_location;    /* temporary pointer, overloaded */
426        __u32 tsval, tsecr;     /* need to include OPTION_TS */
427        struct tcp_fastopen_cookie *fastopen_cookie;    /* Fast open cookie */
428};
429
430/* Write previously computed TCP options to the packet.
431 *
432 * Beware: Something in the Internet is very sensitive to the ordering of
433 * TCP options, we learned this through the hard way, so be careful here.
434 * Luckily we can at least blame others for their non-compliance but from
435 * inter-operability perspective it seems that we're somewhat stuck with
436 * the ordering which we have been using if we want to keep working with
437 * those broken things (not that it currently hurts anybody as there isn't
438 * particular reason why the ordering would need to be changed).
439 *
440 * At least SACK_PERM as the first option is known to lead to a disaster
441 * (but it may well be that other scenarios fail similarly).
442 */
443static void tcp_options_write(__be32 *ptr, struct tcp_sock *tp,
444                              struct tcp_out_options *opts)
445{
446        u16 options = opts->options;    /* mungable copy */
447
448        if (unlikely(OPTION_MD5 & options)) {
449                net_hdr_word(ptr++) =
450                        htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
451                              (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
452                /* overload cookie hash location */
453                opts->hash_location = (__u8 *)ptr;
454                ptr += 4;
455        }
456
457        if (unlikely(opts->mss)) {
458                net_hdr_word(ptr++) =
459                        htonl((TCPOPT_MSS << 24) | (TCPOLEN_MSS << 16) |
460                              opts->mss);
461        }
462
463        if (likely(OPTION_TS & options)) {
464                if (unlikely(OPTION_SACK_ADVERTISE & options)) {
465                        net_hdr_word(ptr++) =
466                                htonl((TCPOPT_SACK_PERM << 24) |
467                                      (TCPOLEN_SACK_PERM << 16) |
468                                      (TCPOPT_TIMESTAMP << 8) |
469                                      TCPOLEN_TIMESTAMP);
470                        options &= ~OPTION_SACK_ADVERTISE;
471                } else {
472                        net_hdr_word(ptr++) =
473                                htonl((TCPOPT_NOP << 24) |
474                                      (TCPOPT_NOP << 16) |
475                                      (TCPOPT_TIMESTAMP << 8) |
476                                      TCPOLEN_TIMESTAMP);
477                }
478                net_hdr_word(ptr++) = htonl(opts->tsval);
479                net_hdr_word(ptr++) = htonl(opts->tsecr);
480        }
481
482        if (unlikely(OPTION_SACK_ADVERTISE & options)) {
483                net_hdr_word(ptr++) =
484                        htonl((TCPOPT_NOP << 24) |
485                              (TCPOPT_NOP << 16) |
486                              (TCPOPT_SACK_PERM << 8) |
487                              TCPOLEN_SACK_PERM);
488        }
489
490        if (unlikely(OPTION_WSCALE & options)) {
491                net_hdr_word(ptr++) =
492                        htonl((TCPOPT_NOP << 24) |
493                              (TCPOPT_WINDOW << 16) |
494                              (TCPOLEN_WINDOW << 8) |
495                              opts->ws);
496        }
497
498        if (unlikely(opts->num_sack_blocks)) {
499                struct tcp_sack_block *sp = tp->rx_opt.dsack ?
500                        tp->duplicate_sack : tp->selective_acks;
501                int this_sack;
502
503                net_hdr_word(ptr++) =
504                        htonl((TCPOPT_NOP << 24) |
505                              (TCPOPT_NOP << 16) |
506                              (TCPOPT_SACK << 8) |
507                              (TCPOLEN_SACK_BASE + (opts->num_sack_blocks *
508                                                     TCPOLEN_SACK_PERBLOCK)));
509
510                for (this_sack = 0; this_sack < opts->num_sack_blocks;
511                     ++this_sack) {
512                        net_hdr_word(ptr++) = htonl(sp[this_sack].start_seq);
513                        net_hdr_word(ptr++) = htonl(sp[this_sack].end_seq);
514                }
515
516                tp->rx_opt.dsack = 0;
517        }
518
519        if (unlikely(OPTION_FAST_OPEN_COOKIE & options)) {
520                struct tcp_fastopen_cookie *foc = opts->fastopen_cookie;
521
522                net_hdr_word(ptr++) =
523                        htonl((TCPOPT_EXP << 24) |
524                              ((TCPOLEN_EXP_FASTOPEN_BASE + foc->len) << 16) |
525                              TCPOPT_FASTOPEN_MAGIC);
526
527                memcpy(ptr, foc->val, foc->len);
528                if ((foc->len & 3) == 2) {
529                        u8 *align = ((u8 *)ptr) + foc->len;
530                        align[0] = align[1] = TCPOPT_NOP;
531                }
532                ptr += (foc->len + 3) >> 2;
533        }
534}
535
536/* Compute TCP options for SYN packets. This is not the final
537 * network wire format yet.
538 */
539static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
540                                struct tcp_out_options *opts,
541                                struct tcp_md5sig_key **md5)
542{
543        struct tcp_sock *tp = tcp_sk(sk);
544        unsigned int remaining = MAX_TCP_OPTION_SPACE;
545        struct tcp_fastopen_request *fastopen = tp->fastopen_req;
546
547#ifdef CONFIG_TCP_MD5SIG
548        *md5 = tp->af_specific->md5_lookup(sk, sk);
549        if (*md5) {
550                opts->options |= OPTION_MD5;
551                remaining -= TCPOLEN_MD5SIG_ALIGNED;
552        }
553#else
554        *md5 = NULL;
555#endif
556
557        /* We always get an MSS option.  The option bytes which will be seen in
558         * normal data packets should timestamps be used, must be in the MSS
559         * advertised.  But we subtract them from tp->mss_cache so that
560         * calculations in tcp_sendmsg are simpler etc.  So account for this
561         * fact here if necessary.  If we don't do this correctly, as a
562         * receiver we won't recognize data packets as being full sized when we
563         * should, and thus we won't abide by the delayed ACK rules correctly.
564         * SACKs don't matter, we never delay an ACK when we have any of those
565         * going out.  */
566        opts->mss = tcp_advertise_mss(sk);
567        remaining -= TCPOLEN_MSS_ALIGNED;
568
569        if (likely(sysctl_tcp_timestamps && *md5 == NULL)) {
570                opts->options |= OPTION_TS;
571                opts->tsval = tcp_skb_timestamp(skb) + tp->tsoffset;
572                opts->tsecr = tp->rx_opt.ts_recent;
573                remaining -= TCPOLEN_TSTAMP_ALIGNED;
574        }
575        if (likely(sysctl_tcp_window_scaling)) {
576                opts->ws = tp->rx_opt.rcv_wscale;
577                opts->options |= OPTION_WSCALE;
578                remaining -= TCPOLEN_WSCALE_ALIGNED;
579        }
580        if (likely(sysctl_tcp_sack)) {
581                opts->options |= OPTION_SACK_ADVERTISE;
582                if (unlikely(!(OPTION_TS & opts->options)))
583                        remaining -= TCPOLEN_SACKPERM_ALIGNED;
584        }
585
586        if (fastopen && fastopen->cookie.len >= 0) {
587                u32 need = TCPOLEN_EXP_FASTOPEN_BASE + fastopen->cookie.len;
588                need = (need + 3) & ~3U;  /* Align to 32 bits */
589                if (remaining >= need) {
590                        opts->options |= OPTION_FAST_OPEN_COOKIE;
591                        opts->fastopen_cookie = &fastopen->cookie;
592                        remaining -= need;
593                        tp->syn_fastopen = 1;
594                }
595        }
596
597        return MAX_TCP_OPTION_SPACE - remaining;
598}
599
600/* Set up TCP options for SYN-ACKs. */
601static unsigned int tcp_synack_options(struct sock *sk,
602                                   struct request_sock *req,
603                                   unsigned int mss, struct sk_buff *skb,
604                                   struct tcp_out_options *opts,
605                                   struct tcp_md5sig_key **md5,
606                                   struct tcp_fastopen_cookie *foc)
607{
608        struct inet_request_sock *ireq = inet_rsk(req);
609        unsigned int remaining = MAX_TCP_OPTION_SPACE;
610
611#ifdef CONFIG_TCP_MD5SIG
612        *md5 = tcp_rsk(req)->af_specific->md5_lookup(sk, req);
613        if (*md5) {
614                opts->options |= OPTION_MD5;
615                remaining -= TCPOLEN_MD5SIG_ALIGNED;
616
617                /* We can't fit any SACK blocks in a packet with MD5 + TS
618                 * options. There was discussion about disabling SACK
619                 * rather than TS in order to fit in better with old,
620                 * buggy kernels, but that was deemed to be unnecessary.
621                 */
622                ireq->tstamp_ok &= !ireq->sack_ok;
623        }
624#else
625        *md5 = NULL;
626#endif
627
628        /* We always send an MSS option. */
629        opts->mss = mss;
630        remaining -= TCPOLEN_MSS_ALIGNED;
631
632        if (likely(ireq->wscale_ok)) {
633                opts->ws = ireq->rcv_wscale;
634                opts->options |= OPTION_WSCALE;
635                remaining -= TCPOLEN_WSCALE_ALIGNED;
636        }
637        if (likely(ireq->tstamp_ok)) {
638                opts->options |= OPTION_TS;
639                opts->tsval = tcp_skb_timestamp(skb);
640                opts->tsecr = req->ts_recent;
641                remaining -= TCPOLEN_TSTAMP_ALIGNED;
642        }
643        if (likely(ireq->sack_ok)) {
644                opts->options |= OPTION_SACK_ADVERTISE;
645                if (unlikely(!ireq->tstamp_ok))
646                        remaining -= TCPOLEN_SACKPERM_ALIGNED;
647        }
648        if (foc != NULL && foc->len >= 0) {
649                u32 need = TCPOLEN_EXP_FASTOPEN_BASE + foc->len;
650                need = (need + 3) & ~3U;  /* Align to 32 bits */
651                if (remaining >= need) {
652                        opts->options |= OPTION_FAST_OPEN_COOKIE;
653                        opts->fastopen_cookie = foc;
654                        remaining -= need;
655                }
656        }
657
658        return MAX_TCP_OPTION_SPACE - remaining;
659}
660
661/* Compute TCP options for ESTABLISHED sockets. This is not the
662 * final wire format yet.
663 */
664static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb,
665                                        struct tcp_out_options *opts,
666                                        struct tcp_md5sig_key **md5)
667{
668        struct tcp_sock *tp = tcp_sk(sk);
669        unsigned int size = 0;
670        unsigned int eff_sacks;
671
672        opts->options = 0;
673
674#ifdef CONFIG_TCP_MD5SIG
675        *md5 = tp->af_specific->md5_lookup(sk, sk);
676        if (unlikely(*md5)) {
677                opts->options |= OPTION_MD5;
678                size += TCPOLEN_MD5SIG_ALIGNED;
679        }
680#else
681        *md5 = NULL;
682#endif
683
684        if (likely(tp->rx_opt.tstamp_ok)) {
685                opts->options |= OPTION_TS;
686                opts->tsval = skb ? tcp_skb_timestamp(skb) + tp->tsoffset : 0;
687                opts->tsecr = tp->rx_opt.ts_recent;
688                size += TCPOLEN_TSTAMP_ALIGNED;
689        }
690
691        eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack;
692        if (unlikely(eff_sacks)) {
693                const unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
694                opts->num_sack_blocks =
695                        min_t(unsigned int, eff_sacks,
696                              (remaining - TCPOLEN_SACK_BASE_ALIGNED) /
697                              TCPOLEN_SACK_PERBLOCK);
698                size += TCPOLEN_SACK_BASE_ALIGNED +
699                        opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK;
700        }
701
702        return size;
703}
704
705
706/* TCP SMALL QUEUES (TSQ)
707 *
708 * TSQ goal is to keep small amount of skbs per tcp flow in tx queues (qdisc+dev)
709 * to reduce RTT and bufferbloat.
710 * We do this using a special skb destructor (tcp_wfree).
711 *
712 * Its important tcp_wfree() can be replaced by sock_wfree() in the event skb
713 * needs to be reallocated in a driver.
714 * The invariant being skb->truesize subtracted from sk->sk_wmem_alloc
715 *
716 * Since transmit from skb destructor is forbidden, we use a tasklet
717 * to process all sockets that eventually need to send more skbs.
718 * We use one tasklet per cpu, with its own queue of sockets.
719 */
720struct tsq_tasklet {
721        struct tasklet_struct   tasklet;
722        struct list_head        head; /* queue of tcp sockets */
723};
724static DEFINE_PER_CPU(struct tsq_tasklet, tsq_tasklet);
725
726static void tcp_tsq_handler(struct sock *sk)
727{
728        if ((1 << sk->sk_state) &
729            (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_CLOSING |
730             TCPF_CLOSE_WAIT  | TCPF_LAST_ACK))
731                tcp_write_xmit(sk, tcp_current_mss(sk), tcp_sk(sk)->nonagle,
732                               0, GFP_ATOMIC);
733}
734/*
735 * One tasklet per cpu tries to send more skbs.
736 * We run in tasklet context but need to disable irqs when
737 * transferring tsq->head because tcp_wfree() might
738 * interrupt us (non NAPI drivers)
739 */
740static void tcp_tasklet_func(unsigned long data)
741{
742        struct tsq_tasklet *tsq = (struct tsq_tasklet *)data;
743        LIST_HEAD(list);
744        unsigned long flags;
745        struct list_head *q, *n;
746        struct tcp_sock *tp;
747        struct sock *sk;
748
749        local_irq_save(flags);
750        list_splice_init(&tsq->head, &list);
751        local_irq_restore(flags);
752
753        list_for_each_safe(q, n, &list) {
754                tp = list_entry(q, struct tcp_sock, tsq_node);
755                list_del(&tp->tsq_node);
756
757                sk = (struct sock *)tp;
758                bh_lock_sock(sk);
759
760                if (!sock_owned_by_user(sk)) {
761                        tcp_tsq_handler(sk);
762                } else {
763                        /* defer the work to tcp_release_cb() */
764                        set_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags);
765                }
766                bh_unlock_sock(sk);
767
768                clear_bit(TSQ_QUEUED, &tp->tsq_flags);
769                sk_free(sk);
770        }
771}
772
773#define TCP_DEFERRED_ALL ((1UL << TCP_TSQ_DEFERRED) |           \
774                          (1UL << TCP_WRITE_TIMER_DEFERRED) |   \
775                          (1UL << TCP_DELACK_TIMER_DEFERRED) |  \
776                          (1UL << TCP_MTU_REDUCED_DEFERRED))
777/**
778 * tcp_release_cb - tcp release_sock() callback
779 * @sk: socket
780 *
781 * called from release_sock() to perform protocol dependent
782 * actions before socket release.
783 */
784void tcp_release_cb(struct sock *sk)
785{
786        struct tcp_sock *tp = tcp_sk(sk);
787        unsigned long flags, nflags;
788
789        /* perform an atomic operation only if at least one flag is set */
790        do {
791                flags = tp->tsq_flags;
792                if (!(flags & TCP_DEFERRED_ALL))
793                        return;
794                nflags = flags & ~TCP_DEFERRED_ALL;
795        } while (cmpxchg(&tp->tsq_flags, flags, nflags) != flags);
796
797        if (flags & (1UL << TCP_TSQ_DEFERRED))
798                tcp_tsq_handler(sk);
799
800        /* Here begins the tricky part :
801         * We are called from release_sock() with :
802         * 1) BH disabled
803         * 2) sk_lock.slock spinlock held
804         * 3) socket owned by us (sk->sk_lock.owned == 1)
805         *
806         * But following code is meant to be called from BH handlers,
807         * so we should keep BH disabled, but early release socket ownership
808         */
809        sock_release_ownership(sk);
810
811        if (flags & (1UL << TCP_WRITE_TIMER_DEFERRED)) {
812                tcp_write_timer_handler(sk);
813                __sock_put(sk);
814        }
815        if (flags & (1UL << TCP_DELACK_TIMER_DEFERRED)) {
816                tcp_delack_timer_handler(sk);
817                __sock_put(sk);
818        }
819        if (flags & (1UL << TCP_MTU_REDUCED_DEFERRED)) {
820                inet_csk(sk)->icsk_af_ops->mtu_reduced(sk);
821                __sock_put(sk);
822        }
823}
824EXPORT_SYMBOL(tcp_release_cb);
825
826void __init tcp_tasklet_init(void)
827{
828        int i;
829
830        for_each_possible_cpu(i) {
831                struct tsq_tasklet *tsq = &per_cpu(tsq_tasklet, i);
832
833                INIT_LIST_HEAD(&tsq->head);
834                tasklet_init(&tsq->tasklet,
835                             tcp_tasklet_func,
836                             (unsigned long)tsq);
837        }
838}
839
840/*
841 * Write buffer destructor automatically called from kfree_skb.
842 * We can't xmit new skbs from this context, as we might already
843 * hold qdisc lock.
844 */
845void tcp_wfree(struct sk_buff *skb)
846{
847        struct sock *sk = skb->sk;
848        struct tcp_sock *tp = tcp_sk(sk);
849        int wmem;
850
851        /* Keep one reference on sk_wmem_alloc.
852         * Will be released by sk_free() from here or tcp_tasklet_func()
853         */
854        wmem = atomic_sub_return(skb->truesize - 1, &sk->sk_wmem_alloc);
855
856        /* If this softirq is serviced by ksoftirqd, we are likely under stress.
857         * Wait until our queues (qdisc + devices) are drained.
858         * This gives :
859         * - less callbacks to tcp_write_xmit(), reducing stress (batches)
860         * - chance for incoming ACK (processed by another cpu maybe)
861         *   to migrate this flow (skb->ooo_okay will be eventually set)
862         */
863        if (wmem >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current)
864                goto out;
865
866        if (test_and_clear_bit(TSQ_THROTTLED, &tp->tsq_flags) &&
867            !test_and_set_bit(TSQ_QUEUED, &tp->tsq_flags)) {
868                unsigned long flags;
869                struct tsq_tasklet *tsq;
870
871                /* queue this socket to tasklet queue */
872                local_irq_save(flags);
873                tsq = this_cpu_ptr(&tsq_tasklet);
874                list_add(&tp->tsq_node, &tsq->head);
875                tasklet_schedule(&tsq->tasklet);
876                local_irq_restore(flags);
877                return;
878        }
879out:
880        sk_free(sk);
881}
882
883/* This routine actually transmits TCP packets queued in by
884 * tcp_do_sendmsg().  This is used by both the initial
885 * transmission and possible later retransmissions.
886 * All SKB's seen here are completely headerless.  It is our
887 * job to build the TCP header, and pass the packet down to
888 * IP so it can do the same plus pass the packet off to the
889 * device.
890 *
891 * We are working here with either a clone of the original
892 * SKB, or a fresh unique copy made by the retransmit engine.
893 */
894static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
895                            gfp_t gfp_mask)
896{
897        const struct inet_connection_sock *icsk = inet_csk(sk);
898        struct inet_sock *inet;
899        struct tcp_sock *tp;
900        struct tcp_skb_cb *tcb;
901        struct tcp_out_options opts;
902        unsigned int tcp_options_size, tcp_header_size;
903        struct tcp_md5sig_key *md5;
904        struct tcphdr *th;
905        int err;
906
907        BUG_ON(!skb || !tcp_skb_pcount(skb));
908
909        if (clone_it) {
910                skb_mstamp_get(&skb->skb_mstamp);
911
912                if (unlikely(skb_cloned(skb)))
913                        skb = pskb_copy(skb, gfp_mask);
914                else
915                        skb = skb_clone(skb, gfp_mask);
916                if (unlikely(!skb))
917                        return -ENOBUFS;
918        }
919
920        inet = inet_sk(sk);
921        tp = tcp_sk(sk);
922        tcb = TCP_SKB_CB(skb);
923        memset(&opts, 0, sizeof(opts));
924
925        if (unlikely(tcb->tcp_flags & TCPHDR_SYN))
926                tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5);
927        else
928                tcp_options_size = tcp_established_options(sk, skb, &opts,
929                                                           &md5);
930        tcp_header_size = tcp_options_size + sizeof(struct tcphdr);
931
932        if (tcp_packets_in_flight(tp) == 0)
933                tcp_ca_event(sk, CA_EVENT_TX_START);
934
935        /* if no packet is in qdisc/device queue, then allow XPS to select
936         * another queue. We can be called from tcp_tsq_handler()
937         * which holds one reference to sk_wmem_alloc.
938         *
939         * TODO: Ideally, in-flight pure ACK packets should not matter here.
940         * One way to get this would be to set skb->truesize = 2 on them.
941         */
942        skb->ooo_okay = sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1);
943
944        skb_push(skb, tcp_header_size);
945        skb_reset_transport_header(skb);
946
947        skb_orphan(skb);
948        skb->sk = sk;
949        skb->destructor = tcp_wfree;
950        skb_set_hash_from_sk(skb, sk);
951        atomic_add(skb->truesize, &sk->sk_wmem_alloc);
952
953        /* Build TCP header and checksum it. */
954        th = tcp_hdr(skb);
955        th->source              = inet->inet_sport;
956        th->dest                = inet->inet_dport;
957        th->seq                 = htonl(tcb->seq);
958        th->ack_seq             = htonl(tp->rcv_nxt);
959        *(((__be16 *)th) + 6)   = htons(((tcp_header_size >> 2) << 12) |
960                                        tcb->tcp_flags);
961
962        if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) {
963                /* RFC1323: The window in SYN & SYN/ACK segments
964                 * is never scaled.
965                 */
966                th->window      = htons(min(tp->rcv_wnd, 65535U));
967        } else {
968                th->window      = htons(tcp_select_window(sk));
969        }
970        th->check               = 0;
971        th->urg_ptr             = 0;
972
973        /* The urg_mode check is necessary during a below snd_una win probe */
974        if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) {
975                if (before(tp->snd_up, tcb->seq + 0x10000)) {
976                        th->urg_ptr = htons(tp->snd_up - tcb->seq);
977                        th->urg = 1;
978                } else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) {
979                        th->urg_ptr = htons(0xFFFF);
980                        th->urg = 1;
981                }
982        }
983
984        tcp_options_write((__be32 *)(th + 1), tp, &opts);
985        if (likely((tcb->tcp_flags & TCPHDR_SYN) == 0))
986                tcp_ecn_send(sk, skb, tcp_header_size);
987
988#ifdef CONFIG_TCP_MD5SIG
989        /* Calculate the MD5 hash, as we have all we need now */
990        if (md5) {
991                sk_nocaps_add(sk, NETIF_F_GSO_MASK);
992                tp->af_specific->calc_md5_hash(opts.hash_location,
993                                               md5, sk, NULL, skb);
994        }
995#endif
996
997        icsk->icsk_af_ops->send_check(sk, skb);
998
999        if (likely(tcb->tcp_flags & TCPHDR_ACK))
1000                tcp_event_ack_sent(sk, tcp_skb_pcount(skb));
1001
1002        if (skb->len != tcp_header_size)
1003                tcp_event_data_sent(tp, sk);
1004
1005        if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
1006                TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS,
1007                              tcp_skb_pcount(skb));
1008
1009        /* OK, its time to fill skb_shinfo(skb)->gso_segs */
1010        skb_shinfo(skb)->gso_segs = tcp_skb_pcount(skb);
1011
1012        /* Our usage of tstamp should remain private */
1013        skb->tstamp.tv64 = 0;
1014
1015        /* Cleanup our debris for IP stacks */
1016        memset(skb->cb, 0, max(sizeof(struct inet_skb_parm),
1017                               sizeof(struct inet6_skb_parm)));
1018
1019        err = icsk->icsk_af_ops->queue_xmit(sk, skb, &inet->cork.fl);
1020
1021        if (likely(err <= 0))
1022                return err;
1023
1024        tcp_enter_cwr(sk);
1025
1026        return net_xmit_eval(err);
1027}
1028
1029/* This routine just queues the buffer for sending.
1030 *
1031 * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames,
1032 * otherwise socket can stall.
1033 */
1034static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
1035{
1036        struct tcp_sock *tp = tcp_sk(sk);
1037
1038        /* Advance write_seq and place onto the write_queue. */
1039        tp->write_seq = TCP_SKB_CB(skb)->end_seq;
1040        __skb_header_release(skb);
1041        tcp_add_write_queue_tail(sk, skb);
1042        sk->sk_wmem_queued += skb->truesize;
1043        sk_mem_charge(sk, skb->truesize);
1044}
1045
1046/* Initialize TSO segments for a packet. */
1047static void tcp_set_skb_tso_segs(const struct sock *sk, struct sk_buff *skb,
1048                                 unsigned int mss_now)
1049{
1050        struct skb_shared_info *shinfo = skb_shinfo(skb);
1051
1052        /* Make sure we own this skb before messing gso_size/gso_segs */
1053        WARN_ON_ONCE(skb_cloned(skb));
1054
1055        if (skb->len <= mss_now || skb->ip_summed == CHECKSUM_NONE) {
1056                /* Avoid the costly divide in the normal
1057                 * non-TSO case.
1058                 */
1059                tcp_skb_pcount_set(skb, 1);
1060                shinfo->gso_size = 0;
1061                shinfo->gso_type = 0;
1062        } else {
1063                tcp_skb_pcount_set(skb, DIV_ROUND_UP(skb->len, mss_now));
1064                shinfo->gso_size = mss_now;
1065                shinfo->gso_type = sk->sk_gso_type;
1066        }
1067}
1068
1069/* When a modification to fackets out becomes necessary, we need to check
1070 * skb is counted to fackets_out or not.
1071 */
1072static void tcp_adjust_fackets_out(struct sock *sk, const struct sk_buff *skb,
1073                                   int decr)
1074{
1075        struct tcp_sock *tp = tcp_sk(sk);
1076
1077        if (!tp->sacked_out || tcp_is_reno(tp))
1078                return;
1079
1080        if (after(tcp_highest_sack_seq(tp), TCP_SKB_CB(skb)->seq))
1081                tp->fackets_out -= decr;
1082}
1083
1084/* Pcount in the middle of the write queue got changed, we need to do various
1085 * tweaks to fix counters
1086 */
1087static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int decr)
1088{
1089        struct tcp_sock *tp = tcp_sk(sk);
1090
1091        tp->packets_out -= decr;
1092
1093        if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
1094                tp->sacked_out -= decr;
1095        if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS)
1096                tp->retrans_out -= decr;
1097        if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST)
1098                tp->lost_out -= decr;
1099
1100        /* Reno case is special. Sigh... */
1101        if (tcp_is_reno(tp) && decr > 0)
1102                tp->sacked_out -= min_t(u32, tp->sacked_out, decr);
1103
1104        tcp_adjust_fackets_out(sk, skb, decr);
1105
1106        if (tp->lost_skb_hint &&
1107            before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->lost_skb_hint)->seq) &&
1108            (tcp_is_fack(tp) || (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)))
1109                tp->lost_cnt_hint -= decr;
1110
1111        tcp_verify_left_out(tp);
1112}
1113
1114static void tcp_fragment_tstamp(struct sk_buff *skb, struct sk_buff *skb2)
1115{
1116        struct skb_shared_info *shinfo = skb_shinfo(skb);
1117
1118        if (unlikely(shinfo->tx_flags & SKBTX_ANY_TSTAMP) &&
1119            !before(shinfo->tskey, TCP_SKB_CB(skb2)->seq)) {
1120                struct skb_shared_info *shinfo2 = skb_shinfo(skb2);
1121                u8 tsflags = shinfo->tx_flags & SKBTX_ANY_TSTAMP;
1122
1123                shinfo->tx_flags &= ~tsflags;
1124                shinfo2->tx_flags |= tsflags;
1125                swap(shinfo->tskey, shinfo2->tskey);
1126        }
1127}
1128
1129/* Function to create two new TCP segments.  Shrinks the given segment
1130 * to the specified size and appends a new segment with the rest of the
1131 * packet to the list.  This won't be called frequently, I hope.
1132 * Remember, these are still headerless SKBs at this point.
1133 */
1134int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len,
1135                 unsigned int mss_now, gfp_t gfp)
1136{
1137        struct tcp_sock *tp = tcp_sk(sk);
1138        struct sk_buff *buff;
1139        int nsize, old_factor;
1140        int nlen;
1141        u8 flags;
1142
1143        if (WARN_ON(len > skb->len))
1144                return -EINVAL;
1145
1146        nsize = skb_headlen(skb) - len;
1147        if (nsize < 0)
1148                nsize = 0;
1149
1150        if (skb_unclone(skb, gfp))
1151                return -ENOMEM;
1152
1153        /* Get a new skb... force flag on. */
1154        buff = sk_stream_alloc_skb(sk, nsize, gfp);
1155        if (buff == NULL)
1156                return -ENOMEM; /* We'll just try again later. */
1157
1158        sk->sk_wmem_queued += buff->truesize;
1159        sk_mem_charge(sk, buff->truesize);
1160        nlen = skb->len - len - nsize;
1161        buff->truesize += nlen;
1162        skb->truesize -= nlen;
1163
1164        /* Correct the sequence numbers. */
1165        TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1166        TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1167        TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1168
1169        /* PSH and FIN should only be set in the second packet. */
1170        flags = TCP_SKB_CB(skb)->tcp_flags;
1171        TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
1172        TCP_SKB_CB(buff)->tcp_flags = flags;
1173        TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked;
1174
1175        if (!skb_shinfo(skb)->nr_frags && skb->ip_summed != CHECKSUM_PARTIAL) {
1176                /* Copy and checksum data tail into the new buffer. */
1177                buff->csum = csum_partial_copy_nocheck(skb->data + len,
1178                                                       skb_put(buff, nsize),
1179                                                       nsize, 0);
1180
1181                skb_trim(skb, len);
1182
1183                skb->csum = csum_block_sub(skb->csum, buff->csum, len);
1184        } else {
1185                skb->ip_summed = CHECKSUM_PARTIAL;
1186                skb_split(skb, buff, len);
1187        }
1188
1189        buff->ip_summed = skb->ip_summed;
1190
1191        buff->tstamp = skb->tstamp;
1192        tcp_fragment_tstamp(skb, buff);
1193
1194        old_factor = tcp_skb_pcount(skb);
1195
1196        /* Fix up tso_factor for both original and new SKB.  */
1197        tcp_set_skb_tso_segs(sk, skb, mss_now);
1198        tcp_set_skb_tso_segs(sk, buff, mss_now);
1199
1200        /* If this packet has been sent out already, we must
1201         * adjust the various packet counters.
1202         */
1203        if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) {
1204                int diff = old_factor - tcp_skb_pcount(skb) -
1205                        tcp_skb_pcount(buff);
1206
1207                if (diff)
1208                        tcp_adjust_pcount(sk, skb, diff);
1209        }
1210
1211        /* Link BUFF into the send queue. */
1212        __skb_header_release(buff);
1213        tcp_insert_write_queue_after(skb, buff, sk);
1214
1215        return 0;
1216}
1217
1218/* This is similar to __pskb_pull_head() (it will go to core/skbuff.c
1219 * eventually). The difference is that pulled data not copied, but
1220 * immediately discarded.
1221 */
1222static void __pskb_trim_head(struct sk_buff *skb, int len)
1223{
1224        struct skb_shared_info *shinfo;
1225        int i, k, eat;
1226
1227        eat = min_t(int, len, skb_headlen(skb));
1228        if (eat) {
1229                __skb_pull(skb, eat);
1230                len -= eat;
1231                if (!len)
1232                        return;
1233        }
1234        eat = len;
1235        k = 0;
1236        shinfo = skb_shinfo(skb);
1237        for (i = 0; i < shinfo->nr_frags; i++) {
1238                int size = skb_frag_size(&shinfo->frags[i]);
1239
1240                if (size <= eat) {
1241                        skb_frag_unref(skb, i);
1242                        eat -= size;
1243                } else {
1244                        shinfo->frags[k] = shinfo->frags[i];
1245                        if (eat) {
1246                                shinfo->frags[k].page_offset += eat;
1247                                skb_frag_size_sub(&shinfo->frags[k], eat);
1248                                eat = 0;
1249                        }
1250                        k++;
1251                }
1252        }
1253        shinfo->nr_frags = k;
1254
1255        skb_reset_tail_pointer(skb);
1256        skb->data_len -= len;
1257        skb->len = skb->data_len;
1258}
1259
1260/* Remove acked data from a packet in the transmit queue. */
1261int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
1262{
1263        if (skb_unclone(skb, GFP_ATOMIC))
1264                return -ENOMEM;
1265
1266        __pskb_trim_head(skb, len);
1267
1268        TCP_SKB_CB(skb)->seq += len;
1269        skb->ip_summed = CHECKSUM_PARTIAL;
1270
1271        skb->truesize        -= len;
1272        sk->sk_wmem_queued   -= len;
1273        sk_mem_uncharge(sk, len);
1274        sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1275
1276        /* Any change of skb->len requires recalculation of tso factor. */
1277        if (tcp_skb_pcount(skb) > 1)
1278                tcp_set_skb_tso_segs(sk, skb, tcp_skb_mss(skb));
1279
1280        return 0;
1281}
1282
1283/* Calculate MSS not accounting any TCP options.  */
1284static inline int __tcp_mtu_to_mss(struct sock *sk, int pmtu)
1285{
1286        const struct tcp_sock *tp = tcp_sk(sk);
1287        const struct inet_connection_sock *icsk = inet_csk(sk);
1288        int mss_now;
1289
1290        /* Calculate base mss without TCP options:
1291           It is MMS_S - sizeof(tcphdr) of rfc1122
1292         */
1293        mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr);
1294
1295        /* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */
1296        if (icsk->icsk_af_ops->net_frag_header_len) {
1297                const struct dst_entry *dst = __sk_dst_get(sk);
1298
1299                if (dst && dst_allfrag(dst))
1300                        mss_now -= icsk->icsk_af_ops->net_frag_header_len;
1301        }
1302
1303        /* Clamp it (mss_clamp does not include tcp options) */
1304        if (mss_now > tp->rx_opt.mss_clamp)
1305                mss_now = tp->rx_opt.mss_clamp;
1306
1307        /* Now subtract optional transport overhead */
1308        mss_now -= icsk->icsk_ext_hdr_len;
1309
1310        /* Then reserve room for full set of TCP options and 8 bytes of data */
1311        if (mss_now < 48)
1312                mss_now = 48;
1313        return mss_now;
1314}
1315
1316/* Calculate MSS. Not accounting for SACKs here.  */
1317int tcp_mtu_to_mss(struct sock *sk, int pmtu)
1318{
1319        /* Subtract TCP options size, not including SACKs */
1320        return __tcp_mtu_to_mss(sk, pmtu) -
1321               (tcp_sk(sk)->tcp_header_len - sizeof(struct tcphdr));
1322}
1323
1324/* Inverse of above */
1325int tcp_mss_to_mtu(struct sock *sk, int mss)
1326{
1327        const struct tcp_sock *tp = tcp_sk(sk);
1328        const struct inet_connection_sock *icsk = inet_csk(sk);
1329        int mtu;
1330
1331        mtu = mss +
1332              tp->tcp_header_len +
1333              icsk->icsk_ext_hdr_len +
1334              icsk->icsk_af_ops->net_header_len;
1335
1336        /* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */
1337        if (icsk->icsk_af_ops->net_frag_header_len) {
1338                const struct dst_entry *dst = __sk_dst_get(sk);
1339
1340                if (dst && dst_allfrag(dst))
1341                        mtu += icsk->icsk_af_ops->net_frag_header_len;
1342        }
1343        return mtu;
1344}
1345
1346/* MTU probing init per socket */
1347void tcp_mtup_init(struct sock *sk)
1348{
1349        struct tcp_sock *tp = tcp_sk(sk);
1350        struct inet_connection_sock *icsk = inet_csk(sk);
1351
1352        icsk->icsk_mtup.enabled = sysctl_tcp_mtu_probing > 1;
1353        icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) +
1354                               icsk->icsk_af_ops->net_header_len;
1355        icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, sysctl_tcp_base_mss);
1356        icsk->icsk_mtup.probe_size = 0;
1357}
1358EXPORT_SYMBOL(tcp_mtup_init);
1359
1360/* This function synchronize snd mss to current pmtu/exthdr set.
1361
1362   tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts
1363   for TCP options, but includes only bare TCP header.
1364
1365   tp->rx_opt.mss_clamp is mss negotiated at connection setup.
1366   It is minimum of user_mss and mss received with SYN.
1367   It also does not include TCP options.
1368
1369   inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function.
1370
1371   tp->mss_cache is current effective sending mss, including
1372   all tcp options except for SACKs. It is evaluated,
1373   taking into account current pmtu, but never exceeds
1374   tp->rx_opt.mss_clamp.
1375
1376   NOTE1. rfc1122 clearly states that advertised MSS
1377   DOES NOT include either tcp or ip options.
1378
1379   NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache
1380   are READ ONLY outside this function.         --ANK (980731)
1381 */
1382unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu)
1383{
1384        struct tcp_sock *tp = tcp_sk(sk);
1385        struct inet_connection_sock *icsk = inet_csk(sk);
1386        int mss_now;
1387
1388        if (icsk->icsk_mtup.search_high > pmtu)
1389                icsk->icsk_mtup.search_high = pmtu;
1390
1391        mss_now = tcp_mtu_to_mss(sk, pmtu);
1392        mss_now = tcp_bound_to_half_wnd(tp, mss_now);
1393
1394        /* And store cached results */
1395        icsk->icsk_pmtu_cookie = pmtu;
1396        if (icsk->icsk_mtup.enabled)
1397                mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low));
1398        tp->mss_cache = mss_now;
1399
1400        return mss_now;
1401}
1402EXPORT_SYMBOL(tcp_sync_mss);
1403
1404/* Compute the current effective MSS, taking SACKs and IP options,
1405 * and even PMTU discovery events into account.
1406 */
1407unsigned int tcp_current_mss(struct sock *sk)
1408{
1409        const struct tcp_sock *tp = tcp_sk(sk);
1410        const struct dst_entry *dst = __sk_dst_get(sk);
1411        u32 mss_now;
1412        unsigned int header_len;
1413        struct tcp_out_options opts;
1414        struct tcp_md5sig_key *md5;
1415
1416        mss_now = tp->mss_cache;
1417
1418        if (dst) {
1419                u32 mtu = dst_mtu(dst);
1420                if (mtu != inet_csk(sk)->icsk_pmtu_cookie)
1421                        mss_now = tcp_sync_mss(sk, mtu);
1422        }
1423
1424        header_len = tcp_established_options(sk, NULL, &opts, &md5) +
1425                     sizeof(struct tcphdr);
1426        /* The mss_cache is sized based on tp->tcp_header_len, which assumes
1427         * some common options. If this is an odd packet (because we have SACK
1428         * blocks etc) then our calculated header_len will be different, and
1429         * we have to adjust mss_now correspondingly */
1430        if (header_len != tp->tcp_header_len) {
1431                int delta = (int) header_len - tp->tcp_header_len;
1432                mss_now -= delta;
1433        }
1434
1435        return mss_now;
1436}
1437
1438/* RFC2861, slow part. Adjust cwnd, after it was not full during one rto.
1439 * As additional protections, we do not touch cwnd in retransmission phases,
1440 * and if application hit its sndbuf limit recently.
1441 */
1442static void tcp_cwnd_application_limited(struct sock *sk)
1443{
1444        struct tcp_sock *tp = tcp_sk(sk);
1445
1446        if (inet_csk(sk)->icsk_ca_state == TCP_CA_Open &&
1447            sk->sk_socket && !test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
1448                /* Limited by application or receiver window. */
1449                u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk));
1450                u32 win_used = max(tp->snd_cwnd_used, init_win);
1451                if (win_used < tp->snd_cwnd) {
1452                        tp->snd_ssthresh = tcp_current_ssthresh(sk);
1453                        tp->snd_cwnd = (tp->snd_cwnd + win_used) >> 1;
1454                }
1455                tp->snd_cwnd_used = 0;
1456        }
1457        tp->snd_cwnd_stamp = tcp_time_stamp;
1458}
1459
1460static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited)
1461{
1462        struct tcp_sock *tp = tcp_sk(sk);
1463
1464        /* Track the maximum number of outstanding packets in each
1465         * window, and remember whether we were cwnd-limited then.
1466         */
1467        if (!before(tp->snd_una, tp->max_packets_seq) ||
1468            tp->packets_out > tp->max_packets_out) {
1469                tp->max_packets_out = tp->packets_out;
1470                tp->max_packets_seq = tp->snd_nxt;
1471                tp->is_cwnd_limited = is_cwnd_limited;
1472        }
1473
1474        if (tcp_is_cwnd_limited(sk)) {
1475                /* Network is feed fully. */
1476                tp->snd_cwnd_used = 0;
1477                tp->snd_cwnd_stamp = tcp_time_stamp;
1478        } else {
1479                /* Network starves. */
1480                if (tp->packets_out > tp->snd_cwnd_used)
1481                        tp->snd_cwnd_used = tp->packets_out;
1482
1483                if (sysctl_tcp_slow_start_after_idle &&
1484                    (s32)(tcp_time_stamp - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto)
1485                        tcp_cwnd_application_limited(sk);
1486        }
1487}
1488
1489/* Minshall's variant of the Nagle send check. */
1490static bool tcp_minshall_check(const struct tcp_sock *tp)
1491{
1492        return after(tp->snd_sml, tp->snd_una) &&
1493                !after(tp->snd_sml, tp->snd_nxt);
1494}
1495
1496/* Update snd_sml if this skb is under mss
1497 * Note that a TSO packet might end with a sub-mss segment
1498 * The test is really :
1499 * if ((skb->len % mss) != 0)
1500 *        tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
1501 * But we can avoid doing the divide again given we already have
1502 *  skb_pcount = skb->len / mss_now
1503 */
1504static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now,
1505                                const struct sk_buff *skb)
1506{
1507        if (skb->len < tcp_skb_pcount(skb) * mss_now)
1508                tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
1509}
1510
1511/* Return false, if packet can be sent now without violation Nagle's rules:
1512 * 1. It is full sized. (provided by caller in %partial bool)
1513 * 2. Or it contains FIN. (already checked by caller)
1514 * 3. Or TCP_CORK is not set, and TCP_NODELAY is set.
1515 * 4. Or TCP_CORK is not set, and all sent packets are ACKed.
1516 *    With Minshall's modification: all sent small packets are ACKed.
1517 */
1518static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp,
1519                            int nonagle)
1520{
1521        return partial &&
1522                ((nonagle & TCP_NAGLE_CORK) ||
1523                 (!nonagle && tp->packets_out && tcp_minshall_check(tp)));
1524}
1525/* Returns the portion of skb which can be sent right away */
1526static unsigned int tcp_mss_split_point(const struct sock *sk,
1527                                        const struct sk_buff *skb,
1528                                        unsigned int mss_now,
1529                                        unsigned int max_segs,
1530                                        int nonagle)
1531{
1532        const struct tcp_sock *tp = tcp_sk(sk);
1533        u32 partial, needed, window, max_len;
1534
1535        window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
1536        max_len = mss_now * max_segs;
1537
1538        if (likely(max_len <= window && skb != tcp_write_queue_tail(sk)))
1539                return max_len;
1540
1541        needed = min(skb->len, window);
1542
1543        if (max_len <= needed)
1544                return max_len;
1545
1546        partial = needed % mss_now;
1547        /* If last segment is not a full MSS, check if Nagle rules allow us
1548         * to include this last segment in this skb.
1549         * Otherwise, we'll split the skb at last MSS boundary
1550         */
1551        if (tcp_nagle_check(partial != 0, tp, nonagle))
1552                return needed - partial;
1553
1554        return needed;
1555}
1556
1557/* Can at least one segment of SKB be sent right now, according to the
1558 * congestion window rules?  If so, return how many segments are allowed.
1559 */
1560static inline unsigned int tcp_cwnd_test(const struct tcp_sock *tp,
1561                                         const struct sk_buff *skb)
1562{
1563        u32 in_flight, cwnd;
1564
1565        /* Don't be strict about the congestion window for the final FIN.  */
1566        if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) &&
1567            tcp_skb_pcount(skb) == 1)
1568                return 1;
1569
1570        in_flight = tcp_packets_in_flight(tp);
1571        cwnd = tp->snd_cwnd;
1572        if (in_flight < cwnd)
1573                return (cwnd - in_flight);
1574
1575        return 0;
1576}
1577
1578/* Initialize TSO state of a skb.
1579 * This must be invoked the first time we consider transmitting
1580 * SKB onto the wire.
1581 */
1582static int tcp_init_tso_segs(const struct sock *sk, struct sk_buff *skb,
1583                             unsigned int mss_now)
1584{
1585        int tso_segs = tcp_skb_pcount(skb);
1586
1587        if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) {
1588                tcp_set_skb_tso_segs(sk, skb, mss_now);
1589                tso_segs = tcp_skb_pcount(skb);
1590        }
1591        return tso_segs;
1592}
1593
1594
1595/* Return true if the Nagle test allows this packet to be
1596 * sent now.
1597 */
1598static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb,
1599                                  unsigned int cur_mss, int nonagle)
1600{
1601        /* Nagle rule does not apply to frames, which sit in the middle of the
1602         * write_queue (they have no chances to get new data).
1603         *
1604         * This is implemented in the callers, where they modify the 'nonagle'
1605         * argument based upon the location of SKB in the send queue.
1606         */
1607        if (nonagle & TCP_NAGLE_PUSH)
1608                return true;
1609
1610        /* Don't use the nagle rule for urgent data (or for the final FIN). */
1611        if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
1612                return true;
1613
1614        if (!tcp_nagle_check(skb->len < cur_mss, tp, nonagle))
1615                return true;
1616
1617        return false;
1618}
1619
1620/* Does at least the first segment of SKB fit into the send window? */
1621static bool tcp_snd_wnd_test(const struct tcp_sock *tp,
1622                             const struct sk_buff *skb,
1623                             unsigned int cur_mss)
1624{
1625        u32 end_seq = TCP_SKB_CB(skb)->end_seq;
1626
1627        if (skb->len > cur_mss)
1628                end_seq = TCP_SKB_CB(skb)->seq + cur_mss;
1629
1630        return !after(end_seq, tcp_wnd_end(tp));
1631}
1632
1633/* This checks if the data bearing packet SKB (usually tcp_send_head(sk))
1634 * should be put on the wire right now.  If so, it returns the number of
1635 * packets allowed by the congestion window.
1636 */
1637static unsigned int tcp_snd_test(const struct sock *sk, struct sk_buff *skb,
1638                                 unsigned int cur_mss, int nonagle)
1639{
1640        const struct tcp_sock *tp = tcp_sk(sk);
1641        unsigned int cwnd_quota;
1642
1643        tcp_init_tso_segs(sk, skb, cur_mss);
1644
1645        if (!tcp_nagle_test(tp, skb, cur_mss, nonagle))
1646                return 0;
1647
1648        cwnd_quota = tcp_cwnd_test(tp, skb);
1649        if (cwnd_quota && !tcp_snd_wnd_test(tp, skb, cur_mss))
1650                cwnd_quota = 0;
1651
1652        return cwnd_quota;
1653}
1654
1655/* Test if sending is allowed right now. */
1656bool tcp_may_send_now(struct sock *sk)
1657{
1658        const struct tcp_sock *tp = tcp_sk(sk);
1659        struct sk_buff *skb = tcp_send_head(sk);
1660
1661        return skb &&
1662                tcp_snd_test(sk, skb, tcp_current_mss(sk),
1663                             (tcp_skb_is_last(sk, skb) ?
1664                              tp->nonagle : TCP_NAGLE_PUSH));
1665}
1666
1667/* Trim TSO SKB to LEN bytes, put the remaining data into a new packet
1668 * which is put after SKB on the list.  It is very much like
1669 * tcp_fragment() except that it may make several kinds of assumptions
1670 * in order to speed up the splitting operation.  In particular, we
1671 * know that all the data is in scatter-gather pages, and that the
1672 * packet has never been sent out before (and thus is not cloned).
1673 */
1674static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
1675                        unsigned int mss_now, gfp_t gfp)
1676{
1677        struct sk_buff *buff;
1678        int nlen = skb->len - len;
1679        u8 flags;
1680
1681        /* All of a TSO frame must be composed of paged data.  */
1682        if (skb->len != skb->data_len)
1683                return tcp_fragment(sk, skb, len, mss_now, gfp);
1684
1685        buff = sk_stream_alloc_skb(sk, 0, gfp);
1686        if (unlikely(buff == NULL))
1687                return -ENOMEM;
1688
1689        sk->sk_wmem_queued += buff->truesize;
1690        sk_mem_charge(sk, buff->truesize);
1691        buff->truesize += nlen;
1692        skb->truesize -= nlen;
1693
1694        /* Correct the sequence numbers. */
1695        TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1696        TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1697        TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1698
1699        /* PSH and FIN should only be set in the second packet. */
1700        flags = TCP_SKB_CB(skb)->tcp_flags;
1701        TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
1702        TCP_SKB_CB(buff)->tcp_flags = flags;
1703
1704        /* This packet was never sent out yet, so no SACK bits. */
1705        TCP_SKB_CB(buff)->sacked = 0;
1706
1707        buff->ip_summed = skb->ip_summed = CHECKSUM_PARTIAL;
1708        skb_split(skb, buff, len);
1709        tcp_fragment_tstamp(skb, buff);
1710
1711        /* Fix up tso_factor for both original and new SKB.  */
1712        tcp_set_skb_tso_segs(sk, skb, mss_now);
1713        tcp_set_skb_tso_segs(sk, buff, mss_now);
1714
1715        /* Link BUFF into the send queue. */
1716        __skb_header_release(buff);
1717        tcp_insert_write_queue_after(skb, buff, sk);
1718
1719        return 0;
1720}
1721
1722/* Try to defer sending, if possible, in order to minimize the amount
1723 * of TSO splitting we do.  View it as a kind of TSO Nagle test.
1724 *
1725 * This algorithm is from John Heffner.
1726 */
1727static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb,
1728                                 bool *is_cwnd_limited)
1729{
1730        struct tcp_sock *tp = tcp_sk(sk);
1731        const struct inet_connection_sock *icsk = inet_csk(sk);
1732        u32 send_win, cong_win, limit, in_flight;
1733        int win_divisor;
1734
1735        if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
1736                goto send_now;
1737
1738        if (icsk->icsk_ca_state != TCP_CA_Open)
1739                goto send_now;
1740
1741        /* Defer for less than two clock ticks. */
1742        if (tp->tso_deferred &&
1743            (((u32)jiffies << 1) >> 1) - (tp->tso_deferred >> 1) > 1)
1744                goto send_now;
1745
1746        in_flight = tcp_packets_in_flight(tp);
1747
1748        BUG_ON(tcp_skb_pcount(skb) <= 1 || (tp->snd_cwnd <= in_flight));
1749
1750        send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
1751
1752        /* From in_flight test above, we know that cwnd > in_flight.  */
1753        cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache;
1754
1755        limit = min(send_win, cong_win);
1756
1757        /* If a full-sized TSO skb can be sent, do it. */
1758        if (limit >= min_t(unsigned int, sk->sk_gso_max_size,
1759                           tp->xmit_size_goal_segs * tp->mss_cache))
1760                goto send_now;
1761
1762        /* Middle in queue won't get any more data, full sendable already? */
1763        if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len))
1764                goto send_now;
1765
1766        win_divisor = ACCESS_ONCE(sysctl_tcp_tso_win_divisor);
1767        if (win_divisor) {
1768                u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache);
1769
1770                /* If at least some fraction of a window is available,
1771                 * just use it.
1772                 */
1773                chunk /= win_divisor;
1774                if (limit >= chunk)
1775                        goto send_now;
1776        } else {
1777                /* Different approach, try not to defer past a single
1778                 * ACK.  Receiver should ACK every other full sized
1779                 * frame, so if we have space for more than 3 frames
1780                 * then send now.
1781                 */
1782                if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache)
1783                        goto send_now;
1784        }
1785
1786        /* Ok, it looks like it is advisable to defer.
1787         * Do not rearm the timer if already set to not break TCP ACK clocking.
1788         */
1789        if (!tp->tso_deferred)
1790                tp->tso_deferred = 1 | (jiffies << 1);
1791
1792        if (cong_win < send_win && cong_win < skb->len)
1793                *is_cwnd_limited = true;
1794
1795        return true;
1796
1797send_now:
1798        tp->tso_deferred = 0;
1799        return false;
1800}
1801
1802/* Create a new MTU probe if we are ready.
1803 * MTU probe is regularly attempting to increase the path MTU by
1804 * deliberately sending larger packets.  This discovers routing
1805 * changes resulting in larger path MTUs.
1806 *
1807 * Returns 0 if we should wait to probe (no cwnd available),
1808 *         1 if a probe was sent,
1809 *         -1 otherwise
1810 */
1811static int tcp_mtu_probe(struct sock *sk)
1812{
1813        struct tcp_sock *tp = tcp_sk(sk);
1814        struct inet_connection_sock *icsk = inet_csk(sk);
1815        struct sk_buff *skb, *nskb, *next;
1816        int len;
1817        int probe_size;
1818        int size_needed;
1819        int copy;
1820        int mss_now;
1821
1822        /* Not currently probing/verifying,
1823         * not in recovery,
1824         * have enough cwnd, and
1825         * not SACKing (the variable headers throw things off) */
1826        if (!icsk->icsk_mtup.enabled ||
1827            icsk->icsk_mtup.probe_size ||
1828            inet_csk(sk)->icsk_ca_state != TCP_CA_Open ||
1829            tp->snd_cwnd < 11 ||
1830            tp->rx_opt.num_sacks || tp->rx_opt.dsack)
1831                return -1;
1832
1833        /* Very simple search strategy: just double the MSS. */
1834        mss_now = tcp_current_mss(sk);
1835        probe_size = 2 * tp->mss_cache;
1836        size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache;
1837        if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high)) {
1838                /* TODO: set timer for probe_converge_event */
1839                return -1;
1840        }
1841
1842        /* Have enough data in the send queue to probe? */
1843        if (tp->write_seq - tp->snd_nxt < size_needed)
1844                return -1;
1845
1846        if (tp->snd_wnd < size_needed)
1847                return -1;
1848        if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp)))
1849                return 0;
1850
1851        /* Do we need to wait to drain cwnd? With none in flight, don't stall */
1852        if (tcp_packets_in_flight(tp) + 2 > tp->snd_cwnd) {
1853                if (!tcp_packets_in_flight(tp))
1854                        return -1;
1855                else
1856                        return 0;
1857        }
1858
1859        /* We're allowed to probe.  Build it now. */
1860        if ((nskb = sk_stream_alloc_skb(sk, probe_size, GFP_ATOMIC)) == NULL)
1861                return -1;
1862        sk->sk_wmem_queued += nskb->truesize;
1863        sk_mem_charge(sk, nskb->truesize);
1864
1865        skb = tcp_send_head(sk);
1866
1867        TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq;
1868        TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size;
1869        TCP_SKB_CB(nskb)->tcp_flags = TCPHDR_ACK;
1870        TCP_SKB_CB(nskb)->sacked = 0;
1871        nskb->csum = 0;
1872        nskb->ip_summed = skb->ip_summed;
1873
1874        tcp_insert_write_queue_before(nskb, skb, sk);
1875
1876        len = 0;
1877        tcp_for_write_queue_from_safe(skb, next, sk) {
1878                copy = min_t(int, skb->len, probe_size - len);
1879                if (nskb->ip_summed) {
1880                        skb_copy_bits(skb, 0, skb_put(nskb, copy), copy);
1881                } else {
1882                        __wsum csum = skb_copy_and_csum_bits(skb, 0,
1883                                                             skb_put(nskb, copy),
1884                                                             copy, 0);
1885                        nskb->csum = csum_block_add(nskb->csum, csum, len);
1886                }
1887
1888                if (skb->len <= copy) {
1889                        /* We've eaten all the data from this skb.
1890                         * Throw it away. */
1891                        TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
1892                        tcp_unlink_write_queue(skb, sk);
1893                        sk_wmem_free_skb(sk, skb);
1894                } else {
1895                        TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags &
1896                                                   ~(TCPHDR_FIN|TCPHDR_PSH);
1897                        if (!skb_shinfo(skb)->nr_frags) {
1898                                skb_pull(skb, copy);
1899                                if (skb->ip_summed != CHECKSUM_PARTIAL)
1900                                        skb->csum = csum_partial(skb->data,
1901                                                                 skb->len, 0);
1902                        } else {
1903                                __pskb_trim_head(skb, copy);
1904                                tcp_set_skb_tso_segs(sk, skb, mss_now);
1905                        }
1906                        TCP_SKB_CB(skb)->seq += copy;
1907                }
1908
1909                len += copy;
1910
1911                if (len >= probe_size)
1912                        break;
1913        }
1914        tcp_init_tso_segs(sk, nskb, nskb->len);
1915
1916        /* We're ready to send.  If this fails, the probe will
1917         * be resegmented into mss-sized pieces by tcp_write_xmit().
1918         */
1919        if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) {
1920                /* Decrement cwnd here because we are sending
1921                 * effectively two packets. */
1922                tp->snd_cwnd--;
1923                tcp_event_new_data_sent(sk, nskb);
1924
1925                icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len);
1926                tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq;
1927                tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq;
1928
1929                return 1;
1930        }
1931
1932        return -1;
1933}
1934
1935/* This routine writes packets to the network.  It advances the
1936 * send_head.  This happens as incoming acks open up the remote
1937 * window for us.
1938 *
1939 * LARGESEND note: !tcp_urg_mode is overkill, only frames between
1940 * snd_up-64k-mss .. snd_up cannot be large. However, taking into
1941 * account rare use of URG, this is not a big flaw.
1942 *
1943 * Send at most one packet when push_one > 0. Temporarily ignore
1944 * cwnd limit to force at most one packet out when push_one == 2.
1945
1946 * Returns true, if no segments are in flight and we have queued segments,
1947 * but cannot send anything now because of SWS or another problem.
1948 */
1949static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
1950                           int push_one, gfp_t gfp)
1951{
1952        struct tcp_sock *tp = tcp_sk(sk);
1953        struct sk_buff *skb;
1954        unsigned int tso_segs, sent_pkts;
1955        int cwnd_quota;
1956        int result;
1957        bool is_cwnd_limited = false;
1958
1959        sent_pkts = 0;
1960
1961        if (!push_one) {
1962                /* Do MTU probing. */
1963                result = tcp_mtu_probe(sk);
1964                if (!result) {
1965                        return false;
1966                } else if (result > 0) {
1967                        sent_pkts = 1;
1968                }
1969        }
1970
1971        while ((skb = tcp_send_head(sk))) {
1972                unsigned int limit;
1973
1974                tso_segs = tcp_init_tso_segs(sk, skb, mss_now);
1975                BUG_ON(!tso_segs);
1976
1977                if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE) {
1978                        /* "skb_mstamp" is used as a start point for the retransmit timer */
1979                        skb_mstamp_get(&skb->skb_mstamp);
1980                        goto repair; /* Skip network transmission */
1981                }
1982
1983                cwnd_quota = tcp_cwnd_test(tp, skb);
1984                if (!cwnd_quota) {
1985                        is_cwnd_limited = true;
1986                        if (push_one == 2)
1987                                /* Force out a loss probe pkt. */
1988                                cwnd_quota = 1;
1989                        else
1990                                break;
1991                }
1992
1993                if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now)))
1994                        break;
1995
1996                if (tso_segs == 1 || !sk->sk_gso_max_segs) {
1997                        if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
1998                                                     (tcp_skb_is_last(sk, skb) ?
1999                                                      nonagle : TCP_NAGLE_PUSH))))
2000                                break;
2001                } else {
2002                        if (!push_one &&
2003                            tcp_tso_should_defer(sk, skb, &is_cwnd_limited))
2004                                break;
2005                }
2006
2007                /* TCP Small Queues :
2008                 * Control number of packets in qdisc/devices to two packets / or ~1 ms.
2009                 * This allows for :
2010                 *  - better RTT estimation and ACK scheduling
2011                 *  - faster recovery
2012                 *  - high rates
2013                 * Alas, some drivers / subsystems require a fair amount
2014                 * of queued bytes to ensure line rate.
2015                 * One example is wifi aggregation (802.11 AMPDU)
2016                 */
2017                limit = max_t(unsigned int, sysctl_tcp_limit_output_bytes,
2018                              sk->sk_pacing_rate >> 10);
2019
2020                if (atomic_read(&sk->sk_wmem_alloc) > limit) {
2021                        set_bit(TSQ_THROTTLED, &tp->tsq_flags);
2022                        /* It is possible TX completion already happened
2023                         * before we set TSQ_THROTTLED, so we must
2024                         * test again the condition.
2025                         */
2026                        smp_mb__after_atomic();
2027                        if (atomic_read(&sk->sk_wmem_alloc) > limit)
2028                                break;
2029                }
2030
2031                limit = mss_now;
2032                if (tso_segs > 1 && sk->sk_gso_max_segs && !tcp_urg_mode(tp))
2033                        limit = tcp_mss_split_point(sk, skb, mss_now,
2034                                                    min_t(unsigned int,
2035                                                          cwnd_quota,
2036                                                          sk->sk_gso_max_segs),
2037                                                    nonagle);
2038
2039                if (skb->len > limit &&
2040                    unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
2041                        break;
2042
2043                if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
2044                        break;
2045
2046repair:
2047                /* Advance the send_head.  This one is sent out.
2048                 * This call will increment packets_out.
2049                 */
2050                tcp_event_new_data_sent(sk, skb);
2051
2052                tcp_minshall_update(tp, mss_now, skb);
2053                sent_pkts += tcp_skb_pcount(skb);
2054
2055                if (push_one)
2056                        break;
2057        }
2058
2059        if (likely(sent_pkts)) {
2060                if (tcp_in_cwnd_reduction(sk))
2061                        tp->prr_out += sent_pkts;
2062
2063                /* Send one loss probe per tail loss episode. */
2064                if (push_one != 2)
2065                        tcp_schedule_loss_probe(sk);
2066                tcp_cwnd_validate(sk, is_cwnd_limited);
2067                return false;
2068        }
2069        return (push_one == 2) || (!tp->packets_out && tcp_send_head(sk));
2070}
2071
2072bool tcp_schedule_loss_probe(struct sock *sk)
2073{
2074        struct inet_connection_sock *icsk = inet_csk(sk);
2075        struct tcp_sock *tp = tcp_sk(sk);
2076        u32 timeout, tlp_time_stamp, rto_time_stamp;
2077        u32 rtt = usecs_to_jiffies(tp->srtt_us >> 3);
2078
2079        if (WARN_ON(icsk->icsk_pending == ICSK_TIME_EARLY_RETRANS))
2080                return false;
2081        /* No consecutive loss probes. */
2082        if (WARN_ON(icsk->icsk_pending == ICSK_TIME_LOSS_PROBE)) {
2083                tcp_rearm_rto(sk);
2084                return false;
2085        }
2086        /* Don't do any loss probe on a Fast Open connection before 3WHS
2087         * finishes.
2088         */
2089        if (sk->sk_state == TCP_SYN_RECV)
2090                return false;
2091
2092        /* TLP is only scheduled when next timer event is RTO. */
2093        if (icsk->icsk_pending != ICSK_TIME_RETRANS)
2094                return false;
2095
2096        /* Schedule a loss probe in 2*RTT for SACK capable connections
2097         * in Open state, that are either limited by cwnd or application.
2098         */
2099        if (sysctl_tcp_early_retrans < 3 || !tp->srtt_us || !tp->packets_out ||
2100            !tcp_is_sack(tp) || inet_csk(sk)->icsk_ca_state != TCP_CA_Open)
2101                return false;
2102
2103        if ((tp->snd_cwnd > tcp_packets_in_flight(tp)) &&
2104             tcp_send_head(sk))
2105                return false;
2106
2107        /* Probe timeout is at least 1.5*rtt + TCP_DELACK_MAX to account
2108         * for delayed ack when there's one outstanding packet.
2109         */
2110        timeout = rtt << 1;
2111        if (tp->packets_out == 1)
2112                timeout = max_t(u32, timeout,
2113                                (rtt + (rtt >> 1) + TCP_DELACK_MAX));
2114        timeout = max_t(u32, timeout, msecs_to_jiffies(10));
2115
2116        /* If RTO is shorter, just schedule TLP in its place. */
2117        tlp_time_stamp = tcp_time_stamp + timeout;
2118        rto_time_stamp = (u32)inet_csk(sk)->icsk_timeout;
2119        if ((s32)(tlp_time_stamp - rto_time_stamp) > 0) {
2120                s32 delta = rto_time_stamp - tcp_time_stamp;
2121                if (delta > 0)
2122                        timeout = delta;
2123        }
2124
2125        inet_csk_reset_xmit_timer(sk, ICSK_TIME_LOSS_PROBE, timeout,
2126                                  TCP_RTO_MAX);
2127        return true;
2128}
2129
2130/* Thanks to skb fast clones, we can detect if a prior transmit of
2131 * a packet is still in a qdisc or driver queue.
2132 * In this case, there is very little point doing a retransmit !
2133 * Note: This is called from BH context only.
2134 */
2135static bool skb_still_in_host_queue(const struct sock *sk,
2136                                    const struct sk_buff *skb)
2137{
2138        if (unlikely(skb_fclone_busy(sk, skb))) {
2139                NET_INC_STATS_BH(sock_net(sk),
2140                                 LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES);
2141                return true;
2142        }
2143        return false;
2144}
2145
2146/* When probe timeout (PTO) fires, send a new segment if one exists, else
2147 * retransmit the last segment.
2148 */
2149void tcp_send_loss_probe(struct sock *sk)
2150{
2151        struct tcp_sock *tp = tcp_sk(sk);
2152        struct sk_buff *skb;
2153        int pcount;
2154        int mss = tcp_current_mss(sk);
2155        int err = -1;
2156
2157        if (tcp_send_head(sk) != NULL) {
2158                err = tcp_write_xmit(sk, mss, TCP_NAGLE_OFF, 2, GFP_ATOMIC);
2159                goto rearm_timer;
2160        }
2161
2162        /* At most one outstanding TLP retransmission. */
2163        if (tp->tlp_high_seq)
2164                goto rearm_timer;
2165
2166        /* Retransmit last segment. */
2167        skb = tcp_write_queue_tail(sk);
2168        if (WARN_ON(!skb))
2169                goto rearm_timer;
2170
2171        if (skb_still_in_host_queue(sk, skb))
2172                goto rearm_timer;
2173
2174        pcount = tcp_skb_pcount(skb);
2175        if (WARN_ON(!pcount))
2176                goto rearm_timer;
2177
2178        if ((pcount > 1) && (skb->len > (pcount - 1) * mss)) {
2179                if (unlikely(tcp_fragment(sk, skb, (pcount - 1) * mss, mss,
2180                                          GFP_ATOMIC)))
2181                        goto rearm_timer;
2182                skb = tcp_write_queue_tail(sk);
2183        }
2184
2185        if (WARN_ON(!skb || !tcp_skb_pcount(skb)))
2186                goto rearm_timer;
2187
2188        err = __tcp_retransmit_skb(sk, skb);
2189
2190        /* Record snd_nxt for loss detection. */
2191        if (likely(!err))
2192                tp->tlp_high_seq = tp->snd_nxt;
2193
2194rearm_timer:
2195        inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
2196                                  inet_csk(sk)->icsk_rto,
2197                                  TCP_RTO_MAX);
2198
2199        if (likely(!err))
2200                NET_INC_STATS_BH(sock_net(sk),
2201                                 LINUX_MIB_TCPLOSSPROBES);
2202}
2203
2204/* Push out any pending frames which were held back due to
2205 * TCP_CORK or attempt at coalescing tiny packets.
2206 * The socket must be locked by the caller.
2207 */
2208void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
2209                               int nonagle)
2210{
2211        /* If we are closed, the bytes will have to remain here.
2212         * In time closedown will finish, we empty the write queue and
2213         * all will be happy.
2214         */
2215        if (unlikely(sk->sk_state == TCP_CLOSE))
2216                return;
2217
2218        if (tcp_write_xmit(sk, cur_mss, nonagle, 0,
2219                           sk_gfp_atomic(sk, GFP_ATOMIC)))
2220                tcp_check_probe_timer(sk);
2221}
2222
2223/* Send _single_ skb sitting at the send head. This function requires
2224 * true push pending frames to setup probe timer etc.
2225 */
2226void tcp_push_one(struct sock *sk, unsigned int mss_now)
2227{
2228        struct sk_buff *skb = tcp_send_head(sk);
2229
2230        BUG_ON(!skb || skb->len < mss_now);
2231
2232        tcp_write_xmit(sk, mss_now, TCP_NAGLE_PUSH, 1, sk->sk_allocation);
2233}
2234
2235/* This function returns the amount that we can raise the
2236 * usable window based on the following constraints
2237 *
2238 * 1. The window can never be shrunk once it is offered (RFC 793)
2239 * 2. We limit memory per socket
2240 *
2241 * RFC 1122:
2242 * "the suggested [SWS] avoidance algorithm for the receiver is to keep
2243 *  RECV.NEXT + RCV.WIN fixed until:
2244 *  RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)"
2245 *
2246 * i.e. don't raise the right edge of the window until you can raise
2247 * it at least MSS bytes.
2248 *
2249 * Unfortunately, the recommended algorithm breaks header prediction,
2250 * since header prediction assumes th->window stays fixed.
2251 *
2252 * Strictly speaking, keeping th->window fixed violates the receiver
2253 * side SWS prevention criteria. The problem is that under this rule
2254 * a stream of single byte packets will cause the right side of the
2255 * window to always advance by a single byte.
2256 *
2257 * Of course, if the sender implements sender side SWS prevention
2258 * then this will not be a problem.
2259 *
2260 * BSD seems to make the following compromise:
2261 *
2262 *      If the free space is less than the 1/4 of the maximum
2263 *      space available and the free space is less than 1/2 mss,
2264 *      then set the window to 0.
2265 *      [ Actually, bsd uses MSS and 1/4 of maximal _window_ ]
2266 *      Otherwise, just prevent the window from shrinking
2267 *      and from being larger than the largest representable value.
2268 *
2269 * This prevents incremental opening of the window in the regime
2270 * where TCP is limited by the speed of the reader side taking
2271 * data out of the TCP receive queue. It does nothing about
2272 * those cases where the window is constrained on the sender side
2273 * because the pipeline is full.
2274 *
2275 * BSD also seems to "accidentally" limit itself to windows that are a
2276 * multiple of MSS, at least until the free space gets quite small.
2277 * This would appear to be a side effect of the mbuf implementation.
2278 * Combining these two algorithms results in the observed behavior
2279 * of having a fixed window size at almost all times.
2280 *
2281 * Below we obtain similar behavior by forcing the offered window to
2282 * a multiple of the mss when it is feasible to do so.
2283 *
2284 * Note, we don't "adjust" for TIMESTAMP or SACK option bytes.
2285 * Regular options like TIMESTAMP are taken into account.
2286 */
2287u32 __tcp_select_window(struct sock *sk)
2288{
2289        struct inet_connection_sock *icsk = inet_csk(sk);
2290        struct tcp_sock *tp = tcp_sk(sk);
2291        /* MSS for the peer's data.  Previous versions used mss_clamp
2292         * here.  I don't know if the value based on our guesses
2293         * of peer's MSS is better for the performance.  It's more correct
2294         * but may be worse for the performance because of rcv_mss
2295         * fluctuations.  --SAW  1998/11/1
2296         */
2297        int mss = icsk->icsk_ack.rcv_mss;
2298        int free_space = tcp_space(sk);
2299        int allowed_space = tcp_full_space(sk);
2300        int full_space = min_t(int, tp->window_clamp, allowed_space);
2301        int window;
2302
2303        if (unlikely(mss > full_space)) {
2304                mss = full_space;
2305                if (mss <= 0)
2306                        return 0;
2307        }
2308        if (free_space < (full_space >> 1)) {
2309                icsk->icsk_ack.quick = 0;
2310
2311                if (sk_under_memory_pressure(sk))
2312                        tp->rcv_ssthresh = min(tp->rcv_ssthresh,
2313                                               4U * tp->advmss);
2314
2315                /* free_space might become our new window, make sure we don't
2316                 * increase it due to wscale.
2317                 */
2318                free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale);
2319
2320                /* if free space is less than mss estimate, or is below 1/16th
2321                 * of the maximum allowed, try to move to zero-window, else
2322                 * tcp_clamp_window() will grow rcv buf up to tcp_rmem[2], and
2323                 * new incoming data is dropped due to memory limits.
2324                 * With large window, mss test triggers way too late in order
2325                 * to announce zero window in time before rmem limit kicks in.
2326                 */
2327                if (free_space < (allowed_space >> 4) || free_space < mss)
2328                        return 0;
2329        }
2330
2331        if (free_space > tp->rcv_ssthresh)
2332                free_space = tp->rcv_ssthresh;
2333
2334        /* Don't do rounding if we are using window scaling, since the
2335         * scaled window will not line up with the MSS boundary anyway.
2336         */
2337        window = tp->rcv_wnd;
2338        if (tp->rx_opt.rcv_wscale) {
2339                window = free_space;
2340
2341                /* Advertise enough space so that it won't get scaled away.
2342                 * Import case: prevent zero window announcement if
2343                 * 1<<rcv_wscale > mss.
2344                 */
2345                if (((window >> tp->rx_opt.rcv_wscale) << tp->rx_opt.rcv_wscale) != window)
2346                        window = (((window >> tp->rx_opt.rcv_wscale) + 1)
2347                                  << tp->rx_opt.rcv_wscale);
2348        } else {
2349                /* Get the largest window that is a nice multiple of mss.
2350                 * Window clamp already applied above.
2351                 * If our current window offering is within 1 mss of the
2352                 * free space we just keep it. This prevents the divide
2353                 * and multiply from happening most of the time.
2354                 * We also don't do any window rounding when the free space
2355                 * is too small.
2356                 */
2357                if (window <= free_space - mss || window > free_space)
2358                        window = (free_space / mss) * mss;
2359                else if (mss == full_space &&
2360                         free_space > window + (full_space >> 1))
2361                        window = free_space;
2362        }
2363
2364        return window;
2365}
2366
2367/* Collapses two adjacent SKB's during retransmission. */
2368static void tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb)
2369{
2370        struct tcp_sock *tp = tcp_sk(sk);
2371        struct sk_buff *next_skb = tcp_write_queue_next(sk, skb);
2372        int skb_size, next_skb_size;
2373
2374        skb_size = skb->len;
2375        next_skb_size = next_skb->len;
2376
2377        BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1);
2378
2379        tcp_highest_sack_combine(sk, next_skb, skb);
2380
2381        tcp_unlink_write_queue(next_skb, sk);
2382
2383        skb_copy_from_linear_data(next_skb, skb_put(skb, next_skb_size),
2384                                  next_skb_size);
2385
2386        if (next_skb->ip_summed == CHECKSUM_PARTIAL)
2387                skb->ip_summed = CHECKSUM_PARTIAL;
2388
2389        if (skb->ip_summed != CHECKSUM_PARTIAL)
2390                skb->csum = csum_block_add(skb->csum, next_skb->csum, skb_size);
2391
2392        /* Update sequence range on original skb. */
2393        TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq;
2394
2395        /* Merge over control information. This moves PSH/FIN etc. over */
2396        TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next_skb)->tcp_flags;
2397
2398        /* All done, get rid of second SKB and account for it so
2399         * packet counting does not break.
2400         */
2401        TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS;
2402
2403        /* changed transmit queue under us so clear hints */
2404        tcp_clear_retrans_hints_partial(tp);
2405        if (next_skb == tp->retransmit_skb_hint)
2406                tp->retransmit_skb_hint = skb;
2407
2408        tcp_adjust_pcount(sk, next_skb, tcp_skb_pcount(next_skb));
2409
2410        sk_wmem_free_skb(sk, next_skb);
2411}
2412
2413/* Check if coalescing SKBs is legal. */
2414static bool tcp_can_collapse(const struct sock *sk, const struct sk_buff *skb)
2415{
2416        if (tcp_skb_pcount(skb) > 1)
2417                return false;
2418        /* TODO: SACK collapsing could be used to remove this condition */
2419        if (skb_shinfo(skb)->nr_frags != 0)
2420                return false;
2421        if (skb_cloned(skb))
2422                return false;
2423        if (skb == tcp_send_head(sk))
2424                return false;
2425        /* Some heurestics for collapsing over SACK'd could be invented */
2426        if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
2427                return false;
2428
2429        return true;
2430}
2431
2432/* Collapse packets in the retransmit queue to make to create
2433 * less packets on the wire. This is only done on retransmission.
2434 */
2435static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to,
2436                                     int space)
2437{
2438        struct tcp_sock *tp = tcp_sk(sk);
2439        struct sk_buff *skb = to, *tmp;
2440        bool first = true;
2441
2442        if (!sysctl_tcp_retrans_collapse)
2443                return;
2444        if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
2445                return;
2446
2447        tcp_for_write_queue_from_safe(skb, tmp, sk) {
2448                if (!tcp_can_collapse(sk, skb))
2449                        break;
2450
2451                space -= skb->len;
2452
2453                if (first) {
2454                        first = false;
2455                        continue;
2456                }
2457
2458                if (space < 0)
2459                        break;
2460                /* Punt if not enough space exists in the first SKB for
2461                 * the data in the second
2462                 */
2463                if (skb->len > skb_availroom(to))
2464                        break;
2465
2466                if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp)))
2467                        break;
2468
2469                tcp_collapse_retrans(sk, to);
2470        }
2471}
2472
2473/* This retransmits one SKB.  Policy decisions and retransmit queue
2474 * state updates are done by the caller.  Returns non-zero if an
2475 * error occurred which prevented the send.
2476 */
2477int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
2478{
2479        struct tcp_sock *tp = tcp_sk(sk);
2480        struct inet_connection_sock *icsk = inet_csk(sk);
2481        unsigned int cur_mss;
2482        int err;
2483
2484        /* Inconslusive MTU probe */
2485        if (icsk->icsk_mtup.probe_size) {
2486                icsk->icsk_mtup.probe_size = 0;
2487        }
2488
2489        /* Do not sent more than we queued. 1/4 is reserved for possible
2490         * copying overhead: fragmentation, tunneling, mangling etc.
2491         */
2492        if (atomic_read(&sk->sk_wmem_alloc) >
2493            min_t(u32, sk->sk_wmem_queued + (sk->sk_wmem_queued >> 2),
2494                  sk->sk_sndbuf))
2495                return -EAGAIN;
2496
2497        if (skb_still_in_host_queue(sk, skb))
2498                return -EBUSY;
2499
2500        if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {
2501                if (before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))
2502                        BUG();
2503                if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
2504                        return -ENOMEM;
2505        }
2506
2507        if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
2508                return -EHOSTUNREACH; /* Routing failure or similar. */
2509
2510        cur_mss = tcp_current_mss(sk);
2511
2512        /* If receiver has shrunk his window, and skb is out of
2513         * new window, do not retransmit it. The exception is the
2514         * case, when window is shrunk to zero. In this case
2515         * our retransmit serves as a zero window probe.
2516         */
2517        if (!before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp)) &&
2518            TCP_SKB_CB(skb)->seq != tp->snd_una)
2519                return -EAGAIN;
2520
2521        if (skb->len > cur_mss) {
2522                if (tcp_fragment(sk, skb, cur_mss, cur_mss, GFP_ATOMIC))
2523                        return -ENOMEM; /* We'll try again later. */
2524        } else {
2525                int oldpcount = tcp_skb_pcount(skb);
2526
2527                if (unlikely(oldpcount > 1)) {
2528                        if (skb_unclone(skb, GFP_ATOMIC))
2529                                return -ENOMEM;
2530                        tcp_init_tso_segs(sk, skb, cur_mss);
2531                        tcp_adjust_pcount(sk, skb, oldpcount - tcp_skb_pcount(skb));
2532                }
2533        }
2534
2535        tcp_retrans_try_collapse(sk, skb, cur_mss);
2536
2537        /* Make a copy, if the first transmission SKB clone we made
2538         * is still in somebody's hands, else make a clone.
2539         */
2540
2541        /* make sure skb->data is aligned on arches that require it
2542         * and check if ack-trimming & collapsing extended the headroom
2543         * beyond what csum_start can cover.
2544         */
2545        if (unlikely((NET_IP_ALIGN && ((unsigned long)skb->data & 3)) ||
2546                     skb_headroom(skb) >= 0xFFFF)) {
2547                struct sk_buff *nskb;
2548
2549                skb_mstamp_get(&skb->skb_mstamp);
2550                nskb = __pskb_copy(skb, MAX_TCP_HEADER, GFP_ATOMIC);
2551                err = nskb ? tcp_transmit_skb(sk, nskb, 0, GFP_ATOMIC) :
2552                             -ENOBUFS;
2553        } else {
2554                err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
2555        }
2556
2557        if (likely(!err)) {
2558                TCP_SKB_CB(skb)->sacked |= TCPCB_EVER_RETRANS;
2559                /* Update global TCP statistics. */
2560                TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS);
2561                if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
2562                        NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
2563                tp->total_retrans++;
2564        }
2565        return err;
2566}
2567
2568int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
2569{
2570        struct tcp_sock *tp = tcp_sk(sk);
2571        int err = __tcp_retransmit_skb(sk, skb);
2572
2573        if (err == 0) {
2574#if FASTRETRANS_DEBUG > 0
2575                if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) {
2576                        net_dbg_ratelimited("retrans_out leaked\n");
2577                }
2578#endif
2579                if (!tp->retrans_out)
2580                        tp->lost_retrans_low = tp->snd_nxt;
2581                TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS;
2582                tp->retrans_out += tcp_skb_pcount(skb);
2583
2584                /* Save stamp of the first retransmit. */
2585                if (!tp->retrans_stamp)
2586                        tp->retrans_stamp = tcp_skb_timestamp(skb);
2587
2588                /* snd_nxt is stored to detect loss of retransmitted segment,
2589                 * see tcp_input.c tcp_sacktag_write_queue().
2590                 */
2591                TCP_SKB_CB(skb)->ack_seq = tp->snd_nxt;
2592        } else if (err != -EBUSY) {
2593                NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL);
2594        }
2595
2596        if (tp->undo_retrans < 0)
2597                tp->undo_retrans = 0;
2598        tp->undo_retrans += tcp_skb_pcount(skb);
2599        return err;
2600}
2601
2602/* Check if we forward retransmits are possible in the current
2603 * window/congestion state.
2604 */
2605static bool tcp_can_forward_retransmit(struct sock *sk)
2606{
2607        const struct inet_connection_sock *icsk = inet_csk(sk);
2608        const struct tcp_sock *tp = tcp_sk(sk);
2609
2610        /* Forward retransmissions are possible only during Recovery. */
2611        if (icsk->icsk_ca_state != TCP_CA_Recovery)
2612                return false;
2613
2614        /* No forward retransmissions in Reno are possible. */
2615        if (tcp_is_reno(tp))
2616                return false;
2617
2618        /* Yeah, we have to make difficult choice between forward transmission
2619         * and retransmission... Both ways have their merits...
2620         *
2621         * For now we do not retransmit anything, while we have some new
2622         * segments to send. In the other cases, follow rule 3 for
2623         * NextSeg() specified in RFC3517.
2624         */
2625
2626        if (tcp_may_send_now(sk))
2627                return false;
2628
2629        return true;
2630}
2631
2632/* This gets called after a retransmit timeout, and the initially
2633 * retransmitted data is acknowledged.  It tries to continue
2634 * resending the rest of the retransmit queue, until either
2635 * we've sent it all or the congestion window limit is reached.
2636 * If doing SACK, the first ACK which comes back for a timeout
2637 * based retransmit packet might feed us FACK information again.
2638 * If so, we use it to avoid unnecessarily retransmissions.
2639 */
2640void tcp_xmit_retransmit_queue(struct sock *sk)
2641{
2642        const struct inet_connection_sock *icsk = inet_csk(sk);
2643        struct tcp_sock *tp = tcp_sk(sk);
2644        struct sk_buff *skb;
2645        struct sk_buff *hole = NULL;
2646        u32 last_lost;
2647        int mib_idx;
2648        int fwd_rexmitting = 0;
2649
2650        if (!tp->packets_out)
2651                return;
2652
2653        if (!tp->lost_out)
2654                tp->retransmit_high = tp->snd_una;
2655
2656        if (tp->retransmit_skb_hint) {
2657                skb = tp->retransmit_skb_hint;
2658                last_lost = TCP_SKB_CB(skb)->end_seq;
2659                if (after(last_lost, tp->retransmit_high))
2660                        last_lost = tp->retransmit_high;
2661        } else {
2662                skb = tcp_write_queue_head(sk);
2663                last_lost = tp->snd_una;
2664        }
2665
2666        tcp_for_write_queue_from(skb, sk) {
2667                __u8 sacked = TCP_SKB_CB(skb)->sacked;
2668
2669                if (skb == tcp_send_head(sk))
2670                        break;
2671                /* we could do better than to assign each time */
2672                if (hole == NULL)
2673                        tp->retransmit_skb_hint = skb;
2674
2675                /* Assume this retransmit will generate
2676                 * only one packet for congestion window
2677                 * calculation purposes.  This works because
2678                 * tcp_retransmit_skb() will chop up the
2679                 * packet to be MSS sized and all the
2680                 * packet counting works out.
2681                 */
2682                if (tcp_packets_in_flight(tp) >= tp->snd_cwnd)
2683                        return;
2684
2685                if (fwd_rexmitting) {
2686begin_fwd:
2687                        if (!before(TCP_SKB_CB(skb)->seq, tcp_highest_sack_seq(tp)))
2688                                break;
2689                        mib_idx = LINUX_MIB_TCPFORWARDRETRANS;
2690
2691                } else if (!before(TCP_SKB_CB(skb)->seq, tp->retransmit_high)) {
2692                        tp->retransmit_high = last_lost;
2693                        if (!tcp_can_forward_retransmit(sk))
2694                                break;
2695                        /* Backtrack if necessary to non-L'ed skb */
2696                        if (hole != NULL) {
2697                                skb = hole;
2698                                hole = NULL;
2699                        }
2700                        fwd_rexmitting = 1;
2701                        goto begin_fwd;
2702
2703                } else if (!(sacked & TCPCB_LOST)) {
2704                        if (hole == NULL && !(sacked & (TCPCB_SACKED_RETRANS|TCPCB_SACKED_ACKED)))
2705                                hole = skb;
2706                        continue;
2707
2708                } else {
2709                        last_lost = TCP_SKB_CB(skb)->end_seq;
2710                        if (icsk->icsk_ca_state != TCP_CA_Loss)
2711                                mib_idx = LINUX_MIB_TCPFASTRETRANS;
2712                        else
2713                                mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS;
2714                }
2715
2716                if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS))
2717                        continue;
2718
2719                if (tcp_retransmit_skb(sk, skb))
2720                        return;
2721
2722                NET_INC_STATS_BH(sock_net(sk), mib_idx);
2723
2724                if (tcp_in_cwnd_reduction(sk))
2725                        tp->prr_out += tcp_skb_pcount(skb);
2726
2727                if (skb == tcp_write_queue_head(sk))
2728                        inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
2729                                                  inet_csk(sk)->icsk_rto,
2730                                                  TCP_RTO_MAX);
2731        }
2732}
2733
2734/* We allow to exceed memory limits for FIN packets to expedite
2735 * connection tear down and (memory) recovery.
2736 * Otherwise tcp_send_fin() could be tempted to either delay FIN
2737 * or even be forced to close flow without any FIN.
2738 */
2739static void sk_forced_wmem_schedule(struct sock *sk, int size)
2740{
2741        int amt, status;
2742
2743        if (size <= sk->sk_forward_alloc)
2744                return;
2745        amt = sk_mem_pages(size);
2746        sk->sk_forward_alloc += amt * SK_MEM_QUANTUM;
2747        sk_memory_allocated_add(sk, amt, &status);
2748}
2749
2750/* Send a FIN. The caller locks the socket for us.
2751 * We should try to send a FIN packet really hard, but eventually give up.
2752 */
2753void tcp_send_fin(struct sock *sk)
2754{
2755        struct sk_buff *skb, *tskb = tcp_write_queue_tail(sk);
2756        struct tcp_sock *tp = tcp_sk(sk);
2757
2758        /* Optimization, tack on the FIN if we have one skb in write queue and
2759         * this skb was not yet sent, or we are under memory pressure.
2760         * Note: in the latter case, FIN packet will be sent after a timeout,
2761         * as TCP stack thinks it has already been transmitted.
2762         */
2763        if (tskb && (tcp_send_head(sk) || sk_under_memory_pressure(sk))) {
2764coalesce:
2765                TCP_SKB_CB(tskb)->tcp_flags |= TCPHDR_FIN;
2766                TCP_SKB_CB(tskb)->end_seq++;
2767                tp->write_seq++;
2768                if (!tcp_send_head(sk)) {
2769                        /* This means tskb was already sent.
2770                         * Pretend we included the FIN on previous transmit.
2771                         * We need to set tp->snd_nxt to the value it would have
2772                         * if FIN had been sent. This is because retransmit path
2773                         * does not change tp->snd_nxt.
2774                         */
2775                        tp->snd_nxt++;
2776                        return;
2777                }
2778        } else {
2779                skb = alloc_skb_fclone(MAX_TCP_HEADER, sk->sk_allocation);
2780                if (unlikely(!skb)) {
2781                        if (tskb)
2782                                goto coalesce;
2783                        return;
2784                }
2785                skb_reserve(skb, MAX_TCP_HEADER);
2786                sk_forced_wmem_schedule(sk, skb->truesize);
2787                /* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */
2788                tcp_init_nondata_skb(skb, tp->write_seq,
2789                                     TCPHDR_ACK | TCPHDR_FIN);
2790                tcp_queue_skb(sk, skb);
2791        }
2792        __tcp_push_pending_frames(sk, tcp_current_mss(sk), TCP_NAGLE_OFF);
2793}
2794
2795/* We get here when a process closes a file descriptor (either due to
2796 * an explicit close() or as a byproduct of exit()'ing) and there
2797 * was unread data in the receive queue.  This behavior is recommended
2798 * by RFC 2525, section 2.17.  -DaveM
2799 */
2800void tcp_send_active_reset(struct sock *sk, gfp_t priority)
2801{
2802        struct sk_buff *skb;
2803
2804        /* NOTE: No TCP options attached and we never retransmit this. */
2805        skb = alloc_skb(MAX_TCP_HEADER, priority);
2806        if (!skb) {
2807                NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
2808                return;
2809        }
2810
2811        /* Reserve space for headers and prepare control bits. */
2812        skb_reserve(skb, MAX_TCP_HEADER);
2813        tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk),
2814                             TCPHDR_ACK | TCPHDR_RST);
2815        skb_mstamp_get(&skb->skb_mstamp);
2816        /* Send it off. */
2817        if (tcp_transmit_skb(sk, skb, 0, priority))
2818                NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
2819
2820        TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS);
2821}
2822
2823/* Send a crossed SYN-ACK during socket establishment.
2824 * WARNING: This routine must only be called when we have already sent
2825 * a SYN packet that crossed the incoming SYN that caused this routine
2826 * to get called. If this assumption fails then the initial rcv_wnd
2827 * and rcv_wscale values will not be correct.
2828 */
2829int tcp_send_synack(struct sock *sk)
2830{
2831        struct sk_buff *skb;
2832
2833        skb = tcp_write_queue_head(sk);
2834        if (skb == NULL || !(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
2835                pr_debug("%s: wrong queue state\n", __func__);
2836                return -EFAULT;
2837        }
2838        if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)) {
2839                if (skb_cloned(skb)) {
2840                        struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC);
2841                        if (nskb == NULL)
2842                                return -ENOMEM;
2843                        tcp_unlink_write_queue(skb, sk);
2844                        __skb_header_release(nskb);
2845                        __tcp_add_write_queue_head(sk, nskb);
2846                        sk_wmem_free_skb(sk, skb);
2847                        sk->sk_wmem_queued += nskb->truesize;
2848                        sk_mem_charge(sk, nskb->truesize);
2849                        skb = nskb;
2850                }
2851
2852                TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ACK;
2853                tcp_ecn_send_synack(sk, skb);
2854        }
2855        return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
2856}
2857
2858/**
2859 * tcp_make_synack - Prepare a SYN-ACK.
2860 * sk: listener socket
2861 * dst: dst entry attached to the SYNACK
2862 * req: request_sock pointer
2863 *
2864 * Allocate one skb and build a SYNACK packet.
2865 * @dst is consumed : Caller should not use it again.
2866 */
2867struct sk_buff *tcp_make_synack(struct sock *sk, struct dst_entry *dst,
2868                                struct request_sock *req,
2869                                struct tcp_fastopen_cookie *foc)
2870{
2871        struct tcp_out_options opts;
2872        struct inet_request_sock *ireq = inet_rsk(req);
2873        struct tcp_sock *tp = tcp_sk(sk);
2874        struct tcphdr *th;
2875        struct sk_buff *skb;
2876        struct tcp_md5sig_key *md5;
2877        int tcp_header_size;
2878        int mss;
2879
2880        skb = sock_wmalloc(sk, MAX_TCP_HEADER, 1, GFP_ATOMIC);
2881        if (unlikely(!skb)) {
2882                dst_release(dst);
2883                return NULL;
2884        }
2885        /* Reserve space for headers. */
2886        skb_reserve(skb, MAX_TCP_HEADER);
2887
2888        skb_dst_set(skb, dst);
2889        security_skb_owned_by(skb, sk);
2890
2891        mss = dst_metric_advmss(dst);
2892        if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < mss)
2893                mss = tp->rx_opt.user_mss;
2894
2895        memset(&opts, 0, sizeof(opts));
2896#ifdef CONFIG_SYN_COOKIES
2897        if (unlikely(req->cookie_ts))
2898                skb->skb_mstamp.stamp_jiffies = cookie_init_timestamp(req);
2899        else
2900#endif
2901        skb_mstamp_get(&skb->skb_mstamp);
2902        tcp_header_size = tcp_synack_options(sk, req, mss, skb, &opts, &md5,
2903                                             foc) + sizeof(*th);
2904
2905        skb_push(skb, tcp_header_size);
2906        skb_reset_transport_header(skb);
2907
2908        th = tcp_hdr(skb);
2909        memset(th, 0, sizeof(struct tcphdr));
2910        th->syn = 1;
2911        th->ack = 1;
2912        tcp_ecn_make_synack(req, th, sk);
2913        th->source = htons(ireq->ir_num);
2914        th->dest = ireq->ir_rmt_port;
2915        /* Setting of flags are superfluous here for callers (and ECE is
2916         * not even correctly set)
2917         */
2918        tcp_init_nondata_skb(skb, tcp_rsk(req)->snt_isn,
2919                             TCPHDR_SYN | TCPHDR_ACK);
2920
2921        th->seq = htonl(TCP_SKB_CB(skb)->seq);
2922        /* XXX data is queued and acked as is. No buffer/window check */
2923        th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt);
2924
2925        /* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */
2926        th->window = htons(min(req->rcv_wnd, 65535U));
2927        tcp_options_write((__be32 *)(th + 1), tp, &opts);
2928        th->doff = (tcp_header_size >> 2);
2929        TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_OUTSEGS);
2930
2931#ifdef CONFIG_TCP_MD5SIG
2932        /* Okay, we have all we need - do the md5 hash if needed */
2933        if (md5) {
2934                tcp_rsk(req)->af_specific->calc_md5_hash(opts.hash_location,
2935                                               md5, NULL, req, skb);
2936        }
2937#endif
2938
2939        /* Do not fool tcpdump (if any), clean our debris */
2940        skb->tstamp.tv64 = 0;
2941        return skb;
2942}
2943EXPORT_SYMBOL(tcp_make_synack);
2944
2945/* Do all connect socket setups that can be done AF independent. */
2946static void tcp_connect_init(struct sock *sk)
2947{
2948        const struct dst_entry *dst = __sk_dst_get(sk);
2949        struct tcp_sock *tp = tcp_sk(sk);
2950        __u8 rcv_wscale;
2951
2952        /* We'll fix this up when we get a response from the other end.
2953         * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT.
2954         */
2955        tp->tcp_header_len = sizeof(struct tcphdr) +
2956                (sysctl_tcp_timestamps ? TCPOLEN_TSTAMP_ALIGNED : 0);
2957
2958#ifdef CONFIG_TCP_MD5SIG
2959        if (tp->af_specific->md5_lookup(sk, sk) != NULL)
2960                tp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED;
2961#endif
2962
2963        /* If user gave his TCP_MAXSEG, record it to clamp */
2964        if (tp->rx_opt.user_mss)
2965                tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
2966        tp->max_window = 0;
2967        tcp_mtup_init(sk);
2968        tcp_sync_mss(sk, dst_mtu(dst));
2969
2970        if (!tp->window_clamp)
2971                tp->window_clamp = dst_metric(dst, RTAX_WINDOW);
2972        tp->advmss = dst_metric_advmss(dst);
2973        if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < tp->advmss)
2974                tp->advmss = tp->rx_opt.user_mss;
2975
2976        tcp_initialize_rcv_mss(sk);
2977
2978        /* limit the window selection if the user enforce a smaller rx buffer */
2979        if (sk->sk_userlocks & SOCK_RCVBUF_LOCK &&
2980            (tp->window_clamp > tcp_full_space(sk) || tp->window_clamp == 0))
2981                tp->window_clamp = tcp_full_space(sk);
2982
2983        tcp_select_initial_window(tcp_full_space(sk),
2984                                  tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0),
2985                                  &tp->rcv_wnd,
2986                                  &tp->window_clamp,
2987                                  sysctl_tcp_window_scaling,
2988                                  &rcv_wscale,
2989                                  dst_metric(dst, RTAX_INITRWND));
2990
2991        tp->rx_opt.rcv_wscale = rcv_wscale;
2992        tp->rcv_ssthresh = tp->rcv_wnd;
2993
2994        sk->sk_err = 0;
2995        sock_reset_flag(sk, SOCK_DONE);
2996        tp->snd_wnd = 0;
2997        tcp_init_wl(tp, 0);
2998        tp->snd_una = tp->write_seq;
2999        tp->snd_sml = tp->write_seq;
3000        tp->snd_up = tp->write_seq;
3001        tp->snd_nxt = tp->write_seq;
3002
3003        if (likely(!tp->repair))
3004                tp->rcv_nxt = 0;
3005        else
3006                tp->rcv_tstamp = tcp_time_stamp;
3007        tp->rcv_wup = tp->rcv_nxt;
3008        tp->copied_seq = tp->rcv_nxt;
3009
3010        inet_csk(sk)->icsk_rto = TCP_TIMEOUT_INIT;
3011        inet_csk(sk)->icsk_retransmits = 0;
3012        tcp_clear_retrans(tp);
3013}
3014
3015static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb)
3016{
3017        struct tcp_sock *tp = tcp_sk(sk);
3018        struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
3019
3020        tcb->end_seq += skb->len;
3021        __skb_header_release(skb);
3022        __tcp_add_write_queue_tail(sk, skb);
3023        sk->sk_wmem_queued += skb->truesize;
3024        sk_mem_charge(sk, skb->truesize);
3025        tp->write_seq = tcb->end_seq;
3026        tp->packets_out += tcp_skb_pcount(skb);
3027}
3028
3029/* Build and send a SYN with data and (cached) Fast Open cookie. However,
3030 * queue a data-only packet after the regular SYN, such that regular SYNs
3031 * are retransmitted on timeouts. Also if the remote SYN-ACK acknowledges
3032 * only the SYN sequence, the data are retransmitted in the first ACK.
3033 * If cookie is not cached or other error occurs, falls back to send a
3034 * regular SYN with Fast Open cookie request option.
3035 */
3036static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)
3037{
3038        struct tcp_sock *tp = tcp_sk(sk);
3039        struct tcp_fastopen_request *fo = tp->fastopen_req;
3040        int syn_loss = 0, space, err = 0;
3041        unsigned long last_syn_loss = 0;
3042        struct sk_buff *syn_data;
3043
3044        tp->rx_opt.mss_clamp = tp->advmss;  /* If MSS is not cached */
3045        tcp_fastopen_cache_get(sk, &tp->rx_opt.mss_clamp, &fo->cookie,
3046                               &syn_loss, &last_syn_loss);
3047        /* Recurring FO SYN losses: revert to regular handshake temporarily */
3048        if (syn_loss > 1 &&
3049            time_before(jiffies, last_syn_loss + (60*HZ << syn_loss))) {
3050                fo->cookie.len = -1;
3051                goto fallback;
3052        }
3053
3054        if (sysctl_tcp_fastopen & TFO_CLIENT_NO_COOKIE)
3055                fo->cookie.len = -1;
3056        else if (fo->cookie.len <= 0)
3057                goto fallback;
3058
3059        /* MSS for SYN-data is based on cached MSS and bounded by PMTU and
3060         * user-MSS. Reserve maximum option space for middleboxes that add
3061         * private TCP options. The cost is reduced data space in SYN :(
3062         */
3063        if (tp->rx_opt.user_mss && tp->rx_opt.user_mss < tp->rx_opt.mss_clamp)
3064                tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
3065        space = __tcp_mtu_to_mss(sk, inet_csk(sk)->icsk_pmtu_cookie) -
3066                MAX_TCP_OPTION_SPACE;
3067
3068        space = min_t(size_t, space, fo->size);
3069
3070        /* limit to order-0 allocations */
3071        space = min_t(size_t, space, SKB_MAX_HEAD(MAX_TCP_HEADER));
3072
3073        syn_data = sk_stream_alloc_skb(sk, space, sk->sk_allocation);
3074        if (!syn_data)
3075                goto fallback;
3076        syn_data->ip_summed = CHECKSUM_PARTIAL;
3077        memcpy(syn_data->cb, syn->cb, sizeof(syn->cb));
3078        if (unlikely(memcpy_fromiovecend(skb_put(syn_data, space),
3079                                         fo->data->msg_iov, 0, space))) {
3080                kfree_skb(syn_data);
3081                goto fallback;
3082        }
3083
3084        /* No more data pending in inet_wait_for_connect() */
3085        if (space == fo->size)
3086                fo->data = NULL;
3087        fo->copied = space;
3088
3089        tcp_connect_queue_skb(sk, syn_data);
3090
3091        err = tcp_transmit_skb(sk, syn_data, 1, sk->sk_allocation);
3092
3093        syn->skb_mstamp = syn_data->skb_mstamp;
3094
3095        /* Now full SYN+DATA was cloned and sent (or not),
3096         * remove the SYN from the original skb (syn_data)
3097         * we keep in write queue in case of a retransmit, as we
3098         * also have the SYN packet (with no data) in the same queue.
3099         */
3100        TCP_SKB_CB(syn_data)->seq++;
3101        TCP_SKB_CB(syn_data)->tcp_flags = TCPHDR_ACK | TCPHDR_PSH;
3102        if (!err) {
3103                tp->syn_data = (fo->copied > 0);
3104                NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT);
3105                goto done;
3106        }
3107
3108fallback:
3109        /* Send a regular SYN with Fast Open cookie request option */
3110        if (fo->cookie.len > 0)
3111                fo->cookie.len = 0;
3112        err = tcp_transmit_skb(sk, syn, 1, sk->sk_allocation);
3113        if (err)
3114                tp->syn_fastopen = 0;
3115done:
3116        fo->cookie.len = -1;  /* Exclude Fast Open option for SYN retries */
3117        return err;
3118}
3119
3120/* Build a SYN and send it off. */
3121int tcp_connect(struct sock *sk)
3122{
3123        struct tcp_sock *tp = tcp_sk(sk);
3124        struct sk_buff *buff;
3125        int err;
3126
3127        tcp_connect_init(sk);
3128
3129        if (unlikely(tp->repair)) {
3130                tcp_finish_connect(sk, NULL);
3131                return 0;
3132        }
3133
3134        buff = sk_stream_alloc_skb(sk, 0, sk->sk_allocation);
3135        if (unlikely(!buff))
3136                return -ENOBUFS;
3137
3138        tcp_init_nondata_skb(buff, tp->write_seq++, TCPHDR_SYN);
3139        tp->retrans_stamp = tcp_time_stamp;
3140        tcp_connect_queue_skb(sk, buff);
3141        tcp_ecn_send_syn(sk, buff);
3142
3143        /* Send off SYN; include data in Fast Open. */
3144        err = tp->fastopen_req ? tcp_send_syn_data(sk, buff) :
3145              tcp_transmit_skb(sk, buff, 1, sk->sk_allocation);
3146        if (err == -ECONNREFUSED)
3147                return err;
3148
3149        /* We change tp->snd_nxt after the tcp_transmit_skb() call
3150         * in order to make this packet get counted in tcpOutSegs.
3151         */
3152        tp->snd_nxt = tp->write_seq;
3153        tp->pushed_seq = tp->write_seq;
3154        TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS);
3155
3156        /* Timer for repeating the SYN until an answer. */
3157        inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
3158                                  inet_csk(sk)->icsk_rto, TCP_RTO_MAX);
3159        return 0;
3160}
3161EXPORT_SYMBOL(tcp_connect);
3162
3163/* Send out a delayed ack, the caller does the policy checking
3164 * to see if we should even be here.  See tcp_input.c:tcp_ack_snd_check()
3165 * for details.
3166 */
3167void tcp_send_delayed_ack(struct sock *sk)
3168{
3169        struct inet_connection_sock *icsk = inet_csk(sk);
3170        int ato = icsk->icsk_ack.ato;
3171        unsigned long timeout;
3172
3173        tcp_ca_event(sk, CA_EVENT_DELAYED_ACK);
3174
3175        if (ato > TCP_DELACK_MIN) {
3176                const struct tcp_sock *tp = tcp_sk(sk);
3177                int max_ato = HZ / 2;
3178
3179                if (icsk->icsk_ack.pingpong ||
3180                    (icsk->icsk_ack.pending & ICSK_ACK_PUSHED))
3181                        max_ato = TCP_DELACK_MAX;
3182
3183                /* Slow path, intersegment interval is "high". */
3184
3185                /* If some rtt estimate is known, use it to bound delayed ack.
3186                 * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements
3187                 * directly.
3188                 */
3189                if (tp->srtt_us) {
3190                        int rtt = max_t(int, usecs_to_jiffies(tp->srtt_us >> 3),
3191                                        TCP_DELACK_MIN);
3192
3193                        if (rtt < max_ato)
3194                                max_ato = rtt;
3195                }
3196
3197                ato = min(ato, max_ato);
3198        }
3199
3200        /* Stay within the limit we were given */
3201        timeout = jiffies + ato;
3202
3203        /* Use new timeout only if there wasn't a older one earlier. */
3204        if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) {
3205                /* If delack timer was blocked or is about to expire,
3206                 * send ACK now.
3207                 */
3208                if (icsk->icsk_ack.blocked ||
3209                    time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) {
3210                        tcp_send_ack(sk);
3211                        return;
3212                }
3213
3214                if (!time_before(timeout, icsk->icsk_ack.timeout))
3215                        timeout = icsk->icsk_ack.timeout;
3216        }
3217        icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
3218        icsk->icsk_ack.timeout = timeout;
3219        sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);
3220}
3221
3222/* This routine sends an ack and also updates the window. */
3223void tcp_send_ack(struct sock *sk)
3224{
3225        struct sk_buff *buff;
3226
3227        /* If we have been reset, we may not send again. */
3228        if (sk->sk_state == TCP_CLOSE)
3229                return;
3230
3231        tcp_ca_event(sk, CA_EVENT_NON_DELAYED_ACK);
3232
3233        /* We are not putting this on the write queue, so
3234         * tcp_transmit_skb() will set the ownership to this
3235         * sock.
3236         */
3237        buff = alloc_skb(MAX_TCP_HEADER, sk_gfp_atomic(sk, GFP_ATOMIC));
3238        if (buff == NULL) {
3239                inet_csk_schedule_ack(sk);
3240                inet_csk(sk)->icsk_ack.ato = TCP_ATO_MIN;
3241                inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK,
3242                                          TCP_DELACK_MAX, TCP_RTO_MAX);
3243                return;
3244        }
3245
3246        /* Reserve space for headers and prepare control bits. */
3247        skb_reserve(buff, MAX_TCP_HEADER);
3248        tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPHDR_ACK);
3249
3250        /* Send it off, this clears delayed acks for us. */
3251        skb_mstamp_get(&buff->skb_mstamp);
3252        tcp_transmit_skb(sk, buff, 0, sk_gfp_atomic(sk, GFP_ATOMIC));
3253}
3254EXPORT_SYMBOL_GPL(tcp_send_ack);
3255
3256/* This routine sends a packet with an out of date sequence
3257 * number. It assumes the other end will try to ack it.
3258 *
3259 * Question: what should we make while urgent mode?
3260 * 4.4BSD forces sending single byte of data. We cannot send
3261 * out of window data, because we have SND.NXT==SND.MAX...
3262 *
3263 * Current solution: to send TWO zero-length segments in urgent mode:
3264 * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is
3265 * out-of-date with SND.UNA-1 to probe window.
3266 */
3267static int tcp_xmit_probe_skb(struct sock *sk, int urgent)
3268{
3269        struct tcp_sock *tp = tcp_sk(sk);
3270        struct sk_buff *skb;
3271
3272        /* We don't queue it, tcp_transmit_skb() sets ownership. */
3273        skb = alloc_skb(MAX_TCP_HEADER, sk_gfp_atomic(sk, GFP_ATOMIC));
3274        if (skb == NULL)
3275                return -1;
3276
3277        /* Reserve space for headers and set control bits. */
3278        skb_reserve(skb, MAX_TCP_HEADER);
3279        /* Use a previous sequence.  This should cause the other
3280         * end to send an ack.  Don't queue or clone SKB, just
3281         * send it.
3282         */
3283        tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPHDR_ACK);
3284        skb_mstamp_get(&skb->skb_mstamp);
3285        return tcp_transmit_skb(sk, skb, 0, GFP_ATOMIC);
3286}
3287
3288void tcp_send_window_probe(struct sock *sk)
3289{
3290        if (sk->sk_state == TCP_ESTABLISHED) {
3291                tcp_sk(sk)->snd_wl1 = tcp_sk(sk)->rcv_nxt - 1;
3292                tcp_xmit_probe_skb(sk, 0);
3293        }
3294}
3295
3296/* Initiate keepalive or window probe from timer. */
3297int tcp_write_wakeup(struct sock *sk)
3298{
3299        struct tcp_sock *tp = tcp_sk(sk);
3300        struct sk_buff *skb;
3301
3302        if (sk->sk_state == TCP_CLOSE)
3303                return -1;
3304
3305        if ((skb = tcp_send_head(sk)) != NULL &&
3306            before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) {
3307                int err;
3308                unsigned int mss = tcp_current_mss(sk);
3309                unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
3310
3311                if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
3312                        tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;
3313
3314                /* We are probing the opening of a window
3315                 * but the window size is != 0
3316                 * must have been a result SWS avoidance ( sender )
3317                 */
3318                if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq ||
3319                    skb->len > mss) {
3320                        seg_size = min(seg_size, mss);
3321                        TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
3322                        if (tcp_fragment(sk, skb, seg_size, mss, GFP_ATOMIC))
3323                                return -1;
3324                } else if (!tcp_skb_pcount(skb))
3325                        tcp_set_skb_tso_segs(sk, skb, mss);
3326
3327                TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
3328                err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3329                if (!err)
3330                        tcp_event_new_data_sent(sk, skb);
3331                return err;
3332        } else {
3333                if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF))
3334                        tcp_xmit_probe_skb(sk, 1);
3335                return tcp_xmit_probe_skb(sk, 0);
3336        }
3337}
3338
3339/* A window probe timeout has occurred.  If window is not closed send
3340 * a partial packet else a zero probe.
3341 */
3342void tcp_send_probe0(struct sock *sk)
3343{
3344        struct inet_connection_sock *icsk = inet_csk(sk);
3345        struct tcp_sock *tp = tcp_sk(sk);
3346        unsigned long probe_max;
3347        int err;
3348
3349        err = tcp_write_wakeup(sk);
3350
3351        if (tp->packets_out || !tcp_send_head(sk)) {
3352                /* Cancel probe timer, if it is not required. */
3353                icsk->icsk_probes_out = 0;
3354                icsk->icsk_backoff = 0;
3355                return;
3356        }
3357
3358        if (err <= 0) {
3359                if (icsk->icsk_backoff < sysctl_tcp_retries2)
3360                        icsk->icsk_backoff++;
3361                icsk->icsk_probes_out++;
3362                probe_max = TCP_RTO_MAX;
3363        } else {
3364                /* If packet was not sent due to local congestion,
3365                 * do not backoff and do not remember icsk_probes_out.
3366                 * Let local senders to fight for local resources.
3367                 *
3368                 * Use accumulated backoff yet.
3369                 */
3370                if (!icsk->icsk_probes_out)
3371                        icsk->icsk_probes_out = 1;
3372                probe_max = TCP_RESOURCE_PROBE_INTERVAL;
3373        }
3374        inet_csk_reset_xmit_timer(sk, ICSK_TIME_PROBE0,
3375                                  inet_csk_rto_backoff(icsk, probe_max),
3376                                  TCP_RTO_MAX);
3377}
3378
3379int tcp_rtx_synack(struct sock *sk, struct request_sock *req)
3380{
3381        const struct tcp_request_sock_ops *af_ops = tcp_rsk(req)->af_specific;
3382        struct flowi fl;
3383        int res;
3384
3385        res = af_ops->send_synack(sk, NULL, &fl, req, 0, NULL);
3386        if (!res) {
3387                TCP_INC_STATS_BH(sock_net(sk), TCP_MIB_RETRANSSEGS);
3388                NET_INC_STATS_BH(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
3389        }
3390        return res;
3391}
3392EXPORT_SYMBOL(tcp_rtx_synack);
Note: See TracBrowser for help on using the repository browser.