[SK_BUFF]: Convert skb->tail to sk_buff_data_t

So that it is also an offset from skb->head, reduces its size from 8 to 4 bytes
on 64bit architectures, allowing us to combine the 4 bytes hole left by the
layer headers conversion, reducing struct sk_buff size to 256 bytes, i.e. 4
64byte cachelines, and since the sk_buff slab cache is SLAB_HWCACHE_ALIGN...
:-)

Many calculations that previously required that skb->{transport,network,
mac}_header be first converted to a pointer now can be done directly, being
meaningful as offsets or pointers.

Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

7 files changed, 58 insertions, 27 deletions

diff --git a/include/linux/netfilter/nfnetlink.h b/include/linux/netfilter/nfnetlink.h
index 6179648a014..e1ea5dfbbbd 100644
--- a/include/linux/netfilter/nfnetlink.h
+++ b/include/linux/netfilter/nfnetlink.h
@@ -62,11 +62,11 @@ struct nfattr
 #define NFA_DATA(nfa)   ((void *)(((char *)(nfa)) + NFA_LENGTH(0)))
 #define NFA_PAYLOAD(nfa) ((int)((nfa)->nfa_len) - NFA_LENGTH(0))
 #define NFA_NEST(skb, type) \
-({	struct nfattr *__start = (struct nfattr *) (skb)->tail; \
+({	struct nfattr *__start = (struct nfattr *)skb_tail_pointer(skb); \
 	NFA_PUT(skb, (NFNL_NFA_NEST | type), 0, NULL); \
 	__start;  })
 #define NFA_NEST_END(skb, start) \
-({      (start)->nfa_len = ((skb)->tail - (unsigned char *) (start)); \
+({      (start)->nfa_len = skb_tail_pointer(skb) - (unsigned char *)(start); \
         (skb)->len; })
 #define NFA_NEST_CANCEL(skb, start) \
 ({      if (start) \
diff --git a/include/linux/netlink.h b/include/linux/netlink.h
index a9d3ad5bc80..68a632b372e 100644
--- a/include/linux/netlink.h
+++ b/include/linux/netlink.h
@@ -229,7 +229,7 @@ __nlmsg_put(struct sk_buff *skb, u32 pid, u32 seq, int type, int len, int flags)
 		  (cb)->nlh->nlmsg_seq, type, len, flags)
 
 #define NLMSG_END(skb, nlh) \
-({	(nlh)->nlmsg_len = (skb)->tail - (unsigned char *) (nlh); \
+({	(nlh)->nlmsg_len = skb_tail_pointer(skb) - (unsigned char *)(nlh); \
 	(skb)->len; })
 
 #define NLMSG_CANCEL(skb, nlh) \
diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h
index 4a629ea70cc..3a4cb242ecd 100644
--- a/include/linux/rtnetlink.h
+++ b/include/linux/rtnetlink.h
@@ -605,7 +605,7 @@ extern void __rta_fill(struct sk_buff *skb, int attrtype, int attrlen, const voi
 
 #define RTA_PUT_NOHDR(skb, attrlen, data) \
 ({	RTA_APPEND(skb, RTA_ALIGN(attrlen), data); \
-	memset(skb->tail - (RTA_ALIGN(attrlen) - attrlen), 0, \
+	memset(skb_tail_pointer(skb) - (RTA_ALIGN(attrlen) - attrlen), 0, \
 	       RTA_ALIGN(attrlen) - attrlen); })
 
 #define RTA_PUT_U8(skb, attrtype, value) \
@@ -637,12 +637,12 @@ extern void __rta_fill(struct sk_buff *skb, int attrtype, int attrlen, const voi
 	RTA_PUT(skb, attrtype, 0, NULL);
 
 #define RTA_NEST(skb, type) \
-({	struct rtattr *__start = (struct rtattr *) (skb)->tail; \
+({	struct rtattr *__start = (struct rtattr *)skb_tail_pointer(skb); \
 	RTA_PUT(skb, type, 0, NULL); \
 	__start;  })
 
 #define RTA_NEST_END(skb, start) \
-({	(start)->rta_len = ((skb)->tail - (unsigned char *) (start)); \
+({	(start)->rta_len = skb_tail_pointer(skb) - (unsigned char *)(start); \
 	(skb)->len; })
 
 #define RTA_NEST_CANCEL(skb, start) \
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 2e740550062..e1c2392ecb5 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -246,9 +246,6 @@ struct sk_buff {
 	int			iif;
 	/* 4 byte hole on 64 bit*/
 
-	sk_buff_data_t		transport_header;
-	sk_buff_data_t		network_header;
-	sk_buff_data_t		mac_header;
 	struct  dst_entry	*dst;
 	struct	sec_path	*sp;
 
@@ -303,13 +300,16 @@ struct sk_buff {
 
 	__u32			mark;
 
+	sk_buff_data_t		transport_header;
+	sk_buff_data_t		network_header;
+	sk_buff_data_t		mac_header;
 	/* These elements must be at the end, see alloc_skb() for details.  */
-	unsigned int		truesize;
-	atomic_t		users;
+	sk_buff_data_t		tail;
 	unsigned char		*head,
 				*data,
-				*tail,
 				*end;
+	unsigned int		truesize;
+	atomic_t		users;
 };
 
 #ifdef __KERNEL__
@@ -812,12 +812,45 @@ static inline void skb_fill_page_desc(struct sk_buff *skb, int i,
 #define SKB_FRAG_ASSERT(skb) 	BUG_ON(skb_shinfo(skb)->frag_list)
 #define SKB_LINEAR_ASSERT(skb)  BUG_ON(skb_is_nonlinear(skb))
 
+#ifdef NET_SKBUFF_DATA_USES_OFFSET
+static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
+{
+	return skb->head + skb->tail;
+}
+
+static inline void skb_reset_tail_pointer(struct sk_buff *skb)
+{
+	skb->tail = skb->data - skb->head;
+}
+
+static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
+{
+	skb_reset_tail_pointer(skb);
+	skb->tail += offset;
+}
+#else /* NET_SKBUFF_DATA_USES_OFFSET */
+static inline unsigned char *skb_tail_pointer(const struct sk_buff *skb)
+{
+	return skb->tail;
+}
+
+static inline void skb_reset_tail_pointer(struct sk_buff *skb)
+{
+	skb->tail = skb->data;
+}
+
+static inline void skb_set_tail_pointer(struct sk_buff *skb, const int offset)
+{
+	skb->tail = skb->data + offset;
+}
+#endif /* NET_SKBUFF_DATA_USES_OFFSET */
+
 /*
  *	Add data to an sk_buff
  */
 static inline unsigned char *__skb_put(struct sk_buff *skb, unsigned int len)
 {
-	unsigned char *tmp = skb->tail;
+	unsigned char *tmp = skb_tail_pointer(skb);
 	SKB_LINEAR_ASSERT(skb);
 	skb->tail += len;
 	skb->len  += len;
@@ -835,11 +868,11 @@ static inline unsigned char *__skb_put(struct sk_buff *skb, unsigned int len)
  */
 static inline unsigned char *skb_put(struct sk_buff *skb, unsigned int len)
 {
-	unsigned char *tmp = skb->tail;
+	unsigned char *tmp = skb_tail_pointer(skb);
 	SKB_LINEAR_ASSERT(skb);
 	skb->tail += len;
 	skb->len  += len;
-	if (unlikely(skb->tail>skb->end))
+	if (unlikely(skb_tail_pointer(skb) > skb->end))
 		skb_over_panic(skb, len, current_text_addr());
 	return tmp;
 }
@@ -935,7 +968,7 @@ static inline int skb_headroom(const struct sk_buff *skb)
  */
 static inline int skb_tailroom(const struct sk_buff *skb)
 {
-	return skb_is_nonlinear(skb) ? 0 : skb->end - skb->tail;
+	return skb_is_nonlinear(skb) ? 0 : skb->end - skb_tail_pointer(skb);
 }
 
 /**
@@ -1127,8 +1160,8 @@ static inline void __skb_trim(struct sk_buff *skb, unsigned int len)
 		WARN_ON(1);
 		return;
 	}
-	skb->len  = len;
-	skb->tail = skb->data + len;
+	skb->len = len;
+	skb_set_tail_pointer(skb, len);
 }
 
 /**
diff --git a/include/net/inet_ecn.h b/include/net/inet_ecn.h
index 06a2c69a89e..de8399a7977 100644
--- a/include/net/inet_ecn.h
+++ b/include/net/inet_ecn.h
@@ -114,14 +114,12 @@ static inline int INET_ECN_set_ce(struct sk_buff *skb)
 {
 	switch (skb->protocol) {
 	case __constant_htons(ETH_P_IP):
-		if (skb_network_header(skb) + sizeof(struct iphdr) <=
-		    skb->tail)
+		if (skb->network_header + sizeof(struct iphdr) <= skb->tail)
 			return IP_ECN_set_ce(ip_hdr(skb));
 		break;
 
 	case __constant_htons(ETH_P_IPV6):
-		if (skb_network_header(skb) + sizeof(struct ipv6hdr) <=
-		    skb->tail)
+		if (skb->network_header + sizeof(struct ipv6hdr) <= skb->tail)
 			return IP6_ECN_set_ce(ipv6_hdr(skb));
 		break;
 	}
diff --git a/include/net/netlink.h b/include/net/netlink.h
index bcaf67b7a19..2c7ab107f20 100644
--- a/include/net/netlink.h
+++ b/include/net/netlink.h
@@ -525,7 +525,7 @@ static inline struct sk_buff *nlmsg_new(size_t payload, gfp_t flags)
  */
 static inline int nlmsg_end(struct sk_buff *skb, struct nlmsghdr *nlh)
 {
-	nlh->nlmsg_len = skb->tail - (unsigned char *) nlh;
+	nlh->nlmsg_len = skb_tail_pointer(skb) - (unsigned char *)nlh;
 
 	return skb->len;
 }
@@ -538,7 +538,7 @@ static inline int nlmsg_end(struct sk_buff *skb, struct nlmsghdr *nlh)
  */
 static inline void *nlmsg_get_pos(struct sk_buff *skb)
 {
-	return skb->tail;
+	return skb_tail_pointer(skb);
 }
 
 /**
@@ -940,7 +940,7 @@ static inline unsigned long nla_get_msecs(struct nlattr *nla)
  */
 static inline struct nlattr *nla_nest_start(struct sk_buff *skb, int attrtype)
 {
-	struct nlattr *start = (struct nlattr *) skb->tail;
+	struct nlattr *start = (struct nlattr *)skb_tail_pointer(skb);
 
 	if (nla_put(skb, attrtype, 0, NULL) < 0)
 		return NULL;
@@ -960,7 +960,7 @@ static inline struct nlattr *nla_nest_start(struct sk_buff *skb, int attrtype)
  */
 static inline int nla_nest_end(struct sk_buff *skb, struct nlattr *start)
 {
-	start->nla_len = skb->tail - (unsigned char *) start;
+	start->nla_len = skb_tail_pointer(skb) - (unsigned char *)start;
 	return skb->len;
 }
 
diff --git a/include/net/pkt_cls.h b/include/net/pkt_cls.h
index dcb3a91f136..4129df70807 100644
--- a/include/net/pkt_cls.h
+++ b/include/net/pkt_cls.h
@@ -337,7 +337,7 @@ static inline unsigned char * tcf_get_base_ptr(struct sk_buff *skb, int layer)
 static inline int tcf_valid_offset(const struct sk_buff *skb,
 				   const unsigned char *ptr, const int len)
 {
-	return unlikely((ptr + len) < skb->tail && ptr > skb->head);
+	return unlikely((ptr + len) < skb_tail_pointer(skb) && ptr > skb->head);
 }
 
 #ifdef CONFIG_NET_CLS_IND