xen/list: Remove prefetching

Xen inherited its list infrastructure from Linux.  One area where has fallen
behind is that of prefetching, which as it turns out is a performance penalty
in most cases.

Prefetch of NULL on x86 is now widely measured to have glacial performance
properties, and will unconditionally hit on every hlist use due to the
termination condition.

Cross-port the following Linux patches:

  75d65a425c (2011) "hlist: remove software prefetching in hlist iterators"
  e66eed651f (2011) "list: remove prefetching from regular list iterators"
  c0d15cc7ee (2013) "linked-list: Remove __list_for_each"

to Xen, which results in the following net diffstat on x86:

  add/remove: 0/1 grow/shrink: 27/83 up/down: 576/-1648 (-1072)

(The code additions comes from a few now-inlined functions, and slightly
different basic block padding.)

No functional change.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Acked-by: Julien Grall <julien@xen.org>
Reviewed-by: Roger Pau Monné <roger.pau@citrix.com>

diff --git a/xen/include/xen/list.h b/xen/include/xen/list.h
index 1387abb21192668f4103620520cf4256902a45aa..dc5a8c461b9ce4bd376fac77587e0be5bfbe3342 100644 (file)
--- a/xen/include/xen/list.h
+++ b/xen/include/xen/list.h
@@ -42,9 +42,6 @@ struct list_head {
 #define LIST_HEAD_READ_MOSTLY(name) \
     struct list_head __read_mostly name = LIST_HEAD_INIT(name)
 
-/* Do not move this ahead of the struct list_head definition! */
-#include <xen/prefetch.h>
-
 static inline void INIT_LIST_HEAD(struct list_head *list)
 {
     list->next = list;
@@ -455,20 +452,6 @@ static inline void list_splice_init(struct list_head *list,
  * @head:    the head for your list.
  */
 #define list_for_each(pos, head)                                        \
-    for (pos = (head)->next; prefetch(pos->next), pos != (head);        \
-         pos = pos->next)
-
-/**
- * __list_for_each - iterate over a list
- * @pos:    the &struct list_head to use as a loop cursor.
- * @head:   the head for your list.
- *
- * This variant differs from list_for_each() in that it's the
- * simplest possible list iteration code, no prefetching is done.
- * Use this for code that knows the list to be very short (empty
- * or 1 entry) most of the time.
- */
-#define __list_for_each(pos, head)                              \
     for (pos = (head)->next; pos != (head); pos = pos->next)
 
 /**
@@ -477,8 +460,7 @@ static inline void list_splice_init(struct list_head *list,
  * @head:   the head for your list.
  */
 #define list_for_each_prev(pos, head)                                   \
-    for (pos = (head)->prev; prefetch(pos->prev), pos != (head);        \
-         pos = pos->prev)
+    for (pos = (head)->prev; pos != (head); pos = pos->prev)
 
 /**
  * list_for_each_safe - iterate over a list safe against removal of list entry
@@ -509,7 +491,7 @@ static inline void list_splice_init(struct list_head *list,
  */
 #define list_for_each_entry(pos, head, member)                          \
     for (pos = list_entry((head)->next, typeof(*pos), member);          \
-         prefetch(pos->member.next), &pos->member != (head);            \
+         &pos->member != (head);                                        \
          pos = list_entry(pos->member.next, typeof(*pos), member))
 
 /**
@@ -520,7 +502,7 @@ static inline void list_splice_init(struct list_head *list,
  */
 #define list_for_each_entry_reverse(pos, head, member)                  \
     for (pos = list_entry((head)->prev, typeof(*pos), member);          \
-         prefetch(pos->member.prev), &pos->member != (head);            \
+         &pos->member != (head);                                        \
          pos = list_entry(pos->member.prev, typeof(*pos), member))
 
 /**
@@ -547,7 +529,7 @@ static inline void list_splice_init(struct list_head *list,
  */
 #define list_for_each_entry_continue(pos, head, member)                 \
     for (pos = list_entry(pos->member.next, typeof(*pos), member);      \
-         prefetch(pos->member.next), &pos->member != (head);            \
+         &pos->member != (head);                                        \
          pos = list_entry(pos->member.next, typeof(*pos), member))
 
 /**
@@ -560,7 +542,7 @@ static inline void list_splice_init(struct list_head *list,
  * Iterate over list of given type, continuing from current position.
  */
 #define list_for_each_entry_from(pos, head, member)                     \
-    for (; prefetch(pos->member.next), &pos->member != (head);          \
+    for (; &pos->member != (head);                                      \
          pos = list_entry(pos->member.next, typeof(*pos), member))
 
 /**
@@ -635,7 +617,7 @@ static inline void list_splice_init(struct list_head *list,
  */
 #define list_for_each_rcu(pos, head)                            \
     for (pos = (head)->next;                                    \
-         prefetch(rcu_dereference(pos)->next), pos != (head);   \
+         rcu_dereference(pos) != (head);                        \
          pos = pos->next)
 
 #define __list_for_each_rcu(pos, head)          \
@@ -672,8 +654,7 @@ static inline void list_splice_init(struct list_head *list,
  */
 #define list_for_each_entry_rcu(pos, head, member)                      \
     for (pos = list_entry((head)->next, typeof(*pos), member);          \
-         prefetch(rcu_dereference(pos)->member.next),                   \
-         &pos->member != (head);                                        \
+         &rcu_dereference(pos)->member != (head);                       \
          pos = list_entry(pos->member.next, typeof(*pos), member))
 
 /**
@@ -689,7 +670,7 @@ static inline void list_splice_init(struct list_head *list,
  */
 #define list_for_each_continue_rcu(pos, head)                           \
     for ((pos) = (pos)->next;                                           \
-         prefetch(rcu_dereference((pos))->next), (pos) != (head);       \
+         rcu_dereference(pos) != (head);                                \
          (pos) = (pos)->next)
 
 /*
@@ -918,8 +899,7 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
 #define hlist_entry(ptr, type, member) container_of(ptr,type,member)
 
 #define hlist_for_each(pos, head)                                       \
-    for (pos = (head)->first; pos && ({ prefetch(pos->next); 1; });     \
-         pos = pos->next)
+    for (pos = (head)->first; pos; pos = pos->next)
 
 #define hlist_for_each_safe(pos, n, head)                       \
     for (pos = (head)->first; pos && ({ n = pos->next; 1; });   \
@@ -934,7 +914,7 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
  */
 #define hlist_for_each_entry(tpos, pos, head, member)                   \
     for (pos = (head)->first;                                           \
-         pos && ({ prefetch(pos->next); 1;}) &&                         \
+         pos &&                                                         \
          ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;});       \
          pos = pos->next)
 
@@ -947,7 +927,7 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
  */
 #define hlist_for_each_entry_continue(tpos, pos, member)                \
     for (pos = (pos)->next;                                             \
-         pos && ({ prefetch(pos->next); 1;}) &&                         \
+         pos &&                                                         \
          ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;});       \
          pos = pos->next)
 
@@ -959,7 +939,7 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
  * @member:    the name of the hlist_node within the struct.
  */
 #define hlist_for_each_entry_from(tpos, pos, member)                    \
-    for (; pos && ({ prefetch(pos->next); 1;}) &&                       \
+    for (; pos &&                                                       \
          ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;});       \
          pos = pos->next)
 
@@ -992,7 +972,7 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev,
  */
 #define hlist_for_each_entry_rcu(tpos, pos, head, member)               \
      for (pos = (head)->first;                                          \
-          rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) &&       \
+          rcu_dereference(pos) &&                                       \
           ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;});      \
           pos = pos->next)