Add an option to awake a single task on waitqueues at a time

With a large number of listeners on a single listen socket, waitqueues tend to
awake too many processes for a single incoming connection (known as the
thundering herd problem).

This patch provides an option WQ_FLAG_LOADBALANCE for waitqueues. When enabled,
this option makes waitqueues awake only one task at a time in a round robin
way. A file mode option FMODE_SINGLE_WAKEUP is also provided to mark files whose
events need be handled with WQ_FLAG_LOADBALANCE set.

Signed-off-by: Xiaofeng Lin <jerrylin.lxf@gmail.com>
Signed-off-by: Junjie Mao <eternal.n08@gmail.com>

Author: Junjie Mao

kernel/fs/eventpoll.c

@@ -979,6 +979,9 @@ static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead,
 		init_waitqueue_func_entry(&pwq->wait, ep_poll_callback);
 		pwq->whead = whead;
 		pwq->base = epi;
+		if (file->f_mode & FMODE_SINGLE_WAKEUP) {
+			pwq->wait.flags |= WQ_FLAG_LOADBALANCE;
+		}
 		add_wait_queue(whead, &pwq->wait);
 		list_add_tail(&pwq->llink, &epi->pwqlist);
 		epi->nwait++;

kernel/include/linux/fs.h

@@ -104,6 +104,8 @@ struct inodes_stat_t {
 #define FMODE_RANDOM		((__force fmode_t)4096)
 /* File for fastsocket */
 #define FMODE_FASTSOCKET	((__force fmode_t)8192)
+/* Wake up the task in a round bobin way, used with fastsocket*/
+#define FMODE_SINGLE_WAKEUP	((__force fmode_t)16384)
 
 /*
  * The below are the various read and write types that we support. Some of

kernel/include/linux/wait.h

@@ -32,6 +32,7 @@ int default_wake_function(wait_queue_t *wait, unsigned mode, int flags, void *ke
 struct __wait_queue {
 	unsigned int flags;
 #define WQ_FLAG_EXCLUSIVE	0x01
+#define WQ_FLAG_LOADBALANCE	0x02
 	void *private;
 	wait_queue_func_t func;
 	struct list_head task_list;

kernel/kernel/sched.c

@@ -6280,6 +6280,12 @@ static void __wake_up_common(wait_queue_head_t *q, unsigned int mode,
 	list_for_each_entry_safe(curr, next, &q->task_list, task_list) {
 		unsigned flags = curr->flags;
 
+		if (flags & WQ_FLAG_LOADBALANCE) {
+			curr->func(curr, mode, wake_flags, key);
+			list_move_tail(&curr->task_list, &q->task_list);
+			break;
+		}
+
 		if (curr->func(curr, mode, wake_flags, key) &&
 				(flags & WQ_FLAG_EXCLUSIVE) && !--nr_exclusive)
 			break;