udp_buffer: add UDP buffer tuning for udp_mem, rmem_max,default

Trace UDP receive fails and based on the error code (-ENOBUFS
for overall UDP memory exhaustion, -ENOMEM for socket receive
buffer exhaustion), modify UDP-related buffer parameters.

Similar to TCP, look for memory pressure/memory exhaustion
approach in UDP and modify udp_mem and rmem appropriately
to avoid exhaustion where possible or mitigate it if unavoidable.

For net.core.rmem_max, if a socket experiences losses with a
rcvbuf ~= rmem_max, increase rmem_max.  If a socket experiences
losses with rcvbuf ~= rmem_default _and_ it is not locked - i.e.
does not have an explicit rcvbuf value set by setsockopt -
increase rmem_default also.

Note that a given socket - because UDP has a fixed rcvbuf
size - will only contribute to such updates where its rcvbuf
is within range of rmem_max,default.  So if a particular socket
cannot cope with load consistently - rathern than the kind of
bursty issues increasing rmem can help with - it will not
keep contributing to rmem updates.

Add tests for rmem updates, rmem locked (where we expect an
explicit setsockopt from iperf3 will lock rcvbuf so no updates
to rmem_default are expected) and udp mem exhaustion.

Signed-off-by: Alan Maguire <alan.maguire@oracle.com>

Author: alan-maguire

README.md

@@ -166,6 +166,8 @@ The key components are
   which helps power namespace awareness for bpftune as a whole.
   Namespace awareness is important as we want to be able to auto-tune
   containers also.  See [bpftune-netns (8)](./docs/bpftune-netns.rst)
+- UDP buffer tuner: auto-tune buffers relating to UDP. See
+  [bpftune-udp-buffer (8)](./docs/bpftune-udp-buffer.rst)
 
 ## Code organization
 

docs/Makefile

@@ -36,7 +36,7 @@ man8dir = $(mandir)/man8
 
 MAN8_RST = bpftune.rst bpftune-sysctl.rst bpftune-tcp-conn.rst \
 	   bpftune-neigh.rst bpftune-tcp-buffer.rst bpftune-netns.rst \
-	   bpftune-net-buffer.rst bpftune-ip-frag.rst
+	   bpftune-net-buffer.rst bpftune-ip-frag.rst bpftune-udp-buffer.rst
 
 _DOC_MAN8 = $(patsubst %.rst,%.8,$(MAN8_RST))
 DOC_MAN8 = $(addprefix $(OUTPUT),$(_DOC_MAN8))

docs/bpftune-udp-buffer.rst

@@ -0,0 +1,37 @@
+==================
+BPFTUNE-UDP-BUFFER
+==================
+-------------------------------------------------------------------------------
+UDP buffer bpftune plugin for managing UDP buffer sizes, memory limits
+-------------------------------------------------------------------------------
+
+:Manual section: 8
+
+
+DESCRIPTION
+===========
+
+        For UDP - like TCP - a triple of min, pressure, max
+        represents UDP memory limits and is specified in
+
+          net.ipv4.udp_mem
+
+        If receive fails with -ENOBUFS this indicates memory
+        limits are being reached; we adaptively increase pressure and
+        max to ensure that memory exhaustion does not occur (as long
+        as we do not approach real memory exhaustion).  As memory
+        exhaustion is approached and we can no longer increase
+        overall memory limits, reduce net.core.rmem* values to limit
+        socket memory overheads.
+
+        For UDP receive buffer memory, bump net.core.rmem_max if
+        a socket experiences receive buffer drops within range of
+        the rmem_max_value.  Similarly bump rmem_default if sockets are
+        within range of it and do not have a locked (via setsockopt)
+        value.
+
+        Tunables:
+
+        - net.ipv4.udp_mem: min, pressure, max UDP memory
+        - net.core.rmem_max: max rcvbuf size specifiable via setsockopt()
+        - net.core.rmem_default: default rcvbuf size where none was set

include/bpftune/bpftune.bpf.h

@@ -233,6 +233,10 @@ unsigned long long bpftune_init_net;
 #define SK_MEM_QUANTUM_SHIFT	ilog2(SK_MEM_QUANTUM)
 #endif
 
+#ifndef SOL_SOCKET
+#define SOL_SOCKET		1
+#endif
+
 #ifndef SOL_TCP
 #define SOL_TCP        		6
 #endif
@@ -274,6 +278,9 @@ unsigned long long bpftune_init_net;
 #endif
 
 #define EINVAL		22
+#define ENOMEM		12
+#define EAGAIN		11
+#define ENOBUFS		105
 
 bool debug;
 
@@ -348,7 +355,6 @@ static __always_inline long send_net_sysctl_event(struct net *net,
 	__u64 event_key = 0;
 	long nscookie = 0;
 	__u64 *last_timep = NULL;
-	int ret = 0;
 
 	nscookie = get_netns_cookie(net);
 	if (nscookie < 0)
@@ -359,7 +365,7 @@ static __always_inline long send_net_sysctl_event(struct net *net,
 	last_timep = bpf_map_lookup_elem(&last_event_map, &event_key);
 	if (last_timep) {
 		if ((now - *last_timep) < (25 * MSEC))
-			return 0;
+			return -EAGAIN;
 		*last_timep = now;
 	} else {
 		bpf_map_update_elem(&last_event_map, &event_key, &now, 0);
@@ -375,11 +381,7 @@ static __always_inline long send_net_sysctl_event(struct net *net,
 	event->update[0].new[0] = new[0];
 	event->update[0].new[1] = new[1];
 	event->update[0].new[2] = new[2];
-	ret = bpf_ringbuf_output(&ring_buffer_map, event, sizeof(*event), 0);
-	bpftune_debug("tuner [%d] scenario [%d]: event send: %d ",
-		    tuner_id, scenario_id, ret);
-	bpftune_debug("\told '%ld %ld %ld'\n", old[0], old[1], old[2]);
-	bpftune_debug("\tnew '%ld %ld %ld'\n", new[0], new[1], new[2]);
+	bpf_ringbuf_output(&ring_buffer_map, event, sizeof(*event), 0);
 	return 0;
 }
 

include/bpftune/bpftune.h

@@ -54,6 +54,9 @@ extern unsigned short bpftune_learning_rate;
 /* shrink by delta (default 25%) */
 #define BPFTUNE_SHRINK_BY_DELTA(val)  ((val) - ((val) >> BPFTUNE_BITSHIFT))
 
+#define BPFTUNE_WITHIN_BITSHIFT(val1, val2, shift)			\
+	((val1 + (val1 >> shift)) > val2)
+
 #define MSEC				((__u64)1000000)
 #define SECOND				((__u64)1000000000)
 #define MINUTE				(60 * SECOND)

include/bpftune/libbpftune.h

@@ -351,10 +351,11 @@ void bpftune_sysctl_name_to_path(const char *name, char *path, size_t path_sz);
 int bpftune_sysctl_read(int netns_fd, const char *name, long *values);
 int bpftune_sysctl_write(int netns_fd, const char *name, __u8 num_values, long *values);
 long long bpftune_ksym_addr(char type, const char *name);
-int bpftune_snmpstat_read(unsigned long netns_cookie, int family, const char *name, long *value);
-int bpftune_netstat_read(unsigned long netns_cookie, int family, const char *name, long *value);
+int bpftune_snmpstat_read(unsigned long netns_cookie, int family, const char *linename, const char *name, long *value);
+int bpftune_netstat_read(unsigned long netns_cookie, int family, const char *linename, const char *name, long *value);
 int bpftune_sched_wait_run_percent_read(void);
 bool bpftune_netns_cookie_supported(void);
+unsigned long bpftune_global_netns_cookie(void);
 int bpftune_netns_set(int fd, int *orig_fd, bool quiet);
 int bpftune_netns_info(int pid, int *fd, unsigned long *cookie);
 int bpftune_netns_init_all(void);

src/Makefile

@@ -99,7 +99,8 @@ submake_extras := feature_display=0
 endif
 
 TUNERS = tcp_buffer_tuner route_table_tuner neigh_table_tuner sysctl_tuner \
-	 tcp_conn_tuner netns_tuner net_buffer_tuner ip_frag_tuner
+	 tcp_conn_tuner netns_tuner net_buffer_tuner ip_frag_tuner \
+	 udp_buffer_tuner
 
 TUNER_OBJS = $(patsubst %,%.o,$(TUNERS))
 TUNER_SRCS = $(patsubst %,%.c,$(TUNERS))

src/ip_frag_tuner.c

@@ -75,9 +75,9 @@ void event_handler(struct bpftuner *tuner,
 	key.netns_cookie = event->netns_cookie;
 
 	af = id == IP_FRAG_MAX_THRESHOLD ? AF_INET : AF_INET6;
-	if (!bpftune_snmpstat_read(event->netns_cookie, af,
+	if (!bpftune_snmpstat_read(event->netns_cookie, af, NULL,
 				   "ReasmFails", &reasmfails) &&
-	    !bpftune_snmpstat_read(event->netns_cookie, af,
+	    !bpftune_snmpstat_read(event->netns_cookie, af, NULL,
 				   "ReasmReqds", &reasmreqds)) {
 		/* % of reasm fails */
 		reasm_failrate = (reasmfails * 100)/reasmreqds;

src/libbpftune.c

@@ -692,6 +692,13 @@ static void bpftune_global_netns_init(void)
 	}
 }
 
+unsigned long bpftune_global_netns_cookie(void)
+{
+	if (!global_netns_cookie)
+		bpftune_global_netns_init();
+	return global_netns_cookie;
+}
+
 /* add a tuner to the list of tuners, or replace existing inactive tuner.
  * If successful, call init().
  */
@@ -725,6 +732,8 @@ struct bpftuner *bpftuner_init(const char *path)
 		free(tuner);
 		return NULL;
 	}
+	tuner->bpf_support = support_level;
+
 	/* If we have a ringbuf fd from any tuner, use its fd to be re-used
  	 * for other ringbuf maps (so we can use the same ring buffer for all
  	 * BPF events.
@@ -1135,7 +1144,8 @@ long long bpftune_ksym_addr(char type, const char *name)
 }
 
 static int bpftune_nstat_read(unsigned long netns_cookie, int family,
-			      const char *file, const char *name, long *value)
+			      const char *file, const char *linename,
+			      const char *name, long *value)
 {
 	int err, netns_fd = 0, orig_netns_fd = 0, stat_index = 0;
 	char line[1024];
@@ -1188,6 +1198,12 @@ static int bpftune_nstat_read(unsigned long netns_cookie, int family,
 			 * have same index on the next line.
 			 */
 			if (strcmp(next, name) == 0) {
+				/* ensure line is Udp: or Tcp: if specified;
+				 * avoids matching InErrs for wrong protocol.
+				 */
+				if (linename &&
+				    strncmp(line, linename, strlen(linename)) != 0)
+					continue;
 				stat_index = index;
 				break;
 			}
@@ -1207,19 +1223,19 @@ static int bpftune_nstat_read(unsigned long netns_cookie, int family,
 }
 
 int bpftune_snmpstat_read(unsigned long netns_cookie, int family,
-                          const char *name, long *value)
+                          const char *linename, const char *name, long *value)
 {
 	return bpftune_nstat_read(netns_cookie, family,
 				  family == AF_INET ? "/proc/net/snmp" :
 				 		      "/proc/net/snmp6",
-				  name, value);
+				  linename, name, value);
 }
 
 int bpftune_netstat_read(unsigned long netns_cookie, int family,
-			 const char *name, long *value)
+			 const char *linename, const char *name, long *value)
 {
 	return bpftune_nstat_read(netns_cookie, family, "/proc/net/netstat",
-				  name, value);
+				  linename, name, value);
 }
 
 /* return % of overall wait/run time on all cpus gathered from

src/libbpftune.map

@@ -41,6 +41,7 @@ LIBBPFTUNE_0.1.1 {
 		bpftuner_bpf_prog_in_strategy;
 		bpftuner_tunables_fini;
 		bpftune_netns_cookie_supported;
+		bpftune_global_netns_cookie;
 		bpftuner_netns_init;
 		bpftuner_netns_fini;
 		bpftuner_netns_from_cookie;

src/tcp_buffer_tuner.c

@@ -373,9 +373,9 @@ void event_handler(struct bpftuner *tuner,
 			break;
 		t = bpftuner_tunable(tuner, TCP_BUFFER_TCP_SYNCOOKIES);
 		if (t && t->current_values[0] > 0 &&
-		    !bpftune_netstat_read(event->netns_cookie, AF_INET,
+		    !bpftune_netstat_read(event->netns_cookie, AF_INET, NULL,
 					  "SyncookiesRecv", &goodcookies) &&
-		    !bpftune_netstat_read(event->netns_cookie, AF_INET,
+		    !bpftune_netstat_read(event->netns_cookie, AF_INET, NULL,
 					  "SyncookiesFailed", &badcookies)) {
 
 			/* syncookies are enabled; are they effective? compare good/bad counts.