Don't set cur=inf RLIM_NOFILE on macOS

virProcessActivateMaxFiles sets rlim_cur to rlim_max.
If rlim_max is RLIM_INFINITY,

2023-08-15 15:17:51.944+0000: 4456752640: debug :
virProcessActivateMaxFiles:1067 : Initial max files was 2560
2023-08-15 15:17:51.944+0000: 4456752640: debug :
virProcessActivateMaxFiles:1077 : Raised max files to
9223372036854775807

then when virCommandMassClose does `int openmax = sysconf(
_SC_OPEN_MAX)`, `openmax < 0` is true and virCommandMassClose
reports an error and bails.  Setting rlim_cur instead to at most
OPEN_MAX, as macOS' documentation suggests, both avoids this problem

2023-08-18 16:01:44.366+0000: 4359562752: debug :
virProcessActivateMaxFiles:1072 : Initial max files was 256
2023-08-18 16:01:44.366+0000: 4359562752: debug :
virProcessActivateMaxFiles:1086 : Raised max files to 10240

and eliminates a case of what the documentation declares
to be invalid input to setrlimit anyway.

Signed-off-by: Laura Hild <lsh@jlab.org>

diff --git a/src/util/virprocess.c b/src/util/virprocess.c
index f8daa786c9c68d91b5f0852e249639dda84900a9..6ed2ac86fd8826a29cc370f64d885161db6b1eae 100644 (file)
--- a/src/util/virprocess.c
+++ b/src/util/virprocess.c
@@ -39,7 +39,7 @@
 # include <sched.h>
 #endif
 
-#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || WITH_BSD_CPU_AFFINITY
+#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || WITH_BSD_CPU_AFFINITY || defined(__APPLE__)
 # include <sys/param.h>
 #endif
 
@@ -61,6 +61,10 @@
 # include <sys/prctl.h>
 #endif
 
+#if defined(__APPLE__)
+# include <sys/syslimits.h>
+#endif
+
 #include "virprocess.h"
 #include "virerror.h"
 #include "viralloc.h"
@@ -1066,7 +1070,20 @@ virProcessActivateMaxFiles(void)
 
     VIR_DEBUG("Initial max files was %llu", (unsigned long long)maxfiles.rlim_cur);
 
+# if defined(__APPLE__)
+    /*
+     * rlim_max may be RLIM_INFINITY, and macOS 12.6.3 getrlimit(2) says
+     *
+     * COMPATIBILITY
+     *      setrlimit() now returns with errno set to EINVAL in places
+     *      that historically succeeded.  It no longer accepts
+     *      "rlim_cur = RLIM_INFINITY" for RLIM_NOFILE.
+     *      Use "rlim_cur = min(OPEN_MAX, rlim_max)".
+     */
+    maxfiles.rlim_cur = MIN(OPEN_MAX, maxfiles.rlim_max);
+# else
     maxfiles.rlim_cur = maxfiles.rlim_max;
+# endif
 
     if (setrlimit(RLIMIT_NOFILE, &maxfiles) < 0) {
         VIR_DEBUG("Unable to set process max files limit to %llu: %s",