Allow custom search path for libnl; try pkg-config when needed

Introduce LIBNL3_LIBDIR configure variable and preprocessor token to
specify a custom path for loading libnl-3 and libnl-genl-3 using
dlopen(3). This is helpful when libnl packages are installed in a
location outside the default search paths of the libraries.
Gentoo and NixOS need this feature to build.

User may specify LIBNL3_LIBDIR manually during configure. If it's not
specified, configure will try the following in order:
(1) find whether libnl exists within the default library search paths
    by try linking it (LIBNL3_LIBDIR being empty),
(2) locate the 'libdir' value of libnl-3.0 package through pkg-config
    and try linking with the directory included, and
(3) if both fail, go back to the default search paths (LIBNL3_LIBDIR
    being empty) and assume the library doesn't exist during the build
    but will exist at htop runtime.

Co-authored-by: Franz Pletz <fpletz@fnordicwalking.de>
Signed-off-by: Kang-Che Sung <explorer09@gmail.com>

Author: Explorer09

configure.ac

@@ -1015,6 +1015,29 @@ case "$enable_capabilities" in
 esac
 
 
+# $1: libnl-3 search path
+htop_try_link_libnl3 () {
+   htop_save_LDFLAGS=$LDFLAGS
+   htop_save_LIBS=$LIBS
+
+   LIBS="-lnl-3 $LIBS"
+   if test "x$1" != x; then
+      # New library path searched after what user has specified
+      LDFLAGS="$LDFLAGS -L$1"
+   fi
+   AC_LINK_IFELSE([AC_LANG_PROGRAM([[
+/* struct nl_sock* nl_socket_alloc(void); */
+void* nl_socket_alloc(void);
+      ]], [[
+void* sock = nl_socket_alloc();
+      ]])],
+      [htop_libnl3_link_succeed=yes],
+      [htop_libnl3_link_succeed=no])
+
+   LDFLAGS=$htop_save_LDFLAGS
+   LIBS=$htop_save_LIBS
+} # htop_try_link_libnl3
+
 AC_ARG_ENABLE([delayacct],
               [AS_HELP_STRING([--enable-delayacct],
                               [enable Linux delay accounting support; requires libnl-3 and libnl-genl-3 @<:@default=check@:>@])],
@@ -1045,7 +1068,7 @@ case "$enable_delayacct" in
          ])
       fi
 
-      old_CFLAGS="$CFLAGS"
+      htop_save_CFLAGS=$CFLAGS
       # New include path searched after what user has specified
       CFLAGS="$CFLAGS $LIBNL3_CFLAGS"
       AC_CHECK_HEADERS([netlink/attr.h netlink/handlers.h netlink/msg.h],
@@ -1054,9 +1077,32 @@ case "$enable_delayacct" in
             AC_MSG_ERROR([can not find required header files netlink/attr.h, netlink/handlers.h, netlink/msg.h])
          fi
          enable_delayacct=no])
-      CFLAGS="$old_CFLAGS"
+      CFLAGS=$htop_save_CFLAGS
 
       if test "$enable_delayacct" != no; then
+         AC_MSG_CHECKING([the search path of libnl-3])
+
+         htop_libnl3_link_succeed=no
+         htop_try_link_libnl3 "$LIBNL3_LIBDIR"
+         if test "$htop_libnl3_link_succeed${LIBNL3_LIBDIR+y}" = no && test "x$PKG_CONFIG" != x; then
+            LIBNL3_LIBDIR=`$PKG_CONFIG --variable=libdir libnl-3.0 2>/dev/null`
+            if test "x$LIBNL3_LIBDIR" != x; then
+               htop_try_link_libnl3 "$LIBNL3_LIBDIR"
+            fi
+         fi
+
+         if test "x$LIBNL3_LIBDIR" = x; then
+            AC_MSG_RESULT([(default)])
+         else
+            # The path must end with a slash
+            LIBNL3_LIBDIR=`echo "x$LIBNL3_LIBDIR" | sed 's/^x//; s|/*$|/|'`
+            AC_MSG_RESULT([$LIBNL3_LIBDIR])
+         fi
+         AC_DEFINE_UNQUOTED([LIBNL3_LIBDIR], ["$LIBNL3_LIBDIR"], [libnl-3 search path; use the default search paths if empty])
+         if test "$htop_libnl3_link_succeed" = no; then
+            AC_MSG_WARN([libnl-3 binary currently not present; will be needed in htop runtime for delay accounting support])
+         fi
+
          enable_delayacct=yes
       fi
       ;;

linux/LibNl.c

@@ -78,17 +78,17 @@ static int load_libnl(void) {
    if (libnlHandle && libnlGenlHandle)
       return 0;
 
-   libnlHandle = dlopen("libnl-3.so", RTLD_LAZY);
+   libnlHandle = dlopen(LIBNL3_LIBDIR "libnl-3.so", RTLD_LAZY);
    if (!libnlHandle) {
-      libnlHandle = dlopen("libnl-3.so.200", RTLD_LAZY);
+      libnlHandle = dlopen(LIBNL3_LIBDIR "libnl-3.so.200", RTLD_LAZY);
       if (!libnlHandle) {
          goto dlfailure;
       }
    }
 
-   libnlGenlHandle = dlopen("libnl-genl-3.so", RTLD_LAZY);
+   libnlGenlHandle = dlopen(LIBNL3_LIBDIR "libnl-genl-3.so", RTLD_LAZY);
    if (!libnlGenlHandle) {
-      libnlGenlHandle = dlopen("libnl-genl-3.so.200", RTLD_LAZY);
+      libnlGenlHandle = dlopen(LIBNL3_LIBDIR "libnl-genl-3.so.200", RTLD_LAZY);
       if (!libnlGenlHandle) {
          goto dlfailure;
       }