strange argv handling / mangled argv ?

[es-fabricemarie]

I'm running Fedora 40 in a UTM VM on MacOS, on an M1 chip.

The relevant software versions I have are:

binfmt-dispatcher-0.1.2-1.fc40.aarch64
fex-emu-2412-3.fc40.aarch64

To test my setup, I made a small ls equivalent in standard C, and compiled it under Fedora 40 x86_64. I then copy it into my as /var/tmp/fexls. The source code for that test is attached as fexls.c. It expects a path argument on the command line and lists the files at that path like ls would. Again this is just to test my fex setup.

When I run /var/tmp/fexls /var/tmp from my shell, I get this:

[binfmt_dispatcher] Using FEX
argc = 4
argv[0] = '/var/tmp/fexls'
argv[1] = '/var/tmp/fexls'
argv[2] = '/var/tmp/fexls'
argv[3] = '/var/tmp/'
Usage: /var/tmp/fexls <target-directory>

Why is my argv mangled?

When I run directly (without binfmt-dispatcher) FEXInterpreter /var/tmp/fexls /var/tmp/ I get the correct output:

argc = 2
argv[0] = '/var/tmp/fexls'
argv[1] = '/var/tmp/'
drwxr-xr-x   1 abrt abrt        0 Apr 16 03:17 abrt
-rw-r--r--   1 root root     2298 Apr 16 03:19 fstab
-rw-r--r--   1 root root      280 Apr 22 20:14 .location
.... [more files] ...

My binfmt-dispatcher configuration file /etc/binfmt-dispatcher.toml is standard (I just disabled muvm only):

[defaults]
interpreter = "fex"
log_level = "info"

[muvm]
path = "/usr/bin/muvm"

[interpreters.box64]
path = "/usr/bin/box64"

[interpreters.fex]
name = "FEX"
path = "/usr/bin/FEXInterpreter"
required_paths = ["/usr/share/fex-emu/RootFS/default.erofs"]
use_muvm = false

[interpreters.qemu]
name = "QEMU User space emulator"
path = "/usr/bin/qemu-x86_64"

[interpreters.qemu-static]
name = "QEMU User space emulator (static)"
path = "/usr/bin/qemu-x86_64-static"

Am I doing something wrong? Or is there a bug here?

The C code for fexls.c:

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <dirent.h>
#include <sys/stat.h>
#include <pwd.h>
#include <grp.h>
#include <time.h>
#include <unistd.h>

void print_file_info(const char *path, const char *filename) {
    char fullpath[4096];
    snprintf(fullpath, sizeof(fullpath), "%s/%s", path, filename);

    struct stat st;
    if (lstat(fullpath, &st) == -1) {
        perror("lstat");
        return;
    }

    // File type and permissions
    char perms[11] = "----------";
    if (S_ISDIR(st.st_mode)) perms[0] = 'd';
    else if (S_ISLNK(st.st_mode)) perms[0] = 'l';
    else if (S_ISCHR(st.st_mode)) perms[0] = 'c';
    else if (S_ISBLK(st.st_mode)) perms[0] = 'b';
    else if (S_ISFIFO(st.st_mode)) perms[0] = 'p';
    else if (S_ISSOCK(st.st_mode)) perms[0] = 's';

    if (st.st_mode & S_IRUSR) perms[1] = 'r';
    if (st.st_mode & S_IWUSR) perms[2] = 'w';
    if (st.st_mode & S_IXUSR) perms[3] = 'x';
    if (st.st_mode & S_IRGRP) perms[4] = 'r';
    if (st.st_mode & S_IWGRP) perms[5] = 'w';
    if (st.st_mode & S_IXGRP) perms[6] = 'x';
    if (st.st_mode & S_IROTH) perms[7] = 'r';
    if (st.st_mode & S_IWOTH) perms[8] = 'w';
    if (st.st_mode & S_IXOTH) perms[9] = 'x';

    // User and group names
    struct passwd *pw = getpwuid(st.st_uid);
    struct group *gr = getgrgid(st.st_gid);

    // Modification time
    char timebuf[64];
    struct tm *mtm = localtime(&st.st_mtime);
    strftime(timebuf, sizeof(timebuf), "%b %e %H:%M", mtm);

    // Output format similar to ls -l
    printf("%s %3lu %s %s %8ld %s %s\n",
           perms,
           st.st_nlink,
           pw ? pw->pw_name : "???",
           gr ? gr->gr_name : "???",
           st.st_size,
           timebuf,
           filename);
}

int main(int argc, char *argv[]) {
    printf("argc = %d\n", argc);
    for (int i = 0; i < argc; ++i) {
        printf("argv[%d] = '%s'\n", i, argv[i]);
    }
    if (argc != 2) {
        fprintf(stderr, "Usage: %s <target-directory>\n", argv[0]);
        return EXIT_FAILURE;
    }

    const char *dirpath = argv[1];
    DIR *dir = opendir(dirpath);
    if (!dir) {
        perror("opendir");
        return EXIT_FAILURE;
    }

    struct dirent *entry;
    while ((entry = readdir(dir))) {
        if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
            continue;
        print_file_info(dirpath, entry->d_name);
    }

    closedir(dir);
    return EXIT_SUCCESS;
}

and I compile it with: gcc -Wall -Wextra -O2 -o fexls fexls.c.

I can confirm the ELF is indeed x86_64:

$ file /var/tmp/fexls 

/var/tmp/fexls: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, BuildID[sha1]=4efe18c6f1e3fa1fffbc8347a43cb3c48d4be70d, for GNU/Linux 3.2.0, not stripped

[davide125]

Thanks for the report. This is odd indeed, I wonder if it's a side effect of the way binfmt_misc passes arguments through to the handler.

[es-fabricemarie]

On Fedora 40, the package is configured to register itself to binfmt-misc with "POCF" flags. So binfmt-dispatcher receives the calling args several times (argv AND fd 3).

[mkurz]

Happened to me yesterday as well, not on Fedora but Asahi Alarm, but still. We ship binfmt-dispatcher as well: https://github.com/asahi-alarm/PKGBUILDs/blob/main/binfmt-dispatcher/PKGBUILD

[es-fabricemarie]

Mine works after changing this:

diff --git a/src/main.rs b/src/main.rs
index b7305ad..3b6a26b 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -42,7 +42,7 @@ fn main() {
     trace!("Configuration:\n{:#?}", settings);
 
     // Collect command line arguments to pass through
-    let args: Vec<OsString> = env::args_os().skip(1).collect();
+    let mut args: Vec<OsString> = env::args_os().skip(1).collect();
     trace!("Args:\n{:#?}", args);
     if args.is_empty() {
         abort("No arguments passed, re-run with --help to learn more.");
@@ -60,13 +60,6 @@ fn main() {
         }
     }
 
-    // File descriptor 3 is where binfmt_misc typically passes the executable
-    let binary: PathBuf = read_link("/proc/self/fd/3").unwrap_or_else(|e| {
-        error!("Failed to read the executable from fd#3: {}", e);
-        exit(1);
-    });
-    trace!("Binary: {:#?}", binary);
-
     let mut interpreter_id = &settings.defaults.interpreter;
     for binary in settings.binaries.values() {
         if Path::new(&binary.path) == canonicalize(&args[0]).unwrap() {
@@ -156,8 +149,9 @@ fn main() {
         command = Command::new(interpreter_path);
     }
 
-    command.arg(binary);
-
+    let new_binary = args.remove(0);
+    args.remove(0);
+    command.arg(new_binary);
     // Pass through all the arguments
     command.args(&args);

Hope this helps.

The reason why I'm dropping what is given through fd-3 is we lose the original command line if we did this: fd-3 will give the full path to the actual binary. And some programs check how they are called to perform different actions (decompress vs compress for example).

[mkurz]

Maybe you should open a pull request?

[davide125]

Ok, there's a few things going on here. Per the spec, when using the O flag the kernel is supposed to open the binary for us and pass the file descriptor to us. This is preferred because it allows the interpreter to work even when the binary isn't readable directly by it. The read_link() hack you see there was the only way I could get it to work. I wasn't expecting this to end up including the args as well. Ideally, we wouldn't do that at all, and just hand over the fd to the underlying implementation to consume.

For the time being, I agree it probably makes sense to drop this logic, at least until it can be implemented correctly. If we do that though we should also drop the O from the flags for clarity. As suggested, please feel free to send a PR for this.