ftrace ftrace_set_filter_ip/ftrace_set_filter return EINVAL

I have asked a question about hooking functions in Linux kernel. Now I'm learning about ftrace.

I wrote a sample program that you can see below, it worked fine on Linux 5.x and 6.x 66-bit as well as Aarch64 on a Raspberry Pi 4 and it worked as expected.

Now I'm testing it on an older kernel, 4.x running inside Qemu as it's Aarch64. But I keep getting EINVAL from ftrace_set_filter() and ftrace_set_filter_ip().

This code is inspired by xcellerator examples https://github.com/xcellerator/linux_kernel_hacking, I modified the source code a lot based on anything I could find online to fix my issue, so any problem with the code is probably mine mistake and not the original authors.

Header,

/*
 * Helper library for ftrace hooking kernel functions
 * Author: Harvey Phillips (xcellerator@gmx.com)
 * License: GPL
 * */

#include <linux/ftrace.h>
#include <linux/linkage.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/version.h>

#if LINUX_VERSION_CODE < KERNEL_VERSION(5,11,0)
#define FTRACE_OPS_FL_RECURSION FTRACE_OPS_FL_RECURSION_SAFE
#endif

#if LINUX_VERSION_CODE < KERNEL_VERSION(5,11,0)
#define ftrace_regs pt_regs

static __always_inline struct pt_regs *ftrace_get_regs(struct ftrace_regs *fregs)
{
        return fregs;
}
#endif

#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,7,0)
#define KPROBE_LOOKUP 1
#include <linux/kprobes.h>
static struct kprobe kp = {
    .symbol_name = "kallsyms_lookup_name"
};
#endif

#define HOOK(_name, _hook, _orig)   \
{                                   \
    .name = (_name),                \
    .function = (_hook),            \
    .original = (_orig),            \
}

#define USE_FENTRY_OFFSET 0
#if !USE_FENTRY_OFFSET
#pragma GCC optimize("-fno-optimize-sibling-calls")
#endif

struct ftrace_hook {
    const char *name;
    void *function;
    void *original;

    unsigned long address;
    struct ftrace_ops ops;
};

static int fh_resolve_hook_address(struct ftrace_hook *hook)
{
#ifdef KPROBE_LOOKUP
    typedef unsigned long (*kallsyms_lookup_name_t)(const char *name);
    kallsyms_lookup_name_t kallsyms_lookup_name;
    int ret;

    ret = register_kprobe(&kp);
    if (ret < 0) {
        printk("register_kprobe failed: %d\n", ret);
        return ret;
    }

    kallsyms_lookup_name = (kallsyms_lookup_name_t) kp.addr;
    unregister_kprobe(&kp);
#endif

    hook->address = kallsyms_lookup_name(hook->name);

    if (!hook->address) {
        printk("unresolved symbol: %s\n", hook->name);
        return -ENOENT;
    }

#if USE_FENTRY_OFFSET
    *((unsigned long*) hook->original) = hook->address + MCOUNT_INSN_SIZE;
#else
    *((unsigned long*) hook->original) = hook->address;
#endif

    printk("Resolved address of %s: %lx\n", hook->name, hook->address);
    return 0;
}

static void notrace fh_ftrace_thunk(unsigned long ip, unsigned long parent_ip,
                                    struct ftrace_ops *ops, struct ftrace_regs *fregs)
{
    printk("fh_ftrace_thunk - Init\n");

    struct pt_regs *regs = ftrace_get_regs(fregs);
    printk("ftrace_get_regs - After\n");

    struct ftrace_hook *hook = container_of(ops, struct ftrace_hook, ops);

#if USE_FENTRY_OFFSET
    regs->pc = (unsigned long)hook->function;
#else
    if (!within_module(parent_ip, THIS_MODULE))
        regs->pc = (unsigned long)hook->function;
#endif
}

int fh_install_hook(struct ftrace_hook *hook)
{
    int err;

    printk("fh_install_hook - prologue\n");

    err = fh_resolve_hook_address(hook);
    printk("fh_resolve_hook - calling fh_resolve_hook_address\n");
    if (err) {
        printk("fh_resolve_hook_address() failed: %d\n", err);
        return err;
    }

    hook->ops.func = fh_ftrace_thunk;
    hook->ops.flags = FTRACE_OPS_FL_SAVE_REGS
                    | FTRACE_OPS_FL_RECURSION
                    | FTRACE_OPS_FL_IPMODIFY;

    printk("fh_install_hook - BEFORE ftrace_set_filter_ip\n");
    err = ftrace_set_filter_ip(&hook->ops, hook->address, 0, 0);
    printk("fh_install_hook - AFTER ftrace_set_filter_ip\n");
    if (err) {
        printk("ftrace_set_filter_ip() failed: %d\n", err);
        return err;
    }

    printk("register_ftrace_function - before\n");
    err = register_ftrace_function(&hook->ops);
    printk("register_ftrace_function - after\n");
    if (err) {
        printk("register_ftrace_function() failed: %d\n", err);
        ftrace_set_filter_ip(&hook->ops, hook->address, 1, 0);
        return err;
    }

    printk("register_ftrace_function - return\n");
    return 0;
}

/**
 * fh_remove_hooks() - disable and unregister a single hook
 * @hook: a hook to remove
 */
void fh_remove_hook(struct ftrace_hook *hook)
{
    int err;

    err = unregister_ftrace_function(&hook->ops);
    if (err) {
        printk("unregister_ftrace_function() failed: %d\n", err);
    }

    err = ftrace_set_filter_ip(&hook->ops, hook->address, 1, 0);
    if (err) {
        printk("ftrace_set_filter_ip() failed: %d\n", err);
    }
}

/**
 * fh_install_hooks() - register and enable multiple hooks
 * @hooks: array of hooks to install
 * @count: number of hooks to install
 *
 * If some hooks fail to install then all hooks will be removed.
 *
 * Returns: zero on success, negative error code otherwise.
 */
int fh_install_hooks(struct ftrace_hook *hooks, size_t count)
{
    int err;
    size_t i;

    for (i = 0; i < count; i++) {
        printk("fh_install_hooks - %lu\n", i);
        err = fh_install_hook(&hooks[i]);
        if (err)
            goto error;
    }

    return 0;

error:
    printk("fh_install_hooks - error statement\n");
    while (i != 0) {
        fh_remove_hook(&hooks[--i]);
    }

    return err;
}

/**
 * fh_remove_hooks() - disable and unregister multiple hooks
 * @hooks: array of hooks to remove
 * @count: number of hooks to remove
 */
void fh_remove_hooks(struct ftrace_hook *hooks, size_t count)
{
    size_t i;

    for (i = 0; i < count; i++)
        fh_remove_hook(&hooks[i]);
}

Kernel module,

#include <linux/init.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/ftrace.h>
#include <linux/ptrace.h>
#include <linux/sched.h>
#include <linux/uaccess.h>
#include <linux/version.h>
#include "ftrace_helper.h"

MODULE_LICENSE("GPL");
MODULE_AUTHOR("jelal");
MODULE_DESCRIPTION("Hook into execve syscall");
MODULE_VERSION("0.01");

#ifdef CONFIG_ARM64
static asmlinkage long (*orig_sys_execve)(const char __user *filename,
                                          const char __user *const __user *argv,
                                          const char __user *const __user *envp);

static asmlinkage long hook_sys_execve(const char __user *filename,
                                       const char __user *const __user *argv,
                                       const char __user *const __user *envp) {
    char fname[256];
    if (strncpy_from_user(fname, filename, sizeof(fname)) > 0) {
        printk(KERN_INFO "execve called with filename: %s\n", fname);
    } else {
        printk(KERN_INFO "execve called with an unknown filename\n");
    }
    return orig_sys_execve(filename, argv, envp);
}

static struct ftrace_hook hooks[] = {
    HOOK("__arm64_sys_execve", hook_sys_execve, &orig_sys_execve),
};

static int __init testftrace_init(void) {
    int err;

    printk(KERN_INFO "testftrace: Init\n");

    err = fh_install_hooks(hooks, ARRAY_SIZE(hooks));
    if (err) {
        printk(KERN_ERR "fh_install_hooks() failed: %d\n", err);
        return err;
    }

    printk(KERN_INFO "testftrace: Loaded >:-)\n");
    return 0;
}

static void __exit testftrace_exit(void) {
    fh_remove_hooks(hooks, ARRAY_SIZE(hooks));
    printk(KERN_INFO "testftrace: Unloaded :-(\n");
}

module_init(testftrace_init);
module_exit(testftrace_exit);
#else
static int __init testftrace_init(void) {
    printk(KERN_INFO "testftrace: Only supported on ARM64 architecture\n");
    return -ENODEV;
}

static void __exit testftrace_exit(void) {
    printk(KERN_INFO "testftrace: Unloaded\n");
}

module_init(testftrace_init);
module_exit(testftrace_exit);
#endif

The Makefile,

obj-m += testftrace.o

all:
    make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules

clean:
    make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean

I am root user, running make followed by insmod testftrace.ko and I'm getting the following error,

insmod: ERROR: could not insert module testftrace.ko: Invalid parameters

and in dmesg -w,

19855.397220] testftrace: Init
[19855.400273] Resolved address of __arm64_sys_execve: ffff00000834a260
[19855.400536] ftrace_set_filter_ip() failed: -22
[19855.400563] fh_install_hooks - error statement
[19855.400581] fh_install_hooks() failed: -22

Error -22 or EINVAL could happen when the address for the function you want to hook is invalid, I checked kallsyms to check the address,

root@localhost execvehook]# cat /proc/kallsyms | grep _sys_exec
ffff00000834a260 T __arm64_sys_execve
...

As you can see, the address for this function seem to be correct. I tried with system call functions and none-system call function but I'm getting the same error.

I tried to use ftrace_set_filter instead of ftrace_set_filter_ip but I got the same EINVAL error.

This is also the kernel config of the Linux that I'm running under Qemu,

CONFIG_HAVE_LIVEPATCH_WO_FTRACE=y
CONFIG_LIVEPATCH_WO_FTRACE=y
# CONFIG_PSTORE_FTRACE is not set
CONFIG_HAVE_DYNAMIC_FTRACE=y
CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
CONFIG_FTRACE=y
CONFIG_FTRACE_SYSCALLS=y
CONFIG_DYNAMIC_FTRACE=y
CONFIG_FTRACE_MCOUNT_RECORD=y
# CONFIG_FTRACE_STARTUP_TEST is not set

At this point, I ran out of ideas to test and my searches didn't lead to anything helpful. I hope to get some help from here.

The problem I'm trying to solve is, to be able to use ftrace on kernel 4.x and older on Arm and Intel.

• Am I missing something in my code? This is basically my very first code for ftrace, so I'm pretty new.
• Have you tried ftrace on 4.x kernels or older on Aarch64? Is there anything I need to watch for, or customize my code for older kernels when using ftrace?
• Are there limitations/differences between ftrace on 5.x/6.x and 4.x and older?

Thanks, Jelal

Thanks everyone for your comments. Here I post the answer for the initial question after some testing. I followed some of the comments and I confirm the issue was related to QEMU, when I tested on a physical machine and a hypervisor it worked fine.