From user-space we can use cpuset to actually isolate a specific core in our system and execute just one specific process to that core.
I'm trying to do the same thing with a kernel module. So I want the module to get executed in an isolated core. In other words: How do I use cpuset's from inside a kernel module? *
Using linux/cpuset.h in my kernel module doesn't work. So, I have a module like this:
#include <linux/module.h>
#include <linux/cpuset.h>
...
#ifdef CONFIG_CPUSETS
printk(KERN_INFO, "cpusets is enabled!");
#endif
cpuset_init(); // this function is declared in cpuset.h
...
When trying to load this module I get (in dmesg) the following message cpusets is enabled!. But I also receive the message Unknown symbol cpu_init (err 0).
Similarly, I tried using sched_setaffinity from linux/sched.h in order to move all running procceses to a specific core and then run my module to an isolated core. I got the same error mesage: Unknown symbol sched_setaffinity (err 0). I guess I got the "unknown symbols" because those functions have no EXPORT_SYMBOL in the kernel. So I went and tried to call the sys_sched_setaffinity system call (based on this question) but again got this mesage: Unknown symbol sys_sched_setaffinity (err 0)!
Furthermore, I am not looking for a solution that uses isolcpus, which is set while booting. I would like to just load the module and afterwards the isolationo to occur.
So I want the module to get executed in an isolated core.
and
actually isolate a specific core in our system and execute just one specific process to that core
This is a working source code compiled and tested on a Debian box using kernel 3.16. I'll describe how to load and unload first and what the parameter passed means.
All sources can be found on github here...
https://github.com/harryjackson/doc/tree/master/linux/kernel/toy/toy
Build and load the module...
make
insmod toy param_cpu_id=2
To unload the module use
rmmod toy
I'm not using modprobe because it expects some configuration etc. The parameter we're passing to the toy kernel module is the CPU we want to isolate. None of the device operations that get called will run unless they're executing on that CPU.
Once the module is loaded you can find it here
/dev/toy
Simple operations like
cat /dev/toy
create events that the kernel module catches and produces some output. You can see the output using dmesg.
Source code...
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/miscdevice.h>
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Harry");
MODULE_DESCRIPTION("toy kernel module");
MODULE_VERSION("0.1");
#define DEVICE_NAME "toy"
#define CLASS_NAME "toy"
static int param_cpu_id;
module_param(param_cpu_id , int, (S_IRUSR | S_IRGRP | S_IROTH));
MODULE_PARM_DESC(param_cpu_id, "CPU ID that operations run on");
//static void bar(void *arg);
//static void foo(void *cpu);
static int toy_open( struct inode *inodep, struct file *fp);
static ssize_t toy_read( struct file *fp , char *buffer, size_t len, loff_t * offset);
static ssize_t toy_write( struct file *fp , const char *buffer, size_t len, loff_t *);
static int toy_release(struct inode *inodep, struct file *fp);
static struct file_operations toy_fops = {
.owner = THIS_MODULE,
.open = toy_open,
.read = toy_read,
.write = toy_write,
.release = toy_release,
};
static struct miscdevice toy_device = {
.minor = MISC_DYNAMIC_MINOR,
.name = "toy",
.fops = &toy_fops
};
//static int CPU_IDS[64] = {0};
static int toy_open(struct inode *inodep, struct file *filep) {
int this_cpu = get_cpu();
printk(KERN_INFO "open: called on CPU:%d\n", this_cpu);
if(this_cpu == param_cpu_id) {
printk(KERN_INFO "open: is on requested CPU: %d\n", smp_processor_id());
}
else {
printk(KERN_INFO "open: not on requested CPU:%d\n", smp_processor_id());
}
put_cpu();
return 0;
}
static ssize_t toy_read(struct file *filep, char *buffer, size_t len, loff_t *offset){
int this_cpu = get_cpu();
printk(KERN_INFO "read: called on CPU:%d\n", this_cpu);
if(this_cpu == param_cpu_id) {
printk(KERN_INFO "read: is on requested CPU: %d\n", smp_processor_id());
}
else {
printk(KERN_INFO "read: not on requested CPU:%d\n", smp_processor_id());
}
put_cpu();
return 0;
}
static ssize_t toy_write(struct file *filep, const char *buffer, size_t len, loff_t *offset){
int this_cpu = get_cpu();
printk(KERN_INFO "write called on CPU:%d\n", this_cpu);
if(this_cpu == param_cpu_id) {
printk(KERN_INFO "write: is on requested CPU: %d\n", smp_processor_id());
}
else {
printk(KERN_INFO "write: not on requested CPU:%d\n", smp_processor_id());
}
put_cpu();
return 0;
}
static int toy_release(struct inode *inodep, struct file *filep){
int this_cpu = get_cpu();
printk(KERN_INFO "release called on CPU:%d\n", this_cpu);
if(this_cpu == param_cpu_id) {
printk(KERN_INFO "release: is on requested CPU: %d\n", smp_processor_id());
}
else {
printk(KERN_INFO "release: not on requested CPU:%d\n", smp_processor_id());
}
put_cpu();
return 0;
}
static int __init toy_init(void) {
int cpu_id;
if(param_cpu_id < 0 || param_cpu_id > 4) {
printk(KERN_INFO "toy: unable to load module without cpu parameter\n");
return -1;
}
printk(KERN_INFO "toy: loading to device driver, param_cpu_id: %d\n", param_cpu_id);
//preempt_disable(); // See notes below
cpu_id = get_cpu();
printk(KERN_INFO "toy init called and running on CPU: %d\n", cpu_id);
misc_register(&toy_device);
//preempt_enable(); // See notes below
put_cpu();
//smp_call_function_single(1,foo,(void *)(uintptr_t) 1,1);
return 0;
}
static void __exit toy_exit(void) {
misc_deregister(&toy_device);
printk(KERN_INFO "toy exit called\n");
}
module_init(toy_init);
module_exit(toy_exit);
The code above contains the two methods you asked for ie isolation of CPU and on init run on an isolated core.
On init get_cpu disables preemption ie anything that comes after it will not be preempted by the kernel and will run on one core. Note, this was done kernel using 3.16, your mileage may vary depending on your kernel version but I think these API's have been around a long time
This is the Makefile...
obj-m += toy.o
all:
make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules
clean:
make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean
Notes. get_cpu is declared in linux/smp.h as
#define get_cpu() ({ preempt_disable(); smp_processor_id(); })
#define put_cpu() preempt_enable()
so you don't actually need to call preempt_disable before calling get_cpu.
The get_cpu call is a wrapper around the following sequence of calls...
preempt_count_inc();
barrier();
and put_cpu is really doing this...
barrier();
if (unlikely(preempt_count_dec_and_test())) {
__preempt_schedule();
}
You can get as fancy as you like using the above. Almost all of this was taken from the following sources..
Google for... smp_call_function_single
Linux Kernel Development, book by Robert Love.
http://derekmolloy.ie/writing-a-linux-kernel-module-part-2-a-character-device/