So I want the module to get executed in an isolated core.
and
actually isolate a specific core in our system and execute just one
specific process to that core
This is a working source code compiled and tested on a Debian box using kernel 3.16. I'll describe how to load and unload first and what the parameter passed means.
All sources can be found on github here...
https://github.com/harryjackson/doc/tree/master/linux/kernel/toy/toy
Build and load the module...
make
insmod toy param_cpu_id=2
To unload the module use
rmmod toy
I'm not using modprobe because it expects some configuration etc. The parameter we're passing to the toy kernel module is the CPU we want to isolate. None of the device operations that get called will run unless they're executing on that CPU.
Once the module is loaded you can find it here
/dev/toy
Simple operations like
cat /dev/toy
create events that the kernel module catches and produces some output. You can see the output using dmesg.
Source code...
#include <linux/module.h>
#include <linux/fs.h>
#include <linux/miscdevice.h>
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Harry");
MODULE_DESCRIPTION("toy kernel module");
MODULE_VERSION("0.1");
#define DEVICE_NAME "toy"
#define CLASS_NAME "toy"
static int param_cpu_id;
module_param(param_cpu_id , int, (S_IRUSR | S_IRGRP | S_IROTH));
MODULE_PARM_DESC(param_cpu_id, "CPU ID that operations run on");
//static void bar(void *arg);
//static void foo(void *cpu);
static int toy_open( struct inode *inodep, struct file *fp);
static ssize_t toy_read( struct file *fp , char *buffer, size_t len, loff_t * offset);
static ssize_t toy_write( struct file *fp , const char *buffer, size_t len, loff_t *);
static int toy_release(struct inode *inodep, struct file *fp);
static struct file_operations toy_fops = {
.owner = THIS_MODULE,
.open = toy_open,
.read = toy_read,
.write = toy_write,
.release = toy_release,
};
static struct miscdevice toy_device = {
.minor = MISC_DYNAMIC_MINOR,
.name = "toy",
.fops = &toy_fops
};
//static int CPU_IDS[64] = {0};
static int toy_open(struct inode *inodep, struct file *filep) {
int this_cpu = get_cpu();
printk(KERN_INFO "open: called on CPU:%d\n", this_cpu);
if(this_cpu == param_cpu_id) {
printk(KERN_INFO "open: is on requested CPU: %d\n", smp_processor_id());
}
else {
printk(KERN_INFO "open: not on requested CPU:%d\n", smp_processor_id());
}
put_cpu();
return 0;
}
static ssize_t toy_read(struct file *filep, char *buffer, size_t len, loff_t *offset){
int this_cpu = get_cpu();
printk(KERN_INFO "read: called on CPU:%d\n", this_cpu);
if(this_cpu == param_cpu_id) {
printk(KERN_INFO "read: is on requested CPU: %d\n", smp_processor_id());
}
else {
printk(KERN_INFO "read: not on requested CPU:%d\n", smp_processor_id());
}
put_cpu();
return 0;
}
static ssize_t toy_write(struct file *filep, const char *buffer, size_t len, loff_t *offset){
int this_cpu = get_cpu();
printk(KERN_INFO "write called on CPU:%d\n", this_cpu);
if(this_cpu == param_cpu_id) {
printk(KERN_INFO "write: is on requested CPU: %d\n", smp_processor_id());
}
else {
printk(KERN_INFO "write: not on requested CPU:%d\n", smp_processor_id());
}
put_cpu();
return 0;
}
static int toy_release(struct inode *inodep, struct file *filep){
int this_cpu = get_cpu();
printk(KERN_INFO "release called on CPU:%d\n", this_cpu);
if(this_cpu == param_cpu_id) {
printk(KERN_INFO "release: is on requested CPU: %d\n", smp_processor_id());
}
else {
printk(KERN_INFO "release: not on requested CPU:%d\n", smp_processor_id());
}
put_cpu();
return 0;
}
static int __init toy_init(void) {
int cpu_id;
if(param_cpu_id < 0 || param_cpu_id > 4) {
printk(KERN_INFO "toy: unable to load module without cpu parameter\n");
return -1;
}
printk(KERN_INFO "toy: loading to device driver, param_cpu_id: %d\n", param_cpu_id);
//preempt_disable(); // See notes below
cpu_id = get_cpu();
printk(KERN_INFO "toy init called and running on CPU: %d\n", cpu_id);
misc_register(&toy_device);
//preempt_enable(); // See notes below
put_cpu();
//smp_call_function_single(1,foo,(void *)(uintptr_t) 1,1);
return 0;
}
static void __exit toy_exit(void) {
misc_deregister(&toy_device);
printk(KERN_INFO "toy exit called\n");
}
module_init(toy_init);
module_exit(toy_exit);
The code above contains the two methods you asked for ie isolation of CPU and on init run on an isolated core.
On init get_cpu disables preemption ie anything that comes after it will not be preempted by the kernel and will run on one core. Note, this was done kernel using 3.16, your mileage may vary depending on your kernel version but I think these API's have been around a long time
This is the Makefile...
obj-m += toy.o
all:
make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules
clean:
make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean
Notes. get_cpu is declared in linux/smp.h as
#define get_cpu() ({ preempt_disable(); smp_processor_id(); })
#define put_cpu() preempt_enable()
so you don't actually need to call preempt_disable before calling get_cpu.
The get_cpu call is a wrapper around the following sequence of calls...
preempt_count_inc();
barrier();
and put_cpu is really doing this...
barrier();
if (unlikely(preempt_count_dec_and_test())) {
__preempt_schedule();
}
You can get as fancy as you like using the above. Almost all of this was taken from the following sources..
Google for... smp_call_function_single
Linux Kernel Development, book by Robert Love.
http://derekmolloy.ie/writing-a-linux-kernel-module-part-2-a-character-device/
https://github.com/vsinitsyn/reverse/blob/master/reverse.c
isolcpus(kernel parameter -- see Documentation/kernel-parameters.txt) "can be used to specify one or more CPUs to isolate from the general SMP balancing and scheduling algorithms. You can move a process onto or off an "isolated" CPU via the CPU affinity syscalls or cpuset." – Roadkthread_bind. Regarding the proposed solutions:isolcpusandmaxcpus, from my understanding those parameters should be used while booting. I would like to isolate without restarting, actually, isolation should happen only when the module is loaded, so I feel this is not the way to go. – Lehmbruckcat /proc/kallsyms | grep cpuset_init. Even more confusing is finding thatcpuset_initstands as undefined (U) in the output ofnm module.ko. – Jola