[Linux Kernel] Kmalloc 분석

int __init test_device_init(void) { if(register_chrdev(300, "test_device", &vd_fops) < 0 ) printk(KERN_ALERT "driver init failed\n"); else printk(KERN_ALERT "driver init successful\n"); buffer = (char*)kmalloc(1024, GFP_KERNEL); if(buffer != NULL) memset(buffer, 0, 1024); return 0; }

#define GFP_ATOMIC (__GFP_HIGH|__GFP_ATOMIC|__GFP_KSWAPD_RECLAIM) #define GFP_KERNEL (__GFP_RECLAIM | __GFP_IO | __GFP_FS) #define GFP_KERNEL_ACCOUNT (GFP_KERNEL | __GFP_ACCOUNT) #define GFP_NOWAIT (__GFP_KSWAPD_RECLAIM) #define GFP_NOIO (__GFP_RECLAIM) #define GFP_NOFS (__GFP_RECLAIM | __GFP_IO) #define GFP_USER (__GFP_RECLAIM | __GFP_IO | __GFP_FS | __GFP_HARDWALL) #define GFP_DMA __GFP_DMA #define GFP_DMA32 __GFP_DMA32 #define GFP_HIGHUSER (GFP_USER | __GFP_HIGHMEM) #define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE) #define GFP_TRANSHUGE_LIGHT ((GFP_HIGHUSER_MOVABLE | __GFP_COMP | __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM) #define GFP_TRANSHUGE (GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM) /* Convert GFP flags to their corresponding migrate type */ #define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE) #define GFP_MOVABLE_SHIFT 3

static __always_inline void *kmalloc(size_t size, gfp_t flags) { if (__builtin_constant_p(size)) { if (size > KMALLOC_MAX_CACHE_SIZE) return kmalloc_large(size, flags); // try buddy allocator #ifndef CONFIG_SLOB if (!(flags & GFP_DMA)) { int index = kmalloc_index(size); if (!index) return ZERO_SIZE_PTR; return kmem_cache_alloc_trace(kmalloc_caches[index], flags, size); } #endif } return __kmalloc(size, flags); }

/* * Figure out which kmalloc slab an allocation of a certain size * belongs to. * 0 = zero alloc * 1 = 65 .. 96 bytes * 2 = 129 .. 192 bytes * n = 2^(n-1)+1 .. 2^n */ static __always_inline int kmalloc_index(size_t size) { if (!size) return 0; if (size <= KMALLOC_MIN_SIZE) return KMALLOC_SHIFT_LOW; if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96) return 1; if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192) return 2; if (size <= 8) return 3; if (size <= 16) return 4; if (size <= 32) return 5; if (size <= 64) return 6; if (size <= 128) return 7; if (size <= 256) return 8; if (size <= 512) return 9; if (size <= 1024) return 10; if (size <= 2 * 1024) return 11; if (size <= 4 * 1024) return 12; if (size <= 8 * 1024) return 13; if (size <= 16 * 1024) return 14; if (size <= 32 * 1024) return 15; if (size <= 64 * 1024) return 16; if (size <= 128 * 1024) return 17; if (size <= 256 * 1024) return 18; if (size <= 512 * 1024) return 19; if (size <= 1024 * 1024) return 20; if (size <= 2 * 1024 * 1024) return 21; if (size <= 4 * 1024 * 1024) return 22; if (size <= 8 * 1024 * 1024) return 23; if (size <= 16 * 1024 * 1024) return 24; if (size <= 32 * 1024 * 1024) return 25; if (size <= 64 * 1024 * 1024) return 26; BUG(); /* Will never be reached. Needed because the compiler may complain */ return -1; }

struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];

void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) { void *ret = slab_alloc(s, gfpflags, _RET_IP_); trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags); kasan_kmalloc(s, ret, size, gfpflags); return ret; }

static __always_inline void *slab_alloc(struct kmem_cache *s, gfp_t gfpflags, unsigned long addr) { return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr); }

static __always_inline void *slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node, unsigned long addr) { void *object; struct kmem_cache_cpu *c; struct page *page; unsigned long tid; s = slab_pre_alloc_hook(s, gfpflags); if (!s) return NULL; redo: /* * Must read kmem_cache cpu data via this cpu ptr. Preemption is * enabled. We may switch back and forth between cpus while * reading from one cpu area. That does not matter as long * as we end up on the original cpu again when doing the cmpxchg. * * We should guarantee that tid and kmem_cache are retrieved on * the same cpu. It could be different if CONFIG_PREEMPT so we need * to check if it is matched or not. */ do { tid = this_cpu_read(s->cpu_slab->tid); //전담 CPU id 값 획득 c = raw_cpu_ptr(s->cpu_slab); // 전담 CPU per-cpu 구조체 포인터 획득 } while (IS_ENABLED(CONFIG_PREEMPT) && unlikely(tid != READ_ONCE(c->tid))); /* * Irqless object alloc/free algorithm used here depends on sequence * of fetching cpu_slab's data. tid should be fetched before anything * on c to guarantee that object and page associated with previous tid * won't be used with current tid. If we fetch tid first, object and * page could be one associated with next tid and our alloc/free * request will be failed. In this case, we will retry. So, no problem. */ barrier(); /* * The transaction ids are globally unique per cpu and per operation on * a per cpu queue. Thus they can be guarantee that the cmpxchg_double * occurs on the right processor and that there was no operation on the * linked list in between. */ object = c->freelist; page = c->page; if (unlikely(!object || !node_match(page, node))) { // slowpath ! object = __slab_alloc(s, gfpflags, node, addr, c); stat(s, ALLOC_SLOWPATH); } else { // fastpath ! void *next_object = get_freepointer_safe(s, object); // freelist 조정 위해 /* * The cmpxchg will only match if there was no additional * operation and if we are on the right processor. * * The cmpxchg does the following atomically (without lock * semantics!) * 1. Relocate first pointer to the current per cpu area. * 2. Verify that tid and freelist have not been changed * 3. If they were not changed replace tid and freelist * * Since this is without lock semantics the protection is only * against code executing on this cpu *not* from access by * other cpus. */ if (unlikely(!this_cpu_cmpxchg_double(s->cpu_slab->freelist, s->cpu_slab->tid, object, tid,next_object, next_tid(tid)))) { note_cmpxchg_failure("slab_alloc", s, tid); goto redo; } prefetch_freepointer(s, next_object); stat(s, ALLOC_FASTPATH); // fastpath로 할당 상태 표시 } if (unlikely(gfpflags & __GFP_ZERO) && object) memset(object, 0, s->object_size); slab_post_alloc_hook(s, gfpflags, 1, &object); return object; }

struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];

/* * Figure out which kmalloc slab an allocation of a certain size * belongs to. * 0 = zero alloc * 1 = 65 .. 96 bytes * 2 = 129 .. 192 bytes * n = 2^(n-1)+1 .. 2^n */ static __always_inline int kmalloc_index(size_t size) { if (!size) return 0; if (size <= KMALLOC_MIN_SIZE) return KMALLOC_SHIFT_LOW; if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96) return 1; if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192) return 2; if (size <= 8) return 3; if (size <= 16) return 4; if (size <= 32) return 5; if (size <= 64) return 6; if (size <= 128) return 7; if (size <= 256) return 8; // here if (size <= 512) return 9; if (size <= 1024) return 10; ... }

pwndbg> p* kmalloc_caches[8] $9 = { cpu_slab = 0x23b80, flags = 1073741824, min_partial = 5, size = 256, object_size = 256, offset = 0, cpu_partial = 13, oo = { x = 16 }, max = { x = 16 }, min = { x = 16 }, allocflags = 0, refcount = 10, ctor = 0x0 <irq_stack_union>, inuse = 256, align = 8, reserved = 0, red_left_pad = 0, name = 0xffffffff81fcac1d "kmalloc-256", list = { next = 0xffff88000e801868, prev = 0xffff88000e801668 }, kobj = { name = 0xffff88000d99d120 ":0000256", entry = { next = 0xffff88000e801880, prev = 0xffff88000e801680 }, parent = 0xffff88000eba0af8, kset = 0xffff88000eba0ae0, ktype = 0xffffffff8225f760 <slab_ktype>, sd = 0xffff88000d9aaf68, kref = { refcount = { refs = { counter = 1 } } }, state_initialized = 1, state_in_sysfs = 1, state_add_uevent_sent = 1, state_remove_uevent_sent = 0, uevent_suppress = 0 }, kobj_remove_work = { data = { counter = 68719476704 }, entry = { next = 0xffff88000e8017c0, prev = 0xffff88000e8017c0 }, func = 0xffffffff811864b0 <sysfs_slab_remove_workfn> }, remote_node_defrag_ratio = 1000, node = {0xffff88000e800e00, 0x0 <irq_stack_union>, 0x0 <irq_stack_union>, 0x0 <irq_stack_union>, 0x23b60, 0x40000000, 0x5 <irq_stack_union+5>, 0xc0000000c0, 0x1e00000000, 0x15 <irq_stack_union+21>, 0x15 <irq_stack_union+21>, 0x15 <irq_stack_union+21>, 0x600000000, 0x0 <irq_stack_union>, 0x8000000c0, 0x0 <irq_stack_union>, 0xffffffff81fcabda, 0xffff88000e801968, 0xffff88000e801768, 0xffff88000d99d140, 0xffff88000e801980, 0xffff88000e801780, 0xffff88000eba0af8, 0xffff88000eba0ae0, 0xffffffff8225f760 <slab_ktype>, 0xffff88000d9a9110, 0x700000001, 0xfffffffe0, 0xffff88000e8018c0, 0xffff88000e8018c0, 0xffffffff811864b0 <sysfs_slab_remove_workfn>, 0x3e8 <irq_stack_union+1000>, 0xffff88000e800e40, 0x0 <irq_stack_union>, 0x0 <irq_stack_union>, 0x0 <irq_stack_union>, 0x23b40, 0x40000000, 0x5 <irq_stack_union+5>, 0x8000000080, 0x1e00000000, 0x20 <irq_stack_union+32>, 0x20 <irq_stack_union+32>, 0x20 <irq_stack_union+32>, 0x800000000, 0x0 <irq_stack_union>, 0x800000080, 0x0 <irq_stack_union>, 0xffffffff81fcac11, 0xffff88000e801a68, 0xffff88000e801868, 0xffff88000d99d160, 0xffff88000e801a80, 0xffff88000e801880, 0xffff88000eba0af8, 0xffff88000eba0ae0, 0xffffffff8225f760 <slab_ktype>, 0xffff88000d96e2a8, 0x700000001, 0xfffffffe0, 0xffff88000e8019c0, 0xffff88000e8019c0, 0xffffffff811864b0 <sysfs_slab_remove_workfn>, 0x3e8 <irq_stack_union+1000>} }

static __always_inline void *slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node, unsigned long addr) { void *object; struct kmem_cache_cpu *c; struct page *page; unsigned long tid; s = slab_pre_alloc_hook(s, gfpflags); if (!s) return NULL; ...

static __always_inline void *slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node, unsigned long addr) ... object = c->freelist; page = c->page; if (unlikely(!object || !node_match(page, node))) { // slowpath ! object = __slab_alloc(s, gfpflags, node, addr, c); stat(s, ALLOC_SLOWPATH); } else { // fastpath ! void *next_object = get_freepointer_safe(s, object); // 여기 진행중 !! if (unlikely(!this_cpu_cmpxchg_double(s->cpu_slab->freelist, s->cpu_slab->tid, object, tid,next_object, next_tid(tid)))) { note_cmpxchg_failure("slab_alloc", s, tid); goto redo; } prefetch_freepointer(s, next_object); stat(s, ALLOC_FASTPATH); // fastpath로 할당 상태 표시 } if (unlikely(gfpflags & __GFP_ZERO) && object) memset(object, 0, s->object_size); slab_post_alloc_hook(s, gfpflags, 1, &object); return object; }

static __always_inline void *slab_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node, unsigned long addr) { void *object; struct kmem_cache_cpu *c; struct page *page; unsigned long tid; ... if (unlikely(!object || !node_match(page, node))) { // slowpath ! object = __slab_alloc(s, gfpflags, node, addr, c); stat(s, ALLOC_SLOWPATH); } else { // fastpath ! void *next_object = get_freepointer_safe(s, object); ... if (unlikely(!this_cpu_cmpxchg_double(s->cpu_slab->freelist, s->cpu_slab->tid, object, tid,next_object, next_tid(tid)))) { this_cpu_cmpxchg_double => 요 함수 !! note_cmpxchg_failure("slab_alloc", s, tid); goto redo; } prefetch_freepointer(s, next_object); stat(s, ALLOC_FASTPATH); // fastpath로 할당 상태 표시 } if (unlikely(gfpflags & __GFP_ZERO) && object) memset(object, 0, s->object_size); slab_post_alloc_hook(s, gfpflags, 1, &object); return object; }

this_cpu_cmpxchg_double() => if ((__this_cpu_ptr(s->cpu_slab->freelist) == object) && (__this_cpu_ptr(s->cpu_slab->tid) == tid)) { __this_cpu_ptr(s->cpu_slab->freelist) = next_object; // freelist 조정 ! __this_cpu_ptr(s->cpu_slab->tid) = next_tid(tid); return true; } else { return false; }

... struct data { unsigned char buffer[BUFFER_SIZE]; }; ... static int chardev_open(struct inode *inode, struct file *file) { char *str = "helloworld"; int ret,i; struct data *p = kmalloc(sizeof(struct data), GFP_KERNEL); printk("The chardev_open() function has been called."); if (p == NULL) { printk(KERN_ERR "kmalloc - Null"); return -ENOMEM; } ret = strlcpy(p->buffer, str, sizeof(p->buffer)); if(ret > strlen(str)){ printk(KERN_ERR "strlcpy - too long (%d)",ret); } file->private_data = p; return 0; } ... static ssize_t chardev_write(struct file *filp, const char __user *buf, size_t count, loff_t *f_pos) { struct data *p = filp->private_data; printk("The chardev_write() function has been called."); printk("Before calling the copy_from_user() function : %p, %s",p->buffer,p->buffer); if (copy_from_user(p->buffer, buf, count) != 0) { return -EFAULT; } printk("After calling the copy_from_user() function : %p, %s",p->buffer,p->buffer); return count; }