針對Memcacged1.4.15代碼

1.完整slabs內存池圖

這是我畫的memcached的slabs內存池對象關系圖：

2.內存池數據結構

typedef struct {unsigned int size;????? /* 每個item的大小 */unsigned int perslab;?? /* 每個page中包含多少個item */void *slots;?????????? //空閑的item列表，指向最后一個空閑的chunk的地址，如果perslab=1那么slots和slab_list[0]指向的地址是同一個unsigned int sl_curr;?? //當前空閑的item位置(也就是實際空閑item個數)，從后往前的unsigned int slabs;???? //已分配chunk數目void **slab_list;?????? //所有的page指針，指向page的地址，也是第一個chunk的地址，可以用(item *)((&slabclass[N])->slab_list[0])獲取第一個itemunsigned int list_size; //每個數組trunk數目,默認是16unsigned int killing;? /* index+1 of dying slab, or zero if none */size_t requested; //已分配總內存大小
} slabclass_t;//slots：指向free chunk塊的指針數組，slots的指針指向一個void //*的數組，該數組中的每一個元素的內容均指向一個空閑的chunk塊，而且相同slabclass上的所有slab中的free chunk塊均掛接到這個鏈表上；
//slabs：當前slabclass中分配的頁內存個數；
//slab_list：當前slabclass所分配的頁內存(slab)的指針數組，每一個數組元素的內容均是一個指向頁內存地址的指針；
//        size_t new_size =  (p->list_size != 0) ? p->list_size * 2 : 16;
//        void *new_list = realloc(p->slab_list, new_size * sizeof(void *));
//        if (new_list == 0) return 0;
//        p->list_size = new_size;
//        p->slab_list = new_list;static slabclass_t slabclass[MAX_NUMBER_OF_SLAB_CLASSES];
static size_t mem_limit = 0;//內存限制大小，如果默認64M?? slabs_init的時候
static size_t mem_malloced = 0;//已分配大小
static int power_largest;//數組最大個數，默認是42static void *mem_base = NULL;
static void *mem_current = NULL;//內存使用當前地址
static size_t mem_avail = 0;//剩余內存/*** slab 線程鎖*/
static pthread_mutex_t slabs_lock = PTHREAD_MUTEX_INITIALIZER;
static pthread_mutex_t slabs_rebalance_lock = PTHREAD_MUTEX_INITIALIZER;

3.初始化slabs_init

/* slab初始化*/
/* limit:內存大小(字節);factor:增長因子;prealloc:是否一次性分配內存*/
void slabs_init(const size_t limit, const double factor, const bool prealloc) {int i = POWER_SMALLEST - 1;//0unsigned int size = sizeof(item) + settings.chunk_size;//chunk_size 最小分配空間mem_limit = limit;//內存限制大小if (prealloc) {//一次分配所有設置的內存/* Allocate everything in a big chunk with malloc */mem_base = malloc(mem_limit);if (mem_base != NULL) {mem_current = mem_base;mem_avail = mem_limit;} else {fprintf(stderr, "Warning: Failed to allocate requested memory in one large chunk.\nWill allocate in smaller chunks\n");}}memset(slabclass, 0, sizeof(slabclass));
//settings.item_size_max = 1024 * 1024=1M; /* The famous 1MB upper limit. */
//settings.item_size_max / factor??? 1048576/1.25=838860.8 這就是單page最大的chunk大小 字節
//slabclass[41]?? ?{size=717184 perslab=1 slots=0x00000000 ...}?? ?slabclass_t
//所以到了42就跳出循環了
//slabclass[42]?? ?{size=1048576 perslab=1 slots=0x00000000 ...}?? ?slabclass_t
//43就不分配了
//slabclass[43]?? ?{size=0 perslab=0 slots=0x00000000 ...}?? ?slabclass_twhile (++i < POWER_LARGEST && size <= settings.item_size_max / factor) {/* Make sure items are always n-byte aligned */if (size % CHUNK_ALIGN_BYTES)//字節數為8的倍數size += CHUNK_ALIGN_BYTES - (size % CHUNK_ALIGN_BYTES);slabclass[i].size = size;//item大小slabclass[i].perslab = settings.item_size_max / slabclass[i].size;//item數目size *= factor;//乘以增長因子if (settings.verbose > 1) {fprintf(stderr, "slab class %3d: chunk size %9u perslab %7u\n",i, slabclass[i].size, slabclass[i].perslab);}}power_largest = i;//默認=42slabclass[power_largest].size = settings.item_size_max;slabclass[power_largest].perslab = 1;//最大的只能存儲一個itemif (settings.verbose > 1) {fprintf(stderr, "slab class %3d: chunk size %9u perslab %7u\n",i, slabclass[i].size, slabclass[i].perslab);}/* for the test suite:? faking of how much we've already malloc'd */{char *t_initial_malloc = getenv("T_MEMD_INITIAL_MALLOC");if (t_initial_malloc) {mem_malloced = (size_t)atol(t_initial_malloc);}}#ifndef DONT_PREALLOC_SLABS{char *pre_alloc = getenv("T_MEMD_SLABS_ALLOC");if (pre_alloc == NULL || atoi(pre_alloc) != 0) {slabs_preallocate(power_largest);}}
#endif
}

4.第一次分配slab

最初是在do_item_alloc中調用do_slabs_alloc

#0? do_item_alloc (key=0x7ffff0013754 "key", nkey=3, flags=0, exptime=0, nbytes=5, cur_hv=0) at items.c:190
#1? 0x0000000000415706 in item_alloc (key=0x7ffff0013754 "key", nkey=3, flags=0, exptime=0, nbytes=5) at thread.c:486
#2? 0x000000000040a38e in process_update_command (c=0x7ffff0013550, tokens=0x7ffff7ae4b00, ntokens=6, comm=2, handle_cas=false) at memcached.c:2917
#3? 0x000000000040b43d in process_command (c=0x7ffff0013550, command=0x7ffff0013750 "set") at memcached.c:3258
#4? 0x000000000040bfa1 in try_read_command (c=0x7ffff0013550) at memcached.c:3504
#5? 0x000000000040cc25 in drive_machine (c=0x7ffff0013550) at memcached.c:3824
#6? 0x000000000040d81f in event_handler (fd=37, which=2, arg=0x7ffff0013550) at memcached.c:4065
#7? 0x00007ffff7dc9e0c in event_process_active_single_queue (base=0x635bb0, flags=0) at event.c:1350
#8? event_process_active (base=0x635bb0, flags=0) at event.c:1420
#9? event_base_loop (base=0x635bb0, flags=0) at?? ?event.c:1621
#10 0x0000000000415416 in worker_libevent (arg=0x628d60) at thread.c:384
#11 0x0000003441607851 in start_thread () from /lib64/libpthread.so.0
#12 0x00000034412e890d in clone?? ?() from /lib64/libc.so.6

客戶端申請存儲key value會調用到do_item_alloc

/*@null@*/
item *do_item_alloc(char *key, const size_t nkey, const int flags,const rel_time_t exptime, const int nbytes,const uint32_t cur_hv) {uint8_t nsuffix;item *it = NULL;char suffix[40];size_t ntotal = item_make_header(nkey + 1, flags, nbytes, suffix, &nsuffix);if (settings.use_cas) {ntotal += sizeof(uint64_t);//算出需要分配的內存的大小，原始長度+結構體自身大小}unsigned int id = slabs_clsid(ntotal);if (id == 0)return 0;mutex_lock(&cache_lock);/* do a quick check if we have any expired items in the tail.. */int tries = 5;int tried_alloc = 0;item *search;void *hold_lock = NULL;rel_time_t oldest_live = settings.oldest_live;search = tails[id];/* We walk up *only* for locked items. Never searching for expired.* Waste of CPU for almost all deployments */for (; tries > 0 && search != NULL; tries--, search=search->prev) {uint32_t hv = hash(ITEM_key(search), search->nkey, 0);/* Attempt to hash item lock the "search" item. If locked, no* other callers can incr the refcount*//* FIXME: I think we need to mask the hv here for comparison? */if (hv != cur_hv && (hold_lock = item_trylock(hv)) == NULL)continue;/* Now see if the item is refcount locked */if (refcount_incr(&search->refcount) != 2) {refcount_decr(&search->refcount);/* Old rare bug could cause a refcount leak. We haven't seen* it in years, but we leave this code in to prevent failures* just in case */if (search->time + TAIL_REPAIR_TIME < current_time) {itemstats[id].tailrepairs++;search->refcount = 1;do_item_unlink_nolock(search, hv);}if (hold_lock)item_trylock_unlock(hold_lock);continue;}/* Expired or flushed */if ((search->exptime != 0 && search->exptime < current_time)|| (search->time <= oldest_live && oldest_live <= current_time)) {itemstats[id].reclaimed++;if ((search->it_flags & ITEM_FETCHED) == 0) {itemstats[id].expired_unfetched++;}it = search;slabs_adjust_mem_requested(it->slabs_clsid, ITEM_ntotal(it), ntotal);do_item_unlink_nolock(it, hv);/* Initialize the item block: */it->slabs_clsid = 0;} else if ((it = slabs_alloc(ntotal, id)) == NULL) {

調用slabs_clsid()

//尋找適合給定大小的item存儲的slab
unsigned int slabs_clsid(const size_t size) {int res = POWER_SMALLEST;if (size == 0)return 0;while (size > slabclass[res].size)//找到第一個比item size大的slabif (res++ == power_largest)return 0;return res;
}

調用do_slabs_alloc返回slots指向的item，并使slots指向下一個item

/*存儲item*/
static void *do_slabs_alloc(const size_t size, unsigned int id) {slabclass_t *p;void *ret = NULL;item *it = NULL;if (id < POWER_SMALLEST || id > power_largest) {MEMCACHED_SLABS_ALLOCATE_FAILED(size, 0);return NULL;}p = &slabclass[id];assert(p->sl_curr == 0 || ((item *)p->slots)->slabs_clsid == 0);/* fail unless we have space at the end of a recently allocated page,we have something on our freelist, or we could allocate a new page *///p->sl_curr != 0 說明還有空閑就不要調用do_slabs_newslab重新分配下一個pageif (! (p->sl_curr != 0 || do_slabs_newslab(id) != 0)) {/* We don't have more memory available */ret = NULL;} else if (p->sl_curr != 0) {/* return off our freelist */it = (item *)p->slots;//將當前p空閑的slots其實也就是第一個，實際上是鏈表最后一個chunk分配給itemp->slots = it->next;//修改p的空閑的slots，為倒數第二個，也就是it的前一個if (it->next) it->next->prev = 0;//因為it的前一個再前一個已經給item分配了，自然沒了p->sl_curr--;//-1，雖然perslab在分割好之后和sl_curr一樣大，但是sl_curr是要遞減的，而perslab是永遠不變的存儲的是chunk個數ret = (void *)it;}if (ret) {p->requested += size;MEMCACHED_SLABS_ALLOCATE(size, id, p->size, ret);} else {MEMCACHED_SLABS_ALLOCATE_FAILED(size, id);}return ret;
}

下面是gdb下調試的函數調用關系：

#0? do_slabs_alloc (size=71, id=1) at slabs.c:241
#1? 0x000000000041161d in slabs_alloc (size=71, id=1) at slabs.c:404
#2? 0x0000000000412ae6 in do_item_alloc (key=0x7ffff0013754 "key", nkey=3, flags=0, exptime=0, nbytes=5, cur_hv=0) at items.c:188
#3? 0x0000000000415706 in item_alloc (key=0x7ffff0013754 "key", nkey=3, flags=0, exptime=0, nbytes=5) at thread.c:486
#4? 0x000000000040a38e in process_update_command (c=0x7ffff0013550, tokens=0x7ffff7ae4b00, ntokens=6, comm=2, handle_cas=false) at memcached.c:2917
#5? 0x000000000040b43d in process_command (c=0x7ffff0013550, command=0x7ffff0013750 "set") at memcached.c:3258
#6? 0x000000000040bfa1 in try_read_command (c=0x7ffff0013550) at memcached.c:3504
#7? 0x000000000040cc25 in drive_machine (c=0x7ffff0013550) at memcached.c:3824
#8? 0x000000000040d81f in event_handler (fd=37,?? ?which=2, arg=0x7ffff0013550) at memcached.c:4065
#9? 0x00007ffff7dc9e0c in event_process_active_single_queue (base=0x635bb0, flags=0) at event.c:1350
#10 event_process_active (base=0x635bb0, flags=0) at event.c:1420
#11 event_base_loop (base=0x635bb0, flags=0) at event.c:1621
#12 0x0000000000415416 in worker_libevent (arg=0x628d60) at thread.c:384
#13 0x0000003441607851 in start_thread () from /lib64/libpthread.so.0
#14 0x00000034412e890d in clone () from /lib64/libc.so.6

5.第一次slab_list

初始化slabs,分配trunk

static int do_slabs_newslab(const unsigned int id) {slabclass_t *p = &slabclass[id];
//settings.item_size_max = 1024 * 1024; /* The famous 1MB upper limit. */int len = settings.slab_reassign ? settings.item_size_max: p->size * p->perslab;char *ptr;if ((mem_limit && mem_malloced + len > mem_limit && p->slabs > 0) ||(grow_slab_list(id) == 0) ||((ptr = memory_allocate((size_t)len)) == 0)) {MEMCACHED_SLABS_SLABCLASS_ALLOCATE_FAILED(id);return 0;}memset(ptr, 0, (size_t)len);split_slab_page_into_freelist(ptr, id);
//這里很巧妙，如果是第一次do_slabs_newslab，那么p->slabs=0，++之后就=1，
//第二次來自然就是2，這次會把第2次申請的page也就是ptr串在p->slab_list[1]上p->slab_list[p->slabs++] = ptr;mem_malloced += len;MEMCACHED_SLABS_SLABCLASS_ALLOCATE(id);return 1;
}

p->slab_list第一次是存16*8個字節

//擴充trunk數目,重新分配slabs個數,默認是分配16個頁,后續按照2倍增加
static int grow_slab_list (const unsigned int id) {slabclass_t *p = &slabclass[id];if (p->slabs == p->list_size) {//默認new_zise=16size_t new_size =  (p->list_size != 0) ? p->list_size * 2 : 16;//因為以后會擴容，所以這里用realloc//p->slab_list一開始=0，64位sizeof(void *)=8，32位sizeof(void *)=4void *new_list = realloc(p->slab_list, new_size * sizeof(void *));if (new_list == 0) return 0;p->list_size = new_size;p->slab_list = new_list;}return 1;
}

調用split_slab_page_into_freelist分割ptr

5.分割ptr

split_slab_page_into_freelist：

static void split_slab_page_into_freelist(char *ptr, const unsigned int id) {slabclass_t *p = &slabclass[id];int x;for (x = 0; x < p->perslab; x++) {do_slabs_free(ptr, 0, id);ptr += p->size;//ptr偏移一個size，第一次是96字節，如果是默認配置的話}
}

這個函數很短，就是循環去調用do_slabs_free函數

//分割item內存
static void do_slabs_free(void *ptr, const size_t size, unsigned int id) {slabclass_t *p;item *it;//首先聲明一個item對象assert(((item *)ptr)->slabs_clsid == 0);assert(id >= POWER_SMALLEST && id <= power_largest);if (id < POWER_SMALLEST || id > power_largest)return;MEMCACHED_SLABS_FREE(size, id, ptr);p = &slabclass[id];//獲取第id個slabclass_t并聲明為pit = (item *)ptr;//將需要切割的內存空間實例化為item *it->it_flags |= ITEM_SLABBED;it->prev = 0;//it一開始無頭it->next = p->slots;//it的新卡一個是p的當前的slots，也就是ptr上一次切割的item，也可以理解為是當前item的上一個itemif (it->next) it->next->prev = it;//如果當前的it的下一個item也就是也就是ptr上一次切割的item存在（第2次切割以后才會有）//前一個item的后一個item指向當前，其實也就是正常的雙鏈表指向操作p->slots = it;//p的slots永遠指向當前的item，每次如果執行do_slabs_free就會劃走一塊item，并串前后的指針p->sl_curr++;//p的當前空閑的item個數+1p->requested -= size;return;
}

第一次：
(gdb) p?? ?ptr
$5 = 0x7ffff51e1010 ""

vs2012+?? ??? ?ptr?? ?0x0000000000230070 ""?? ?char *

>??? split_slab_page_into_freelist(ptr, id);

第二次：
(gdb) p?? ?ptr
$25 = (void *) 0x7ffff51e1070
0x70-0x10=0x60=96? 正好是一個chunk大小

第三次
(gdb) p ptr
$50 = (void *) 0x7ffff51e10d0
(gdb) p *(it-2)
$46 = {next = 0x7ffff51e1010, prev = 0x7ffff51e10d0, h_next = 0x0, time = 0, exptime = 0, nbytes = 0, refcount = 0, nsuffix = 0 '\000', it_flags = 4 '\004
', slabs_clsid = 0 '\000', nkey = 0 '\000', data = 0x7ffff51e1070}
(gdb) p *(it-4)
$54 = {next = 0x0, prev = 0x7ffff51e1070, h_next = 0x0, time = 0, exptime = 0, nbytes = 0, refcount = 0, nsuffix = 0 '\000', it_flags = 4 '\004', slabs_cl
sid = 0 '\000', nkey = 0 '\000', data = 0x7ffff51e1010}

為什么是it-2才是上一個呢？
因為
(gdb) p sizeof(it)
$57 = 8
(gdb) p sizeof(item)
$58 = 48
it是指針自然是8字節，而item結構是48字節，96正好是48的2倍，其實這里是巧合，求上一個item不應該這么求！
static void split_slab_page_into_freelist(char *ptr, const unsigned int id) {
??? slabclass_t *p = &slabclass[id];
??? int x;
??? for (x = 0; x < p->perslab; x++) {
??????? do_slabs_free(ptr, 0, id);
??????? ptr += p->size;
??? }
}
因為每次是Ptr增加p->size的長度。
所以直接用next指針就可以了
(gdb) p it->next
$67 = (struct _stritem *) 0x7ffff51e1070
(gdb) p *it->next
$68 = {next = 0x7ffff51e1010, prev = 0x7ffff51e10d0, h_next = 0x0, time = 0, exptime = 0, nbytes = 0, refcount = 0, nsuffix = 0 '\000', it_flags = 4 '\004
', slabs_clsid = 0 '\000', nkey = 0 '\000', data = 0x7ffff51e1070}

下面是在vs2012下的調試數據，一看就清楚了

7.分割完ptr之后

回到do_slabs_newslab中，修改了p->slab_list[p->slabs++] = ptr;mem_malloced += len;

然后又回到do_slabs_alloc函數中：

it = (item *)p->slots;//將當前p空閑的slots其實也就是第一個，實際上是鏈表最后一個chunk分配給itemp->slots = it->next;//修改p的空閑的slots，為倒數第二個，也就是it的前一個if (it->next) it->next->prev = 0;//因為it的前一個再前一個已經給item分配了，自然沒了p->sl_curr--;//-1，雖然perslab在分割好之后和sl_curr一樣大，但是sl_curr是要遞減的，而perslab是永遠不變的存儲的是chunk個數ret = (void *)it;

(gdb) p?? ?*p
$72 = {size = 96, perslab = 10922, slots = 0x7ffff52e0f70, sl_curr = 10922, slabs = 1, slab_list = 0x7ffff00169e0, list_size = 16, killing = 0, requested
= 0}
(gdb) n
(gdb) p *p
$73 = {size = 96, perslab = 10922, slots = 0x7ffff52e0f10, sl_curr = 10922, slabs = 1, slab_list = 0x7ffff00169e0, list_size = 16, killing = 0, requested

= 0}

?p->sl_curr--;之后的調試數據，可以看到sl_curry已經減1 ，而perslab是不變的。

(gdb) p *p
$75 = {size = 96, perslab = 10922, slots = 0x7ffff52e0f10, sl_curr = 10921, slabs = 1, slab_list = 0x7ffff00169e0, list_size = 16, killing = 0, requested
= 0}

下面是vs2012中的結果，一樣的：

8.分配好item之后

回到do_item_alloc，已經得到分配的item

(gdb) p *it
$79 = {next = 0x7ffff52e0f10, prev = 0x0, h_next = 0x0, time = 0, exptime = 0, nbytes = 0, refcount = 0, nsuffix = 0 '\000', it_flags = 4 '\004', slabs_cl
sid = 0 '\000', nkey = 0 '\000', data = 0x7ffff52e0f70}

9.模擬memcached調用slabs

3次分配一樣大小的

2.故意設一個比默認chunk=96大的需求設置100，從后面的調試信息可以看出id2已經被算出應該是第2個slabs上：

int main()
{item *it1=NULL;item *it2 = NULL;item *it3 = NULL;item *it4 = NULL;int preallocate = 0;size_t ntotal =63;// item_make_header(nkey + 1, flags, nbytes, suffix, &nsuffix);size_t ntotal2 =100;unsigned int id =0;	unsigned int id2 =0;settings_init();slabs_init(settings.maxbytes, settings.factor, preallocate);if (settings.use_cas) {ntotal += sizeof(uint64_t);}id=slabs_clsid(ntotal);it1=slabs_alloc(ntotal, id);it2=slabs_alloc(ntotal, id);it3=slabs_alloc(ntotal, id);if (settings.use_cas) {ntotal2 += sizeof(uint64_t);}id2=slabs_clsid(ntotal2);it4=slabs_alloc(ntotal2, id2);
return 0;
}

制造perslab=1的情景

分配超過settings.item_size_max / factor=717184的size

這樣就會命中id=41的slabs[41]，同時我們設置2個需要分配717184的：

從調試結果可以看出來slabclass[41]已經分配了2個slab_list，由于這2個slab_list指向的page都只有一個chunk，所以這個chunk中存儲的item的next和prev都是0，

因為第一次分割do_slabs_free的時候：

it->prev = 0;it->next = p->slots;

由于p->slots一開始=0，因為后面不會再切割chunk了，所以這個item的next和prev都是0。

	item *it1=NULL;item *it2 = NULL;item *it3 = NULL;item *it4 = NULL;item *it5 = NULL;int preallocate = 0;size_t ntotal =63;// item_make_header(nkey + 1, flags, nbytes, suffix, &nsuffix);size_t ntotal2 =717184;unsigned int id =0;	unsigned int id2 =0;settings_init();slabs_init(settings.maxbytes, settings.factor, preallocate);if (settings.use_cas) {ntotal += sizeof(uint64_t);}id=slabs_clsid(ntotal);it1=slabs_alloc(ntotal, id);it2=slabs_alloc(ntotal, id);it3=slabs_alloc(ntotal, id);if (settings.use_cas) {ntotal2 += sizeof(uint64_t);}id2=slabs_clsid(ntotal2);it4=slabs_alloc(ntotal2, id2);it5=slabs_alloc(ntotal2, id2);

總結

以上是生活随笔為你收集整理的Memcached内存池分析的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。