X-Git-Url: https://pd.if.org/git/?p=nbds;a=blobdiff_plain;f=runtime%2Fmem.c;h=2f55ff4d2108d93522d878e4549ef5812cc10721;hp=7ffda4378208bb60daa78d29577bace2b634bfd3;hb=HEAD;hpb=7d658a03f83e64690d0c71b4733dd3f9a2c60208 diff --git a/runtime/mem.c b/runtime/mem.c index 7ffda43..2f55ff4 100644 --- a/runtime/mem.c +++ b/runtime/mem.c @@ -14,20 +14,19 @@ #include "rlocal.h" #include "lwt.h" -#define RECYCLE_PAGES - -#define MAX_SCALE 31 // allocate blocks up to 4GB (arbitrary, could be bigger) #ifndef NBD32 +#define MAX_SCALE 36 // allocate blocks up to 64GB (arbitrary, could be bigger) #define MIN_SCALE 3 // smallest allocated block is 8 bytes #define MAX_POINTER_BITS 48 #define PAGE_SCALE 21 // 2MB pages #else +#define MAX_SCALE 31 #define MIN_SCALE 2 // smallest allocated block is 4 bytes #define MAX_POINTER_BITS 32 #define PAGE_SCALE 12 // 4KB pages #endif -#define PAGE_SIZE (1 << PAGE_SCALE) -#define HEADERS_SIZE (((size_t)1 << (MAX_POINTER_BITS - PAGE_SCALE)) * sizeof(header_t)) +#define PAGE_SIZE (1ULL << PAGE_SCALE) +#define HEADERS_SIZE (((size_t)1ULL << (MAX_POINTER_BITS - PAGE_SCALE)) * sizeof(header_t)) typedef struct block { struct block *next; @@ -75,9 +74,9 @@ static inline header_t *get_header (void *r) { } static void *get_new_region (int block_scale) { - LOCALIZE_THREAD_LOCAL(tid_, int); + int thread_index = GET_THREAD_INDEX(); #ifdef RECYCLE_PAGES - tl_t *tl = &tl_[tid_]; // thread-local data + tl_t *tl = &tl_[thread_index]; // thread-local data if (block_scale <= PAGE_SCALE && tl->free_pages != NULL) { void *region = tl->free_pages; tl->free_pages = tl->free_pages->next; @@ -85,11 +84,11 @@ static void *get_new_region (int block_scale) { return region; } #endif//RECYCLE_PAGES - size_t region_size = (1 << block_scale); + size_t region_size = (1ULL << block_scale); if (region_size < PAGE_SIZE) { region_size = PAGE_SIZE; } - void *region = mmap(NULL, region_size, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); + void *region = mmap(NULL, region_size, PROT_READ|PROT_WRITE, MAP_NORESERVE|MAP_ANON|MAP_PRIVATE, -1, 0); TRACE("m1", "get_new_region: mmapped new region %p (size %p)", region, region_size); if (region == (void *)-1) { perror("get_new_region: mmap"); @@ -98,7 +97,7 @@ static void *get_new_region (int block_scale) { if ((size_t)region & (region_size - 1)) { TRACE("m0", "get_new_region: region not aligned", 0, 0); munmap(region, region_size); - region = mmap(NULL, region_size * 2, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); + region = mmap(NULL, region_size * 2, PROT_READ|PROT_WRITE, MAP_NORESERVE|MAP_ANON|MAP_PRIVATE, -1, 0); if (region == (void *)-1) { perror("get_new_region: mmap"); exit(-1); @@ -123,7 +122,7 @@ static void *get_new_region (int block_scale) { TRACE("m1", "get_new_region: header %p (%p)", h, h - headers_); assert(h->scale == 0); h->scale = block_scale; - h->owner = tid_; + h->owner = thread_index; return region; } @@ -133,9 +132,8 @@ void mem_init (void) { // Allocate space for the page headers. This could be a big chunk of memory on 64 bit systems, // but it just takes up virtual address space. Physical space used by the headers is still // proportional to the amount of memory the user mallocs. - headers_ = (header_t *)malloc(HEADERS_SIZE); + headers_ = mmap(NULL, HEADERS_SIZE, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); TRACE("m1", "mem_init: header page %p", headers_, 0); - memset(headers_, 0, HEADERS_SIZE); // initialize spsc queues for (int i = 0; i < MAX_NUM_THREADS; ++i) { @@ -150,7 +148,6 @@ void mem_init (void) { void nbd_free (void *x) { TRACE("m1", "nbd_free: block %p page %p", x, (size_t)x & ~MASK(PAGE_SCALE)); ASSERT(x); - LOCALIZE_THREAD_LOCAL(tid_, int); block_t *b = (block_t *)x; header_t *h = get_header(x); int b_scale = h->scale; @@ -158,16 +155,17 @@ void nbd_free (void *x) { ASSERT(b_scale && b_scale <= MAX_SCALE); #ifdef RECYCLE_PAGES if (b_scale > PAGE_SCALE) { - int rc = munmap(x, 1 << b_scale); + int rc = munmap(x, 1ULL << b_scale); ASSERT(rc == 0); rc = rc; } #endif #ifndef NDEBUG - memset(b, 0xcd, (1 << b_scale)); // bear trap + memset(b, 0xcd, (1ULL << b_scale)); // bear trap #endif - tl_t *tl = &tl_[tid_]; // thread-local data - if (h->owner == tid_) { + int thread_index = GET_THREAD_INDEX(); + tl_t *tl = &tl_[thread_index]; // thread-local data + if (h->owner == thread_index) { TRACE("m1", "nbd_free: private block, old free list head %p", tl->free_list[b_scale], 0); #ifndef RECYCLE_PAGES @@ -266,13 +264,13 @@ void *nbd_malloc (size_t n) { if (EXPECT_FALSE(b_scale < MIN_SCALE)) { b_scale = MIN_SCALE; } if (EXPECT_FALSE(b_scale > MAX_SCALE)) { return NULL; } - LOCALIZE_THREAD_LOCAL(tid_, int); - tl_t *tl = &tl_[tid_]; // thread-local data + tl_t *tl = &tl_[GET_THREAD_INDEX()]; // thread-local data block_t *b = pop_free_list(tl, b_scale); if (b != NULL) { TRACE("m1", "nbd_malloc: returning block %p", b, 0); return b; + assert(b); } // The free list is empty so process blocks freed from other threads and then check again. @@ -281,6 +279,7 @@ void *nbd_malloc (size_t n) { if (b != NULL) { TRACE("m1", "nbd_malloc: returning block %p", b, 0); return b; + assert(b); } #ifdef RECYCLE_PAGES @@ -295,6 +294,7 @@ void *nbd_malloc (size_t n) { ASSERT(b != NULL); TRACE("m1", "nbd_malloc: returning block %p", b, 0); return b; + assert(b); } // There are no partially allocated pages so get a new page. @@ -307,7 +307,7 @@ void *nbd_malloc (size_t n) { // Break up the remainder of the page into blocks and put them on the free list. Start at the // end of the page so that the free list ends up in increasing order, for ease of debugging. if (b_scale < PAGE_SCALE) { - size_t block_size = (1 << b_scale); + size_t block_size = (1ULL << b_scale); block_t *head = NULL; for (int offset = PAGE_SIZE - block_size; offset > 0; offset -= block_size) { block_t *x = (block_t *)(page + offset); @@ -322,16 +322,20 @@ void *nbd_malloc (size_t n) { } TRACE("m1", "nbd_malloc: returning block %p from new region %p", b, (size_t)b & ~MASK(PAGE_SCALE)); + assert(b); return b; } #else//USE_SYSTEM_MALLOC #include +#include "common.h" +#include "rlocal.h" +#include "lwt.h" void mem_init (void) { return; } -void ndb_free (void *x) { +void nbd_free (void *x) { TRACE("m1", "nbd_free: %p", x, 0); #ifndef NDEBUG memset(x, 0xcd, sizeof(void *)); // bear trap