X-Git-Url: https://pd.if.org/git/?p=nbds;a=blobdiff_plain;f=struct%2Fskiplist.c;h=5e9066c280165b078ddc3cbd5a7613d442df4502;hp=2d81fdd42364be4fd766f1b354dafdc58d6427df;hb=052506ff082f134f00834f666653819e18d9bddf;hpb=691dcd0dad1a04358c4f5a8849e4f69788bbf733 diff --git a/struct/skiplist.c b/struct/skiplist.c index 2d81fdd..5e9066c 100644 --- a/struct/skiplist.c +++ b/struct/skiplist.c @@ -2,22 +2,29 @@ * Written by Josh Dybnis and released to the public domain, as explained at * http://creativecommons.org/licenses/publicdomain * - * C implementation of the lock-free skiplist data-structure created by Maurice Herlihy, - * Yossi Lev, and Nir Shavit. See "The Art of Multiprocessor Programming" + * Implementation of the lock-free skiplist data-structure created by Maurice Herlihy, Yossi Lev, + * and Nir Shavit. See Herlihy's and Shivit's book "The Art of Multiprocessor Programming". * http://www.amazon.com/Art-Multiprocessor-Programming-Maurice-Herlihy/dp/0123705916/ * - * See also Kir Fraser's dissertation "Practical Lock Freedom" + * See also Kir Fraser's dissertation "Practical Lock Freedom". * www.cl.cam.ac.uk/techreports/UCAM-CL-TR-579.pdf * + * This code is written for the x86 memory-model. The algorithim depends on certain stores and + * loads being ordered. Be careful, this code probably won't work correctly on platforms with + * weaker memory models if you don't add memory barriers in the right places. */ #include #include #include "common.h" +#include "runtime.h" #include "struct.h" #include "mem.h" +#include "tls.h" -#define MAX_LEVEL 3 +// Setting MAX_LEVEL to 0 essentially makes this data structure the Harris-Michael lock-free list +// in list.c +#define MAX_LEVEL 31 typedef struct node { uint64_t key; @@ -28,57 +35,67 @@ typedef struct node { typedef struct skiplist { node_t *head; - node_t *last; - int top_level; } skiplist_t; -static int random_level (int r) { +static int random_level (void) { + unsigned r = nbd_rand(); if (r&1) return 0; - int n = __builtin_ctz(r); - if (n < MAX_LEVEL) - return n; - return MAX_LEVEL; + int n = __builtin_ctz(r)-1; +#if MAX_LEVEL < 31 + if (n > MAX_LEVEL) + return MAX_LEVEL; +#endif + assert(n <= MAX_LEVEL); + return n; } -node_t *node_alloc (int top_level, uint64_t key, uint64_t value) { - assert(top_level >= 0 && top_level <= MAX_LEVEL); - size_t sz = sizeof(node_t) + (top_level + 1) * sizeof(node_t *); +node_t *node_alloc (int level, uint64_t key, uint64_t value) { + assert(level >= 0 && level <= MAX_LEVEL); + size_t sz = sizeof(node_t) + (level + 1) * sizeof(node_t *); node_t *item = (node_t *)nbd_malloc(sz); memset(item, 0, sz); item->key = key; item->value = value; - item->top_level = top_level; + item->top_level = level; return item; } -skiplist_t *skiplist_alloc (void) { +skiplist_t *sl_alloc (void) { skiplist_t *skiplist = (skiplist_t *)nbd_malloc(sizeof(skiplist_t)); skiplist->head = node_alloc(MAX_LEVEL, 0, 0); - skiplist->last = node_alloc(MAX_LEVEL, (uint64_t)-1, 0); - for (int level = 0; level <= MAX_LEVEL; ++level) { - skiplist->head->next[level] = skiplist->last; - } + memset(skiplist->head->next, 0, (MAX_LEVEL+1) * sizeof(skiplist_t *)); return skiplist; } -static node_t *find_preds (node_t **preds, skiplist_t *skiplist, uint64_t key, int help_remove) { +static node_t *find_preds (node_t *preds[MAX_LEVEL+1], int n, skiplist_t *skiplist, uint64_t key, int help_remove) { node_t *pred = skiplist->head; node_t *item = NULL; TRACE("s3", "find_preds: searching for key %p in skiplist (head is %p)", key, pred); -#ifndef NDEBUG - int count = 0; -#endif + + // Optimization for small lists. No need to traverse empty higher levels. + assert(MAX_LEVEL > 2); + int start_level = 2; + while (pred->next[start_level+1] != NULL) { + start_level += start_level - 1; + if (EXPECT_FALSE(start_level >= MAX_LEVEL)) { + start_level = MAX_LEVEL; + break; + } + } + if (EXPECT_FALSE(start_level < n)) { + start_level = n; + } // Traverse the levels of the skiplist from the top level to the bottom - for (int level = MAX_LEVEL; level >= 0; --level) { + for (int level = start_level; level >= 0; --level) { TRACE("s3", "find_preds: level %llu", level, 0); item = pred->next[level]; - if (IS_TAGGED(item)) { + if (EXPECT_FALSE(IS_TAGGED(item))) { TRACE("s3", "find_preds: pred %p is marked for removal (item %p); retry", pred, item); - return find_preds(preds, skiplist, key, help_remove); // retry + return find_preds(preds, n, skiplist, key, help_remove); // retry } - do { + while (item != NULL) { node_t *next = item->next[level]; TRACE("s3", "find_preds: visiting item %p (next %p)", item, next); TRACE("s3", "find_preds: key %p", item->key, 0); @@ -89,6 +106,8 @@ static node_t *find_preds (node_t **preds, skiplist_t *skiplist, uint64_t key, i // Skip over partially removed items. if (!help_remove) { item = (node_t *)STRIP_TAG(item->next); + if (EXPECT_FALSE(item == NULL)) + break; next = item->next[level]; continue; } @@ -97,6 +116,8 @@ static node_t *find_preds (node_t **preds, skiplist_t *skiplist, uint64_t key, i node_t *other; if ((other = SYNC_CAS(&pred->next[level], item, STRIP_TAG(next))) == item) { item = (node_t *)STRIP_TAG(next); + if (EXPECT_FALSE(item == NULL)) + break; next = item->next[level]; TRACE("s3", "find_preds: unlinked item %p from pred %p", item, pred); TRACE("s3", "find_preds: now item is %p next is %p", item, next); @@ -104,82 +125,98 @@ static node_t *find_preds (node_t **preds, skiplist_t *skiplist, uint64_t key, i // The thread that completes the unlink should free the memory. if (level == 0) { nbd_defer_free(other); } } else { - TRACE("s3", "find_preds: lost race to unlink item from pred %p; its link changed to %p",pred,other); + TRACE("s3", "find_preds: lost race to unlink from pred %p; its link changed to %p", pred, other); if (IS_TAGGED(other)) - return find_preds(preds, skiplist, key, help_remove); // retry + return find_preds(preds, n, skiplist, key, help_remove); // retry item = other; + if (EXPECT_FALSE(item == NULL)) + break; next = item->next[level]; } } + if (EXPECT_FALSE(item == NULL)) + break; + // If we reached the key (or passed where it should be), we found a pred. Save it and continue down. if (item->key >= key) { TRACE("s3", "find_preds: found pred %p item %p", pred, item); - if (preds != NULL) { - preds[level] = pred; - } break; } - assert(count++ < 18); pred = item; item = next; - - } while (1); + } + if (preds != NULL) { + preds[level] = pred; + } + } + if (n == -1 && item != NULL) { + assert(preds != NULL); + for (int level = start_level + 1; level <= item->top_level; ++level) { + preds[level] = skiplist->head; + } } return item; } // Fast find that does not help unlink partially removed nodes and does not return the node's predecessors. -uint64_t skiplist_lookup (skiplist_t *skiplist, uint64_t key) { - TRACE("s3", "skiplist_lookup: searching for key %p in skiplist %p", key, skiplist); - node_t *item = find_preds(NULL, skiplist, key, FALSE); +uint64_t sl_lookup (skiplist_t *skiplist, uint64_t key) { + TRACE("s3", "sl_lookup: searching for key %p in skiplist %p", key, skiplist); + node_t *item = find_preds(NULL, 0, skiplist, key, FALSE); // If we found an matching the return its value. - return (item->key == key) ? item->value : DOES_NOT_EXIST; + return (item && item->key == key) ? item->value : DOES_NOT_EXIST; } // Insert the if it doesn't already exist in the -uint64_t skiplist_add_i (int n, skiplist_t *skiplist, uint64_t key, uint64_t value) { - TRACE("s3", "skiplist_add: inserting key %p value %p", key, value); +uint64_t sl_add (skiplist_t *skiplist, uint64_t key, uint64_t value) { + TRACE("s3", "sl_add: inserting key %p value %p", key, value); node_t *preds[MAX_LEVEL+1]; node_t *item = NULL; do { - node_t *next = find_preds(preds, skiplist, key, TRUE); + int n = random_level(); + node_t *next = find_preds(preds, n, skiplist, key, TRUE); // If a node matching already exists in the skiplist, return its value. - if (next->key == key) { - TRACE("s3", "skiplist_add: there is already an item %p (value %p) with the same key", next, next->value); + if (next != NULL && next->key == key) { + TRACE("s3", "sl_add: there is already an item %p (value %p) with the same key", next, next->value); if (EXPECT_FALSE(item != NULL)) { nbd_free(item); } return next->value; } // First insert into the bottom level. - if (EXPECT_TRUE(item == NULL)) { item = node_alloc(random_level(n), key, value); } - TRACE("s3", "skiplist_add: attempting to insert item between %p and %p", preds[0], next); + if (EXPECT_TRUE(item == NULL)) { item = node_alloc(n, key, value); } + TRACE("s3", "sl_add: attempting to insert item between %p and %p", preds[0], next); item->next[0] = next; for (int level = 1; level <= item->top_level; ++level) { - item->next[level] = preds[level]->next[level]; + node_t *pred = preds[level]; + item->next[level] = pred->next[level]; } - node_t *other = SYNC_CAS(&preds[0]->next[0], next, item); + node_t *pred = preds[0]; + node_t *other = SYNC_CAS(&pred->next[0], next, item); if (other == next) { - TRACE("s3", "skiplist_add: successfully inserted item %p at level 0", item, 0); + TRACE("s3", "sl_add: successfully inserted item %p at level 0", item, 0); break; // success } - TRACE("s3", "skiplist_add: failed to change pred's link: expected %p found %p", next, other); + TRACE("s3", "sl_add: failed to change pred's link: expected %p found %p", next, other); } while (1); // Insert into the skiplist from the bottom level up. for (int level = 1; level <= item->top_level; ++level) { do { - node_t *pred = preds[level]; - node_t *next = pred->next[level]; - while (EXPECT_FALSE(IS_TAGGED(next) || next->key < key)) { - find_preds(preds, skiplist, key, TRUE); + node_t *pred; + node_t *next; + do { pred = preds[level]; next = pred->next[level]; - } + if (next == NULL) // item goes at the end of the list + break; + if (!IS_TAGGED(next) && next->key > key) // pred's link changed + break; + find_preds(preds, item->top_level, skiplist, key, TRUE); + } while (1); do { // There in no need to continue linking in the item if another thread removed it. @@ -192,25 +229,25 @@ uint64_t skiplist_add_i (int n, skiplist_t *skiplist, uint64_t key, uint64_t val break; } while (1); - TRACE("s3", "skiplist_add: attempting to insert item between %p and %p", pred, next); + TRACE("s3", "sl_add: attempting to insert item between %p and %p", pred, next); node_t *other = SYNC_CAS(&pred->next[level], next, item); if (other == next) { - TRACE("s3", "skiplist_add: successfully inserted item %p at level %llu", item, level); + TRACE("s3", "sl_add: successfully inserted item %p at level %llu", item, level); break; // success } - TRACE("s3", "skiplist_add: failed to change pred's link: expected %p found %p", next, other); + TRACE("s3", "sl_add: failed to change pred's link: expected %p found %p", next, other); } while (1); } return value; } -uint64_t skiplist_remove (skiplist_t *skiplist, uint64_t key) { - TRACE("s3", "skiplist_remove: removing item with key %p from skiplist %p", key, skiplist); +uint64_t sl_remove (skiplist_t *skiplist, uint64_t key) { + TRACE("s3", "sl_remove: removing item with key %p from skiplist %p", key, skiplist); node_t *preds[MAX_LEVEL+1]; - node_t *item = find_preds(preds, skiplist, key, TRUE); - if (item->key != key) { - TRACE("s3", "skiplist_remove: remove failed, an item with a matching key does not exist in the skiplist", 0, 0); + node_t *item = find_preds(preds, -1, skiplist, key, TRUE); + if (item == NULL || item->key != key) { + TRACE("s3", "sl_remove: remove failed, an item with a matching key does not exist in the skiplist", 0, 0); return DOES_NOT_EXIST; } @@ -219,14 +256,14 @@ uint64_t skiplist_remove (skiplist_t *skiplist, uint64_t key) { // them succeeds. for (int level = item->top_level; level >= 0; --level) { if (EXPECT_FALSE(IS_TAGGED(item->next[level]))) { - TRACE("s3", "skiplist_remove: %p is already marked for removal by another thread", item, 0); + TRACE("s3", "sl_remove: %p is already marked for removal by another thread", item, 0); if (level == 0) return DOES_NOT_EXIST; continue; } node_t *next = SYNC_FETCH_AND_OR(&item->next[level], TAG); if (EXPECT_FALSE(IS_TAGGED(next))) { - TRACE("s3", "skiplist_remove: lost race -- %p is already marked for removal by another thread", item, 0); + TRACE("s3", "sl_remove: lost race -- %p is already marked for removal by another thread", item, 0); if (level == 0) return DOES_NOT_EXIST; continue; @@ -240,10 +277,10 @@ uint64_t skiplist_remove (skiplist_t *skiplist, uint64_t key) { while (level >= 0) { node_t *pred = preds[level]; node_t *next = item->next[level]; - TRACE("s3", "skiplist_remove: link item's pred %p to it's successor %p", pred, STRIP_TAG(next)); + TRACE("s3", "sl_remove: link item's pred %p to it's successor %p", pred, STRIP_TAG(next)); node_t *other = NULL; if ((other = SYNC_CAS(&pred->next[level], item, STRIP_TAG(next))) != item) { - TRACE("s3", "skiplist_remove: unlink failed; pred's link changed from %p to %p", item, other); + TRACE("s3", "sl_remove: unlink failed; pred's link changed from %p to %p", item, other); // By marking the item earlier, we logically removed it. It is safe to leave the item partially // unlinked. Another thread will finish physically removing it from the skiplist. return value; @@ -256,29 +293,34 @@ uint64_t skiplist_remove (skiplist_t *skiplist, uint64_t key) { return value; } -void skiplist_print (skiplist_t *skiplist) { +void sl_print (skiplist_t *skiplist) { for (int level = MAX_LEVEL; level >= 0; --level) { - printf("(%d) ", level); node_t *item = skiplist->head; + if (item->next[level] == NULL) + continue; + printf("(%d) ", level); while (item) { node_t *next = item->next[level]; - printf("%s%p:0x%llx ", IS_TAGGED(next) ? "*" : "", item, item->key); + printf("%s%p ", IS_TAGGED(next) ? "*" : "", item); item = (node_t *)STRIP_TAG(next); } printf("\n"); fflush(stdout); } + printf("\n"); node_t *item = skiplist->head; while (item) { - assert(item->top_level <= MAX_LEVEL); int is_marked = IS_TAGGED(item->next[0]); - printf("%s%p:0x%llx (%d", is_marked ? "*" : "", item, item->key, item->top_level); - for (int i = 1; i <= item->top_level; ++i) { - node_t *next = (node_t *)STRIP_TAG(item->next[i]); - printf(" %p:0x%llx", item->next[i], next ? next->key : 0); + printf("%s%p:0x%llx [%d]", is_marked ? "*" : "", item, item->key, item->top_level); + for (int level = 1; level <= item->top_level; ++level) { + node_t *next = (node_t *)STRIP_TAG(item->next[level]); + is_marked = IS_TAGGED(item->next[0]); + printf(" %p%s", next, is_marked ? "*" : ""); + if (item == skiplist->head && item->next[level] == NULL) + break; } - printf(")\n"); + printf("\n"); fflush(stdout); item = (node_t *)STRIP_TAG(item->next[0]); } @@ -295,24 +337,21 @@ void skiplist_print (skiplist_t *skiplist) { static volatile int wait_; static long num_threads_; -static skiplist_t *skiplist_; +static skiplist_t *sl_; void *worker (void *arg) { - int id = (int)(size_t)arg; - - unsigned int rand_seed = id+1;//rdtsc_l(); // Wait for all the worker threads to be ready. SYNC_ADD(&wait_, -1); do {} while (wait_); for (int i = 0; i < NUM_ITERATIONS/num_threads_; ++i) { - int n = rand_r(&rand_seed); - int key = (n & 0xF) + 1; - if (n & (1 << 8)) { - skiplist_add_i(n, skiplist_, key, 1); + unsigned r = nbd_rand(); + int key = (r & 0xF) + 1; + if (r & (1 << 8)) { + sl_add(sl_, key, 1); } else { - skiplist_remove(skiplist_, key); + sl_remove(sl_, key); } rcu_update(); @@ -352,7 +391,7 @@ int main (int argc, char **argv) { } } - skiplist_ = skiplist_alloc(); + sl_ = sl_alloc(); struct timeval tv1, tv2; gettimeofday(&tv1, NULL); @@ -370,9 +409,8 @@ int main (int argc, char **argv) { gettimeofday(&tv2, NULL); int ms = (int)(1000000*(tv2.tv_sec - tv1.tv_sec) + tv2.tv_usec - tv1.tv_usec) / 1000; + sl_print(sl_); printf("Th:%ld Time:%dms\n", num_threads_, ms); - skiplist_print(skiplist_); - lwt_dump("lwt.out"); return 0; }