fix for buffer pools > 4GB. Recommend btree2u for linux

diff --git a/README.md b/README.md
index d24a063ed99e0431073545feb012d99df9893434..2c7898a44c3d850f5a0796a6a96b97abcf0e1cfd 100644 (file)
--- a/README.md
+++ b/README.md
@@ -9,13 +9,13 @@ btree2s.c       Single Threaded/MultiProcess version that removes keys all the w
 
 btree2t.c       Single Threaded/MultiProcess version similar to btree2s except that fcntl locking has been replaced by test & set latches in the first few btree pages.  Uses either memory mapping or file I/O.
 
-btree2u.c              Single Threaded/MultiProcess version that implements a traditional buffer pool manager in the first n pages of the btree file.  The buffer pool accesses its pages with mmap.  Evicted pages are written back to the btree file from the buffer pool pages with pwrite.
+btree2u.c              Single Threaded/MultiProcess version that implements a traditional buffer pool manager in the first n pages of the btree file.  The buffer pool accesses its pages with mmap.  Evicted pages are written back to the btree file from the buffer pool pages with pwrite. Recommended for linux.
 
 threads2h.c     Multi-Threaded/Multi-Process with latching implemented by a latch manager with pthreads/SRW latches in the first few btree pages. Recommended for Windows.
 
 threads2i.c     Multi-Threaded/Multi-Process with latching implemented by a latch manager with test & set latches in the first few btree pages with thread yield system calls during contention.
 
-threads2j.c     Multi-Threaded/Multi-Process with latching implemented by a latch manager with test & set locks in the first few btree pages with Linux futex system calls during contention. Recommended for linux.
+threads2j.c     Multi-Threaded/Multi-Process with latching implemented by a latch manager with test & set locks in the first few btree pages with Linux futex system calls during contention.
 
 Compilation is achieved on linux or Windows by:
 

diff --git a/btree2u.c b/btree2u.c
index 1f7edcbd57d66d3a30b85fd7c8745c20e2864425..bd97dd2a609a397e5bae9106b27baccd06033944 100644 (file)
--- a/btree2u.c
+++ b/btree2u.c
@@ -473,8 +473,10 @@ BTERR bt_readpage (BtDb *bt, BtPage page, uid page_no)
 off64_t off = page_no << bt->page_bits;
 
 #ifdef unix
-       if( pread (bt->idx, page, bt->page_size, page_no << bt->page_bits) < bt->page_size )
+       if( pread (bt->idx, page, bt->page_size, page_no << bt->page_bits) < bt->page_size ) {
+               fprintf (stderr, "Unable to read page %.8x errno = %d\n", page_no, errno);
                return bt->err = BTERR_read;
+       }
 #else
 OVERLAPPED ovl[1];
 uint amt[1];
@@ -483,10 +485,14 @@ uint amt[1];
        ovl->Offset = off;
        ovl->OffsetHigh = off >> 32;
 
-       if( !ReadFile(bt->idx, page, bt->page_size, amt, ovl))
+       if( !ReadFile(bt->idx, page, bt->page_size, amt, ovl)) {
+               fprintf (stderr, "Unable to read page %.8x GetLastError = %d\n", page_no, GetLastError());
                return bt->err = BTERR_read;
-       if( *amt <  bt->page_size )
+       }
+       if( *amt <  bt->page_size ) {
+               fprintf (stderr, "Unable to read page %.8x GetLastError = %d\n", page_no, GetLastError());
                return bt->err = BTERR_read;
+       }
 #endif
        return 0;
 }
@@ -525,7 +531,7 @@ uint amt[1];
 
 BTERR bt_latchlink (BtDb *bt, uint hashidx, uint slot, uid page_no)
 {
-BtPage page = (BtPage)(slot * bt->page_size + bt->pagepool);
+BtPage page = (BtPage)((uid)slot * bt->page_size + bt->pagepool);
 BtLatchSet *latch = bt->latchsets + slot;
 
        if( latch->next = bt->table[hashidx].slot )
@@ -656,7 +662,7 @@ BtPage page;
 
        //  update permanent page area in btree
 
-       page = (BtPage)(slot * bt->page_size + bt->pagepool);
+       page = (BtPage)((uid)slot * bt->page_size + bt->pagepool);
 
        if( page->dirty )
          if( bt_writepage (bt, page, latch->page_no) )
@@ -908,21 +914,21 @@ btlatch:
                fprintf (stderr, "Unable to mmap page zero, errno = %d", errno);
                return bt_close (bt), NULL;
        }
-       bt->table = (void *)mmap (0, nlatchpage * bt->page_size, flag, MAP_SHARED, bt->idx, LATCH_page * bt->page_size);
+       bt->table = (void *)mmap (0, (uid)nlatchpage * bt->page_size, flag, MAP_SHARED, bt->idx, LATCH_page * bt->page_size);
        if( bt->table == MAP_FAILED ) {
                fprintf (stderr, "Unable to mmap buffer pool, errno = %d", errno);
                return bt_close (bt), NULL;
        }
 #else
        flag = PAGE_READWRITE;
-       bt->halloc = CreateFileMapping(bt->idx, NULL, flag, 0, (nlatchpage + LATCH_page) * bt->page_size, NULL);
+       bt->halloc = CreateFileMapping(bt->idx, NULL, flag, 0, ((uid)nlatchpage + LATCH_page) * bt->page_size, NULL);
        if( !bt->halloc ) {
                fprintf (stderr, "Unable to create file mapping for buffer pool mgr, GetLastError = %d\n", GetLastError());
                return bt_close (bt), NULL;
        }
 
        flag = FILE_MAP_WRITE;
-       bt->latchmgr = MapViewOfFile(bt->halloc, flag, 0, 0, (nlatchpage + LATCH_page) * bt->page_size);
+       bt->latchmgr = MapViewOfFile(bt->halloc, flag, 0, 0, ((uid)nlatchpage + LATCH_page) * bt->page_size);
        if( !bt->latchmgr ) {
                fprintf (stderr, "Unable to map buffer pool, GetLastError = %d\n", GetLastError());
                return bt_close (bt), NULL;
@@ -930,8 +936,8 @@ btlatch:
 
        bt->table = (void *)((char *)bt->latchmgr + LATCH_page * bt->page_size);
 #endif
-       bt->pagepool = (unsigned char *)bt->table + (nlatchpage - bt->latchmgr->latchtotal) * bt->page_size;
-       bt->latchsets = (BtLatchSet *)(bt->pagepool - bt->latchmgr->latchtotal * sizeof(BtLatchSet));
+       bt->pagepool = (unsigned char *)bt->table + (uid)(nlatchpage - bt->latchmgr->latchtotal) * bt->page_size;
+       bt->latchsets = (BtLatchSet *)(bt->pagepool - (uid)bt->latchmgr->latchtotal * sizeof(BtLatchSet));
 
 #ifdef unix
        bt->mem = malloc (2 * bt->page_size);
@@ -1057,7 +1063,7 @@ void bt_update (BtDb *bt, BtPage page)
 {
 #ifdef unix
        msync (page, bt->page_size, MS_ASYNC);
-//#else
+#else
 //     FlushViewOfFile (page, bt->page_size);
 #endif
        page->dirty = 1;
@@ -1067,7 +1073,7 @@ void bt_update (BtDb *bt, BtPage page)
 
 BtPage bt_mappage (BtDb *bt, BtLatchSet *latch)
 {
-       return (BtPage)((latch - bt->latchsets) * bt->page_size + bt->pagepool);
+       return (BtPage)((uid)(latch - bt->latchsets) * bt->page_size + bt->pagepool);
 }
 
 //     deallocate a deleted page 
@@ -1882,7 +1888,6 @@ BtPage page;
 uint amt[1];
 BtKey ptr;
 
-#ifdef unix
        if( *(ushort *)(bt->latchmgr->lock) )
                fprintf(stderr, "Alloc page locked\n");
        *(ushort *)(bt->latchmgr->lock) = 0;
@@ -1905,7 +1910,7 @@ BtKey ptr;
                        fprintf(stderr, "latchset %d pinned for page %.8x\n", idx, latch->page_no);
                        latch->pin = 0;
                }
-               page = (BtPage)(idx * bt->page_size + bt->pagepool);
+               page = (BtPage)((uid)idx * bt->page_size + bt->pagepool);
 
            if( page->dirty )
                 if( bt_writepage (bt, page, latch->page_no) )
@@ -1929,7 +1934,8 @@ BtKey ptr;
        page_no = LEAF_page;
 
        while( page_no < bt_getid(bt->latchmgr->alloc->right) ) {
-               pread (bt->idx, bt->frame, bt->page_size, page_no << bt->page_bits);
+               if( bt_readpage (bt, bt->frame, page_no) )
+                 fprintf(stderr, "page %.8x unreadable\n", page_no);
                if( !bt->frame->free ) {
                 for( idx = 0; idx++ < bt->frame->cnt - 1; ) {
                  ptr = keyptr(bt->frame, idx+1);
@@ -1945,9 +1951,6 @@ BtKey ptr;
                page_no = next;
        }
        return cnt - 1;
-#else
-       return 0;
-#endif
 }
 
 #ifndef unix