diff --git a/headers/private/kernel/util/RadixBitmap.h b/headers/private/kernel/util/RadixBitmap.h new file mode 100644 index 0000000000..e9b6f2ddb5 --- /dev/null +++ b/headers/private/kernel/util/RadixBitmap.h @@ -0,0 +1,76 @@ +/*- + * Copyright (c) 1998 Matthew Dillon. All Rights Reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * The radix bitmap structure is ported from FreeBSD and is only used + * in swap system at the time. + */ + + +#ifndef _KERNEL_UTIL_RADIX_BITMAP_H +#define _KERNEL_UTIL_RADIX_BITMAP_H + + +#include + + +typedef uint32 swap_addr_t; +typedef uint32 bitmap_t; + +typedef struct radix_node { + union { + bitmap_t bitmap; // bitmap for the slots if we are a leaf + int32 available; // available slots under us if we are not a leaf + } u; + int32 big_hint; // the biggest continuous slots under us +} radix_node; + +// Bitmap which uses radix tree for hinting. +// The radix tree is stored in an array. +typedef struct radix_bitmap { + swap_addr_t free_slots; // # of free slots + swap_addr_t radix; // coverage radix + swap_addr_t skip; // starting skip + radix_node *root; // root of radix tree, actually it is an array + swap_addr_t root_size; // size of the array(# of nodes in the tree) +} radix_bitmap; + + +#define BITMAP_RADIX (sizeof(swap_addr_t) * 8) +#define NODE_RADIX 16 + +#define SWAP_SLOT_NONE ((swap_addr_t)-1) + + +extern radix_bitmap *radix_bitmap_create(uint32 slots); +extern swap_addr_t radix_bitmap_alloc(radix_bitmap *bmp, uint32 count); +extern void radix_bitmap_dealloc(radix_bitmap *bmp, swap_addr_t slotIndex, + uint32 count); +extern void radix_bitmap_destroy(radix_bitmap *bmp); + +#endif // _KERNEL_UTIL_RADIX_BITMAP_H + diff --git a/src/system/kernel/util/Jamfile b/src/system/kernel/util/Jamfile index add563160a..54a62aa7ed 100644 --- a/src/system/kernel/util/Jamfile +++ b/src/system/kernel/util/Jamfile @@ -10,6 +10,7 @@ KernelMergeObject kernel_util.o : list.c queue.c ring_buffer.cpp + RadixBitmap.cpp : $(TARGET_KERNEL_PIC_CCFLAGS) -DUSING_LIBGCC ; diff --git a/src/system/kernel/util/RadixBitmap.cpp b/src/system/kernel/util/RadixBitmap.cpp new file mode 100644 index 0000000000..7a2fc303b4 --- /dev/null +++ b/src/system/kernel/util/RadixBitmap.cpp @@ -0,0 +1,330 @@ +/*- + * Copyright (c) 1998 Matthew Dillon. All Rights Reserved. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * BLIST.C - Bitmap allocator/deallocator, using a radix tree with hinting + * + * This module implements a general bitmap allocator/deallocator. The + * allocator eats around 2 bits per 'block'. The module does not + * try to interpret the meaning of a 'block' other then to return + * SWAPBLK_NONE on an allocation failure. + * + * A radix tree is used to maintain the bitmap. Two radix constants are + * involved: One for the bitmaps contained in the leaf nodes (typically + * 32), and one for the meta nodes (typically 16). Both meta and leaf + * nodes have a hint field. This field gives us a hint as to the largest + * free contiguous range of blocks under the node. It may contain a + * value that is too high, but will never contain a value that is too + * low. When the radix tree is searched, allocation failures in subtrees + * update the hint. + * + * The radix tree also implements two collapsed states for meta nodes: + * the ALL-ALLOCATED state and the ALL-FREE state. If a meta node is + * in either of these two states, all information contained underneath + * the node is considered stale. These states are used to optimize + * allocation and freeing operations. + * + * The hinting greatly increases code efficiency for allocations while + * the general radix structure optimizes both allocations and frees. The + * radix tree should be able to operate well no matter how much + * fragmentation there is and no matter how large a bitmap is used. + * + * Unlike the rlist code, the blist code wires all necessary memory at + * creation time. Neither allocations nor frees require interaction with + * the memory subsystem. In contrast, the rlist code may allocate memory + * on an rlist_free() call. The non-blocking features of the blist code + * are used to great advantage in the swap code (vm/nswap_pager.c). The + * rlist code uses a little less overall memory then the blist code (but + * due to swap interleaving not all that much less), but the blist code + * scales much, much better. + * + * LAYOUT: The radix tree is layed out recursively using a + * linear array. Each meta node is immediately followed (layed out + * sequentially in memory) by BLIST_META_RADIX lower level nodes. This + * is a recursive structure but one that can be easily scanned through + * a very simple 'skip' calculation. In order to support large radixes, + * portions of the tree may reside outside our memory allocation. We + * handle this with an early-termination optimization (when bighint is + * set to -1) on the scan. The memory allocation is only large enough + * to cover the number of blocks requested at creation time even if it + * must be encompassed in larger root-node radix. + * + * NOTE: the allocator cannot currently allocate more then + * BLIST_BMAP_RADIX blocks per call. It will panic with 'allocation too + * large' if you try. This is an area that could use improvement. The + * radix is large enough that this restriction does not effect the swap + * system, though. Currently only the allocation code is effected by + * this algorithmic unfeature. The freeing code can handle arbitrary + * ranges. + * + * This code can be compiled stand-alone for debugging. + */ + +/* + * NOTE: a few modifications is made in the orginal FreeBSD code: + * 1. change the name of some variables/constants + * 2. discard the code handling ALL_FREE and ALL_ALLOCATED state + * 3. allocate more than 32 slots won't lead to kernel panic + * 4. remove all the code for debugging + * + * Zhao Shuai + * upczhsh@163.com + */ + + +#include +#include + +#define TERMINATOR -1 + + +static uint32 +radix_bitmap_init(radix_node *node, uint32 radix, uint32 skip, uint32 slots) +{ + uint32 index = 0; + int32 big_hint = radix < slots ? radix : slots; + + // leaf node + if (radix == BITMAP_RADIX) { + if (node) { + node->big_hint = big_hint; + if (big_hint == BITMAP_RADIX) + node->u.bitmap = 0; + else + node->u.bitmap = (bitmap_t)-1 << big_hint; + } + return index; + } + + // not leaf node + if (node) { + node->big_hint = big_hint; + node->u.available = big_hint; + } + + radix /= NODE_RADIX; + uint32 next_skip = skip / NODE_RADIX; + + uint32 i; + for (i = 1; i <= skip; i += next_skip) { + if (slots >= radix) { + index = i + radix_bitmap_init(node ? &node[i] : NULL, + radix, next_skip - 1, radix); + slots -= radix; + } else if (slots > 0) { + index = i + radix_bitmap_init(node ? &node[i] : NULL, + radix, next_skip - 1, slots); + slots = 0; + } else { // add a terminator + if (node) + node[i].big_hint = TERMINATOR; + break; + } + } + + if (index < i) + index = i; + + return index; +} + + +radix_bitmap * +radix_bitmap_create(uint32 slots) +{ + uint32 radix = BITMAP_RADIX; + uint32 skip = 0; + + while (radix < slots) { + radix *= NODE_RADIX; + skip = (skip + 1) * NODE_RADIX; + } + + radix_bitmap *bmp = (radix_bitmap *)malloc(sizeof(radix_bitmap)); + if (bmp == NULL) + return NULL; + + bmp->radix = radix; + bmp->skip = skip; + bmp->free_slots = slots; + bmp->root_size = 1 + radix_bitmap_init(NULL, radix, skip, slots); + + bmp->root = (radix_node *)malloc(bmp->root_size * sizeof(radix_node)); + if (bmp->root == NULL) { + free(bmp); + return NULL; + } + + radix_bitmap_init(bmp->root, radix, skip, slots); + + return bmp; +} + + +void +radix_bitmap_destroy(radix_bitmap *bmp) +{ + free(bmp->root); + free(bmp); +} + + +static swap_addr_t +radix_leaf_alloc(radix_node *leaf, swap_addr_t slotIndex, int32 count) +{ + if (count <= BITMAP_RADIX) { + bitmap_t bitmap = ~leaf->u.bitmap; + uint32 n = BITMAP_RADIX - count; + bitmap_t mask = (bitmap_t)-1 >> n; + for (uint32 j = 0; j <= n; j++) { + if ((bitmap & mask) == mask) { + leaf->u.bitmap |= mask; + return (slotIndex + j); + } + mask <<= 1; + } + } + + // we could not allocate count here, update big_hint + if (leaf->big_hint >= count) + leaf->big_hint = count - 1; + return SWAP_SLOT_NONE; +} + + +static swap_addr_t +radix_node_alloc(radix_node *node, swap_addr_t slotIndex, int32 count, + uint32 radix, uint32 skip) +{ + uint32 next_skip = skip / NODE_RADIX; + radix /= NODE_RADIX; + + for (uint32 i = 1; i <= skip; i += next_skip) { + if (node[i].big_hint == TERMINATOR) // TERMINATOR + break; + + if (count <= node[i].big_hint) { + swap_addr_t addr = SWAP_SLOT_NONE; + if (next_skip == 1) + addr = radix_leaf_alloc(&node[i], slotIndex, count); + else + addr = radix_node_alloc(&node[i], slotIndex, count, radix, + next_skip - 1); + if (addr != SWAP_SLOT_NONE) { + node->u.available -= count; + if (node->big_hint > node->u.available) + node->big_hint = node->u.available; + + return addr; + } + } + slotIndex += radix; + } + + // we could not allocate count in the subtree, update big_hint + if (node->big_hint >= count) + node->big_hint = count - 1; + return SWAP_SLOT_NONE; +} + + +swap_addr_t +radix_bitmap_alloc(radix_bitmap *bmp, uint32 count) +{ + swap_addr_t addr = SWAP_SLOT_NONE; + + if (bmp->radix == BITMAP_RADIX) + addr = radix_leaf_alloc(bmp->root, 0, count); + else + addr = radix_node_alloc(bmp->root, 0, count, bmp->radix, bmp->skip); + + if (addr != SWAP_SLOT_NONE) + bmp->free_slots -= count; + + return addr; +} + + +static void +radix_leaf_dealloc(radix_node *leaf, swap_addr_t slotIndex, uint32 count) +{ + uint32 n = slotIndex & (BITMAP_RADIX - 1); + bitmap_t mask = ((bitmap_t)-1 >> (BITMAP_RADIX - count - n)) + & ((bitmap_t)-1 << n); + leaf->u.bitmap &= ~mask; + + leaf->big_hint = BITMAP_RADIX; +} + + +static void +radix_node_dealloc(radix_node *node, swap_addr_t slotIndex, uint32 count, + uint32 radix, uint32 skip, swap_addr_t index) +{ + node->u.available += count; + + uint32 next_skip = skip / NODE_RADIX; + radix /= NODE_RADIX; + + uint32 i = (slotIndex - index) / radix; + index += i * radix; + i = i * next_skip + 1; + + while (i <= skip && index < slotIndex + count) { + uint32 v = index + radix - slotIndex; + if (v > count) + v = count; + + if (next_skip == 1) + radix_leaf_dealloc(&node[i], slotIndex, v); + else + radix_node_dealloc(&node[i], slotIndex, v, radix, + next_skip - 1, index); + + if (node->big_hint < node[i].big_hint) + node->big_hint = node[i].big_hint; + + count -= v; + slotIndex += v; + index += radix; + i += next_skip; + } +} + + +void +radix_bitmap_dealloc(radix_bitmap *bmp, swap_addr_t slotIndex, uint32 count) +{ + if (bmp->radix == BITMAP_RADIX) + radix_leaf_dealloc(bmp->root, slotIndex, count); + else + radix_node_dealloc(bmp->root, slotIndex, count, bmp->radix, + bmp->skip, 0); + + bmp->free_slots += count; +} + diff --git a/src/system/kernel/vm/VMAnonymousCache.cpp b/src/system/kernel/vm/VMAnonymousCache.cpp index acf11088d9..cf009f666c 100644 --- a/src/system/kernel/vm/VMAnonymousCache.cpp +++ b/src/system/kernel/vm/VMAnonymousCache.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -61,34 +62,13 @@ #define SWAP_BLOCK_SHIFT 5 /* 1 << SWAP_BLOCK_SHIFT == SWAP_BLOCK_PAGES */ #define SWAP_BLOCK_MASK (SWAP_BLOCK_PAGES - 1) -#define SWAP_SLOT_NONE (~(swap_addr_t)0) - -// bitmap allocation macros -#define NUM_BYTES_PER_WORD 4 -#define NUM_BITS_PER_WORD (8 * NUM_BYTES_PER_WORD) -#define MAP_SHIFT 5 // 1 << MAP_SHIFT == NUM_BITS_PER_WORD - -#define TESTBIT(map, i) \ - (((map)[(i) >> MAP_SHIFT] & (1 << (i) % NUM_BITS_PER_WORD))) -#define SETBIT(map, i) \ - (((map)[(i) >> MAP_SHIFT] |= (1 << (i) % NUM_BITS_PER_WORD))) -#define CLEARBIT(map, i) \ - (((map)[(i) >> MAP_SHIFT] &= ~(1 << (i) % NUM_BITS_PER_WORD))) - -// The stack functionality looks like a good candidate to put into its own -// store. I have not done this because once we have a swap file backing up -// the memory, it would probably not be a good idea to separate this -// anymore. - struct swap_file : DoublyLinkedListLinkImpl { int fd; struct vnode *vnode; void *cookie; swap_addr_t first_slot; swap_addr_t last_slot; - swap_addr_t used; // # of slots used - uint32 *maps; // bitmap for the slots - swap_addr_t hint; // next free slot + radix_bitmap *bmp; }; struct swap_hash_key { @@ -96,7 +76,8 @@ struct swap_hash_key { off_t page_index; // page index in the cache }; -// Each swap block contains SWAP_BLOCK_PAGES pages +// Each swap block contains swap address information for +// SWAP_BLOCK_PAGES continuous pages from the same cache struct swap_block : HashTableLink { swap_hash_key key; uint32 used; @@ -222,23 +203,22 @@ private: #endif - static int dump_swap_info(int argc, char** argv) { swap_addr_t totalSwapPages = 0; - swap_addr_t usedSwapPages = 0; + swap_addr_t freeSwapPages = 0; kprintf("swap files:\n"); for (SwapFileList::Iterator it = sSwapFileList.GetIterator(); swap_file* file = it.Next();) { swap_addr_t total = file->last_slot - file->first_slot; - kprintf(" vnode: %p, pages: total: %lu, used: %lu\n", - file->vnode, total, file->used); + kprintf(" vnode: %p, pages: total: %lu, free: %lu\n", + file->vnode, total, file->bmp->free_slots); totalSwapPages += total; - usedSwapPages += file->used; + freeSwapPages += file->bmp->free_slots; } kprintf("\n"); @@ -247,8 +227,8 @@ dump_swap_info(int argc, char** argv) kprintf("available: %9llu\n", sAvailSwapSpace / B_PAGE_SIZE); kprintf("reserved: %9llu\n", totalSwapPages - sAvailSwapSpace / B_PAGE_SIZE); - kprintf("used: %9lu\n", usedSwapPages); - kprintf("free: %9lu\n", totalSwapPages - usedSwapPages); + kprintf("used: %9lu\n", totalSwapPages - freeSwapPages); + kprintf("free: %9lu\n", freeSwapPages); return 0; } @@ -265,76 +245,43 @@ swap_slot_alloc(uint32 count) return SWAP_SLOT_NONE; } - // compute how many pages are free in all swap files - uint32 freeSwapPages = 0; - for (SwapFileList::Iterator it = sSwapFileList.GetIterator(); - swap_file *file = it.Next();) - freeSwapPages += file->last_slot - file->first_slot - file->used; - - if (freeSwapPages < count) { + // since radix bitmap could not handle more than 32 pages, we return + // SWAP_SLOT_NONE, this forces Write() adjust allocation amount + if (count > BITMAP_RADIX) { mutex_unlock(&sSwapFileListLock); - panic("swap_slot_alloc(): swap space exhausted!\n"); return SWAP_SLOT_NONE; } - swap_addr_t hint = 0; - swap_addr_t j; + swap_addr_t j, addr = SWAP_SLOT_NONE; for (j = 0; j < sSwapFileCount; j++) { if (sSwapFileAlloc == NULL) sSwapFileAlloc = sSwapFileList.First(); - hint = sSwapFileAlloc->hint; - swap_addr_t pageCount = sSwapFileAlloc->last_slot - - sSwapFileAlloc->first_slot; - - swap_addr_t i = 0; - while (i < count && (hint + count) <= pageCount) { - if (TESTBIT(sSwapFileAlloc->maps, hint + i)) { - hint++; - i = 0; - } else - i++; - } - - if (i == count) + addr = radix_bitmap_alloc(sSwapFileAlloc->bmp, count); + if (addr != SWAP_SLOT_NONE) { + addr += sSwapFileAlloc->first_slot; break; + } // this swap_file is full, find another sSwapFileAlloc = sSwapFileList.GetNext(sSwapFileAlloc); } - // no swap file can alloc so many pages, we return SWAP_SLOT_NONE - // and VMAnonymousCache::Write() will adjust allocation amount if (j == sSwapFileCount) { mutex_unlock(&sSwapFileListLock); + panic("swap_slot_alloc: swap space exhausted!\n"); return SWAP_SLOT_NONE; } - swap_addr_t slotIndex = sSwapFileAlloc->first_slot + hint; - - for (uint32 i = 0; i < count; i++) - SETBIT(sSwapFileAlloc->maps, hint + i); - if (hint == sSwapFileAlloc->hint) { - sSwapFileAlloc->hint += count; - swap_addr_t pageCount = sSwapFileAlloc->last_slot - - sSwapFileAlloc->first_slot; - while (TESTBIT(sSwapFileAlloc->maps, sSwapFileAlloc->hint) - && sSwapFileAlloc->hint < pageCount) { - sSwapFileAlloc->hint++; - } - } - - sSwapFileAlloc->used += count; - - // if this swap file has used more than 90% percent of its slots + // if this swap file has used more than 90% percent of its space // switch to another - if (sSwapFileAlloc->used - > 9 * (sSwapFileAlloc->last_slot - sSwapFileAlloc->first_slot) / 10) + if (sSwapFileAlloc->bmp->free_slots + < (sSwapFileAlloc->last_slot - sSwapFileAlloc->first_slot) / 10) sSwapFileAlloc = sSwapFileList.GetNext(sSwapFileAlloc); mutex_unlock(&sSwapFileListLock); - return slotIndex; + return addr; } @@ -361,16 +308,8 @@ swap_slot_dealloc(swap_addr_t slotIndex, uint32 count) mutex_lock(&sSwapFileListLock); swap_file *swapFile = find_swap_file(slotIndex); - slotIndex -= swapFile->first_slot; - - for (uint32 i = 0; i < count; i++) - CLEARBIT(swapFile->maps, slotIndex + i); - - if (swapFile->hint > slotIndex) - swapFile->hint = slotIndex; - - swapFile->used -= count; + radix_bitmap_dealloc(swapFile->bmp, slotIndex, count); mutex_unlock(&sSwapFileListLock); } @@ -1101,16 +1040,12 @@ swap_file_add(char *path) swap->cookie = descriptor->cookie; int32 pageCount = st.st_size >> PAGE_SHIFT; - swap->used = 0; - - swap->maps = (uint32 *)malloc((pageCount + 7) / 8); - if (swap->maps == NULL) { + swap->bmp = radix_bitmap_create(pageCount); + if (swap->bmp == NULL) { free(swap); close(fd); return B_NO_MEMORY; } - memset(swap->maps, 0, (pageCount + 7) / 8); - swap->hint = 0; // set slot index and add this file to swap file list mutex_lock(&sSwapFileListLock); @@ -1160,7 +1095,7 @@ swap_file_delete(char *path) // if this file is currently used, we can't delete // TODO: mark this swap file deleting, and remove it after releasing // all the swap space - if (swapFile->used > 0) + if (swapFile->bmp->free_slots < swapFile->last_slot - swapFile->first_slot) return B_ERROR; sSwapFileList.Remove(swapFile); @@ -1172,7 +1107,7 @@ swap_file_delete(char *path) mutex_unlock(&sAvailSwapSpaceLock); close(swapFile->fd); - free(swapFile->maps); + radix_bitmap_destroy(swapFile->bmp); free(swapFile); return B_OK;