Merged branch haiku/branches/developer/bonefish/vm into trunk. This

introduces the following relevant changes:
* VMCache:
  - Renamed vm_cache to VMCache, merged it with vm_store and made it a
    C++ class with virtual methods (replacing the store operations).
    Turned the different store implementations into subclasses.
  - Introduced MergeStore() callback, changed semantics of Commit().
  - Changed locking and referencing semantics. A reference can only be
    acquired/released with the cache locked. An unreferenced cache is
    deleted and a mergeable cache merged when it is unlocked. This
    removes the "busy" state of a cache and simplifies the page fault
    code.
* Added VMAnonymousCache, which will implement swap support (work by
  Zhao Shuai). It is not integrated and used yet, though.
* Enabled the mutex/recursive lock holder asserts.
* Fixed DoublyLinkedList::Swap().
* Generalized the low memory handler to a low resource handler. And made
  semaphores and reserved memory handled resources. Made
  vm_try_resource_memory() optionally wait (with timeout), and used that
  feature to reserve memory for areas.
...


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@26572 a95241bf-73f2-0310-859d-f6bbb57e9c96

headers/private/kernel/condition_variable.h

@@ -61,6 +61,10 @@ public:
 
 			void				Add(ConditionVariableEntry* entry);
 
+			status_t			Wait(uint32 flags = 0, bigtime_t timeout = 0);
+									// all-in one, i.e. doesn't need a
+									// ConditionVariableEntry
+
 			const void*			Object() const	{ return fObject; }
 
 	static	void				ListAll();

headers/private/kernel/file_cache.h

@@ -49,8 +49,6 @@ extern status_t file_map_init(void);
 extern status_t file_cache_init_post_boot_device(void);
 extern status_t file_cache_init(void);
 
-extern vm_store *vm_create_vnode_store(struct vnode *vnode);
-
 #ifdef __cplusplus
 }
 #endif

headers/private/kernel/heap.h

@@ -44,4 +44,21 @@ status_t heap_init_post_thread();
 }
 #endif
 
+
+#ifdef __cplusplus
+
+#include <new>
+
+static const struct nogrow_t {
+} nogrow = {};
+
+inline void*
+operator new(size_t size, const nogrow_t& nogrow)
+{
+	return malloc_nogrow(size);
+}
+
+#endif	/* __cplusplus */
+
+
 #endif	/* _KERNEL_MEMHEAP_H */

headers/private/kernel/lock.h

@@ -53,13 +53,13 @@ typedef struct rw_lock {
 #define RW_LOCK_FLAG_CLONE_NAME	0x1
 
 
-#if 0 && KDEBUG // XXX disable this for now, it causes problems when including thread.h here
-#	include <thread.h>
-#define ASSERT_LOCKED_RECURSIVE(r) { ASSERT(thread_get_current_thread_id() == (r)->holder); }
-#define ASSERT_LOCKED_MUTEX(m) { ASSERT(thread_get_current_thread_id() == (m)->holder); }
+#if KDEBUG
+#	define ASSERT_LOCKED_RECURSIVE(r) \
+		{ ASSERT(find_thread(NULL) == (r)->lock.holder); }
+#	define ASSERT_LOCKED_MUTEX(m) { ASSERT(find_thread(NULL) == (m)->holder); }
 #else
-#define ASSERT_LOCKED_RECURSIVE(r)
-#define ASSERT_LOCKED_MUTEX(m)
+#	define ASSERT_LOCKED_RECURSIVE(r)	do {} while (false)
+#	define ASSERT_LOCKED_MUTEX(m)		do {} while (false)
 #endif
 
 
@@ -102,10 +102,15 @@ extern void mutex_init(mutex* lock, const char* name);
 	// name is *not* cloned nor freed in mutex_destroy()
 extern void mutex_init_etc(mutex* lock, const char* name, uint32 flags);
 extern void mutex_destroy(mutex* lock);
+extern status_t mutex_switch_lock(mutex* from, mutex* to);
+	// Unlocks "from" and locks "to" such that unlocking and starting to wait
+	// for the lock is atomically. I.e. if "from" guards the object "to" belongs
+	// to, the operation is safe as long as "from" is held while destroying
+	// "to".
 
 // implementation private:
 extern status_t _mutex_lock(mutex* lock, bool threadsLocked);
-extern void _mutex_unlock(mutex* lock);
+extern void _mutex_unlock(mutex* lock, bool threadsLocked);
 extern status_t _mutex_trylock(mutex* lock);
 
 
@@ -151,12 +156,10 @@ mutex_trylock(mutex* lock)
 static inline void
 mutex_unlock(mutex* lock)
 {
-#ifdef KDEBUG
-	_mutex_unlock(lock);
-#else
+#if !defined(KDEBUG)
 	if (atomic_add(&lock->count, 1) < -1)
-		_mutex_unlock(lock);
 #endif
+		_mutex_unlock(lock, false);
 }
 
 

headers/private/kernel/low_resource_manager.h

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2008, Ingo Weinhold, [email protected].
+ * Copyright 2005-2007, Axel Dörfler, [email protected]. All rights reserved.
+ * Distributed under the terms of the MIT License.
+ */
+#ifndef _KERNEL_LOW_RESOURCE_MANAGER_H
+#define _KERNEL_LOW_RESOURCE_MANAGER_H
+
+
+#include <SupportDefs.h>
+
+
+/* warning levels for low resource handlers */
+enum {
+	B_NO_LOW_RESOURCE = 0,
+	B_LOW_RESOURCE_NOTE,
+	B_LOW_RESOURCE_WARNING,
+	B_LOW_RESOURCE_CRITICAL,
+};
+
+enum {
+	B_KERNEL_RESOURCE_PAGES			= 0x01,	/* physical pages */
+	B_KERNEL_RESOURCE_MEMORY		= 0x02,	/* reservable memory */
+	B_KERNEL_RESOURCE_SEMAPHORES	= 0x04,	/* semaphores */
+
+	B_ALL_KERNEL_RESOURCES			= B_KERNEL_RESOURCE_PAGES
+										| B_KERNEL_RESOURCE_MEMORY
+										| B_KERNEL_RESOURCE_SEMAPHORES
+};
+
+typedef void (*low_resource_func)(void *data, uint32 resources, int32 level);
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+status_t low_resource_manager_init(void);
+status_t low_resource_manager_init_post_thread(void);
+int32 low_resource_state(uint32 resources);
+void low_resource(uint32 resource, uint64 requirements, uint32 flags,
+	uint32 timeout);
+
+// these calls might get public some day
+status_t register_low_resource_handler(low_resource_func function, void *data,
+			uint32 resources, int32 priority);
+status_t unregister_low_resource_handler(low_resource_func function,
+			void *data);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* _KERNEL_LOW_RESOURCE_MANAGER_H */

headers/private/kernel/util/DoublyLinkedList.h

@@ -484,34 +484,20 @@ void
 DOUBLY_LINKED_LIST_CLASS_NAME::Swap(Element *a, Element *b)
 {
 	if (a && b && a != b) {
-		Link *aLink = sGetLink(a);
-		Link *bLink = sGetLink(b);
-		Element *aPrev = aLink->previous;
-		Element *bPrev = bLink->previous;
-		Element *aNext = aLink->next;
-		Element *bNext = bLink->next;
-		// place a
-		if (bPrev)
-			sGetLink(bPrev)->next = a;
-		else
-			fFirst = a;
-		if (bNext)
-			sGetLink(bNext)->previous = a;
-		else
-			fLast = a;
-		aLink->previous = bPrev;
-		aLink->next = bNext;
-		// place b
-		if (aPrev)
-			sGetLink(aPrev)->next = b;
-		else
-			fFirst = b;
-		if (aNext)
-			sGetLink(aNext)->previous = b;
-		else
-			fLast = b;
-		bLink->previous = aPrev;
-		bLink->next = aNext;
+		Element *aNext = sGetLink(a)->next;
+		Element *bNext = sGetLink(b)->next;
+		if (a == bNext) {
+			Remove(a);
+			Insert(b, a);
+		} else if (b == aNext) {
+			Remove(b);
+			Insert(a, b);
+		} else {
+			Remove(a);
+			Remove(b);
+			Insert(aNext, b);
+			Insert(bNext, a);
+		}
 	}
 }
 

headers/private/kernel/vfs.h

@@ -37,7 +37,7 @@ struct net_stat;
 struct pollfd;
 struct selectsync;
 struct select_info;
-struct vm_cache;
+struct VMCache;
 struct vnode;
 
 
@@ -97,7 +97,7 @@ status_t vfs_read_pages(struct vnode *vnode, void *cookie, off_t pos,
 			const iovec *vecs, size_t count, size_t *_numBytes, bool fsReenter);
 status_t vfs_write_pages(struct vnode *vnode, void *cookie, off_t pos,
 			const iovec *vecs, size_t count, size_t *_numBytes, bool fsReenter);
-status_t vfs_get_vnode_cache(struct vnode *vnode, struct vm_cache **_cache,
+status_t vfs_get_vnode_cache(struct vnode *vnode, struct VMCache **_cache,
 			bool allocate);
 status_t vfs_get_file_map(struct vnode *vnode, off_t offset, size_t size,
 			struct file_io_vec *vecs, size_t *_count);

headers/private/kernel/vm.h

@@ -17,7 +17,7 @@
 struct kernel_args;
 struct team;
 struct vm_page;
-struct vm_cache;
+struct VMCache;
 struct vm_area;
 struct vm_address_space;
 struct vnode;
@@ -60,8 +60,8 @@ area_id vm_map_physical_memory(team_id team, const char *name, void **address,
 area_id vm_map_file(team_id aid, const char *name, void **address,
 			uint32 addressSpec, addr_t size, uint32 protection, uint32 mapping,
 			int fd, off_t offset);
-struct vm_cache *vm_area_get_locked_cache(struct vm_area *area);
-void vm_area_put_locked_cache(struct vm_cache *cache);
+struct VMCache *vm_area_get_locked_cache(struct vm_area *area);
+void vm_area_put_locked_cache(struct VMCache *cache);
 area_id vm_create_null_area(team_id team, const char *name, void **address,
 			uint32 addressSpec, addr_t size);
 area_id vm_copy_area(team_id team, const char *name, void **_address,
@@ -70,7 +70,7 @@ area_id vm_clone_area(team_id team, const char *name, void **address,
 			uint32 addressSpec, uint32 protection, uint32 mapping,
 			area_id sourceArea, bool kernel);
 status_t vm_delete_area(team_id teamID, area_id areaID, bool kernel);
-status_t vm_create_vnode_cache(struct vnode *vnode, struct vm_cache **_cache);
+status_t vm_create_vnode_cache(struct vnode *vnode, struct VMCache **_cache);
 struct vm_area *vm_area_lookup(struct vm_address_space *addressSpace,
 			addr_t address);
 status_t vm_set_area_memory_type(area_id id, addr_t physicalBase, uint32 type);
@@ -87,6 +87,7 @@ status_t vm_get_physical_page(addr_t paddr, addr_t *vaddr, uint32 flags);
 status_t vm_put_physical_page(addr_t vaddr);
 
 off_t vm_available_memory(void);
+off_t vm_available_not_needed_memory(void);
 
 // user syscalls
 area_id _user_create_area(const char *name, void **address, uint32 addressSpec,

headers/private/kernel/vm_cache.h

@@ -15,31 +15,14 @@
 
 struct kernel_args;
 
-//typedef struct vm_store vm_store;
-
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 status_t vm_cache_init(struct kernel_args *args);
-struct vm_cache *vm_cache_create(struct vm_store *store);
-void vm_cache_acquire_ref(struct vm_cache *cache);
-void vm_cache_release_ref(struct vm_cache *cache);
-struct vm_cache *vm_cache_acquire_page_cache_ref(struct vm_page *page);
-struct vm_page *vm_cache_lookup_page(struct vm_cache *cache, off_t page);
-void vm_cache_insert_page(struct vm_cache *cache, struct vm_page *page,
-	off_t offset);
-void vm_cache_remove_page(struct vm_cache *cache, struct vm_page *page);
-void vm_cache_remove_consumer(struct vm_cache *cache, struct vm_cache *consumer);
-void vm_cache_add_consumer_locked(struct vm_cache *cache,
-	struct vm_cache *consumer);
-status_t vm_cache_write_modified(struct vm_cache *cache, bool fsReenter);
-status_t vm_cache_set_minimal_commitment_locked(struct vm_cache *cache,
-	off_t commitment);
-status_t vm_cache_resize(struct vm_cache *cache, off_t newSize);
-status_t vm_cache_insert_area_locked(struct vm_cache *cache, vm_area *area);
-status_t vm_cache_remove_area(struct vm_cache *cache, struct vm_area *area);
+struct VMCache *vm_cache_acquire_locked_page_cache(struct vm_page *page,
+	bool dontWait);
 
 #ifdef __cplusplus
 }

headers/private/kernel/vm_low_memory.h

@@ -1,40 +0,0 @@
-/*
- * Copyright 2005-2007, Axel Dörfler, [email protected]. All rights reserved.
- * Distributed under the terms of the MIT License.
- */
-#ifndef _KERNEL_VM_LOW_MEMORY_H
-#define _KERNEL_VM_LOW_MEMORY_H
-
-
-#include <SupportDefs.h>
-
-
-/* warning levels for low memory handlers */
-enum {
-	B_NO_LOW_MEMORY = 0,
-	B_LOW_MEMORY_NOTE,
-	B_LOW_MEMORY_WARNING,
-	B_LOW_MEMORY_CRITICAL,
-};
-
-typedef void (*low_memory_func)(void *data, int32 level);
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-status_t vm_low_memory_init(void);
-status_t vm_low_memory_init_post_thread(void);
-int32 vm_low_memory_state(void);
-void vm_low_memory(size_t requirements);
-
-// these calls might get public some day
-status_t register_low_memory_handler(low_memory_func function, void *data,
-			int32 priority);
-status_t unregister_low_memory_handler(low_memory_func function, void *data);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif	/* _KERNEL_VM_LOW_MEMORY_H */

headers/private/kernel/vm_page.h

@@ -26,7 +26,7 @@ status_t vm_page_init_post_thread(struct kernel_args *args);
 
 status_t vm_mark_page_inuse(addr_t page);
 status_t vm_mark_page_range_inuse(addr_t startPage, addr_t length);
-void vm_page_free(struct vm_cache *cache, struct vm_page *page);
+void vm_page_free(struct VMCache *cache, struct vm_page *page);
 status_t vm_page_set_state(struct vm_page *page, int state);
 void vm_page_requeue(struct vm_page *page, bool tail);
 
@@ -35,11 +35,11 @@ size_t vm_page_num_pages(void);
 size_t vm_page_num_free_pages(void);
 size_t vm_page_num_available_pages(void);
 
-status_t vm_page_write_modified_page_range(struct vm_cache *cache,
+status_t vm_page_write_modified_page_range(struct VMCache *cache,
 	uint32 firstPage, uint32 endPage, bool fsReenter);
-status_t vm_page_write_modified_pages(struct vm_cache *cache, bool fsReenter);
+status_t vm_page_write_modified_pages(struct VMCache *cache, bool fsReenter);
 void vm_page_schedule_write_page(struct vm_page *page);
-void vm_page_schedule_write_page_range(struct vm_cache *cache,
+void vm_page_schedule_write_page_range(struct VMCache *cache,
 	uint32 firstPage, uint32 endPage);
 
 void vm_page_unreserve_pages(uint32 count);

headers/private/kernel/vm_priv.h

@@ -33,7 +33,7 @@ extern "C" {
 status_t vm_page_fault(addr_t address, addr_t faultAddress, bool isWrite,
 	bool isUser, addr_t *newip);
 void vm_unreserve_memory(size_t bytes);
-status_t vm_try_reserve_memory(size_t bytes);
+status_t vm_try_reserve_memory(size_t bytes, bigtime_t timeout);
 void vm_schedule_page_scanner(uint32 target);
 status_t vm_daemon_init(void);
 

headers/private/kernel/vm_types.h

@@ -13,6 +13,7 @@
 #include <arch/vm_translation_map.h>
 #include <condition_variable.h>
 #include <kernel.h>
+#include <lock.h>
 #include <util/DoublyLinkedQueue.h>
 
 #include <sys/uio.h>
@@ -34,6 +35,7 @@
 #include <util/SplayTree.h>
 
 struct vm_page_mapping;
+struct VMCache;
 typedef DoublyLinkedListLink<vm_page_mapping> vm_page_mapping_link;
 
 typedef struct vm_page_mapping {
@@ -80,7 +82,7 @@ struct vm_page {
 
 	addr_t				physical_page_number;
 
-	struct vm_cache		*cache;
+	VMCache				*cache;
 	page_num_t			cache_offset;
 							// in page size units
 
@@ -165,36 +167,121 @@ struct VMCachePagesTreeDefinition {
 
 typedef IteratableSplayTree<VMCachePagesTreeDefinition> VMCachePagesTree;
 
-struct vm_cache {
-	mutex				lock;
+struct VMCache {
+public:
+	VMCache();
+	virtual ~VMCache();
+
+	status_t Init(uint32 cacheType);
+
+	virtual	void		Delete();
+
+			bool		Lock()
+							{ return mutex_lock(&fLock) == B_OK; }
+			bool		TryLock()
+							{ return mutex_trylock(&fLock) == B_OK; }
+			bool		SwitchLock(mutex* from)
+							{ return mutex_switch_lock(from, &fLock) == B_OK; }
+			void		Unlock();
+			void		AssertLocked()
+							{ ASSERT_LOCKED_MUTEX(&fLock); }
+
+			void		AcquireRefLocked();
+			void		AcquireRef();
+			void		ReleaseRefLocked();
+			void		ReleaseRef();
+			void		ReleaseRefAndUnlock()
+							{ ReleaseRefLocked(); Unlock(); }
+
+			vm_page*	LookupPage(off_t offset);
+			void		InsertPage(vm_page* page, off_t offset);
+			void		RemovePage(vm_page* page);
+
+			void		AddConsumer(VMCache* consumer);
+
+			status_t	InsertAreaLocked(vm_area* area);
+			status_t	RemoveArea(vm_area* area);
+
+			status_t	WriteModified(bool fsReenter);
+			status_t	SetMinimalCommitment(off_t commitment);
+			status_t	Resize(off_t newSize);
+
+			// for debugging only
+			mutex*		GetLock()
+							{ return &fLock; }
+			int32		RefCount() const
+							{ return fRefCount; }
+
+	// backing store operations
+	virtual	status_t	Commit(off_t size);
+	virtual	bool		HasPage(off_t offset);
+
+	virtual	status_t	Read(off_t offset, const iovec *vecs, size_t count,
+							size_t *_numBytes, bool fsReenter);
+	virtual	status_t	Write(off_t offset, const iovec *vecs, size_t count,
+							size_t *_numBytes, bool fsReenter);
+
+	virtual	status_t	Fault(struct vm_address_space *aspace, off_t offset);
+
+	virtual	void		MergeStore(VMCache* source);
+
+	virtual	status_t	AcquireUnreferencedStoreRef();
+	virtual	void		AcquireStoreRef();
+	virtual	void		ReleaseStoreRef();
+
+private:
+	inline	bool		_IsMergeable() const;
+
+			void		_MergeWithOnlyConsumer();
+			void		_RemoveConsumer(VMCache* consumer);
+
+
+public:
 	struct vm_area		*areas;
-	vint32				ref_count;
 	struct list_link	consumer_link;
 	struct list			consumers;
 		// list of caches that use this cache as a source
 	VMCachePagesTree	pages;
-	struct vm_cache		*source;
-	struct vm_store		*store;
+	VMCache				*source;
 	off_t				virtual_base;
-	off_t				virtual_size;
-		// the size is absolute, and independent from virtual_base
+	off_t				virtual_end;
+	off_t				committed_size;
+		// TODO: Remove!
 	uint32				page_count;
 	uint32				temporary : 1;
 	uint32				scan_skip : 1;
-	uint32				busy : 1;
-	uint32				type : 5;
+	uint32				type : 6;
 
 #if DEBUG_CACHE_LIST
-	struct vm_cache*	debug_previous;
-	struct vm_cache*	debug_next;
+	struct VMCache*		debug_previous;
+	struct VMCache*		debug_next;
 #endif
+
+private:
+	int32				fRefCount;
+	mutex				fLock;
 };
 
+typedef VMCache vm_cache;
+	// TODO: Remove!
+
 
 #if DEBUG_CACHE_LIST
 extern vm_cache* gDebugCacheList;
 #endif
 
+
+class VMCacheFactory {
+public:
+	static	status_t	CreateAnonymousCache(VMCache*& cache,
+							bool canOvercommit, int32 numPrecommittedPages,
+							int32 numGuardPages);
+	static	status_t	CreateVnodeCache(VMCache*& cache, struct vnode* vnode);
+	static	status_t	CreateDeviceCache(VMCache*& cache, addr_t baseAddress);
+	static	status_t	CreateNullCache(VMCache*& cache);
+};
+
+
 struct vm_area {
 	char				*name;
 	area_id				id;
@@ -204,7 +291,7 @@ struct vm_area {
 	uint16				wiring;
 	uint16				memory_type;
 
-	struct vm_cache		*cache;
+	VMCache				*cache;
 	vint32				no_cache_change;
 	off_t				cache_offset;
 	uint32				cache_type;
@@ -239,25 +326,4 @@ struct vm_address_space {
 	struct vm_address_space *hash_next;
 };
 
-struct vm_store {
-	struct vm_store_ops	*ops;
-	struct vm_cache		*cache;
-	off_t				committed_size;
-};
-
-typedef struct vm_store_ops {
-	void (*destroy)(struct vm_store *backingStore);
-	status_t (*commit)(struct vm_store *backingStore, off_t size);
-	bool (*has_page)(struct vm_store *backingStore, off_t offset);
-	status_t (*read)(struct vm_store *backingStore, off_t offset,
-		const iovec *vecs, size_t count, size_t *_numBytes, bool fsReenter);
-	status_t (*write)(struct vm_store *backingStore, off_t offset,
-		const iovec *vecs, size_t count, size_t *_numBytes, bool fsReenter);
-	status_t (*fault)(struct vm_store *backingStore,
-		struct vm_address_space *aspace, off_t offset);
-	status_t (*acquire_unreferenced_ref)(struct vm_store *backingStore);
-	void (*acquire_ref)(struct vm_store *backingStore);
-	void (*release_ref)(struct vm_store *backingStore);
-} vm_store_ops;
-
 #endif	/* _KERNEL_VM_TYPES_H */