* Removed the VMArea::Wire() version that has to allocate a VMAreaWiredRange.

Since the requirement is that the area's top cache is locked, allocating
  memory isn't allowed.
* lock_memory_etc(): Create the VMAreaWiredRange object explicitly before
  locking the area's top cache.

Fixes #5680 (deadlocks when using the slab as malloc() backend).


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@36033 a95241bf-73f2-0310-859d-f6bbb57e9c96

headers/private/kernel/vm/VMArea.h

@@ -94,7 +94,6 @@ struct VMArea {
 			bool				IsWired(addr_t base, size_t size) const;
 
 			void				Wire(VMAreaWiredRange* range);
-			VMAreaWiredRange*	Wire(addr_t base, size_t size, bool writable);
 			void				Unwire(VMAreaWiredRange* range);
 			void				Unwire(addr_t base, size_t size, bool writable);
 

src/system/kernel/vm/VMArea.cpp

@@ -106,25 +106,6 @@ VMArea::Wire(VMAreaWiredRange* range)
 }
 
 
-/*!	Adds a wired range to this area.
-	The area's top cache must be locked.
-
-	\return The newly created wired area object. \c NULL when out of memory.
-*/
-VMAreaWiredRange*
-VMArea::Wire(addr_t base, size_t size, bool writable)
-{
-	VMAreaWiredRange* range = new(std::nothrow) VMAreaWiredRange(base, size,
-		writable, true);
-	if (range == NULL)
-		return NULL;
-
-	Wire(range);
-
-	return range;
-}
-
-
 /*!	Removes the given wired range from this area.
 	Must balance a previous Wire() call.
 	The area's top cache must be locked.

src/system/kernel/vm/vm.cpp

@@ -4792,17 +4792,23 @@ lock_memory_etc(team_id team, void* address, size_t numBytes, uint32 flags)
 		addr_t areaStart = nextAddress;
 		addr_t areaEnd = std::min(lockEndAddress, area->Base() + area->Size());
 
-		// Lock the area's top cache. This is a requirement for VMArea::Wire().
-		VMCacheChainLocker cacheChainLocker(vm_area_get_locked_cache(area));
-
-		// mark the area range wired
-		VMAreaWiredRange* range = area->Wire(areaStart, areaEnd - areaStart,
-			writable);
+		// allocate the wired range (do that before locking the cache to avoid
+		// deadlocks)
+		uint32 mallocFlags = isUser
+			? 0 : HEAP_DONT_WAIT_FOR_MEMORY | HEAP_DONT_LOCK_KERNEL_SPACE;
+		VMAreaWiredRange* range = new(malloc_flags(mallocFlags))
+			VMAreaWiredRange(areaStart, areaEnd - areaStart, writable, true);
 		if (range == NULL) {
 			error = B_NO_MEMORY;
 			break;
 		}
 
+		// Lock the area's top cache. This is a requirement for VMArea::Wire().
+		VMCacheChainLocker cacheChainLocker(vm_area_get_locked_cache(area));
+
+		// mark the area range wired
+		area->Wire(range);
+
 		// Depending on the area cache type and the wiring, we may not need to
 		// look at the individual pages.
 		if (area->cache_type == CACHE_TYPE_NULL