vm_soft_fault(): Avoid deadlock waiting for wired ranges

* VMArea::AddWaiterIfWired(): Replace the ignoreRange argument by a
flags argument and introduce (currently only) flag
IGNORE_WRITE_WIRED_RANGES. If specified, ranges wired for writing
are ignored. Ignoring just a single specified range doesn't cut it
in vm_soft_fault(), and there aren't any other users of that feature.
* vm_soft_fault(): When having to unmap a page of a lower cache, this
page cannot be wired for writing. So we can safely ignore all
writed-wired ranges, instead of just our own. We even have to do that
in case there's another thread that concurrently tries to write-wire
the same page, since otherwise we'd deadlock waiting for each other.

headers/private/kernel/vm/VMArea.h

@@ -89,6 +89,14 @@ struct VMPageWiringInfo {
 
 
 struct VMArea {
+public:
+	enum {
+		// AddWaiterIfWired() flags
+		IGNORE_WRITE_WIRED_RANGES	= 0x01,	// ignore existing ranges that
+											// wire for writing
+	};
+
+public:
 	char*					name;
 	area_id					id;
 	uint32					protection;
@@ -130,8 +138,7 @@ public:
 
 			bool				AddWaiterIfWired(VMAreaUnwiredWaiter* waiter);
 			bool				AddWaiterIfWired(VMAreaUnwiredWaiter* waiter,
-									addr_t base, size_t size,
+									addr_t base, size_t size, uint32 flags = 0);
-									VMAreaWiredRange* ignoreRange = NULL);
 
 protected:
 								VMArea(VMAddressSpace* addressSpace,

src/system/kernel/vm/VMArea.cpp

@@ -185,19 +185,20 @@ VMArea::AddWaiterIfWired(VMAreaUnwiredWaiter* waiter)
 		that intersects with the given address range.
 	\param base The base of the address range to check.
 	\param size The size of the address range to check.
-	\param ignoreRange If given, this wired range of the area is not checked
+	\param flags
-		whether it intersects with the given address range. Useful when the
+		- \c IGNORE_WRITE_WIRED_RANGES: Ignore ranges wired for writing.
-		caller has added the range and only wants to check intersection with
-		other ranges.
 	\return \c true, if the waiter has been added, \c false otherwise.
 */
 bool
 VMArea::AddWaiterIfWired(VMAreaUnwiredWaiter* waiter, addr_t base, size_t size,
-	VMAreaWiredRange* ignoreRange)
+	uint32 flags)
 {
 	for (VMAreaWiredRangeList::Iterator it = fWiredRanges.GetIterator();
 			VMAreaWiredRange* range = it.Next();) {
-		if (range != ignoreRange && range->IntersectsWith(base, size)) {
+		if ((flags & IGNORE_WRITE_WIRED_RANGES) != 0 && range->writable)
+			continue;
+
+		if (range->IntersectsWith(base, size)) {
 			waiter->area = this;
 			waiter->base = base;
 			waiter->size = size;

src/system/kernel/vm/vm.cpp

@@ -4681,10 +4681,14 @@ vm_soft_fault(VMAddressSpace* addressSpace, addr_t originalAddress,
 
 		if (unmapPage) {
 			// If the page is wired, we can't unmap it. Wait until it is unwired
-			// again and restart.
+			// again and restart. Note that the page cannot be wired for
+			// writing, since it it isn't in the topmost cache. So we can safely
+			// ignore ranges wired for writing (our own and other concurrent
+			// wiring attempts in progress) and in fact have to do that to avoid
+			// a deadlock.
 			VMAreaUnwiredWaiter waiter;
 			if (area->AddWaiterIfWired(&waiter, address, B_PAGE_SIZE,
-					wiredRange)) {
+					VMArea::IGNORE_WRITE_WIRED_RANGES)) {
 				// unlock everything and wait
 				if (context.pageAllocated) {
 					// ... but since we allocated a page and inserted it into