kernel/vm: Insertion into the Areas AVL tree can fail.

So we need to check that it didn't when creating areas.

Change-Id: I4342463113046b543722faa7a51ca269ed67e8bf
Reviewed-on: https://review.haiku-os.org/c/haiku/+/8137
Reviewed-by: waddlesplash <[email protected]>

headers/private/kernel/vm/VMArea.h

@@ -219,7 +219,7 @@ struct VMAreas {
 									{ return sTree.Find(id); }
 	static	VMArea*				Lookup(area_id id);
 	static	area_id				Find(const char* name);
-	static	void				Insert(VMArea* area);
+	static	status_t			Insert(VMArea* area);
 	static	void				Remove(VMArea* area);
 
 	static	VMAreasTree::Iterator GetIterator()

src/system/kernel/vm/VMArea.cpp

@@ -244,12 +244,13 @@ VMAreas::Find(const char* name)
 }
 
 
-/*static*/ void
+/*static*/ status_t
 VMAreas::Insert(VMArea* area)
 {
 	WriteLock();
-	sTree.Insert(area);
+	status_t status = sTree.Insert(area);
 	WriteUnlock();
+	return status;
 }
 
 

src/system/kernel/vm/vm.cpp

@@ -1219,7 +1219,9 @@ map_backing_store(VMAddressSpace* addressSpace, VMCache* cache, off_t offset,
 		cache->Unlock();
 
 	// insert the area in the global areas map
-	VMAreas::Insert(area);
+	status = VMAreas::Insert(area);
+	if (status != B_OK)
+		goto err3;
 
 	// grab a ref to the address space (the area holds this)
 	addressSpace->Get();
@@ -1230,6 +1232,10 @@ map_backing_store(VMAddressSpace* addressSpace, VMCache* cache, off_t offset,
 	*_area = area;
 	return B_OK;
 
+err3:
+	cache->Lock();
+	cache->RemoveArea(area);
+	area->cache = NULL;
 err2:
 	if (mapping == REGION_PRIVATE_MAP) {
 		// We created this cache, so we must delete it again. Note, that we