pthread & os/locks: Add some more assertions and error checks.

The first of these assertions in the pthread code is actually possible
to trigger under some specific circumstances, which is ticket #18436.
This makes that problem more obvious when it does happen.

Change-Id: I026ea6e4c569a7c20d82b70722f752d87e57c5a1
Reviewed-on: https://review.haiku-os.org/c/haiku/+/6536
Reviewed-by: waddlesplash <[email protected]>
Tested-by: Commit checker robot <[email protected]>
Reviewed-by: Adrien Destugues <[email protected]>

src/system/libroot/os/locks/mutex.cpp

@@ -95,4 +95,7 @@ __mutex_unlock(mutex *lock)
 			&& (oldValue & B_USER_MUTEX_DISABLED) == 0) {
 		_kern_mutex_unlock(&lock->lock, 0);
 	}
+
+	if ((oldValue & B_USER_MUTEX_LOCKED) == 0)
+		debugger("mutex was not actually locked!");
 }

src/system/libroot/posix/pthread/pthread_mutex.cpp

@@ -8,6 +8,7 @@
 #include <pthread.h>
 #include "pthread_private.h"
 
+#include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -91,6 +92,7 @@ __pthread_mutex_lock(pthread_mutex_t* mutex, uint32 flags, bigtime_t timeout)
 	}
 
 	// we have locked the mutex for the first time
+	assert(mutex->owner == -1);
 	mutex->owner = thisThread;
 	mutex->owner_count = 1;
 
@@ -176,6 +178,12 @@ pthread_mutex_unlock(pthread_mutex_t* mutex)
 	if ((oldValue & B_USER_MUTEX_WAITING) != 0)
 		_kern_mutex_unlock((int32*)&mutex->lock, 0);
 
+	if (MUTEX_TYPE(mutex) == PTHREAD_MUTEX_ERRORCHECK
+		|| MUTEX_TYPE(mutex) == PTHREAD_MUTEX_DEFAULT) {
+		if ((oldValue & B_USER_MUTEX_LOCKED) == 0)
+			return EPERM;
+	}
+
 	return 0;
 }