Fixed some incorrect behavior pointed out in #2990:

* The pthread_mutex_*lock() family should return EDEADLK when re-locking
  an error-checked mutex.
* pthread_mutex_trylock() is supposed to return EBUSY, not
  B_WOULD_BLOCK.
* pthread_mutex_unlock() should return EPERM when the caller is not the
  owner. It used to print a message and try to unlock anyway.


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@28354 a95241bf-73f2-0310-859d-f6bbb57e9c96

src/system/libroot/posix/pthread/pthread_mutex.c

@@ -85,7 +85,7 @@ mutex_lock(pthread_mutex_t *mutex, bigtime_t timeout)
 	if (MUTEX_TYPE(mutex) == PTHREAD_MUTEX_ERRORCHECK
 		&& mutex->owner == thisThread) {
 		// we detect this kind of deadlock and return an error
-		return B_BUSY;
+		return EDEADLK;
 	}
 
 	if (MUTEX_TYPE(mutex) == PTHREAD_MUTEX_RECURSIVE
@@ -123,7 +123,8 @@ pthread_mutex_lock(pthread_mutex_t *mutex)
 int
 pthread_mutex_trylock(pthread_mutex_t *mutex)
 {
-	return mutex_lock(mutex, 0);
+	status_t status = mutex_lock(mutex, 0);
+	return status == B_WOULD_BLOCK ? EBUSY : status;
 }
 
 
@@ -156,11 +157,8 @@ pthread_mutex_unlock(pthread_mutex_t *mutex)
 	if (mutex == NULL)
 		return B_BAD_VALUE;
 
-	if (mutex->owner != find_thread(NULL)) {
-		// this is a bug in the calling application!
-		// ToDo: should we handle it in another way?
-		fprintf(stderr, "mutex unlocked from foreign thread!\n");
-	}
+	if (mutex->owner != find_thread(NULL))
+		return EPERM;
 
 	if (MUTEX_TYPE(mutex) == PTHREAD_MUTEX_RECURSIVE) {
 		if (mutex->owner_count-- > 1)