Enforce team and thread limits

Also fixes incorrect team accounting in case of error when creating
a team. The previously incremented sUsedTeams wasn't decremented again.

src/system/kernel/team.cpp

@@ -1707,7 +1707,9 @@ load_image_internal(char**& _flatArgs, size_t flatArgsSize, int32 argCount,
  	InterruptsSpinLocker teamsLocker(sTeamHashLock);
 
 	sTeamHash.Insert(team);
-	sUsedTeams++;
+	bool teamLimitReached = sUsedTeams >= sMaxTeams;
+	if (!teamLimitReached)
+		sUsedTeams++;
 
 	teamsLocker.Unlock();
 
@@ -1727,6 +1729,11 @@ load_image_internal(char**& _flatArgs, size_t flatArgsSize, int32 argCount,
 	// check the executable's set-user/group-id permission
 	update_set_id_user_and_group(team, path);
 
+	if (teamLimitReached) {
+		status = B_NO_MORE_TEAMS;
+		goto err1;
+	}
+
 	status = create_team_arg(&teamArgs, path, flatArgs, flatArgsSize, argCount,
 		envCount, (mode_t)-1, errorPort, errorToken);
 	if (status != B_OK)
@@ -1834,6 +1841,8 @@ err1:
 
 	teamsLocker.Lock();
 	sTeamHash.Remove(team);
+	if (!teamLimitReached)
+		sUsedTeams--;
 	teamsLocker.Unlock();
 
 	sNotificationService.Notify(TEAM_REMOVED, team);
@@ -2038,7 +2047,9 @@ fork_team(void)
 	InterruptsSpinLocker teamsLocker(sTeamHashLock);
 
 	sTeamHash.Insert(team);
-	sUsedTeams++;
+	bool teamLimitReached = sUsedTeams >= sMaxTeams;
+	if (!teamLimitReached)
+		sUsedTeams++;
 
 	teamsLocker.Unlock();
 
@@ -2055,6 +2066,11 @@ fork_team(void)
 	team->debug_info.flags |= atomic_get(&parentTeam->debug_info.flags)
 		& B_TEAM_DEBUG_INHERITED_FLAGS;
 
+	if (teamLimitReached) {
+		status = B_NO_MORE_TEAMS;
+		goto err1;
+	}
+
 	forkArgs = (arch_fork_arg*)malloc(sizeof(arch_fork_arg));
 	if (forkArgs == NULL) {
 		status = B_NO_MEMORY;
@@ -2185,6 +2201,8 @@ err1:
 
 	teamsLocker.Lock();
 	sTeamHash.Remove(team);
+	if (!teamLimitReached)
+		sUsedTeams--;
 	teamsLocker.Unlock();
 
 	sNotificationService.Notify(TEAM_REMOVED, team);

src/system/kernel/thread.cpp

@@ -1015,19 +1015,38 @@ thread_create_thread(const ThreadCreationAttributes& attributes, bool kernel)
 	}
 
 	// We're going to make the thread live, now. The thread itself will take
-	// over a reference to its Thread object. We acquire another reference for
+	// over a reference to its Thread object. We'll acquire another reference
-	// our own use (and threadReference remains armed).
+	// for our own use (and threadReference remains armed).
-	thread->AcquireReference();
 
 	ThreadLocker threadLocker(thread);
 	InterruptsSpinLocker schedulerLocker(gSchedulerLock);
 	SpinLocker threadHashLocker(sThreadHashLock);
 
+	// check the thread limit
+	if (sUsedThreads >= sMaxThreads) {
+		// Clean up the user_thread structure. It's a bit unfortunate that the
+		// Thread destructor cannot do that, so we have to do that explicitly.
+		threadHashLocker.Unlock();
+		schedulerLocker.Unlock();
+
+		user_thread* userThread = thread->user_thread;
+		thread->user_thread = NULL;
+
+		threadLocker.Unlock();
+
+		if (userThread != NULL)
+			team_free_user_thread(team, userThread);
+
+		return B_NO_MORE_THREADS;
+	}
+
 	// make thread visible in global hash/list
 	thread->visible = true;
 	sUsedThreads++;
 	scheduler_on_thread_init(thread);
 
+	thread->AcquireReference();
+
 	// Debug the new thread, if the parent thread required that (see above),
 	// or the respective global team debug flag is set. But only, if a
 	// debugger is installed for the team.