diff --git a/src/add-ons/input_server/devices/keyboard/KeyboardInputDevice.cpp b/src/add-ons/input_server/devices/keyboard/KeyboardInputDevice.cpp
index 3642d184ba..278719b4aa 100644
--- a/src/add-ons/input_server/devices/keyboard/KeyboardInputDevice.cpp
+++ b/src/add-ons/input_server/devices/keyboard/KeyboardInputDevice.cpp
@@ -677,7 +677,7 @@ KeyboardInputDevice::DeviceWatcher(void *arg)
 					|| (is_keydown && !(dev->modifiers & modifiers)))
 					dev->modifiers |= modifiers;
 				else
-					dev->modifiers ^= modifiers;
+					dev->modifiers &= ~modifiers;
 					
 				msg->AddInt32("modifiers", dev->modifiers);
 				msg->AddData("states", B_UINT8_TYPE, states, 16);