diff --git a/src/preferences/keymap/KeyboardLayoutView.cpp b/src/preferences/keymap/KeyboardLayoutView.cpp
index 3b7676ab89..e463cc9114 100644
--- a/src/preferences/keymap/KeyboardLayoutView.cpp
+++ b/src/preferences/keymap/KeyboardLayoutView.cpp
@@ -126,6 +126,7 @@ KeyboardLayoutView::MouseDown(BPoint point)
 {
 	fClickPoint = point;
 	fDragKey = NULL;
+	fDropPoint.x = -1;
 
 	Key* key = _KeyAt(point);
 	if (key != NULL) {
@@ -165,14 +166,9 @@ KeyboardLayoutView::MouseMoved(BPoint point, uint32 transit,
 
 	if (dragMessage != NULL) {
 		_InvalidateKey(fDropTarget);
+		fDropPoint = point;
 
-		fDropTarget = _KeyAt(point);
-		if (fDropTarget != NULL) {
-			if (fDropTarget == fDragKey)
-				fDropTarget = NULL;
-			else
-				_InvalidateKey(fDropTarget);
-		}
+		_EvaluateDropTarget(point);
 	} else if (fDragKey == NULL && (fabs(point.x - fClickPoint.x) > 4
 		|| fabs(point.y - fClickPoint.y) > 4)) {
 		// start dragging
@@ -220,6 +216,7 @@ KeyboardLayoutView::MouseMoved(BPoint point, uint32 transit,
 
 		DragMessage(&drag, bitmap, B_OP_ALPHA, offset);
 		fDragKey = key;
+		fDragModifiers = fModifiers;
 
 		fKeyState[key->code / 8] &= ~(1 << (7 - (key->code & 7)));
 		_InvalidateKey(key);
@@ -252,11 +249,15 @@ KeyboardLayoutView::MessageReceived(BMessage* message)
 				size--;
 
 			int32 keyCode;
+			int32 buttons;
 			if (!message->IsSourceRemote()
+				&& message->FindInt32("buttons", &buttons) == B_OK
+				&& (buttons & B_SECONDARY_MOUSE_BUTTON) != 0
 				&& message->FindInt32("key", &keyCode) == B_OK) {
 				// switch keys if the dropped object came from us
 				Key* key = _KeyForCode(keyCode);
-				if (key == NULL || key == fDropTarget)
+				if (key == NULL
+					|| (key == fDropTarget && fDragModifiers == fModifiers))
 					return;
 
 				char* string;
@@ -266,7 +267,7 @@ KeyboardLayoutView::MessageReceived(BMessage* message)
 				if (string != NULL) {
 					fKeymap->SetKey(fDropTarget->code, fModifiers, fDeadKey,
 						(const char*)data, size);
-					fKeymap->SetKey(key->code, fModifiers, fDeadKey,
+					fKeymap->SetKey(key->code, fDragModifiers, fDeadKey,
 						string, numBytes);
 					delete[] string;
 				}
@@ -281,6 +282,7 @@ KeyboardLayoutView::MessageReceived(BMessage* message)
 
 		_InvalidateKey(fDropTarget);
 		fDropTarget = NULL;
+		fDropPoint.x = -1;
 	}
 
 	switch (message->what) {
@@ -290,8 +292,10 @@ KeyboardLayoutView::MessageReceived(BMessage* message)
 			break;
 
 		case B_MODIFIERS_CHANGED:
-			if (message->FindInt32("modifiers", &fModifiers) == B_OK)
+			if (message->FindInt32("modifiers", &fModifiers) == B_OK) {
+				_EvaluateDropTarget(fDropPoint);
 				Invalidate();
+			}
 			break;
 
 		default:
@@ -692,6 +696,19 @@ KeyboardLayoutView::_SetFontSize(BView* view, key_kind keyKind)
 }
 
 
+void
+KeyboardLayoutView::_EvaluateDropTarget(BPoint point)
+{
+	fDropTarget = _KeyAt(point);
+	if (fDropTarget != NULL) {
+		if (fDropTarget == fDragKey && fModifiers == fDragModifiers)
+			fDropTarget = NULL;
+		else
+			_InvalidateKey(fDropTarget);
+	}
+}
+
+
 void
 KeyboardLayoutView::_SendFakeKeyDown(const Key* key)
 {
diff --git a/src/preferences/keymap/KeyboardLayoutView.h b/src/preferences/keymap/KeyboardLayoutView.h
index 7912d3401a..87a8bb83b2 100644
--- a/src/preferences/keymap/KeyboardLayoutView.h
+++ b/src/preferences/keymap/KeyboardLayoutView.h
@@ -70,6 +70,7 @@ private:
 			Key*			_KeyAt(BPoint point);
 			BRect			_FrameFor(const Key* key);
 			void			_SetFontSize(BView* view, key_kind keyKind);
+			void			_EvaluateDropTarget(BPoint point);
 			void			_SendFakeKeyDown(const Key* key);
 
 			KeyboardLayout*	fLayout;
@@ -82,7 +83,9 @@ private:
 
 			BPoint			fClickPoint;
 			Key*			fDragKey;
+			int32			fDragModifiers;
 			Key*			fDropTarget;
+			BPoint			fDropPoint;
 
 			BFont			fFont;
 			BFont			fSpecialFont;