app_server: allow drawing mode changes in opaque layers

* For better performance, we allow doing drawing mode changes
  (and thus, B_OP_COPY) again when inside an opaque layer which
  has only other opaque layers below it in the layer stack.

  As soon as the first non-opaque layer turns up in the stack, the
  drawing mode will be locked to alpha composite mode, until this
  layer stack is ended entirely.

  This allows using B_OP_COPY in many cases as used by WebKit.

* In the long term it would be nice to get rid of the drawing-mode
  lock altogether, however that would need some larger refactoring
  work in Painter (i.e. remove the offsetting from renderer_region
  again and instead implement an "exit-level transform" (support
  for offsets is enough) in Painter which is applied after all other
  transforms).

src/servers/app/DrawState.cpp

@@ -47,6 +47,7 @@ DrawState::DrawState()
 	fDrawingMode(B_OP_COPY),
 	fAlphaSrcMode(B_PIXEL_ALPHA),
 	fAlphaFncMode(B_ALPHA_OVERLAY),
+	fDrawingModeLocked(false),
 
 	fPenLocation(0.0f, 0.0f),
 	fPenSize(1.0f),
@@ -81,6 +82,7 @@ DrawState::DrawState(const DrawState& other)
 	fDrawingMode(other.fDrawingMode),
 	fAlphaSrcMode(other.fAlphaSrcMode),
 	fAlphaFncMode(other.fAlphaFncMode),
+	fDrawingModeLocked(other.fDrawingModeLocked),
 
 	fPenLocation(other.fPenLocation),
 	fPenSize(other.fPenSize),
@@ -527,18 +529,29 @@ DrawState::SetPattern(const Pattern& pattern)
 void
 DrawState::SetDrawingMode(drawing_mode mode)
 {
-	fDrawingMode = mode;
+	if (!fDrawingModeLocked)
+		fDrawingMode = mode;
 }
 
 
 void
 DrawState::SetBlendingMode(source_alpha srcMode, alpha_function fncMode)
 {
-	fAlphaSrcMode = srcMode;
+	if (!fDrawingModeLocked) {
-	fAlphaFncMode = fncMode;
+		fAlphaSrcMode = srcMode;
+		fAlphaFncMode = fncMode;
+	}
 }
 
 
+void
+DrawState::SetDrawingModeLocked(bool locked)
+{
+	fDrawingModeLocked = locked;
+}
+
+
+
 void
 DrawState::SetPenLocation(BPoint location)
 {

src/servers/app/DrawState.h

@@ -111,6 +111,8 @@ public:
 		alpha_function		AlphaFncMode() const
 								{ return fAlphaFncMode; }
 
+		void				SetDrawingModeLocked(bool locked);
+
 							// pen
 		void				SetPenLocation(BPoint location);
 		BPoint				PenLocation() const;
@@ -173,6 +175,7 @@ protected:
 		drawing_mode		fDrawingMode;
 		source_alpha		fAlphaSrcMode;
 		alpha_function		fAlphaFncMode;
+		bool				fDrawingModeLocked;
 
 		BPoint				fPenLocation;
 		float				fPenSize;

src/servers/app/ServerWindow.cpp

@@ -2197,6 +2197,13 @@ fDesktop->LockSingleWindow();
 			Layer* layer = new(std::nothrow) Layer(opacity);
 			if (layer == NULL)
 				break;
+
+			if (opacity != 255) {
+				fCurrentView->CurrentState()->SetDrawingMode(B_OP_ALPHA);
+				fCurrentView->CurrentState()->SetBlendingMode(B_PIXEL_ALPHA, B_ALPHA_COMPOSITE);
+				fCurrentView->CurrentState()->SetDrawingModeLocked(true);
+			}
+
 			fCurrentView->SetPicture(layer);
 			break;
 		}
@@ -2945,6 +2952,7 @@ ServerWindow::_DispatchViewDrawingMessage(int32 code,
 				Title()));
 			fCurrentView->BlendAllLayers();
 			fCurrentView->SetPicture(NULL);
+			fCurrentView->CurrentState()->SetDrawingModeLocked(false);
 			break;
 		}
 
@@ -3009,11 +3017,6 @@ ServerWindow::_DispatchPictureMessage(int32 code, BPrivate::LinkReceiver& link)
 			int8 drawingMode;
 			link.Read<int8>(&drawingMode);
 
-			if (dynamic_cast<Layer*>(picture) != NULL) {
-				// drawing mode changes not allowed in layers
-				break;
-			}
-
 			picture->WriteSetDrawingMode((drawing_mode)drawingMode);
 
 			fCurrentView->CurrentState()->SetDrawingMode(
@@ -3461,6 +3464,12 @@ ServerWindow::_DispatchPictureMessage(int32 code, BPrivate::LinkReceiver& link)
 			if (nextLayer == NULL)
 				break;
 
+			if (opacity != 255) {
+				fCurrentView->CurrentState()->SetDrawingMode(B_OP_ALPHA);
+				fCurrentView->CurrentState()->SetBlendingMode(B_PIXEL_ALPHA, B_ALPHA_COMPOSITE);
+				fCurrentView->CurrentState()->SetDrawingModeLocked(true);
+			}
+
 			nextLayer->PushLayer(layer);
 			fCurrentView->SetPicture(nextLayer);
 			break;