app_server/Interface Kit: add new clipping API

* Add new clipping API for rectangles (ClipToRect, ClipToInverseRect)
  and shapes (ClipToShape, ClipToInverseShape)

* Works with affine transforms and automatically switches from fast
  region-based clipping to alpha-mask based clipping as necessary.

* Always self-intersecting, i.e. no state push required to further
  narrow down the clipping region. All of the 4 new methods can be
  mixed in any order.

headers/os/interface/View.h

@@ -223,6 +223,11 @@ public:
 			void				ClipToInversePicture(BPicture* picture,
 									BPoint where = B_ORIGIN, bool sync = true);
 
+			void				ClipToRect(BRect rect);
+			void				ClipToInverseRect(BRect rect);
+			void				ClipToShape(BShape* shape);
+			void				ClipToInverseShape(BShape* shape);
+
 	virtual	void				SetDrawingMode(drawing_mode mode);
 			drawing_mode		DrawingMode() const;
 
@@ -651,6 +656,9 @@ private:
 			void				_ClipToPicture(BPicture* picture, BPoint where,
 									bool invert, bool sync);
 
+			void				_ClipToRect(BRect rect, bool inverse);
+			void				_ClipToShape(BShape* shape, bool inverse);
+
 			bool				_CheckOwnerLockAndSwitchCurrent() const;
 			bool				_CheckOwnerLock() const;
 			void				_CheckLockAndSwitchCurrent() const;

headers/private/app/ServerProtocol.h

@@ -347,6 +347,10 @@ enum {
 	AS_VIEW_SET_FILL_RULE,
 	AS_VIEW_GET_FILL_RULE,
 
+	// New clipping: cumulative, transformed
+	AS_VIEW_CLIP_TO_RECT,
+	AS_VIEW_CLIP_TO_SHAPE,
+
 	AS_LAST_CODE
 };
 

headers/private/interface/PictureDataWriter.h

@@ -96,6 +96,11 @@ public:
 									const int32& token);
 
 			status_t			WriteBlendLayer(Layer* layer);
+			status_t			WriteClipToRect(const BRect& rect,
+									bool inverse);
+			status_t			WriteClipToShape(int32 opCount,
+									const void* opList, int32 ptCount,
+									const void* ptList, bool inverse);
 
 protected:
 	// throw a status_t on error

headers/private/interface/PicturePlayer.h

@@ -85,6 +85,9 @@ struct picture_player_callbacks {
 		alpha_function alphaFunctionMode);
 	void (*set_transform)(void* userData, const BAffineTransform& transform);
 	void (*blend_layer)(void* userData, Layer* layer);
+	void (*clip_to_rect)(void* userData, const BRect& rect, bool inverse);
+	void (*clip_to_shape)(void* userData, int32 opCount, const uint32 opList[],
+		int32 ptCount, const BPoint ptList[], bool inverse);
 };
 
 

headers/private/interface/PictureProtocol.h

@@ -29,6 +29,8 @@ enum {
 	B_PIC_PUSH_STATE			= 0x0203,
 	B_PIC_POP_STATE				= 0x0204,
 	B_PIC_CLEAR_CLIPPING_RECTS	= 0x0205,
+	B_PIC_CLIP_TO_RECT			= 0x0206,
+	B_PIC_CLIP_TO_SHAPE			= 0x0207,
 
 	B_PIC_SET_ORIGIN			= 0x0300,
 	B_PIC_SET_PEN_LOCATION		= 0x0301,

src/kits/interface/PictureDataWriter.cpp

@@ -676,6 +676,42 @@ PictureDataWriter::WriteBlendLayer(Layer* layer)
 }
 
 
+status_t
+PictureDataWriter::WriteClipToRect(const BRect& rect, bool inverse)
+{
+	try {
+		BeginOp(B_PIC_CLIP_TO_RECT);
+		Write<bool>(inverse);
+		Write<BRect>(rect);
+		EndOp();
+	} catch (status_t& status) {
+		return status;
+	}
+
+	return B_OK;
+}
+
+
+status_t
+PictureDataWriter::WriteClipToShape(int32 opCount, const void* opList,
+	int32 ptCount, const void* ptList, bool inverse)
+{
+	try {
+		BeginOp(B_PIC_CLIP_TO_SHAPE);
+		Write<bool>(inverse);
+		Write<int32>(opCount);
+		Write<int32>(ptCount);
+		WriteData(opList, opCount * sizeof(uint32));
+		WriteData(ptList, ptCount * sizeof(BPoint));
+		EndOp();
+	} catch (status_t& status) {
+		return status;
+	}
+
+	return B_OK;
+}
+
+
 // private
 void
 PictureDataWriter::BeginOp(const int16& op)

src/kits/interface/PicturePlayer.cpp

@@ -1223,6 +1223,39 @@ PicturePlayer::_Play(const picture_player_callbacks& callbacks, void* userData,
 				break;
 			}
 
+			case B_PIC_CLIP_TO_RECT:
+			{
+				const bool* inverse;
+				const BRect* rect;
+
+				if (callbacks.clip_to_rect == NULL || !reader.Get(inverse)
+					|| !reader.Get(rect)) {
+					break;
+				}
+
+				callbacks.clip_to_rect(userData, *rect, *inverse);
+				break;
+			}
+
+			case B_PIC_CLIP_TO_SHAPE:
+			{
+				const bool* inverse;
+				const uint32* opCount;
+				const uint32* pointCount;
+				const uint32* opList;
+				const BPoint* pointList;
+				if (callbacks.clip_to_shape == NULL || !reader.Get(inverse)
+					|| !reader.Get(opCount) || !reader.Get(pointCount)
+					|| !reader.Get(opList, *opCount)
+					|| !reader.Get(pointList, *pointCount)) {
+					break;
+				}
+
+				callbacks.clip_to_shape(userData, *opCount, opList,
+					*pointCount, pointList, *inverse);
+				break;
+			}
+
 			default:
 				break;
 		}

src/kits/interface/View.cpp

@@ -2586,6 +2586,34 @@ BView::ConstrainClippingRegion(BRegion* region)
 }
 
 
+void
+BView::ClipToRect(BRect rect)
+{
+	_ClipToRect(rect, false);
+}
+
+
+void
+BView::ClipToInverseRect(BRect rect)
+{
+	_ClipToRect(rect, true);
+}
+
+
+void
+BView::ClipToShape(BShape* shape)
+{
+	_ClipToShape(shape, false);
+}
+
+
+void
+BView::ClipToInverseShape(BShape* shape)
+{
+	_ClipToShape(shape, true);
+}
+
+
 //	#pragma mark - Drawing Functions
 
 
@@ -5271,6 +5299,40 @@ BView::_ClipToPicture(BPicture* picture, BPoint where, bool invert, bool sync)
 }
 
 
+void
+BView::_ClipToRect(BRect rect, bool inverse)
+{
+	if (_CheckOwnerLockAndSwitchCurrent()) {
+		fOwner->fLink->StartMessage(AS_VIEW_CLIP_TO_RECT);
+		fOwner->fLink->Attach<bool>(inverse);
+		fOwner->fLink->Attach<BRect>(rect);
+		_FlushIfNotInTransaction();
+	}
+}
+
+
+void
+BView::_ClipToShape(BShape* shape, bool inverse)
+{
+	if (shape == NULL)
+		return;
+
+	shape_data* sd = (shape_data*)shape->fPrivateData;
+	if (sd->opCount == 0 || sd->ptCount == 0)
+		return;
+
+	if (_CheckOwnerLockAndSwitchCurrent()) {
+		fOwner->fLink->StartMessage(AS_VIEW_CLIP_TO_SHAPE);
+		fOwner->fLink->Attach<bool>(inverse);
+		fOwner->fLink->Attach<int32>(sd->opCount);
+		fOwner->fLink->Attach<int32>(sd->ptCount);
+		fOwner->fLink->Attach(sd->opList, sd->opCount * sizeof(uint32));
+		fOwner->fLink->Attach(sd->ptList, sd->ptCount * sizeof(BPoint));
+		_FlushIfNotInTransaction();
+	}
+}
+
+
 bool
 BView::_RemoveChildFromList(BView* child)
 {

src/servers/app/Canvas.cpp

@@ -149,6 +149,22 @@ Canvas::SetUserClipping(const BRegion* region)
 }
 
 
+bool
+Canvas::ClipToRect(BRect rect, bool inverse)
+{
+	bool needDrawStateUpdate = fDrawState->ClipToRect(rect, inverse);
+	RebuildClipping(false);
+	return needDrawStateUpdate;
+}
+
+
+void
+Canvas::ClipToShape(shape_data* shape, bool inverse)
+{
+	fDrawState->ClipToShape(shape, inverse);
+}
+
+
 void
 Canvas::SetAlphaMask(AlphaMask* mask)
 {
@@ -287,3 +303,13 @@ OffscreenCanvas::ResyncDrawState()
 {
 	fDrawingEngine->SetDrawState(fDrawState);
 }
+
+
+void
+OffscreenCanvas::UpdateCurrentDrawingRegion()
+{
+	if (fDrawState->HasClipping()) {
+		fDrawState->GetCombinedClippingRegion(&fCurrentDrawingRegion);
+		fDrawingEngine->ConstrainClippingRegion(&fCurrentDrawingRegion);
+	}
+}

src/servers/app/Canvas.h

@@ -29,6 +29,7 @@ class IntPoint;
 class IntRect;
 class Layer;
 class ServerPicture;
+class shape_data;
 
 
 class Canvas {
@@ -53,6 +54,9 @@ public:
 			void			SetUserClipping(const BRegion* region);
 				// region is expected in view coordinates
 
+			bool			ClipToRect(BRect rect, bool inverse);
+			void			ClipToShape(shape_data* shape, bool inverse);
+
 			void			SetAlphaMask(AlphaMask* mask);
 			AlphaMask*		GetAlphaMask() const;
 
@@ -90,6 +94,7 @@ public:
 
 	virtual void			RebuildClipping(bool deep) { /* TODO */ }
 	virtual void			ResyncDrawState();
+	virtual void			UpdateCurrentDrawingRegion();
 	virtual ServerPicture*	GetPicture(int32 token) const
 								{ /* TODO */ return NULL; }
 
@@ -99,6 +104,7 @@ protected:
 
 private:
 			DrawingEngine*	fDrawingEngine;
+			BRegion			fCurrentDrawingRegion;
 };
 
 

src/servers/app/DrawState.cpp

@@ -19,6 +19,7 @@
 #include <stdio.h>
 
 #include <Region.h>
+#include <ShapePrivate.h>
 
 #include "AlphaMask.h"
 #include "LinkReceiver.h"
@@ -449,6 +450,79 @@ DrawState::GetCombinedClippingRegion(BRegion* region) const
 }
 
 
+bool
+DrawState::ClipToRect(BRect rect, bool inverse)
+{
+	if (!rect.IsValid())
+		return false;
+
+	if (!fCombinedTransform.IsIdentity()) {
+		if (fCombinedTransform.IsDilation()) {
+			BPoint points[2] = { rect.LeftTop(), rect.RightBottom() };
+			fCombinedTransform.Apply(&points[0], 2);
+			rect.Set(points[0].x, points[0].y, points[1].x, points[1].y);
+		} else {
+			uint32 ops[] = {
+				OP_MOVETO | OP_LINETO | 3,
+				OP_CLOSE
+			};
+			BPoint points[4] = {
+				BPoint(rect.left,  rect.top),
+				BPoint(rect.right, rect.top),
+				BPoint(rect.right, rect.bottom),
+				BPoint(rect.left,  rect.bottom)
+			};
+			shape_data rectShape;
+			rectShape.opList = &ops[0];
+			rectShape.opCount = 2;
+			rectShape.opSize = sizeof(uint32) * 2;
+			rectShape.ptList = &points[0];
+			rectShape.ptCount = 4;
+			rectShape.ptSize = sizeof(BPoint) * 4;
+
+			ClipToShape(&rectShape, inverse);
+			return true;
+		}
+	}
+
+	if (inverse) {
+		if (fClippingRegion == NULL) {
+			fClippingRegion = new(nothrow) BRegion(BRect(
+				-(1 << 16), -(1 << 16), (1 << 16), (1 << 16)));
+				// TODO: we should have a definition for a rect (or region)
+				// with "infinite" area. For now, this region size should do...
+		}
+		fClippingRegion->Exclude(rect);
+	} else {
+		if (fClippingRegion == NULL)
+			fClippingRegion = new(nothrow) BRegion(rect);
+		else {
+			BRegion rectRegion(rect);
+			fClippingRegion->IntersectWith(&rectRegion);
+		}
+	}
+
+	return false;
+}
+
+
+void
+DrawState::ClipToShape(shape_data* shape, bool inverse)
+{
+	if (shape->ptCount == 0)
+		return;
+
+	if (!fCombinedTransform.IsIdentity())
+		fCombinedTransform.Apply(shape->ptList, shape->ptCount);
+
+	AlphaMask* const mask = new ShapeAlphaMask(GetAlphaMask(), *shape,
+		BPoint(0, 0), inverse);
+	SetAlphaMask(mask);
+	if (mask != NULL)
+		mask->ReleaseReference();
+}
+
+
 void
 DrawState::SetAlphaMask(AlphaMask* mask)
 {

src/servers/app/DrawState.h

@@ -26,6 +26,7 @@
 
 class AlphaMask;
 class BRegion;
+class shape_data;
 
 namespace BPrivate {
 	class LinkReceiver;
@@ -80,6 +81,9 @@ public:
 		bool				HasAdditionalClipping() const;
 		bool				GetCombinedClippingRegion(BRegion* region) const;
 
+		bool				ClipToRect(BRect rect, bool inverse);
+		void				ClipToShape(shape_data* shape, bool inverse);
+
 			void			SetAlphaMask(AlphaMask* mask);
 			AlphaMask*		GetAlphaMask() const;
 

src/servers/app/Layer.cpp

@@ -16,10 +16,12 @@
 
 class LayerCanvas : public Canvas {
 public:
-	LayerCanvas(DrawingEngine* drawingEngine, DrawState* drawState)
+	LayerCanvas(DrawingEngine* drawingEngine, DrawState* drawState,
+		BRect bitmapBounds)
 		:
 		Canvas(),
-		fDrawingEngine(drawingEngine)
+		fDrawingEngine(drawingEngine),
+		fBitmapBounds(bitmapBounds)
 	{
 		delete fDrawState;
 		fDrawState = drawState;
@@ -44,6 +46,20 @@ public:
 		fDrawingEngine->SetDrawState(fDrawState);
 	}
 
+	virtual void UpdateCurrentDrawingRegion()
+	{
+		bool hasDrawStateClipping = fDrawState->GetCombinedClippingRegion(
+			&fCurrentDrawingRegion);
+
+		BRegion bitmapRegion(fBitmapBounds);
+		if (hasDrawStateClipping)
+			fCurrentDrawingRegion.IntersectWith(&bitmapRegion);
+		else
+			fCurrentDrawingRegion = bitmapRegion;
+
+		fDrawingEngine->ConstrainClippingRegion(&fCurrentDrawingRegion);
+	}
+
 protected:
 	virtual void _LocalToScreenTransform(SimpleTransform&) const
 	{
@@ -55,6 +71,8 @@ protected:
 
 private:
 	DrawingEngine*	fDrawingEngine;
+	BRegion			fCurrentDrawingRegion;
+	BRect			fBitmapBounds;
 };
 
 
@@ -118,7 +136,7 @@ Layer::RenderToBitmap(Canvas* canvas)
 		// Painter), to prevent this origin from being further transformed by
 		// e.g. scaling.
 
-	LayerCanvas layerCanvas(layerEngine, canvas->CurrentState());
+	LayerCanvas layerCanvas(layerEngine, canvas->CurrentState(), boundingBox);
 
 	AlphaMask* const mask = layerCanvas.GetAlphaMask();
 	IntPoint oldOffset;
@@ -134,8 +152,7 @@ Layer::RenderToBitmap(Canvas* canvas)
 		// Apply state to the new drawing engine of the layer canvas
 
 	if (layerEngine->LockParallelAccess()) {
-		const BRegion region(boundingBox);
+		layerCanvas.UpdateCurrentDrawingRegion();
-		layerEngine->ConstrainClippingRegion(&region);
 
 		// Draw recorded picture into bitmap
 		Play(&layerCanvas);

src/servers/app/ServerPicture.cpp

@@ -760,6 +760,43 @@ blend_layer(void* _canvas, Layer* layer)
 }
 
 
+static void
+clip_to_rect(void* _canvas, const BRect& rect, bool inverse)
+{
+	Canvas* const canvas = reinterpret_cast<Canvas*>(_canvas);
+	bool needDrawStateUpdate = canvas->ClipToRect(rect, inverse);
+	if (needDrawStateUpdate)
+		canvas->ResyncDrawState();
+	canvas->UpdateCurrentDrawingRegion();
+}
+
+
+static void
+clip_to_shape(void* _canvas, int32 opCount, const uint32 opList[],
+	int32 ptCount, const BPoint ptList[], bool inverse)
+{
+	Canvas* const canvas = reinterpret_cast<Canvas*>(_canvas);
+	shape_data shapeData;
+
+	// TODO: avoid copies
+	shapeData.opList = (uint32*)malloc(opCount * sizeof(uint32));
+	memcpy(shapeData.opList, opList, opCount * sizeof(uint32));
+	shapeData.ptList = (BPoint*)malloc(ptCount * sizeof(BPoint));
+	memcpy(shapeData.ptList, ptList, ptCount * sizeof(BPoint));
+
+	shapeData.opCount = opCount;
+	shapeData.opSize = opCount * sizeof(uint32);
+	shapeData.ptCount = ptCount;
+	shapeData.ptSize = ptCount * sizeof(BPoint);
+
+	canvas->ClipToShape(&shapeData, inverse);
+	canvas->ResyncDrawState();
+
+	free(shapeData.opList);
+	free(shapeData.ptList);
+}
+
+
 static const BPrivate::picture_player_callbacks kPicturePlayerCallbacks = {
 	move_pen_by,
 	stroke_line,
@@ -801,7 +838,9 @@ static const BPrivate::picture_player_callbacks kPicturePlayerCallbacks = {
 	set_font_face,
 	set_blending_mode,
 	set_transform,
-	blend_layer
+	blend_layer,
+	clip_to_rect,
+	clip_to_shape
 };
 
 

src/servers/app/ServerWindow.cpp

@@ -44,6 +44,7 @@
 
 #include <MessagePrivate.h>
 #include <PortLink.h>
+#include <ShapePrivate.h>
 #include <ServerProtocolStructs.h>
 #include <ViewPrivate.h>
 #include <WindowInfo.h>
@@ -2016,6 +2017,51 @@ fDesktop->LockSingleWindow();
 			break;
 		}
 
+		case AS_VIEW_CLIP_TO_RECT:
+		{
+			bool inverse;
+			BRect rect;
+
+			link.Read<bool>(&inverse);
+			link.Read<BRect>(&rect);
+
+			bool needDrawStateUpdate = fCurrentView->ClipToRect(
+				rect, inverse);
+			fCurrentDrawingRegionValid = false;
+
+			if (needDrawStateUpdate)
+				_UpdateDrawState(fCurrentView);
+
+			_UpdateCurrentDrawingRegion();
+
+			BRegion region(fCurrentDrawingRegion);
+			fCurrentView->ScreenToLocalTransform().Apply(&region);
+
+			break;
+		}
+
+		case AS_VIEW_CLIP_TO_SHAPE:
+		{
+			bool inverse;
+			link.Read<bool>(&inverse);
+
+			shape_data shape;
+			link.Read<int32>(&shape.opCount);
+			link.Read<int32>(&shape.ptCount);
+			shape.opList = new(nothrow) uint32[shape.opCount];
+			shape.ptList = new(nothrow) BPoint[shape.ptCount];
+			if (link.Read(shape.opList, shape.opCount * sizeof(uint32)) >= B_OK
+				&& link.Read(shape.ptList,
+					shape.ptCount * sizeof(BPoint)) >= B_OK) {
+				fCurrentView->ClipToShape(&shape, inverse);
+				_UpdateDrawState(fCurrentView);
+			}
+
+			delete[] shape.opList;
+			delete[] shape.ptList;
+			break;
+		}
+
 		case AS_VIEW_INVALIDATE_RECT:
 		{
 			// NOTE: looks like this call is NOT affected by origin and scale
@@ -3437,6 +3483,39 @@ ServerWindow::_DispatchPictureMessage(int32 code, BPrivate::LinkReceiver& link)
 			break;
 		}
 
+		case AS_VIEW_CLIP_TO_RECT:
+		{
+			bool inverse;
+			BRect rect;
+			link.Read<bool>(&inverse);
+			link.Read<BRect>(&rect);
+			picture->WriteClipToRect(rect, inverse);
+
+			break;
+		}
+
+		case AS_VIEW_CLIP_TO_SHAPE:
+		{
+			bool inverse;
+			link.Read<bool>(&inverse);
+
+			shape_data shape;
+			link.Read<int32>(&shape.opCount);
+			link.Read<int32>(&shape.ptCount);
+			shape.opList = new(nothrow) uint32[shape.opCount];
+			shape.ptList = new(nothrow) BPoint[shape.ptCount];
+			if (link.Read(shape.opList, shape.opCount * sizeof(uint32)) >= B_OK
+				&& link.Read(shape.ptList,
+					shape.ptCount * sizeof(BPoint)) >= B_OK) {
+				picture->WriteClipToShape(shape.opCount, shape.opList,
+					shape.ptCount, shape.ptList, inverse);
+			}
+
+			delete[] shape.opList;
+			delete[] shape.ptList;
+			break;
+		}
+
 		case AS_VIEW_BEGIN_PICTURE:
 		{
 			ServerPicture* newPicture = App()->CreatePicture();