Add Alpha Masking support in ClipToPicture

Use AGG to implement ClipToPicture in a faster and better way.
There are things missing in this initial implementation:
* No support for PushState/PopState saving and restoring the picture.
* No support for nested clipping through PushState
* The clipping doesn't happen where you expect it when using SetScale()
* There are artifacts when scrolling and resizing clipped views
* The implementation uses more memory than it needs, as the clipping
bitmap is stored as RGBA32, yet only the alpha channel is used
* The clipping bitmap is rendered more times than it needs to. We need
some caching here.

src/servers/app/Jamfile

@@ -46,6 +46,7 @@ if $(TARGET_ARCH) != x86_64 {
 }
 
 Server app_server :
+	AlphaMask.cpp
 	Angle.cpp
 	AppServer.cpp
 	#BitfieldRegion.cpp

src/servers/app/View.cpp

@@ -15,6 +15,8 @@
 #include <new>
 #include <stdio.h>
 
+#include "AlphaMask.h"
+#include "BitmapHWInterface.h"
 #include "BitmapManager.h"
 #include "Desktop.h"
 #include "DrawingEngine.h"
@@ -113,6 +115,7 @@ View::View(IntRect frame, IntPoint scrollingOffset, const char* name,
 
 	fCursor(NULL),
 	fPicture(NULL),
+	fAlphaMask(NULL),
 
 	fLocalClipping((BRect)Bounds()),
 	fScreenClipping(),
@@ -133,6 +136,7 @@ View::~View()
 	delete fScreenAndUserClipping;
 	delete fUserClipping;
 	delete fDrawState;
+	delete fAlphaMask;
 
 //	if (fWindow && this == fWindow->TopView())
 //		fWindow->SetTopView(NULL);
@@ -1573,6 +1577,98 @@ View::InvalidateScreenClipping()
 }
 
 
+// TODO we should be storing the ServerPicture here, so we can recompute the
+// bitmap mask when the view is scaled, resized or the origin is changed.
+// This would allow us to keep a bitmap mask matching exactly the view size.
+// Moreover, we should clip that bitmap mask using the region-based clipping,
+// so it can mask out all the clipped regions, and we don't have to worry about
+// them whendoing further drawing. Essentially, switch from region-based to
+// bitmap based clipping for all ourdrawing.
+void
+View::SetAlphaMask(ServerPicture* picture, bool inverse, BPoint origin)
+{
+	// BeOS compatibility: they implemented ClipToPicture by converting the
+	// picture to a complex BRegion and used that as a clipping region. As a
+	// result, youcan't have a picture and a region clipping at the same level
+	// (but you can either using PushState/PopState, or using
+	// ConstrainClippingRegion after ClipToPicture...)
+	// SetUserClipping(NULL);
+
+	delete fAlphaMask;
+	if (picture != NULL) {
+		fAlphaMask = new(std::nothrow) AlphaMask(*this, *picture, inverse,
+			origin);
+	} else
+		fAlphaMask = NULL;
+}
+
+
+ServerBitmap*
+View::_RenderPicture(ServerPicture* picture, bool inverse)
+{
+	BRect bounds(Bounds());
+
+	// TODO: Only the alpha channel is relevant, but there is no B_ALPHA8
+	// color space, so we use 300% more memory than needed.
+	UtilityBitmap* bitmap = new(std::nothrow) UtilityBitmap(bounds, B_RGBA32,
+		0);
+	if (bitmap == NULL)
+		return NULL;
+
+#if 0
+	/*
+	 * TODO stippi says we could use OffscreenWindow to do this, but there
+	 * doesn't seem to be a way to create a View without a BView on
+	 * application side (the constructor wants a token).
+	 * This would be better, as it would avoid the DrawingContext abstraction
+	 * above View and OffscreenContext and allow for inlining more methods.
+	 */
+	OffscreenWindow window(bitmap, "ClipToPicture", Window());
+	View view(bounds, IntPoint(0, 0), "ClipToPicture");
+	window->SetTopView(view);
+#endif
+
+	// Clear the bitmap with the transparent color
+	memset(bitmap->Bits(), 0, bitmap->BitsLength());
+
+	// Render the picture to the bitmap
+	BitmapHWInterface interface(bitmap);
+	DrawingEngine* engine = interface.CreateDrawingEngine();
+	if (engine == NULL) {
+		delete bitmap;
+		return NULL;
+	}
+
+	// Copy the current state of the client view, so we draw with the right
+	// font, color and everything
+	engine->SetDrawState(CurrentState());
+	OffscreenContext context(engine);
+	if (engine->LockParallelAccess())
+	{
+		// FIXME ConstrainClippingRegion docs says passing NULL disables
+		// all clipping. This doesn't work and will crash in Painter.
+		BRegion clipping;
+		clipping.Include(bounds);
+		engine->ConstrainClippingRegion(&clipping);
+		picture->Play(&context);
+		engine->UnlockParallelAccess();
+	}
+	delete engine;
+
+	if (!inverse)
+		return bitmap;
+
+	// Compute the inverse of our bitmap. There probably is a better way.
+	uint32 size = bitmap->BitsLength();
+	uint8* bits = (uint8*)bitmap->Bits();
+
+	for(uint32 i = 0; i < size; i++)
+		bits[i] = 255 - bits[i];
+
+	return bitmap;
+}
+
+
 BRegion&
 View::_ScreenClipping(BRegion* windowContentClipping, bool force) const
 {

src/servers/app/View.h

@@ -29,6 +29,7 @@ namespace BPrivate {
 	class PortLink;
 };
 
+class AlphaMask;
 class DrawingEngine;
 class Overlay;
 class Window;
@@ -220,6 +221,10 @@ class View: public DrawingContext {
 										&& fScreenAndUserClipping != NULL));
 								}
 
+			void			SetAlphaMask(ServerPicture* picture, bool inverse,
+								BPoint where);
+			AlphaMask*		GetAlphaMask() { return fAlphaMask; }
+
 			// debugging
 			void			PrintToStream() const;
 #if 0
@@ -228,6 +233,9 @@ class View: public DrawingContext {
 #endif
 
 	protected:
+			friend class	AlphaMask;
+			ServerBitmap*	_RenderPicture(ServerPicture* picture,
+								bool inverse);
 			BRegion&		_ScreenClipping(BRegion* windowContentClipping,
 								bool force = false) const;
 			void			_MoveScreenClipping(int32 x, int32 y,
@@ -269,6 +277,7 @@ class View: public DrawingContext {
 
 			ServerCursor*	fCursor;
 			ServerPicture*	fPicture;
+			AlphaMask*		fAlphaMask;
 
 			// clipping
 			BRegion			fLocalClipping;

src/servers/app/drawing/DrawingEngine.cpp

@@ -17,6 +17,7 @@
 #include <algorithm>
 #include <stack>
 
+#include "AlphaMask.h"
 #include "DrawState.h"
 #include "GlyphLayoutEngine.h"
 #include "Painter.h"
@@ -234,6 +235,16 @@ DrawingEngine::ConstrainClippingRegion(const BRegion* region)
 }
 
 
+void
+DrawingEngine::SetAlphaMask(AlphaMask* mask)
+{
+	scanline_unpacked_masked_type* scanline = NULL;
+	if (mask != NULL)
+		scanline = mask->Generate();
+	fPainter->SetAlphaMask(scanline);
+}
+
+
 void
 DrawingEngine::SetDrawState(const DrawState* state, int32 xOffset,
 	int32 yOffset)

src/servers/app/drawing/DrawingEngine.h

@@ -25,6 +25,7 @@ class BPoint;
 class BRect;
 class BRegion;
 
+class AlphaMask;
 class DrawState;
 class Painter;
 class ServerBitmap;
@@ -67,6 +68,7 @@ public:
 	// clipping for all drawing functions, passing a NULL region
 	// will remove any clipping (drawing allowed everywhere)
 	virtual	void			ConstrainClippingRegion(const BRegion* region);
+			void			SetAlphaMask(AlphaMask* mask);
 
 	virtual	void			SetDrawState(const DrawState* state,
 								int32 xOffset = 0, int32 yOffset = 0);

src/servers/app/drawing/Painter/Painter.cpp

@@ -51,6 +51,7 @@
 #include <AutoDeleter.h>
 #include <View.h>
 
+#include "AlphaMask.h"
 #include "DrawingMode.h"
 #include "GlobalSubpixelSettings.h"
 #include "PatternHandler.h"
@@ -172,13 +173,14 @@ Painter::Painter()
 	fBaseRenderer(fPixelFormat),
 	fUnpackedScanline(),
 	fPackedScanline(),
+	fRasterizer(),
+	fRenderer(fBaseRenderer),
+	fRendererBin(fBaseRenderer),
 	fSubpixPackedScanline(),
 	fSubpixUnpackedScanline(),
 	fSubpixRasterizer(),
-	fRasterizer(),
 	fSubpixRenderer(fBaseRenderer),
-	fRenderer(fBaseRenderer),
-	fRendererBin(fBaseRenderer),
+	fMaskedUnpackedScanline(NULL),
 
 	fPath(),
 	fCurve(fPath),
@@ -334,6 +336,13 @@ Painter::ConstrainClipping(const BRegion* region)
 }
 
 
+void
+Painter::SetAlphaMask(scanline_unpacked_masked_type* mask)
+{
+	fMaskedUnpackedScanline = mask;
+}
+
+
 // SetHighColor
 void
 Painter::SetHighColor(const rgb_color& color)
@@ -454,7 +463,8 @@ Painter::StrokeLine(BPoint a, BPoint b)
 
 	// first, try an optimized version
 	if (fPenSize == 1.0
-		&& (fDrawingMode == B_OP_COPY || fDrawingMode == B_OP_OVER)) {
+		&& (fDrawingMode == B_OP_COPY || fDrawingMode == B_OP_OVER)
+		&& fMaskedUnpackedScanline == NULL) {
 		pattern pat = *fPatternHandler.GetR5Pattern();
 		if (pat == B_SOLID_HIGH
 			&& StraightLine(a, b, fPatternHandler.HighColor())) {
@@ -862,8 +872,9 @@ Painter::StrokeRect(const BRect& r) const
 	_Transform(&b, false);
 
 	// first, try an optimized version
-	if (fPenSize == 1.0 &&
-		(fDrawingMode == B_OP_COPY || fDrawingMode == B_OP_OVER)) {
+	if (fPenSize == 1.0
+			&& (fDrawingMode == B_OP_COPY || fDrawingMode == B_OP_OVER)
+			&& fMaskedUnpackedScanline == NULL) {
 		pattern p = *fPatternHandler.GetR5Pattern();
 		if (p == B_SOLID_HIGH) {
 			BRect rect(a, b);
@@ -924,7 +935,8 @@ Painter::FillRect(const BRect& r) const
 	_Transform(&b, false);
 
 	// first, try an optimized version
-	if (fDrawingMode == B_OP_COPY || fDrawingMode == B_OP_OVER) {
+	if ((fDrawingMode == B_OP_COPY || fDrawingMode == B_OP_OVER)
+			&& fMaskedUnpackedScanline == NULL) {
 		pattern p = *fPatternHandler.GetR5Pattern();
 		if (p == B_SOLID_HIGH) {
 			BRect rect(a, b);
@@ -983,7 +995,8 @@ Painter::FillRect(const BRect& r, const BGradient& gradient) const
 
 	// first, try an optimized version
 	if (gradient.GetType() == BGradient::TYPE_LINEAR
-		&& (fDrawingMode == B_OP_COPY || fDrawingMode == B_OP_OVER)) {
+		&& (fDrawingMode == B_OP_COPY || fDrawingMode == B_OP_OVER)
+		&& fMaskedUnpackedScanline == NULL) {
 		const BGradientLinear* linearGradient
 			= dynamic_cast<const BGradientLinear*>(&gradient);
 		if (linearGradient->Start().x == linearGradient->End().x
@@ -1220,7 +1233,10 @@ Painter::StrokeRoundRect(const BRect& r, float xRadius, float yRadius) const
 		// make the inner rect work as a hole
 		fRasterizer.filling_rule(agg::fill_even_odd);
 
-		if (fPenSize > 2)
+		if (fMaskedUnpackedScanline != NULL) {
+			agg::render_scanlines(fRasterizer, *fMaskedUnpackedScanline,
+				fRenderer);
+		} else if (fPenSize > 2)
 			agg::render_scanlines(fRasterizer, fPackedScanline, fRenderer);
 		else
 			agg::render_scanlines(fRasterizer, fUnpackedScanline, fRenderer);
@@ -1362,7 +1378,10 @@ Painter::DrawEllipse(BRect r, bool fill) const
 		// make the inner ellipse work as a hole
 		fRasterizer.filling_rule(agg::fill_even_odd);
 
-		if (fPenSize > 4)
+		if (fMaskedUnpackedScanline != NULL) {
+			agg::render_scanlines(fRasterizer, *fMaskedUnpackedScanline,
+				fRenderer);
+		} else if (fPenSize > 4)
 			agg::render_scanlines(fRasterizer, fPackedScanline, fRenderer);
 		else
 			agg::render_scanlines(fRasterizer, fUnpackedScanline, fRenderer);
@@ -2750,8 +2769,13 @@ Painter::_DrawBitmapGeneric32(agg::rendering_buffer& srcBuffer,
 		span_gen_type spanGenerator(source, interpolator);
 
 		// render the path with the bitmap as scanline fill
-		agg::render_scanlines_aa(fRasterizer, fUnpackedScanline, fBaseRenderer,
-			spanAllocator, spanGenerator);
+		if (fMaskedUnpackedScanline != NULL) {
+			agg::render_scanlines_aa(fRasterizer, *fMaskedUnpackedScanline,
+				fBaseRenderer, spanAllocator, spanGenerator);
+		} else {
+			agg::render_scanlines_aa(fRasterizer, fUnpackedScanline,
+				fBaseRenderer, spanAllocator, spanGenerator);
+		}
 	} else {
 		// image filter (nearest neighbor)
 		typedef agg::span_image_filter_rgba_nn<
@@ -2759,8 +2783,13 @@ Painter::_DrawBitmapGeneric32(agg::rendering_buffer& srcBuffer,
 		span_gen_type spanGenerator(source, interpolator);
 
 		// render the path with the bitmap as scanline fill
-		agg::render_scanlines_aa(fRasterizer, fUnpackedScanline, fBaseRenderer,
-			spanAllocator, spanGenerator);
+		if (fMaskedUnpackedScanline != NULL) {
+			agg::render_scanlines_aa(fRasterizer, *fMaskedUnpackedScanline,
+				fBaseRenderer, spanAllocator, spanGenerator);
+		} else {
+			agg::render_scanlines_aa(fRasterizer, fUnpackedScanline,
+				fBaseRenderer, spanAllocator, spanGenerator);
+		}
 	}
 }
 
@@ -2887,6 +2916,12 @@ Painter::_StrokePath(VertexSource& path) const
 
 		agg::render_scanlines(fSubpixRasterizer,
 			fSubpixPackedScanline, fSubpixRenderer);
+	} else if(fMaskedUnpackedScanline != NULL) {
+		// TODO: we can't do both alpha-masking and subpixel AA.
+		fRasterizer.reset();
+		fRasterizer.add_path(path);
+		agg::render_scanlines(fRasterizer, *fMaskedUnpackedScanline,
+			fRenderer);
 	} else {
 		fRasterizer.reset();
 		fRasterizer.add_path(stroke);
@@ -2912,6 +2947,12 @@ Painter::_FillPath(VertexSource& path) const
 		fSubpixRasterizer.add_path(path);
 		agg::render_scanlines(fSubpixRasterizer,
 			fSubpixPackedScanline, fSubpixRenderer);
+	} else if(fMaskedUnpackedScanline != NULL) {
+		// TODO: we can't do both alpha-masking and subpixel AA.
+		fRasterizer.reset();
+		fRasterizer.add_path(path);
+		agg::render_scanlines(fRasterizer, *fMaskedUnpackedScanline,
+			fRenderer);
 	} else {
 		fRasterizer.reset();
 		fRasterizer.add_path(path);
@@ -3133,7 +3174,12 @@ Painter::_FillPathGradientLinear(VertexSource& path,
 
 	fRasterizer.reset();
 	fRasterizer.add_path(path);
-	agg::render_scanlines(fRasterizer, fPackedScanline, gradientRenderer);
+	if (fMaskedUnpackedScanline == NULL)
+		agg::render_scanlines(fRasterizer, fPackedScanline, gradientRenderer);
+	else {
+		agg::render_scanlines(fRasterizer, *fMaskedUnpackedScanline,
+			gradientRenderer);
+	}
 }
 
 
@@ -3180,7 +3226,12 @@ Painter::_FillPathGradientRadial(VertexSource& path,
 
 	fRasterizer.reset();
 	fRasterizer.add_path(path);
-	agg::render_scanlines(fRasterizer, fPackedScanline, gradientRenderer);
+	if (fMaskedUnpackedScanline == NULL)
+		agg::render_scanlines(fRasterizer, fPackedScanline, gradientRenderer);
+	else {
+		agg::render_scanlines(fRasterizer, *fMaskedUnpackedScanline,
+			gradientRenderer);
+	}
 }
 
 
@@ -3228,7 +3279,12 @@ Painter::_FillPathGradientRadialFocus(VertexSource& path,
 
 	fRasterizer.reset();
 	fRasterizer.add_path(path);
-	agg::render_scanlines(fRasterizer, fPackedScanline, gradientRenderer);
+	if (fMaskedUnpackedScanline == NULL)
+		agg::render_scanlines(fRasterizer, fPackedScanline, gradientRenderer);
+	else {
+		agg::render_scanlines(fRasterizer, *fMaskedUnpackedScanline,
+			gradientRenderer);
+	}
 }
 
 
@@ -3274,7 +3330,12 @@ Painter::_FillPathGradientDiamond(VertexSource& path,
 
 	fRasterizer.reset();
 	fRasterizer.add_path(path);
-	agg::render_scanlines(fRasterizer, fPackedScanline, gradientRenderer);
+	if (fMaskedUnpackedScanline == NULL)
+		agg::render_scanlines(fRasterizer, fPackedScanline, gradientRenderer);
+	else {
+		agg::render_scanlines(fRasterizer, *fMaskedUnpackedScanline,
+			gradientRenderer);
+	}
 }
 
 
@@ -3320,5 +3381,10 @@ Painter::_FillPathGradientConic(VertexSource& path,
 
 	fRasterizer.reset();
 	fRasterizer.add_path(path);
-	agg::render_scanlines(fRasterizer, fPackedScanline, gradientRenderer);
+	if (fMaskedUnpackedScanline == NULL)
+		agg::render_scanlines(fRasterizer, fPackedScanline, gradientRenderer);
+	else {
+		agg::render_scanlines(fRasterizer, *fMaskedUnpackedScanline,
+			gradientRenderer);
+	}
 }

src/servers/app/drawing/Painter/Painter.h

@@ -54,6 +54,8 @@ public:
 			void				ConstrainClipping(const BRegion* region);
 			const BRegion*		ClippingRegion() const
 									{ return fClippingRegion; }
+			void				SetAlphaMask(
+									scanline_unpacked_masked_type* mask);
 
 			void				SetDrawState(const DrawState* data,
 									int32 xOffset = 0,
@@ -331,15 +333,24 @@ private:
 			pixfmt				fPixelFormat;
 	mutable	renderer_base		fBaseRenderer;
 
+	// Regular drawing mode: pixel-aligned, no alpha masking
 	mutable	scanline_unpacked_type fUnpackedScanline;
 	mutable	scanline_packed_type fPackedScanline;
+	mutable	rasterizer_type		fRasterizer;
+	mutable	renderer_type		fRenderer;
+
+	// Fast mode: no antialiasing needed (horizontal/vertical lines, ...)
+	mutable	renderer_bin_type	fRendererBin;
+
+	// Subpixel mode
 	mutable	scanline_packed_subpix_type fSubpixPackedScanline;
 	mutable	scanline_unpacked_subpix_type fSubpixUnpackedScanline;
 	mutable	rasterizer_subpix_type fSubpixRasterizer;
-	mutable	rasterizer_type		fRasterizer;
 	mutable	renderer_subpix_type fSubpixRenderer;
-	mutable	renderer_type		fRenderer;
-	mutable	renderer_bin_type	fRendererBin;
+
+	// Alpha-Masked mode: for ClipToPicture
+	// (this uses the standard rasterizer and renderer)
+	mutable	scanline_unpacked_masked_type* fMaskedUnpackedScanline;
 
 	mutable	agg::path_storage	fPath;
 	mutable	agg::conv_curve<agg::path_storage> fCurve;

src/servers/app/drawing/Painter/defines.h

@@ -11,6 +11,7 @@
 #ifndef DEFINES_H
 #define DEFINES_H
 
+#include <agg_alpha_mask_u8.h>
 #include <agg_rasterizer_outline.h>
 #include <agg_rasterizer_outline_aa.h>
 #include <agg_rasterizer_scanline_aa.h>
@@ -25,6 +26,7 @@
 #include <agg_span_interpolator_linear.h>
 #include <agg_rendering_buffer.h>
 
+#include "agg_clipped_alpha_mask.h"
 #include "agg_rasterizer_scanline_aa_subpix.h"
 #include "agg_renderer_region.h"
 #include "agg_renderer_scanline_subpix.h"
@@ -34,7 +36,7 @@
 #include "agg_scanline_u_subpix_avrg_filtering.h"
 
 #include "GlobalSubpixelSettings.h"
-#include "PixelFormat.h"
+#include "drawing_modes/PixelFormat.h"
 
 
 #define ALIASED_DRAWING 0
@@ -62,6 +64,9 @@
 	typedef agg::scanline_p8_subpix								scanline_packed_subpix_type;
 	typedef agg::scanline_u8_subpix								scanline_unpacked_subpix_type;
 #endif
+
+	typedef agg::scanline_u8_am<agg::clipped_alpha_mask>		scanline_unpacked_masked_type;
+
 	typedef agg::renderer_scanline_aa_solid<renderer_base>		renderer_type;
 #endif // !ALIASED_DRAWING