libprint: remove arbitrary limitation of allocation size

libprint had a very conservative size limitation (4MB) for the bitmap it
allocates. This leads to splitting a page to print in several "bands"
(about 5 in my testing). However, these may still be too large for the
printer driver to handle, which means the driver may be further slicing
things up, or other drivers may need the full page anyway and recompose
it in some way.

Instead of an hardcoded limit, now try to allocate a bitmap for the
whole page, and if that doesn't work, progressively increase the number
of bands until we manage to allocate a bitmap. Stop when we have split
the page in 256 bands, as it seems rather pointless to be that far. Call
debugger when this happens, as there doesn't seem to be a way to do
better error handling here (the code used to raise std::bad_alloc if
BBitmap allocation failed, or just return an invalid bitmap and view).

Change-Id: Iba690f68c748d20828709244a23e82a08185390e
Reviewed-on: https://review.haiku-os.org/c/haiku/+/1922
Reviewed-by: Adrien Destugues <[email protected]>

src/libs/print/libprint/GraphicsDriver.cpp

@@ -38,11 +38,6 @@ using namespace std;
 #define MEASURE_PRINT_JOB_TIME false
 
 
-enum {
-	kMaxMemorySize = 4 * 1024 * 1024
-};
-
-
 GraphicsDriver::GraphicsDriver(BMessage* message, PrinterData* printerData,
 	const PrinterCap* printerCap)
 	:
@@ -167,18 +162,11 @@ GraphicsDriver::_SetupBitmap()
 	fPageHeight = (fRealJobData->GetPhysicalRect().IntegerHeight()
 		* fOrgJobData->GetYres() + 71) / 72;
 
-	int widthByte = (fPageWidth * fPixelDepth + 7) / 8;
-	int size = widthByte * fPageHeight;
-#ifdef USE_PREVIEW_FOR_DEBUG
-	size = 0;
-#endif
+	fBitmap = NULL;
+	fRotatedBitmap = NULL;
+	BRect rect;
 
-	if (size < kMaxMemorySize) {
-		fBandCount  = 0;
-		fBandWidth  = fPageWidth;
-		fBandHeight = fPageHeight;
-	} else {
-		fBandCount  = (size + kMaxMemorySize - 1) / kMaxMemorySize;
+	for(fBandCount = 1; fBandCount < 256; fBandCount++) {
 		if (_NeedRotateBitmapBand()) {
 			fBandWidth  = (fPageWidth + fBandCount - 1) / fBandCount;
 			fBandHeight = fPageHeight;
@@ -186,26 +174,52 @@ GraphicsDriver::_SetupBitmap()
 			fBandWidth  = fPageWidth;
 			fBandHeight = (fPageHeight + fBandCount - 1) / fBandCount;
 		}
+
+		rect.Set(0, 0, fBandWidth - 1, fBandHeight - 1);
+		fBitmap = new(std::nothrow) BBitmap(rect, fOrgJobData->GetSurfaceType(),
+			true);
+		if (fBitmap == NULL || fBitmap->InitCheck() != B_OK) {
+			delete fBitmap;
+			fBitmap = NULL;
+			// Try with smaller bands
+			continue;
+		}
+
+		if (_NeedRotateBitmapBand()) {
+			BRect rotatedRect(0, 0, rect.bottom, rect.right);
+			delete fRotatedBitmap;
+			fRotatedBitmap = new(std::nothrow) BBitmap(rotatedRect,
+				fOrgJobData->GetSurfaceType(), false);
+			if (fRotatedBitmap == NULL || fRotatedBitmap->InitCheck() != B_OK) {
+				delete fBitmap;
+				fBitmap = NULL;
+				delete fRotatedBitmap;
+				fRotatedBitmap = NULL;
+
+				// Try with smaller bands
+				continue;
+			}
+		}
+
+		// If we get here, all needed allocations have succeeded, we can safely
+		// go ahead.
+		break;
+	};
+
+	if (fBitmap == NULL) {
+		debugger("Failed to allocate bitmaps for print rasterization");
+		return;
 	}
 
+	fView = new BView(rect, "", B_FOLLOW_ALL, B_WILL_DRAW);
+	fBitmap->AddChild(fView);
+
 	DBGMSG(("****************\n"));
 	DBGMSG(("page_width  = %d\n", fPageWidth));
 	DBGMSG(("page_height = %d\n", fPageHeight));
 	DBGMSG(("band_count  = %d\n", fBandCount));
 	DBGMSG(("band_height = %d\n", fBandHeight));
 	DBGMSG(("****************\n"));
-
-	BRect rect;
-	rect.Set(0, 0, fBandWidth - 1, fBandHeight - 1);
-	fBitmap = new BBitmap(rect, fOrgJobData->GetSurfaceType(), true);
-	fView   = new BView(rect, "", B_FOLLOW_ALL, B_WILL_DRAW);
-	fBitmap->AddChild(fView);
-
-	if (_NeedRotateBitmapBand()) {
-		BRect rotatedRect(0, 0, rect.bottom, rect.right);
-		fRotatedBitmap = new BBitmap(rotatedRect, fOrgJobData->GetSurfaceType(),
-			false);
-	}
 }