Shared: JSON Parse Perf

Improve the mechanics for JSON parsing by reusing
text buffers during the parse.

Change-Id: I7fb2cae31e6558a5a0c63fd02e1fc6fec4f9e4b3
Reviewed-on: https://review.haiku-os.org/c/haiku/+/7106
Tested-by: Commit checker robot <[email protected]>
Reviewed-by: Jérôme Duval <[email protected]>

headers/private/shared/Json.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017, Andrew Lindesay <[email protected]>
+ * Copyright 2017-2023, Andrew Lindesay <[email protected]>
  * Copyright 2014, Augustin Cavalier (waddlesplash)
  * Distributed under the terms of the MIT License.
  */
@@ -39,14 +39,12 @@ private:
 									JsonParseContext& jsonParseContext);
 	static	bool				ParseObject(JsonParseContext& jsonParseContext);
 	static	bool				ParseArray(JsonParseContext& jsonParseContext);
-	static	bool				ParseEscapeUnicodeSequence(
+
-									JsonParseContext& jsonParseContext,
+	static	bool				ParseEscapeUnicodeSequence(JsonParseContext& jsonParseContext);
-									BString& stringResult);
+	static	bool				ParseStringEscapeSequence(JsonParseContext& jsonParseContext);
-	static	bool				ParseStringEscapeSequence(
-									JsonParseContext& jsonParseContext,
-									BString& stringResult);
 	static	bool				ParseString(JsonParseContext& jsonParseContext,
 									json_event_type eventType);
+
 	static	bool				ParseExpectedVerbatimStringAndRaiseEvent(
 									JsonParseContext& jsonParseContext,
 									const char* expectedString,
@@ -59,7 +57,7 @@ private:
 									size_t expectedStringLength,
 									char leadingChar);
 
-	static bool					IsValidNumber(BString& number);
+	static bool					IsValidNumber(const char* value);
 	static bool					ParseNumber(JsonParseContext& jsonParseContext);
 };
 

src/kits/shared/Json.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>
+ * Copyright 2017-2023, Andrew Lindesay <apl@lindesay.co.nz>
  * Copyright 2014-2017, Augustin Cavalier (waddlesplash)
  * Copyright 2014, Stephan Aßmus <superstippi@gmx.de>
  * Distributed under the terms of the MIT License.
@@ -25,26 +25,151 @@
 
 namespace BPrivate {
 
+/*!	A buffer is used to assemble strings into. This will be the initial size
+	of this buffer.
+*/
 
-static bool
+static const size_t kInitialAssemblyBufferSize = 64;
-b_jsonparse_is_hex(char c)
+
-{
+/*!	A buffer is used to assemble strings into. This buffer starts off small
-	return isdigit(c)
+	but is able to grow as the string it needs to process as encountered. To
-		|| (c > 0x41 && c <= 0x46)
+	avoid frequent reallocation of the buffer, the buffer will be retained
-		|| (c > 0x61 && c <= 0x66);
+	between strings. This is the maximum size of buffer that will be retained.
-}
+*/
+
+static const size_t kRetainedAssemblyBufferSize = 32 * 1024;
+
+static const size_t kAssemblyBufferSizeIncrement = 256;
+
+static const size_t kMaximumUtf8SequenceLength = 7;
 
 
-static bool
+class JsonParseAssemblyBuffer {
-b_jsonparse_all_hex(const char* c)
+public:
-{
+	JsonParseAssemblyBuffer()
-	for (int i = 0; i < 4; i++) {
+		:
-		if (!b_jsonparse_is_hex(c[i]))
+		fAssemblyBuffer(NULL),
-			return false;
+		fAssemblyBufferAllocatedSize(0),
+		fAssemblyBufferUsedSize(0)
+	{
+		fAssemblyBuffer = (char*) malloc(kInitialAssemblyBufferSize);
+		if (fAssemblyBuffer != NULL)
+			fAssemblyBufferAllocatedSize = kInitialAssemblyBufferSize;
 	}
 
-	return true;
+	~JsonParseAssemblyBuffer()
-}
+	{
+		if (fAssemblyBuffer != NULL)
+			free(fAssemblyBuffer);
+	}
+
+	const char* Buffer() const
+	{
+		return fAssemblyBuffer;
+	}
+
+	/*! This method should be used each time that the assembly buffer has
+		been finished with by some section of logic.
+	*/
+
+	status_t Reset()
+	{
+		fAssemblyBufferUsedSize = 0;
+
+		if (fAssemblyBufferAllocatedSize > kRetainedAssemblyBufferSize) {
+			fAssemblyBuffer = (char*) realloc(fAssemblyBuffer, kRetainedAssemblyBufferSize);
+			if (fAssemblyBuffer == NULL) {
+				fAssemblyBufferAllocatedSize = 0;
+				return B_NO_MEMORY;
+			}
+			fAssemblyBufferAllocatedSize = kRetainedAssemblyBufferSize;
+		}
+
+		return B_OK;
+	}
+
+	status_t AppendCharacter(char c)
+	{
+		status_t result = _EnsureAssemblyBufferAllocatedSize(fAssemblyBufferUsedSize + 1);
+
+		if (result == B_OK) {
+			fAssemblyBuffer[fAssemblyBufferUsedSize] = c;
+			fAssemblyBufferUsedSize++;
+		}
+
+		return result;
+	}
+
+	status_t AppendCharacters(char* str, size_t len)
+	{
+		status_t result = _EnsureAssemblyBufferAllocatedSize(fAssemblyBufferUsedSize + len);
+
+		if (result == B_OK) {
+			memcpy(&fAssemblyBuffer[fAssemblyBufferUsedSize], str, len);
+			fAssemblyBufferUsedSize += len;
+		}
+
+		return result;
+	}
+
+	status_t AppendUnicodeCharacter(uint32 c)
+	{
+		status_t result = _EnsureAssemblyBufferAllocatedSize(
+			fAssemblyBufferUsedSize + kMaximumUtf8SequenceLength);
+		if (result == B_OK) {
+			char* insertPtr = &fAssemblyBuffer[fAssemblyBufferUsedSize];
+			char* ptr = insertPtr;
+			BUnicodeChar::ToUTF8(c, &ptr);
+			size_t sequenceLength = static_cast<uint32>(ptr - insertPtr);
+			fAssemblyBufferUsedSize += sequenceLength;
+		}
+
+		return result;
+	}
+
+private:
+
+	/*!	This method will return the assembly buffer ensuring that it has at
+		least `minimumSize` bytes available.
+	*/
+
+	status_t _EnsureAssemblyBufferAllocatedSize(size_t minimumSize)
+	{
+		if (fAssemblyBufferAllocatedSize < minimumSize) {
+			fAssemblyBuffer = (char*) realloc(fAssemblyBuffer, minimumSize);
+			if (fAssemblyBuffer == NULL) {
+				fAssemblyBufferAllocatedSize = 0;
+				return B_NO_MEMORY;
+			}
+			fAssemblyBufferAllocatedSize = minimumSize;
+		}
+		return B_OK;
+	}
+
+private:
+	char*					fAssemblyBuffer;
+	size_t					fAssemblyBufferAllocatedSize;
+	size_t					fAssemblyBufferUsedSize;
+};
+
+
+class JsonParseAssemblyBufferResetter {
+public:
+	JsonParseAssemblyBufferResetter(JsonParseAssemblyBuffer* assemblyBuffer)
+		:
+		fAssemblyBuffer(assemblyBuffer)
+	{
+	}
+
+	~JsonParseAssemblyBufferResetter()
+	{
+		fAssemblyBuffer->Reset();
+	}
+
+private:
+	JsonParseAssemblyBuffer*
+							fAssemblyBuffer;
+};
 
 
 /*! This class carries state around the parsing process. */
@@ -57,11 +182,18 @@ public:
 		fData(data),
 		fLineNumber(1), // 1 is the first line
 		fPushbackChar(0),
-		fHasPushbackChar(false)
+		fHasPushbackChar(false),
+		fAssemblyBuffer(new JsonParseAssemblyBuffer())
 	{
 	}
 
 
+	~JsonParseContext()
+	{
+		delete fAssemblyBuffer;
+	}
+
+
 	BJsonEventListener* Listener() const
 	{
 		return fListener;
@@ -85,14 +217,8 @@ public:
 		fLineNumber++;
 	}
 
-
-		// TODO; there is considerable opportunity for performance improvements
-		// here by buffering the input and then feeding it into the parse
-		// algorithm character by character.
-
 	status_t NextChar(char* buffer)
 	{
-
 		if (fHasPushbackChar) {
 			buffer[0] = fPushbackChar;
 			fHasPushbackChar = false;
@@ -102,19 +228,27 @@ public:
 		return Data()->ReadExactly(buffer, 1);
 	}
 
-
 	void PushbackChar(char c)
 	{
 		fPushbackChar = c;
 		fHasPushbackChar = true;
 	}
 
+
+	JsonParseAssemblyBuffer* AssemblyBuffer()
+	{
+		return fAssemblyBuffer;
+	}
+
+
 private:
 	BJsonEventListener*		fListener;
 	BDataIO*				fData;
 	uint32					fLineNumber;
 	char					fPushbackChar;
 	bool					fHasPushbackChar;
+	JsonParseAssemblyBuffer*
+							fAssemblyBuffer;
 };
 
 
@@ -464,90 +598,86 @@ BJson::ParseArray(JsonParseContext& jsonParseContext)
 
 
 bool
-BJson::ParseEscapeUnicodeSequence(JsonParseContext& jsonParseContext,
+BJson::ParseEscapeUnicodeSequence(JsonParseContext& jsonParseContext)
-	BString& stringResult)
 {
-	char buffer[5];
+	char ch;
-	buffer[4] = 0;
+	uint32 unicodeCh = 0;
 
-	if (!NextChar(jsonParseContext, &buffer[0])
+	for (int i = 3; i >= 0; i--) {
-		|| !NextChar(jsonParseContext, &buffer[1])
+		if (!NextChar(jsonParseContext, &ch)) {
-		|| !NextChar(jsonParseContext, &buffer[2])
+			jsonParseContext.Listener()->HandleError(B_ERROR, jsonParseContext.LineNumber(),
-		|| !NextChar(jsonParseContext, &buffer[3])) {
+				"unable to read unicode sequence");
-		return false;
+			return false;
+		}
+
+		if (ch >= '0' && ch <= '9')
+			unicodeCh |= static_cast<uint32>(ch - '0') << (i * 4);
+		else if (ch >= 'a' && ch <= 'f')
+			unicodeCh |= (10 + static_cast<uint32>(ch - 'a')) << (i * 4);
+		else if (ch >= 'A' && ch <= 'F')
+			unicodeCh |= (10 + static_cast<uint32>(ch - 'A')) << (i * 4);
+		else {
+			BString errorMessage;
+			errorMessage.SetToFormat(
+				"malformed hex character [%c] in unicode sequence in string parsing", ch);
+			jsonParseContext.Listener()->HandleError(B_BAD_DATA, jsonParseContext.LineNumber(),
+				errorMessage.String());
+			return false;
+		}
 	}
 
-	if (!b_jsonparse_all_hex(buffer)) {
+	JsonParseAssemblyBuffer* assemblyBuffer = jsonParseContext.AssemblyBuffer();
-		BString errorMessage;
+	status_t result = assemblyBuffer->AppendUnicodeCharacter(unicodeCh);
-		errorMessage.SetToFormat(
+
-			"malformed unicode sequence [%s] in string parsing",
+	if (result != B_OK) {
-			buffer);
+		jsonParseContext.Listener()->HandleError(result, jsonParseContext.LineNumber(),
-		jsonParseContext.Listener()->HandleError(B_BAD_DATA,
+			"unable to store unicode char as utf-8");
-			jsonParseContext.LineNumber(), errorMessage.String());
 		return false;
 	}
 
-	uint intValue;
-
-	if (sscanf(buffer, "%4x", &intValue) != 1) {
-		BString errorMessage;
-		errorMessage.SetToFormat(
-			"unable to process unicode sequence [%s] in string "
-			" parsing", buffer);
-		jsonParseContext.Listener()->HandleError(B_BAD_DATA,
-			jsonParseContext.LineNumber(), errorMessage.String());
-		return false;
-	}
-
-	char character[7];
-	char* ptr = character;
-	BUnicodeChar::ToUTF8(intValue, &ptr);
-	int32 sequenceLength = ptr - character;
-	stringResult.Append(character, sequenceLength);
-
 	return true;
 }
 
 
 bool
-BJson::ParseStringEscapeSequence(JsonParseContext& jsonParseContext,
+BJson::ParseStringEscapeSequence(JsonParseContext& jsonParseContext)
-	BString& stringResult)
 {
 	char c;
 
 	if (!NextChar(jsonParseContext, &c))
 		return false;
 
+	JsonParseAssemblyBuffer* assemblyBuffer = jsonParseContext.AssemblyBuffer();
+
 	switch (c) {
 		case 'n':
-			stringResult += "\n";
+			assemblyBuffer->AppendCharacter('\n');
 			break;
 		case 'r':
-			stringResult += "\r";
+			assemblyBuffer->AppendCharacter('\r');
 			break;
 		case 'b':
-			stringResult += "\b";
+			assemblyBuffer->AppendCharacter('\b');
 			break;
 		case 'f':
-			stringResult += "\f";
+			assemblyBuffer->AppendCharacter('\f');
 			break;
 		case '\\':
-			stringResult += "\\";
+			assemblyBuffer->AppendCharacter('\\');
 			break;
 		case '/':
-			stringResult += "/";
+			assemblyBuffer->AppendCharacter('/');
 			break;
 		case 't':
-			stringResult += "\t";
+			assemblyBuffer->AppendCharacter('\t');
 			break;
 		case '"':
-			stringResult += "\"";
+			assemblyBuffer->AppendCharacter('"');
 			break;
 		case 'u':
 		{
 				// unicode escape sequence.
-			if (!ParseEscapeUnicodeSequence(jsonParseContext,
+			if (!ParseEscapeUnicodeSequence(jsonParseContext)) {
-					stringResult)) {
 				return false;
 			}
 			break;
@@ -555,9 +685,7 @@ BJson::ParseStringEscapeSequence(JsonParseContext& jsonParseContext,
 		default:
 		{
 			BString errorMessage;
-			errorMessage.SetToFormat(
+			errorMessage.SetToFormat("unexpected escaped character [%c] in string parsing", c);
-				"unexpected escaped character [%c] in string parsing",
-				c);
 			jsonParseContext.Listener()->HandleError(B_BAD_DATA,
 				jsonParseContext.LineNumber(), errorMessage.String());
 			return false;
@@ -573,7 +701,8 @@ BJson::ParseString(JsonParseContext& jsonParseContext,
 	json_event_type eventType)
 {
 	char c;
-	BString stringResult;
+	JsonParseAssemblyBuffer* assemblyBuffer = jsonParseContext.AssemblyBuffer();
+	JsonParseAssemblyBufferResetter assembleBufferResetter(assemblyBuffer);
 
 	while(true) {
 		if (!NextChar(jsonParseContext, &c))
@@ -583,17 +712,16 @@ BJson::ParseString(JsonParseContext& jsonParseContext,
 			case '"':
 			{
 					// terminates the string assembled so far.
+				assemblyBuffer->AppendCharacter(0);
 				jsonParseContext.Listener()->Handle(
-					BJsonEvent(eventType, stringResult.String()));
+					BJsonEvent(eventType, assemblyBuffer->Buffer()));
 				return true;
 			}
 
 			case '\\':
 			{
-				if (!ParseStringEscapeSequence(jsonParseContext,
+				if (!ParseStringEscapeSequence(jsonParseContext))
-					stringResult)) {
 					return false;
-				}
 				break;
 			}
 
@@ -611,7 +739,7 @@ BJson::ParseString(JsonParseContext& jsonParseContext,
 					return false;
 				}
 
-				stringResult.Append(&c, 1);
+				assemblyBuffer->AppendCharacter(c);
 				break;
 			}
 		}
@@ -671,47 +799,47 @@ BJson::ParseExpectedVerbatimString(JsonParseContext& jsonParseContext,
 */
 
 bool
-BJson::IsValidNumber(BString& number)
+BJson::IsValidNumber(const char* value)
 {
 	int32 offset = 0;
-	int32 len = number.Length();
+	int32 len = strlen(value);
 
-	if (offset < len && number[offset] == '-')
+	if (offset < len && value[offset] == '-')
 		offset++;
 
 	if (offset >= len)
 		return false;
 
-	if (isdigit(number[offset]) && number[offset] != '0') {
+	if (isdigit(value[offset]) && value[offset] != '0') {
-		while (offset < len && isdigit(number[offset]))
+		while (offset < len && isdigit(value[offset]))
 			offset++;
 	} else {
-		if (number[offset] == '0')
+		if (value[offset] == '0')
 			offset++;
 		else
 			return false;
 	}
 
-	if (offset < len && number[offset] == '.') {
+	if (offset < len && value[offset] == '.') {
 		offset++;
 
 		if (offset >= len)
 			return false;
 
-		while (offset < len && isdigit(number[offset]))
+		while (offset < len && isdigit(value[offset]))
 			offset++;
 	}
 
-	if (offset < len && (number[offset] == 'E' || number[offset] == 'e')) {
+	if (offset < len && (value[offset] == 'E' || value[offset] == 'e')) {
 		offset++;
 
-		if(offset < len && (number[offset] == '+' || number[offset] == '-'))
+		if(offset < len && (value[offset] == '+' || value[offset] == '-'))
 		 	offset++;
 
 		if (offset >= len)
 			return false;
 
-		while (offset < len && isdigit(number[offset]))
+		while (offset < len && isdigit(value[offset]))
 			offset++;
 	}
 
@@ -728,7 +856,8 @@ BJson::IsValidNumber(BString& number)
 bool
 BJson::ParseNumber(JsonParseContext& jsonParseContext)
 {
-	BString value;
+	JsonParseAssemblyBuffer* assemblyBuffer = jsonParseContext.AssemblyBuffer();
+	JsonParseAssemblyBufferResetter assembleBufferResetter(assemblyBuffer);
 
 	while (true) {
 		char c;
@@ -737,13 +866,8 @@ BJson::ParseNumber(JsonParseContext& jsonParseContext)
 		switch (result) {
 			case B_OK:
 			{
-				if (isdigit(c)) {
+				if (isdigit(c) || c == '.' || c == '-' || c == 'e' || c == 'E' || c == '+') {
-					value += c;
+					assemblyBuffer->AppendCharacter(c);
-					break;
-				}
-
-				if (NULL != strchr("+-eE.", c)) {
-					value += c;
 					break;
 				}
 
@@ -753,15 +877,16 @@ BJson::ParseNumber(JsonParseContext& jsonParseContext)
 			case B_PARTIAL_READ:
 			{
 				errno = 0;
+				assemblyBuffer->AppendCharacter(0);
 
-				if (!IsValidNumber(value)) {
+				if (!IsValidNumber(assemblyBuffer->Buffer())) {
 					jsonParseContext.Listener()->HandleError(B_BAD_DATA,
 						jsonParseContext.LineNumber(), "malformed number");
 					return false;
 				}
 
 				jsonParseContext.Listener()->Handle(BJsonEvent(B_JSON_NUMBER,
-					value.String()));
+					assemblyBuffer->Buffer()));
 
 				return true;
 			}

src/tests/kits/shared/ChecksumJsonEventListener.cpp

@@ -0,0 +1,87 @@
+/*
+ * Copyright 2023, Andrew Lindesay <apl@lindesay.co.nz>.
+ * All rights reserved. Distributed under the terms of the MIT License.
+ */
+
+
+#include "ChecksumJsonEventListener.h"
+
+
+ChecksumJsonEventListener::ChecksumJsonEventListener(int32 checksumLimit)
+	:
+	fChecksum(0),
+	fChecksumLimit(checksumLimit),
+	fError(B_OK),
+	fCompleted(false)
+{
+}
+
+
+ChecksumJsonEventListener::~ChecksumJsonEventListener()
+{
+}
+
+
+bool
+ChecksumJsonEventListener::Handle(const BJsonEvent& event)
+{
+	if (fCompleted || B_OK != fError)
+		return false;
+
+	switch (event.EventType()) {
+		case B_JSON_NUMBER:
+		{
+			const char* content = event.Content();
+			_ChecksumProcessCharacters(content, strlen(content));
+			break;
+		}
+		case B_JSON_STRING:
+		case B_JSON_OBJECT_NAME:
+		{
+			const char* content = event.Content();
+			_ChecksumProcessCharacters(content, strlen(content));
+			break;
+		}
+		default:
+			break;
+	}
+
+	return true;
+}
+
+
+void
+ChecksumJsonEventListener::HandleError(status_t status, int32 line, const char* message)
+{
+	fError = status;
+}
+
+
+void
+ChecksumJsonEventListener::Complete()
+{
+	fCompleted = true;
+}
+
+
+uint32
+ChecksumJsonEventListener::Checksum() const
+{
+	return fChecksum;
+}
+
+
+status_t
+ChecksumJsonEventListener::Error() const
+{
+	return fError;
+}
+
+
+void
+ChecksumJsonEventListener::_ChecksumProcessCharacters(const char* content, size_t len)
+{
+	for (size_t i = 0; i < len; i++) {
+		fChecksum = (fChecksum + static_cast<int32>(content[i])) % fChecksumLimit;
+	}
+}

src/tests/kits/shared/ChecksumJsonEventListener.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2023, Andrew Lindesay <apl@lindesay.co.nz>.
+ * All rights reserved. Distributed under the terms of the MIT License.
+ */
+#ifndef CHECKSUM_JSON_EVENT_LISTENER_H
+#define CHECKSUM_JSON_EVENT_LISTENER_H
+
+
+#include <JsonEventListener.h>
+
+
+/*!	This class can be used by a text to accept JSON events and then to maintain a checksum of the
+	strings and numbers that it encounters in order to get some sort of a checksum on the data
+	that it is seeing. It can then compare this to the data that was emitted in order to verify
+	that the JSON parser has parsed and passed-through all of the data correctly.
+*/
+
+class ChecksumJsonEventListener : public BJsonEventListener {
+public:
+								ChecksumJsonEventListener(int32 checksumLimit);
+	virtual						~ChecksumJsonEventListener();
+
+	virtual bool				Handle(const BJsonEvent& event);
+	virtual void				HandleError(status_t status, int32 line, const char* message);
+	virtual void				Complete();
+
+			uint32				Checksum() const;
+			status_t			Error() const;
+
+private:
+			void				_ChecksumProcessCharacters(const char* content, size_t len);
+
+private:
+			uint32				fChecksum;
+			uint32				fChecksumLimit;
+			status_t			fError;
+			bool				fCompleted;
+};
+
+#endif // CHECKSUM_JSON_EVENT_LISTENER_H

src/tests/kits/shared/FakeJsonDataGenerator.cpp

@@ -0,0 +1,215 @@
+/*
+ * Copyright 2023, Andrew Lindesay <apl@lindesay.co.nz>.
+ * All rights reserved. Distributed under the terms of the MIT License.
+ */
+#include "FakeJsonDataGenerator.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <strings.h>
+#include <string.h>
+
+
+static const char* kTextData = "abcdefghijklmnopqrstuvwxyz!@#$"
+	"ABCDEFGHIJKLMNOPQRSTUVWXYZ)(*&"
+	"0123456789{}[]|:;'<>,.?/~`_+-="
+	"abcdefghijklmnopqrstuvwxyz!@#$"
+	"ABCDEFGHIJKLMNOPQRSTUVWXYZ)(*&"
+	"0123456789{}[]|:;'<>,.?/~`_+-=";
+static const int kTextDataLength = 180;
+
+
+FakeJsonStreamDataIO::FakeJsonStreamDataIO(int count, uint32 checksumLimit)
+	:
+	fFeedOutState(OPEN_ARRAY),
+	fItemCount(count),
+	fItemUpto(0),
+	fChecksum(0),
+	fChecksumLimit(checksumLimit)
+{
+}
+
+
+FakeJsonStreamDataIO::~FakeJsonStreamDataIO()
+{
+}
+
+
+ssize_t
+FakeJsonStreamDataIO::Read(void* buffer, size_t size)
+{
+	char* buffer_c = static_cast<char*>(buffer);
+	status_t result = B_OK;
+	size_t i = 0;
+
+	while (i < size && result == B_OK) {
+		result = NextChar(&buffer_c[i]);
+		if (result == B_OK)
+			i++;
+	}
+
+	if(0 != i)
+		return i;
+
+	return result;
+}
+
+
+ssize_t
+FakeJsonStreamDataIO::Write(const void* buffer, size_t size)
+{
+	return B_NOT_SUPPORTED;
+}
+
+
+status_t
+FakeJsonStreamDataIO::Flush()
+{
+	return B_OK;
+}
+
+
+uint32
+FakeJsonStreamDataIO::Checksum() const
+{
+	return fChecksum;
+}
+
+
+void
+FakeJsonStreamDataIO::_ChecksumProcessCharacter(const char c)
+{
+	fChecksum = (fChecksum + static_cast<int32>(c)) % fChecksumLimit;
+}
+
+
+// #pragma mark - FakeJsonStringStreamDataIO
+
+
+FakeJsonStringStreamDataIO::FakeJsonStringStreamDataIO(int count, uint32 checksumLimit)
+	:
+	FakeJsonStreamDataIO(count, checksumLimit),
+	fItemBufferSize(0),
+	fItemBufferUpto(0)
+{
+	FillBuffer();
+}
+
+
+FakeJsonStringStreamDataIO::~FakeJsonStringStreamDataIO()
+{
+}
+
+
+status_t
+FakeJsonStringStreamDataIO::NextChar(char* c)
+{
+	switch (fFeedOutState) {
+		case OPEN_ARRAY:
+			c[0] = '[';
+			fFeedOutState = OPEN_QUOTE;
+			return B_OK;
+		case OPEN_QUOTE:
+			c[0] = '"';
+			fFeedOutState = ITEM;
+			return B_OK;
+		case ITEM:
+			c[0] = kTextData[fItemBufferUpto];
+			_ChecksumProcessCharacter(kTextData[fItemBufferUpto]);
+			fItemBufferUpto++;
+			if (fItemBufferUpto >= fItemBufferSize) {
+				fFeedOutState = CLOSE_QUOTE;
+				FillBuffer();
+			}
+			return B_OK;
+		case CLOSE_QUOTE:
+			c[0] = '"';
+			fItemUpto++;
+			if (fItemUpto >= fItemCount)
+				fFeedOutState = CLOSE_ARRAY;
+			else
+				fFeedOutState = SEPARATOR;
+			return B_OK;
+		case SEPARATOR:
+			c[0] = ',';
+			fFeedOutState = OPEN_QUOTE;
+			return B_OK;
+		case CLOSE_ARRAY:
+			c[0] = ']';
+			fFeedOutState = END;
+			return B_OK;
+		default:
+			return -1; // end of file
+	}
+}
+
+
+void
+FakeJsonStringStreamDataIO::FillBuffer()
+{
+	fItemBufferSize = random() % kTextDataLength;
+	fItemBufferUpto = 0;
+}
+
+
+// #pragma mark - FakeJsonStringStreamDataIO
+
+
+FakeJsonNumberStreamDataIO::FakeJsonNumberStreamDataIO(int count, uint32 checksumLimit)
+	:
+	FakeJsonStreamDataIO(count, checksumLimit),
+	fItemBufferSize(0),
+	fItemBufferUpto(0)
+{
+	bzero(fBuffer, 32);
+	FillBuffer();
+}
+
+
+FakeJsonNumberStreamDataIO::~FakeJsonNumberStreamDataIO()
+{
+}
+
+
+status_t
+FakeJsonNumberStreamDataIO::NextChar(char* c)
+{
+	switch (fFeedOutState) {
+		case OPEN_ARRAY:
+			c[0] = '[';
+			fFeedOutState = ITEM;
+			return B_OK;
+		case ITEM:
+			c[0] = fBuffer[fItemBufferUpto];
+			_ChecksumProcessCharacter(fBuffer[fItemBufferUpto]);
+			fItemBufferUpto++;
+			if (fItemBufferUpto >= fItemBufferSize) {
+				fItemUpto++;
+				if (fItemUpto >= fItemCount)
+					fFeedOutState = CLOSE_ARRAY;
+				else
+					fFeedOutState = SEPARATOR;
+				FillBuffer();
+			}
+			return B_OK;
+		case SEPARATOR:
+			c[0] = ',';
+			fFeedOutState = ITEM;
+			return B_OK;
+		case CLOSE_ARRAY:
+			c[0] = ']';
+			fFeedOutState = END;
+			return B_OK;
+		default:
+			return -1; // end of file
+	}
+}
+
+
+void
+FakeJsonNumberStreamDataIO::FillBuffer()
+{
+	int32 value = static_cast<int32>(random());
+	fItemBufferSize = snprintf(fBuffer, 32, "%" B_PRIu32, value);
+	fItemBufferUpto = 0;
+}

src/tests/kits/shared/FakeJsonDataGenerator.h

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2023, Andrew Lindesay <apl@lindesay.co.nz>.
+ * All rights reserved. Distributed under the terms of the MIT License.
+ */
+#ifndef FAKE_JSON_DATA_GENERATOR_H
+#define FAKE_JSON_DATA_GENERATOR_H
+
+
+#include <DataIO.h>
+
+
+typedef enum FeedOutState {
+	OPEN_ARRAY,
+	OPEN_QUOTE, // only used for strings
+	ITEM,
+	CLOSE_QUOTE, // only used for strings
+	SEPARATOR,
+	CLOSE_ARRAY,
+	END
+} FeedOutState;
+
+
+class FakeJsonStreamDataIO : public BDataIO {
+public:
+								FakeJsonStreamDataIO(int count, uint32 checksumLimit);
+	virtual						~FakeJsonStreamDataIO();
+
+	virtual	ssize_t				Read(void* buffer, size_t size);
+	virtual	ssize_t				Write(const void* buffer, size_t size);
+
+	virtual	status_t			Flush();
+
+			uint32				Checksum() const;
+
+protected:
+	virtual	void				FillBuffer() = 0;
+	virtual	status_t			NextChar(char* c) = 0;
+
+			void				_ChecksumProcessCharacter(const char c);
+
+protected:
+			FeedOutState		fFeedOutState;
+			int					fItemCount;
+			int					fItemUpto;
+
+private:
+			uint32				fChecksum;
+			uint32				fChecksumLimit;
+};
+
+
+class FakeJsonStringStreamDataIO : public FakeJsonStreamDataIO {
+public:
+								FakeJsonStringStreamDataIO(int count, uint32 checksumLimit);
+	virtual						~FakeJsonStringStreamDataIO();
+
+protected:
+	virtual	void				FillBuffer();
+			status_t			NextChar(char* c);
+
+protected:
+			int					fItemBufferSize;
+			int					fItemBufferUpto;
+};
+
+
+class FakeJsonNumberStreamDataIO : public FakeJsonStreamDataIO {
+public:
+								FakeJsonNumberStreamDataIO(int count, uint32 checksumLimit);
+	virtual						~FakeJsonNumberStreamDataIO();
+
+protected:
+	virtual	void				FillBuffer();
+			status_t			NextChar(char* c);
+
+protected:
+			int					fItemBufferSize;
+			int					fItemBufferUpto;
+			char				fBuffer[32];
+};
+
+
+#endif // FAKE_JSON_DATA_GENERATOR_H

src/tests/kits/shared/Jamfile

@@ -8,7 +8,9 @@ UnitTestLib libsharedtest.so :
 	SharedTestAddon.cpp
 
 	CalendarViewTest.cpp
+	ChecksumJsonEventListener.cpp
 	DriverSettingsMessageAdapterTest.cpp
+	FakeJsonDataGenerator.cpp
 	JsonEndToEndTest.cpp
 	JsonErrorHandlingTest.cpp
 	JsonTextWriterTest.cpp

src/tests/kits/shared/JsonEndToEndTest.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>
+ * Copyright 2017-2023, Andrew Lindesay <apl@lindesay.co.nz>
  * Distributed under the terms of the MIT License.
  */
 #include "JsonEndToEndTest.h"
@@ -12,12 +12,18 @@
 #include <cppunit/TestCaller.h>
 #include <cppunit/TestSuite.h>
 
+#include "ChecksumJsonEventListener.h"
+#include "FakeJsonDataGenerator.h"
 #include "JsonSamples.h"
 
 
 using namespace BPrivate;
 
 
+static const size_t kHighVolumeItemCount = 10000;
+static const uint32 kChecksumLimit = 100000;
+
+
 JsonEndToEndTest::JsonEndToEndTest()
 {
 }
@@ -28,6 +34,70 @@ JsonEndToEndTest::~JsonEndToEndTest()
 }
 
 
+/*! Just here so it is possible to extract timings for the data generation cost.
+*/
+
+void
+JsonEndToEndTest::TestHighVolumeStringSampleGenerationOnly()
+{
+	FakeJsonStreamDataIO* inputData = new FakeJsonStringStreamDataIO(kHighVolumeItemCount,
+		kChecksumLimit);
+	char c;
+
+	while (inputData->Read(&c, 1) == 1) {
+		// do nothing
+	}
+}
+
+
+/*! Just here so it is possible to extract timings for the data generation cost.
+*/
+
+void
+JsonEndToEndTest::TestHighVolumeNumberSampleGenerationOnly()
+{
+	FakeJsonStreamDataIO* inputData = new FakeJsonNumberStreamDataIO(kHighVolumeItemCount,
+		kChecksumLimit);
+	char c;
+
+	while (inputData->Read(&c, 1) == 1) {
+		// do nothing
+	}
+}
+
+
+void
+JsonEndToEndTest::TestHighVolumeStringParsing()
+{
+	FakeJsonStreamDataIO* inputData = new FakeJsonStringStreamDataIO(kHighVolumeItemCount,
+		kChecksumLimit);
+	ChecksumJsonEventListener* listener = new ChecksumJsonEventListener(kChecksumLimit);
+
+	// ----------------------
+    BPrivate::BJson::Parse(inputData, listener);
+    // ----------------------
+
+	CPPUNIT_ASSERT_EQUAL(B_OK, listener->Error());
+	CPPUNIT_ASSERT_EQUAL(inputData->Checksum(), listener->Checksum());
+}
+
+
+void
+JsonEndToEndTest::TestHighVolumeNumberParsing()
+{
+	FakeJsonStreamDataIO* inputData = new FakeJsonNumberStreamDataIO(kHighVolumeItemCount,
+		kChecksumLimit);
+	ChecksumJsonEventListener* listener = new ChecksumJsonEventListener(kChecksumLimit);
+
+	// ----------------------
+    BPrivate::BJson::Parse(inputData, listener);
+    // ----------------------
+
+	CPPUNIT_ASSERT_EQUAL(B_OK, listener->Error());
+	CPPUNIT_ASSERT_EQUAL(inputData->Checksum(), listener->Checksum());
+}
+
+
 void
 JsonEndToEndTest::TestParseAndWrite(const char* input, const char* expectedOutput)
 {
@@ -208,8 +278,17 @@ JsonEndToEndTest::AddTests(BTestSuite& parent)
 		"JsonEndToEndTest::TestArrayUnterminated",
 		&JsonEndToEndTest::TestArrayUnterminated));
 	suite.addTest(new CppUnit::TestCaller<JsonEndToEndTest>(
-		"JsonEndToEndTest::TestObjectUnterminated",
+		"JsonEndToEndTest::TestHighVolumeStringParsing",
-		&JsonEndToEndTest::TestObjectUnterminated));
+		&JsonEndToEndTest::TestHighVolumeStringParsing));
+	suite.addTest(new CppUnit::TestCaller<JsonEndToEndTest>(
+		"JsonEndToEndTest::TestHighVolumeNumberParsing",
+		&JsonEndToEndTest::TestHighVolumeNumberParsing));
+	suite.addTest(new CppUnit::TestCaller<JsonEndToEndTest>(
+		"JsonEndToEndTest::TestHighVolumeStringSampleGenerationOnly",
+		&JsonEndToEndTest::TestHighVolumeStringSampleGenerationOnly));
+	suite.addTest(new CppUnit::TestCaller<JsonEndToEndTest>(
+		"JsonEndToEndTest::TestHighVolumeNumberSampleGenerationOnly",
+		&JsonEndToEndTest::TestHighVolumeNumberSampleGenerationOnly));
 
 	parent.addTest("JsonEndToEndTest", &suite);
 }

src/tests/kits/shared/JsonEndToEndTest.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2017, Andrew Lindesay <apl@lindesay.co.nz>
+ * Copyright 2017-2023, Andrew Lindesay <apl@lindesay.co.nz>
  * Distributed under the terms of the MIT License.
  */
 #ifndef JSON_END_TO_END_TEST_H
@@ -18,6 +18,11 @@ public:
 								JsonEndToEndTest();
 	virtual						~JsonEndToEndTest();
 
+			void				TestHighVolumeStringParsing();
+			void				TestHighVolumeNumberParsing();
+			void				TestHighVolumeStringSampleGenerationOnly();
+			void				TestHighVolumeNumberSampleGenerationOnly();
+
 			void				TestNullA();
 			void				TestTrueA();
 			void				TestFalseA();