From 35c4600e215ec42e39fe5fbaeda10a9f4e2d8884 Mon Sep 17 00:00:00 2001 From: Andrew Lindesay Date: Mon, 13 Nov 2023 23:35:19 +1300 Subject: [PATCH] Shared: JSON Parse Perf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Improve the mechanics for JSON parsing by reusing text buffers during the parse. Change-Id: I7fb2cae31e6558a5a0c63fd02e1fc6fec4f9e4b3 Reviewed-on: https://review.haiku-os.org/c/haiku/+/7106 Tested-by: Commit checker robot Reviewed-by: Jérôme Duval --- headers/private/shared/Json.h | 14 +- src/kits/shared/Json.cpp | 325 ++++++++++++------ .../kits/shared/ChecksumJsonEventListener.cpp | 87 +++++ .../kits/shared/ChecksumJsonEventListener.h | 40 +++ .../kits/shared/FakeJsonDataGenerator.cpp | 215 ++++++++++++ src/tests/kits/shared/FakeJsonDataGenerator.h | 83 +++++ src/tests/kits/shared/Jamfile | 2 + src/tests/kits/shared/JsonEndToEndTest.cpp | 85 ++++- src/tests/kits/shared/JsonEndToEndTest.h | 7 +- 9 files changed, 746 insertions(+), 112 deletions(-) create mode 100644 src/tests/kits/shared/ChecksumJsonEventListener.cpp create mode 100644 src/tests/kits/shared/ChecksumJsonEventListener.h create mode 100644 src/tests/kits/shared/FakeJsonDataGenerator.cpp create mode 100644 src/tests/kits/shared/FakeJsonDataGenerator.h diff --git a/headers/private/shared/Json.h b/headers/private/shared/Json.h index 3ec9c4dee8..55985f1117 100644 --- a/headers/private/shared/Json.h +++ b/headers/private/shared/Json.h @@ -1,5 +1,5 @@ /* - * Copyright 2017, Andrew Lindesay + * Copyright 2017-2023, Andrew Lindesay * Copyright 2014, Augustin Cavalier (waddlesplash) * Distributed under the terms of the MIT License. */ @@ -39,14 +39,12 @@ private: JsonParseContext& jsonParseContext); static bool ParseObject(JsonParseContext& jsonParseContext); static bool ParseArray(JsonParseContext& jsonParseContext); - static bool ParseEscapeUnicodeSequence( - JsonParseContext& jsonParseContext, - BString& stringResult); - static bool ParseStringEscapeSequence( - JsonParseContext& jsonParseContext, - BString& stringResult); + + static bool ParseEscapeUnicodeSequence(JsonParseContext& jsonParseContext); + static bool ParseStringEscapeSequence(JsonParseContext& jsonParseContext); static bool ParseString(JsonParseContext& jsonParseContext, json_event_type eventType); + static bool ParseExpectedVerbatimStringAndRaiseEvent( JsonParseContext& jsonParseContext, const char* expectedString, @@ -59,7 +57,7 @@ private: size_t expectedStringLength, char leadingChar); - static bool IsValidNumber(BString& number); + static bool IsValidNumber(const char* value); static bool ParseNumber(JsonParseContext& jsonParseContext); }; diff --git a/src/kits/shared/Json.cpp b/src/kits/shared/Json.cpp index 14c4bcf950..83390e8f8b 100644 --- a/src/kits/shared/Json.cpp +++ b/src/kits/shared/Json.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2017, Andrew Lindesay + * Copyright 2017-2023, Andrew Lindesay * Copyright 2014-2017, Augustin Cavalier (waddlesplash) * Copyright 2014, Stephan Aßmus * Distributed under the terms of the MIT License. @@ -25,26 +25,151 @@ namespace BPrivate { +/*! A buffer is used to assemble strings into. This will be the initial size + of this buffer. +*/ -static bool -b_jsonparse_is_hex(char c) -{ - return isdigit(c) - || (c > 0x41 && c <= 0x46) - || (c > 0x61 && c <= 0x66); -} +static const size_t kInitialAssemblyBufferSize = 64; + +/*! A buffer is used to assemble strings into. This buffer starts off small + but is able to grow as the string it needs to process as encountered. To + avoid frequent reallocation of the buffer, the buffer will be retained + between strings. This is the maximum size of buffer that will be retained. +*/ + +static const size_t kRetainedAssemblyBufferSize = 32 * 1024; + +static const size_t kAssemblyBufferSizeIncrement = 256; + +static const size_t kMaximumUtf8SequenceLength = 7; -static bool -b_jsonparse_all_hex(const char* c) -{ - for (int i = 0; i < 4; i++) { - if (!b_jsonparse_is_hex(c[i])) - return false; +class JsonParseAssemblyBuffer { +public: + JsonParseAssemblyBuffer() + : + fAssemblyBuffer(NULL), + fAssemblyBufferAllocatedSize(0), + fAssemblyBufferUsedSize(0) + { + fAssemblyBuffer = (char*) malloc(kInitialAssemblyBufferSize); + if (fAssemblyBuffer != NULL) + fAssemblyBufferAllocatedSize = kInitialAssemblyBufferSize; } - return true; -} + ~JsonParseAssemblyBuffer() + { + if (fAssemblyBuffer != NULL) + free(fAssemblyBuffer); + } + + const char* Buffer() const + { + return fAssemblyBuffer; + } + + /*! This method should be used each time that the assembly buffer has + been finished with by some section of logic. + */ + + status_t Reset() + { + fAssemblyBufferUsedSize = 0; + + if (fAssemblyBufferAllocatedSize > kRetainedAssemblyBufferSize) { + fAssemblyBuffer = (char*) realloc(fAssemblyBuffer, kRetainedAssemblyBufferSize); + if (fAssemblyBuffer == NULL) { + fAssemblyBufferAllocatedSize = 0; + return B_NO_MEMORY; + } + fAssemblyBufferAllocatedSize = kRetainedAssemblyBufferSize; + } + + return B_OK; + } + + status_t AppendCharacter(char c) + { + status_t result = _EnsureAssemblyBufferAllocatedSize(fAssemblyBufferUsedSize + 1); + + if (result == B_OK) { + fAssemblyBuffer[fAssemblyBufferUsedSize] = c; + fAssemblyBufferUsedSize++; + } + + return result; + } + + status_t AppendCharacters(char* str, size_t len) + { + status_t result = _EnsureAssemblyBufferAllocatedSize(fAssemblyBufferUsedSize + len); + + if (result == B_OK) { + memcpy(&fAssemblyBuffer[fAssemblyBufferUsedSize], str, len); + fAssemblyBufferUsedSize += len; + } + + return result; + } + + status_t AppendUnicodeCharacter(uint32 c) + { + status_t result = _EnsureAssemblyBufferAllocatedSize( + fAssemblyBufferUsedSize + kMaximumUtf8SequenceLength); + if (result == B_OK) { + char* insertPtr = &fAssemblyBuffer[fAssemblyBufferUsedSize]; + char* ptr = insertPtr; + BUnicodeChar::ToUTF8(c, &ptr); + size_t sequenceLength = static_cast(ptr - insertPtr); + fAssemblyBufferUsedSize += sequenceLength; + } + + return result; + } + +private: + + /*! This method will return the assembly buffer ensuring that it has at + least `minimumSize` bytes available. + */ + + status_t _EnsureAssemblyBufferAllocatedSize(size_t minimumSize) + { + if (fAssemblyBufferAllocatedSize < minimumSize) { + fAssemblyBuffer = (char*) realloc(fAssemblyBuffer, minimumSize); + if (fAssemblyBuffer == NULL) { + fAssemblyBufferAllocatedSize = 0; + return B_NO_MEMORY; + } + fAssemblyBufferAllocatedSize = minimumSize; + } + return B_OK; + } + +private: + char* fAssemblyBuffer; + size_t fAssemblyBufferAllocatedSize; + size_t fAssemblyBufferUsedSize; +}; + + +class JsonParseAssemblyBufferResetter { +public: + JsonParseAssemblyBufferResetter(JsonParseAssemblyBuffer* assemblyBuffer) + : + fAssemblyBuffer(assemblyBuffer) + { + } + + ~JsonParseAssemblyBufferResetter() + { + fAssemblyBuffer->Reset(); + } + +private: + JsonParseAssemblyBuffer* + fAssemblyBuffer; +}; /*! This class carries state around the parsing process. */ @@ -57,11 +182,18 @@ public: fData(data), fLineNumber(1), // 1 is the first line fPushbackChar(0), - fHasPushbackChar(false) + fHasPushbackChar(false), + fAssemblyBuffer(new JsonParseAssemblyBuffer()) { } + ~JsonParseContext() + { + delete fAssemblyBuffer; + } + + BJsonEventListener* Listener() const { return fListener; @@ -85,14 +217,8 @@ public: fLineNumber++; } - - // TODO; there is considerable opportunity for performance improvements - // here by buffering the input and then feeding it into the parse - // algorithm character by character. - status_t NextChar(char* buffer) { - if (fHasPushbackChar) { buffer[0] = fPushbackChar; fHasPushbackChar = false; @@ -102,19 +228,27 @@ public: return Data()->ReadExactly(buffer, 1); } - void PushbackChar(char c) { fPushbackChar = c; fHasPushbackChar = true; } + + JsonParseAssemblyBuffer* AssemblyBuffer() + { + return fAssemblyBuffer; + } + + private: BJsonEventListener* fListener; BDataIO* fData; uint32 fLineNumber; char fPushbackChar; bool fHasPushbackChar; + JsonParseAssemblyBuffer* + fAssemblyBuffer; }; @@ -464,90 +598,86 @@ BJson::ParseArray(JsonParseContext& jsonParseContext) bool -BJson::ParseEscapeUnicodeSequence(JsonParseContext& jsonParseContext, - BString& stringResult) +BJson::ParseEscapeUnicodeSequence(JsonParseContext& jsonParseContext) { - char buffer[5]; - buffer[4] = 0; + char ch; + uint32 unicodeCh = 0; - if (!NextChar(jsonParseContext, &buffer[0]) - || !NextChar(jsonParseContext, &buffer[1]) - || !NextChar(jsonParseContext, &buffer[2]) - || !NextChar(jsonParseContext, &buffer[3])) { - return false; + for (int i = 3; i >= 0; i--) { + if (!NextChar(jsonParseContext, &ch)) { + jsonParseContext.Listener()->HandleError(B_ERROR, jsonParseContext.LineNumber(), + "unable to read unicode sequence"); + return false; + } + + if (ch >= '0' && ch <= '9') + unicodeCh |= static_cast(ch - '0') << (i * 4); + else if (ch >= 'a' && ch <= 'f') + unicodeCh |= (10 + static_cast(ch - 'a')) << (i * 4); + else if (ch >= 'A' && ch <= 'F') + unicodeCh |= (10 + static_cast(ch - 'A')) << (i * 4); + else { + BString errorMessage; + errorMessage.SetToFormat( + "malformed hex character [%c] in unicode sequence in string parsing", ch); + jsonParseContext.Listener()->HandleError(B_BAD_DATA, jsonParseContext.LineNumber(), + errorMessage.String()); + return false; + } } - if (!b_jsonparse_all_hex(buffer)) { - BString errorMessage; - errorMessage.SetToFormat( - "malformed unicode sequence [%s] in string parsing", - buffer); - jsonParseContext.Listener()->HandleError(B_BAD_DATA, - jsonParseContext.LineNumber(), errorMessage.String()); + JsonParseAssemblyBuffer* assemblyBuffer = jsonParseContext.AssemblyBuffer(); + status_t result = assemblyBuffer->AppendUnicodeCharacter(unicodeCh); + + if (result != B_OK) { + jsonParseContext.Listener()->HandleError(result, jsonParseContext.LineNumber(), + "unable to store unicode char as utf-8"); return false; } - uint intValue; - - if (sscanf(buffer, "%4x", &intValue) != 1) { - BString errorMessage; - errorMessage.SetToFormat( - "unable to process unicode sequence [%s] in string " - " parsing", buffer); - jsonParseContext.Listener()->HandleError(B_BAD_DATA, - jsonParseContext.LineNumber(), errorMessage.String()); - return false; - } - - char character[7]; - char* ptr = character; - BUnicodeChar::ToUTF8(intValue, &ptr); - int32 sequenceLength = ptr - character; - stringResult.Append(character, sequenceLength); - return true; } bool -BJson::ParseStringEscapeSequence(JsonParseContext& jsonParseContext, - BString& stringResult) +BJson::ParseStringEscapeSequence(JsonParseContext& jsonParseContext) { char c; if (!NextChar(jsonParseContext, &c)) return false; + JsonParseAssemblyBuffer* assemblyBuffer = jsonParseContext.AssemblyBuffer(); + switch (c) { case 'n': - stringResult += "\n"; + assemblyBuffer->AppendCharacter('\n'); break; case 'r': - stringResult += "\r"; + assemblyBuffer->AppendCharacter('\r'); break; case 'b': - stringResult += "\b"; + assemblyBuffer->AppendCharacter('\b'); break; case 'f': - stringResult += "\f"; + assemblyBuffer->AppendCharacter('\f'); break; case '\\': - stringResult += "\\"; + assemblyBuffer->AppendCharacter('\\'); break; case '/': - stringResult += "/"; + assemblyBuffer->AppendCharacter('/'); break; case 't': - stringResult += "\t"; + assemblyBuffer->AppendCharacter('\t'); break; case '"': - stringResult += "\""; + assemblyBuffer->AppendCharacter('"'); break; case 'u': { // unicode escape sequence. - if (!ParseEscapeUnicodeSequence(jsonParseContext, - stringResult)) { + if (!ParseEscapeUnicodeSequence(jsonParseContext)) { return false; } break; @@ -555,9 +685,7 @@ BJson::ParseStringEscapeSequence(JsonParseContext& jsonParseContext, default: { BString errorMessage; - errorMessage.SetToFormat( - "unexpected escaped character [%c] in string parsing", - c); + errorMessage.SetToFormat("unexpected escaped character [%c] in string parsing", c); jsonParseContext.Listener()->HandleError(B_BAD_DATA, jsonParseContext.LineNumber(), errorMessage.String()); return false; @@ -573,7 +701,8 @@ BJson::ParseString(JsonParseContext& jsonParseContext, json_event_type eventType) { char c; - BString stringResult; + JsonParseAssemblyBuffer* assemblyBuffer = jsonParseContext.AssemblyBuffer(); + JsonParseAssemblyBufferResetter assembleBufferResetter(assemblyBuffer); while(true) { if (!NextChar(jsonParseContext, &c)) @@ -583,17 +712,16 @@ BJson::ParseString(JsonParseContext& jsonParseContext, case '"': { // terminates the string assembled so far. + assemblyBuffer->AppendCharacter(0); jsonParseContext.Listener()->Handle( - BJsonEvent(eventType, stringResult.String())); + BJsonEvent(eventType, assemblyBuffer->Buffer())); return true; } case '\\': { - if (!ParseStringEscapeSequence(jsonParseContext, - stringResult)) { + if (!ParseStringEscapeSequence(jsonParseContext)) return false; - } break; } @@ -611,7 +739,7 @@ BJson::ParseString(JsonParseContext& jsonParseContext, return false; } - stringResult.Append(&c, 1); + assemblyBuffer->AppendCharacter(c); break; } } @@ -671,47 +799,47 @@ BJson::ParseExpectedVerbatimString(JsonParseContext& jsonParseContext, */ bool -BJson::IsValidNumber(BString& number) +BJson::IsValidNumber(const char* value) { int32 offset = 0; - int32 len = number.Length(); + int32 len = strlen(value); - if (offset < len && number[offset] == '-') + if (offset < len && value[offset] == '-') offset++; if (offset >= len) return false; - if (isdigit(number[offset]) && number[offset] != '0') { - while (offset < len && isdigit(number[offset])) + if (isdigit(value[offset]) && value[offset] != '0') { + while (offset < len && isdigit(value[offset])) offset++; } else { - if (number[offset] == '0') + if (value[offset] == '0') offset++; else return false; } - if (offset < len && number[offset] == '.') { + if (offset < len && value[offset] == '.') { offset++; if (offset >= len) return false; - while (offset < len && isdigit(number[offset])) + while (offset < len && isdigit(value[offset])) offset++; } - if (offset < len && (number[offset] == 'E' || number[offset] == 'e')) { + if (offset < len && (value[offset] == 'E' || value[offset] == 'e')) { offset++; - if(offset < len && (number[offset] == '+' || number[offset] == '-')) + if(offset < len && (value[offset] == '+' || value[offset] == '-')) offset++; if (offset >= len) return false; - while (offset < len && isdigit(number[offset])) + while (offset < len && isdigit(value[offset])) offset++; } @@ -728,7 +856,8 @@ BJson::IsValidNumber(BString& number) bool BJson::ParseNumber(JsonParseContext& jsonParseContext) { - BString value; + JsonParseAssemblyBuffer* assemblyBuffer = jsonParseContext.AssemblyBuffer(); + JsonParseAssemblyBufferResetter assembleBufferResetter(assemblyBuffer); while (true) { char c; @@ -737,13 +866,8 @@ BJson::ParseNumber(JsonParseContext& jsonParseContext) switch (result) { case B_OK: { - if (isdigit(c)) { - value += c; - break; - } - - if (NULL != strchr("+-eE.", c)) { - value += c; + if (isdigit(c) || c == '.' || c == '-' || c == 'e' || c == 'E' || c == '+') { + assemblyBuffer->AppendCharacter(c); break; } @@ -753,15 +877,16 @@ BJson::ParseNumber(JsonParseContext& jsonParseContext) case B_PARTIAL_READ: { errno = 0; + assemblyBuffer->AppendCharacter(0); - if (!IsValidNumber(value)) { + if (!IsValidNumber(assemblyBuffer->Buffer())) { jsonParseContext.Listener()->HandleError(B_BAD_DATA, jsonParseContext.LineNumber(), "malformed number"); return false; } jsonParseContext.Listener()->Handle(BJsonEvent(B_JSON_NUMBER, - value.String())); + assemblyBuffer->Buffer())); return true; } diff --git a/src/tests/kits/shared/ChecksumJsonEventListener.cpp b/src/tests/kits/shared/ChecksumJsonEventListener.cpp new file mode 100644 index 0000000000..619d402205 --- /dev/null +++ b/src/tests/kits/shared/ChecksumJsonEventListener.cpp @@ -0,0 +1,87 @@ +/* + * Copyright 2023, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ + + +#include "ChecksumJsonEventListener.h" + + +ChecksumJsonEventListener::ChecksumJsonEventListener(int32 checksumLimit) + : + fChecksum(0), + fChecksumLimit(checksumLimit), + fError(B_OK), + fCompleted(false) +{ +} + + +ChecksumJsonEventListener::~ChecksumJsonEventListener() +{ +} + + +bool +ChecksumJsonEventListener::Handle(const BJsonEvent& event) +{ + if (fCompleted || B_OK != fError) + return false; + + switch (event.EventType()) { + case B_JSON_NUMBER: + { + const char* content = event.Content(); + _ChecksumProcessCharacters(content, strlen(content)); + break; + } + case B_JSON_STRING: + case B_JSON_OBJECT_NAME: + { + const char* content = event.Content(); + _ChecksumProcessCharacters(content, strlen(content)); + break; + } + default: + break; + } + + return true; +} + + +void +ChecksumJsonEventListener::HandleError(status_t status, int32 line, const char* message) +{ + fError = status; +} + + +void +ChecksumJsonEventListener::Complete() +{ + fCompleted = true; +} + + +uint32 +ChecksumJsonEventListener::Checksum() const +{ + return fChecksum; +} + + +status_t +ChecksumJsonEventListener::Error() const +{ + return fError; +} + + +void +ChecksumJsonEventListener::_ChecksumProcessCharacters(const char* content, size_t len) +{ + for (size_t i = 0; i < len; i++) { + fChecksum = (fChecksum + static_cast(content[i])) % fChecksumLimit; + } +} diff --git a/src/tests/kits/shared/ChecksumJsonEventListener.h b/src/tests/kits/shared/ChecksumJsonEventListener.h new file mode 100644 index 0000000000..fb75f41abe --- /dev/null +++ b/src/tests/kits/shared/ChecksumJsonEventListener.h @@ -0,0 +1,40 @@ +/* + * Copyright 2023, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ +#ifndef CHECKSUM_JSON_EVENT_LISTENER_H +#define CHECKSUM_JSON_EVENT_LISTENER_H + + +#include + + +/*! This class can be used by a text to accept JSON events and then to maintain a checksum of the + strings and numbers that it encounters in order to get some sort of a checksum on the data + that it is seeing. It can then compare this to the data that was emitted in order to verify + that the JSON parser has parsed and passed-through all of the data correctly. +*/ + +class ChecksumJsonEventListener : public BJsonEventListener { +public: + ChecksumJsonEventListener(int32 checksumLimit); + virtual ~ChecksumJsonEventListener(); + + virtual bool Handle(const BJsonEvent& event); + virtual void HandleError(status_t status, int32 line, const char* message); + virtual void Complete(); + + uint32 Checksum() const; + status_t Error() const; + +private: + void _ChecksumProcessCharacters(const char* content, size_t len); + +private: + uint32 fChecksum; + uint32 fChecksumLimit; + status_t fError; + bool fCompleted; +}; + +#endif // CHECKSUM_JSON_EVENT_LISTENER_H diff --git a/src/tests/kits/shared/FakeJsonDataGenerator.cpp b/src/tests/kits/shared/FakeJsonDataGenerator.cpp new file mode 100644 index 0000000000..c3754b4e38 --- /dev/null +++ b/src/tests/kits/shared/FakeJsonDataGenerator.cpp @@ -0,0 +1,215 @@ +/* + * Copyright 2023, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ +#include "FakeJsonDataGenerator.h" + +#include +#include +#include +#include + + +static const char* kTextData = "abcdefghijklmnopqrstuvwxyz!@#$" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ)(*&" + "0123456789{}[]|:;'<>,.?/~`_+-=" + "abcdefghijklmnopqrstuvwxyz!@#$" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ)(*&" + "0123456789{}[]|:;'<>,.?/~`_+-="; +static const int kTextDataLength = 180; + + +FakeJsonStreamDataIO::FakeJsonStreamDataIO(int count, uint32 checksumLimit) + : + fFeedOutState(OPEN_ARRAY), + fItemCount(count), + fItemUpto(0), + fChecksum(0), + fChecksumLimit(checksumLimit) +{ +} + + +FakeJsonStreamDataIO::~FakeJsonStreamDataIO() +{ +} + + +ssize_t +FakeJsonStreamDataIO::Read(void* buffer, size_t size) +{ + char* buffer_c = static_cast(buffer); + status_t result = B_OK; + size_t i = 0; + + while (i < size && result == B_OK) { + result = NextChar(&buffer_c[i]); + if (result == B_OK) + i++; + } + + if(0 != i) + return i; + + return result; +} + + +ssize_t +FakeJsonStreamDataIO::Write(const void* buffer, size_t size) +{ + return B_NOT_SUPPORTED; +} + + +status_t +FakeJsonStreamDataIO::Flush() +{ + return B_OK; +} + + +uint32 +FakeJsonStreamDataIO::Checksum() const +{ + return fChecksum; +} + + +void +FakeJsonStreamDataIO::_ChecksumProcessCharacter(const char c) +{ + fChecksum = (fChecksum + static_cast(c)) % fChecksumLimit; +} + + +// #pragma mark - FakeJsonStringStreamDataIO + + +FakeJsonStringStreamDataIO::FakeJsonStringStreamDataIO(int count, uint32 checksumLimit) + : + FakeJsonStreamDataIO(count, checksumLimit), + fItemBufferSize(0), + fItemBufferUpto(0) +{ + FillBuffer(); +} + + +FakeJsonStringStreamDataIO::~FakeJsonStringStreamDataIO() +{ +} + + +status_t +FakeJsonStringStreamDataIO::NextChar(char* c) +{ + switch (fFeedOutState) { + case OPEN_ARRAY: + c[0] = '['; + fFeedOutState = OPEN_QUOTE; + return B_OK; + case OPEN_QUOTE: + c[0] = '"'; + fFeedOutState = ITEM; + return B_OK; + case ITEM: + c[0] = kTextData[fItemBufferUpto]; + _ChecksumProcessCharacter(kTextData[fItemBufferUpto]); + fItemBufferUpto++; + if (fItemBufferUpto >= fItemBufferSize) { + fFeedOutState = CLOSE_QUOTE; + FillBuffer(); + } + return B_OK; + case CLOSE_QUOTE: + c[0] = '"'; + fItemUpto++; + if (fItemUpto >= fItemCount) + fFeedOutState = CLOSE_ARRAY; + else + fFeedOutState = SEPARATOR; + return B_OK; + case SEPARATOR: + c[0] = ','; + fFeedOutState = OPEN_QUOTE; + return B_OK; + case CLOSE_ARRAY: + c[0] = ']'; + fFeedOutState = END; + return B_OK; + default: + return -1; // end of file + } +} + + +void +FakeJsonStringStreamDataIO::FillBuffer() +{ + fItemBufferSize = random() % kTextDataLength; + fItemBufferUpto = 0; +} + + +// #pragma mark - FakeJsonStringStreamDataIO + + +FakeJsonNumberStreamDataIO::FakeJsonNumberStreamDataIO(int count, uint32 checksumLimit) + : + FakeJsonStreamDataIO(count, checksumLimit), + fItemBufferSize(0), + fItemBufferUpto(0) +{ + bzero(fBuffer, 32); + FillBuffer(); +} + + +FakeJsonNumberStreamDataIO::~FakeJsonNumberStreamDataIO() +{ +} + + +status_t +FakeJsonNumberStreamDataIO::NextChar(char* c) +{ + switch (fFeedOutState) { + case OPEN_ARRAY: + c[0] = '['; + fFeedOutState = ITEM; + return B_OK; + case ITEM: + c[0] = fBuffer[fItemBufferUpto]; + _ChecksumProcessCharacter(fBuffer[fItemBufferUpto]); + fItemBufferUpto++; + if (fItemBufferUpto >= fItemBufferSize) { + fItemUpto++; + if (fItemUpto >= fItemCount) + fFeedOutState = CLOSE_ARRAY; + else + fFeedOutState = SEPARATOR; + FillBuffer(); + } + return B_OK; + case SEPARATOR: + c[0] = ','; + fFeedOutState = ITEM; + return B_OK; + case CLOSE_ARRAY: + c[0] = ']'; + fFeedOutState = END; + return B_OK; + default: + return -1; // end of file + } +} + + +void +FakeJsonNumberStreamDataIO::FillBuffer() +{ + int32 value = static_cast(random()); + fItemBufferSize = snprintf(fBuffer, 32, "%" B_PRIu32, value); + fItemBufferUpto = 0; +} \ No newline at end of file diff --git a/src/tests/kits/shared/FakeJsonDataGenerator.h b/src/tests/kits/shared/FakeJsonDataGenerator.h new file mode 100644 index 0000000000..17c29c771b --- /dev/null +++ b/src/tests/kits/shared/FakeJsonDataGenerator.h @@ -0,0 +1,83 @@ +/* + * Copyright 2023, Andrew Lindesay . + * All rights reserved. Distributed under the terms of the MIT License. + */ +#ifndef FAKE_JSON_DATA_GENERATOR_H +#define FAKE_JSON_DATA_GENERATOR_H + + +#include + + +typedef enum FeedOutState { + OPEN_ARRAY, + OPEN_QUOTE, // only used for strings + ITEM, + CLOSE_QUOTE, // only used for strings + SEPARATOR, + CLOSE_ARRAY, + END +} FeedOutState; + + +class FakeJsonStreamDataIO : public BDataIO { +public: + FakeJsonStreamDataIO(int count, uint32 checksumLimit); + virtual ~FakeJsonStreamDataIO(); + + virtual ssize_t Read(void* buffer, size_t size); + virtual ssize_t Write(const void* buffer, size_t size); + + virtual status_t Flush(); + + uint32 Checksum() const; + +protected: + virtual void FillBuffer() = 0; + virtual status_t NextChar(char* c) = 0; + + void _ChecksumProcessCharacter(const char c); + +protected: + FeedOutState fFeedOutState; + int fItemCount; + int fItemUpto; + +private: + uint32 fChecksum; + uint32 fChecksumLimit; +}; + + +class FakeJsonStringStreamDataIO : public FakeJsonStreamDataIO { +public: + FakeJsonStringStreamDataIO(int count, uint32 checksumLimit); + virtual ~FakeJsonStringStreamDataIO(); + +protected: + virtual void FillBuffer(); + status_t NextChar(char* c); + +protected: + int fItemBufferSize; + int fItemBufferUpto; +}; + + +class FakeJsonNumberStreamDataIO : public FakeJsonStreamDataIO { +public: + FakeJsonNumberStreamDataIO(int count, uint32 checksumLimit); + virtual ~FakeJsonNumberStreamDataIO(); + +protected: + virtual void FillBuffer(); + status_t NextChar(char* c); + +protected: + int fItemBufferSize; + int fItemBufferUpto; + char fBuffer[32]; +}; + + +#endif // FAKE_JSON_DATA_GENERATOR_H diff --git a/src/tests/kits/shared/Jamfile b/src/tests/kits/shared/Jamfile index 7403d6829e..f6a8dc8902 100644 --- a/src/tests/kits/shared/Jamfile +++ b/src/tests/kits/shared/Jamfile @@ -8,7 +8,9 @@ UnitTestLib libsharedtest.so : SharedTestAddon.cpp CalendarViewTest.cpp + ChecksumJsonEventListener.cpp DriverSettingsMessageAdapterTest.cpp + FakeJsonDataGenerator.cpp JsonEndToEndTest.cpp JsonErrorHandlingTest.cpp JsonTextWriterTest.cpp diff --git a/src/tests/kits/shared/JsonEndToEndTest.cpp b/src/tests/kits/shared/JsonEndToEndTest.cpp index 4f7d9fe6bb..bf2fd77e8e 100644 --- a/src/tests/kits/shared/JsonEndToEndTest.cpp +++ b/src/tests/kits/shared/JsonEndToEndTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright 2017, Andrew Lindesay + * Copyright 2017-2023, Andrew Lindesay * Distributed under the terms of the MIT License. */ #include "JsonEndToEndTest.h" @@ -12,12 +12,18 @@ #include #include +#include "ChecksumJsonEventListener.h" +#include "FakeJsonDataGenerator.h" #include "JsonSamples.h" using namespace BPrivate; +static const size_t kHighVolumeItemCount = 10000; +static const uint32 kChecksumLimit = 100000; + + JsonEndToEndTest::JsonEndToEndTest() { } @@ -28,6 +34,70 @@ JsonEndToEndTest::~JsonEndToEndTest() } +/*! Just here so it is possible to extract timings for the data generation cost. +*/ + +void +JsonEndToEndTest::TestHighVolumeStringSampleGenerationOnly() +{ + FakeJsonStreamDataIO* inputData = new FakeJsonStringStreamDataIO(kHighVolumeItemCount, + kChecksumLimit); + char c; + + while (inputData->Read(&c, 1) == 1) { + // do nothing + } +} + + +/*! Just here so it is possible to extract timings for the data generation cost. +*/ + +void +JsonEndToEndTest::TestHighVolumeNumberSampleGenerationOnly() +{ + FakeJsonStreamDataIO* inputData = new FakeJsonNumberStreamDataIO(kHighVolumeItemCount, + kChecksumLimit); + char c; + + while (inputData->Read(&c, 1) == 1) { + // do nothing + } +} + + +void +JsonEndToEndTest::TestHighVolumeStringParsing() +{ + FakeJsonStreamDataIO* inputData = new FakeJsonStringStreamDataIO(kHighVolumeItemCount, + kChecksumLimit); + ChecksumJsonEventListener* listener = new ChecksumJsonEventListener(kChecksumLimit); + + // ---------------------- + BPrivate::BJson::Parse(inputData, listener); + // ---------------------- + + CPPUNIT_ASSERT_EQUAL(B_OK, listener->Error()); + CPPUNIT_ASSERT_EQUAL(inputData->Checksum(), listener->Checksum()); +} + + +void +JsonEndToEndTest::TestHighVolumeNumberParsing() +{ + FakeJsonStreamDataIO* inputData = new FakeJsonNumberStreamDataIO(kHighVolumeItemCount, + kChecksumLimit); + ChecksumJsonEventListener* listener = new ChecksumJsonEventListener(kChecksumLimit); + + // ---------------------- + BPrivate::BJson::Parse(inputData, listener); + // ---------------------- + + CPPUNIT_ASSERT_EQUAL(B_OK, listener->Error()); + CPPUNIT_ASSERT_EQUAL(inputData->Checksum(), listener->Checksum()); +} + + void JsonEndToEndTest::TestParseAndWrite(const char* input, const char* expectedOutput) { @@ -208,8 +278,17 @@ JsonEndToEndTest::AddTests(BTestSuite& parent) "JsonEndToEndTest::TestArrayUnterminated", &JsonEndToEndTest::TestArrayUnterminated)); suite.addTest(new CppUnit::TestCaller( - "JsonEndToEndTest::TestObjectUnterminated", - &JsonEndToEndTest::TestObjectUnterminated)); + "JsonEndToEndTest::TestHighVolumeStringParsing", + &JsonEndToEndTest::TestHighVolumeStringParsing)); + suite.addTest(new CppUnit::TestCaller( + "JsonEndToEndTest::TestHighVolumeNumberParsing", + &JsonEndToEndTest::TestHighVolumeNumberParsing)); + suite.addTest(new CppUnit::TestCaller( + "JsonEndToEndTest::TestHighVolumeStringSampleGenerationOnly", + &JsonEndToEndTest::TestHighVolumeStringSampleGenerationOnly)); + suite.addTest(new CppUnit::TestCaller( + "JsonEndToEndTest::TestHighVolumeNumberSampleGenerationOnly", + &JsonEndToEndTest::TestHighVolumeNumberSampleGenerationOnly)); parent.addTest("JsonEndToEndTest", &suite); } diff --git a/src/tests/kits/shared/JsonEndToEndTest.h b/src/tests/kits/shared/JsonEndToEndTest.h index cd536482c5..b6fe7e8f98 100644 --- a/src/tests/kits/shared/JsonEndToEndTest.h +++ b/src/tests/kits/shared/JsonEndToEndTest.h @@ -1,5 +1,5 @@ /* - * Copyright 2017, Andrew Lindesay + * Copyright 2017-2023, Andrew Lindesay * Distributed under the terms of the MIT License. */ #ifndef JSON_END_TO_END_TEST_H @@ -18,6 +18,11 @@ public: JsonEndToEndTest(); virtual ~JsonEndToEndTest(); + void TestHighVolumeStringParsing(); + void TestHighVolumeNumberParsing(); + void TestHighVolumeStringSampleGenerationOnly(); + void TestHighVolumeNumberSampleGenerationOnly(); + void TestNullA(); void TestTrueA(); void TestFalseA();