Shared: JSON Parse Perf
Improve the mechanics for JSON parsing by reusing text buffers during the parse. Change-Id: I7fb2cae31e6558a5a0c63fd02e1fc6fec4f9e4b3 Reviewed-on: https://review.haiku-os.org/c/haiku/+/7106 Tested-by: Commit checker robot <[email protected]> Reviewed-by: Jérôme Duval <[email protected]>
This commit is contained in:
Andrew Lindesay
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2017, Andrew Lindesay <[email protected]>
|
||||
* Copyright 2017-2023, Andrew Lindesay <[email protected]>
|
||||
* Copyright 2014, Augustin Cavalier (waddlesplash)
|
||||
* Distributed under the terms of the MIT License.
|
||||
*/
|
||||
@@ -39,14 +39,12 @@ private:
|
||||
JsonParseContext& jsonParseContext);
|
||||
static bool ParseObject(JsonParseContext& jsonParseContext);
|
||||
static bool ParseArray(JsonParseContext& jsonParseContext);
|
||||
static bool ParseEscapeUnicodeSequence(
|
||||
JsonParseContext& jsonParseContext,
|
||||
BString& stringResult);
|
||||
static bool ParseStringEscapeSequence(
|
||||
JsonParseContext& jsonParseContext,
|
||||
BString& stringResult);
|
||||
|
||||
static bool ParseEscapeUnicodeSequence(JsonParseContext& jsonParseContext);
|
||||
static bool ParseStringEscapeSequence(JsonParseContext& jsonParseContext);
|
||||
static bool ParseString(JsonParseContext& jsonParseContext,
|
||||
json_event_type eventType);
|
||||
|
||||
static bool ParseExpectedVerbatimStringAndRaiseEvent(
|
||||
JsonParseContext& jsonParseContext,
|
||||
const char* expectedString,
|
||||
@@ -59,7 +57,7 @@ private:
|
||||
size_t expectedStringLength,
|
||||
char leadingChar);
|
||||
|
||||
static bool IsValidNumber(BString& number);
|
||||
static bool IsValidNumber(const char* value);
|
||||
static bool ParseNumber(JsonParseContext& jsonParseContext);
|
||||
};
|
||||
|
||||
|
||||
+225
-100
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2017, Andrew Lindesay <[email protected]>
|
||||
* Copyright 2017-2023, Andrew Lindesay <[email protected]>
|
||||
* Copyright 2014-2017, Augustin Cavalier (waddlesplash)
|
||||
* Copyright 2014, Stephan Aßmus <[email protected]>
|
||||
* Distributed under the terms of the MIT License.
|
||||
@@ -25,26 +25,151 @@
|
||||
|
||||
namespace BPrivate {
|
||||
|
||||
/*! A buffer is used to assemble strings into. This will be the initial size
|
||||
of this buffer.
|
||||
*/
|
||||
|
||||
static bool
|
||||
b_jsonparse_is_hex(char c)
|
||||
{
|
||||
return isdigit(c)
|
||||
|| (c > 0x41 && c <= 0x46)
|
||||
|| (c > 0x61 && c <= 0x66);
|
||||
}
|
||||
static const size_t kInitialAssemblyBufferSize = 64;
|
||||
|
||||
/*! A buffer is used to assemble strings into. This buffer starts off small
|
||||
but is able to grow as the string it needs to process as encountered. To
|
||||
avoid frequent reallocation of the buffer, the buffer will be retained
|
||||
between strings. This is the maximum size of buffer that will be retained.
|
||||
*/
|
||||
|
||||
static const size_t kRetainedAssemblyBufferSize = 32 * 1024;
|
||||
|
||||
static const size_t kAssemblyBufferSizeIncrement = 256;
|
||||
|
||||
static const size_t kMaximumUtf8SequenceLength = 7;
|
||||
|
||||
|
||||
static bool
|
||||
b_jsonparse_all_hex(const char* c)
|
||||
{
|
||||
for (int i = 0; i < 4; i++) {
|
||||
if (!b_jsonparse_is_hex(c[i]))
|
||||
return false;
|
||||
class JsonParseAssemblyBuffer {
|
||||
public:
|
||||
JsonParseAssemblyBuffer()
|
||||
:
|
||||
fAssemblyBuffer(NULL),
|
||||
fAssemblyBufferAllocatedSize(0),
|
||||
fAssemblyBufferUsedSize(0)
|
||||
{
|
||||
fAssemblyBuffer = (char*) malloc(kInitialAssemblyBufferSize);
|
||||
if (fAssemblyBuffer != NULL)
|
||||
fAssemblyBufferAllocatedSize = kInitialAssemblyBufferSize;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
~JsonParseAssemblyBuffer()
|
||||
{
|
||||
if (fAssemblyBuffer != NULL)
|
||||
free(fAssemblyBuffer);
|
||||
}
|
||||
|
||||
const char* Buffer() const
|
||||
{
|
||||
return fAssemblyBuffer;
|
||||
}
|
||||
|
||||
/*! This method should be used each time that the assembly buffer has
|
||||
been finished with by some section of logic.
|
||||
*/
|
||||
|
||||
status_t Reset()
|
||||
{
|
||||
fAssemblyBufferUsedSize = 0;
|
||||
|
||||
if (fAssemblyBufferAllocatedSize > kRetainedAssemblyBufferSize) {
|
||||
fAssemblyBuffer = (char*) realloc(fAssemblyBuffer, kRetainedAssemblyBufferSize);
|
||||
if (fAssemblyBuffer == NULL) {
|
||||
fAssemblyBufferAllocatedSize = 0;
|
||||
return B_NO_MEMORY;
|
||||
}
|
||||
fAssemblyBufferAllocatedSize = kRetainedAssemblyBufferSize;
|
||||
}
|
||||
|
||||
return B_OK;
|
||||
}
|
||||
|
||||
status_t AppendCharacter(char c)
|
||||
{
|
||||
status_t result = _EnsureAssemblyBufferAllocatedSize(fAssemblyBufferUsedSize + 1);
|
||||
|
||||
if (result == B_OK) {
|
||||
fAssemblyBuffer[fAssemblyBufferUsedSize] = c;
|
||||
fAssemblyBufferUsedSize++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
status_t AppendCharacters(char* str, size_t len)
|
||||
{
|
||||
status_t result = _EnsureAssemblyBufferAllocatedSize(fAssemblyBufferUsedSize + len);
|
||||
|
||||
if (result == B_OK) {
|
||||
memcpy(&fAssemblyBuffer[fAssemblyBufferUsedSize], str, len);
|
||||
fAssemblyBufferUsedSize += len;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
status_t AppendUnicodeCharacter(uint32 c)
|
||||
{
|
||||
status_t result = _EnsureAssemblyBufferAllocatedSize(
|
||||
fAssemblyBufferUsedSize + kMaximumUtf8SequenceLength);
|
||||
if (result == B_OK) {
|
||||
char* insertPtr = &fAssemblyBuffer[fAssemblyBufferUsedSize];
|
||||
char* ptr = insertPtr;
|
||||
BUnicodeChar::ToUTF8(c, &ptr);
|
||||
size_t sequenceLength = static_cast<uint32>(ptr - insertPtr);
|
||||
fAssemblyBufferUsedSize += sequenceLength;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
/*! This method will return the assembly buffer ensuring that it has at
|
||||
least `minimumSize` bytes available.
|
||||
*/
|
||||
|
||||
status_t _EnsureAssemblyBufferAllocatedSize(size_t minimumSize)
|
||||
{
|
||||
if (fAssemblyBufferAllocatedSize < minimumSize) {
|
||||
fAssemblyBuffer = (char*) realloc(fAssemblyBuffer, minimumSize);
|
||||
if (fAssemblyBuffer == NULL) {
|
||||
fAssemblyBufferAllocatedSize = 0;
|
||||
return B_NO_MEMORY;
|
||||
}
|
||||
fAssemblyBufferAllocatedSize = minimumSize;
|
||||
}
|
||||
return B_OK;
|
||||
}
|
||||
|
||||
private:
|
||||
char* fAssemblyBuffer;
|
||||
size_t fAssemblyBufferAllocatedSize;
|
||||
size_t fAssemblyBufferUsedSize;
|
||||
};
|
||||
|
||||
|
||||
class JsonParseAssemblyBufferResetter {
|
||||
public:
|
||||
JsonParseAssemblyBufferResetter(JsonParseAssemblyBuffer* assemblyBuffer)
|
||||
:
|
||||
fAssemblyBuffer(assemblyBuffer)
|
||||
{
|
||||
}
|
||||
|
||||
~JsonParseAssemblyBufferResetter()
|
||||
{
|
||||
fAssemblyBuffer->Reset();
|
||||
}
|
||||
|
||||
private:
|
||||
JsonParseAssemblyBuffer*
|
||||
fAssemblyBuffer;
|
||||
};
|
||||
|
||||
|
||||
/*! This class carries state around the parsing process. */
|
||||
@@ -57,11 +182,18 @@ public:
|
||||
fData(data),
|
||||
fLineNumber(1), // 1 is the first line
|
||||
fPushbackChar(0),
|
||||
fHasPushbackChar(false)
|
||||
fHasPushbackChar(false),
|
||||
fAssemblyBuffer(new JsonParseAssemblyBuffer())
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
~JsonParseContext()
|
||||
{
|
||||
delete fAssemblyBuffer;
|
||||
}
|
||||
|
||||
|
||||
BJsonEventListener* Listener() const
|
||||
{
|
||||
return fListener;
|
||||
@@ -85,14 +217,8 @@ public:
|
||||
fLineNumber++;
|
||||
}
|
||||
|
||||
|
||||
// TODO; there is considerable opportunity for performance improvements
|
||||
// here by buffering the input and then feeding it into the parse
|
||||
// algorithm character by character.
|
||||
|
||||
status_t NextChar(char* buffer)
|
||||
{
|
||||
|
||||
if (fHasPushbackChar) {
|
||||
buffer[0] = fPushbackChar;
|
||||
fHasPushbackChar = false;
|
||||
@@ -102,19 +228,27 @@ public:
|
||||
return Data()->ReadExactly(buffer, 1);
|
||||
}
|
||||
|
||||
|
||||
void PushbackChar(char c)
|
||||
{
|
||||
fPushbackChar = c;
|
||||
fHasPushbackChar = true;
|
||||
}
|
||||
|
||||
|
||||
JsonParseAssemblyBuffer* AssemblyBuffer()
|
||||
{
|
||||
return fAssemblyBuffer;
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
BJsonEventListener* fListener;
|
||||
BDataIO* fData;
|
||||
uint32 fLineNumber;
|
||||
char fPushbackChar;
|
||||
bool fHasPushbackChar;
|
||||
JsonParseAssemblyBuffer*
|
||||
fAssemblyBuffer;
|
||||
};
|
||||
|
||||
|
||||
@@ -464,90 +598,86 @@ BJson::ParseArray(JsonParseContext& jsonParseContext)
|
||||
|
||||
|
||||
bool
|
||||
BJson::ParseEscapeUnicodeSequence(JsonParseContext& jsonParseContext,
|
||||
BString& stringResult)
|
||||
BJson::ParseEscapeUnicodeSequence(JsonParseContext& jsonParseContext)
|
||||
{
|
||||
char buffer[5];
|
||||
buffer[4] = 0;
|
||||
char ch;
|
||||
uint32 unicodeCh = 0;
|
||||
|
||||
if (!NextChar(jsonParseContext, &buffer[0])
|
||||
|| !NextChar(jsonParseContext, &buffer[1])
|
||||
|| !NextChar(jsonParseContext, &buffer[2])
|
||||
|| !NextChar(jsonParseContext, &buffer[3])) {
|
||||
return false;
|
||||
for (int i = 3; i >= 0; i--) {
|
||||
if (!NextChar(jsonParseContext, &ch)) {
|
||||
jsonParseContext.Listener()->HandleError(B_ERROR, jsonParseContext.LineNumber(),
|
||||
"unable to read unicode sequence");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (ch >= '0' && ch <= '9')
|
||||
unicodeCh |= static_cast<uint32>(ch - '0') << (i * 4);
|
||||
else if (ch >= 'a' && ch <= 'f')
|
||||
unicodeCh |= (10 + static_cast<uint32>(ch - 'a')) << (i * 4);
|
||||
else if (ch >= 'A' && ch <= 'F')
|
||||
unicodeCh |= (10 + static_cast<uint32>(ch - 'A')) << (i * 4);
|
||||
else {
|
||||
BString errorMessage;
|
||||
errorMessage.SetToFormat(
|
||||
"malformed hex character [%c] in unicode sequence in string parsing", ch);
|
||||
jsonParseContext.Listener()->HandleError(B_BAD_DATA, jsonParseContext.LineNumber(),
|
||||
errorMessage.String());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!b_jsonparse_all_hex(buffer)) {
|
||||
BString errorMessage;
|
||||
errorMessage.SetToFormat(
|
||||
"malformed unicode sequence [%s] in string parsing",
|
||||
buffer);
|
||||
jsonParseContext.Listener()->HandleError(B_BAD_DATA,
|
||||
jsonParseContext.LineNumber(), errorMessage.String());
|
||||
JsonParseAssemblyBuffer* assemblyBuffer = jsonParseContext.AssemblyBuffer();
|
||||
status_t result = assemblyBuffer->AppendUnicodeCharacter(unicodeCh);
|
||||
|
||||
if (result != B_OK) {
|
||||
jsonParseContext.Listener()->HandleError(result, jsonParseContext.LineNumber(),
|
||||
"unable to store unicode char as utf-8");
|
||||
return false;
|
||||
}
|
||||
|
||||
uint intValue;
|
||||
|
||||
if (sscanf(buffer, "%4x", &intValue) != 1) {
|
||||
BString errorMessage;
|
||||
errorMessage.SetToFormat(
|
||||
"unable to process unicode sequence [%s] in string "
|
||||
" parsing", buffer);
|
||||
jsonParseContext.Listener()->HandleError(B_BAD_DATA,
|
||||
jsonParseContext.LineNumber(), errorMessage.String());
|
||||
return false;
|
||||
}
|
||||
|
||||
char character[7];
|
||||
char* ptr = character;
|
||||
BUnicodeChar::ToUTF8(intValue, &ptr);
|
||||
int32 sequenceLength = ptr - character;
|
||||
stringResult.Append(character, sequenceLength);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
bool
|
||||
BJson::ParseStringEscapeSequence(JsonParseContext& jsonParseContext,
|
||||
BString& stringResult)
|
||||
BJson::ParseStringEscapeSequence(JsonParseContext& jsonParseContext)
|
||||
{
|
||||
char c;
|
||||
|
||||
if (!NextChar(jsonParseContext, &c))
|
||||
return false;
|
||||
|
||||
JsonParseAssemblyBuffer* assemblyBuffer = jsonParseContext.AssemblyBuffer();
|
||||
|
||||
switch (c) {
|
||||
case 'n':
|
||||
stringResult += "\n";
|
||||
assemblyBuffer->AppendCharacter('\n');
|
||||
break;
|
||||
case 'r':
|
||||
stringResult += "\r";
|
||||
assemblyBuffer->AppendCharacter('\r');
|
||||
break;
|
||||
case 'b':
|
||||
stringResult += "\b";
|
||||
assemblyBuffer->AppendCharacter('\b');
|
||||
break;
|
||||
case 'f':
|
||||
stringResult += "\f";
|
||||
assemblyBuffer->AppendCharacter('\f');
|
||||
break;
|
||||
case '\\':
|
||||
stringResult += "\\";
|
||||
assemblyBuffer->AppendCharacter('\\');
|
||||
break;
|
||||
case '/':
|
||||
stringResult += "/";
|
||||
assemblyBuffer->AppendCharacter('/');
|
||||
break;
|
||||
case 't':
|
||||
stringResult += "\t";
|
||||
assemblyBuffer->AppendCharacter('\t');
|
||||
break;
|
||||
case '"':
|
||||
stringResult += "\"";
|
||||
assemblyBuffer->AppendCharacter('"');
|
||||
break;
|
||||
case 'u':
|
||||
{
|
||||
// unicode escape sequence.
|
||||
if (!ParseEscapeUnicodeSequence(jsonParseContext,
|
||||
stringResult)) {
|
||||
if (!ParseEscapeUnicodeSequence(jsonParseContext)) {
|
||||
return false;
|
||||
}
|
||||
break;
|
||||
@@ -555,9 +685,7 @@ BJson::ParseStringEscapeSequence(JsonParseContext& jsonParseContext,
|
||||
default:
|
||||
{
|
||||
BString errorMessage;
|
||||
errorMessage.SetToFormat(
|
||||
"unexpected escaped character [%c] in string parsing",
|
||||
c);
|
||||
errorMessage.SetToFormat("unexpected escaped character [%c] in string parsing", c);
|
||||
jsonParseContext.Listener()->HandleError(B_BAD_DATA,
|
||||
jsonParseContext.LineNumber(), errorMessage.String());
|
||||
return false;
|
||||
@@ -573,7 +701,8 @@ BJson::ParseString(JsonParseContext& jsonParseContext,
|
||||
json_event_type eventType)
|
||||
{
|
||||
char c;
|
||||
BString stringResult;
|
||||
JsonParseAssemblyBuffer* assemblyBuffer = jsonParseContext.AssemblyBuffer();
|
||||
JsonParseAssemblyBufferResetter assembleBufferResetter(assemblyBuffer);
|
||||
|
||||
while(true) {
|
||||
if (!NextChar(jsonParseContext, &c))
|
||||
@@ -583,17 +712,16 @@ BJson::ParseString(JsonParseContext& jsonParseContext,
|
||||
case '"':
|
||||
{
|
||||
// terminates the string assembled so far.
|
||||
assemblyBuffer->AppendCharacter(0);
|
||||
jsonParseContext.Listener()->Handle(
|
||||
BJsonEvent(eventType, stringResult.String()));
|
||||
BJsonEvent(eventType, assemblyBuffer->Buffer()));
|
||||
return true;
|
||||
}
|
||||
|
||||
case '\\':
|
||||
{
|
||||
if (!ParseStringEscapeSequence(jsonParseContext,
|
||||
stringResult)) {
|
||||
if (!ParseStringEscapeSequence(jsonParseContext))
|
||||
return false;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -611,7 +739,7 @@ BJson::ParseString(JsonParseContext& jsonParseContext,
|
||||
return false;
|
||||
}
|
||||
|
||||
stringResult.Append(&c, 1);
|
||||
assemblyBuffer->AppendCharacter(c);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -671,47 +799,47 @@ BJson::ParseExpectedVerbatimString(JsonParseContext& jsonParseContext,
|
||||
*/
|
||||
|
||||
bool
|
||||
BJson::IsValidNumber(BString& number)
|
||||
BJson::IsValidNumber(const char* value)
|
||||
{
|
||||
int32 offset = 0;
|
||||
int32 len = number.Length();
|
||||
int32 len = strlen(value);
|
||||
|
||||
if (offset < len && number[offset] == '-')
|
||||
if (offset < len && value[offset] == '-')
|
||||
offset++;
|
||||
|
||||
if (offset >= len)
|
||||
return false;
|
||||
|
||||
if (isdigit(number[offset]) && number[offset] != '0') {
|
||||
while (offset < len && isdigit(number[offset]))
|
||||
if (isdigit(value[offset]) && value[offset] != '0') {
|
||||
while (offset < len && isdigit(value[offset]))
|
||||
offset++;
|
||||
} else {
|
||||
if (number[offset] == '0')
|
||||
if (value[offset] == '0')
|
||||
offset++;
|
||||
else
|
||||
return false;
|
||||
}
|
||||
|
||||
if (offset < len && number[offset] == '.') {
|
||||
if (offset < len && value[offset] == '.') {
|
||||
offset++;
|
||||
|
||||
if (offset >= len)
|
||||
return false;
|
||||
|
||||
while (offset < len && isdigit(number[offset]))
|
||||
while (offset < len && isdigit(value[offset]))
|
||||
offset++;
|
||||
}
|
||||
|
||||
if (offset < len && (number[offset] == 'E' || number[offset] == 'e')) {
|
||||
if (offset < len && (value[offset] == 'E' || value[offset] == 'e')) {
|
||||
offset++;
|
||||
|
||||
if(offset < len && (number[offset] == '+' || number[offset] == '-'))
|
||||
if(offset < len && (value[offset] == '+' || value[offset] == '-'))
|
||||
offset++;
|
||||
|
||||
if (offset >= len)
|
||||
return false;
|
||||
|
||||
while (offset < len && isdigit(number[offset]))
|
||||
while (offset < len && isdigit(value[offset]))
|
||||
offset++;
|
||||
}
|
||||
|
||||
@@ -728,7 +856,8 @@ BJson::IsValidNumber(BString& number)
|
||||
bool
|
||||
BJson::ParseNumber(JsonParseContext& jsonParseContext)
|
||||
{
|
||||
BString value;
|
||||
JsonParseAssemblyBuffer* assemblyBuffer = jsonParseContext.AssemblyBuffer();
|
||||
JsonParseAssemblyBufferResetter assembleBufferResetter(assemblyBuffer);
|
||||
|
||||
while (true) {
|
||||
char c;
|
||||
@@ -737,13 +866,8 @@ BJson::ParseNumber(JsonParseContext& jsonParseContext)
|
||||
switch (result) {
|
||||
case B_OK:
|
||||
{
|
||||
if (isdigit(c)) {
|
||||
value += c;
|
||||
break;
|
||||
}
|
||||
|
||||
if (NULL != strchr("+-eE.", c)) {
|
||||
value += c;
|
||||
if (isdigit(c) || c == '.' || c == '-' || c == 'e' || c == 'E' || c == '+') {
|
||||
assemblyBuffer->AppendCharacter(c);
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -753,15 +877,16 @@ BJson::ParseNumber(JsonParseContext& jsonParseContext)
|
||||
case B_PARTIAL_READ:
|
||||
{
|
||||
errno = 0;
|
||||
assemblyBuffer->AppendCharacter(0);
|
||||
|
||||
if (!IsValidNumber(value)) {
|
||||
if (!IsValidNumber(assemblyBuffer->Buffer())) {
|
||||
jsonParseContext.Listener()->HandleError(B_BAD_DATA,
|
||||
jsonParseContext.LineNumber(), "malformed number");
|
||||
return false;
|
||||
}
|
||||
|
||||
jsonParseContext.Listener()->Handle(BJsonEvent(B_JSON_NUMBER,
|
||||
value.String()));
|
||||
assemblyBuffer->Buffer()));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
/*
|
||||
* Copyright 2023, Andrew Lindesay <[email protected]>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
|
||||
|
||||
#include "ChecksumJsonEventListener.h"
|
||||
|
||||
|
||||
ChecksumJsonEventListener::ChecksumJsonEventListener(int32 checksumLimit)
|
||||
:
|
||||
fChecksum(0),
|
||||
fChecksumLimit(checksumLimit),
|
||||
fError(B_OK),
|
||||
fCompleted(false)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
ChecksumJsonEventListener::~ChecksumJsonEventListener()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
bool
|
||||
ChecksumJsonEventListener::Handle(const BJsonEvent& event)
|
||||
{
|
||||
if (fCompleted || B_OK != fError)
|
||||
return false;
|
||||
|
||||
switch (event.EventType()) {
|
||||
case B_JSON_NUMBER:
|
||||
{
|
||||
const char* content = event.Content();
|
||||
_ChecksumProcessCharacters(content, strlen(content));
|
||||
break;
|
||||
}
|
||||
case B_JSON_STRING:
|
||||
case B_JSON_OBJECT_NAME:
|
||||
{
|
||||
const char* content = event.Content();
|
||||
_ChecksumProcessCharacters(content, strlen(content));
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
ChecksumJsonEventListener::HandleError(status_t status, int32 line, const char* message)
|
||||
{
|
||||
fError = status;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
ChecksumJsonEventListener::Complete()
|
||||
{
|
||||
fCompleted = true;
|
||||
}
|
||||
|
||||
|
||||
uint32
|
||||
ChecksumJsonEventListener::Checksum() const
|
||||
{
|
||||
return fChecksum;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
ChecksumJsonEventListener::Error() const
|
||||
{
|
||||
return fError;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
ChecksumJsonEventListener::_ChecksumProcessCharacters(const char* content, size_t len)
|
||||
{
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
fChecksum = (fChecksum + static_cast<int32>(content[i])) % fChecksumLimit;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Copyright 2023, Andrew Lindesay <[email protected]>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
#ifndef CHECKSUM_JSON_EVENT_LISTENER_H
|
||||
#define CHECKSUM_JSON_EVENT_LISTENER_H
|
||||
|
||||
|
||||
#include <JsonEventListener.h>
|
||||
|
||||
|
||||
/*! This class can be used by a text to accept JSON events and then to maintain a checksum of the
|
||||
strings and numbers that it encounters in order to get some sort of a checksum on the data
|
||||
that it is seeing. It can then compare this to the data that was emitted in order to verify
|
||||
that the JSON parser has parsed and passed-through all of the data correctly.
|
||||
*/
|
||||
|
||||
class ChecksumJsonEventListener : public BJsonEventListener {
|
||||
public:
|
||||
ChecksumJsonEventListener(int32 checksumLimit);
|
||||
virtual ~ChecksumJsonEventListener();
|
||||
|
||||
virtual bool Handle(const BJsonEvent& event);
|
||||
virtual void HandleError(status_t status, int32 line, const char* message);
|
||||
virtual void Complete();
|
||||
|
||||
uint32 Checksum() const;
|
||||
status_t Error() const;
|
||||
|
||||
private:
|
||||
void _ChecksumProcessCharacters(const char* content, size_t len);
|
||||
|
||||
private:
|
||||
uint32 fChecksum;
|
||||
uint32 fChecksumLimit;
|
||||
status_t fError;
|
||||
bool fCompleted;
|
||||
};
|
||||
|
||||
#endif // CHECKSUM_JSON_EVENT_LISTENER_H
|
||||
@@ -0,0 +1,215 @@
|
||||
/*
|
||||
* Copyright 2023, Andrew Lindesay <[email protected]>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
#include "FakeJsonDataGenerator.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <stdio.h>
|
||||
#include <strings.h>
|
||||
#include <string.h>
|
||||
|
||||
|
||||
static const char* kTextData = "abcdefghijklmnopqrstuvwxyz!@#$"
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZ)(*&"
|
||||
"0123456789{}[]|:;'<>,.?/~`_+-="
|
||||
"abcdefghijklmnopqrstuvwxyz!@#$"
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZ)(*&"
|
||||
"0123456789{}[]|:;'<>,.?/~`_+-=";
|
||||
static const int kTextDataLength = 180;
|
||||
|
||||
|
||||
FakeJsonStreamDataIO::FakeJsonStreamDataIO(int count, uint32 checksumLimit)
|
||||
:
|
||||
fFeedOutState(OPEN_ARRAY),
|
||||
fItemCount(count),
|
||||
fItemUpto(0),
|
||||
fChecksum(0),
|
||||
fChecksumLimit(checksumLimit)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
FakeJsonStreamDataIO::~FakeJsonStreamDataIO()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
ssize_t
|
||||
FakeJsonStreamDataIO::Read(void* buffer, size_t size)
|
||||
{
|
||||
char* buffer_c = static_cast<char*>(buffer);
|
||||
status_t result = B_OK;
|
||||
size_t i = 0;
|
||||
|
||||
while (i < size && result == B_OK) {
|
||||
result = NextChar(&buffer_c[i]);
|
||||
if (result == B_OK)
|
||||
i++;
|
||||
}
|
||||
|
||||
if(0 != i)
|
||||
return i;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
ssize_t
|
||||
FakeJsonStreamDataIO::Write(const void* buffer, size_t size)
|
||||
{
|
||||
return B_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
FakeJsonStreamDataIO::Flush()
|
||||
{
|
||||
return B_OK;
|
||||
}
|
||||
|
||||
|
||||
uint32
|
||||
FakeJsonStreamDataIO::Checksum() const
|
||||
{
|
||||
return fChecksum;
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
FakeJsonStreamDataIO::_ChecksumProcessCharacter(const char c)
|
||||
{
|
||||
fChecksum = (fChecksum + static_cast<int32>(c)) % fChecksumLimit;
|
||||
}
|
||||
|
||||
|
||||
// #pragma mark - FakeJsonStringStreamDataIO
|
||||
|
||||
|
||||
FakeJsonStringStreamDataIO::FakeJsonStringStreamDataIO(int count, uint32 checksumLimit)
|
||||
:
|
||||
FakeJsonStreamDataIO(count, checksumLimit),
|
||||
fItemBufferSize(0),
|
||||
fItemBufferUpto(0)
|
||||
{
|
||||
FillBuffer();
|
||||
}
|
||||
|
||||
|
||||
FakeJsonStringStreamDataIO::~FakeJsonStringStreamDataIO()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
FakeJsonStringStreamDataIO::NextChar(char* c)
|
||||
{
|
||||
switch (fFeedOutState) {
|
||||
case OPEN_ARRAY:
|
||||
c[0] = '[';
|
||||
fFeedOutState = OPEN_QUOTE;
|
||||
return B_OK;
|
||||
case OPEN_QUOTE:
|
||||
c[0] = '"';
|
||||
fFeedOutState = ITEM;
|
||||
return B_OK;
|
||||
case ITEM:
|
||||
c[0] = kTextData[fItemBufferUpto];
|
||||
_ChecksumProcessCharacter(kTextData[fItemBufferUpto]);
|
||||
fItemBufferUpto++;
|
||||
if (fItemBufferUpto >= fItemBufferSize) {
|
||||
fFeedOutState = CLOSE_QUOTE;
|
||||
FillBuffer();
|
||||
}
|
||||
return B_OK;
|
||||
case CLOSE_QUOTE:
|
||||
c[0] = '"';
|
||||
fItemUpto++;
|
||||
if (fItemUpto >= fItemCount)
|
||||
fFeedOutState = CLOSE_ARRAY;
|
||||
else
|
||||
fFeedOutState = SEPARATOR;
|
||||
return B_OK;
|
||||
case SEPARATOR:
|
||||
c[0] = ',';
|
||||
fFeedOutState = OPEN_QUOTE;
|
||||
return B_OK;
|
||||
case CLOSE_ARRAY:
|
||||
c[0] = ']';
|
||||
fFeedOutState = END;
|
||||
return B_OK;
|
||||
default:
|
||||
return -1; // end of file
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
FakeJsonStringStreamDataIO::FillBuffer()
|
||||
{
|
||||
fItemBufferSize = random() % kTextDataLength;
|
||||
fItemBufferUpto = 0;
|
||||
}
|
||||
|
||||
|
||||
// #pragma mark - FakeJsonStringStreamDataIO
|
||||
|
||||
|
||||
FakeJsonNumberStreamDataIO::FakeJsonNumberStreamDataIO(int count, uint32 checksumLimit)
|
||||
:
|
||||
FakeJsonStreamDataIO(count, checksumLimit),
|
||||
fItemBufferSize(0),
|
||||
fItemBufferUpto(0)
|
||||
{
|
||||
bzero(fBuffer, 32);
|
||||
FillBuffer();
|
||||
}
|
||||
|
||||
|
||||
FakeJsonNumberStreamDataIO::~FakeJsonNumberStreamDataIO()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
status_t
|
||||
FakeJsonNumberStreamDataIO::NextChar(char* c)
|
||||
{
|
||||
switch (fFeedOutState) {
|
||||
case OPEN_ARRAY:
|
||||
c[0] = '[';
|
||||
fFeedOutState = ITEM;
|
||||
return B_OK;
|
||||
case ITEM:
|
||||
c[0] = fBuffer[fItemBufferUpto];
|
||||
_ChecksumProcessCharacter(fBuffer[fItemBufferUpto]);
|
||||
fItemBufferUpto++;
|
||||
if (fItemBufferUpto >= fItemBufferSize) {
|
||||
fItemUpto++;
|
||||
if (fItemUpto >= fItemCount)
|
||||
fFeedOutState = CLOSE_ARRAY;
|
||||
else
|
||||
fFeedOutState = SEPARATOR;
|
||||
FillBuffer();
|
||||
}
|
||||
return B_OK;
|
||||
case SEPARATOR:
|
||||
c[0] = ',';
|
||||
fFeedOutState = ITEM;
|
||||
return B_OK;
|
||||
case CLOSE_ARRAY:
|
||||
c[0] = ']';
|
||||
fFeedOutState = END;
|
||||
return B_OK;
|
||||
default:
|
||||
return -1; // end of file
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
FakeJsonNumberStreamDataIO::FillBuffer()
|
||||
{
|
||||
int32 value = static_cast<int32>(random());
|
||||
fItemBufferSize = snprintf(fBuffer, 32, "%" B_PRIu32, value);
|
||||
fItemBufferUpto = 0;
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Copyright 2023, Andrew Lindesay <[email protected]>.
|
||||
* All rights reserved. Distributed under the terms of the MIT License.
|
||||
*/
|
||||
#ifndef FAKE_JSON_DATA_GENERATOR_H
|
||||
#define FAKE_JSON_DATA_GENERATOR_H
|
||||
|
||||
|
||||
#include <DataIO.h>
|
||||
|
||||
|
||||
typedef enum FeedOutState {
|
||||
OPEN_ARRAY,
|
||||
OPEN_QUOTE, // only used for strings
|
||||
ITEM,
|
||||
CLOSE_QUOTE, // only used for strings
|
||||
SEPARATOR,
|
||||
CLOSE_ARRAY,
|
||||
END
|
||||
} FeedOutState;
|
||||
|
||||
|
||||
class FakeJsonStreamDataIO : public BDataIO {
|
||||
public:
|
||||
FakeJsonStreamDataIO(int count, uint32 checksumLimit);
|
||||
virtual ~FakeJsonStreamDataIO();
|
||||
|
||||
virtual ssize_t Read(void* buffer, size_t size);
|
||||
virtual ssize_t Write(const void* buffer, size_t size);
|
||||
|
||||
virtual status_t Flush();
|
||||
|
||||
uint32 Checksum() const;
|
||||
|
||||
protected:
|
||||
virtual void FillBuffer() = 0;
|
||||
virtual status_t NextChar(char* c) = 0;
|
||||
|
||||
void _ChecksumProcessCharacter(const char c);
|
||||
|
||||
protected:
|
||||
FeedOutState fFeedOutState;
|
||||
int fItemCount;
|
||||
int fItemUpto;
|
||||
|
||||
private:
|
||||
uint32 fChecksum;
|
||||
uint32 fChecksumLimit;
|
||||
};
|
||||
|
||||
|
||||
class FakeJsonStringStreamDataIO : public FakeJsonStreamDataIO {
|
||||
public:
|
||||
FakeJsonStringStreamDataIO(int count, uint32 checksumLimit);
|
||||
virtual ~FakeJsonStringStreamDataIO();
|
||||
|
||||
protected:
|
||||
virtual void FillBuffer();
|
||||
status_t NextChar(char* c);
|
||||
|
||||
protected:
|
||||
int fItemBufferSize;
|
||||
int fItemBufferUpto;
|
||||
};
|
||||
|
||||
|
||||
class FakeJsonNumberStreamDataIO : public FakeJsonStreamDataIO {
|
||||
public:
|
||||
FakeJsonNumberStreamDataIO(int count, uint32 checksumLimit);
|
||||
virtual ~FakeJsonNumberStreamDataIO();
|
||||
|
||||
protected:
|
||||
virtual void FillBuffer();
|
||||
status_t NextChar(char* c);
|
||||
|
||||
protected:
|
||||
int fItemBufferSize;
|
||||
int fItemBufferUpto;
|
||||
char fBuffer[32];
|
||||
};
|
||||
|
||||
|
||||
#endif // FAKE_JSON_DATA_GENERATOR_H
|
||||
@@ -8,7 +8,9 @@ UnitTestLib libsharedtest.so :
|
||||
SharedTestAddon.cpp
|
||||
|
||||
CalendarViewTest.cpp
|
||||
ChecksumJsonEventListener.cpp
|
||||
DriverSettingsMessageAdapterTest.cpp
|
||||
FakeJsonDataGenerator.cpp
|
||||
JsonEndToEndTest.cpp
|
||||
JsonErrorHandlingTest.cpp
|
||||
JsonTextWriterTest.cpp
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2017, Andrew Lindesay <[email protected]>
|
||||
* Copyright 2017-2023, Andrew Lindesay <[email protected]>
|
||||
* Distributed under the terms of the MIT License.
|
||||
*/
|
||||
#include "JsonEndToEndTest.h"
|
||||
@@ -12,12 +12,18 @@
|
||||
#include <cppunit/TestCaller.h>
|
||||
#include <cppunit/TestSuite.h>
|
||||
|
||||
#include "ChecksumJsonEventListener.h"
|
||||
#include "FakeJsonDataGenerator.h"
|
||||
#include "JsonSamples.h"
|
||||
|
||||
|
||||
using namespace BPrivate;
|
||||
|
||||
|
||||
static const size_t kHighVolumeItemCount = 10000;
|
||||
static const uint32 kChecksumLimit = 100000;
|
||||
|
||||
|
||||
JsonEndToEndTest::JsonEndToEndTest()
|
||||
{
|
||||
}
|
||||
@@ -28,6 +34,70 @@ JsonEndToEndTest::~JsonEndToEndTest()
|
||||
}
|
||||
|
||||
|
||||
/*! Just here so it is possible to extract timings for the data generation cost.
|
||||
*/
|
||||
|
||||
void
|
||||
JsonEndToEndTest::TestHighVolumeStringSampleGenerationOnly()
|
||||
{
|
||||
FakeJsonStreamDataIO* inputData = new FakeJsonStringStreamDataIO(kHighVolumeItemCount,
|
||||
kChecksumLimit);
|
||||
char c;
|
||||
|
||||
while (inputData->Read(&c, 1) == 1) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*! Just here so it is possible to extract timings for the data generation cost.
|
||||
*/
|
||||
|
||||
void
|
||||
JsonEndToEndTest::TestHighVolumeNumberSampleGenerationOnly()
|
||||
{
|
||||
FakeJsonStreamDataIO* inputData = new FakeJsonNumberStreamDataIO(kHighVolumeItemCount,
|
||||
kChecksumLimit);
|
||||
char c;
|
||||
|
||||
while (inputData->Read(&c, 1) == 1) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
JsonEndToEndTest::TestHighVolumeStringParsing()
|
||||
{
|
||||
FakeJsonStreamDataIO* inputData = new FakeJsonStringStreamDataIO(kHighVolumeItemCount,
|
||||
kChecksumLimit);
|
||||
ChecksumJsonEventListener* listener = new ChecksumJsonEventListener(kChecksumLimit);
|
||||
|
||||
// ----------------------
|
||||
BPrivate::BJson::Parse(inputData, listener);
|
||||
// ----------------------
|
||||
|
||||
CPPUNIT_ASSERT_EQUAL(B_OK, listener->Error());
|
||||
CPPUNIT_ASSERT_EQUAL(inputData->Checksum(), listener->Checksum());
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
JsonEndToEndTest::TestHighVolumeNumberParsing()
|
||||
{
|
||||
FakeJsonStreamDataIO* inputData = new FakeJsonNumberStreamDataIO(kHighVolumeItemCount,
|
||||
kChecksumLimit);
|
||||
ChecksumJsonEventListener* listener = new ChecksumJsonEventListener(kChecksumLimit);
|
||||
|
||||
// ----------------------
|
||||
BPrivate::BJson::Parse(inputData, listener);
|
||||
// ----------------------
|
||||
|
||||
CPPUNIT_ASSERT_EQUAL(B_OK, listener->Error());
|
||||
CPPUNIT_ASSERT_EQUAL(inputData->Checksum(), listener->Checksum());
|
||||
}
|
||||
|
||||
|
||||
void
|
||||
JsonEndToEndTest::TestParseAndWrite(const char* input, const char* expectedOutput)
|
||||
{
|
||||
@@ -208,8 +278,17 @@ JsonEndToEndTest::AddTests(BTestSuite& parent)
|
||||
"JsonEndToEndTest::TestArrayUnterminated",
|
||||
&JsonEndToEndTest::TestArrayUnterminated));
|
||||
suite.addTest(new CppUnit::TestCaller<JsonEndToEndTest>(
|
||||
"JsonEndToEndTest::TestObjectUnterminated",
|
||||
&JsonEndToEndTest::TestObjectUnterminated));
|
||||
"JsonEndToEndTest::TestHighVolumeStringParsing",
|
||||
&JsonEndToEndTest::TestHighVolumeStringParsing));
|
||||
suite.addTest(new CppUnit::TestCaller<JsonEndToEndTest>(
|
||||
"JsonEndToEndTest::TestHighVolumeNumberParsing",
|
||||
&JsonEndToEndTest::TestHighVolumeNumberParsing));
|
||||
suite.addTest(new CppUnit::TestCaller<JsonEndToEndTest>(
|
||||
"JsonEndToEndTest::TestHighVolumeStringSampleGenerationOnly",
|
||||
&JsonEndToEndTest::TestHighVolumeStringSampleGenerationOnly));
|
||||
suite.addTest(new CppUnit::TestCaller<JsonEndToEndTest>(
|
||||
"JsonEndToEndTest::TestHighVolumeNumberSampleGenerationOnly",
|
||||
&JsonEndToEndTest::TestHighVolumeNumberSampleGenerationOnly));
|
||||
|
||||
parent.addTest("JsonEndToEndTest", &suite);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2017, Andrew Lindesay <[email protected]>
|
||||
* Copyright 2017-2023, Andrew Lindesay <[email protected]>
|
||||
* Distributed under the terms of the MIT License.
|
||||
*/
|
||||
#ifndef JSON_END_TO_END_TEST_H
|
||||
@@ -18,6 +18,11 @@ public:
|
||||
JsonEndToEndTest();
|
||||
virtual ~JsonEndToEndTest();
|
||||
|
||||
void TestHighVolumeStringParsing();
|
||||
void TestHighVolumeNumberParsing();
|
||||
void TestHighVolumeStringSampleGenerationOnly();
|
||||
void TestHighVolumeNumberSampleGenerationOnly();
|
||||
|
||||
void TestNullA();
|
||||
void TestTrueA();
|
||||
void TestFalseA();
|
||||
|
||||
Reference in New Issue
Block a user