bun-src/scripts/build/jsonByteClass.ts

// Generates `json_byte_class.h` (nibble LUTs, Highway SIMD kernel) and `json_byte_class.rs` (the
// derived 256-entry table, Rust scalar indexer): both come from this one table so they agree.

import { mkdirSync } from "node:fs";
import { resolve } from "node:path";
import type { Config } from "./config.ts";
import { writeIfChanged } from "./fs.ts";

const STRUCTURAL = 0x07;
const WHITESPACE = 0x18;
const ODDITY = 0x20;
const CONTROL = 0x40;

const LUT_LO = [0x50, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x60, 0x40, 0x48, 0x4c, 0x41, 0x42, 0x49, 0x40, 0x60];
const LUT_HI = [0x48, 0x40, 0x32, 0x04, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00];

const classOf = (b: number): number => LUT_LO[b & 0xf] & LUT_HI[b >> 4];

function check() {
  const classesOf = (b: number): number[] => {
    const ch = String.fromCharCode(b);
    const classes: number[] = [];
    if ("{}[]:,".includes(ch)) classes.push(STRUCTURAL);
    if (" \t\n\r".includes(ch)) classes.push(WHITESPACE);
    if ("/'".includes(ch)) classes.push(ODDITY);
    if (b < 0x20) classes.push(CONTROL);
    return classes;
  };
  for (let b = 0; b < 0x80; b++) {
    const got = classOf(b);
    const classes = classesOf(b);
    const allowed = classes.reduce((acc, c) => acc | c, 0);
    const ok = classes.every(c => (got & c) !== 0) && (got & ~allowed) === 0;
    if (!ok) {
      throw new Error(
        `json_byte_class: byte 0x${b.toString(16)} classifies as 0x${got.toString(16)}, expected classes 0x${allowed.toString(16)}`,
      );
    }
  }
  for (let b = 0x80; b < 0x100; b++) {
    if (classOf(b) !== 0) {
      throw new Error(`json_byte_class: non-ASCII byte 0x${b.toString(16)} must not classify`);
    }
  }
}

export function generateJsonByteClass(cfg: Config): { h: string; rs: string } {
  check();

  const banner = (comment: string) => [
    `${comment} Generated by scripts/build/jsonByteClass.ts at configure time. Do not`,
    `${comment} edit. The same definition feeds the Highway JSON kernel (these nibble`,
    `${comment} LUTs) and the Rust scalar indexer (the derived 256-entry table); see`,
    `${comment} the generator for why they must agree byte for byte.`,
    `${comment}`,
    `${comment} cls = LUT_LO[b & 0xF] & LUT_HI[b >> 4]`,
    `${comment}   0x07 structural { } [ ] : ,    0x18 whitespace sp \\t \\n \\r`,
    `${comment}   0x20 oddity / '                0x40 control < 0x20`,
    "",
  ];
  const hex = (v: number) => `0x${v.toString(16).padStart(2, "0")}`;

  const h = [
    ...banner("//"),
    "#pragma once",
    "#include <stdint.h>",
    "",
    `#define BUN_JSON_CLASS_STRUCTURAL ${hex(STRUCTURAL)}`,
    `#define BUN_JSON_CLASS_WHITESPACE ${hex(WHITESPACE)}`,
    `#define BUN_JSON_CLASS_ODDITY ${hex(ODDITY)}`,
    `#define BUN_JSON_CLASS_CONTROL ${hex(CONTROL)}`,
    "",
    `alignas(16) static const uint8_t kBunJsonLutLo[16] = { ${LUT_LO.map(hex).join(", ")} };`,
    `alignas(16) static const uint8_t kBunJsonLutHi[16] = { ${LUT_HI.map(hex).join(", ")} };`,
    "",
  ].join("\n");

  const table: string[] = [];
  for (let row = 0; row < 256; row += 16) {
    const cells = [];
    for (let b = row; b < row + 16; b++) cells.push(hex(classOf(b)));
    table.push(`    ${cells.join(", ")},`);
  }
  const allow = "#[allow(dead_code, unreachable_pub, unused)]";
  const rs = [
    ...banner("//"),
    allow,
    `pub const CLASS_STRUCTURAL: u8 = ${hex(STRUCTURAL)};`,
    allow,
    `pub const CLASS_WHITESPACE: u8 = ${hex(WHITESPACE)};`,
    "",
    "/// `LUT_LO[b & 0xF] & LUT_HI[b >> 4]` for every byte `b`.",
    allow,
    "#[rustfmt::skip]",
    "pub const JSON_BYTE_CLASS: [u8; 256] = [",
    ...table,
    "];",
    "",
  ].join("\n");

  mkdirSync(cfg.codegenDir, { recursive: true });
  const hPath = resolve(cfg.codegenDir, "json_byte_class.h");
  const rsPath = resolve(cfg.codegenDir, "json_byte_class.rs");
  writeIfChanged(hPath, h);
  writeIfChanged(rsPath, rs);
  return { h: hPath, rs: rsPath };
}