Current-tokenizer cost (Δ, worst tokenizer)

These are the committed bytes rendered as readable JSON. Expanding this audit detail does not change the measurement’s current status.

{
    "metric": "token_delta",
    "construct": "may-as-permission / may-as-possibility",
    "models": [
        "cl100k_base",
        "o200k_base"
    ],
    "population": "twenty-four fresh short affirmative operational modal statements, balanced twelve authority-permission claims and twelve speaker-evidence possibility claims",
    "careful_english_controls": {
        "permission": "is permitted to",
        "possibility": "might"
    },
    "selection": "Twelve wholly new complete meaning-matched pairs per force, frozen before tokenizer load; strata are equally weighted; no negated may; no exact pair overlaps the target manifest.",
    "method": "For each pinned tokenizer, count len(encode(ainglish))-len(encode(english)) without special tokens. Average equally within each twelve-item force stratum, then average the two force means equally. The filed token_delta is the maximum balanced mean across tokenizers; value_lo/value_hi are the minimum/maximum balanced means. Report each force separately under every tokenizer.",
    "seed": "none — deterministic tokenizer counts, no sampling",
    "environment": {
        "python": "3.12.3",
        "library": "tiktoken",
        "version": "0.13.0",
        "ainglish_sdk": "0.2.47"
    },
    "replicates_hash": "ead8571ce276ebc166511b4a1561b4a89ccf4af7275117c036e2de63ec6383c5",
    "test_set": [
        {
            "force": "permission",
            "ainglish": "The research assistant may-as-permission query the restricted dataset.",
            "english": "The research assistant is permitted to query the restricted dataset."
        },
        {
            "force": "permission",
            "ainglish": "The deployment controller may-as-permission promote the candidate build.",
            "english": "The deployment controller is permitted to promote the candidate build."
        },
        {
            "force": "permission",
            "ainglish": "The billing agent may-as-permission issue the customer credit.",
            "english": "The billing agent is permitted to issue the customer credit."
        },
        {
            "force": "permission",
            "ainglish": "The triage nurse may-as-permission access the emergency chart.",
            "english": "The triage nurse is permitted to access the emergency chart."
        },
        {
            "force": "permission",
            "ainglish": "The drone operator may-as-permission enter the protected airspace.",
            "english": "The drone operator is permitted to enter the protected airspace."
        },
        {
            "force": "permission",
            "ainglish": "The data steward may-as-permission erase the expired records.",
            "english": "The data steward is permitted to erase the expired records."
        },
        {
            "force": "permission",
            "ainglish": "The moderator may-as-permission unseal the archived thread.",
            "english": "The moderator is permitted to unseal the archived thread."
        },
        {
            "force": "permission",
            "ainglish": "The warehouse robot may-as-permission cross the quarantine line.",
            "english": "The warehouse robot is permitted to cross the quarantine line."
        },
        {
            "force": "permission",
            "ainglish": "The incident commander may-as-permission disable the failover.",
            "english": "The incident commander is permitted to disable the failover."
        },
        {
            "force": "permission",
            "ainglish": "The procurement service may-as-permission approve the supplier.",
            "english": "The procurement service is permitted to approve the supplier."
        },
        {
            "force": "permission",
            "ainglish": "The lab technician may-as-permission export the genome file.",
            "english": "The lab technician is permitted to export the genome file."
        },
        {
            "force": "permission",
            "ainglish": "The translator may-as-permission publish the redacted transcript.",
            "english": "The translator is permitted to publish the redacted transcript."
        },
        {
            "force": "possibility",
            "ainglish": "The northern cache may-as-possibility evict the hot key.",
            "english": "The northern cache might evict the hot key."
        },
        {
            "force": "possibility",
            "ainglish": "The solar inverter may-as-possibility disconnect during the voltage spike.",
            "english": "The solar inverter might disconnect during the voltage spike."
        },
        {
            "force": "possibility",
            "ainglish": "The batch job may-as-possibility exceed the memory budget.",
            "english": "The batch job might exceed the memory budget."
        },
        {
            "force": "possibility",
            "ainglish": "The satellite clock may-as-possibility drift beyond tolerance.",
            "english": "The satellite clock might drift beyond tolerance."
        },
        {
            "force": "possibility",
            "ainglish": "The queued notification may-as-possibility arrive twice.",
            "english": "The queued notification might arrive twice."
        },
        {
            "force": "possibility",
            "ainglish": "The image classifier may-as-possibility mislabel the damaged sign.",
            "english": "The image classifier might mislabel the damaged sign."
        },
        {
            "force": "possibility",
            "ainglish": "The replica may-as-possibility promote before the log catches up.",
            "english": "The replica might promote before the log catches up."
        },
        {
            "force": "possibility",
            "ainglish": "The safety relay may-as-possibility trip during the self-test.",
            "english": "The safety relay might trip during the self-test."
        },
        {
            "force": "possibility",
            "ainglish": "The currency feed may-as-possibility revise the closing rate.",
            "english": "The currency feed might revise the closing rate."
        },
        {
            "force": "possibility",
            "ainglish": "The recovery script may-as-possibility skip the orphaned block.",
            "english": "The recovery script might skip the orphaned block."
        },
        {
            "force": "possibility",
            "ainglish": "The DNS answer may-as-possibility persist after expiration.",
            "english": "The DNS answer might persist after expiration."
        },
        {
            "force": "possibility",
            "ainglish": "The archive index may-as-possibility rebuild incompletely.",
            "english": "The archive index might rebuild incompletely."
        }
    ]
}