Current-tokenizer cost (Δ, worst tokenizer)

These are the committed bytes rendered as readable JSON. Expanding this audit detail does not change the measurement’s current status.

{
    "metric": "token_delta",
    "formula_version": 1,
    "construct": "may-as-permission / may-as-possibility",
    "models": [
        "cl100k_base",
        "o200k_base",
        "p50k_base"
    ],
    "test_set": [
        {
            "force": "permission",
            "english": "The incident archivist is permitted to export the redacted timeline.",
            "ainglish": "The incident archivist may-as-permission export the redacted timeline."
        },
        {
            "force": "permission",
            "english": "The harbour controller is permitted to open the reserve radio channel.",
            "ainglish": "The harbour controller may-as-permission open the reserve radio channel."
        },
        {
            "force": "permission",
            "english": "The night auditor is permitted to inspect the quarantined invoices.",
            "ainglish": "The night auditor may-as-permission inspect the quarantined invoices."
        },
        {
            "force": "permission",
            "english": "The deployment witness is permitted to unlock the signed release bundle.",
            "ainglish": "The deployment witness may-as-permission unlock the signed release bundle."
        },
        {
            "force": "permission",
            "english": "The safety marshal is permitted to enter the decommissioned turbine hall.",
            "ainglish": "The safety marshal may-as-permission enter the decommissioned turbine hall."
        },
        {
            "force": "permission",
            "english": "The records clerk is permitted to copy the sealed hearing transcript.",
            "ainglish": "The records clerk may-as-permission copy the sealed hearing transcript."
        },
        {
            "force": "permission",
            "english": "The recovery operator is permitted to mount the offline backup volume.",
            "ainglish": "The recovery operator may-as-permission mount the offline backup volume."
        },
        {
            "force": "permission",
            "english": "The laboratory custodian is permitted to move the retained tissue samples.",
            "ainglish": "The laboratory custodian may-as-permission move the retained tissue samples."
        },
        {
            "force": "permission",
            "english": "The appeals reviewer is permitted to read the unredacted case appendix.",
            "ainglish": "The appeals reviewer may-as-permission read the unredacted case appendix."
        },
        {
            "force": "permission",
            "english": "The traffic coordinator is permitted to reroute vehicles through the service lane.",
            "ainglish": "The traffic coordinator may-as-permission reroute vehicles through the service lane."
        },
        {
            "force": "permission",
            "english": "The escrow agent is permitted to release the disputed maintenance deposit.",
            "ainglish": "The escrow agent may-as-permission release the disputed maintenance deposit."
        },
        {
            "force": "permission",
            "english": "The continuity lead is permitted to activate the secondary control centre.",
            "ainglish": "The continuity lead may-as-permission activate the secondary control centre."
        },
        {
            "force": "permission",
            "english": "The registry maintainer is permitted to merge the corrected ownership record.",
            "ainglish": "The registry maintainer may-as-permission merge the corrected ownership record."
        },
        {
            "force": "permission",
            "english": "The emergency dispatcher is permitted to broadcast on the protected frequency.",
            "ainglish": "The emergency dispatcher may-as-permission broadcast on the protected frequency."
        },
        {
            "force": "permission",
            "english": "The compliance observer is permitted to download the raw access ledger.",
            "ainglish": "The compliance observer may-as-permission download the raw access ledger."
        },
        {
            "force": "permission",
            "english": "The facilities engineer is permitted to isolate the eastern power bus.",
            "ainglish": "The facilities engineer may-as-permission isolate the eastern power bus."
        },
        {
            "force": "possibility",
            "english": "The coastal relay might lose power during the storm surge.",
            "ainglish": "The coastal relay may-as-possibility lose power during the storm surge."
        },
        {
            "force": "possibility",
            "english": "A stale policy replica might accept the withdrawn credential.",
            "ainglish": "A stale policy replica may-as-possibility accept the withdrawn credential."
        },
        {
            "force": "possibility",
            "english": "The overnight transfer might settle after the reporting cutoff.",
            "ainglish": "The overnight transfer may-as-possibility settle after the reporting cutoff."
        },
        {
            "force": "possibility",
            "english": "The damaged sensor might recover after thermal cycling.",
            "ainglish": "The damaged sensor may-as-possibility recover after thermal cycling."
        },
        {
            "force": "possibility",
            "english": "The revised forecast might move the flood peak into Friday.",
            "ainglish": "The revised forecast may-as-possibility move the flood peak into Friday."
        },
        {
            "force": "possibility",
            "english": "An incomplete replay might duplicate the final ledger entry.",
            "ainglish": "An incomplete replay may-as-possibility duplicate the final ledger entry."
        },
        {
            "force": "possibility",
            "english": "The reserve generator might stall during the load test.",
            "ainglish": "The reserve generator may-as-possibility stall during the load test."
        },
        {
            "force": "possibility",
            "english": "A delayed revocation might reach the gateway after authentication.",
            "ainglish": "A delayed revocation may-as-possibility reach the gateway after authentication."
        },
        {
            "force": "possibility",
            "english": "The next calibration run might expose a nonlinear drift.",
            "ainglish": "The next calibration run may-as-possibility expose a nonlinear drift."
        },
        {
            "force": "possibility",
            "english": "The archive migration might change the ordering of equal timestamps.",
            "ainglish": "The archive migration may-as-possibility change the ordering of equal timestamps."
        },
        {
            "force": "possibility",
            "english": "A second inspection might find corrosion beneath the new coating.",
            "ainglish": "A second inspection may-as-possibility find corrosion beneath the new coating."
        },
        {
            "force": "possibility",
            "english": "The isolated shard might rejoin before the repair window closes.",
            "ainglish": "The isolated shard may-as-possibility rejoin before the repair window closes."
        },
        {
            "force": "possibility",
            "english": "The currency refresh might alter the converted settlement amount.",
            "ainglish": "The currency refresh may-as-possibility alter the converted settlement amount."
        },
        {
            "force": "possibility",
            "english": "The amended route might increase the convoy's fuel requirement.",
            "ainglish": "The amended route may-as-possibility increase the convoy's fuel requirement."
        },
        {
            "force": "possibility",
            "english": "A late witness statement might change the appeal recommendation.",
            "ainglish": "A late witness statement may-as-possibility change the appeal recommendation."
        },
        {
            "force": "possibility",
            "english": "The fallback resolver might return an obsolete service address.",
            "ainglish": "The fallback resolver may-as-possibility return an obsolete service address."
        }
    ],
    "seed": "none — deterministic tokenizer counts, no sampling",
    "population": "32 fresh affirmative operational modal statements, balanced 16 per force",
    "selection": "Complete pairs were authored by Excelsior before tokenizer exposure for this run. Each permission control uses the proposal's shortest adequate 'is permitted to' comparator; each possibility control uses 'might'. Subjects, predicates, and operational domains are fresh, and exact pairs were checked against every served prior test_set immediately before mint. Negated may is excluded.",
    "method": "For each pinned tokenizer, compute len(encode(ainglish))-len(encode(english)) for every pair. Average equally within each 16-item force, then equally across forces. Report the maximum tokenizer balanced mean as the least-favourable token_delta; value_lo and value_hi are the minimum and maximum tokenizer means. File every finite outcome once regardless of sign, the proposal's +4 prediction, or agreement.",
    "estimand": {
        "population": "the 32 complete fresh pairs frozen in this manifest",
        "aggregation": "balanced force mean per tokenizer; headline is the maximum tokenizer mean",
        "comparator": {
            "permission": "is permitted to",
            "possibility": "might"
        },
        "declared_prediction": "worst-tokenizer balanced mean no greater than +4 tokens"
    },
    "environment": {
        "tiktoken": "0.13.0",
        "python": "3.12.3"
    },
    "replicates_hash": "ead8571ce276ebc166511b4a1561b4a89ccf4af7275117c036e2de63ec6383c5",
    "freeze": "The API retained these canonical manifest bytes and their SHA-256 in an open attempt before this process imported tiktoken or observed any token counts."
}