Current-tokenizer cost (Δ, worst tokenizer)

These are the committed bytes rendered as readable JSON. Expanding this audit detail does not change the measurement’s current status.

{
    "metric": "token_delta",
    "formula_version": 1,
    "construct": "we-including-you / we-excluding-you",
    "models": [
        "cl100k_base",
        "o200k_base",
        "p50k_base"
    ],
    "test_set": [
        {
            "form": "we-including-you",
            "ainglish": "we-including-you will cross-check the orchard frost sensors before sunrise.",
            "english": "we, and that includes you, will cross-check the orchard frost sensors before sunrise."
        },
        {
            "form": "we-including-you",
            "ainglish": "we-including-you must catalog the borrowed field microscopes before return.",
            "english": "we, and that includes you, must catalog the borrowed field microscopes before return."
        },
        {
            "form": "we-including-you",
            "ainglish": "we-including-you should review the ferry loading sequence after rehearsal.",
            "english": "we, and that includes you, should review the ferry loading sequence after rehearsal."
        },
        {
            "form": "we-including-you",
            "ainglish": "we-including-you can inspect the community battery dashboard during the workshop.",
            "english": "we, and that includes you, can inspect the community battery dashboard during the workshop."
        },
        {
            "form": "we-including-you",
            "ainglish": "we-including-you need to map the accessible entrances before the festival.",
            "english": "we, and that includes you, need to map the accessible entrances before the festival."
        },
        {
            "form": "we-including-you",
            "ainglish": "we-including-you will calibrate the portable air monitors beside the school.",
            "english": "we, and that includes you, will calibrate the portable air monitors beside the school."
        },
        {
            "form": "we-including-you",
            "ainglish": "we-including-you must compare the archive moisture readings at noon.",
            "english": "we, and that includes you, must compare the archive moisture readings at noon."
        },
        {
            "form": "we-including-you",
            "ainglish": "we-including-you may attend the coastal erosion survey on Saturday.",
            "english": "we, and that includes you, may attend the coastal erosion survey on Saturday."
        },
        {
            "form": "we-including-you",
            "ainglish": "we-including-you should reconcile the bilingual signage inventory before printing.",
            "english": "we, and that includes you, should reconcile the bilingual signage inventory before printing."
        },
        {
            "form": "we-including-you",
            "ainglish": "we-including-you will test the emergency radio relay from the hilltop.",
            "english": "we, and that includes you, will test the emergency radio relay from the hilltop."
        },
        {
            "form": "we-including-you",
            "ainglish": "we-including-you need to approve the volunteer rota before registration opens.",
            "english": "we, and that includes you, need to approve the volunteer rota before registration opens."
        },
        {
            "form": "we-including-you",
            "ainglish": "we-including-you can challenge the draft trail closure plan during consultation.",
            "english": "we, and that includes you, can challenge the draft trail closure plan during consultation."
        },
        {
            "form": "we-excluding-you",
            "ainglish": "we-excluding-you will decode the sealed donor records before the trustees meet.",
            "english": "we, not including you, will decode the sealed donor records before the trustees meet."
        },
        {
            "form": "we-excluding-you",
            "ainglish": "we-excluding-you must review the confidential hiring notes inside the secure room.",
            "english": "we, not including you, must review the confidential hiring notes inside the secure room."
        },
        {
            "form": "we-excluding-you",
            "ainglish": "we-excluding-you should rotate the private signing key after the incident.",
            "english": "we, not including you, should rotate the private signing key after the incident."
        },
        {
            "form": "we-excluding-you",
            "ainglish": "we-excluding-you can inspect the embargoed prototype during the closed session.",
            "english": "we, not including you, can inspect the embargoed prototype during the closed session."
        },
        {
            "form": "we-excluding-you",
            "ainglish": "we-excluding-you need to redact the protected address list before disclosure.",
            "english": "we, not including you, need to redact the protected address list before disclosure."
        },
        {
            "form": "we-excluding-you",
            "ainglish": "we-excluding-you will negotiate the unreleased licensing terms tomorrow.",
            "english": "we, not including you, will negotiate the unreleased licensing terms tomorrow."
        },
        {
            "form": "we-excluding-you",
            "ainglish": "we-excluding-you must reconcile the privileged audit trail before counsel arrives.",
            "english": "we, not including you, must reconcile the privileged audit trail before counsel arrives."
        },
        {
            "form": "we-excluding-you",
            "ainglish": "we-excluding-you may enter the quarantined lab while the alarm is disabled.",
            "english": "we, not including you, may enter the quarantined lab while the alarm is disabled."
        },
        {
            "form": "we-excluding-you",
            "ainglish": "we-excluding-you should revoke the hidden service credentials before migration.",
            "english": "we, not including you, should revoke the hidden service credentials before migration."
        },
        {
            "form": "we-excluding-you",
            "ainglish": "we-excluding-you will brief the mediator on the confidential settlement range.",
            "english": "we, not including you, will brief the mediator on the confidential settlement range."
        },
        {
            "form": "we-excluding-you",
            "ainglish": "we-excluding-you need to examine the unmasked safeguarding referral in private.",
            "english": "we, not including you, need to examine the unmasked safeguarding referral in private."
        },
        {
            "form": "we-excluding-you",
            "ainglish": "we-excluding-you can authorize the sealed escrow release during recovery.",
            "english": "we, not including you, can authorize the sealed escrow release during recovery."
        }
    ],
    "seed": "none — deterministic tokenizer counts, no sampling",
    "population": "24 complete fresh subject-position clauses, balanced 12 per clusivity form",
    "selection": "Predicates were frozen before tokenizer exposure. The inclusive and exclusive sets balance obligations, future actions, permissions, advice, capability, and operational domains. Every control uses the proposal's complete careful-English expansion with the predicate unchanged. Exact complete pairs were checked against the target original before mint.",
    "method": "For each pinned tokenizer, compute len(encode(ainglish))-len(encode(english)) for every complete pair. Average equally within each 12-item form stratum, then equally across forms. Report the maximum tokenizer mean as the least-favourable token_delta; value_lo and value_hi are the minimum and maximum tokenizer means. File every finite outcome once regardless of sign or agreement.",
    "estimand": {
        "population": "the 24 complete clusivity pairs frozen in this manifest",
        "aggregation": "balanced form mean per tokenizer; headline is the maximum tokenizer mean",
        "comparator": "the proposal's complete, meaning-matched careful-English expansion",
        "comparator_class": "careful_expansion"
    },
    "environment": {
        "tiktoken": "0.13.0",
        "python": "3.12.3"
    },
    "freeze": "The API retains these canonical manifest bytes and their SHA-256 in an open attempt before this process imports tiktoken or observes any token counts."
}