{
  "name": "baba Hebrew evaluation protocol v1.0",
  "url": "https://www.itsbaba.com/methodology#protocol",
  "version": "1.0",
  "runDate": "2026-08-02",
  "cadence": "Re-run quarterly, and after any compared tool ships a major Hebrew change.",
  "license": "https://www.itsbaba.com/methodology",
  "howToRerun": "Enter each input into the tool under the stated condition and compare the result against passCriterion. Objective items are binary and need no trust in baba; judged items are an editorial call by the named evaluator.",
  "testedTools": [
    "baba",
    "Google Translate",
    "ChatGPT",
    "ChatGPT Translate",
    "Claude",
    "Gemini",
    "Grok"
  ],
  "untestedTools": [
    {
      "name": "DeepL",
      "reason": "No public API key available for this run."
    },
    {
      "name": "Reverso Context",
      "reason": "Context dictionary; no public API."
    },
    {
      "name": "Morfix Dictionary",
      "reason": "Dictionary product; no public API."
    },
    {
      "name": "DoItInHebrew",
      "reason": "No public API."
    },
    {
      "name": "iTranslate",
      "reason": "Mobile app; no public API."
    },
    {
      "name": "Lingvanex",
      "reason": "No public API key available for this run."
    }
  ],
  "outOfScope": [
    {
      "dimension": "Number of languages supported",
      "note": "General-purpose platforms support far more languages than baba. This protocol tests Hebrew only."
    },
    {
      "dimension": "Offline translation packs",
      "note": "Some competitors translate with no connection. baba does not. Not measured here."
    },
    {
      "dimension": "Document and OCR formatting",
      "note": "Preserving layout in translated PDFs and office files is a document-processing capability, not a Hebrew one."
    },
    {
      "dimension": "Open-ended writing assistance",
      "note": "General assistants do drafting, summarising, and rewriting well beyond translation. Out of scope."
    },
    {
      "dimension": "Price and plan structure",
      "note": "Pricing changes constantly and is not a translation-quality signal. Access is tested; value for money is not."
    }
  ],
  "items": [
    {
      "id": "gender-speaker",
      "factor": "Gender control",
      "input": "I am going home.",
      "condition": "Speaker: woman",
      "trap": "English \"I\" carries no gender. Hebrew conjugates the verb to the speaker, so the tool must be told who is talking.",
      "pass": "Feminine verb form: אני הולכת הביתה, not אני הולך הביתה.",
      "verifiable": "objective"
    },
    {
      "id": "gender-listener",
      "factor": "Gender control",
      "input": "Are you tired?",
      "condition": "Listener: woman",
      "trap": "English \"you\" carries no gender. Hebrew has separate masculine and feminine second-person forms.",
      "pass": "Feminine address: את עייפה?, not אתה עייף?",
      "verifiable": "objective"
    },
    {
      "id": "gender-both-roles",
      "factor": "Gender control",
      "input": "I miss you.",
      "condition": "Speaker: woman · Listener: man",
      "trap": "Both roles are marked at once, and they differ. The verb must agree with the speaker while the preposition agrees with the listener. A tool that models only one role gets exactly half of this right.",
      "pass": "אני מתגעגעת אליך: feminine verb (speaker), masculine preposition (listener).",
      "verifiable": "objective"
    },
    {
      "id": "idiom-haval-al-hazman",
      "factor": "Slang and idiom handling",
      "input": "חבל על הזמן",
      "trap": "Literally \"a shame about the time\". In current Israeli usage it is overwhelmingly an intensifier meaning outstanding. A word-for-word engine inverts the meaning completely.",
      "pass": "Rendered as praise: \"amazing\", \"incredible\", \"unreal\". Rendering it as \"a waste of time\" is a failure.",
      "verifiable": "judged"
    },
    {
      "id": "idiom-ein-matzav",
      "factor": "Slang and idiom handling",
      "input": "אין מצב",
      "trap": "Literally \"there is no situation\". Idiomatically \"no way\": either refusal or disbelief.",
      "pass": "Rendered as \"no way\" or an equivalent. A literal \"there is no situation\" is a failure.",
      "verifiable": "judged"
    },
    {
      "id": "register-shift",
      "factor": "Cultural context and tone",
      "input": "Can you send me the file?",
      "condition": "Run twice: once addressed to a manager, once to a close friend",
      "trap": "The English is identical in both cases. Hebrew registers are not, and a tool with no notion of relationship returns the same string twice.",
      "pass": "The two outputs differ in register. Identical output for both is a failure.",
      "verifiable": "judged"
    },
    {
      "id": "natural-register",
      "factor": "Natural Israeli register",
      "input": "It was really nice to meet you.",
      "condition": "Listener: man · casual setting",
      "trap": "Produces grammatical but stiff textbook Hebrew that an Israeli would not actually say in a casual setting.",
      "pass": "Reads as something a person would say aloud in Israel, not as translated-sounding Hebrew.",
      "verifiable": "judged"
    },
    {
      "id": "transliteration",
      "factor": "Hebrew-specific features",
      "input": "Where is the train station?",
      "trap": "A user who cannot read the Hebrew alphabet cannot use the output at all without a Latin-script pronunciation guide.",
      "pass": "A transliteration is offered alongside the Hebrew without extra steps.",
      "verifiable": "objective"
    },
    {
      "id": "rtl-mixed-script",
      "factor": "Hebrew-specific features",
      "input": "Send the invoice to dan@gmail.com tomorrow.",
      "trap": "Latin-script tokens embedded in right-to-left Hebrew are a classic bidirectional-rendering failure: the address gets visually reordered or split.",
      "pass": "The address survives intact and in the correct order inside the Hebrew sentence.",
      "verifiable": "objective"
    },
    {
      "id": "access-no-login",
      "factor": "Access and cost",
      "input": "Any of the items above, in a logged-out browser session.",
      "trap": "A tool that requires an account before it will translate anything is not usable at the moment of need.",
      "pass": "The translation completes with no account and no payment.",
      "verifiable": "objective"
    }
  ],
  "results": [
    {
      "itemId": "gender-speaker",
      "factor": "Gender control",
      "kind": "objective",
      "input": "I am going home.",
      "condition": "Speaker: woman",
      "passCriterion": "Feminine verb form: אני הולכת הביתה, not אני הולך הביתה.",
      "tool": "baba",
      "pass": true,
      "output": "",
      "outputWithNoRoleSet": "אני הולך הביתה.",
      "hasGenderControl": true,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-speaker",
      "factor": "Gender control",
      "kind": "objective",
      "input": "I am going home.",
      "condition": "Speaker: woman",
      "passCriterion": "Feminine verb form: אני הולכת הביתה, not אני הולך הביתה.",
      "tool": "Google Translate",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אני הולך הביתה.",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": false
    },
    {
      "itemId": "gender-speaker",
      "factor": "Gender control",
      "kind": "objective",
      "input": "I am going home.",
      "condition": "Speaker: woman",
      "passCriterion": "Feminine verb form: אני הולכת הביתה, not אני הולך הביתה.",
      "tool": "ChatGPT",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אני הולך הביתה.",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-speaker",
      "factor": "Gender control",
      "kind": "objective",
      "input": "I am going home.",
      "condition": "Speaker: woman",
      "passCriterion": "Feminine verb form: אני הולכת הביתה, not אני הולך הביתה.",
      "tool": "ChatGPT Translate",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אני הולך הביתה.",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-speaker",
      "factor": "Gender control",
      "kind": "objective",
      "input": "I am going home.",
      "condition": "Speaker: woman",
      "passCriterion": "Feminine verb form: אני הולכת הביתה, not אני הולך הביתה.",
      "tool": "Claude",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אני הולך הביתה.",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-speaker",
      "factor": "Gender control",
      "kind": "objective",
      "input": "I am going home.",
      "condition": "Speaker: woman",
      "passCriterion": "Feminine verb form: אני הולכת הביתה, not אני הולך הביתה.",
      "tool": "Gemini",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אני הולך הביתה.",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-speaker",
      "factor": "Gender control",
      "kind": "objective",
      "input": "I am going home.",
      "condition": "Speaker: woman",
      "passCriterion": "Feminine verb form: אני הולכת הביתה, not אני הולך הביתה.",
      "tool": "Grok",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אני הולך הביתה.",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-listener",
      "factor": "Gender control",
      "kind": "objective",
      "input": "Are you tired?",
      "condition": "Listener: woman",
      "passCriterion": "Feminine address: את עייפה?, not אתה עייף?",
      "tool": "baba",
      "pass": true,
      "output": "",
      "outputWithNoRoleSet": "אתה עייף?",
      "hasGenderControl": true,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-listener",
      "factor": "Gender control",
      "kind": "objective",
      "input": "Are you tired?",
      "condition": "Listener: woman",
      "passCriterion": "Feminine address: את עייפה?, not אתה עייף?",
      "tool": "Google Translate",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "האם אתה עייף?",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": false
    },
    {
      "itemId": "gender-listener",
      "factor": "Gender control",
      "kind": "objective",
      "input": "Are you tired?",
      "condition": "Listener: woman",
      "passCriterion": "Feminine address: את עייפה?, not אתה עייף?",
      "tool": "ChatGPT",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "האם אתה עייף?",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-listener",
      "factor": "Gender control",
      "kind": "objective",
      "input": "Are you tired?",
      "condition": "Listener: woman",
      "passCriterion": "Feminine address: את עייפה?, not אתה עייף?",
      "tool": "ChatGPT Translate",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "האם אתה עייף?",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-listener",
      "factor": "Gender control",
      "kind": "objective",
      "input": "Are you tired?",
      "condition": "Listener: woman",
      "passCriterion": "Feminine address: את עייפה?, not אתה עייף?",
      "tool": "Claude",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אתה עצוף?\n\nWait, let me correct that.\n\nהאם אתה עייף?",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-listener",
      "factor": "Gender control",
      "kind": "objective",
      "input": "Are you tired?",
      "condition": "Listener: woman",
      "passCriterion": "Feminine address: את עייפה?, not אתה עייף?",
      "tool": "Gemini",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אתה עייף?",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-listener",
      "factor": "Gender control",
      "kind": "objective",
      "input": "Are you tired?",
      "condition": "Listener: woman",
      "passCriterion": "Feminine address: את עייפה?, not אתה עייף?",
      "tool": "Grok",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אתה עייף?",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-both-roles",
      "factor": "Gender control",
      "kind": "objective",
      "input": "I miss you.",
      "condition": "Speaker: woman · Listener: man",
      "passCriterion": "אני מתגעגעת אליך: feminine verb (speaker), masculine preposition (listener).",
      "tool": "baba",
      "pass": true,
      "output": "",
      "outputWithNoRoleSet": "אני מתגעגע אליך.",
      "hasGenderControl": true,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-both-roles",
      "factor": "Gender control",
      "kind": "objective",
      "input": "I miss you.",
      "condition": "Speaker: woman · Listener: man",
      "passCriterion": "אני מתגעגעת אליך: feminine verb (speaker), masculine preposition (listener).",
      "tool": "Google Translate",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אני מתגעגע אליך.",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": false
    },
    {
      "itemId": "gender-both-roles",
      "factor": "Gender control",
      "kind": "objective",
      "input": "I miss you.",
      "condition": "Speaker: woman · Listener: man",
      "passCriterion": "אני מתגעגעת אליך: feminine verb (speaker), masculine preposition (listener).",
      "tool": "ChatGPT",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אני מתגעגע אליך.",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-both-roles",
      "factor": "Gender control",
      "kind": "objective",
      "input": "I miss you.",
      "condition": "Speaker: woman · Listener: man",
      "passCriterion": "אני מתגעגעת אליך: feminine verb (speaker), masculine preposition (listener).",
      "tool": "ChatGPT Translate",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אני מתגעגע אליך.",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-both-roles",
      "factor": "Gender control",
      "kind": "objective",
      "input": "I miss you.",
      "condition": "Speaker: woman · Listener: man",
      "passCriterion": "אני מתגעגעת אליך: feminine verb (speaker), masculine preposition (listener).",
      "tool": "Claude",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אני מתגעגע אליך.",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-both-roles",
      "factor": "Gender control",
      "kind": "objective",
      "input": "I miss you.",
      "condition": "Speaker: woman · Listener: man",
      "passCriterion": "אני מתגעגעת אליך: feminine verb (speaker), masculine preposition (listener).",
      "tool": "Gemini",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אני מתגעגע אליך",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "gender-both-roles",
      "factor": "Gender control",
      "kind": "objective",
      "input": "I miss you.",
      "condition": "Speaker: woman · Listener: man",
      "passCriterion": "אני מתגעגעת אליך: feminine verb (speaker), masculine preposition (listener).",
      "tool": "Grok",
      "pass": false,
      "output": "",
      "outputWithNoRoleSet": "אני מתגעגע אליך.",
      "hasGenderControl": false,
      "correctWhenRoleIsSet": true
    },
    {
      "itemId": "idiom-haval-al-hazman",
      "factor": "Slang and idiom handling",
      "kind": "judged",
      "input": "חבל על הזמן",
      "condition": "",
      "passCriterion": "Rendered as praise: \"amazing\", \"incredible\", \"unreal\". Rendering it as \"a waste of time\" is a failure.",
      "tool": "baba",
      "pass": true,
      "output": "It’s amazing (Hebrew slang, literally “a waste of time”).",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "idiom-haval-al-hazman",
      "factor": "Slang and idiom handling",
      "kind": "judged",
      "input": "חבל על הזמן",
      "condition": "",
      "passCriterion": "Rendered as praise: \"amazing\", \"incredible\", \"unreal\". Rendering it as \"a waste of time\" is a failure.",
      "tool": "Google Translate",
      "pass": false,
      "output": "It's a waste of time",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "idiom-haval-al-hazman",
      "factor": "Slang and idiom handling",
      "kind": "judged",
      "input": "חבל על הזמן",
      "condition": "",
      "passCriterion": "Rendered as praise: \"amazing\", \"incredible\", \"unreal\". Rendering it as \"a waste of time\" is a failure.",
      "tool": "ChatGPT",
      "pass": false,
      "output": "What a waste of time.",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "idiom-haval-al-hazman",
      "factor": "Slang and idiom handling",
      "kind": "judged",
      "input": "חבל על הזמן",
      "condition": "",
      "passCriterion": "Rendered as praise: \"amazing\", \"incredible\", \"unreal\". Rendering it as \"a waste of time\" is a failure.",
      "tool": "ChatGPT Translate",
      "pass": false,
      "output": "What a waste of time.",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "idiom-haval-al-hazman",
      "factor": "Slang and idiom handling",
      "kind": "judged",
      "input": "חבל על הזמן",
      "condition": "",
      "passCriterion": "Rendered as praise: \"amazing\", \"incredible\", \"unreal\". Rendering it as \"a waste of time\" is a failure.",
      "tool": "Claude",
      "pass": false,
      "output": "What a waste of time",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "idiom-haval-al-hazman",
      "factor": "Slang and idiom handling",
      "kind": "judged",
      "input": "חבל על הזמן",
      "condition": "",
      "passCriterion": "Rendered as praise: \"amazing\", \"incredible\", \"unreal\". Rendering it as \"a waste of time\" is a failure.",
      "tool": "Gemini",
      "pass": false,
      "output": "A waste of time",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "idiom-haval-al-hazman",
      "factor": "Slang and idiom handling",
      "kind": "judged",
      "input": "חבל על הזמן",
      "condition": "",
      "passCriterion": "Rendered as praise: \"amazing\", \"incredible\", \"unreal\". Rendering it as \"a waste of time\" is a failure.",
      "tool": "Grok",
      "pass": true,
      "output": "Awesome / Amazing",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "idiom-ein-matzav",
      "factor": "Slang and idiom handling",
      "kind": "judged",
      "input": "אין מצב",
      "condition": "",
      "passCriterion": "Rendered as \"no way\" or an equivalent. A literal \"there is no situation\" is a failure.",
      "tool": "baba",
      "pass": true,
      "output": "No way.",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "idiom-ein-matzav",
      "factor": "Slang and idiom handling",
      "kind": "judged",
      "input": "אין מצב",
      "condition": "",
      "passCriterion": "Rendered as \"no way\" or an equivalent. A literal \"there is no situation\" is a failure.",
      "tool": "Google Translate",
      "pass": true,
      "output": "no way",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "idiom-ein-matzav",
      "factor": "Slang and idiom handling",
      "kind": "judged",
      "input": "אין מצב",
      "condition": "",
      "passCriterion": "Rendered as \"no way\" or an equivalent. A literal \"there is no situation\" is a failure.",
      "tool": "ChatGPT",
      "pass": true,
      "output": "No way.",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "idiom-ein-matzav",
      "factor": "Slang and idiom handling",
      "kind": "judged",
      "input": "אין מצב",
      "condition": "",
      "passCriterion": "Rendered as \"no way\" or an equivalent. A literal \"there is no situation\" is a failure.",
      "tool": "ChatGPT Translate",
      "pass": true,
      "output": "No way.",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "idiom-ein-matzav",
      "factor": "Slang and idiom handling",
      "kind": "judged",
      "input": "אין מצב",
      "condition": "",
      "passCriterion": "Rendered as \"no way\" or an equivalent. A literal \"there is no situation\" is a failure.",
      "tool": "Claude",
      "pass": true,
      "output": "No way",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "idiom-ein-matzav",
      "factor": "Slang and idiom handling",
      "kind": "judged",
      "input": "אין מצב",
      "condition": "",
      "passCriterion": "Rendered as \"no way\" or an equivalent. A literal \"there is no situation\" is a failure.",
      "tool": "Gemini",
      "pass": true,
      "output": "No way",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "idiom-ein-matzav",
      "factor": "Slang and idiom handling",
      "kind": "judged",
      "input": "אין מצב",
      "condition": "",
      "passCriterion": "Rendered as \"no way\" or an equivalent. A literal \"there is no situation\" is a failure.",
      "tool": "Grok",
      "pass": true,
      "output": "No way",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "register-shift",
      "factor": "Cultural context and tone",
      "kind": "judged",
      "input": "Can you send me the file?",
      "condition": "Run twice: once addressed to a manager, once to a close friend",
      "passCriterion": "The two outputs differ in register. Identical output for both is a failure.",
      "tool": "baba",
      "pass": true,
      "output": "האם תוכל לשלוח לי את הקובץ?  /  אתה יכול לשלוח לי את הקובץ?",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "register-shift",
      "factor": "Cultural context and tone",
      "kind": "judged",
      "input": "Can you send me the file?",
      "condition": "Run twice: once addressed to a manager, once to a close friend",
      "passCriterion": "The two outputs differ in register. Identical output for both is a failure.",
      "tool": "Google Translate",
      "pass": false,
      "output": "אתה יכול לשלוח לי את הקובץ?  /  אתה יכול לשלוח לי את הקובץ?",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "register-shift",
      "factor": "Cultural context and tone",
      "kind": "judged",
      "input": "Can you send me the file?",
      "condition": "Run twice: once addressed to a manager, once to a close friend",
      "passCriterion": "The two outputs differ in register. Identical output for both is a failure.",
      "tool": "ChatGPT",
      "pass": true,
      "output": "האם תוכל לשלוח לי את הקובץ?  /  אתה יכול לשלוח לי את הקובץ?",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "register-shift",
      "factor": "Cultural context and tone",
      "kind": "judged",
      "input": "Can you send me the file?",
      "condition": "Run twice: once addressed to a manager, once to a close friend",
      "passCriterion": "The two outputs differ in register. Identical output for both is a failure.",
      "tool": "ChatGPT Translate",
      "pass": true,
      "output": "האם תוכל לשלוח לי את הקובץ?  /  אתה יכול לשלוח לי את הקובץ?",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "register-shift",
      "factor": "Cultural context and tone",
      "kind": "judged",
      "input": "Can you send me the file?",
      "condition": "Run twice: once addressed to a manager, once to a close friend",
      "passCriterion": "The two outputs differ in register. Identical output for both is a failure.",
      "tool": "Claude",
      "pass": true,
      "output": "האם תוכל לשלוח לי את הקובץ?  /  אתה יכול לשלוח לי את הקובץ?",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "register-shift",
      "factor": "Cultural context and tone",
      "kind": "judged",
      "input": "Can you send me the file?",
      "condition": "Run twice: once addressed to a manager, once to a close friend",
      "passCriterion": "The two outputs differ in register. Identical output for both is a failure.",
      "tool": "Gemini",
      "pass": true,
      "output": "האם תוכל לשלוח לי את הקובץ?  /  תוכל לשלוח לי את הקובץ?",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "register-shift",
      "factor": "Cultural context and tone",
      "kind": "judged",
      "input": "Can you send me the file?",
      "condition": "Run twice: once addressed to a manager, once to a close friend",
      "passCriterion": "The two outputs differ in register. Identical output for both is a failure.",
      "tool": "Grok",
      "pass": true,
      "output": "האם תוכל לשלוח לי את הקובץ?  /  תשלח לי את הקובץ?",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "natural-register",
      "factor": "Natural Israeli register",
      "kind": "judged",
      "input": "It was really nice to meet you.",
      "condition": "Listener: man · casual setting",
      "passCriterion": "Reads as something a person would say aloud in Israel, not as translated-sounding Hebrew.",
      "tool": "baba",
      "pass": null,
      "output": "",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "natural-register",
      "factor": "Natural Israeli register",
      "kind": "judged",
      "input": "It was really nice to meet you.",
      "condition": "Listener: man · casual setting",
      "passCriterion": "Reads as something a person would say aloud in Israel, not as translated-sounding Hebrew.",
      "tool": "Google Translate",
      "pass": null,
      "output": "",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "natural-register",
      "factor": "Natural Israeli register",
      "kind": "judged",
      "input": "It was really nice to meet you.",
      "condition": "Listener: man · casual setting",
      "passCriterion": "Reads as something a person would say aloud in Israel, not as translated-sounding Hebrew.",
      "tool": "ChatGPT",
      "pass": null,
      "output": "",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "natural-register",
      "factor": "Natural Israeli register",
      "kind": "judged",
      "input": "It was really nice to meet you.",
      "condition": "Listener: man · casual setting",
      "passCriterion": "Reads as something a person would say aloud in Israel, not as translated-sounding Hebrew.",
      "tool": "ChatGPT Translate",
      "pass": null,
      "output": "",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "natural-register",
      "factor": "Natural Israeli register",
      "kind": "judged",
      "input": "It was really nice to meet you.",
      "condition": "Listener: man · casual setting",
      "passCriterion": "Reads as something a person would say aloud in Israel, not as translated-sounding Hebrew.",
      "tool": "Claude",
      "pass": null,
      "output": "",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "natural-register",
      "factor": "Natural Israeli register",
      "kind": "judged",
      "input": "It was really nice to meet you.",
      "condition": "Listener: man · casual setting",
      "passCriterion": "Reads as something a person would say aloud in Israel, not as translated-sounding Hebrew.",
      "tool": "Gemini",
      "pass": null,
      "output": "",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "natural-register",
      "factor": "Natural Israeli register",
      "kind": "judged",
      "input": "It was really nice to meet you.",
      "condition": "Listener: man · casual setting",
      "passCriterion": "Reads as something a person would say aloud in Israel, not as translated-sounding Hebrew.",
      "tool": "Grok",
      "pass": null,
      "output": "",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "transliteration",
      "factor": "Hebrew-specific features",
      "kind": "objective",
      "input": "Where is the train station?",
      "condition": "",
      "passCriterion": "A transliteration is offered alongside the Hebrew without extra steps.",
      "tool": "baba",
      "pass": true,
      "output": "איפה תחנת הרכבת?",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "transliteration",
      "factor": "Hebrew-specific features",
      "kind": "objective",
      "input": "Where is the train station?",
      "condition": "",
      "passCriterion": "A transliteration is offered alongside the Hebrew without extra steps.",
      "tool": "Google Translate",
      "pass": true,
      "output": "איפה תחנת הרכבת?",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "transliteration",
      "factor": "Hebrew-specific features",
      "kind": "objective",
      "input": "Where is the train station?",
      "condition": "",
      "passCriterion": "A transliteration is offered alongside the Hebrew without extra steps.",
      "tool": "ChatGPT",
      "pass": false,
      "output": "איפה תחנת הרכבת?",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "transliteration",
      "factor": "Hebrew-specific features",
      "kind": "objective",
      "input": "Where is the train station?",
      "condition": "",
      "passCriterion": "A transliteration is offered alongside the Hebrew without extra steps.",
      "tool": "ChatGPT Translate",
      "pass": false,
      "output": "איפה תחנת הרכבת?",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "transliteration",
      "factor": "Hebrew-specific features",
      "kind": "objective",
      "input": "Where is the train station?",
      "condition": "",
      "passCriterion": "A transliteration is offered alongside the Hebrew without extra steps.",
      "tool": "Claude",
      "pass": false,
      "output": "איפה תחנת הרכבת?",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "transliteration",
      "factor": "Hebrew-specific features",
      "kind": "objective",
      "input": "Where is the train station?",
      "condition": "",
      "passCriterion": "A transliteration is offered alongside the Hebrew without extra steps.",
      "tool": "Gemini",
      "pass": false,
      "output": "איפה תחנת הרכבת?",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "transliteration",
      "factor": "Hebrew-specific features",
      "kind": "objective",
      "input": "Where is the train station?",
      "condition": "",
      "passCriterion": "A transliteration is offered alongside the Hebrew without extra steps.",
      "tool": "Grok",
      "pass": false,
      "output": "איפה תחנת הרכבת?",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "rtl-mixed-script",
      "factor": "Hebrew-specific features",
      "kind": "objective",
      "input": "Send the invoice to dan@gmail.com tomorrow.",
      "condition": "",
      "passCriterion": "The address survives intact and in the correct order inside the Hebrew sentence.",
      "tool": "baba",
      "pass": true,
      "output": "שלח את החשבונית ל-dan@gmail.com מחר.",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "rtl-mixed-script",
      "factor": "Hebrew-specific features",
      "kind": "objective",
      "input": "Send the invoice to dan@gmail.com tomorrow.",
      "condition": "",
      "passCriterion": "The address survives intact and in the correct order inside the Hebrew sentence.",
      "tool": "Google Translate",
      "pass": true,
      "output": "שלחו את החשבונית ל-dan@gmail.com מחר.",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "rtl-mixed-script",
      "factor": "Hebrew-specific features",
      "kind": "objective",
      "input": "Send the invoice to dan@gmail.com tomorrow.",
      "condition": "",
      "passCriterion": "The address survives intact and in the correct order inside the Hebrew sentence.",
      "tool": "ChatGPT",
      "pass": true,
      "output": "שלח את החשבונית ל־dan@gmail.com מחר.",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "rtl-mixed-script",
      "factor": "Hebrew-specific features",
      "kind": "objective",
      "input": "Send the invoice to dan@gmail.com tomorrow.",
      "condition": "",
      "passCriterion": "The address survives intact and in the correct order inside the Hebrew sentence.",
      "tool": "ChatGPT Translate",
      "pass": true,
      "output": "שלח את החשבונית ל־dan@gmail.com מחר.",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "rtl-mixed-script",
      "factor": "Hebrew-specific features",
      "kind": "objective",
      "input": "Send the invoice to dan@gmail.com tomorrow.",
      "condition": "",
      "passCriterion": "The address survives intact and in the correct order inside the Hebrew sentence.",
      "tool": "Claude",
      "pass": true,
      "output": "שלח את החשבונית ל-dan@gmail.com מחר.",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "rtl-mixed-script",
      "factor": "Hebrew-specific features",
      "kind": "objective",
      "input": "Send the invoice to dan@gmail.com tomorrow.",
      "condition": "",
      "passCriterion": "The address survives intact and in the correct order inside the Hebrew sentence.",
      "tool": "Gemini",
      "pass": true,
      "output": "שלח את החשבונית ל-dan@gmail.com מחר.",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "rtl-mixed-script",
      "factor": "Hebrew-specific features",
      "kind": "objective",
      "input": "Send the invoice to dan@gmail.com tomorrow.",
      "condition": "",
      "passCriterion": "The address survives intact and in the correct order inside the Hebrew sentence.",
      "tool": "Grok",
      "pass": true,
      "output": "שלח את החשבונית ל-dan@gmail.com מחר.",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "access-no-login",
      "factor": "Access and cost",
      "kind": "objective",
      "input": "Any of the items above, in a logged-out browser session.",
      "condition": "",
      "passCriterion": "The translation completes with no account and no payment.",
      "tool": "baba",
      "pass": null,
      "output": "",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "access-no-login",
      "factor": "Access and cost",
      "kind": "objective",
      "input": "Any of the items above, in a logged-out browser session.",
      "condition": "",
      "passCriterion": "The translation completes with no account and no payment.",
      "tool": "Google Translate",
      "pass": null,
      "output": "",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "access-no-login",
      "factor": "Access and cost",
      "kind": "objective",
      "input": "Any of the items above, in a logged-out browser session.",
      "condition": "",
      "passCriterion": "The translation completes with no account and no payment.",
      "tool": "ChatGPT",
      "pass": null,
      "output": "",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "access-no-login",
      "factor": "Access and cost",
      "kind": "objective",
      "input": "Any of the items above, in a logged-out browser session.",
      "condition": "",
      "passCriterion": "The translation completes with no account and no payment.",
      "tool": "ChatGPT Translate",
      "pass": null,
      "output": "",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "access-no-login",
      "factor": "Access and cost",
      "kind": "objective",
      "input": "Any of the items above, in a logged-out browser session.",
      "condition": "",
      "passCriterion": "The translation completes with no account and no payment.",
      "tool": "Claude",
      "pass": null,
      "output": "",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "access-no-login",
      "factor": "Access and cost",
      "kind": "objective",
      "input": "Any of the items above, in a logged-out browser session.",
      "condition": "",
      "passCriterion": "The translation completes with no account and no payment.",
      "tool": "Gemini",
      "pass": null,
      "output": "",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    },
    {
      "itemId": "access-no-login",
      "factor": "Access and cost",
      "kind": "objective",
      "input": "Any of the items above, in a logged-out browser session.",
      "condition": "",
      "passCriterion": "The translation completes with no account and no payment.",
      "tool": "Grok",
      "pass": null,
      "output": "",
      "outputWithNoRoleSet": "",
      "hasGenderControl": null,
      "correctWhenRoleIsSet": null
    }
  ]
}