{
 "id": "Poppo/KoGEM",
 "owner": "Poppo",
 "name": "KoGEM",
 "downloads": 191,
 "likes": 0,
 "last_modified": "2025-08-24T14:07:54.000Z",
 "stated_licence": "cc-by-4.0",
 "stated_from": "tag",
 "licence_name": null,
 "licence_link": null,
 "file_licence": null,
 "licence_files": [],
 "sources": [],
 "gated": false,
 "tasks": [
  "question-answering",
  "zero-shot-classification"
 ],
 "citation": "@inproceedings{kim-etal-2025-polishing,\n    title = \"Polishing Every Facet of the {GEM}: Testing Linguistic Competence of {LLM}s and Humans in {K}orean\",\n    author = \"Kim, SungHo  and\n      Kim, Nayeon  and\n      Jeon, Taehee  and\n      Lee, SangKeun\",\n    editor = \"Che, Wanxiang  and\n      Nabende, Joyce  and\n      Shutova, Ekaterina  and\n      Pilehvar, Mohammad Taher\",\n    booktitle = \"Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)\",\n    month = jul,\n    year = \"2025\",\n    address = \"Vienna, Austria\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2025.acl-long.492/\",\n    doi = \"10.18653/v1/2025.acl-long.492\",\n    pages = \"9955--9984\",\n    ISBN = \"979-8-89176-251-0\",\n    abstract = \"We introduce the $\\underline{Ko}rean \\underline{G}rammar \\underline{E}valuation Bench\\underline{M}ark (KoGEM)$, designed to assess the linguistic competence of LLMs and humans in Korean. KoGEM consists of 1.5k multiple-choice QA pairs covering five main categories and 16 subcategories. The zero-shot evaluation of 27 LLMs of various sizes and types reveals that while LLMs perform remarkably well on straightforward tasks requiring primarily definitional knowledge, they struggle with tasks that demand the integration of real-world experiential knowledge, such as phonological rules and pronunciation. Furthermore, our in-depth analysis suggests that incorporating such experiential know",
 "signal": "permissive",
 "reason": "The declared licence (cc-by-4.0) is classified as permissive.",
 "revision": "338e8cd374466560daad1ef815a7855216323191",
 "stated_evidence": "https://huggingface.co/datasets/Poppo/KoGEM/blob/338e8cd374466560daad1ef815a7855216323191/README.md",
 "file_evidence": "https://huggingface.co/datasets/Poppo/KoGEM/tree/338e8cd374466560daad1ef815a7855216323191",
 "card_nc": false
}