{
 "id": "commoncrawl/CommonLID",
 "owner": "commoncrawl",
 "name": "CommonLID",
 "downloads": 370,
 "likes": 58,
 "last_modified": "2026-09-22T09:12:58.000Z",
 "stated_licence": "other",
 "stated_from": "tag",
 "licence_name": "common-crawl-tou",
 "licence_link": "https://commoncrawl.org/terms-of-use",
 "file_licence": null,
 "licence_files": [
  "LICENSE"
 ],
 "sources": [],
 "gated": "auto",
 "tasks": [
  "text-classification"
 ],
 "citation": "```text\n@inproceedings{suarez-etal-2026-commonlid,\n    title = \"{C}ommon{LID}: Re-evaluating State-of-the-Art Language Identification Performance on Web Data\",\n    author = \"Suarez, Pedro Ortiz  and\n      Burchell, Laurie  and\n      Arnett, Catherine  and\n      Mosquera, Rafael  and\n      Monsalve, Sara Hincapi{\\'e}  and\n      Vaughan, Thom  and\n      Stewart, Damian  and\n      Ostendorff, Malte  and\n      Abdulmumin, Idris  and\n      Marivate, Vukosi  and\n      Muhammad, Shamsuddeen Hassan  and\n      Tonja, Atnafu Lambebo  and\n      Al-Khalifa, Hend  and\n      Hammouda, Nadia Ghezaiel  and\n      Otiende, Verrah Akinyi  and\n      Wong, Tack Hwa  and\n      Saydaliev, Jakhongir  and\n      Nobakhtian, Melika  and\n      Habibi, Muhammad Ravi Shulthan  and\n      Kranti, Chalamalasetti  and\n      Muchemi, Carol  and\n      Nguyen, Khang  and\n      Adam, Faisal Muhammad  and\n      Salim, Luis Frentzen  and\n      Alqifari, Reem  and\n      Amol, Cynthia Jayne  and\n      Imperial, Joseph Marvin  and\n      Kesen, Ilker  and\n      Mustafid, Ahmad  and\n      Stepachev, Pavel  and\n      Choshen, Leshem  and\n      Anugraha, David  and\n      Nayel, Hamada  and\n      Yimam, Seid Muhie  and\n      Alexandra Putra, Vallerie  and\n      Nguyen, My Chiffon  and\n      Wasi, Azmine Toushik  and\n      Vadithya, Gouthami  and\n      Van Der Goot, Rob  and\n      C{'}horr, Lanwenn ar  and\n      Dua, Karan  and\n      Yates, Andrew  and\n      Bangera, Mithil  and\n      Bangera, Yeshil  and\n      Patel, Hites",
 "signal": "unclear",
 "reason": "The checks could not identify a recognised licence ID in the dataset's metadata.",
 "revision": "848125552728d584b7adcae253c8b8ceb963cf2e",
 "stated_evidence": "https://huggingface.co/datasets/commoncrawl/CommonLID/blob/848125552728d584b7adcae253c8b8ceb963cf2e/README.md",
 "file_evidence": "https://huggingface.co/datasets/commoncrawl/CommonLID/blob/848125552728d584b7adcae253c8b8ceb963cf2e/LICENSE",
 "card_nc": false
}