{
 "id": "eduagarcia/multilingual_tokenizer_benchmark",
 "owner": "eduagarcia",
 "name": "multilingual_tokenizer_benchmark",
 "downloads": 404,
 "likes": 2,
 "last_modified": "2025-07-12T17:12:01.000Z",
 "stated_licence": null,
 "stated_from": "none",
 "licence_name": null,
 "licence_link": null,
 "file_licence": null,
 "licence_files": [],
 "sources": [
  {
   "id": "bigcode/the-stack-smol",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigcode/the-stack-smol/blob/4a6938ce94446f324c6629e7de00ac591710044b/README.md"
  },
  {
   "id": "badranx/opus_raw",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/badranx/opus_raw/blob/fabb9115e9f173c9aaa4897e5b2f3a49e74758a0/README.md"
  },
  {
   "id": "HuggingFaceFW/fineweb-2",
   "licence": "odc-by",
   "found": true,
   "evidence": "https://huggingface.co/datasets/HuggingFaceFW/fineweb-2/blob/af9c13333eb981300149d5ca60a8e9d659b276b9/README.md"
  },
  {
   "id": "sebastiandizon/genius-song-lyrics",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/sebastiandizon/genius-song-lyrics/blob/065f07cbdbbce5f5b13d9a39dd872c77ad757474/README.md"
  },
  {
   "id": "rntc/pubmed_preprocess",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/rntc/pubmed_preprocess/blob/9b87d90e6720a7e071d302fafe0d4264c4e0cf46/README.md"
  },
  {
   "id": "wikimedia/wikipedia",
   "licence": "cc-by-sa-3.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/wikimedia/wikipedia/blob/b04c8d1ceb2f5cd4588862100d08de323dccfbaa/README.md"
  },
  {
   "id": "joelniklaus/Multi_Legal_Pile",
   "licence": "cc-by-nc-sa-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/joelniklaus/Multi_Legal_Pile/blob/911e1d214162fd11d2c78d3f1428cbfcbe07782c/README.md"
  },
  {
   "id": "almanach/HALvest",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/almanach/halvest/blob/258dd9479aad96d7c7b7e36139e97f637bd6c309/README.md"
  },
  {
   "id": "premio-ai/TheArabicPile_Medical",
   "licence": "cc-by-nc-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/premio-ai/TheArabicPile_Medical/blob/670fb50cb0b3e2997880fb4f9de56fcaded8944c/README.md"
  },
  {
   "id": "ImruQays/Rasaif-Classical-Arabic-English-Parallel-texts",
   "licence": "cc-by-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/ImruQays/Rasaif-Classical-Arabic-English-Parallel-texts/blob/3373ae33443ddbc93f92d439350152428d6228a8/README.md"
  },
  {
   "id": "asas-ai/financial_news",
   "licence": "apache-2.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/asas-ai/financial_news/blob/00dd476287d596e9720cb2533512c4c07698ec45/README.md"
  },
  {
   "id": "bigscience-data/roots_ar_ted_talks_iwslt",
   "licence": "cc-by-nc-nd-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigscience-data/roots_ar_ted_talks_iwslt/blob/40be87e6c1944e6244cae8d62b630a16c8904563/README.md"
  },
  {
   "id": "premio-ai/TheArabicPile_Lyrics",
   "licence": "cc-by-nc-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/premio-ai/TheArabicPile_Lyrics/blob/857a082a25f869416ac07015a4b12249e5ca1bb0/README.md"
  },
  {
   "id": "AIR-Bench/qa_news_ar",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/AIR-Bench/qa_news_ar/blob/bc0c15d3dbf34184a3a47a016a753855e57ee0d3/README.md"
  },
  {
   "id": "premio-ai/TheArabicPile_Poetry",
   "licence": "cc-by-nc-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/premio-ai/TheArabicPile_Poetry/blob/aa6e4b61b67f0253220e3e4137e2210fd345bdc9/README.md"
  },
  {
   "id": "bigscience-data/roots_ar_wikipedia",
   "licence": "cc-by-sa-3.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigscience-data/roots_ar_wikipedia/blob/3427690241eeb836fe9abd1aa3645761451f48d9/README.md"
  },
  {
   "id": "bigcode/starcoderdata",
   "licence": "other",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigcode/starcoderdata/blob/9fc30b578cedaec69e47302df72cf00feed7c8c4/README.md"
  },
  {
   "id": "AIR-Bench/qa_news_de",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/AIR-Bench/qa_news_de/blob/2d44e9d88087f4e42c35e3ded4bb680e6652666c/README.md"
  },
  {
   "id": "orionweller/books_mds_incremental",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/orionweller/books_mds_incremental/blob/aeb37952cd4e19129f86dc2ae9209774fb23c362/README.md"
  },
  {
   "id": "ElKulako/stocktwits-emoji",
   "licence": "mit",
   "found": true,
   "evidence": "https://huggingface.co/datasets/ElKulako/stocktwits-emoji/blob/a9d5e55298cbeb13414417782609c5a018124cd8/README.md"
  },
  {
   "id": "Orion-zhen/dpo-codealpaca-emoji",
   "licence": "mit",
   "found": true,
   "evidence": "https://huggingface.co/datasets/Orion-zhen/dpo-codealpaca-emoji/blob/fcc4e217ea6d43fcaa81388d0a38da351e2ef060/README.md"
  },
  {
   "id": "Orion-zhen/dpo-mathinstuct-emoji",
   "licence": "mit",
   "found": true,
   "evidence": "https://huggingface.co/datasets/Orion-zhen/dpo-mathinstuct-emoji/blob/98e7c7ec31f34ae575f8a0e26a01233a03fa67fa/README.md"
  },
  {
   "id": "Orion-zhen/dpo-physical-reasoning-emoji",
   "licence": "mit",
   "found": true,
   "evidence": "https://huggingface.co/datasets/Orion-zhen/dpo-physical-reasoning-emoji/blob/e188431db9e269e1d0aa1ed16b7f30d612c76b70/README.md"
  },
  {
   "id": "AIR-Bench/qa_finance_en",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/AIR-Bench/qa_finance_en/blob/3c8291525b021fcba51b2a1d6606cf50545a2fe8/README.md"
  },
  {
   "id": "bigscience-data/roots_en_ted_talks_iwslt",
   "licence": "cc-by-nc-nd-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigscience-data/roots_en_ted_talks_iwslt/blob/e71d6de7a5487448fb5a9a3e094c9e010a5e256c/README.md"
  },
  {
   "id": "orionweller/cc_news_mds_incremental",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/orionweller/cc_news_mds_incremental/blob/272cf9cd1765240cf9c61208aefa611266998fdc/README.md"
  },
  {
   "id": "DanFosing/public-domain-poetry",
   "licence": "cc0-1.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/DanFosing/public-domain-poetry/blob/84a87909d09ff0c3ae040c4e0af25a6344d96531/README.md"
  },
  {
   "id": "orionweller/arxiv_mds_incremental",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/orionweller/arxiv_mds_incremental/blob/2efa6b280069d4fd392f45d342fb9dc93903a438/README.md"
  },
  {
   "id": "orionweller/pes2o_mds_incremental",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/orionweller/pes2o_mds_incremental/blob/71ed50bcfd714e2360c4fcd59d601d4eecc9d1a2/README.md"
  },
  {
   "id": "orionweller/reddit_mds_incremental",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/orionweller/reddit_mds_incremental/blob/53d1edb1053ffa4b519ba45d9daed14fd82cfd68/README.md"
  },
  {
   "id": "HuggingFaceFW/fineweb",
   "licence": "odc-by",
   "found": true,
   "evidence": "https://huggingface.co/datasets/HuggingFaceFW/fineweb/blob/9bb295ddab0e05d785b879661af7260fed5140fc/README.md"
  },
  {
   "id": "orionweller/wikipedia_mds_incremental",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/orionweller/wikipedia_mds_incremental/blob/15c6e2da9ce4f660df875914a9cb6984b1b1e0ed/README.md"
  },
  {
   "id": "jorgeortizfuentes/spanish_books",
   "licence": "cc-by-sa-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/jorgeortizfuentes/spanish_books/blob/b13f7ed769444c5b010034ed8ac5f0f0b6c87af9/README.md"
  },
  {
   "id": "bigscience-data/roots_es_ted_talks_iwslt",
   "licence": "cc-by-nc-nd-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigscience-data/roots_es_ted_talks_iwslt/blob/675ce3d6f8a7da97eacfa7c6d37af59795056658/README.md"
  },
  {
   "id": "LeoCordoba/CC-NEWS-ES",
   "licence": "mit",
   "found": true,
   "evidence": "https://huggingface.co/datasets/LeoCordoba/CC-NEWS-ES/blob/ffc909433ee918658635edf15958e8e4d82cbdd6/README.md"
  },
  {
   "id": "andreamorgar/spanish_poetry",
   "licence": "gpl-3.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/andreamorgar/spanish_poetry/blob/e0f3a5567f5c8db711fce1d5dcf244000c5ab587/README.md"
  },
  {
   "id": "jorge-henao/disco_poetry_spanish",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/jorge-henao/disco_poetry_spanish/blob/689f949a36ec83a2a6f14e1fc4a52cf22a704d56/README.md"
  },
  {
   "id": "eduagarcia/scielo_abstracts",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/eduagarcia/scielo_abstracts/blob/27270a55ee179365f2c77a80ba92557bd2b02264/README.md"
  },
  {
   "id": "Arconte/Dominican_reddit_raw_corpus",
   "licence": "mit",
   "found": true,
   "evidence": "https://huggingface.co/datasets/Arconte/Dominican_reddit_raw_corpus/blob/8f71c88aa99dfa47241caf9de803fd508ee21bf5/README.md"
  },
  {
   "id": "bigscience-data/roots_es_wikipedia",
   "licence": "cc-by-sa-3.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigscience-data/roots_es_wikipedia/blob/7f97555408b362bd56ffd16474c827a43ebf2ef3/README.md"
  },
  {
   "id": "AIR-Bench/qa_finance_fr",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/AIR-Bench/qa_finance_fr/blob/3a7726a15f002942a85dd4a147cde01b56e5cb39/README.md"
  },
  {
   "id": "bigscience-data/roots_fr_ted_talks_iwslt",
   "licence": "cc-by-nc-nd-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigscience-data/roots_fr_ted_talks_iwslt/blob/a9ec2d844a44fc98d2059da3835515217fb249f6/README.md"
  },
  {
   "id": "AIR-Bench/qa_news_fr",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/AIR-Bench/qa_news_fr/blob/3f4b181f2e29a25885e4c5948af01cc5aad74352/README.md"
  },
  {
   "id": "kamarko/ccnews-french-subset",
   "licence": "cc-by-nc-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/kamarko/ccnews-french-subset/blob/443ea4b53b9551742a04c9b7ad4d583107a5ca65/README.md"
  },
  {
   "id": "manu/french_poetry",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/manu/french_poetry/blob/5e2f486995e97034ef68a711d5c8fa8a1269ae5c/README.md"
  },
  {
   "id": "bigscience-data/roots_fr_wikipedia",
   "licence": "cc-by-sa-3.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigscience-data/roots_fr_wikipedia/blob/fd0af8f43bc547d64c66013149db236e072625f6/README.md"
  },
  {
   "id": "HiTZ/Multilingual-Medical-Corpus",
   "licence": "apache-2.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/HiTZ/Multilingual-Medical-Corpus/blob/48ca9537138c0d18c396944735bf7a93a1fb292e/README.md"
  },
  {
   "id": "globis-university/aozorabunko-clean",
   "licence": "cc-by-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/globis-university/aozorabunko-clean/blob/42a9c9c0f1d67e6a5554d9bea4201973dc9b049c/README.md"
  },
  {
   "id": "geniacllm/hanrei",
   "licence": "cc0-1.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/geniacllm/hanrei/blob/8efcf8340f6af03ba0cada497db00a85959ed25b/README.md"
  },
  {
   "id": "kajuma/CC-news-2024-July-October-cleaned",
   "licence": "odc-by",
   "found": true,
   "evidence": "https://huggingface.co/datasets/kajuma/CC-news-2024-July-October-cleaned/blob/510b225698ffe38683ec52fa44eefd7bd968b2d7/README.md"
  },
  {
   "id": "p1atdev/modern_haiku",
   "licence": "mit",
   "found": true,
   "evidence": "https://huggingface.co/datasets/p1atdev/modern_haiku/blob/889a4d9072cde4d55d1af7a44ecba67b58d77b85/README.md"
  },
  {
   "id": "kunishou/J-ResearchCorpus",
   "licence": "other",
   "found": true,
   "evidence": "https://huggingface.co/datasets/kunishou/J-ResearchCorpus/blob/a1896d0d7fc771351ce170b92d9ff2b6f826bf6b/README.md"
  },
  {
   "id": "p1atdev/open2ch",
   "licence": "apache-2.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/p1atdev/open2ch/blob/8a48274854909717f0f9acd1c3400a320f57ba85/README.md"
  },
  {
   "id": "BCCard/BCCard-Finance-Kor-QnA",
   "licence": "apache-2.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/BCCard/BCCard-Finance-Kor-QnA/blob/cd803e0bc1e88b56001fad130dc67f5a9b5d9238/README.md"
  },
  {
   "id": "jihye-moon/klac_legal_aid_counseling",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/jihye-moon/klac_legal_aid_counseling/blob/2460d3a56b7a767acbfa7b77fea879332f0082d6/README.md"
  },
  {
   "id": "joonhok-exo-ai/korean_law_open_data_precedents",
   "licence": "openrail",
   "found": true,
   "evidence": "https://huggingface.co/datasets/joonhok-exo-ai/korean_law_open_data_precedents/blob/0e2049bf3f3bae288b40e95a0553923d8d5246c0/README.md"
  },
  {
   "id": "AIR-Bench/qa_news_ko",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/AIR-Bench/qa_news_ko/blob/76980f15241c9edf2b64259456e16412cab7a3be/README.md"
  },
  {
   "id": "daekeun-ml/naver-news-summarization-ko",
   "licence": "apache-2.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/daekeun-ml/naver-news-summarization-ko/blob/916dfe6aed651b2ecd9d04fa0a704f8953e90762/README.md"
  },
  {
   "id": "werty1248/Korea-Related-Reddit-comments",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/werty1248/Korea-Related-Reddit-comments/blob/93bd2d4df93508c29b5e41247ade5d0c28d38597/README.md"
  },
  {
   "id": "werty1248/Korea-Related-Reddit-posts",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/werty1248/Korea-Related-Reddit-posts/blob/a0457eb30d359f2ae8c8dd4e3c0ca79ab59897d4/README.md"
  },
  {
   "id": "WiktorS/polish-news",
   "licence": "apache-2.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/WiktorS/polish-news/blob/2e3abfe7d3fbdfe02fab92a9980fca42cac3838c/README.md"
  },
  {
   "id": "winterkitsune/elka-pl-news",
   "licence": "apache-2.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/winterkitsune/elka-pl-news/blob/cdb943d633f244d3d1c85b7a53888276496b0b20/README.md"
  },
  {
   "id": "eduagarcia/pd_books_pt",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/eduagarcia/pd_books_pt/blob/5be463ed6bec30b213c6bb5c23570a0593077615/README.md"
  },
  {
   "id": "eduagarcia/LegalPT_dedup",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/eduagarcia/LegalPT_dedup/blob/faf40c878d5a12d2729cf2fe26bb558f88231e8a/README.md"
  },
  {
   "id": "bigscience-data/roots_pt_ted_talks_iwslt",
   "licence": "cc-by-nc-nd-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigscience-data/roots_pt_ted_talks_iwslt/blob/c1858a5f6400c42283ba217bfd10cbde16cc34cc/README.md"
  },
  {
   "id": "eduagarcia/cc_news_pt_v2",
   "licence": "unknown",
   "found": true,
   "evidence": "https://huggingface.co/datasets/eduagarcia/cc_news_pt_v2/blob/4b5c5e4f47554e6631f5dce1aadf70ba2ba08069/README.md"
  },
  {
   "id": "eduagarcia/scielo_papers",
   "licence": null,
   "found": false,
   "evidence": "https://huggingface.co/datasets/eduagarcia/scielo_papers"
  },
  {
   "id": "carolina-c4ai/corpus-carolina",
   "licence": "cc-by-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/carolina-c4ai/corpus-carolina/blob/55e63a519393c70a48dcfa14a558499c6bb0583b/README.md"
  },
  {
   "id": "tallesl/quinquilharia",
   "licence": "unlicense",
   "found": true,
   "evidence": "https://huggingface.co/datasets/tallesl/quinquilharia/blob/7773fac4d8933752a50198c77e51bee184dbf286/README.md"
  },
  {
   "id": "bigscience-data/roots_pt_wikipedia",
   "licence": "cc-by-sa-3.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigscience-data/roots_pt_wikipedia/blob/7258c542b0aebc0537de200ece4fc9c107925144/README.md"
  },
  {
   "id": "blinoff/medical_qa_ru_data",
   "licence": "unknown",
   "found": true,
   "evidence": "https://huggingface.co/datasets/blinoff/medical_qa_ru_data/blob/b7c9fe729920578a60d6a294a6f6a81496d6c6fc/README.md"
  },
  {
   "id": "sevenreasons/genius-lyrics-russian",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/sevenreasons/genius-lyrics-russian/blob/236a5a939c650fa234b9456ff7e58918bc85705f/README.md"
  },
  {
   "id": "AIR-Bench/qa_news_ru",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/AIR-Bench/qa_news_ru/blob/d2c27941dad1b830f4454a1251dd4bb832f57f57/README.md"
  },
  {
   "id": "IlyaGusev/ru_news",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/IlyaGusev/ru_news/blob/d85158aae54fbdea8e52cf9541f60e7f1bbb3fa7/README.md"
  },
  {
   "id": "0x7o/poemma-10k",
   "licence": "apache-2.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/0x7o/poemma-10k/blob/d9f17677aa37207705daa8a86855b5b979b6122e/README.md"
  },
  {
   "id": "AnyaSchen/russian_poetry_with_keywords",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/AnyaSchen/russian_poetry_with_keywords/blob/573301ba20219a35166586b9619c04e891ddd801/README.md"
  },
  {
   "id": "mlsa-iai-msu-lab/ru_sci_bench",
   "licence": "mit",
   "found": true,
   "evidence": "https://huggingface.co/datasets/mlsa-iai-msu-lab/ru_sci_bench/blob/9401420f018a5cb7767c0263141ed6c8863f433b/README.md"
  },
  {
   "id": "VietAI/vi_pubmed",
   "licence": "cc",
   "found": true,
   "evidence": "https://huggingface.co/datasets/VietAI/vi_pubmed/blob/decfdcc57efa83466449ccaa658ad431a8a416d4/README.md"
  },
  {
   "id": "bigscience-data/roots_vi_ted_talks_iwslt",
   "licence": "cc-by-nc-nd-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigscience-data/roots_vi_ted_talks_iwslt/blob/3dc9b93db41ce9b5e03031a84902d663634a2f49/README.md"
  },
  {
   "id": "bigscience-data/roots_vi_binhvq_news_corpus",
   "licence": "apache-2.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigscience-data/roots_vi_binhvq_news_corpus/blob/1af265b7581d562e88dbc50e56a96aa6c0d4b3ba/README.md"
  },
  {
   "id": "bigscience-data/roots_vi_vietnamese_poetry",
   "licence": "mit",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigscience-data/roots_vi_vietnamese_poetry/blob/890a2f440eddceae92e9d77b1b2ee5a530741721/README.md"
  },
  {
   "id": "DavidLanz/medical_pretrain",
   "licence": null,
   "found": false,
   "evidence": "https://huggingface.co/datasets/DavidLanz/medical_pretrain"
  },
  {
   "id": "Orion-zhen/dpo-emoji-zh",
   "licence": "gpl-3.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/Orion-zhen/dpo-emoji-zh/blob/42d5bd54e3641a9feef7651de2c31f005a8ead12/README.md"
  },
  {
   "id": "Orion-zhen/dpo-ruozhiba-emoji",
   "licence": "mit",
   "found": true,
   "evidence": "https://huggingface.co/datasets/Orion-zhen/dpo-ruozhiba-emoji/blob/f2dcdc34b4eaa49ba736f52358c8c30787bb0ef1/README.md"
  },
  {
   "id": "shareAI/DPO-zh-en-emoji",
   "licence": "apache-2.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/shareAI/DPO-zh-en-emoji/blob/db2e31ec01aec98b1de7db6cb9dcef581f78b49e/README.md"
  },
  {
   "id": "Duxiaoman-DI/FinCorpus",
   "licence": "apache-2.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/Duxiaoman-DI/FinCorpus/blob/dae52562603787e65ee00cc95561bab4fa818d9b/README.md"
  },
  {
   "id": "TigerResearch/tigerbot-law-plugin",
   "licence": "apache-2.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/TigerResearch/tigerbot-law-plugin/blob/d6837fa6596d8c9dd1aa051c07a046597b1d5092/README.md"
  },
  {
   "id": "bigscience-data/roots_zh_ted_talks_iwslt",
   "licence": "cc-by-nc-nd-4.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigscience-data/roots_zh_ted_talks_iwslt/blob/d34d95a4267e244ada5c58021f09647510f12024/README.md"
  },
  {
   "id": "dirtycomputer/chinese_lyrics",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/dirtycomputer/chinese_lyrics/blob/7886fa9ac66b11c88ea9a5db8172b8137f7c2a6f/README.md"
  },
  {
   "id": "AIR-Bench/qa_news_zh",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/AIR-Bench/qa_news_zh/blob/2b586bcca112b4000b3df66dbf83b4925b63234d/README.md"
  },
  {
   "id": "liswei/news-collection-zhtw",
   "licence": null,
   "found": true,
   "evidence": "https://huggingface.co/datasets/liswei/news-collection-zhtw/blob/681cc6539771bf3914713389058de37f8b46754a/README.md"
  },
  {
   "id": "erhwenkuo/poetry-chinese-zhtw",
   "licence": "mit",
   "found": true,
   "evidence": "https://huggingface.co/datasets/erhwenkuo/poetry-chinese-zhtw/blob/8e53cb324c7feec2b3889140c428720d8a834831/README.md"
  },
  {
   "id": "bigscience-data/roots_zh-cn_wikipedia",
   "licence": "cc-by-sa-3.0",
   "found": true,
   "evidence": "https://huggingface.co/datasets/bigscience-data/roots_zh-cn_wikipedia/blob/ba4990a9de8e3ceaf3f15a4ee97e1aea8659caf7/README.md"
  }
 ],
 "gated": false,
 "tasks": [
  "text-generation"
 ],
 "citation": null,
 "signal": "unclear",
 "reason": "The checks could not identify a recognised licence ID in the dataset's metadata.",
 "revision": "74fa5907af0265f42c09e21379bf21847e066990",
 "stated_evidence": "https://huggingface.co/datasets/eduagarcia/multilingual_tokenizer_benchmark/blob/74fa5907af0265f42c09e21379bf21847e066990/README.md",
 "file_evidence": "https://huggingface.co/datasets/eduagarcia/multilingual_tokenizer_benchmark/tree/74fa5907af0265f42c09e21379bf21847e066990",
 "card_nc": false
}