{
 "id": "daman1209arora/jeebench",
 "owner": "daman1209arora",
 "name": "jeebench",
 "downloads": 1882,
 "likes": 8,
 "last_modified": "2023-12-18T09:31:37.000Z",
 "stated_licence": "mit",
 "stated_from": "tag",
 "licence_name": null,
 "licence_link": null,
 "file_licence": null,
 "licence_files": [],
 "sources": [],
 "gated": false,
 "tasks": [
  "question-answering"
 ],
 "citation": "If you use our dataset in your research, please cite it using the following\n```latex\n@inproceedings{arora-etal-2023-llms,\n    title = \"Have {LLM}s Advanced Enough? A Challenging Problem Solving Benchmark For Large Language Models\",\n    author = \"Arora, Daman  and\n      Singh, Himanshu  and\n      {Mausam}\",\n    editor = \"Bouamor, Houda  and\n      Pino, Juan  and\n      Bali, Kalika\",\n    booktitle = \"Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing\",\n    month = dec,\n    year = \"2023\",\n    address = \"Singapore\",\n    publisher = \"Association for Computational Linguistics\",\n    url = \"https://aclanthology.org/2023.emnlp-main.468\",\n    doi = \"10.18653/v1/2023.emnlp-main.468\",\n    pages = \"7527--7543\",\n    abstract = \"The performance of large language models (LLMs) on existing reasoning benchmarks has significantly improved over the past years. In response, we present JEEBench, a considerably more challenging benchmark dataset for evaluating the problem solving abilities of LLMs. We curate 515 challenging pre-engineering mathematics, physics and chemistry problems from the highly competitive IIT JEE-Advanced exam. Long-horizon reasoning on top of deep in-domain knowledge is essential for solving problems in this benchmark. Our evaluation on various open-source and proprietary models reveals that the highest performance, even after using techniques like self-consistency, self-refinement and chain-of-thought prompting, is less than 40{\\%}. The ty",
 "signal": "permissive",
 "reason": "The declared licence (mit) is classified as permissive.",
 "revision": "d070d1113ce365df05903763a2017ec2d0d56118",
 "stated_evidence": "https://huggingface.co/datasets/daman1209arora/jeebench/blob/d070d1113ce365df05903763a2017ec2d0d56118/README.md",
 "file_evidence": "https://huggingface.co/datasets/daman1209arora/jeebench/tree/d070d1113ce365df05903763a2017ec2d0d56118",
 "card_nc": false
}