{
"cells": [
{
"cell_type": "markdown",
"id": "a744d473",
"metadata": {},
"source": [
"# Quiz\n",
"\n",
"Companion notebook for the video [*Benchmarking Local AI on Android with PythonHere and Pydantic Evals*](https://youtu.be/CSwZkyGWrhU).\n",
"\n",
"Run a six-question Python quiz against a model on your Android device, then\n",
"score its answers locally with [Pydantic Evals](https://pydantic.dev/docs/ai/evals/evals/).\n",
"\n",
"## Connect and configure"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "3e4a67e4",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-29T22:20:56.757283Z",
"iopub.status.busy": "2026-10-29T22:20:56.757089Z",
"iopub.status.idle": "2026-10-29T22:20:56.768734Z",
"shell.execute_reply": "2026-10-29T22:20:56.768000Z"
}
},
"outputs": [],
"source": [
"# %pip install pydantic-evals"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "17088e87",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-29T22:20:56.770675Z",
"iopub.status.busy": "2026-10-29T22:20:56.770472Z",
"iopub.status.idle": "2026-10-29T22:20:57.193150Z",
"shell.execute_reply": "2026-10-29T22:20:57.192101Z"
}
},
"outputs": [],
"source": [
"%load_ext pythonhere\n",
"%connect-there"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "7830e68f",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-29T22:20:57.196703Z",
"iopub.status.busy": "2026-10-29T22:20:57.196406Z",
"iopub.status.idle": "2026-10-29T22:20:57.475530Z",
"shell.execute_reply": "2026-10-29T22:20:57.474119Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"'pong'"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from herethere.everywhere import ConnectionConfig\n",
"from herethere.there import Client\n",
"\n",
"client = Client()\n",
"await client.connect(ConnectionConfig.load(path=\"there.env\", prefix=\"there\"))\n",
"await client.ping()"
]
},
{
"cell_type": "markdown",
"id": "7a621f42",
"metadata": {},
"source": [
"We use [Qwen2.5-Coder-3B-Instruct](https://huggingface.co/litert-community/Qwen2.5-Coder-3B-Instruct) model, with greedy decoding\n",
"and a 16-token answer limit. `%there put` copies these settings to the device for model loading and inference."
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "0b517fac",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-29T22:20:57.479423Z",
"iopub.status.busy": "2026-10-29T22:20:57.479145Z",
"iopub.status.idle": "2026-10-29T22:20:57.734029Z",
"shell.execute_reply": "2026-10-29T22:20:57.732961Z"
}
},
"outputs": [],
"source": [
"import os\n",
"\n",
"settings = {\n",
" \"model_repo\": \"litert-community/Qwen2.5-Coder-3B-Instruct\",\n",
" \"model_file\": \"Qwen2.5_Coder_3B_It.litertlm\",\n",
" \"sampler\": {\"temperature\": 1.0, \"top_p\": 1.0, \"top_k\": 1, \"seed\": 42},\n",
" \"max_output_tokens\": 16,\n",
"}\n",
"hf_token = os.environ.get(\"HF_TOKEN\")\n",
"\n",
"%there put settings\n",
"%there put hf_token"
]
},
{
"cell_type": "markdown",
"id": "4c1e2e6e",
"metadata": {},
"source": [
"## Download the model"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "4a0a5849",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-29T22:20:57.738094Z",
"iopub.status.busy": "2026-10-29T22:20:57.737919Z",
"iopub.status.idle": "2026-10-29T22:20:58.273888Z",
"shell.execute_reply": "2026-10-29T22:20:58.272690Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/storage/emulated/0/Android/data/me.herethere.pythonhere_dev/files/models/litert-community/Qwen2.5-Coder-3B-Instruct/Qwen2.5_Coder_3B_It.litertlm\n"
]
}
],
"source": [
"%%there --worker\n",
"from ml_here import download_hf_model, require_model\n",
"try:\n",
" model = require_model(settings[\"model_repo\"], settings[\"model_file\"])\n",
"except FileNotFoundError:\n",
" def report_progress(value):\n",
" if value.percent is not None:\n",
" print(f\"\\rDownloading: {value.percent:.1f}%\", end=\"\", flush=True)\n",
"\n",
" model = download_hf_model(\n",
" repo_id=settings[\"model_repo\"],\n",
" filename=settings[\"model_file\"],\n",
" token=hf_token,\n",
" progress=report_progress,\n",
" )\n",
"\n",
"print(model)"
]
},
{
"cell_type": "markdown",
"id": "128ea9a8",
"metadata": {},
"source": [
"## Load the model\n",
"\n",
"The engine stays loaded across questions, but each answer uses a fresh\n",
"conversation so questions are answered independently."
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "a1ec8330",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-29T22:20:58.277214Z",
"iopub.status.busy": "2026-10-29T22:20:58.276977Z",
"iopub.status.idle": "2026-10-29T22:21:10.452466Z",
"shell.execute_reply": "2026-10-29T22:21:10.452135Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Engine(model_path='/storage/emulated/0/Android/data/me.herethere.pythonhere_dev/files/models/litert-community/Qwen2.5-Coder-3B-Instruct/Qwen2.5_Coder_3B_It.litertlm', backend=GPU(gpu_decode_steps_per_sync=None), max_num_tokens=None, max_num_images=None, cache_dir=None, vision_backend=None, audio_backend=None, enable_speculative_decoding=None, lora_rank_config=None, activation_data_type=None, use_ringbuffers_local_attention=None, enable_ynnpack=False)\n"
]
}
],
"source": [
"%%there --worker\n",
"import litert_lm\n",
"from ml_here import require_model\n",
"\n",
"if globals().get(\"quiz_engine\") is not None:\n",
" quiz_engine.close()\n",
"quiz_engine = None\n",
"\n",
"quiz_engine = litert_lm.Engine(\n",
" require_model(settings[\"model_repo\"], settings[\"model_file\"]),\n",
" backend=litert_lm.Backend.GPU(),\n",
")\n",
"\n",
"def quiz_answer(prompt):\n",
" with quiz_engine.create_conversation(\n",
" sampler_config=litert_lm.SamplerConfig(**settings[\"sampler\"]),\n",
" max_output_tokens=settings[\"max_output_tokens\"],\n",
" ) as conversation:\n",
" response = conversation.send_message(prompt)\n",
" return \"\".join(block[\"text\"] for block in response[\"content\"] if \"text\" in block)\n",
"\n",
"print(quiz_engine)"
]
},
{
"cell_type": "markdown",
"id": "adb801b0",
"metadata": {},
"source": [
"## Define questions\n",
"\n",
"For this simple quiz, we define the questions directly in a Pydantic Evals\n",
"`Dataset`. Each `Case` contains a question, answer choices, and the expected\n",
"letter; `EqualsExpected()` scores the model's response against that letter."
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "7c3ffe34",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-29T22:21:10.454347Z",
"iopub.status.busy": "2026-10-29T22:21:10.454149Z",
"iopub.status.idle": "2026-10-29T22:21:12.189592Z",
"shell.execute_reply": "2026-10-29T22:21:12.189148Z"
}
},
"outputs": [],
"source": [
"from typing_extensions import TypedDict\n",
"\n",
"from pydantic_evals import Case, Dataset\n",
"from pydantic_evals.evaluators import EqualsExpected\n",
"\n",
"\n",
"class QuestionInputs(TypedDict):\n",
" question: str\n",
" options: dict[str, str]\n",
"\n",
"\n",
"dataset = Dataset[QuestionInputs, str, None](\n",
" name=\"python-quiz\",\n",
" cases=[\n",
" Case(\n",
" name=\"mutable-container\",\n",
" inputs={\n",
" \"question\": \"Which built-in type is mutable?\",\n",
" \"options\": {\n",
" \"A\": \"tuple\",\n",
" \"B\": \"list\",\n",
" \"C\": \"str\",\n",
" \"D\": \"int\",\n",
" },\n",
" },\n",
" expected_output=\"B\",\n",
" ),\n",
" Case(\n",
" name=\"floor-division\",\n",
" inputs={\n",
" \"question\": \"What is the value of 7 // 2 in Python 3?\",\n",
" \"options\": {\n",
" \"A\": \"3.5\",\n",
" \"B\": \"4\",\n",
" \"C\": \"2\",\n",
" \"D\": \"3\",\n",
" },\n",
" },\n",
" expected_output=\"D\",\n",
" ),\n",
" Case(\n",
" name=\"list-alias\",\n",
" inputs={\n",
" \"question\": (\n",
" \"What does this print?\\n\\n\"\n",
" \"a = [1, 2]\\n\"\n",
" \"b = a\\n\"\n",
" \"b.append(3)\\n\"\n",
" \"print(a)\"\n",
" ),\n",
" \"options\": {\n",
" \"A\": \"[1, 2, 3]\",\n",
" \"B\": \"[1, 2]\",\n",
" \"C\": \"[3]\",\n",
" \"D\": \"None\",\n",
" },\n",
" },\n",
" expected_output=\"A\",\n",
" ),\n",
" Case(\n",
" name=\"empty-set\",\n",
" inputs={\n",
" \"question\": \"Which expression creates an empty set?\",\n",
" \"options\": {\n",
" \"A\": \"{}\",\n",
" \"B\": \"[]\",\n",
" \"C\": \"set()\",\n",
" \"D\": \"()\",\n",
" },\n",
" },\n",
" expected_output=\"C\",\n",
" ),\n",
" Case(\n",
" name=\"mutable-default\",\n",
" inputs={\n",
" \"question\": (\n",
" \"What does the second call print?\\n\\n\"\n",
" \"def add(x, items=[]):\\n\"\n",
" \" items.append(x)\\n\"\n",
" \" return items\\n\\n\"\n",
" \"add(1)\\n\"\n",
" \"print(add(2))\"\n",
" ),\n",
" \"options\": {\n",
" \"A\": \"[2]\",\n",
" \"B\": \"[1, 2]\",\n",
" \"C\": \"[]\",\n",
" \"D\": \"Raises TypeError\",\n",
" },\n",
" },\n",
" expected_output=\"B\",\n",
" ),\n",
" Case(\n",
" name=\"unpacking-comprehension-315\",\n",
" inputs={\n",
" \"question\": (\n",
" \"In Python 3.15, what does this print?\\n\\n\"\n",
" \"groups = [[1, 2], [3]]\\n\"\n",
" \"print([*group for group in groups])\"\n",
" ),\n",
" \"options\": {\n",
" \"A\": \"SyntaxError\",\n",
" \"B\": \"[[1, 2], [3]]\",\n",
" \"C\": \"[1, 2, 3]\",\n",
" \"D\": \"TypeError\",\n",
" },\n",
" },\n",
" expected_output=\"C\",\n",
" ),\n",
" ],\n",
" evaluators=[EqualsExpected()],\n",
")"
]
},
{
"cell_type": "markdown",
"id": "9a1e7638",
"metadata": {},
"source": [
"## Define the task"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "ec5cf293",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-29T22:21:12.191677Z",
"iopub.status.busy": "2026-10-29T22:21:12.191415Z",
"iopub.status.idle": "2026-10-29T22:21:12.194483Z",
"shell.execute_reply": "2026-10-29T22:21:12.194048Z"
}
},
"outputs": [],
"source": [
"def build_prompt(inputs: QuestionInputs) -> str:\n",
" options = \"\\n\".join(\n",
" f\"{letter}: {inputs['options'][letter]}\"\n",
" for letter in \"ABCD\"\n",
" )\n",
" return (\n",
" \"Answer the Python multiple-choice question.\\n\"\n",
" \"Exactly one option is correct.\\n\"\n",
" \"Reply with only A, B, C, or D. Do not explain.\\n\\n\"\n",
" f\"{inputs['question'].rstrip()}\\n\\n\"\n",
" f\"{options}\\n\\n\"\n",
" \"Answer:\"\n",
" )\n",
"\n",
"async def answer_question(inputs: QuestionInputs) -> str:\n",
" prompt = build_prompt(inputs)\n",
" response = await client.get_worker(f\"quiz_answer({prompt!r})\")\n",
" return response.strip()"
]
},
{
"cell_type": "markdown",
"id": "f12c99c5",
"metadata": {},
"source": [
"Inspect the prompt and try one question:"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "93a68a98",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-29T22:21:12.195959Z",
"iopub.status.busy": "2026-10-29T22:21:12.195797Z",
"iopub.status.idle": "2026-10-29T22:21:12.198005Z",
"shell.execute_reply": "2026-10-29T22:21:12.197679Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Answer the Python multiple-choice question.\n",
"Exactly one option is correct.\n",
"Reply with only A, B, C, or D. Do not explain.\n",
"\n",
"Which built-in type is mutable?\n",
"\n",
"A: tuple\n",
"B: list\n",
"C: str\n",
"D: int\n",
"\n",
"Answer:\n"
]
}
],
"source": [
"example = dataset.cases[0]\n",
"print(build_prompt(example.inputs))"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "06c8a3e1",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-29T22:21:12.199393Z",
"iopub.status.busy": "2026-10-29T22:21:12.199243Z",
"iopub.status.idle": "2026-10-29T22:21:18.720854Z",
"shell.execute_reply": "2026-10-29T22:21:18.719628Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"'B'\n"
]
}
],
"source": [
"print(repr(await answer_question(example.inputs)))"
]
},
{
"cell_type": "markdown",
"id": "526e9a94",
"metadata": {},
"source": [
"## Run and inspect\n",
"\n",
"Scoring requires the expected letter alone, ignoring surrounding whitespace.\n",
"Extra explanation counts as a failed answer. Report durations include the client round trip."
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "badd6dea",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-29T22:21:18.725076Z",
"iopub.status.busy": "2026-10-29T22:21:18.724666Z",
"iopub.status.idle": "2026-10-29T22:21:52.227442Z",
"shell.execute_reply": "2026-10-29T22:21:52.227053Z"
}
},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "0887eede9c504a3b8ac7e3518b9ca65f",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Output()"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n"
],
"text/plain": []
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
" Evaluation Summary: answer_question \n",
"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━┓\n",
"┃ Case ID ┃ Inputs ┃ Expected Output ┃ Outputs ┃ Assertions ┃ Duration ┃\n",
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━┩\n",
"│ mutable-container │ {'question': 'Which built-in │ B │ B │ ✔ │ 5.4s │\n",
"│ │ type is mutable?', 'options': │ │ │ │ │\n",
"│ │ {'A': 'tuple', 'B': 'list', │ │ │ │ │\n",
"│ │ 'C': 'str', 'D': 'int'}} │ │ │ │ │\n",
"├─────────────────────────────┼───────────────────────────────┼─────────────────┼─────────┼────────────┼──────────┤\n",
"│ floor-division │ {'question': 'What is the │ D │ D │ ✔ │ 5.5s │\n",
"│ │ value of 7 // 2 in Python │ │ │ │ │\n",
"│ │ 3?', 'options': {'A': '3.5', │ │ │ │ │\n",
"│ │ 'B': '4', 'C': '2', 'D': │ │ │ │ │\n",
"│ │ '3'}} │ │ │ │ │\n",
"├─────────────────────────────┼───────────────────────────────┼─────────────────┼─────────┼────────────┼──────────┤\n",
"│ list-alias │ {'question': 'What does this │ A │ A │ ✔ │ 5.4s │\n",
"│ │ print?\\n\\na = [1, 2]\\nb = │ │ │ │ │\n",
"│ │ a\\nb.append(3)\\nprint(a)', │ │ │ │ │\n",
"│ │ 'options': {'A': '[1, 2, 3]', │ │ │ │ │\n",
"│ │ 'B': '[1, 2]', 'C': '[3]', │ │ │ │ │\n",
"│ │ 'D': 'None'}} │ │ │ │ │\n",
"├─────────────────────────────┼───────────────────────────────┼─────────────────┼─────────┼────────────┼──────────┤\n",
"│ empty-set │ {'question': 'Which │ C │ C │ ✔ │ 5.4s │\n",
"│ │ expression creates an empty │ │ │ │ │\n",
"│ │ set?', 'options': {'A': '{}', │ │ │ │ │\n",
"│ │ 'B': '[]', 'C': 'set()', 'D': │ │ │ │ │\n",
"│ │ '()'}} │ │ │ │ │\n",
"├─────────────────────────────┼───────────────────────────────┼─────────────────┼─────────┼────────────┼──────────┤\n",
"│ mutable-default │ {'question': 'What does the │ B │ B │ ✔ │ 5.5s │\n",
"│ │ second call print?\\n\\ndef │ │ │ │ │\n",
"│ │ add(x, items=[]):\\n │ │ │ │ │\n",
"│ │ items.append(x)\\n return │ │ │ │ │\n",
"│ │ items\\n\\nadd(1)\\nprint(add(2) │ │ │ │ │\n",
"│ │ )', 'options': {'A': '[2]', │ │ │ │ │\n",
"│ │ 'B': '[1, 2]', 'C': '[]', │ │ │ │ │\n",
"│ │ 'D': 'Raises TypeError'}} │ │ │ │ │\n",
"├─────────────────────────────┼───────────────────────────────┼─────────────────┼─────────┼────────────┼──────────┤\n",
"│ unpacking-comprehension-315 │ {'question': 'In Python 3.15, │ C │ B │ ✗ │ 5.7s │\n",
"│ │ what does this │ │ │ │ │\n",
"│ │ print?\\n\\ngroups = [[1, 2], │ │ │ │ │\n",
"│ │ [3]]\\nprint([*group for group │ │ │ │ │\n",
"│ │ in groups])', 'options': │ │ │ │ │\n",
"│ │ {'A': 'SyntaxError', 'B': │ │ │ │ │\n",
"│ │ '[[1, 2], [3]]', 'C': '[1, 2, │ │ │ │ │\n",
"│ │ 3]', 'D': 'TypeError'}} │ │ │ │ │\n",
"├─────────────────────────────┼───────────────────────────────┼─────────────────┼─────────┼────────────┼──────────┤\n",
"│ Averages │ │ │ │ 83.3% ✔ │ 5.5s │\n",
"└─────────────────────────────┴───────────────────────────────┴─────────────────┴─────────┴────────────┴──────────┘\n",
"\n"
],
"text/plain": [
"\u001b[3m Evaluation Summary: answer_question \u001b[0m\n",
"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mCase ID \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mInputs \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mExpected Output\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutputs\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mAssertions\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mDuration\u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━┩\n",
"│\u001b[1m \u001b[0m\u001b[1mmutable-container \u001b[0m\u001b[1m \u001b[0m│ {'question': 'Which built-in │ B │ B │ \u001b[32m✔\u001b[0m │ 5.4s │\n",
"│\u001b[1m \u001b[0m│ type is mutable?', 'options': │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ {'A': 'tuple', 'B': 'list', │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ 'C': 'str', 'D': 'int'}} │ │ │ │ │\n",
"├─────────────────────────────┼───────────────────────────────┼─────────────────┼─────────┼────────────┼──────────┤\n",
"│\u001b[1m \u001b[0m\u001b[1mfloor-division \u001b[0m\u001b[1m \u001b[0m│ {'question': 'What is the │ D │ D │ \u001b[32m✔\u001b[0m │ 5.5s │\n",
"│\u001b[1m \u001b[0m│ value of 7 // 2 in Python │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ 3?', 'options': {'A': '3.5', │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ 'B': '4', 'C': '2', 'D': │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ '3'}} │ │ │ │ │\n",
"├─────────────────────────────┼───────────────────────────────┼─────────────────┼─────────┼────────────┼──────────┤\n",
"│\u001b[1m \u001b[0m\u001b[1mlist-alias \u001b[0m\u001b[1m \u001b[0m│ {'question': 'What does this │ A │ A │ \u001b[32m✔\u001b[0m │ 5.4s │\n",
"│\u001b[1m \u001b[0m│ print?\\n\\na = [1, 2]\\nb = │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ a\\nb.append(3)\\nprint(a)', │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ 'options': {'A': '[1, 2, 3]', │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ 'B': '[1, 2]', 'C': '[3]', │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ 'D': 'None'}} │ │ │ │ │\n",
"├─────────────────────────────┼───────────────────────────────┼─────────────────┼─────────┼────────────┼──────────┤\n",
"│\u001b[1m \u001b[0m\u001b[1mempty-set \u001b[0m\u001b[1m \u001b[0m│ {'question': 'Which │ C │ C │ \u001b[32m✔\u001b[0m │ 5.4s │\n",
"│\u001b[1m \u001b[0m│ expression creates an empty │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ set?', 'options': {'A': '{}', │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ 'B': '[]', 'C': 'set()', 'D': │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ '()'}} │ │ │ │ │\n",
"├─────────────────────────────┼───────────────────────────────┼─────────────────┼─────────┼────────────┼──────────┤\n",
"│\u001b[1m \u001b[0m\u001b[1mmutable-default \u001b[0m\u001b[1m \u001b[0m│ {'question': 'What does the │ B │ B │ \u001b[32m✔\u001b[0m │ 5.5s │\n",
"│\u001b[1m \u001b[0m│ second call print?\\n\\ndef │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ add(x, items=[]):\\n │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ items.append(x)\\n return │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ items\\n\\nadd(1)\\nprint(add(2) │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ )', 'options': {'A': '[2]', │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ 'B': '[1, 2]', 'C': '[]', │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ 'D': 'Raises TypeError'}} │ │ │ │ │\n",
"├─────────────────────────────┼───────────────────────────────┼─────────────────┼─────────┼────────────┼──────────┤\n",
"│\u001b[1m \u001b[0m\u001b[1munpacking-comprehension-315\u001b[0m\u001b[1m \u001b[0m│ {'question': 'In Python 3.15, │ C │ B │ \u001b[31m✗\u001b[0m │ 5.7s │\n",
"│\u001b[1m \u001b[0m│ what does this │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ print?\\n\\ngroups = [[1, 2], │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ [3]]\\nprint([*group for group │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ in groups])', 'options': │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ {'A': 'SyntaxError', 'B': │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ '[[1, 2], [3]]', 'C': '[1, 2, │ │ │ │ │\n",
"│\u001b[1m \u001b[0m│ 3]', 'D': 'TypeError'}} │ │ │ │ │\n",
"├─────────────────────────────┼───────────────────────────────┼─────────────────┼─────────┼────────────┼──────────┤\n",
"│\u001b[1m \u001b[0m\u001b[1;3mAverages\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m│ │ │ │ 83.3% \u001b[32m✔\u001b[0m │ 5.5s │\n",
"└─────────────────────────────┴───────────────────────────────┴─────────────────┴─────────┴────────────┴──────────┘\n"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"report = await dataset.evaluate(answer_question, max_concurrency=1)\n",
"report.print(include_input=True, include_output=True, include_expected_output=True)"
]
},
{
"cell_type": "markdown",
"id": "76eb7d58",
"metadata": {},
"source": [
"## Close the model and client"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "c9af1c1b",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-29T22:21:52.615168Z",
"iopub.status.busy": "2026-10-29T22:21:52.615064Z",
"iopub.status.idle": "2026-10-29T22:21:53.236517Z",
"shell.execute_reply": "2026-10-29T22:21:53.235461Z"
}
},
"outputs": [],
"source": [
"%%there --worker\n",
"quiz_engine.close()\n",
"quiz_engine = None"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "4b508b7a",
"metadata": {
"execution": {
"iopub.execute_input": "2026-10-29T22:21:53.241153Z",
"iopub.status.busy": "2026-10-29T22:21:53.240772Z",
"iopub.status.idle": "2026-10-29T22:21:53.245970Z",
"shell.execute_reply": "2026-10-29T22:21:53.244958Z"
}
},
"outputs": [],
"source": [
"await client.disconnect()"
]
}
],
"metadata": {
"jupytext": {
"default_lexer": "ipython3"
},
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "there-ipython3",
"version": "3.12.3"
},
"widgets": {
"application/vnd.jupyter.widget-state+json": {
"state": {
"0887eede9c504a3b8ac7e3518b9ca65f": {
"model_module": "@jupyter-widgets/output",
"model_module_version": "1.0.0",
"model_name": "OutputModel",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/output",
"_model_module_version": "1.0.0",
"_model_name": "OutputModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/output",
"_view_module_version": "1.0.0",
"_view_name": "OutputView",
"layout": "IPY_MODEL_10468712126a49f38d57e48953ece25f",
"msg_id": "",
"outputs": [
{
"data": {
"text/html": "Evaluating answer_question ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% 0:00:00\n
\n",
"text/plain": "Evaluating answer_question \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m100%\u001b[0m \u001b[36m0:00:00\u001b[0m\n"
},
"metadata": {},
"output_type": "display_data"
}
],
"tabbable": null,
"tooltip": null
}
},
"10468712126a49f38d57e48953ece25f": {
"model_module": "@jupyter-widgets/base",
"model_module_version": "2.0.0",
"model_name": "LayoutModel",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "2.0.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "2.0.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border_bottom": null,
"border_left": null,
"border_right": null,
"border_top": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
}
},
"version_major": 2,
"version_minor": 0
}
}
},
"nbformat": 4,
"nbformat_minor": 5
}