{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "828c33ea", "metadata": {}, "outputs": [], "source": [ "from dotenv import load_dotenv\n", "from openai import OpenAI\n", "load_dotenv()\n", "openai_client = OpenAI()" ] }, { "cell_type": "code", "execution_count": 2, "id": "d3ba7531", "metadata": {}, "outputs": [], "source": [ "import os\n", "import re\n", "import json\n", "import time\n", "import traceback\n", "import subprocess\n", "import textwrap\n", "from pathlib import Path\n", "import traceback\n", "from datetime import datetime, timezone\n", "\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 3, "id": "ef2bc44a", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
| \n", " | date | \n", "category | \n", "region | \n", "product | \n", "sales | \n", "units | \n", "ad_spend | \n", "margin | \n", "revenue | \n", "month | \n", "weekday | \n", "hour | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", "2024-03-06 | \n", "Hardware | \n", "South | \n", "A | \n", "111.347115 | \n", "12 | \n", "399.616639 | \n", "0.269241 | \n", "1267.593020 | \n", "2024-03 | \n", "Wednesday | \n", "3 | \n", "
| 1 | \n", "2025-07-19 | \n", "Hardware | \n", "North | \n", "C | \n", "642.969184 | \n", "4 | \n", "294.445952 | \n", "0.302791 | \n", "2909.872851 | \n", "2025-07 | \n", "Saturday | \n", "5 | \n", "
| 2 | \n", "2025-04-23 | \n", "Services | \n", "East | \n", "C | \n", "475.814998 | \n", "12 | \n", "191.320860 | \n", "0.283313 | \n", "5772.769029 | \n", "2025-04 | \n", "Wednesday | \n", "9 | \n", "
| 3 | \n", "2024-11-16 | \n", "Services | \n", "East | \n", "D | \n", "414.966635 | \n", "14 | \n", "90.354281 | \n", "0.288207 | \n", "5855.648592 | \n", "2024-11 | \n", "Saturday | \n", "19 | \n", "
| 4 | \n", "2024-11-12 | \n", "Subscriptions | \n", "South | \n", "C | \n", "499.248053 | \n", "11 | \n", "497.961157 | \n", "0.271504 | \n", "4788.295506 | \n", "2024-11 | \n", "Tuesday | \n", "4 | \n", "
| \n", " | datetime.utcnow().isoformat() | \n", "provider | \n", "model | \n", "task_id | \n", "task_name | \n", "library | \n", "run_id | \n", "temperature | \n", "initial_prompt | \n", "initial_response | \n", "... | \n", "final_error_message | \n", "final_code | \n", "final_code_chars | \n", "final_code_lines | \n", "total_agentic_tokens | \n", "execution_script_path | \n", "prompt_track | \n", "experiment_name | \n", "timestamp | \n", "error | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", "2026-07-05T21:27:29.452167 | \n", "openai | \n", "gpt-4.1-mini | \n", "T1 | \n", "Bar chart | \n", "Matplotlib | \n", "1 | \n", "0.0 | \n", "You are given a pandas DataFrame named df with... | \n", "import matplotlib.pyplot as plt\\n\\nrevenue_by_... | \n", "... | \n", "NaN | \n", "import matplotlib.pyplot as plt\\n\\nrevenue_by_... | \n", "385.0 | \n", "11.0 | \n", "392.0 | \n", "experiment_outputs_openai_quality_v2_full/open... | \n", "quality_controlled_v2 | \n", "openai_quality_v2_all_models_all_tasks | \n", "2026-07-05T21:27:29.452251+00:00 | \n", "NaN | \n", "
| 1 | \n", "2026-07-05T21:27:33.107684 | \n", "openai | \n", "gpt-4.1-mini | \n", "T1 | \n", "Bar chart | \n", "Matplotlib | \n", "2 | \n", "0.0 | \n", "You are given a pandas DataFrame named df with... | \n", "import matplotlib.pyplot as plt\\n\\nrevenue_by_... | \n", "... | \n", "NaN | \n", "import matplotlib.pyplot as plt\\n\\nrevenue_by_... | \n", "385.0 | \n", "11.0 | \n", "392.0 | \n", "experiment_outputs_openai_quality_v2_full/open... | \n", "quality_controlled_v2 | \n", "openai_quality_v2_all_models_all_tasks | \n", "2026-07-05T21:27:33.107738+00:00 | \n", "NaN | \n", "
| 2 | \n", "2026-07-05T21:27:36.402421 | \n", "openai | \n", "gpt-4.1-mini | \n", "T1 | \n", "Bar chart | \n", "Matplotlib | \n", "3 | \n", "0.0 | \n", "You are given a pandas DataFrame named df with... | \n", "import matplotlib.pyplot as plt\\n\\nrevenue_by_... | \n", "... | \n", "NaN | \n", "import matplotlib.pyplot as plt\\n\\nrevenue_by_... | \n", "385.0 | \n", "11.0 | \n", "392.0 | \n", "experiment_outputs_openai_quality_v2_full/open... | \n", "quality_controlled_v2 | \n", "openai_quality_v2_all_models_all_tasks | \n", "2026-07-05T21:27:36.402488+00:00 | \n", "NaN | \n", "
| 3 | \n", "2026-07-05T21:27:39.615520 | \n", "openai | \n", "gpt-4.1-mini | \n", "T1 | \n", "Bar chart | \n", "Altair | \n", "1 | \n", "0.0 | \n", "You are given a pandas DataFrame named df with... | \n", "import altair as alt\\n\\nchart = alt.Chart(df).... | \n", "... | \n", "NaN | \n", "import altair as alt\\n\\nchart = alt.Chart(df).... | \n", "279.0 | \n", "12.0 | \n", "382.0 | \n", "experiment_outputs_openai_quality_v2_full/open... | \n", "quality_controlled_v2 | \n", "openai_quality_v2_all_models_all_tasks | \n", "2026-07-05T21:27:39.615571+00:00 | \n", "NaN | \n", "
| 4 | \n", "2026-07-05T21:27:43.206672 | \n", "openai | \n", "gpt-4.1-mini | \n", "T1 | \n", "Bar chart | \n", "Altair | \n", "2 | \n", "0.0 | \n", "You are given a pandas DataFrame named df with... | \n", "import altair as alt\\n\\nchart = alt.Chart(df).... | \n", "... | \n", "NaN | \n", "import altair as alt\\n\\nchart = alt.Chart(df).... | \n", "286.0 | \n", "12.0 | \n", "384.0 | \n", "experiment_outputs_openai_quality_v2_full/open... | \n", "quality_controlled_v2 | \n", "openai_quality_v2_all_models_all_tasks | \n", "2026-07-05T21:27:43.206726+00:00 | \n", "NaN | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 259 | \n", "2026-07-06T07:54:47.076169+00:00 | \n", "openai | \n", "gpt-4o-mini | \n", "T11 | \n", "Linked views | \n", "Matplotlib | \n", "2 | \n", "0.0 | \n", "You are given a pandas DataFrame named df with... | \n", "import pandas as pd\\nimport matplotlib.pyplot ... | \n", "... | \n", "NaN | \n", "import pandas as pd\\nimport matplotlib.pyplot ... | \n", "1232.0 | \n", "36.0 | \n", "1637.0 | \n", "experiment_outputs_openai_quality_v2_full/open... | \n", "quality_controlled_v2 | \n", "openai_quality_v2_all_models_all_tasks | \n", "2026-07-06T07:54:47.076282+00:00 | \n", "NaN | \n", "
| 260 | \n", "2026-07-06T07:54:54.032455+00:00 | \n", "openai | \n", "gpt-4o-mini | \n", "T11 | \n", "Linked views | \n", "Matplotlib | \n", "3 | \n", "0.0 | \n", "You are given a pandas DataFrame named df with... | \n", "import pandas as pd\\nimport matplotlib.pyplot ... | \n", "... | \n", "NaN | \n", "import pandas as pd\\nimport matplotlib.pyplot ... | \n", "947.0 | \n", "29.0 | \n", "551.0 | \n", "experiment_outputs_openai_quality_v2_full/open... | \n", "quality_controlled_v2 | \n", "openai_quality_v2_all_models_all_tasks | \n", "2026-07-06T07:54:54.032598+00:00 | \n", "NaN | \n", "
| 261 | \n", "2026-07-06T07:55:06.024020+00:00 | \n", "openai | \n", "gpt-4o-mini | \n", "T11 | \n", "Linked views | \n", "Altair | \n", "1 | \n", "0.0 | \n", "You are given a pandas DataFrame named df with... | \n", "import altair as alt\\nimport pandas as pd\\n\\ns... | \n", "... | \n", "NaN | \n", "import altair as alt\\nimport pandas as pd\\n\\n#... | \n", "1081.0 | \n", "41.0 | \n", "1352.0 | \n", "experiment_outputs_openai_quality_v2_full/open... | \n", "quality_controlled_v2 | \n", "openai_quality_v2_all_models_all_tasks | \n", "2026-07-06T07:55:06.024195+00:00 | \n", "NaN | \n", "
| 262 | \n", "2026-07-06T07:55:19.110010+00:00 | \n", "openai | \n", "gpt-4o-mini | \n", "T11 | \n", "Linked views | \n", "Altair | \n", "2 | \n", "0.0 | \n", "You are given a pandas DataFrame named df with... | \n", "import altair as alt\\nimport pandas as pd\\n\\ns... | \n", "... | \n", "/Users/eduardodisanti/git/altair_as_agentic_se... | \n", "import altair as alt\\nimport pandas as pd\\n\\n#... | \n", "1097.0 | \n", "39.0 | \n", "1378.0 | \n", "experiment_outputs_openai_quality_v2_full/open... | \n", "quality_controlled_v2 | \n", "openai_quality_v2_all_models_all_tasks | \n", "2026-07-06T07:55:19.110167+00:00 | \n", "NaN | \n", "
| 263 | \n", "2026-07-06T07:55:33.102056+00:00 | \n", "openai | \n", "gpt-4o-mini | \n", "T11 | \n", "Linked views | \n", "Altair | \n", "3 | \n", "0.0 | \n", "You are given a pandas DataFrame named df with... | \n", "import altair as alt\\nimport pandas as pd\\n\\ns... | \n", "... | \n", "/Users/eduardodisanti/git/altair_as_agentic_se... | \n", "import altair as alt\\nimport pandas as pd\\n\\n#... | \n", "1100.0 | \n", "39.0 | \n", "1357.0 | \n", "experiment_outputs_openai_quality_v2_full/open... | \n", "quality_controlled_v2 | \n", "openai_quality_v2_all_models_all_tasks | \n", "2026-07-06T07:55:33.102323+00:00 | \n", "NaN | \n", "
264 rows × 32 columns
\n", "| \n", " | model | \n", "complexity | \n", "library | \n", "mean_initial_tokens | \n", "std_initial_tokens | \n", "median_initial_tokens | \n", "mean_code_lines | \n", "std_code_lines | \n", "first_pass_success_rate | \n", "final_success_rate | \n", "mean_repair_tokens | \n", "std_repair_tokens | \n", "n | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", "gpt-4.1-mini | \n", "low | \n", "Altair | \n", "375.888889 | \n", "14.953632 | \n", "377.0 | \n", "11.444444 | \n", "1.130388 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 1 | \n", "gpt-4.1-mini | \n", "low | \n", "Matplotlib | \n", "376.666667 | \n", "19.371371 | \n", "387.0 | \n", "10.333333 | \n", "1.000000 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 2 | \n", "gpt-4.1-mini | \n", "medium | \n", "Altair | \n", "424.111111 | \n", "37.681044 | \n", "403.0 | \n", "15.666667 | \n", "2.179449 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 3 | \n", "gpt-4.1-mini | \n", "medium | \n", "Matplotlib | \n", "512.333333 | \n", "62.285632 | \n", "473.0 | \n", "25.333333 | \n", "7.399324 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 4 | \n", "gpt-4.1-mini | \n", "medium-high | \n", "Altair | \n", "419.500000 | \n", "26.120873 | \n", "415.5 | \n", "15.000000 | \n", "1.095445 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "6 | \n", "
| 5 | \n", "gpt-4.1-mini | \n", "medium-high | \n", "Matplotlib | \n", "467.000000 | \n", "46.008695 | \n", "467.0 | \n", "20.000000 | \n", "3.286335 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "6 | \n", "
| 6 | \n", "gpt-4.1-mini | \n", "high | \n", "Altair | \n", "472.222222 | \n", "33.825943 | \n", "454.0 | \n", "22.777778 | \n", "5.517648 | \n", "0.666667 | \n", "0.666667 | \n", "359.666667 | \n", "539.500000 | \n", "9 | \n", "
| 7 | \n", "gpt-4.1-mini | \n", "high | \n", "Matplotlib | \n", "663.222222 | \n", "111.024522 | \n", "591.0 | \n", "42.888889 | \n", "13.204587 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 8 | \n", "gpt-4o-mini | \n", "low | \n", "Altair | \n", "421.777778 | \n", "21.735787 | \n", "433.0 | \n", "16.000000 | \n", "0.866025 | \n", "0.888889 | \n", "1.000000 | \n", "48.222222 | \n", "144.666667 | \n", "9 | \n", "
| 9 | \n", "gpt-4o-mini | \n", "low | \n", "Matplotlib | \n", "381.333333 | \n", "16.755596 | \n", "392.0 | \n", "11.555556 | \n", "1.236033 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 10 | \n", "gpt-4o-mini | \n", "medium | \n", "Altair | \n", "435.555556 | \n", "20.952990 | \n", "422.0 | \n", "16.444444 | \n", "1.878238 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 11 | \n", "gpt-4o-mini | \n", "medium | \n", "Matplotlib | \n", "470.111111 | \n", "85.430446 | \n", "436.0 | \n", "19.888889 | \n", "8.894443 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 12 | \n", "gpt-4o-mini | \n", "medium-high | \n", "Altair | \n", "434.000000 | \n", "7.348469 | \n", "435.0 | \n", "19.166667 | \n", "1.940790 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "6 | \n", "
| 13 | \n", "gpt-4o-mini | \n", "medium-high | \n", "Matplotlib | \n", "422.000000 | \n", "39.436024 | \n", "422.0 | \n", "16.500000 | \n", "3.834058 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "6 | \n", "
| 14 | \n", "gpt-4o-mini | \n", "high | \n", "Altair | \n", "469.888889 | \n", "14.172548 | \n", "467.0 | \n", "22.555556 | \n", "4.850544 | \n", "0.333333 | \n", "0.777778 | \n", "552.777778 | \n", "418.341003 | \n", "9 | \n", "
| 15 | \n", "gpt-4o-mini | \n", "high | \n", "Matplotlib | \n", "561.111111 | \n", "73.304919 | \n", "551.0 | \n", "28.444444 | \n", "4.216370 | \n", "0.333333 | \n", "0.777778 | \n", "657.000000 | \n", "543.063993 | \n", "9 | \n", "
| 16 | \n", "gpt-5.4-mini | \n", "low | \n", "Altair | \n", "427.111111 | \n", "24.609167 | \n", "429.0 | \n", "16.888889 | \n", "4.075673 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 17 | \n", "gpt-5.4-mini | \n", "low | \n", "Matplotlib | \n", "402.555556 | \n", "38.373530 | \n", "414.0 | \n", "12.000000 | \n", "2.738613 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 18 | \n", "gpt-5.4-mini | \n", "medium | \n", "Altair | \n", "463.000000 | \n", "21.627529 | \n", "466.0 | \n", "23.111111 | \n", "1.691482 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 19 | \n", "gpt-5.4-mini | \n", "medium | \n", "Matplotlib | \n", "562.888889 | \n", "97.173099 | \n", "524.0 | \n", "34.444444 | \n", "14.748823 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 20 | \n", "gpt-5.4-mini | \n", "medium-high | \n", "Altair | \n", "441.666667 | \n", "22.303961 | \n", "454.0 | \n", "20.333333 | \n", "3.326660 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "6 | \n", "
| 21 | \n", "gpt-5.4-mini | \n", "medium-high | \n", "Matplotlib | \n", "508.833333 | \n", "66.291528 | \n", "491.0 | \n", "24.166667 | \n", "5.741661 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "6 | \n", "
| 22 | \n", "gpt-5.4-mini | \n", "high | \n", "Altair | \n", "572.888889 | \n", "79.808277 | \n", "539.0 | \n", "33.666667 | \n", "8.440972 | \n", "0.888889 | \n", "1.000000 | \n", "119.777778 | \n", "359.333333 | \n", "9 | \n", "
| 23 | \n", "gpt-5.4-mini | \n", "high | \n", "Matplotlib | \n", "856.000000 | \n", "163.908511 | \n", "936.0 | \n", "62.666667 | \n", "17.635192 | \n", "0.666667 | \n", "1.000000 | \n", "374.000000 | \n", "592.787272 | \n", "9 | \n", "
| 24 | \n", "gpt-5.5 | \n", "low | \n", "Altair | \n", "755.777778 | \n", "118.164904 | \n", "712.0 | \n", "12.222222 | \n", "1.201850 | \n", "0.888889 | \n", "1.000000 | \n", "76.111111 | \n", "228.333333 | \n", "9 | \n", "
| 25 | \n", "gpt-5.5 | \n", "low | \n", "Matplotlib | \n", "636.888889 | \n", "195.865671 | \n", "533.0 | \n", "14.222222 | \n", "4.265495 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 26 | \n", "gpt-5.5 | \n", "medium | \n", "Altair | \n", "778.111111 | \n", "118.744941 | \n", "816.0 | \n", "17.888889 | \n", "2.088327 | \n", "0.888889 | \n", "1.000000 | \n", "130.888889 | \n", "392.666667 | \n", "9 | \n", "
| 27 | \n", "gpt-5.5 | \n", "medium | \n", "Matplotlib | \n", "1168.333333 | \n", "213.318424 | \n", "1099.0 | \n", "41.000000 | \n", "7.071068 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 28 | \n", "gpt-5.5 | \n", "medium-high | \n", "Altair | \n", "873.666667 | \n", "57.618284 | \n", "876.0 | \n", "16.666667 | \n", "2.943920 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "6 | \n", "
| 29 | \n", "gpt-5.5 | \n", "medium-high | \n", "Matplotlib | \n", "1069.166667 | \n", "298.177408 | \n", "1031.0 | \n", "29.833333 | \n", "6.306082 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "6 | \n", "
| 30 | \n", "gpt-5.5 | \n", "high | \n", "Altair | \n", "1365.333333 | \n", "613.316599 | \n", "1096.0 | \n", "27.111111 | \n", "11.095545 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| 31 | \n", "gpt-5.5 | \n", "high | \n", "Matplotlib | \n", "2098.333333 | \n", "332.961334 | \n", "2297.0 | \n", "29.111111 | \n", "43.913109 | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "9 | \n", "
| library | \n", "model | \n", "complexity | \n", "Altair | \n", "Matplotlib | \n", "token_reduction_abs | \n", "token_reduction_pct | \n", "
|---|---|---|---|---|---|---|
| 0 | \n", "gpt-4.1-mini | \n", "low | \n", "375.888889 | \n", "376.666667 | \n", "0.777778 | \n", "0.206490 | \n", "
| 1 | \n", "gpt-4.1-mini | \n", "medium | \n", "424.111111 | \n", "512.333333 | \n", "88.222222 | \n", "17.219692 | \n", "
| 2 | \n", "gpt-4.1-mini | \n", "medium-high | \n", "419.500000 | \n", "467.000000 | \n", "47.500000 | \n", "10.171306 | \n", "
| 3 | \n", "gpt-4.1-mini | \n", "high | \n", "472.222222 | \n", "663.222222 | \n", "191.000000 | \n", "28.798794 | \n", "
| 4 | \n", "gpt-4o-mini | \n", "low | \n", "421.777778 | \n", "381.333333 | \n", "-40.444444 | \n", "-10.606061 | \n", "
| 5 | \n", "gpt-4o-mini | \n", "medium | \n", "435.555556 | \n", "470.111111 | \n", "34.555556 | \n", "7.350508 | \n", "
| 6 | \n", "gpt-4o-mini | \n", "medium-high | \n", "434.000000 | \n", "422.000000 | \n", "-12.000000 | \n", "-2.843602 | \n", "
| 7 | \n", "gpt-4o-mini | \n", "high | \n", "469.888889 | \n", "561.111111 | \n", "91.222222 | \n", "16.257426 | \n", "
| 8 | \n", "gpt-5.4-mini | \n", "low | \n", "427.111111 | \n", "402.555556 | \n", "-24.555556 | \n", "-6.099917 | \n", "
| 9 | \n", "gpt-5.4-mini | \n", "medium | \n", "463.000000 | \n", "562.888889 | \n", "99.888889 | \n", "17.745756 | \n", "
| 10 | \n", "gpt-5.4-mini | \n", "medium-high | \n", "441.666667 | \n", "508.833333 | \n", "67.166667 | \n", "13.200131 | \n", "
| 11 | \n", "gpt-5.4-mini | \n", "high | \n", "572.888889 | \n", "856.000000 | \n", "283.111111 | \n", "33.073728 | \n", "
| 12 | \n", "gpt-5.5 | \n", "low | \n", "755.777778 | \n", "636.888889 | \n", "-118.888889 | \n", "-18.667132 | \n", "
| 13 | \n", "gpt-5.5 | \n", "medium | \n", "778.111111 | \n", "1168.333333 | \n", "390.222222 | \n", "33.399905 | \n", "
| 14 | \n", "gpt-5.5 | \n", "medium-high | \n", "873.666667 | \n", "1069.166667 | \n", "195.500000 | \n", "18.285269 | \n", "
| 15 | \n", "gpt-5.5 | \n", "high | \n", "1365.333333 | \n", "2098.333333 | \n", "733.000000 | \n", "34.932486 | \n", "
| library | \n", "complexity | \n", "Altair | \n", "Matplotlib | \n", "Altair_std | \n", "Matplotlib_std | \n", "token_reduction_abs | \n", "token_reduction_pct | \n", "
|---|---|---|---|---|---|---|---|
| 0 | \n", "low | \n", "495.138889 | \n", "449.361111 | \n", "164.888040 | \n", "146.322765 | \n", "-45.777778 | \n", "-10.187303 | \n", "
| 1 | \n", "medium | \n", "525.194444 | \n", "678.416667 | \n", "160.908815 | \n", "313.872620 | \n", "153.222222 | \n", "22.585268 | \n", "
| 2 | \n", "medium-high | \n", "542.208333 | \n", "616.750000 | \n", "198.166153 | \n", "305.383472 | \n", "74.541667 | \n", "12.086205 | \n", "
| 3 | \n", "high | \n", "720.083333 | \n", "1044.666667 | \n", "481.942165 | \n", "653.990913 | \n", "324.583333 | \n", "31.070517 | \n", "
| library | \n", "model | \n", "task_id | \n", "task_name | \n", "complexity | \n", "Altair | \n", "Matplotlib | \n", "token_reduction_abs | \n", "token_reduction_pct | \n", "
|---|---|---|---|---|---|---|---|---|
| 0 | \n", "gpt-4.1-mini | \n", "T1 | \n", "Bar chart | \n", "low | \n", "382.666667 | \n", "392.000000 | \n", "9.333333 | \n", "2.380952 | \n", "
| 3 | \n", "gpt-4.1-mini | \n", "T2 | \n", "Line chart | \n", "low | \n", "386.000000 | \n", "387.000000 | \n", "1.000000 | \n", "0.258398 | \n", "
| 5 | \n", "gpt-4.1-mini | \n", "T4 | \n", "Histogram | \n", "low | \n", "359.000000 | \n", "351.000000 | \n", "-8.000000 | \n", "-2.279202 | \n", "
| 4 | \n", "gpt-4.1-mini | \n", "T3 | \n", "Scatter plot with tooltip | \n", "medium | \n", "395.000000 | \n", "594.666667 | \n", "199.666667 | \n", "33.576233 | \n", "
| 6 | \n", "gpt-4.1-mini | \n", "T5 | \n", "Grouped bar chart | \n", "medium | \n", "403.333333 | \n", "469.000000 | \n", "65.666667 | \n", "14.001421 | \n", "
| 9 | \n", "gpt-4.1-mini | \n", "T8 | \n", "Heatmap | \n", "medium | \n", "474.000000 | \n", "473.333333 | \n", "-0.666667 | \n", "-0.140845 | \n", "
| 7 | \n", "gpt-4.1-mini | \n", "T6 | \n", "Layered line chart | \n", "medium-high | \n", "396.000000 | \n", "425.000000 | \n", "29.000000 | \n", "6.823529 | \n", "
| 8 | \n", "gpt-4.1-mini | \n", "T7 | \n", "Faceted chart | \n", "medium-high | \n", "443.000000 | \n", "509.000000 | \n", "66.000000 | \n", "12.966601 | \n", "
| 1 | \n", "gpt-4.1-mini | \n", "T10 | \n", "Dropdown filter | \n", "high | \n", "454.000000 | \n", "577.666667 | \n", "123.666667 | \n", "21.407963 | \n", "
| 2 | \n", "gpt-4.1-mini | \n", "T11 | \n", "Linked views | \n", "high | \n", "517.000000 | \n", "609.333333 | \n", "92.333333 | \n", "15.153173 | \n", "
| 10 | \n", "gpt-4.1-mini | \n", "T9 | \n", "Interactive category selection | \n", "high | \n", "445.666667 | \n", "802.666667 | \n", "357.000000 | \n", "44.476744 | \n", "
| 11 | \n", "gpt-4o-mini | \n", "T1 | \n", "Bar chart | \n", "low | \n", "436.666667 | \n", "393.000000 | \n", "-43.666667 | \n", "-11.111111 | \n", "
| 14 | \n", "gpt-4o-mini | \n", "T2 | \n", "Line chart | \n", "low | \n", "435.666667 | \n", "392.000000 | \n", "-43.666667 | \n", "-11.139456 | \n", "
| 16 | \n", "gpt-4o-mini | \n", "T4 | \n", "Histogram | \n", "low | \n", "393.000000 | \n", "359.000000 | \n", "-34.000000 | \n", "-9.470752 | \n", "
| 15 | \n", "gpt-4o-mini | \n", "T3 | \n", "Scatter plot with tooltip | \n", "medium | \n", "421.000000 | \n", "580.333333 | \n", "159.333333 | \n", "27.455485 | \n", "
| 17 | \n", "gpt-4o-mini | \n", "T5 | \n", "Grouped bar chart | \n", "medium | \n", "423.333333 | \n", "394.000000 | \n", "-29.333333 | \n", "-7.445008 | \n", "
| 20 | \n", "gpt-4o-mini | \n", "T8 | \n", "Heatmap | \n", "medium | \n", "462.333333 | \n", "436.000000 | \n", "-26.333333 | \n", "-6.039755 | \n", "
| 18 | \n", "gpt-4o-mini | \n", "T6 | \n", "Layered line chart | \n", "medium-high | \n", "436.666667 | \n", "386.000000 | \n", "-50.666667 | \n", "-13.126079 | \n", "
| 19 | \n", "gpt-4o-mini | \n", "T7 | \n", "Faceted chart | \n", "medium-high | \n", "431.333333 | \n", "458.000000 | \n", "26.666667 | \n", "5.822416 | \n", "
| 12 | \n", "gpt-4o-mini | \n", "T10 | \n", "Dropdown filter | \n", "high | \n", "467.000000 | \n", "487.333333 | \n", "20.333333 | \n", "4.172367 | \n", "
| 13 | \n", "gpt-4o-mini | \n", "T11 | \n", "Linked views | \n", "high | \n", "487.333333 | \n", "554.000000 | \n", "66.666667 | \n", "12.033694 | \n", "
| 21 | \n", "gpt-4o-mini | \n", "T9 | \n", "Interactive category selection | \n", "high | \n", "455.333333 | \n", "642.000000 | \n", "186.666667 | \n", "29.075805 | \n", "
| 22 | \n", "gpt-5.4-mini | \n", "T1 | \n", "Bar chart | \n", "low | \n", "446.333333 | \n", "414.000000 | \n", "-32.333333 | \n", "-7.809984 | \n", "
| 25 | \n", "gpt-5.4-mini | \n", "T2 | \n", "Line chart | \n", "low | \n", "433.333333 | \n", "438.000000 | \n", "4.666667 | \n", "1.065449 | \n", "
| 27 | \n", "gpt-5.4-mini | \n", "T4 | \n", "Histogram | \n", "low | \n", "401.666667 | \n", "355.666667 | \n", "-46.000000 | \n", "-12.933458 | \n", "
| 26 | \n", "gpt-5.4-mini | \n", "T3 | \n", "Scatter plot with tooltip | \n", "medium | \n", "465.000000 | \n", "686.666667 | \n", "221.666667 | \n", "32.281553 | \n", "
| 28 | \n", "gpt-5.4-mini | \n", "T5 | \n", "Grouped bar chart | \n", "medium | \n", "450.333333 | \n", "483.666667 | \n", "33.333333 | \n", "6.891799 | \n", "
| 31 | \n", "gpt-5.4-mini | \n", "T8 | \n", "Heatmap | \n", "medium | \n", "473.666667 | \n", "518.333333 | \n", "44.666667 | \n", "8.617363 | \n", "
| 29 | \n", "gpt-5.4-mini | \n", "T6 | \n", "Layered line chart | \n", "medium-high | \n", "443.333333 | \n", "459.000000 | \n", "15.666667 | \n", "3.413217 | \n", "
| 30 | \n", "gpt-5.4-mini | \n", "T7 | \n", "Faceted chart | \n", "medium-high | \n", "440.000000 | \n", "558.666667 | \n", "118.666667 | \n", "21.241050 | \n", "
| 23 | \n", "gpt-5.4-mini | \n", "T10 | \n", "Dropdown filter | \n", "high | \n", "552.333333 | \n", "646.666667 | \n", "94.333333 | \n", "14.587629 | \n", "
| 24 | \n", "gpt-5.4-mini | \n", "T11 | \n", "Linked views | \n", "high | \n", "671.666667 | \n", "951.000000 | \n", "279.333333 | \n", "29.372590 | \n", "
| 32 | \n", "gpt-5.4-mini | \n", "T9 | \n", "Interactive category selection | \n", "high | \n", "494.666667 | \n", "970.333333 | \n", "475.666667 | \n", "49.020955 | \n", "
| 33 | \n", "gpt-5.5 | \n", "T1 | \n", "Bar chart | \n", "low | \n", "677.666667 | \n", "574.333333 | \n", "-103.333333 | \n", "-17.991875 | \n", "
| 36 | \n", "gpt-5.5 | \n", "T2 | \n", "Line chart | \n", "low | \n", "801.333333 | \n", "826.000000 | \n", "24.666667 | \n", "2.986279 | \n", "
| 38 | \n", "gpt-5.5 | \n", "T4 | \n", "Histogram | \n", "low | \n", "788.333333 | \n", "510.333333 | \n", "-278.000000 | \n", "-54.474200 | \n", "
| 37 | \n", "gpt-5.5 | \n", "T3 | \n", "Scatter plot with tooltip | \n", "medium | \n", "646.333333 | \n", "1185.666667 | \n", "539.333333 | \n", "45.487771 | \n", "
| 39 | \n", "gpt-5.5 | \n", "T5 | \n", "Grouped bar chart | \n", "medium | \n", "843.666667 | \n", "1037.666667 | \n", "194.000000 | \n", "18.695792 | \n", "
| 42 | \n", "gpt-5.5 | \n", "T8 | \n", "Heatmap | \n", "medium | \n", "844.333333 | \n", "1281.666667 | \n", "437.333333 | \n", "34.122237 | \n", "
| 40 | \n", "gpt-5.5 | \n", "T6 | \n", "Layered line chart | \n", "medium-high | \n", "890.333333 | \n", "872.333333 | \n", "-18.000000 | \n", "-2.063431 | \n", "
| 41 | \n", "gpt-5.5 | \n", "T7 | \n", "Faceted chart | \n", "medium-high | \n", "857.000000 | \n", "1266.000000 | \n", "409.000000 | \n", "32.306477 | \n", "
| 34 | \n", "gpt-5.5 | \n", "T10 | \n", "Dropdown filter | \n", "high | \n", "1808.333333 | \n", "2297.000000 | \n", "488.666667 | \n", "21.274126 | \n", "
| 35 | \n", "gpt-5.5 | \n", "T11 | \n", "Linked views | \n", "high | \n", "1434.000000 | \n", "2309.000000 | \n", "875.000000 | \n", "37.895193 | \n", "
| 43 | \n", "gpt-5.5 | \n", "T9 | \n", "Interactive category selection | \n", "high | \n", "853.666667 | \n", "1689.000000 | \n", "835.333333 | \n", "49.457273 | \n", "
| \n", " | model | \n", "complexity | \n", "library | \n", "first_pass_success_rate | \n", "final_success_rate | \n", "mean_repair_attempts | \n", "mean_repair_tokens | \n", "max_repair_tokens | \n", "n | \n", "
|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", "gpt-4.1-mini | \n", "low | \n", "Altair | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 1 | \n", "gpt-4.1-mini | \n", "low | \n", "Matplotlib | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 2 | \n", "gpt-4.1-mini | \n", "medium | \n", "Altair | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 3 | \n", "gpt-4.1-mini | \n", "medium | \n", "Matplotlib | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 4 | \n", "gpt-4.1-mini | \n", "medium-high | \n", "Altair | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "6 | \n", "
| 5 | \n", "gpt-4.1-mini | \n", "medium-high | \n", "Matplotlib | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "6 | \n", "
| 6 | \n", "gpt-4.1-mini | \n", "high | \n", "Altair | \n", "0.666667 | \n", "0.666667 | \n", "0.333333 | \n", "359.666667 | \n", "1079.0 | \n", "9 | \n", "
| 7 | \n", "gpt-4.1-mini | \n", "high | \n", "Matplotlib | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 8 | \n", "gpt-4o-mini | \n", "low | \n", "Altair | \n", "0.888889 | \n", "1.000000 | \n", "0.111111 | \n", "48.222222 | \n", "434.0 | \n", "9 | \n", "
| 9 | \n", "gpt-4o-mini | \n", "low | \n", "Matplotlib | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 10 | \n", "gpt-4o-mini | \n", "medium | \n", "Altair | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 11 | \n", "gpt-4o-mini | \n", "medium | \n", "Matplotlib | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 12 | \n", "gpt-4o-mini | \n", "medium-high | \n", "Altair | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "6 | \n", "
| 13 | \n", "gpt-4o-mini | \n", "medium-high | \n", "Matplotlib | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "6 | \n", "
| 14 | \n", "gpt-4o-mini | \n", "high | \n", "Altair | \n", "0.333333 | \n", "0.777778 | \n", "0.666667 | \n", "552.777778 | \n", "886.0 | \n", "9 | \n", "
| 15 | \n", "gpt-4o-mini | \n", "high | \n", "Matplotlib | \n", "0.333333 | \n", "0.777778 | \n", "0.666667 | \n", "657.000000 | \n", "1258.0 | \n", "9 | \n", "
| 16 | \n", "gpt-5.4-mini | \n", "low | \n", "Altair | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 17 | \n", "gpt-5.4-mini | \n", "low | \n", "Matplotlib | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 18 | \n", "gpt-5.4-mini | \n", "medium | \n", "Altair | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 19 | \n", "gpt-5.4-mini | \n", "medium | \n", "Matplotlib | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 20 | \n", "gpt-5.4-mini | \n", "medium-high | \n", "Altair | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "6 | \n", "
| 21 | \n", "gpt-5.4-mini | \n", "medium-high | \n", "Matplotlib | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "6 | \n", "
| 22 | \n", "gpt-5.4-mini | \n", "high | \n", "Altair | \n", "0.888889 | \n", "1.000000 | \n", "0.111111 | \n", "119.777778 | \n", "1078.0 | \n", "9 | \n", "
| 23 | \n", "gpt-5.4-mini | \n", "high | \n", "Matplotlib | \n", "0.666667 | \n", "1.000000 | \n", "0.333333 | \n", "374.000000 | \n", "1561.0 | \n", "9 | \n", "
| 24 | \n", "gpt-5.5 | \n", "low | \n", "Altair | \n", "0.888889 | \n", "1.000000 | \n", "0.111111 | \n", "76.111111 | \n", "685.0 | \n", "9 | \n", "
| 25 | \n", "gpt-5.5 | \n", "low | \n", "Matplotlib | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 26 | \n", "gpt-5.5 | \n", "medium | \n", "Altair | \n", "0.888889 | \n", "1.000000 | \n", "0.111111 | \n", "130.888889 | \n", "1178.0 | \n", "9 | \n", "
| 27 | \n", "gpt-5.5 | \n", "medium | \n", "Matplotlib | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 28 | \n", "gpt-5.5 | \n", "medium-high | \n", "Altair | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "6 | \n", "
| 29 | \n", "gpt-5.5 | \n", "medium-high | \n", "Matplotlib | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "6 | \n", "
| 30 | \n", "gpt-5.5 | \n", "high | \n", "Altair | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "
| 31 | \n", "gpt-5.5 | \n", "high | \n", "Matplotlib | \n", "1.000000 | \n", "1.000000 | \n", "0.000000 | \n", "0.000000 | \n", "0.0 | \n", "9 | \n", "