Skip to content

Break ties in rank calculation #168

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 8, 2025
Merged

Break ties in rank calculation #168

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
db1e0a5
Break ties in rank calculation
vivekjoshy Jun 8, 2025
b129a57
Add changelog fragment
vivekjoshy Jun 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 11 additions & 13 deletions benchmark/draw.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,8 +156,8 @@
"Requirement already satisfied: arrow>=0.15.0 in /opt/conda/lib/python3.10/site-packages (from isoduration->jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.2.3)\r\n",
"Downloading rbo-0.1.3-py3-none-any.whl (7.8 kB)\r\n",
"Downloading openskill-6.0.0-py3-none-any.whl (50 kB)\r\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.9/50.9 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
"\u001b[?25hInstalling collected packages: rbo, openskill\r\n",
"\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m50.9/50.9 kB\u001B[0m \u001B[31m1.9 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
"\u001B[?25hInstalling collected packages: rbo, openskill\r\n",
"Successfully installed openskill-6.0.0 rbo-0.1.3\r\n",
"Note: you may need to restart the kernel to use updated packages.\n"
]
Expand Down Expand Up @@ -210,14 +210,17 @@
"source": [
"import gc\n",
"from dataclasses import dataclass\n",
"from enum import Enum\n",
"from pathlib import Path\n",
"from typing import Dict, List\n",
"\n",
"import ipywidgets as widgets\n",
"import numpy as np\n",
"import pandas as pd\n",
"import rich\n",
"from pooch import DOIDownloader\n",
"from rbo import rbo\n",
"from rich.table import Table\n",
"from sklearn.model_selection import train_test_split\n",
"from tqdm.notebook import tqdm\n",
"\n",
"import openskill\n",
Expand All @@ -227,12 +230,7 @@
" PlackettLuce,\n",
" ThurstoneMostellerFull,\n",
" ThurstoneMostellerPart,\n",
")\n",
"\n",
"from pooch import DOIDownloader\n",
"from pathlib import Path\n",
"from sklearn.model_selection import train_test_split\n",
"from enum import Enum"
")"
]
},
{
Expand Down Expand Up @@ -1139,12 +1137,12 @@
"</pre>\n"
],
"text/plain": [
"\u001b[3m Benchmark Results \u001b[0m\n",
"\u001B[3m Benchmark Results \u001B[0m\n",
"┏━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1m Information\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mValue \u001b[0m\u001b[1m \u001b[0m┃\n",
"┃\u001B[1m \u001B[0m\u001B[1m Information\u001B[0m\u001B[1m \u001B[0m┃\u001B[1m \u001B[0m\u001B[1mValue \u001B[0m\u001B[1m \u001B[0m┃\n",
"┡━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩\n",
"│\u001b[36m \u001b[0m\u001b[36m Available Matches\u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m20064 \u001b[0m\u001b[35m \u001b[0m│\n",
"│\u001b[36m \u001b[0m\u001b[36mPlackettLuce Accuracy\u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m19335/729 [ 96.37%]\u001b[0m\u001b[35m \u001b[0m│\n",
"│\u001B[36m \u001B[0m\u001B[36m Available Matches\u001B[0m\u001B[36m \u001B[0m│\u001B[35m \u001B[0m\u001B[35m20064 \u001B[0m\u001B[35m \u001B[0m│\n",
"│\u001B[36m \u001B[0m\u001B[36mPlackettLuce Accuracy\u001B[0m\u001B[36m \u001B[0m│\u001B[35m \u001B[0m\u001B[35m19335/729 [ 96.37%]\u001B[0m\u001B[35m \u001B[0m│\n",
"└───────────────────────┴─────────────────────┘\n"
]
},
Expand Down
61 changes: 30 additions & 31 deletions benchmark/rank.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@
"Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (1.26.4)\r\n",
"Collecting numpy\r\n",
" Downloading numpy-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\r\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.9/60.9 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
"\u001b[?25hRequirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (4.66.4)\r\n",
"\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m60.9/60.9 kB\u001B[0m \u001B[31m1.9 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
"\u001B[?25hRequirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (4.66.4)\r\n",
"Requirement already satisfied: pooch in /opt/conda/lib/python3.10/site-packages (1.8.2)\r\n",
"Requirement already satisfied: ipywidgets in /opt/conda/lib/python3.10/site-packages (7.7.1)\r\n",
"Collecting ipywidgets\r\n",
Expand Down Expand Up @@ -96,16 +96,16 @@
"Requirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.10/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0)\r\n",
"Downloading rbo-0.1.3-py3-none-any.whl (7.8 kB)\r\n",
"Downloading rich-13.7.1-py3-none-any.whl (240 kB)\r\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m240.7/240.7 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
"\u001b[?25hDownloading ipywidgets-8.1.3-py3-none-any.whl (139 kB)\r\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.4/139.4 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
"\u001b[?25hDownloading openskill-6.0.0-py3-none-any.whl (50 kB)\r\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.9/50.9 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
"\u001b[?25hDownloading jupyterlab_widgets-3.0.11-py3-none-any.whl (214 kB)\r\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m214.4/214.4 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
"\u001b[?25hDownloading widgetsnbextension-4.0.11-py3-none-any.whl (2.3 MB)\r\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.3/2.3 MB\u001b[0m \u001b[31m42.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
"\u001b[?25hInstalling collected packages: widgetsnbextension, rbo, openskill, jupyterlab-widgets, rich, ipywidgets\r\n",
"\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m240.7/240.7 kB\u001B[0m \u001B[31m9.1 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
"\u001B[?25hDownloading ipywidgets-8.1.3-py3-none-any.whl (139 kB)\r\n",
"\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m139.4/139.4 kB\u001B[0m \u001B[31m6.3 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
"\u001B[?25hDownloading openskill-6.0.0-py3-none-any.whl (50 kB)\r\n",
"\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m50.9/50.9 kB\u001B[0m \u001B[31m2.2 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
"\u001B[?25hDownloading jupyterlab_widgets-3.0.11-py3-none-any.whl (214 kB)\r\n",
"\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m214.4/214.4 kB\u001B[0m \u001B[31m8.9 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
"\u001B[?25hDownloading widgetsnbextension-4.0.11-py3-none-any.whl (2.3 MB)\r\n",
"\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m2.3/2.3 MB\u001B[0m \u001B[31m42.2 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
"\u001B[?25hInstalling collected packages: widgetsnbextension, rbo, openskill, jupyterlab-widgets, rich, ipywidgets\r\n",
" Attempting uninstall: widgetsnbextension\r\n",
" Found existing installation: widgetsnbextension 3.6.7\r\n",
" Uninstalling widgetsnbextension-3.6.7:\r\n",
Expand All @@ -122,10 +122,10 @@
" Found existing installation: ipywidgets 7.7.1\r\n",
" Uninstalling ipywidgets-7.7.1:\r\n",
" Successfully uninstalled ipywidgets-7.7.1\r\n",
"\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",
"\u001B[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",
"tensorflow-decision-forests 1.8.1 requires wurlitzer, which is not installed.\r\n",
"tensorflow 2.15.0 requires keras<2.16,>=2.15.0, but you have keras 3.4.1 which is incompatible.\u001b[0m\u001b[31m\r\n",
"\u001b[0mSuccessfully installed ipywidgets-8.1.3 jupyterlab-widgets-3.0.11 openskill-6.0.0 rbo-0.1.3 rich-13.7.1 widgetsnbextension-4.0.11\r\n",
"tensorflow 2.15.0 requires keras<2.16,>=2.15.0, but you have keras 3.4.1 which is incompatible.\u001B[0m\u001B[31m\r\n",
"\u001B[0mSuccessfully installed ipywidgets-8.1.3 jupyterlab-widgets-3.0.11 openskill-6.0.0 rbo-0.1.3 rich-13.7.1 widgetsnbextension-4.0.11\r\n",
"Note: you may need to restart the kernel to use updated packages.\n"
]
}
Expand Down Expand Up @@ -175,15 +175,17 @@
},
"outputs": [],
"source": [
"import time\n",
"import gc\n",
"import time\n",
"from dataclasses import dataclass\n",
"from pathlib import Path\n",
"from typing import Dict\n",
"\n",
"import ipywidgets as widgets\n",
"import numpy as np\n",
"import polars as pl\n",
"import rich\n",
"from pooch import DOIDownloader\n",
"from rbo import rbo\n",
"from rich.table import Table\n",
"from tqdm.notebook import tqdm\n",
Expand All @@ -195,10 +197,7 @@
" PlackettLuce,\n",
" ThurstoneMostellerFull,\n",
" ThurstoneMostellerPart,\n",
")\n",
"\n",
"from pooch import DOIDownloader\n",
"from pathlib import Path"
")"
]
},
{
Expand Down Expand Up @@ -1307,13 +1306,13 @@
"</pre>\n"
],
"text/plain": [
"\u001b[3m Benchmark Results \u001b[0m\n",
"\u001B[3m Benchmark Results \u001B[0m\n",
"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1m Information\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mValue \u001b[0m\u001b[1m \u001b[0m┃\n",
"┃\u001B[1m \u001B[0m\u001B[1m Information\u001B[0m\u001B[1m \u001B[0m┃\u001B[1m \u001B[0m\u001B[1mValue \u001B[0m\u001B[1m \u001B[0m┃\n",
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
"│\u001b[36m \u001b[0m\u001b[36m Available Matches\u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m145994 \u001b[0m\u001b[35m \u001b[0m│\n",
"│\u001b[36m \u001b[0m\u001b[36m PlackettLuce Accuracy\u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m134323/11671 [ 92.01%]\u001b[0m\u001b[35m \u001b[0m│\n",
"│\u001b[36m \u001b[0m\u001b[36m Rank-Biased Overlap Score:\u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 64.46 \u001b[0m\u001b[35m \u001b[0m│\n",
"│\u001B[36m \u001B[0m\u001B[36m Available Matches\u001B[0m\u001B[36m \u001B[0m│\u001B[35m \u001B[0m\u001B[35m145994 \u001B[0m\u001B[35m \u001B[0m│\n",
"│\u001B[36m \u001B[0m\u001B[36m PlackettLuce Accuracy\u001B[0m\u001B[36m \u001B[0m│\u001B[35m \u001B[0m\u001B[35m134323/11671 [ 92.01%]\u001B[0m\u001B[35m \u001B[0m│\n",
"│\u001B[36m \u001B[0m\u001B[36m Rank-Biased Overlap Score:\u001B[0m\u001B[36m \u001B[0m│\u001B[35m \u001B[0m\u001B[35m 64.46 \u001B[0m\u001B[35m \u001B[0m│\n",
"└─────────────────────────────┴────────────────────────┘\n"
]
},
Expand Down Expand Up @@ -1624,13 +1623,13 @@
"description": "",
"description_allow_html": false,
"layout": "IPY_MODEL_0d1a94edd9e4441f800b85c272cd4174",
"max": 51704546.0,
"max": 5.1704546E7,
"min": 0.0,
"orientation": "horizontal",
"style": "IPY_MODEL_c05b6b0fe0b14f8dba8c8532d5becad1",
"tabbable": null,
"tooltip": null,
"value": 51704546.0
"value": 5.1704546E7
}
},
"1fa24287923243499d42735aa608dff0": {
Expand Down Expand Up @@ -2076,13 +2075,13 @@
"description": "",
"description_allow_html": false,
"layout": "IPY_MODEL_5f83e32b011b4503b1e67a736ec1153f",
"max": 11487236.0,
"max": 1.1487236E7,
"min": 0.0,
"orientation": "horizontal",
"style": "IPY_MODEL_e87634fce8704dd3824c2a8685c5fd3b",
"tabbable": null,
"tooltip": null,
"value": 11487236.0
"value": 1.1487236E7
}
},
"6eb3fa3d7ca84770ab2d24b2f6a694f0": {
Expand Down Expand Up @@ -2666,13 +2665,13 @@
"description": "",
"description_allow_html": false,
"layout": "IPY_MODEL_f2d82cacecc9493295741ae970381eea",
"max": 51704546.0,
"max": 5.1704546E7,
"min": 0.0,
"orientation": "horizontal",
"style": "IPY_MODEL_651e7062e6e94d4495ccd2f06cb05c2a",
"tabbable": null,
"tooltip": null,
"value": 51703476.0
"value": 5.1703476E7
}
},
"bdfb5a775ba04c458828718e6926edc7": {
Expand Down
Loading