Skip to content

Commit e79afc8

Browse files
f-PLTf-PLT
authored
Merge pull request #10 from RolnickLab/downloader_refactoring
Downloader refactoring
2 parents 22e29d8 + 07303f9 commit e79afc8

35 files changed

+1277
-804
lines changed

‎climateset/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@
99
LOAD_DATA = DATA_DIR / "load"
1010
META_DATA = DATA_DIR / "meta"
1111
SCRIPT_DIR = PROJECT_ROOT / "scripts"
12+
TEST_DIR = PROJECT_ROOT / "tests"

‎climateset/cli.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import typer
2+
from click import Context
3+
from typer.core import TyperGroup
4+
5+
from climateset import CONFIGS
6+
from climateset.download.downloader import download_from_config_file
7+
8+
MINIMAL_DATASET = CONFIGS / "minimal_dataset.yaml"
9+
10+
11+
class OrderCommands(TyperGroup):
12+
def list_commands(self, ctx: Context):
13+
return list(self.commands)
14+
15+
16+
app = typer.Typer(
17+
no_args_is_help=True,
18+
cls=OrderCommands,
19+
)
20+
21+
22+
@app.command(name="info", help="General info.")
23+
def info():
24+
typer.echo("\nWelcome to the ClimateSet CLI!\n")
25+
typer.echo("More information to come!\n")
26+
27+
28+
@app.command(name="download", help="Download ClimateSet data via configuration file.")
29+
def download_command(config: str = MINIMAL_DATASET):
30+
download_from_config_file(config)
31+
32+
33+
if __name__ == "__main__":
34+
app()

‎climateset/download/constants/data_constants.py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,3 @@
1-
import pandas as pd
2-
3-
from climateset import APP_ROOT
4-
5-
# TODO figure out a better/cleaner way to handle this...
6-
DATA_CSV = pd.read_csv(APP_ROOT / "download/constants/selected_scenariosMIPs.csv")
7-
81
EMISSIONS_ENDINGS = ["_em_openburning", "_em_anthro", "_em_AIR_anthro"]
92

103
META_ENDINGS_PRC = [

‎climateset/download/constants/esm_constants.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@
4343
# model
4444
MODELS = ["nan"]
4545

46-
# number of esemble members to be considered
47-
NUM_ESEMBLE = 1
46+
# number of ensemble members to be considered
47+
NUM_ENSEMBLE = 1
4848

4949
# which type of grid
5050
GRID = "grid"

‎climateset/download/constants/selected_scenariosMIPs.csv‎

Lines changed: 0 additions & 37 deletions
This file was deleted.

‎climateset/download/constants/selected_scenariosMIPs.json‎

Lines changed: 290 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,290 @@
1+
[
2+
{
3+
"Unnamed: 0": 0,
4+
"source_id": "ACCESS-CM2",
5+
"nominal_resolution_km": 250,
6+
"scenarios": "ssp126 ssp245 ssp370 ssp534-over ssp585",
7+
"num_ensemble_members": "5 5 5 1 5",
8+
"data_node_available": true
9+
},
10+
{
11+
"Unnamed: 0": 1,
12+
"source_id": "ACCESS-ESM1-5",
13+
"nominal_resolution_km": 250,
14+
"scenarios": "ssp126 ssp245 ssp370 ssp534-over ssp585",
15+
"num_ensemble_members": "40 40 40 12 40",
16+
"data_node_available": true
17+
},
18+
{
19+
"Unnamed: 0": 2,
20+
"source_id": "AWI-CM-1-1-MR",
21+
"nominal_resolution_km": 100,
22+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
23+
"num_ensemble_members": "1 1 5 1",
24+
"data_node_available": true
25+
},
26+
{
27+
"Unnamed: 0": 3,
28+
"source_id": "BCC-CSM2-MR",
29+
"nominal_resolution_km": 100,
30+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
31+
"num_ensemble_members": "1 1 1 1",
32+
"data_node_available": true
33+
},
34+
{
35+
"Unnamed: 0": 4,
36+
"source_id": "CAMS-CSM1-0",
37+
"nominal_resolution_km": 100,
38+
"scenarios": "ssp119 ssp126 ssp245 ssp370 ssp585",
39+
"num_ensemble_members": "2 2 2 2 2",
40+
"data_node_available": true
41+
},
42+
{
43+
"Unnamed: 0": 7,
44+
"source_id": "CAS-ESM2-0",
45+
"nominal_resolution_km": 100,
46+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
47+
"num_ensemble_members": "2 2 2 2",
48+
"data_node_available": true
49+
},
50+
{
51+
"Unnamed: 0": 8,
52+
"source_id": "CESM2",
53+
"nominal_resolution_km": 100,
54+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
55+
"num_ensemble_members": "3 3 3 3",
56+
"data_node_available": true
57+
},
58+
{
59+
"Unnamed: 0": 9,
60+
"source_id": "CESM2-WACCM",
61+
"nominal_resolution_km": 100,
62+
"scenarios": "ssp126 ssp245 ssp370 ssp534-over ssp585",
63+
"num_ensemble_members": "1 5 3 3 5",
64+
"data_node_available": true
65+
},
66+
{
67+
"Unnamed: 0": 11,
68+
"source_id": "CMCC-CM2-SR5",
69+
"nominal_resolution_km": 100,
70+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
71+
"num_ensemble_members": "1 1 1 1",
72+
"data_node_available": true
73+
},
74+
{
75+
"Unnamed: 0": 12,
76+
"source_id": "CMCC-ESM2",
77+
"nominal_resolution_km": 100,
78+
"scenarios": "ssp126 ssp245 ssp370 ssp534-over ssp585",
79+
"num_ensemble_members": "1 1 1 1 1",
80+
"data_node_available": true
81+
},
82+
{
83+
"Unnamed: 0": 13,
84+
"source_id": "CNRM-CM6-1",
85+
"nominal_resolution_km": 250,
86+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
87+
"num_ensemble_members": "6 10 6 6",
88+
"data_node_available": true
89+
},
90+
{
91+
"Unnamed: 0": 14,
92+
"source_id": "CNRM-CM6-1-HR",
93+
"nominal_resolution_km": 100,
94+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
95+
"num_ensemble_members": "1 1 1 1",
96+
"data_node_available": true
97+
},
98+
{
99+
"Unnamed: 0": 15,
100+
"source_id": "CNRM-ESM2-1",
101+
"nominal_resolution_km": 250,
102+
"scenarios": "ssp119 ssp126 ssp245 ssp370 ssp434 ssp460 ssp534-over ssp585",
103+
"num_ensemble_members": "5 5 10 5 5 5 5 5",
104+
"data_node_available": true
105+
},
106+
{
107+
"Unnamed: 0": 19,
108+
"source_id": "EC-Earth3",
109+
"nominal_resolution_km": 100,
110+
"scenarios": "ssp119 ssp126 ssp245 ssp370 ssp434 ssp460 ssp534-over ssp585",
111+
"num_ensemble_members": "52 57 97 57 50 50 50 58",
112+
"data_node_available": false
113+
},
114+
{
115+
"Unnamed: 0": 22,
116+
"source_id": "EC-Earth3-Veg",
117+
"nominal_resolution_km": 100,
118+
"scenarios": "ssp119 ssp126 ssp245 ssp370 ssp585",
119+
"num_ensemble_members": "3 7 8 6 8",
120+
"data_node_available": false
121+
},
122+
{
123+
"Unnamed: 0": 23,
124+
"source_id": "EC-Earth3-Veg-LR",
125+
"nominal_resolution_km": 250,
126+
"scenarios": "ssp119 ssp126 ssp245 ssp370 ssp585",
127+
"num_ensemble_members": "3 3 3 3 3",
128+
"data_node_available": true
129+
},
130+
{
131+
"Unnamed: 0": 24,
132+
"source_id": "FGOALS-f3-L",
133+
"nominal_resolution_km": 100,
134+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
135+
"num_ensemble_members": "1 1 1 1",
136+
"data_node_available": true
137+
},
138+
{
139+
"Unnamed: 0": 25,
140+
"source_id": "FGOALS-g3",
141+
"nominal_resolution_km": 250,
142+
"scenarios": "ssp119 ssp126 ssp245 ssp370 ssp434 ssp460 ssp534-over ssp585",
143+
"num_ensemble_members": "1 4 4 5 1 1 1 4",
144+
"data_node_available": true
145+
},
146+
{
147+
"Unnamed: 0": 27,
148+
"source_id": "GFDL-ESM4",
149+
"nominal_resolution_km": 100,
150+
"scenarios": "ssp119 ssp126 ssp245 ssp370 ssp585",
151+
"num_ensemble_members": "1 1 3 1 1",
152+
"data_node_available": true
153+
},
154+
{
155+
"Unnamed: 0": 28,
156+
"source_id": "GISS-E2-1-G",
157+
"nominal_resolution_km": 250,
158+
"scenarios": "ssp119 ssp126 ssp245 ssp370 ssp434 ssp460 ssp534-over ssp585",
159+
"num_ensemble_members": "7 16 36 27 7 8 11 15",
160+
"data_node_available": true
161+
},
162+
{
163+
"Unnamed: 0": 29,
164+
"source_id": "GISS-E2-1-H",
165+
"nominal_resolution_km": 250,
166+
"scenarios": "ssp119 ssp126 ssp245 ssp370 ssp434 ssp460 ssp534-over ssp585",
167+
"num_ensemble_members": "2 10 10 6 2 2 6 10",
168+
"data_node_available": true
169+
},
170+
{
171+
"Unnamed: 0": 30,
172+
"source_id": "GISS-E2-2-G",
173+
"nominal_resolution_km": 250,
174+
"scenarios": "ssp126 ssp245 ssp370 ssp534-over ssp585",
175+
"num_ensemble_members": "5 5 5 5 5",
176+
"data_node_available": true
177+
},
178+
{
179+
"Unnamed: 0": 31,
180+
"source_id": "IITM-ESM",
181+
"nominal_resolution_km": 250,
182+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
183+
"num_ensemble_members": "1 1 1 1",
184+
"data_node_available": true
185+
},
186+
{
187+
"Unnamed: 0": 32,
188+
"source_id": "INM-CM4-8",
189+
"nominal_resolution_km": 100,
190+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
191+
"num_ensemble_members": "1 1 1 1",
192+
"data_node_available": true
193+
},
194+
{
195+
"Unnamed: 0": 33,
196+
"source_id": "INM-CM5-0",
197+
"nominal_resolution_km": 100,
198+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
199+
"num_ensemble_members": "1 1 5 1",
200+
"data_node_available": true
201+
},
202+
{
203+
"Unnamed: 0": 35,
204+
"source_id": "IPSL-CM6A-LR",
205+
"nominal_resolution_km": 250,
206+
"scenarios": "ssp119 ssp126 ssp245 ssp370 ssp434 ssp460 ssp534-over ssp585",
207+
"num_ensemble_members": "6 6 11 11 2 7 1 7",
208+
"data_node_available": true
209+
},
210+
{
211+
"Unnamed: 0": 36,
212+
"source_id": "KACE-1-0-G",
213+
"nominal_resolution_km": 250,
214+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
215+
"num_ensemble_members": "3 3 3 3",
216+
"data_node_available": true
217+
},
218+
{
219+
"Unnamed: 0": 38,
220+
"source_id": "MCM-UA-1-0",
221+
"nominal_resolution_km": 250,
222+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
223+
"num_ensemble_members": "1 1 1 1",
224+
"data_node_available": true
225+
},
226+
{
227+
"Unnamed: 0": 40,
228+
"source_id": "MIROC6",
229+
"nominal_resolution_km": 250,
230+
"scenarios": "ssp119 ssp126 ssp245 ssp370 ssp434 ssp460 ssp534-over ssp585",
231+
"num_ensemble_members": "50 50 50 50 1 1 1 50",
232+
"data_node_available": true
233+
},
234+
{
235+
"Unnamed: 0": 42,
236+
"source_id": "MPI-ESM1-2-HR",
237+
"nominal_resolution_km": 100,
238+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
239+
"num_ensemble_members": "2 2 10 2",
240+
"data_node_available": true
241+
},
242+
{
243+
"Unnamed: 0": 43,
244+
"source_id": "MPI-ESM1-2-LR",
245+
"nominal_resolution_km": 250,
246+
"scenarios": "ssp119 ssp126 ssp245 ssp370 ssp585",
247+
"num_ensemble_members": "30 30 30 30 30",
248+
"data_node_available": true
249+
},
250+
{
251+
"Unnamed: 0": 44,
252+
"source_id": "MRI-ESM2-0",
253+
"nominal_resolution_km": 100,
254+
"scenarios": "ssp119 ssp126 ssp245 ssp370 ssp434 ssp460 ssp534-over ssp585",
255+
"num_ensemble_members": "5 5 10 5 1 1 1 6",
256+
"data_node_available": true
257+
},
258+
{
259+
"Unnamed: 0": 46,
260+
"source_id": "NorESM2-LM",
261+
"nominal_resolution_km": 250,
262+
"scenarios": "ssp126 ssp245 ssp370 ssp534-over ssp585",
263+
"num_ensemble_members": "1 13 3 1 1",
264+
"data_node_available": true
265+
},
266+
{
267+
"Unnamed: 0": 47,
268+
"source_id": "NorESM2-MM",
269+
"nominal_resolution_km": 100,
270+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
271+
"num_ensemble_members": "1 2 1 1",
272+
"data_node_available": true
273+
},
274+
{
275+
"Unnamed: 0": 48,
276+
"source_id": "TaiESM1",
277+
"nominal_resolution_km": 100,
278+
"scenarios": "ssp126 ssp245 ssp370 ssp585",
279+
"num_ensemble_members": "1 1 1 1",
280+
"data_node_available": false
281+
},
282+
{
283+
"Unnamed: 0": 51,
284+
"source_id": "UKESM1-0-LL",
285+
"nominal_resolution_km": 250,
286+
"scenarios": "ssp119 ssp126 ssp245 ssp370 ssp434 ssp534-over ssp585",
287+
"num_ensemble_members": "5 16 17 16 5 5 5",
288+
"data_node_available": true
289+
}
290+
]

0 commit comments

Comments
 (0)