Merge branch 'main' into issue/209-xrdatarray-b

Author: peterdudfield

notebooks/move_EUMETSAT_files_to_shallower_directory_structure.ipynb

@@ -0,0 +1,121 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "0d39c43a-afae-4a54-b1ea-eb48f00f10ae",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pathlib import Path"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "41dcea91-eaab-4e46-ba83-fada36357067",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "EUMETSAT_PATH = Path(\"/mnt/storage_a/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/satellite/EUMETSAT/SEVIRI_RSS/native/\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "3533e596-57c7-4e52-b83d-0da0f2c72ea2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def list_directories(path: Path, pattern: str) -> list[Path]:\n",
+    "    return [x for x in path.glob(pattern) if x.is_dir()]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "d1db021b-9a19-483b-ad3e-fddcc72b194c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 10.8 ms, sys: 4.65 ms, total: 15.5 ms\n",
+      "Wall time: 32.1 ms\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "613"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "year_month_day_directories = list_directories(path=EUMETSAT_PATH, pattern='*/*/*')\n",
+    "n = len(year_month_day_directories)\n",
+    "n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "a422f554-a8c1-4e48-9070-54d9a74cbf83",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "613/613: /mnt/storage_a/data/ocf/solar_pv_nowcasting/nowcasting_dataset_pipeline/satellite/EUMETSAT/SEVIRI_RSS/native/2020/02/09"
+     ]
+    }
+   ],
+   "source": [
+    "for i, year_month_day_directory in enumerate(year_month_day_directories):\n",
+    "    print(f'\\r{i+1:3d}/{n:3d}: {year_month_day_directory}', end='', flush=True)\n",
+    "    \n",
+    "    # Find all the '.nat' and '.nat.bz2' files:\n",
+    "    files_for_day = [x for x in year_month_day_directory.glob('*/*/*.*') if '.nat' in x.suffixes]\n",
+    "    \n",
+    "    # Move the files:\n",
+    "    for file in files_for_day:\n",
+    "        target_filename = year_month_day_directory / file.name\n",
+    "        file.rename(target_filename)\n",
+    "        \n",
+    "    # Remove the empty hour/day directories:\n",
+    "    for minute_dir in list_directories(year_month_day_directory, '*/*'):\n",
+    "        minute_dir.rmdir()\n",
+    "    for hour_dir in list_directories(year_month_day_directory, '*'):\n",
+    "        hour_dir.rmdir()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "nowcasting_dataset",
+   "language": "python",
+   "name": "nowcasting_dataset"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}