From 1fe2d4bb667b50bdf7c20c0a4c46694bcfa78820 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 30 Dec 2024 21:25:12 -0600 Subject: [PATCH 1/4] build profiling image in CI --- .dockerignore | 1 - .github/workflows/main.yml | 3 +++ Dockerfile-profiling | 14 ++++++++------ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.dockerignore b/.dockerignore index f852e82..230e8a1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,5 +13,4 @@ !LightGBM/python-package !LightGBM/src !LightGBM/swig -!LightGBM/windows !LightGBM/VERSION.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 20a4fdf..2ad0a60 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,6 +31,9 @@ jobs: - name: Build cluster image run: | make cluster-image + - name: Build profiling image + run: | + make profiling-image all-tests-successful: if: always() runs-on: ubuntu-latest diff --git a/Dockerfile-profiling b/Dockerfile-profiling index 1b2dc36..d55dd86 100644 --- a/Dockerfile-profiling +++ b/Dockerfile-profiling @@ -3,12 +3,14 @@ ARG BASE_IMAGE=unset # hadolint ignore=DL3006 FROM ${BASE_IMAGE} -RUN pip install --no-cache-dir --prefer-binary \ - memray \ - pytest \ - pytest-memray \ - pytest-profiling \ - snakeviz +RUN << Date: Mon, 30 Dec 2024 22:12:21 -0600 Subject: [PATCH 2/4] more changes --- Dockerfile-cluster | 2 +- Dockerfile-notebook | 2 +- Makefile | 3 ++ notebooks/demo-aws.ipynb | 88 ++++++++++++++++++++++++++++++++-------- 4 files changed, 77 insertions(+), 18 deletions(-) diff --git a/Dockerfile-cluster b/Dockerfile-cluster index 0a446cf..b3d9c91 100644 --- a/Dockerfile-cluster +++ b/Dockerfile-cluster @@ -3,7 +3,7 @@ ARG BASE_IMAGE=unset # hadolint ignore=DL3006 FROM ${BASE_IMAGE} -RUN --mount=type=bind,source=LightGBM,target=/tmp/LightGBM \ +RUN --mount=type=bind,source=LightGBM,target=/tmp/LightGBM,rw \ < 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mlightgbm\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdask\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m DaskLGBMRegressor\n\u001b[1;32m 3\u001b[0m dask_reg \u001b[38;5;241m=\u001b[39m DaskLGBMRegressor(\n\u001b[1;32m 4\u001b[0m client\u001b[38;5;241m=\u001b[39mclient,\n\u001b[1;32m 5\u001b[0m max_depth\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m5\u001b[39m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 10\u001b[0m min_child_samples\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m,\n\u001b[1;32m 11\u001b[0m )\n\u001b[1;32m 13\u001b[0m dask_reg\u001b[38;5;241m.\u001b[39mfit(\n\u001b[1;32m 14\u001b[0m X\u001b[38;5;241m=\u001b[39mdX,\n\u001b[1;32m 15\u001b[0m y\u001b[38;5;241m=\u001b[39mdy,\n\u001b[1;32m 16\u001b[0m )\n", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'lightgbm'" + ] + } + ], "source": [ "from lightgbm.dask import DaskLGBMRegressor\n", "\n", @@ -300,7 +356,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.4" + "version": "3.12.8" } }, "nbformat": 4, From f21e7865b5ecb5c052bdd6ffe31600c7f10a22fd Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 30 Dec 2024 22:16:01 -0600 Subject: [PATCH 3/4] clear outputs --- notebooks/demo-aws.ipynb | 62 +++++++++------------------------------- 1 file changed, 13 insertions(+), 49 deletions(-) diff --git a/notebooks/demo-aws.ipynb b/notebooks/demo-aws.ipynb index e5d1ada..7e2a64f 100644 --- a/notebooks/demo-aws.ipynb +++ b/notebooks/demo-aws.ipynb @@ -40,18 +40,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "expanded-declaration", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "scheduler and worker image: public.ecr.aws/w8s1c8b1/lightgbm-dask-testing-cluster-jlamb:py3.12-dask2024.6.2\n" - ] - } - ], + "outputs": [], "source": [ "import json\n", "import os\n", @@ -79,16 +71,17 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "4a10d61c-5251-46a7-9f16-bd6eef606a82", "metadata": {}, "outputs": [], "source": [ "import platform\n", + "\n", "if platform.machine().lower() in {\"aarch64\", \"arm64\"}:\n", - " cpu_architecture=\"ARM64\"\n", + " cpu_architecture = \"ARM64\"\n", "else:\n", - " cpu_architecture=\"X86_64\"" + " cpu_architecture = \"X86_64\"" ] }, { @@ -104,16 +97,7 @@ "execution_count": null, "id": "respective-collect", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.12/contextlib.py:144: UserWarning: Creating your cluster is taking a surprisingly long time. This is likely due to pending resources on AWS. Hang tight! \n", - " next(self.gen)\n" - ] - } - ], + "outputs": [], "source": [ "from dask.distributed import Client\n", "from dask_cloudprovider.aws import FargateCluster\n", @@ -134,18 +118,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "raising-mauritius", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "View the dashboard: http://52.55.229.38:8787/status\n" - ] - } - ], + "outputs": [], "source": [ "print(f\"View the dashboard: {cluster.dashboard_link}\")" ] @@ -174,7 +150,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "structural-street", "metadata": {}, "outputs": [], @@ -199,7 +175,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "quiet-nicaragua", "metadata": {}, "outputs": [], @@ -225,22 +201,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "pleased-brunei", "metadata": {}, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'lightgbm'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[13], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mlightgbm\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdask\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m DaskLGBMRegressor\n\u001b[1;32m 3\u001b[0m dask_reg \u001b[38;5;241m=\u001b[39m DaskLGBMRegressor(\n\u001b[1;32m 4\u001b[0m client\u001b[38;5;241m=\u001b[39mclient,\n\u001b[1;32m 5\u001b[0m max_depth\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m5\u001b[39m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 10\u001b[0m min_child_samples\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m,\n\u001b[1;32m 11\u001b[0m )\n\u001b[1;32m 13\u001b[0m dask_reg\u001b[38;5;241m.\u001b[39mfit(\n\u001b[1;32m 14\u001b[0m X\u001b[38;5;241m=\u001b[39mdX,\n\u001b[1;32m 15\u001b[0m y\u001b[38;5;241m=\u001b[39mdy,\n\u001b[1;32m 16\u001b[0m )\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'lightgbm'" - ] - } - ], + "outputs": [], "source": [ "from lightgbm.dask import DaskLGBMRegressor\n", "\n", From e2da62facf38be2123ddc598f45c7e809838900a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 30 Dec 2024 22:29:28 -0600 Subject: [PATCH 4/4] fix syntax --- Dockerfile-profiling | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-profiling b/Dockerfile-profiling index d55dd86..d1b4095 100644 --- a/Dockerfile-profiling +++ b/Dockerfile-profiling @@ -3,7 +3,7 @@ ARG BASE_IMAGE=unset # hadolint ignore=DL3006 FROM ${BASE_IMAGE} -RUN <<