From 31c958d81c55ce478a757a18dce155ffbe5c4c4c Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Thu, 21 Aug 2025 11:44:00 +0100 Subject: [PATCH 01/12] Test build OSB forrk/branch --- applications/netpyne/deploy/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/applications/netpyne/deploy/values.yaml b/applications/netpyne/deploy/values.yaml index 69f9a87d5..e55d6abd0 100644 --- a/applications/netpyne/deploy/values.yaml +++ b/applications/netpyne/deploy/values.yaml @@ -19,8 +19,8 @@ harness: hard: - jupyterhub git: - - url: https://github.com/MetaCell/NetPyNE-UI.git - branch_tag: 1.1.0 + - url: https://github.com/OpenSourceBrain/NetPyNE-UI.git + branch_tag: osbv2_tests singleuser: cpu: limit: 1 From fd8842e817176fdb30008489e289cf53e1b2a0ba Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Thu, 21 Aug 2025 13:09:38 +0100 Subject: [PATCH 02/12] Update requirements for latest netpyne and neuroml libraries --- applications/netpyne/overrides/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/applications/netpyne/overrides/requirements.txt b/applications/netpyne/overrides/requirements.txt index 5326f342c..7640ea03c 100644 --- a/applications/netpyne/overrides/requirements.txt +++ b/applications/netpyne/overrides/requirements.txt @@ -71,7 +71,7 @@ jupyterthemes==0.20.0 kiwisolver==1.4.8 lesscpy==0.15.1 LFPykit==0.5.1 -libNeuroML==0.5.1 +libNeuroML==0.6.7 lxml==5.3.1 MarkupSafe==1.1.1 matplotlib==3.10.0 @@ -88,9 +88,9 @@ nbconvert==5.6.1 nbformat==5.2.0 ndindex==1.9.2 nest-asyncio==1.6.0 -netpyne @ git+https://github.com/Neurosim-lab/netpyne.git@3d633bcda9a3ab3fe4a90b7c705cd3692a729185 +netpyne @ git+https://github.com/Neurosim-lab/netpyne.git@osbv2-dev#egg=netpyne networkx==3.4.2 -neuromllite==0.5.4 +neuromllite==0.6.1 NEURON==8.2.6 notebook==6.4.5 notebook_shim==0.2.3 @@ -117,9 +117,9 @@ pycparser==2.22 pyecore==0.15.2 pygeppetto==0.9.0 Pygments==2.19.1 -PyLEMS==0.5.9 +PyLEMS==0.6.8 pymongo==4.11.1 -pyNeuroML==1.0.10 +pyNeuroML==1.3.21 pyparsing==3.2.1 pytest==6.2.5 python-dateutil==2.9.0.post0 From be3630504b09ad9d3799204b724ff1aff5239daa Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Thu, 21 Aug 2025 14:51:20 +0100 Subject: [PATCH 03/12] Quicker download of head model --- applications/netpyne/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/applications/netpyne/Dockerfile b/applications/netpyne/Dockerfile index eca06eea7..3b1ed003d 100644 --- a/applications/netpyne/Dockerfile +++ b/applications/netpyne/Dockerfile @@ -16,7 +16,9 @@ RUN yarn build-dev ### Download on a separate stage to run in parallel with buildkit FROM quay.io/jupyter/base-notebook:python-3.12 AS downloads USER root -RUN wget --no-check-certificate -O /nyhead.mat https://www.parralab.org/nyhead/sa_nyhead.mat +RUN apt-get update -qq && apt-get install aria2 -y +RUN aria2c -j10 -x 10 https://www.parralab.org/nyhead/sa_nyhead.mat +RUN mv sa_nyhead.mat /nyhead.mat ### FROM quay.io/jupyter/base-notebook:python-3.12 @@ -87,7 +89,7 @@ RUN ln -s /opt/workspace workspace # RUN jupyter labextension disable @jupyterlab/hub-extension -COPY --from=downloads --chown=1000:1000 /nyhead.mat $NP_LFPYKIT_HEAD_FILE +COPY --from=downloads --chown=1000:1000 nyhead.mat $NP_LFPYKIT_HEAD_FILE COPY --from=jsbuild --chown=1000:1000 $FOLDER/webapp/build webapp/build RUN chown -R $NB_UID /home/jovyan/.jupyter From 3e5593b128640be5293a9141fcef4e81392ff05c Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Fri, 22 Aug 2025 19:32:37 +0200 Subject: [PATCH 04/12] Revert change netpyne branch --- deployment/codefresh-dev.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/codefresh-dev.yaml b/deployment/codefresh-dev.yaml index 78433979b..ef43d1125 100644 --- a/deployment/codefresh-dev.yaml +++ b/deployment/codefresh-dev.yaml @@ -30,8 +30,8 @@ steps: clone_NetPyNE-UI_git_netpyne: title: Cloning NetPyNE-UI.git repository... type: git-clone - repo: https://github.com/MetaCell/NetPyNE-UI.git - revision: dev_netpyne_updates + repo: https://github.com/OpenSourceBrain/NetPyNE-UI.git + revision: osbv2_tests working_directory: applications/netpyne/dependencies/ git: github clone_nwb-explorer_git_nwb-explorer: From 1860ff55fbd7cb202cbcf43d44d6be6a6fe5eb4e Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Tue, 26 Aug 2025 10:41:34 +0100 Subject: [PATCH 05/12] Consolidated download of nyhead.mat into one line --- applications/netpyne/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/applications/netpyne/Dockerfile b/applications/netpyne/Dockerfile index 3b1ed003d..b13c90f25 100644 --- a/applications/netpyne/Dockerfile +++ b/applications/netpyne/Dockerfile @@ -16,9 +16,7 @@ RUN yarn build-dev ### Download on a separate stage to run in parallel with buildkit FROM quay.io/jupyter/base-notebook:python-3.12 AS downloads USER root -RUN apt-get update -qq && apt-get install aria2 -y -RUN aria2c -j10 -x 10 https://www.parralab.org/nyhead/sa_nyhead.mat -RUN mv sa_nyhead.mat /nyhead.mat +RUN apt-get update -qq && apt-get install aria2 -y && aria2c -j10 -x 10 https://www.parralab.org/nyhead/sa_nyhead.mat && mv sa_nyhead.mat /nyhead.mat ### FROM quay.io/jupyter/base-notebook:python-3.12 From 6892d4332fe9638f3ef524bdbba8594e353810c7 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Tue, 26 Aug 2025 12:20:45 +0100 Subject: [PATCH 06/12] Allow a local directory, shared, to be accessed inside the docker container for netpyneui --- .gitignore | 1 + applications/netpyne/run_local.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 082c4e5b0..9df469a54 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ skaffold-overrides.yaml /libraries/client/netpyne-web/ /.venv /libraries/client/*.pyc +/applications/netpyne/shared diff --git a/applications/netpyne/run_local.sh b/applications/netpyne/run_local.sh index 595ccb418..88fce41a1 100755 --- a/applications/netpyne/run_local.sh +++ b/applications/netpyne/run_local.sh @@ -3,6 +3,6 @@ set -e # A script to run the NetPyNE container locally (build it first with ./build_local.sh) -docker run --network host -it --rm --name mynp mynetpyneosb +docker run --network host -v $PWD/shared:/opt/workspace/local:rw -it --rm --name mynp mynetpyneosb From 2331908bcc8e348028a80c67780a1cc3678c370a Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Tue, 26 Aug 2025 14:22:14 +0200 Subject: [PATCH 07/12] Remove netpyne gatekeeper --- applications/netpyne/deploy/values.yaml | 2 +- applications/workspaces/api/openapi.yaml | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/applications/netpyne/deploy/values.yaml b/applications/netpyne/deploy/values.yaml index 7b538e8f8..4ecf10686 100644 --- a/applications/netpyne/deploy/values.yaml +++ b/applications/netpyne/deploy/values.yaml @@ -1,6 +1,6 @@ harness: subdomain: netpyne - secured: true + secured: false uri_role_mapping: [] service: auto: false diff --git a/applications/workspaces/api/openapi.yaml b/applications/workspaces/api/openapi.yaml index 8b2037ce0..83f37a7e1 100644 --- a/applications/workspaces/api/openapi.yaml +++ b/applications/workspaces/api/openapi.yaml @@ -271,6 +271,8 @@ paths: security: - bearerAuth: [] + - + cookieAuth: [] operationId: workspaces.controllers.workspace_controller.setthumbnail summary: Sets the thumbnail of the workspace. parameters: @@ -302,6 +304,8 @@ paths: security: - bearerAuth: [] + - + cookieAuth: [] operationId: workspaces.controllers.workspace_controller.addimage summary: Adds and image to the workspace. parameters: @@ -324,6 +328,8 @@ paths: security: - bearerAuth: [] + - + cookieAuth: [] operationId: delimage summary: Delete a Workspace Image from the workspace. x-openapi-router-controller: workspaces.controllers.workspace_controller @@ -365,6 +371,8 @@ paths: security: - bearerAuth: [] + - + cookieAuth: [] summary: Used to save a WorkspaceResource to the repository. '/workspaceresource/{id}': get: @@ -382,6 +390,8 @@ paths: security: - bearerAuth: [] + - + cookieAuth: [] summary: Used to retrieve a WorkspaceResource. put: requestBody: @@ -1517,6 +1527,11 @@ components: bearerFormat: JWT type: http x-bearerInfoFunc: cloudharness.auth.AuthClient.decode_token + cookieAuth: + type: apiKey + name: kc-access + in: cookie + x-apikeyInfoFunc: cloudharness.auth.decode_token tags: - name: rest From 05ba58a74e693a9d70a6b8392e878bb0d15e19a6 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 27 Aug 2025 18:45:56 +0100 Subject: [PATCH 08/12] Minor changes in Dockerfile to aid admin bash scripting --- applications/netpyne/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/applications/netpyne/Dockerfile b/applications/netpyne/Dockerfile index b13c90f25..eb88bf469 100644 --- a/applications/netpyne/Dockerfile +++ b/applications/netpyne/Dockerfile @@ -27,10 +27,13 @@ ENV NP_LFPYKIT_HEAD_FILE=/home/jovyan/nyhead.mat USER root +### Some helpful aliases +RUN echo -e '\n\nalias cd..="cd .." \nalias h=history \nalias ll="ls -alt" \n' >> ~/.bashrc + RUN rm -rf /var/lib/apt/lists RUN apt-get update -qq &&\ apt-get install python3-tk vim nano unzip git make libtool g++ -qq pkg-config libfreetype6-dev libpng-dev libopenmpi-dev -y -RUN apt-get install openjdk-11-jre-headless -y +RUN apt-get install openjdk-11-jre-headless htop ncdu -y # RUN conda install python=3.7 -y From fa93fc827f8c19d0f2cfe334615c755b4c61e472 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Wed, 27 Aug 2025 19:10:37 +0100 Subject: [PATCH 09/12] Update About box: Add Gleeson lab, full url to logos Url update for https://github.com/OpenSourceBrain/OSBv2/issues/990. This may not be an issue on live, so could be reverted... --- .../src/components/MainDrawer/MainDrawer.tsx | 2 +- .../src/components/dialogs/AboutDialog.tsx | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/applications/osb-portal/src/components/MainDrawer/MainDrawer.tsx b/applications/osb-portal/src/components/MainDrawer/MainDrawer.tsx index 41c18be0c..6b0c7d408 100644 --- a/applications/osb-portal/src/components/MainDrawer/MainDrawer.tsx +++ b/applications/osb-portal/src/components/MainDrawer/MainDrawer.tsx @@ -338,7 +338,7 @@ export const MainDrawer = (props: { - + { - const logoMetaCell = "/images/metacell.png"; - const logoWellcome = "/images/wellcome.png"; + const logoMetaCell = "https://raw.githubusercontent.com/OpenSourceBrain/OSBv2/refs/heads/master/applications/osb-portal/src/assets/images/metacell.png"; + const logoWellcome = "https://raw.githubusercontent.com/OpenSourceBrain/OSBv2/refs/heads/master/applications/osb-portal/src/assets/images/wellcome.png"; return ( @@ -60,9 +60,16 @@ export const AboutContent = (props: any) => { OSBv2 is being developed by the{" "} - Silver Lab at University College London + Silver Lab {" "} - in collaboration with{" "} + and the{" "} + + Gleeson Lab + {" "} + at {" "} + + University College London + , in collaboration with{" "} MetaCell From 94a9954460a6c8722b80dc793cf1943fbc3c4d7f Mon Sep 17 00:00:00 2001 From: Filippo Ledda Date: Thu, 28 Aug 2025 12:02:43 +0200 Subject: [PATCH 10/12] Patch keycloak bootstrap issue --- .../accounts/deploy/resources/realm.json | 57 ------------------- 1 file changed, 57 deletions(-) diff --git a/applications/accounts/deploy/resources/realm.json b/applications/accounts/deploy/resources/realm.json index 80888301c..8a3f11593 100644 --- a/applications/accounts/deploy/resources/realm.json +++ b/applications/accounts/deploy/resources/realm.json @@ -158,63 +158,6 @@ ] }, "clients": [ - { - "id": "18893fbb-8252-4aaa-bc9b-60799ceb9932", - "clientId": "account-console", - "name": "${client_account-console}", - "rootUrl": "${authBaseUrl}", - "baseUrl": {{ printf "/realms/%s/account" .Values.namespace | quote }}, - "surrogateAuthRequired": false, - "enabled": true, - "alwaysDisplayInConsole": false, - "clientAuthenticatorType": "client-secret", - "redirectUris": [ - {{ printf "/realms/%s/account/*" .Values.namespace | quote }} - ], - "webOrigins": [], - "notBefore": 0, - "bearerOnly": false, - "consentRequired": false, - "standardFlowEnabled": true, - "implicitFlowEnabled": false, - "directAccessGrantsEnabled": false, - "serviceAccountsEnabled": false, - "publicClient": true, - "frontchannelLogout": false, - "protocol": "openid-connect", - "attributes": { - "realm_client": "false", - "post.logout.redirect.uris": "+", - "pkce.code.challenge.method": "S256" - }, - "authenticationFlowBindingOverrides": {}, - "fullScopeAllowed": false, - "nodeReRegistrationTimeout": 0, - "protocolMappers": [ - { - "id": "9a68ec2d-943d-49cb-9fdd-cd821d606210", - "name": "audience resolve", - "protocol": "openid-connect", - "protocolMapper": "oidc-audience-resolve-mapper", - "consentRequired": false, - "config": {} - } - ], - "defaultClientScopes": [ - "service_account", - "web-origins", - "acr", - "address", - "administrator-scope", - "phone", - "profile", - "roles", - "microprofile-jwt", - "basic", - "email" - ], - "optionalClientScopes": [] - }, { "id": "9a6a2560-c6be-4493-8bd5-3fdc4522d82b", "clientId": {{ .Values.apps.accounts.client.id | quote }}, From 3d54f1cd4138669eed67d09decf9806ff36b8114 Mon Sep 17 00:00:00 2001 From: "Ankur Sinha (Ankur Sinha Gmail)" Date: Thu, 28 Aug 2025 11:33:12 +0100 Subject: [PATCH 11/12] feat(local-script): do not make a skaffold failure stop minikube --- local-scripts/osbv2-local.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/local-scripts/osbv2-local.sh b/local-scripts/osbv2-local.sh index c5d3c1908..57d59b5d8 100755 --- a/local-scripts/osbv2-local.sh +++ b/local-scripts/osbv2-local.sh @@ -153,8 +153,8 @@ deploy () { harness_deployment echo "-> running skaffold" - $SKAFFOLD dev --cleanup=false || { notify_fail "Failed: skaffold" ; minikube stop; } - #$SKAFFOLD dev || notify_fail "Failed: skaffold" + $SKAFFOLD dev --cleanup=false || { notify_fail "Failed: skaffold" ; } + minkube status popd } @@ -193,10 +193,18 @@ harness_deployment() { if [ "YES" == "$LIVE" ] then echo "-> harnessing live configuration deployment, and deploying" + set -x + set -v harness-deployment ../cloud-harness . -l -n ${OSB_NAMESPACE} -d osb.local -r gcr.io/metacellllc -e "local" -t "$LIVE_TAG" || notify_fail "Failed: harness-deployment (live)" + set +x + set +v else echo "-> harnessing development deployment" + set -x + set -v harness-deployment ../cloud-harness . -l -n ${OSB_NAMESPACE} -d osb.local -dtls -e "local" ${DEPLOYMENT_APP:+-i $DEPLOYMENT_APP} || notify_fail "Failed: harness-deployment (dev)" + set +x + set +v fi popd } @@ -242,10 +250,12 @@ deactivate_venv() { print_versions() { echo "** docker **" docker version + echo -e "\n** kubernetes **" + kubectl version echo -e "\n** minikube **" minikube version echo -e "\n** cloud harness **" - pushd "${CLOUD_HARNESS_DIR}" && git log --oneline | head -1 && popd + pushd "${CLOUD_HARNESS_DIR}" 2>&1 > /dev/null && git log --oneline | head -1 && popd 2>&1 > /dev/null echo -e "\n** helm **" helm version echo -e "\n** skaffold **" From f44774e8aad75f03f0dffe9da143c68f10fa93b5 Mon Sep 17 00:00:00 2001 From: Padraig Gleeson Date: Thu, 28 Aug 2025 11:34:48 +0100 Subject: [PATCH 12/12] Updates to Jupyterlab: uses osbv2 netpyne branch & add scripts --- .github/workflows/docker-jupyterlab.yml | 9 +++++---- applications/jupyterlab/build_local.sh | 13 +++++++++++++ applications/jupyterlab/pip_omv_info.sh | 8 ++++++++ applications/jupyterlab/rebuild_local.sh | 13 +++++++++++++ applications/jupyterlab/requirements.txt | 2 +- applications/jupyterlab/run_local.sh | 8 ++++++++ 6 files changed, 48 insertions(+), 5 deletions(-) create mode 100755 applications/jupyterlab/build_local.sh create mode 100755 applications/jupyterlab/pip_omv_info.sh create mode 100755 applications/jupyterlab/rebuild_local.sh create mode 100755 applications/jupyterlab/run_local.sh diff --git a/.github/workflows/docker-jupyterlab.yml b/.github/workflows/docker-jupyterlab.yml index 56a8ab49c..103f698be 100644 --- a/.github/workflows/docker-jupyterlab.yml +++ b/.github/workflows/docker-jupyterlab.yml @@ -1,4 +1,4 @@ -name: Docker Image Build Jupyterlab +name: Docker Image Build JupyterLab on: push: @@ -13,12 +13,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build the Docker image run: | cd applications/jupyterlab - docker build -t myjlab -f Dockerfile --no-cache . + ./build_local.sh - name: Info on Docker image sizes run: | @@ -26,4 +26,5 @@ jobs: - name: Run the Docker container and list python installs run: | - docker run -t --rm --entrypoint /bin/bash myjlab -c "pip3 list" + cd applications/jupyterlab + ./pip_omv_info.sh diff --git a/applications/jupyterlab/build_local.sh b/applications/jupyterlab/build_local.sh new file mode 100755 index 000000000..4eebcecc9 --- /dev/null +++ b/applications/jupyterlab/build_local.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -ex + +# Set the platform flag if we're on ARM +arch=$(uname -m) +if [[ "$arch" == "arm64" || "$arch" == "aarch64" ]]; then + platform_flag="--platform linux/amd64" +else + platform_flag="" +fi + +time DOCKER_BUILDKIT=1 docker build $platform_flag -t myjlab -f Dockerfile . + diff --git a/applications/jupyterlab/pip_omv_info.sh b/applications/jupyterlab/pip_omv_info.sh new file mode 100755 index 000000000..c3b56a48c --- /dev/null +++ b/applications/jupyterlab/pip_omv_info.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +# A script to print info versions of packages in the JupyterLab container + +docker run -t --rm --entrypoint /bin/bash myjlab -c "pip3 list" +echo "--------------------------------------------------------" +docker run -t --rm --entrypoint /bin/bash myjlab -c "omv list -V" diff --git a/applications/jupyterlab/rebuild_local.sh b/applications/jupyterlab/rebuild_local.sh new file mode 100755 index 000000000..15eca6be3 --- /dev/null +++ b/applications/jupyterlab/rebuild_local.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -ex + +# Set the platform flag if we're on ARM +arch=$(uname -m) +if [[ "$arch" == "arm64" || "$arch" == "aarch64" ]]; then + platform_flag="--platform linux/amd64" +else + platform_flag="" +fi + +time DOCKER_BUILDKIT=1 docker build $platform_flag -t myjlab -f Dockerfile --no-cache . + diff --git a/applications/jupyterlab/requirements.txt b/applications/jupyterlab/requirements.txt index cce0760db..9874151aa 100644 --- a/applications/jupyterlab/requirements.txt +++ b/applications/jupyterlab/requirements.txt @@ -6,7 +6,7 @@ neurotune neuron # Install specific version of NetPyNE -git+https://github.com/Neurosim-lab/netpyne.git@osbv2#egg=netpyne +git+https://github.com/Neurosim-lab/netpyne.git@osbv2-dev#egg=netpyne #### Other simulators diff --git a/applications/jupyterlab/run_local.sh b/applications/jupyterlab/run_local.sh new file mode 100755 index 000000000..269054b73 --- /dev/null +++ b/applications/jupyterlab/run_local.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +# A script to run the JupyterLab container locally (build it first with ./build_local.sh) + +docker run --network host -it --rm --name myjupyterlab myjlab + +