Switch site to MultiDocumenter.jl and add package docs

.github/workflows/Deploy.yml

@@ -1,50 +1,30 @@
-name: Build and Deploy
+name: Build and deploy
+
 on:
+  schedule:
+    - cron: "0 */6 * * *" # Run every 6 hours
+  pull_request:
   push:
     branches:
-      - main
-    paths:
-      - 'website/**'
+      - master
+    tags: "*"
+  workflow_dispatch:
+
+permissions:
+  actions: write
+  contents: write
+
 jobs:
-  build-and-deploy:
+  docs:
+    name: Documentation
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-      with:
-        persist-credentials: false
-    # NOTE: Python is necessary for the pre-rendering (minification) step
-    - name: Install python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.8'
-    # NOTE: Here you can install dependencies such as matplotlib if you use
-    # packages such as PyPlot.
-    # - run: pip install matplotlib
-    - name: Install Julia
-      uses: julia-actions/setup-julia@v1
-      with:
-        version: 1.8
-    # NOTE
-    #   The steps below ensure that NodeJS and Franklin are loaded then it
-    #   installs highlight.js which is needed for the prerendering step
-    #   (code highlighting + katex prerendering).
-    #   Then the environment is activated and instantiated to install all
-    #   Julia packages which may be required to successfully build your site.
-    #   The last line should be `optimize()` though you may want to give it
-    #   specific arguments, see the documentation or ?optimize in the REPL.
-    - run: julia -e '
-            cd("website");
-            using Pkg; Pkg.activate("."); Pkg.instantiate();
-            using NodeJS; run(`$(npm_cmd()) install highlight.js`);
-            run(`$(npm_cmd()) install lunr`);
-            run(`$(npm_cmd()) install cheerio`);
-            using Franklin;
-            lunr();
-            optimize()'
-    - name: Build and Deploy
-      uses: JamesIves/github-pages-deploy-action@releases/v3
-      with:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        BRANCH: gh-pages
-        FOLDER: website/__site
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+      - uses: julia-actions/cache@v2
+      - run: |
+          julia --project=docs -e 'using Pkg; Pkg.instantiate()'
+      - run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          julia --project=docs docs/make.jl deploy

.gitignore

@@ -1,3 +1,6 @@
 .DS_Store
 .vscode/
-.ipynb_checkpoints/
+.ipynb_checkpoints/
+docs/build
+docs/clones
+*/Manifest.toml

JuliaHPC/Project.toml

@@ -0,0 +1,4 @@
+name = "JuliaHPC"
+uuid = "785f4173-1021-4a9f-a8d5-3f9e0cca329a"
+authors = ["JamesWrigley <james@puiterwijk.org>"]
+version = "0.1.0"

JuliaHPC/src/JuliaHPC.jl

@@ -0,0 +1,5 @@
+module JuliaHPC
+
+greet() = print("Hello World!")
+
+end # module JuliaHPC

docs/Project.toml

@@ -0,0 +1,4 @@
+[deps]
+Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+LiveServer = "16fef848-5104-11e9-1b77-fb7a48bbb589"
+MultiDocumenter = "87ed4bf0-c935-4a67-83c3-2a03bee4197c"

docs/make.jl

@@ -0,0 +1,116 @@
+import Documenter
+import MultiDocumenter
+
+clonedir = joinpath(@__DIR__, "clones")
+deploying = "deploy" in ARGS
+
+# Build local docs
+Documenter.makedocs(
+    sitename = "JuliaHPC",
+    pages = [
+        "index.md",
+        "For Users" => [
+            "user_gettingstarted.md",
+            "user_vscode.md",
+            "user_hpcprofiling/index.md",
+            "user_hpcsystems.md",
+            "user_faq.md"
+        ],
+        "For System Admins" => [
+            "sysadmin_julia.md"
+        ]
+    ],
+    format = Documenter.HTML(assets = ["assets/favicon.ico"])
+)
+
+# Helper function stolen from DynamicalSystemsDocs.jl
+function multidocref(package, descr = "")
+    name = "$(package).jl"
+
+    MultiDocumenter.MultiDocRef(;
+        upstream = joinpath(clonedir, package),
+        path = lowercase(package),
+        name = isempty(descr) ? "$(name)" : "$(name) - $(descr)",
+        giturl = "https://github.com/JuliaParallel/$(name).git",
+    )
+end
+
+link(package) = MultiDocumenter.Link("$(package).jl", "https://github.com/JuliaParallel/$(package).jl")
+
+docs = [
+    MultiDocumenter.MultiDocRef(
+        upstream = joinpath(@__DIR__, "build"),
+        path = "Overview",
+        name = "Home",
+        fix_canonical_url = false,),
+    MultiDocumenter.DropdownNav("HPC packages", [
+        multidocref("MPI", "Julia bindings to MPI"),
+        multidocref("Dagger", "DAG-based computing"),
+        MultiDocumenter.MultiDocRef(;
+           upstream = joinpath(clonedir, "ParallelProcessingTools"),
+           path = "parallelprocessingtools",
+           name = "ParallelProcessingTools.jl - Utilities for doing parallel/distributed computing",
+           giturl = "https://github.com/oschulz/ParallelProcessingTools.jl.git")
+    ]),
+    MultiDocumenter.MegaDropdownNav("Distributed packages", [
+        MultiDocumenter.Column("Implementations", [
+            MultiDocumenter.Link("Distributed.jl - Stdlib for distributed computing", "https://docs.julialang.org/en/v1/stdlib/Distributed/"),
+            multidocref("DistributedNext", "Bleeding-edge fork of Distributed.jl")           
+        ]),
+        MultiDocumenter.Column("Cluster managers", [
+            link("SlurmClusterManager"),
+            link("MPIClusterManagers"),
+            link("ElasticClusterManager"),
+            link("LSFClusterManager")
+        ])
+    ])
+]
+
+outpath = deploying ? mktempdir() : joinpath(@__DIR__, "build")
+
+MultiDocumenter.make(
+    outpath,
+    docs;
+    search_engine = MultiDocumenter.SearchConfig(
+        index_versions = ["stable"],
+        engine = MultiDocumenter.FlexSearch
+    ),
+    assets_dir = "docs/src/assets",
+    brand_image = MultiDocumenter.BrandImage("https://juliahpc.github.io", "assets/logo.png")
+)
+
+if "deploy" in ARGS
+    @info "Deploying to GitHub" ARGS
+    gitroot = normpath(joinpath(@__DIR__, ".."))
+    run(`git pull`)
+    outbranch = "gh-pages"
+    has_outbranch = true
+    if !success(`git checkout $outbranch`)
+        has_outbranch = false
+        if !success(`git switch --orphan $outbranch`)
+            @error "Cannot create new orphaned branch $outbranch."
+            exit(1)
+        end
+    end
+    for file in readdir(gitroot; join = true)
+        endswith(file, ".git") && continue
+        rm(file; force = true, recursive = true)
+    end
+    for file in readdir(outpath)
+        cp(joinpath(outpath, file), joinpath(gitroot, file))
+    end
+    run(`git add .`)
+    if success(`git commit -m 'Aggregate documentation'`)
+        @info "Pushing updated documentation."
+        if has_outbranch
+            run(`git push`)
+        else
+            run(`git push -u origin $outbranch`)
+        end
+        run(`git checkout master`)
+    else
+        @info "No changes to aggregated documentation."
+    end
+else
+    @info "Skipping deployment, 'deploy' not passed. Generated files in $(outpath)." ARGS
+end

docs/src/index.md

@@ -0,0 +1,24 @@
+# Julia on HPC clusters
+
+👋 Welcome to the notes about the [Julia programming
+language](http://www.julialang.org/) on HPC clusters.
+
+## Objective
+The goal of these notes is to offer good practices and tips and tricks about how
+to use and set up Julia on HPC clusters. The target audiences are therefore
+Julia users on clusters as well as HPC system administrators (see the respective
+sections in the navigation bar on the left).
+
+## Acknowledgements
+
+These notes don't come out of nowhere but stem from endless conversations with
+many people on the Julia discourse, over Slack and Discord, as well as in-person
+discussions at Julia HPC minisymposia, BoFs, and workshops.
+
+They also build upon previous efforts to document best practices, most notably
+[https://github.com/hlrs-tasc/julia-on-hpc-systems/](https://github.com/hlrs-tasc/julia-on-hpc-systems/),
+from which we took (among other things) the [HPC systems with Julia
+support](user_hpcsystems.md) information.
+
+The creator and main author is [Carsten Bauer](https://github.com/carstenbauer)
+but many members of the Julia HPC community have made valuable contributions.