Improve CUTEst benchmarks

.JuliaFormatter.toml

@@ -0,0 +1,3 @@
+style = "sciml"
+format_markdown = true
+format_docstrings = true

.buildkite/run_benchmark.yml

@@ -19,8 +19,8 @@ steps:
       - JuliaCI/julia#v1:
           version: "1.10"
       - staticfloat/sandbox:
-          rootfs_url: "https://github.com/thazhemadam/openmodelica-rootfs-image/releases/download/v1.23.0/rootfs-openmodelica-v1.23.0.amd64.tar.gz"
-          rootfs_treehash: "82970243dc4f188e599a976abc20951f4aba2912"
+          rootfs_url: "https://github.com/thazhemadam/ScimlBenchmarks-rootfs-image/releases/download/rootfs-image/rootfs-script.amd64.tar.gz"
+          rootfs_treehash: "943bde824554f0afad5a939341f05e3bdb393051"
           uid: 1000
           gid: 1000
           workspaces:

.buildkite/test_sciml.yml

@@ -1,8 +1,8 @@
 steps:
-  - label: ":julia: Run tests on 1.8"
+  - label: ":julia: Run tests on LTS"
     plugins:
       - JuliaCI/julia#v1:
-          version: 1.9
+          version: '1.10'
       - JuliaCI/julia-test#v1:
     timeout_in_minutes: 20
     artifact_paths:

.github/workflows/update.jl

@@ -12,14 +12,14 @@ date = Dates.format(now(), "yyyy-mm-dd")
 benchpath = joinpath(@__DIR__, "..", "..", "benchmarks")
 
 # Get all the open PRs and their number
-gh_prs = GitHub.pull_requests("SciML/SciMLBenchmarks.jl"; auth=myauth)
+gh_prs = GitHub.pull_requests("SciML/SciMLBenchmarks.jl"; auth = myauth)
 prs = Dict{String, Int64}()
 for i in 1:length(gh_prs[1])
     prs[gh_prs[1][i].head.ref] = gh_prs[1][i].number
 end
 
 # Get all the branches from the repo
-gh_branches = GitHub.branches("SciML/SciMLBenchmarks.jl"; auth=myauth)
+gh_branches = GitHub.branches("SciML/SciMLBenchmarks.jl"; auth = myauth)
 branches = [gh_branches[1][i].name for i in 1:length(gh_branches[1])]
 
 @info("PRs and branches", prs, branches)
@@ -50,11 +50,11 @@ for dir in readdir(benchpath)
         if dir ∉ keys(prs)
             params = Dict(
                 "title" => "Updated $(dir) for benchmarks",
-                "head"  => "$(dir)",
-                "base"  => "master"
+                "head" => "$(dir)",
+                "base" => "master"
             )
-            @info("Creating a pull request from head: ", dir) 
-            GitHub.create_pull_request("SciML/SciMLBenchmarks.jl"; params=params, auth=myauth)
+            @info("Creating a pull request from head: ", dir)
+            GitHub.create_pull_request("SciML/SciMLBenchmarks.jl"; params = params, auth = myauth)
         else
             @info("Updating the pull request numbered: ", prs[dir])
             GitHub.update_pull_request("SciML/SciMLBenchmarks.jl", prs[dir]; auth=myauth)

README.md

@@ -31,6 +31,9 @@ or building work-precision diagrams for direct comparison of speed at given erro
 
 **If any of the code from any of the languages can be improved, please open a pull request**.
 
+For critiques of benchmarks, please open a pull request that changes the code in the desired manner. Issues with recommended
+changes are generally vague and not actionable, while pull requests with code changes are exact. Thus if there is something
+you think should be changed in the code, please make the recommended change in the code!
 
 ## Results
 

benchmarks/AdaptiveSDE/AdaptiveEfficiencyTests.jmd

@@ -8,138 +8,163 @@ author: Chris Rackauckas
 using Distributed
 addprocs(2)
 
-p1 = Vector{Any}(undef,3)
-p2 = Vector{Any}(undef,3)
-p3 = Vector{Any}(undef,3)
+p1 = Vector{Any}(undef, 3)
+p2 = Vector{Any}(undef, 3)
+p3 = Vector{Any}(undef, 3)
 
 @everywhere begin
-  using StochasticDiffEq, SDEProblemLibrary, DiffEqNoiseProcess, Plots, ParallelDataTransfer
-  import SDEProblemLibrary: prob_sde_additive,
-              prob_sde_linear, prob_sde_wave
+    using StochasticDiffEq, SDEProblemLibrary, DiffEqNoiseProcess, Plots,
+          ParallelDataTransfer
+    import SDEProblemLibrary: prob_sde_additive,
+                              prob_sde_linear, prob_sde_wave
 end
 
 using StochasticDiffEq, SDEProblemLibrary, DiffEqNoiseProcess, Plots, ParallelDataTransfer
 import SDEProblemLibrary: prob_sde_additive,
-            prob_sde_linear, prob_sde_wave
+                          prob_sde_linear, prob_sde_wave
 
-probs = Matrix{SDEProblem}(undef,3,3)
+probs = Matrix{SDEProblem}(undef, 3, 3)
 ## Problem 1
 prob = prob_sde_linear
-probs[1,1] = SDEProblem(prob.f,prob.g,prob.u0,prob.tspan,prob.p,noise=WienerProcess(0.0,0.0,0.0,rswm=RSWM(adaptivealg=:RSwM1)))
-probs[1,2] = SDEProblem(prob.f,prob.g,prob.u0,prob.tspan,prob.p,noise=WienerProcess(0.0,0.0,0.0,rswm=RSWM(adaptivealg=:RSwM2)))
-probs[1,3] = SDEProblem(prob.f,prob.g,prob.u0,prob.tspan,prob.p,noise=WienerProcess(0.0,0.0,0.0,rswm=RSWM(adaptivealg=:RSwM3)))
+probs[1, 1] = SDEProblem(prob.f, prob.g, prob.u0, prob.tspan, prob.p,
+    noise = WienerProcess(0.0, 0.0, 0.0, rswm = RSWM(adaptivealg = :RSwM1)))
+probs[1, 2] = SDEProblem(prob.f, prob.g, prob.u0, prob.tspan, prob.p,
+    noise = WienerProcess(0.0, 0.0, 0.0, rswm = RSWM(adaptivealg = :RSwM2)))
+probs[1, 3] = SDEProblem(prob.f, prob.g, prob.u0, prob.tspan, prob.p,
+    noise = WienerProcess(0.0, 0.0, 0.0, rswm = RSWM(adaptivealg = :RSwM3)))
 ## Problem 2
 prob = prob_sde_wave
-probs[2,1] = SDEProblem(prob.f,prob.g,prob.u0,prob.tspan,prob.p,noise=WienerProcess(0.0,0.0,0.0,rswm=RSWM(adaptivealg=:RSwM1)))
-probs[2,2] = SDEProblem(prob.f,prob.g,prob.u0,prob.tspan,prob.p,noise=WienerProcess(0.0,0.0,0.0,rswm=RSWM(adaptivealg=:RSwM2)))
-probs[2,3] = SDEProblem(prob.f,prob.g,prob.u0,prob.tspan,prob.p,noise=WienerProcess(0.0,0.0,0.0,rswm=RSWM(adaptivealg=:RSwM3)))
+probs[2, 1] = SDEProblem(prob.f, prob.g, prob.u0, prob.tspan, prob.p,
+    noise = WienerProcess(0.0, 0.0, 0.0, rswm = RSWM(adaptivealg = :RSwM1)))
+probs[2, 2] = SDEProblem(prob.f, prob.g, prob.u0, prob.tspan, prob.p,
+    noise = WienerProcess(0.0, 0.0, 0.0, rswm = RSWM(adaptivealg = :RSwM2)))
+probs[2, 3] = SDEProblem(prob.f, prob.g, prob.u0, prob.tspan, prob.p,
+    noise = WienerProcess(0.0, 0.0, 0.0, rswm = RSWM(adaptivealg = :RSwM3)))
 ## Problem 3
 prob = prob_sde_additive
-probs[3,1] = SDEProblem(prob.f,prob.g,prob.u0,prob.tspan,prob.p,noise=WienerProcess(0.0,0.0,0.0,rswm=RSWM(adaptivealg=:RSwM1)))
-probs[3,2] = SDEProblem(prob.f,prob.g,prob.u0,prob.tspan,prob.p,noise=WienerProcess(0.0,0.0,0.0,rswm=RSWM(adaptivealg=:RSwM2)))
-probs[3,3] = SDEProblem(prob.f,prob.g,prob.u0,prob.tspan,prob.p,noise=WienerProcess(0.0,0.0,0.0,rswm=RSWM(adaptivealg=:RSwM3)))
-
-fullMeans = Vector{Array}(undef,3)
-fullMedians = Vector{Array}(undef,3)
-fullElapsed = Vector{Array}(undef,3)
-fullTols = Vector{Array}(undef,3)
+probs[3, 1] = SDEProblem(prob.f, prob.g, prob.u0, prob.tspan, prob.p,
+    noise = WienerProcess(0.0, 0.0, 0.0, rswm = RSWM(adaptivealg = :RSwM1)))
+probs[3, 2] = SDEProblem(prob.f, prob.g, prob.u0, prob.tspan, prob.p,
+    noise = WienerProcess(0.0, 0.0, 0.0, rswm = RSWM(adaptivealg = :RSwM2)))
+probs[3, 3] = SDEProblem(prob.f, prob.g, prob.u0, prob.tspan, prob.p,
+    noise = WienerProcess(0.0, 0.0, 0.0, rswm = RSWM(adaptivealg = :RSwM3)))
+
+fullMeans = Vector{Array}(undef, 3)
+fullMedians = Vector{Array}(undef, 3)
+fullElapsed = Vector{Array}(undef, 3)
+fullTols = Vector{Array}(undef, 3)
 offset = 0
 
-Ns = [17,23,
-17]
+Ns = [17, 23,
+    17]
 ```
 
 Timings are only valid if no workers die. Workers die if you run out of memory.
 
 ```julia
-for k in 1:size(probs,1)
-  global probs, Ns, fullMeans, fullMedians, fullElapsed, fullTols
-  println("Problem $k")
-  ## Setup
-  N = Ns[k]
-
-  msims = Vector{Any}(undef,N)
-  elapsed = Array{Float64}(undef,N,3)
-  medians = Array{Float64}(undef,N,3)
-  means   = Array{Float64}(undef,N,3)
-  tols    = Array{Float64}(undef,N,3)
-
-  #Compile
-  prob = probs[k,1]
-  ParallelDataTransfer.sendto(workers(), prob=prob)
-  monte_prob = EnsembleProblem(prob)
-  solve(monte_prob,SRIW1(),dt=1/2^(4),adaptive=true,trajectories=1000,abstol=2.0^(-1),reltol=0)
-
-  println("RSwM1")
-  for i=1+offset:N+offset
-    tols[i-offset,1] = 2.0^(-i-1)
-    msims[i-offset] = DiffEqBase.calculate_monte_errors(solve(monte_prob,SRIW1(),
-                                            trajectories=1000,abstol=2.0^(-i-1),
-                                            reltol=0,force_dtmin=true))
-    elapsed[i-offset,1] = msims[i-offset].elapsedTime
-    medians[i-offset,1] = msims[i-offset].error_medians[:final]
-    means[i-offset,1]   = msims[i-offset].error_means[:final]
-  end
-
-  println("RSwM2")
-  prob = probs[k,2]
-
-  ParallelDataTransfer.sendto(workers(), prob=prob)
-  monte_prob = EnsembleProblem(prob)
-  solve(monte_prob,SRIW1(),dt=1/2^(4),adaptive=true,trajectories=1000,abstol=2.0^(-1),reltol=0)
-
-  for i=1+offset:N+offset
-    tols[i-offset,2] = 2.0^(-i-1)
-    msims[i-offset] = DiffEqBase.calculate_monte_errors(solve(monte_prob,SRIW1(),
-                                            trajectories=1000,abstol=2.0^(-i-1),
-                                            reltol=0,force_dtmin=true))
-    elapsed[i-offset,2] = msims[i-offset].elapsedTime
-    medians[i-offset,2] = msims[i-offset].error_medians[:final]
-    means[i-offset,2]   = msims[i-offset].error_means[:final]
-  end
-
-  println("RSwM3")
-  prob = probs[k,3]
-  ParallelDataTransfer.sendto(workers(), prob=prob)
-  monte_prob = EnsembleProblem(prob)
-  solve(monte_prob,SRIW1(),dt=1/2^(4),adaptive=true,trajectories=1000,abstol=2.0^(-1),reltol=0)
-
-  for i=1+offset:N+offset
-    tols[i-offset,3] = 2.0^(-i-1)
-        msims[i-offset] = DiffEqBase.calculate_monte_errors(solve(monte_prob,SRIW1(),
-                                    adaptive=true,trajectories=1000,abstol=2.0^(-i-1),
-                                    reltol=0,force_dtmin=true))
-    elapsed[i-offset,3] = msims[i-offset].elapsedTime
-    medians[i-offset,3] = msims[i-offset].error_medians[:final]
-    means[i-offset,3]   = msims[i-offset].error_means[:final]
-  end
-
-  fullMeans[k] = means
-  fullMedians[k] =medians
-  fullElapsed[k] = elapsed
-  fullTols[k] = tols
+for k in 1:size(probs, 1)
+    global probs, Ns, fullMeans, fullMedians, fullElapsed, fullTols
+    println("Problem $k")
+    ## Setup
+    N = Ns[k]
+
+    msims = Vector{Any}(undef, N)
+    elapsed = Array{Float64}(undef, N, 3)
+    medians = Array{Float64}(undef, N, 3)
+    means = Array{Float64}(undef, N, 3)
+    tols = Array{Float64}(undef, N, 3)
+
+    #Compile
+    prob = probs[k, 1]
+    ParallelDataTransfer.sendto(workers(), prob = prob)
+    monte_prob = EnsembleProblem(prob)
+    solve(monte_prob, SRIW1(), dt = 1/2^(4), adaptive = true,
+        trajectories = 1000, abstol = 2.0^(-1), reltol = 0)
+
+    println("RSwM1")
+    for i in (1 + offset):(N + offset)
+        tols[i - offset, 1] = 2.0^(-i-1)
+        msims[i - offset] = DiffEqBase.calculate_monte_errors(solve(monte_prob, SRIW1(),
+            trajectories = 1000, abstol = 2.0^(-i-1),
+            reltol = 0, force_dtmin = true))
+        elapsed[i - offset, 1] = msims[i - offset].elapsedTime
+        medians[i - offset, 1] = msims[i - offset].error_medians[:final]
+        means[i - offset, 1] = msims[i - offset].error_means[:final]
+    end
+
+    println("RSwM2")
+    prob = probs[k, 2]
+
+    ParallelDataTransfer.sendto(workers(), prob = prob)
+    monte_prob = EnsembleProblem(prob)
+    solve(monte_prob, SRIW1(), dt = 1/2^(4), adaptive = true,
+        trajectories = 1000, abstol = 2.0^(-1), reltol = 0)
+
+    for i in (1 + offset):(N + offset)
+        tols[i - offset, 2] = 2.0^(-i-1)
+        msims[i - offset] = DiffEqBase.calculate_monte_errors(solve(monte_prob, SRIW1(),
+            trajectories = 1000, abstol = 2.0^(-i-1),
+            reltol = 0, force_dtmin = true))
+        elapsed[i - offset, 2] = msims[i - offset].elapsedTime
+        medians[i - offset, 2] = msims[i - offset].error_medians[:final]
+        means[i - offset, 2] = msims[i - offset].error_means[:final]
+    end
+
+    println("RSwM3")
+    prob = probs[k, 3]
+    ParallelDataTransfer.sendto(workers(), prob = prob)
+    monte_prob = EnsembleProblem(prob)
+    solve(monte_prob, SRIW1(), dt = 1/2^(4), adaptive = true,
+        trajectories = 1000, abstol = 2.0^(-1), reltol = 0)
+
+    for i in (1 + offset):(N + offset)
+        tols[i - offset, 3] = 2.0^(-i-1)
+        msims[i - offset] = DiffEqBase.calculate_monte_errors(solve(monte_prob, SRIW1(),
+            adaptive = true, trajectories = 1000, abstol = 2.0^(-i-1),
+            reltol = 0, force_dtmin = true))
+        elapsed[i - offset, 3] = msims[i - offset].elapsedTime
+        medians[i - offset, 3] = msims[i - offset].error_medians[:final]
+        means[i - offset, 3] = msims[i - offset].error_means[:final]
+    end
+
+    fullMeans[k] = means
+    fullMedians[k] = medians
+    fullElapsed[k] = elapsed
+    fullTols[k] = tols
 end
 ```
 
 ```julia
-gr(fmt=:svg)
+gr(fmt = :svg)
 lw=3
-leg=String["RSwM1","RSwM2","RSwM3"]
+leg=String["RSwM1", "RSwM2", "RSwM3"]
 
 titleFontSize = 16
 guideFontSize = 14
-legendFontSize= 14
-tickFontSize  = 12
-
-for k in 1:size(probs,1)
-  global probs, Ns, fullMeans, fullMedians, fullElapsed, fullTols
-  p1[k] = Plots.plot(fullTols[k],fullMeans[k],xscale=:log10,yscale=:log10,  xguide="Absolute Tolerance",yguide="Mean Final Error",title="Example $k"  ,linewidth=lw,grid=false,lab=leg,titlefont=font(titleFontSize),legendfont=font(legendFontSize),tickfont=font(tickFontSize),guidefont=font(guideFontSize))
-  p2[k] = Plots.plot(fullTols[k],fullMedians[k],xscale=:log10,yscale=:log10,xguide="Absolute Tolerance",yguide="Median Final Error",title="Example $k",linewidth=lw,grid=false,lab=leg,titlefont=font(titleFontSize),legendfont=font(legendFontSize),tickfont=font(tickFontSize),guidefont=font(guideFontSize))
-  p3[k] = Plots.plot(fullTols[k],fullElapsed[k],xscale=:log10,yscale=:log10,xguide="Absolute Tolerance",yguide="Elapsed Time",title="Example $k"      ,linewidth=lw,grid=false,lab=leg,titlefont=font(titleFontSize),legendfont=font(legendFontSize),tickfont=font(tickFontSize),guidefont=font(guideFontSize))
+legendFontSize = 14
+tickFontSize = 12
+
+for k in 1:size(probs, 1)
+    global probs, Ns, fullMeans, fullMedians, fullElapsed, fullTols
+    p1[k] = Plots.plot(fullTols[k], fullMeans[k], xscale = :log10, yscale = :log10,
+        xguide = "Absolute Tolerance", yguide = "Mean Final Error",
+        title = "Example $k", linewidth = lw, grid = false, lab = leg,
+        titlefont = font(titleFontSize), legendfont = font(legendFontSize),
+        tickfont = font(tickFontSize), guidefont = font(guideFontSize))
+    p2[k] = Plots.plot(fullTols[k], fullMedians[k], xscale = :log10, yscale = :log10,
+        xguide = "Absolute Tolerance", yguide = "Median Final Error",
+        title = "Example $k", linewidth = lw, grid = false, lab = leg,
+        titlefont = font(titleFontSize), legendfont = font(legendFontSize),
+        tickfont = font(tickFontSize), guidefont = font(guideFontSize))
+    p3[k] = Plots.plot(fullTols[k], fullElapsed[k], xscale = :log10, yscale = :log10,
+        xguide = "Absolute Tolerance", yguide = "Elapsed Time",
+        title = "Example $k", linewidth = lw, grid = false, lab = leg,
+        titlefont = font(titleFontSize), legendfont = font(legendFontSize),
+        tickfont = font(tickFontSize), guidefont = font(guideFontSize))
 end
 
 Plots.plot!(p1[1])
-Plots.plot(p1[1],p1[2],p1[3],layout=(3,1),size=(1000,800))
+Plots.plot(p1[1], p1[2], p1[3], layout = (3, 1), size = (1000, 800))
 ```
 
 ```julia
@@ -148,17 +173,17 @@ Plots.plot(p1[1],p1[2],p1[3],layout=(3,1),size=(1000,800))
 ```
 
 ```julia
-plot(p3[1],p3[2],p3[3],layout=(3,1),size=(1000,800))
+plot(p3[1], p3[2], p3[3], layout = (3, 1), size = (1000, 800))
 #savefig("timevstol.png")
 #savefig("timevstol.pdf")
 ```
 
 ```julia
-plot(p1[1],p3[1],p1[2],p3[2],p1[3],p3[3],layout=(3,2),size=(1000,800))
+plot(p1[1], p3[1], p1[2], p3[2], p1[3], p3[3], layout = (3, 2), size = (1000, 800))
 ```
 
 ```julia
 
 using SciMLBenchmarks
-SciMLBenchmarks.bench_footer(WEAVE_ARGS[:folder],WEAVE_ARGS[:file])
+SciMLBenchmarks.bench_footer(WEAVE_ARGS[:folder], WEAVE_ARGS[:file])
 ```

benchmarks/AdaptiveSDE/Project.toml

@@ -4,14 +4,14 @@ Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 ParallelDataTransfer = "2dcacdae-9679-587a-88bb-8b444fb7085b"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
-SciMLBenchmarks = "31c91b34-3c75-11e9-0341-95557aab0344"
 SDEProblemLibrary = "c72e72a9-a271-4b2b-8966-303ed956772e"
+SciMLBenchmarks = "31c91b34-3c75-11e9-0341-95557aab0344"
 StochasticDiffEq = "789caeaf-c7a9-5a7d-9973-96adeb23e2a0"
 
 [compat]
 DiffEqNoiseProcess = "5.0"
 ParallelDataTransfer = "0.5"
 Plots = "1.4"
+SDEProblemLibrary = "0.1, 1"
 SciMLBenchmarks = "0.1"
-SDEProblemLibrary = "0.1"
 StochasticDiffEq = "6.23"