From 03898371809e24569915e115bf983942c6931e79 Mon Sep 17 00:00:00 2001
From: Ben <bbbales2@gmail.com>
Date: Fri, 22 May 2020 15:07:49 -0400
Subject: [PATCH 001/157] Revert "Revert back to forward mode sensitivity for
 cvodes (to make separate branch for adjoint ODEs)"

This reverts commit 962ba839e10177ff9440e980ff856ea747884424.
---
 stan/math/rev/functor/cvodes_integrator.hpp | 662 +++++++++++++++-----
 stan/math/rev/functor/ode_bdf.hpp           |  12 +-
 2 files changed, 521 insertions(+), 153 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator.hpp b/stan/math/rev/functor/cvodes_integrator.hpp
index bb54b685031..a2df7919fa6 100644
--- a/stan/math/rev/functor/cvodes_integrator.hpp
+++ b/stan/math/rev/functor/cvodes_integrator.hpp
@@ -2,6 +2,7 @@
 #define STAN_MATH_REV_FUNCTOR_INTEGRATE_ODE_CVODES_HPP
 
 #include <stan/math/rev/meta.hpp>
+#include <stan/math/rev/core/save_varis.hpp>
 #include <stan/math/rev/functor/cvodes_utils.hpp>
 #include <stan/math/rev/functor/coupled_ode_system.hpp>
 #include <stan/math/rev/functor/ode_store_sensitivities.hpp>
@@ -16,6 +17,115 @@
 namespace stan {
 namespace math {
 
+template <typename T_Return>
+inline std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> build_varis(
+    vari* ode_vari, vari**& non_chaining_varis,
+    const std::vector<Eigen::VectorXd>& y);
+
+template <>
+inline std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> build_varis<var>(
+    vari* ode_vari, vari**& non_chaining_varis,
+    const std::vector<Eigen::VectorXd>& y) {
+  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y_return(y.size());
+
+  if (y.size() == 0) {
+    return y_return;
+  }
+
+  int N = y[0].size();
+
+  non_chaining_varis
+      = ChainableStack::instance_->memalloc_.alloc_array<vari*>(y.size() * N);
+
+  for (size_t i = 0; i < y.size(); ++i) {
+    for (size_t j = 0; j < N; j++) {
+      non_chaining_varis[i * N + j] = new vari(y[i][j], false);
+    }
+  }
+
+  for (size_t i = 0; i < y.size(); ++i) {
+    y_return[i].resize(N);
+    for (size_t j = 0; j < N; j++)
+      y_return[i][j] = var(non_chaining_varis[i * N + j]);
+  }
+
+  return y_return;
+}
+
+/*
+ * If theta and y are both doubles, just pass the values through (there's
+ * no autodiff to handle here).
+ */
+template <>
+inline std::vector<Eigen::VectorXd> build_varis<double>(
+    vari* ode_vari, vari**& non_chaining_varis,
+    const std::vector<Eigen::VectorXd>& y) {
+  return y;
+}
+
+template <int Lmm, typename F, typename T_y0, typename T_t0, typename T_ts,
+          typename... T_Args>
+class cvodes_integrator_vari;
+
+template <int Lmm, typename F, typename T_y0, typename T_t0, typename T_ts,
+          typename... T_Args>
+class cvodes_integrator_memory : public chainable_alloc {
+  const size_t N_;
+  const F f_;
+  const Eigen::Matrix<T_y0, Eigen::Dynamic, 1> y0_;
+  const T_t0 t0_;
+  const std::vector<T_ts> ts_;
+  std::tuple<T_Args...> args_tuple_;
+  std::tuple<decltype(value_of(T_Args()))...> value_of_args_tuple_;
+  std::vector<Eigen::VectorXd> y_;
+  void* cvodes_mem_;
+  Eigen::VectorXd state;
+  N_Vector nv_state_;
+  SUNMatrix A_;
+  SUNLinearSolver LS_;
+
+  cvodes_integrator_memory(const F& f,
+                           const Eigen::Matrix<T_y0, Eigen::Dynamic, 1>& y0,
+                           const T_t0& t0, const std::vector<T_ts>& ts,
+                           const T_Args&... args)
+      : N_(y0.size()),
+        f_(f),
+        y0_(y0),
+        t0_(t0),
+        ts_(ts),
+        args_tuple_(std::make_tuple(args...)),
+        value_of_args_tuple_(std::make_tuple(value_of(args)...)),
+        y_(ts_.size()),
+        cvodes_mem_(nullptr),
+        state(value_of(y0)) {
+    if (N_ > 0) {
+      nv_state_ = N_VMake_Serial(N_, state.data());
+      A_ = SUNDenseMatrix(N_, N_);
+      LS_ = SUNDenseLinearSolver(nv_state_, A_);
+
+      cvodes_mem_ = CVodeCreate(Lmm);
+      if (cvodes_mem_ == nullptr) {
+        throw std::runtime_error("CVodeCreate failed to allocate memory");
+      }
+    }
+  }
+
+  ~cvodes_integrator_memory() {
+    if (N_ > 0) {
+      SUNLinSolFree(LS_);
+      SUNMatDestroy(A_);
+
+      N_VDestroy_Serial(nv_state_);
+
+      if (cvodes_mem_) {
+        CVodeFree(&cvodes_mem_);
+      }
+    }
+  }
+
+  friend class cvodes_integrator_vari<Lmm, F, T_y0, T_t0, T_ts, T_Args...>;
+};
+
 /**
  * Integrator interface for CVODES' ODE solvers (Adams & BDF
  * methods).
@@ -29,54 +139,62 @@ namespace math {
  */
 template <int Lmm, typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args>
-class cvodes_integrator {
+class cvodes_integrator_vari : public vari {
   using T_Return = return_type_t<T_y0, T_t0, T_ts, T_Args...>;
-  using T_y0_t0 = return_type_t<T_y0, T_t0>;
 
-  const F& f_;
-  const Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1> y0_;
-  const T_t0 t0_;
-  const std::vector<T_ts>& ts_;
-  std::tuple<const T_Args&...> args_tuple_;
-  std::tuple<decltype(value_of(T_Args()))...> value_of_args_tuple_;
   const size_t N_;
+  bool returned_;
   std::ostream* msgs_;
   double relative_tolerance_;
   double absolute_tolerance_;
   long int max_num_steps_;
 
+  const size_t t0_vars_;
+  const size_t ts_vars_;
   const size_t y0_vars_;
   const size_t args_vars_;
 
-  coupled_ode_system<F, T_y0_t0, T_Args...> coupled_ode_;
+  vari** non_chaining_varis_;
 
-  Eigen::VectorXd coupled_state_;
-  N_Vector nv_state_;
-  N_Vector* nv_state_sens_;
-  SUNMatrix A_;
-  SUNLinearSolver LS_;
+  vari** t0_varis_;
+  vari** ts_varis_;
+  vari** y0_varis_;
+  vari** args_varis_;
+
+  cvodes_integrator_memory<Lmm, F, T_y0, T_t0, T_ts, T_Args...>* memory;
 
   /**
    * Implements the function of type CVRhsFn which is the user-defined
    * ODE RHS passed to CVODES.
    */
   static int cv_rhs(realtype t, N_Vector y, N_Vector ydot, void* user_data) {
-    const cvodes_integrator* integrator
-        = static_cast<const cvodes_integrator*>(user_data);
+    const cvodes_integrator_vari* integrator
+        = static_cast<const cvodes_integrator_vari*>(user_data);
     integrator->rhs(t, NV_DATA_S(y), NV_DATA_S(ydot));
     return 0;
   }
 
   /**
-   * Implements the function of type CVSensRhsFn which is the
-   * RHS of the sensitivity ODE system.
+   * Implements the function of type CVRhsFnB which is the
+   * RHS of the backward ODE system.
+   */
+  static int cv_rhs_adj_sens(realtype t, N_Vector y, N_Vector yB,
+                             N_Vector yBdot, void* user_data) {
+    const cvodes_integrator_vari* integrator
+        = static_cast<const cvodes_integrator_vari*>(user_data);
+    integrator->rhs_adj_sens(t, y, yB, yBdot);
+    return 0;
+  }
+
+  /**
+   * Implements the function of type CVQuadRhsFnB which is the
+   * RHS of the backward ODE system's quadrature.
    */
-  static int cv_rhs_sens(int Ns, realtype t, N_Vector y, N_Vector ydot,
-                         N_Vector* yS, N_Vector* ySdot, void* user_data,
-                         N_Vector tmp1, N_Vector tmp2) {
-    const cvodes_integrator* integrator
-        = static_cast<const cvodes_integrator*>(user_data);
-    integrator->rhs_sens(t, NV_DATA_S(y), yS, ySdot);
+  static int cv_quad_rhs_adj(realtype t, N_Vector y, N_Vector yB,
+                             N_Vector qBdot, void* user_data) {
+    const cvodes_integrator_vari* integrator
+        = static_cast<const cvodes_integrator_vari*>(user_data);
+    integrator->quad_rhs_adj(t, y, yB, qBdot);
     return 0;
   }
 
@@ -89,9 +207,22 @@ class cvodes_integrator {
   static int cv_jacobian_states(realtype t, N_Vector y, N_Vector fy,
                                 SUNMatrix J, void* user_data, N_Vector tmp1,
                                 N_Vector tmp2, N_Vector tmp3) {
-    const cvodes_integrator* integrator
-        = static_cast<const cvodes_integrator*>(user_data);
-    integrator->jacobian_states(t, NV_DATA_S(y), J);
+    const cvodes_integrator_vari* integrator
+        = static_cast<const cvodes_integrator_vari*>(user_data);
+    integrator->jacobian_states(t, y, J);
+    return 0;
+  }
+
+  /**
+   * Implements the CVLsJacFnB function for evaluating the jacobian of
+   * the adjoint problem.
+   */
+  static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
+                             SUNMatrix J, void* user_data, N_Vector tmp1,
+                             N_Vector tmp2, N_Vector tmp3) {
+    const cvodes_integrator_vari* integrator
+        = static_cast<const cvodes_integrator_vari*>(user_data);
+    integrator->jacobian_adj(t, y, J);
     return 0;
   }
 
@@ -102,9 +233,9 @@ class cvodes_integrator {
   inline void rhs(double t, const double y[], double dy_dt[]) const {
     const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
 
-    Eigen::VectorXd dy_dt_vec
-        = apply([&](auto&&... args) { return f_(t, y_vec, msgs_, args...); },
-                value_of_args_tuple_);
+    Eigen::VectorXd dy_dt_vec = apply(
+        [&](auto&&... args) { return memory->f_(t, y_vec, msgs_, args...); },
+        memory->value_of_args_tuple_);
 
     check_size_match("cvodes_integrator::rhs", "dy_dt", dy_dt_vec.size(),
                      "states", N_);
@@ -112,20 +243,105 @@ class cvodes_integrator {
     std::copy(dy_dt_vec.data(), dy_dt_vec.data() + dy_dt_vec.size(), dy_dt);
   }
 
+  /*
+   * Calculate the adjoint sensitivity RHS for varying initial conditions
+   * and parameters
+   *
+   * @param[in] initial var vector
+   * @param[in] param var vector
+   * @param[in] t time
+   * @param[in] y state of the base ODE system
+   * @param[in] yB state of the adjoint ODE system
+   * @param[out] yBdot evaluation of adjoint ODE RHS
+   */
+  void rhs_adj_sens(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
+    Eigen::Map<Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
+    Eigen::Map<Eigen::VectorXd> lambda(NV_DATA_S(yB), N_);
+    Eigen::Map<Eigen::VectorXd> lambda_dot(NV_DATA_S(yBdot), N_);
+    lambda_dot = Eigen::VectorXd::Zero(N_);
+
+    const nested_rev_autodiff nested;
+
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y_vars(y_vec.size());
+    for (size_t i = 0; i < y_vars.size(); ++i)
+      y_vars(i) = new vari(y_vec(i));
+
+    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars = apply(
+        [&](auto&&... args) { return memory->f_(t, y_vars, msgs_, args...); },
+        memory->value_of_args_tuple_);
+
+    check_size_match("coupled_ode_system1", "dy_dt", f_y_t_vars.size(),
+                     "states", N_);
+
+    for (size_t i = 0; i < f_y_t_vars.size(); ++i) {
+      f_y_t_vars(i).vi_->adj_ = -lambda(i);
+    }
+
+    grad();
+
+    for (size_t i = 0; i < y_vars.size(); ++i) {
+      lambda_dot(i) = y_vars(i).adj();
+    }
+  }
+
+  /*
+   * Calculate the RHS for the quadrature part of the adjoint ODE problem.
+   *
+   * @param[in] initial var vector
+   * @param[in] param var vector
+   * @param[in] t time
+   * @param[in] y state of the base ODE system
+   * @param[in] yB state of the adjoint ODE system
+   * @param[out] qBdot evaluation of adjoint ODE quadrature RHS
+   */
+  void quad_rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector qBdot) const {
+    Eigen::VectorXd y_vec = Eigen::Map<Eigen::VectorXd>(NV_DATA_S(y), N_);
+    Eigen::Map<Eigen::VectorXd> lambda(NV_DATA_S(yB), N_);
+    Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(qBdot), args_vars_);
+    mu_dot = Eigen::VectorXd::Zero(args_vars_);
+
+    nested_rev_autodiff nested;
+
+    auto local_args_tuple = apply(
+        [&](auto&&... args) {
+          return std::tuple<decltype(deep_copy_vars(args))...>(
+              deep_copy_vars(args)...);
+        },
+        memory->args_tuple_);
+
+    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars = apply(
+        [&](auto&&... args) { return memory->f_(t, y_vec, msgs_, args...); },
+        local_args_tuple);
+
+    check_size_match("coupled_ode_system2", "dy_dt", f_y_t_vars.size(),
+                     "states", N_);
+
+    for (size_t i = 0; i < f_y_t_vars.size(); ++i) {
+      f_y_t_vars(i).vi_->adj_ = -lambda(i);
+    }
+
+    grad();
+
+    apply([&](auto&&... args) { accumulate_adjoints(mu_dot.data(), args...); },
+          local_args_tuple);
+  }
+
   /**
    * Calculates the jacobian of the ODE RHS wrt to its states y at the
    * given time-point t and state y.
    */
-  inline void jacobian_states(double t, const double y[], SUNMatrix J) const {
+  inline void jacobian_states(double t, N_Vector y, SUNMatrix J) const {
     Eigen::VectorXd fy;
     Eigen::MatrixXd Jfy;
 
     auto f_wrapped = [&](const Eigen::Matrix<var, Eigen::Dynamic, 1>& y) {
-      return apply([&](auto&&... args) { return f_(t, y, msgs_, args...); },
-                   value_of_args_tuple_);
+      return apply(
+          [&](auto&&... args) { return memory->f_(t, y, msgs_, args...); },
+          memory->value_of_args_tuple_);
     };
 
-    jacobian(f_wrapped, Eigen::Map<const Eigen::VectorXd>(y, N_), fy, Jfy);
+    jacobian(f_wrapped, Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_), fy,
+             Jfy);
 
     for (size_t j = 0; j < Jfy.cols(); ++j) {
       for (size_t i = 0; i < Jfy.rows(); ++i) {
@@ -134,25 +350,31 @@ class cvodes_integrator {
     }
   }
 
-  /**
-   * Calculates the RHS of the sensitivity ODE system which
-   * corresponds to the coupled ode system from which the first N
-   * states are omitted, since the first N states are the ODE RHS
-   * which CVODES separates from the main ODE RHS.
+  /*
+   * Calculate the Jacobian of the RHS of the adjoint ODE (see rhs_adj_sens
+   * below for citation for how this is done)
+   *
+   * @param[in] y State of system
+   * @param[in] t Time
+   * @param[out] J CVode structure where output is to be stored
    */
-  inline void rhs_sens(double t, const double y[], N_Vector* yS,
-                       N_Vector* ySdot) const {
-    Eigen::VectorXd z(coupled_state_.size());
-    Eigen::VectorXd dz_dt(coupled_state_.size());
-    std::copy(y, y + N_, z.data());
-    for (std::size_t s = 0; s < y0_vars_ + args_vars_; s++) {
-      std::copy(NV_DATA_S(yS[s]), NV_DATA_S(yS[s]) + N_,
-                z.data() + (s + 1) * N_);
-    }
-    coupled_ode_(z, dz_dt, t);
-    for (std::size_t s = 0; s < y0_vars_ + args_vars_; s++) {
-      std::move(dz_dt.data() + (s + 1) * N_, dz_dt.data() + (s + 2) * N_,
-                NV_DATA_S(ySdot[s]));
+  inline void jacobian_adj(double t, N_Vector y, SUNMatrix J) const {
+    Eigen::VectorXd fy;
+    Eigen::MatrixXd Jfy;
+
+    auto f_wrapped = [&](const Eigen::Matrix<var, Eigen::Dynamic, 1>& y) {
+      return apply(
+          [&](auto&&... args) { return memory->f_(t, y, msgs_, args...); },
+          memory->value_of_args_tuple_);
+    };
+
+    jacobian(f_wrapped, Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_), fy,
+             Jfy);
+
+    for (size_t j = 0; j < Jfy.cols(); ++j) {
+      for (size_t i = 0; i < Jfy.rows(); ++i) {
+        SM_ELEMENT_D(J, j, i) = -Jfy(i, j);
+      }
     }
   }
 
@@ -176,32 +398,45 @@ class cvodes_integrator {
    * @return a vector of states, each state being a vector of the
    * same size as the state variable, corresponding to a time in ts.
    */
-  cvodes_integrator(const F& f,
-                    const Eigen::Matrix<T_y0, Eigen::Dynamic, 1>& y0,
-                    const T_t0& t0, const std::vector<T_ts>& ts,
-                    double relative_tolerance, double absolute_tolerance,
-                    long int max_num_steps, std::ostream* msgs,
-                    const T_Args&... args)
-      : f_(f),
-        y0_(y0.unaryExpr([](const T_y0& val) { return T_y0_t0(val); })),
-        t0_(t0),
-        ts_(ts),
-        args_tuple_(args...),
-        value_of_args_tuple_(value_of(args)...),
+  cvodes_integrator_vari(const F& f,
+                         const Eigen::Matrix<T_y0, Eigen::Dynamic, 1>& y0,
+                         const T_t0& t0, const std::vector<T_ts>& ts,
+                         double relative_tolerance, double absolute_tolerance,
+                         long int max_num_steps, std::ostream* msgs,
+                         const T_Args&... args)
+      : vari(NOT_A_NUMBER),
         N_(y0.size()),
+        returned_(false),
+        memory(NULL),
         msgs_(msgs),
         relative_tolerance_(relative_tolerance),
         absolute_tolerance_(absolute_tolerance),
         max_num_steps_(max_num_steps),
-        y0_vars_(count_vars(y0_)),
+        t0_vars_(count_vars(t0)),
+        ts_vars_(count_vars(ts)),
+        y0_vars_(count_vars(y0)),
         args_vars_(count_vars(args...)),
-        coupled_ode_(f, y0_, msgs, args...),
-        coupled_state_(coupled_ode_.initial_state()) {
+        t0_varis_(
+            ChainableStack::instance_->memalloc_.alloc_array<vari*>(t0_vars_)),
+        ts_varis_(
+            ChainableStack::instance_->memalloc_.alloc_array<vari*>(ts_vars_)),
+        y0_varis_(
+            ChainableStack::instance_->memalloc_.alloc_array<vari*>(y0_vars_)),
+        args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
+            args_vars_)) {
     const char* fun = "cvodes_integrator::integrate";
 
-    check_finite(fun, "initial state", y0_);
-    check_finite(fun, "initial time", t0_);
-    check_finite(fun, "times", ts_);
+    memory = new cvodes_integrator_memory<Lmm, F, T_y0, T_t0, T_ts, T_Args...>(
+        f, y0, t0, ts, args...);
+
+    save_varis(t0_varis_, t0);
+    save_varis(ts_varis_, ts);
+    save_varis(y0_varis_, y0);
+    save_varis(args_varis_, args...);
+
+    check_finite(fun, "initial state", y0);
+    check_finite(fun, "initial time", t0);
+    check_finite(fun, "times", ts);
 
     // Code from: https://stackoverflow.com/a/17340003 . Should probably do
     // something better
@@ -210,38 +445,18 @@ class cvodes_integrator {
           std::vector<int> unused_temp{
               0, (check_finite(fun, "ode parameters and data", args), 0)...};
         },
-        args_tuple_);
+        memory->args_tuple_);
 
-    check_nonzero_size(fun, "times", ts_);
-    check_nonzero_size(fun, "initial state", y0_);
-    check_ordered(fun, "times", ts_);
-    check_less(fun, "initial time", t0_, ts_[0]);
+    check_nonzero_size(fun, "times", ts);
+    check_nonzero_size(fun, "initial state", y0);
+    check_ordered(fun, "times", ts);
+    check_less(fun, "initial time", t0, ts[0]);
     check_positive_finite(fun, "relative_tolerance", relative_tolerance_);
     check_positive_finite(fun, "absolute_tolerance", absolute_tolerance_);
     check_positive(fun, "max_num_steps", max_num_steps_);
-
-    nv_state_ = N_VMake_Serial(N_, &coupled_state_[0]);
-    nv_state_sens_ = nullptr;
-    A_ = SUNDenseMatrix(N_, N_);
-    LS_ = SUNDenseLinearSolver(nv_state_, A_);
-
-    if (y0_vars_ + args_vars_ > 0) {
-      nv_state_sens_
-          = N_VCloneVectorArrayEmpty_Serial(y0_vars_ + args_vars_, nv_state_);
-      for (std::size_t i = 0; i < y0_vars_ + args_vars_; i++) {
-        NV_DATA_S(nv_state_sens_[i]) = &coupled_state_[N_] + i * N_;
-      }
-    }
   }
 
-  ~cvodes_integrator() {
-    SUNLinSolFree(LS_);
-    SUNMatDestroy(A_);
-    N_VDestroy_Serial(nv_state_);
-    if (y0_vars_ + args_vars_ > 0) {
-      N_VDestroyVectorArray_Serial(nv_state_sens_, y0_vars_ + args_vars_);
-    }
-  }
+  ~cvodes_integrator_vari() {}
 
   /**
    * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
@@ -251,85 +466,238 @@ class cvodes_integrator {
    * @return std::vector of Eigen::Matrix of the states of the ODE, one for each
    *   solution time (excluding the initial state)
    */
-  std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>>
-  operator()() {  // NOLINT(runtime/int)
-    std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y;
+  std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> operator()() {
+    const double t0_dbl = value_of(memory->t0_);
+    const std::vector<double> ts_dbl = value_of(memory->ts_);
+
+    check_flag_sundials(
+        CVodeInit(memory->cvodes_mem_, &cvodes_integrator_vari::cv_rhs, t0_dbl,
+                  memory->nv_state_),
+        "CVodeInit");
+
+    // Assign pointer to this as user data
+    check_flag_sundials(
+        CVodeSetUserData(memory->cvodes_mem_, reinterpret_cast<void*>(this)),
+        "CVodeSetUserData");
+
+    cvodes_set_options(memory->cvodes_mem_, relative_tolerance_,
+                       absolute_tolerance_, max_num_steps_);
+
+    // for the stiff solvers we need to reserve additional memory
+    // and provide a Jacobian function call. new API since 3.0.0:
+    // create matrix object and linear solver object; resource
+    // (de-)allocation is handled in the cvodes_ode_data
+    check_flag_sundials(
+        CVodeSetLinearSolver(memory->cvodes_mem_, memory->LS_, memory->A_),
+        "CVodeSetLinearSolver");
+
+    check_flag_sundials(
+        CVodeSetJacFn(memory->cvodes_mem_,
+                      &cvodes_integrator_vari::cv_jacobian_states),
+        "CVodeSetJacFn");
+
+    // initialize forward sensitivity system of CVODES as needed
+    if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
+      check_flag_sundials(CVodeAdjInit(memory->cvodes_mem_, 25, CV_HERMITE),
+                          "CVodeAdjInit");
+    }
 
-    const double t0_dbl = value_of(t0_);
-    const std::vector<double> ts_dbl = value_of(ts_);
+    double t_init = t0_dbl;
+    for (size_t n = 0; n < ts_dbl.size(); ++n) {
+      double t_final = ts_dbl[n];
 
-    void* cvodes_mem = CVodeCreate(Lmm);
-    if (cvodes_mem == nullptr) {
-      throw std::runtime_error("CVodeCreate failed to allocate memory");
+      if (t_final != t_init) {
+        if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
+          int ncheck;
+          check_flag_sundials(
+              CVodeF(memory->cvodes_mem_, t_final, memory->nv_state_, &t_init,
+                     CV_NORMAL, &ncheck),
+              "CVodeF");
+        } else {
+          check_flag_sundials(CVode(memory->cvodes_mem_, t_final,
+                                    memory->nv_state_, &t_init, CV_NORMAL),
+                              "CVode");
+        }
+      }
+
+      memory->y_[n] = memory->state;
+
+      t_init = t_final;
+    }
+
+    returned_ = true;
+    return build_varis<T_Return>(this, non_chaining_varis_, memory->y_);
+  }
+
+  virtual void chain() {
+    // std::cout << "chain" << std::endl; <-- Good way to verify it's only
+    //  being called once
+    if (memory == NULL)
+      return;
+
+    if (memory->cvodes_mem_ == NULL)
+      return;
+
+    if (returned_ == false)
+      return;
+
+    if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ == 0) {
+      return;
     }
 
+    Eigen::VectorXd state_sens(N_);
+    Eigen::VectorXd quad(args_vars_);
+    N_Vector nv_state_sens
+        = N_VMake_Serial(state_sens.size(), state_sens.data());
+    N_Vector nv_quad = N_VMake_Serial(quad.size(), quad.data());
+    N_VConst(0.0, nv_state_sens);
+    N_VConst(0.0, nv_quad);
+
+    SUNMatrix AB_ = SUNDenseMatrix(N_, N_);
+    SUNLinearSolver LSB_ = SUNDenseLinearSolver(nv_state_sens, AB_);
+
     try {
+      int indexB;
+
+      // This is all boilerplate CVODES setting up the adjoint ODE to solve
+      check_flag_sundials(CVodeCreateB(memory->cvodes_mem_, Lmm, &indexB),
+                          "CVodeCreateB");
+
+      check_flag_sundials(CVodeSetUserDataB(memory->cvodes_mem_, indexB,
+                                            reinterpret_cast<void*>(this)),
+                          "CVodeSetUserDataB");
+
+      // The ode_rhs_adj_sense functions passed in here cause problems with
+      // the autodiff stack (they can cause reallocations of the internal
+      // vectors and cause segfaults)
+      check_flag_sundials(
+          CVodeInitB(memory->cvodes_mem_, indexB,
+                     &cvodes_integrator_vari::cv_rhs_adj_sens,
+                     value_of(memory->ts_.back()), nv_state_sens),
+          "CVodeInitB");
+
       check_flag_sundials(
-          CVodeInit(cvodes_mem, &cvodes_integrator::cv_rhs, t0_dbl, nv_state_),
-          "CVodeInit");
+          CVodeSStolerancesB(memory->cvodes_mem_, indexB, relative_tolerance_,
+                             absolute_tolerance_),
+          "CVodeSStolerancesB");
 
-      // Assign pointer to this as user data
       check_flag_sundials(
-          CVodeSetUserData(cvodes_mem, reinterpret_cast<void*>(this)),
-          "CVodeSetUserData");
-
-      cvodes_set_options(cvodes_mem, relative_tolerance_, absolute_tolerance_,
-                         max_num_steps_);
-
-      // for the stiff solvers we need to reserve additional memory
-      // and provide a Jacobian function call. new API since 3.0.0:
-      // create matrix object and linear solver object; resource
-      // (de-)allocation is handled in the cvodes_ode_data
-      check_flag_sundials(CVodeSetLinearSolver(cvodes_mem, LS_, A_),
-                          "CVodeSetLinearSolver");
+          CVodeSetLinearSolverB(memory->cvodes_mem_, indexB, LSB_, AB_),
+          "CVodeSetLinearSolverB");
+
+      // The same autodiff issue that applies to ode_rhs_adj_sense applies
+      // here
       check_flag_sundials(
-          CVodeSetJacFn(cvodes_mem, &cvodes_integrator::cv_jacobian_states),
-          "CVodeSetJacFn");
+          CVodeSetJacFnB(memory->cvodes_mem_, indexB,
+                         &cvodes_integrator_vari::cv_jacobian_adj),
+          "CVodeSetJacFnB");
 
-      // initialize forward sensitivity system of CVODES as needed
-      if (y0_vars_ + args_vars_ > 0) {
+      // Allocate space for backwards quadrature
+      if (args_vars_ > 0) {
         check_flag_sundials(
-            CVodeSensInit(cvodes_mem, static_cast<int>(y0_vars_ + args_vars_),
-                          CV_STAGGERED, &cvodes_integrator::cv_rhs_sens,
-                          nv_state_sens_),
-            "CVodeSensInit");
+            CVodeQuadInitB(memory->cvodes_mem_, indexB,
+                           &cvodes_integrator_vari::cv_quad_rhs_adj, nv_quad),
+            "CVodeQuadInitB");
 
-        check_flag_sundials(CVodeSensEEtolerances(cvodes_mem),
-                            "CVodeSensEEtolerances");
+        check_flag_sundials(
+            CVodeQuadSStolerancesB(memory->cvodes_mem_, indexB,
+                                   relative_tolerance_, absolute_tolerance_),
+            "CVodeQuadSStolerancesB");
+
+        check_flag_sundials(
+            CVodeSetQuadErrConB(memory->cvodes_mem_, indexB, SUNTRUE),
+            "CVodeSetQuadErrConB");
       }
 
-      double t_init = t0_dbl;
-      for (size_t n = 0; n < ts_.size(); ++n) {
-        double t_final = ts_dbl[n];
+      // At every time step, collect the adjoints from the output
+      // variables and re-initialize the solver
+      double t_init = value_of(memory->ts_.back());
+      for (int i = memory->ts_.size() - 1; i >= 0; --i) {
+        // Take in the adjoints from all the output variables at this point
+        // in time
+        Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
+        for (int j = 0; j < N_; j++) {
+          // std::cout << "i: " << i << ", j: " << j << std::endl;
+          state_sens(j) += non_chaining_varis_[i * N_ + j]->adj_;
+          step_sens(j) += non_chaining_varis_[i * N_ + j]->adj_;
+        }
 
+        if (ts_vars_ > 0 && i >= 0) {
+          ts_varis_[i]->adj_ += apply(
+              [&](auto&&... args) {
+                double adj = step_sens.dot(
+                    memory->f_(t_init, memory->y_[i], msgs_, args...));
+                // std::cout << "adj: " << adj << ", i: " << i << std::endl;
+                return adj;
+              },
+              memory->value_of_args_tuple_);
+        }
+
+        double t_final = value_of((i > 0) ? memory->ts_[i - 1] : memory->t0_);
         if (t_final != t_init) {
           check_flag_sundials(
-              CVode(cvodes_mem, t_final, nv_state_, &t_init, CV_NORMAL),
-              "CVode");
-        }
+              CVodeReInitB(memory->cvodes_mem_, indexB, t_init, nv_state_sens),
+              "CVodeReInitB");
+
+          if (args_vars_ > 0) {
+            check_flag_sundials(
+                CVodeQuadReInitB(memory->cvodes_mem_, indexB, nv_quad),
+                "CVodeQuadReInitB");
+          }
+
+          check_flag_sundials(CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL),
+                              "CVodeB");
 
-        if (y0_vars_ + args_vars_ > 0) {
-          check_flag_sundials(CVodeGetSens(cvodes_mem, &t_init, nv_state_sens_),
-                              "CVodeGetSens");
+          check_flag_sundials(
+              CVodeGetB(memory->cvodes_mem_, indexB, &t_init, nv_state_sens),
+              "CVodeGetB");
+
+          if (args_vars_ > 0) {
+            check_flag_sundials(
+                CVodeGetQuadB(memory->cvodes_mem_, indexB, &t_init, nv_quad),
+                "CVodeGetQuadB");
+          }
         }
+      }
 
-        y.emplace_back(apply(
+      if (t0_vars_ > 0) {
+        Eigen::VectorXd y0d = value_of(memory->y0_);
+        t0_varis_[0]->adj_ += apply(
             [&](auto&&... args) {
-              return ode_store_sensitivities(f_, coupled_state_, y0_, t0_,
-                                             ts_[n], msgs_, args...);
+              return -state_sens.dot(memory->f_(t_init, y0d, msgs_, args...));
             },
-            args_tuple_));
+            memory->value_of_args_tuple_);
+      }
 
-        t_init = t_final;
+      if (args_vars_ > 0) {
+        check_flag_sundials(
+            CVodeGetQuadB(memory->cvodes_mem_, indexB, &t_init, nv_quad),
+            "CVodeGetQuadB");
+      }
+
+      // After integrating all the way back to t0, we finally have the
+      // the adjoints we wanted
+      // These are the dlog_density / d(initial_conditions[s]) adjoints
+      for (size_t s = 0; s < y0_vars_; s++) {
+        y0_varis_[s]->adj_ += state_sens(s);
+      }
+
+      // These are the dlog_density / d(parameters[s]) adjoints
+      for (size_t s = 0; s < args_vars_; s++) {
+        args_varis_[s]->adj_ += quad(s);
       }
     } catch (const std::exception& e) {
-      CVodeFree(&cvodes_mem);
+      SUNLinSolFree(LSB_);
+      SUNMatDestroy(AB_);
+      N_VDestroy_Serial(nv_state_sens);
+      N_VDestroy_Serial(nv_quad);
       throw;
     }
 
-    CVodeFree(&cvodes_mem);
-
-    return y;
+    SUNLinSolFree(LSB_);
+    SUNMatDestroy(AB_);
+    N_VDestroy_Serial(nv_state_sens);
+    N_VDestroy_Serial(nv_quad);
   }
 };  // cvodes integrator
 
diff --git a/stan/math/rev/functor/ode_bdf.hpp b/stan/math/rev/functor/ode_bdf.hpp
index 0c08236cc92..e6336429d62 100644
--- a/stan/math/rev/functor/ode_bdf.hpp
+++ b/stan/math/rev/functor/ode_bdf.hpp
@@ -99,13 +99,13 @@ ode_bdf_tol(const F& f, const Eigen::Matrix<T_initial, Eigen::Dynamic, 1>& y0,
             const T_t0& t0, const std::vector<T_ts>& ts,
             double relative_tolerance, double absolute_tolerance,
             long int max_num_steps, std::ostream* msgs, const T_Args&... args) {
-  cvodes_integrator<CV_BDF, F, T_initial, T_t0, T_ts,
-		    T_Args...> integrator(f, y0, t0, ts,
-					  relative_tolerance, absolute_tolerance,
-					  max_num_steps,
-					  msgs, args...);
+  auto integrator
+      = new stan::math::cvodes_integrator_vari<CV_BDF, F, T_initial, T_t0, T_ts,
+                                               T_Args...>(
+          f, y0, t0, ts, relative_tolerance, absolute_tolerance, max_num_steps,
+          msgs, args...);
 
-  return integrator();
+  return (*integrator)();
 }
 
 }  // namespace math

From 27fbb73786abc059901c1cc6264e30185607953d Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sat, 2 Jan 2021 12:53:09 +0100
Subject: [PATCH 002/157] add adjoint integrator first version

---
 stan/math/rev/functor.hpp                     |   1 +
 .../rev/functor/cvodes_integrator_adjoint.hpp | 708 ++++++++++++++++++
 stan/math/rev/functor/ode_bdf_adjoint.hpp     | 113 +++
 .../rev/functor/ode_bdf_adjoint_rev_test.cpp  | 522 +++++++++++++
 4 files changed, 1344 insertions(+)
 create mode 100644 stan/math/rev/functor/cvodes_integrator_adjoint.hpp
 create mode 100644 stan/math/rev/functor/ode_bdf_adjoint.hpp
 create mode 100644 test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp

diff --git a/stan/math/rev/functor.hpp b/stan/math/rev/functor.hpp
index 0be37ccb17b..5229ef674ff 100644
--- a/stan/math/rev/functor.hpp
+++ b/stan/math/rev/functor.hpp
@@ -18,6 +18,7 @@
 #include <stan/math/rev/functor/integrate_ode_bdf.hpp>
 #include <stan/math/rev/functor/ode_adams.hpp>
 #include <stan/math/rev/functor/ode_bdf.hpp>
+#include <stan/math/rev/functor/ode_bdf_adjoint.hpp>
 #include <stan/math/rev/functor/ode_store_sensitivities.hpp>
 #include <stan/math/rev/functor/jacobian.hpp>
 #include <stan/math/rev/functor/kinsol_data.hpp>
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
new file mode 100644
index 00000000000..2cbed8af12c
--- /dev/null
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -0,0 +1,708 @@
+#ifndef STAN_MATH_REV_FUNCTOR_INTEGRATE_ODE_CVODES_ADJOINT_HPP
+#define STAN_MATH_REV_FUNCTOR_INTEGRATE_ODE_CVODES_ADJOINT_HPP
+
+#include <stan/math/rev/meta.hpp>
+#include <stan/math/rev/core/save_varis.hpp>
+#include <stan/math/rev/functor/cvodes_utils.hpp>
+#include <stan/math/rev/functor/coupled_ode_system.hpp>
+#include <stan/math/rev/functor/ode_store_sensitivities.hpp>
+#include <stan/math/prim/err.hpp>
+#include <stan/math/prim/fun/value_of.hpp>
+#include <cvodes/cvodes.h>
+#include <sunlinsol/sunlinsol_dense.h>
+#include <algorithm>
+#include <ostream>
+#include <utility>
+#include <vector>
+
+namespace stan {
+namespace math {
+
+template <typename T_Return>
+inline std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> build_varis(
+    vari* ode_vari, vari**& non_chaining_varis,
+    const std::vector<Eigen::VectorXd>& y);
+
+template <>
+inline std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> build_varis<var>(
+    vari* ode_vari, vari**& non_chaining_varis,
+    const std::vector<Eigen::VectorXd>& y) {
+  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y_return(y.size());
+
+  if (y.size() == 0) {
+    return y_return;
+  }
+
+  int N = y[0].size();
+
+  non_chaining_varis
+      = ChainableStack::instance_->memalloc_.alloc_array<vari*>(y.size() * N);
+
+  for (size_t i = 0; i < y.size(); ++i) {
+    for (size_t j = 0; j < N; j++) {
+      non_chaining_varis[i * N + j] = new vari(y[i][j], false);
+    }
+  }
+
+  for (size_t i = 0; i < y.size(); ++i) {
+    y_return[i].resize(N);
+    for (size_t j = 0; j < N; j++)
+      y_return[i][j] = var(non_chaining_varis[i * N + j]);
+  }
+
+  return y_return;
+}
+
+/*
+ * If theta and y are both doubles, just pass the values through (there's
+ * no autodiff to handle here).
+ */
+template <>
+inline std::vector<Eigen::VectorXd> build_varis<double>(
+    vari* ode_vari, vari**& non_chaining_varis,
+    const std::vector<Eigen::VectorXd>& y) {
+  return y;
+}
+
+template <int Lmm, typename F, typename T_y0, typename T_t0, typename T_ts,
+          typename... T_Args>
+class cvodes_integrator_adjoint_vari;
+
+template <int Lmm, typename F, typename T_y0, typename T_t0, typename T_ts,
+          typename... T_Args>
+class cvodes_integrator_adjoint_memory : public chainable_alloc {
+  const size_t N_;
+  const F f_;
+  const Eigen::Matrix<T_y0, Eigen::Dynamic, 1> y0_;
+  const T_t0 t0_;
+  const std::vector<T_ts> ts_;
+  std::tuple<T_Args...> args_tuple_;
+  //std::tuple<decltype(value_of(T_Args()))...> value_of_args_tuple_;
+  std::tuple<decltype(value_of(std::declval<T_Args>()))...> value_of_args_tuple_;
+  std::vector<Eigen::VectorXd> y_;
+  void* cvodes_mem_;
+  Eigen::VectorXd state;
+  N_Vector nv_state_;
+  SUNMatrix A_;
+  SUNLinearSolver LS_;
+
+  cvodes_integrator_adjoint_memory(const F& f,
+                           const Eigen::Matrix<T_y0, Eigen::Dynamic, 1>& y0,
+                           const T_t0& t0, const std::vector<T_ts>& ts,
+                           const T_Args&... args)
+      : N_(y0.size()),
+        f_(f),
+        y0_(y0),
+        t0_(t0),
+        ts_(ts),
+        args_tuple_(std::make_tuple(args...)),
+        value_of_args_tuple_(std::make_tuple(value_of(args)...)),
+        y_(ts_.size()),
+        cvodes_mem_(nullptr),
+        state(value_of(y0)) {
+    if (N_ > 0) {
+      nv_state_ = N_VMake_Serial(N_, state.data());
+      A_ = SUNDenseMatrix(N_, N_);
+      LS_ = SUNDenseLinearSolver(nv_state_, A_);
+
+      cvodes_mem_ = CVodeCreate(Lmm);
+      if (cvodes_mem_ == nullptr) {
+        throw std::runtime_error("CVodeCreate failed to allocate memory");
+      }
+    }
+  }
+
+  ~cvodes_integrator_adjoint_memory() {
+    if (N_ > 0) {
+      SUNLinSolFree(LS_);
+      SUNMatDestroy(A_);
+
+      N_VDestroy_Serial(nv_state_);
+
+      if (cvodes_mem_) {
+        CVodeFree(&cvodes_mem_);
+      }
+    }
+  }
+
+  friend class cvodes_integrator_adjoint_vari<Lmm, F, T_y0, T_t0, T_ts, T_Args...>;
+};
+
+/**
+ * Integrator interface for CVODES' ODE solvers (Adams & BDF
+ * methods).
+ *
+ * @tparam Lmm ID of ODE solver (1: ADAMS, 2: BDF)
+ * @tparam F Type of ODE right hand side
+ * @tparam T_y0 Type of scalars for initial state
+ * @tparam T_param Type of scalars for parameters
+ * @tparam T_t0 Type of scalar of initial time point
+ * @tparam T_ts Type of time-points where ODE solution is returned
+ */
+template <int Lmm, typename F, typename T_y0, typename T_t0, typename T_ts,
+          typename... T_Args>
+class cvodes_integrator_adjoint_vari : public vari {
+  using T_Return = return_type_t<T_y0, T_t0, T_ts, T_Args...>;
+
+  const size_t N_;
+  bool returned_;
+  std::ostream* msgs_;
+  double relative_tolerance_;
+  double absolute_tolerance_;
+  long int max_num_steps_;
+
+  const size_t t0_vars_;
+  const size_t ts_vars_;
+  const size_t y0_vars_;
+  const size_t args_vars_;
+
+  vari** non_chaining_varis_;
+
+  vari** t0_varis_;
+  vari** ts_varis_;
+  vari** y0_varis_;
+  vari** args_varis_;
+
+  cvodes_integrator_adjoint_memory<Lmm, F, T_y0, T_t0, T_ts, T_Args...>* memory;
+
+  /**
+   * Implements the function of type CVRhsFn which is the user-defined
+   * ODE RHS passed to CVODES.
+   */
+  static int cv_rhs(realtype t, N_Vector y, N_Vector ydot, void* user_data) {
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+    integrator->rhs(t, NV_DATA_S(y), NV_DATA_S(ydot));
+    return 0;
+  }
+
+  /**
+   * Implements the function of type CVRhsFnB which is the
+   * RHS of the backward ODE system.
+   */
+  static int cv_rhs_adj_sens(realtype t, N_Vector y, N_Vector yB,
+                             N_Vector yBdot, void* user_data) {
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+    integrator->rhs_adj_sens(t, y, yB, yBdot);
+    return 0;
+  }
+
+  /**
+   * Implements the function of type CVQuadRhsFnB which is the
+   * RHS of the backward ODE system's quadrature.
+   */
+  static int cv_quad_rhs_adj(realtype t, N_Vector y, N_Vector yB,
+                             N_Vector qBdot, void* user_data) {
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+    integrator->quad_rhs_adj(t, y, yB, qBdot);
+    return 0;
+  }
+
+  /**
+   * Implements the function of type CVDlsJacFn which is the
+   * user-defined callback for CVODES to calculate the jacobian of the
+   * ode_rhs wrt to the states y. The jacobian is stored in column
+   * major format.
+   */
+  static int cv_jacobian_states(realtype t, N_Vector y, N_Vector fy,
+                                SUNMatrix J, void* user_data, N_Vector tmp1,
+                                N_Vector tmp2, N_Vector tmp3) {
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+    integrator->jacobian_states(t, y, J);
+    return 0;
+  }
+
+  /**
+   * Implements the CVLsJacFnB function for evaluating the jacobian of
+   * the adjoint problem.
+   */
+  static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
+                             SUNMatrix J, void* user_data, N_Vector tmp1,
+                             N_Vector tmp2, N_Vector tmp3) {
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+    integrator->jacobian_adj(t, y, J);
+    return 0;
+  }
+
+  /**
+   * Calculates the ODE RHS, dy_dt, using the user-supplied functor at
+   * the given time t and state y.
+   */
+  inline void rhs(double t, const double y[], double dy_dt[]) const {
+    const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
+
+    Eigen::VectorXd dy_dt_vec = apply(
+        [&](auto&&... args) { return memory->f_(t, y_vec, msgs_, args...); },
+        memory->value_of_args_tuple_);
+
+    check_size_match("cvodes_integrator::rhs", "dy_dt", dy_dt_vec.size(),
+                     "states", N_);
+
+    std::copy(dy_dt_vec.data(), dy_dt_vec.data() + dy_dt_vec.size(), dy_dt);
+  }
+
+  /*
+   * Calculate the adjoint sensitivity RHS for varying initial conditions
+   * and parameters
+   *
+   * @param[in] initial var vector
+   * @param[in] param var vector
+   * @param[in] t time
+   * @param[in] y state of the base ODE system
+   * @param[in] yB state of the adjoint ODE system
+   * @param[out] yBdot evaluation of adjoint ODE RHS
+   */
+  void rhs_adj_sens(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
+    Eigen::Map<Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
+    Eigen::Map<Eigen::VectorXd> lambda(NV_DATA_S(yB), N_);
+    Eigen::Map<Eigen::VectorXd> lambda_dot(NV_DATA_S(yBdot), N_);
+    lambda_dot = Eigen::VectorXd::Zero(N_);
+
+    const nested_rev_autodiff nested;
+
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y_vars(y_vec.size());
+    for (size_t i = 0; i < y_vars.size(); ++i)
+      y_vars(i) = new vari(y_vec(i));
+
+    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars = apply(
+        [&](auto&&... args) { return memory->f_(t, y_vars, msgs_, args...); },
+        memory->value_of_args_tuple_);
+
+    check_size_match("coupled_ode_system1", "dy_dt", f_y_t_vars.size(),
+                     "states", N_);
+
+    for (size_t i = 0; i < f_y_t_vars.size(); ++i) {
+      f_y_t_vars(i).vi_->adj_ = -lambda(i);
+    }
+
+    grad();
+
+    for (size_t i = 0; i < y_vars.size(); ++i) {
+      lambda_dot(i) = y_vars(i).adj();
+    }
+  }
+
+  /*
+   * Calculate the RHS for the quadrature part of the adjoint ODE problem.
+   *
+   * @param[in] initial var vector
+   * @param[in] param var vector
+   * @param[in] t time
+   * @param[in] y state of the base ODE system
+   * @param[in] yB state of the adjoint ODE system
+   * @param[out] qBdot evaluation of adjoint ODE quadrature RHS
+   */
+  void quad_rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector qBdot) const {
+    Eigen::VectorXd y_vec = Eigen::Map<Eigen::VectorXd>(NV_DATA_S(y), N_);
+    Eigen::Map<Eigen::VectorXd> lambda(NV_DATA_S(yB), N_);
+    Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(qBdot), args_vars_);
+    mu_dot = Eigen::VectorXd::Zero(args_vars_);
+
+    nested_rev_autodiff nested;
+
+    auto local_args_tuple = apply(
+        [&](auto&&... args) {
+          return std::tuple<decltype(deep_copy_vars(args))...>(
+              deep_copy_vars(args)...);
+        },
+        memory->args_tuple_);
+
+    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars = apply(
+        [&](auto&&... args) { return memory->f_(t, y_vec, msgs_, args...); },
+        local_args_tuple);
+
+    check_size_match("coupled_ode_system2", "dy_dt", f_y_t_vars.size(),
+                     "states", N_);
+
+    for (size_t i = 0; i < f_y_t_vars.size(); ++i) {
+      f_y_t_vars(i).vi_->adj_ = -lambda(i);
+    }
+
+    grad();
+
+    apply([&](auto&&... args) { accumulate_adjoints(mu_dot.data(), args...); },
+          local_args_tuple);
+  }
+
+  /**
+   * Calculates the jacobian of the ODE RHS wrt to its states y at the
+   * given time-point t and state y.
+   */
+  inline void jacobian_states(double t, N_Vector y, SUNMatrix J) const {
+    Eigen::VectorXd fy;
+    Eigen::MatrixXd Jfy;
+
+    auto f_wrapped = [&](const Eigen::Matrix<var, Eigen::Dynamic, 1>& y) {
+      return apply(
+          [&](auto&&... args) { return memory->f_(t, y, msgs_, args...); },
+          memory->value_of_args_tuple_);
+    };
+
+    jacobian(f_wrapped, Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_), fy,
+             Jfy);
+
+    for (size_t j = 0; j < Jfy.cols(); ++j) {
+      for (size_t i = 0; i < Jfy.rows(); ++i) {
+        SM_ELEMENT_D(J, i, j) = Jfy(i, j);
+      }
+    }
+  }
+
+  /*
+   * Calculate the Jacobian of the RHS of the adjoint ODE (see rhs_adj_sens
+   * below for citation for how this is done)
+   *
+   * @param[in] y State of system
+   * @param[in] t Time
+   * @param[out] J CVode structure where output is to be stored
+   */
+  inline void jacobian_adj(double t, N_Vector y, SUNMatrix J) const {
+    Eigen::VectorXd fy;
+    Eigen::MatrixXd Jfy;
+
+    auto f_wrapped = [&](const Eigen::Matrix<var, Eigen::Dynamic, 1>& y) {
+      return apply(
+          [&](auto&&... args) { return memory->f_(t, y, msgs_, args...); },
+          memory->value_of_args_tuple_);
+    };
+
+    jacobian(f_wrapped, Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_), fy,
+             Jfy);
+
+    for (size_t j = 0; j < Jfy.cols(); ++j) {
+      for (size_t i = 0; i < Jfy.rows(); ++i) {
+        SM_ELEMENT_D(J, j, i) = -Jfy(i, j);
+      }
+    }
+  }
+
+ public:
+  /**
+   * Construct cvodes_integrator object
+   *
+   * @param f Right hand side of the ODE
+   * @param y0 Initial state
+   * @param t0 Initial time
+   * @param ts Times at which to solve the ODE at. All values must be sorted and
+   *   not less than t0.
+   * @param relative_tolerance Relative tolerance passed to CVODES
+   * @param absolute_tolerance Absolute tolerance passed to CVODES
+   * @param max_num_steps Upper limit on the number of integration steps to
+   *   take between each output (error if exceeded)
+   * @param[in, out] msgs the print stream for warning messages
+   * @param args Extra arguments passed unmodified through to ODE right hand
+   * side function
+   * @return Solution to ODE at times \p ts
+   * @return a vector of states, each state being a vector of the
+   * same size as the state variable, corresponding to a time in ts.
+   */
+  cvodes_integrator_adjoint_vari(const F& f,
+                         const Eigen::Matrix<T_y0, Eigen::Dynamic, 1>& y0,
+                         const T_t0& t0, const std::vector<T_ts>& ts,
+                         double relative_tolerance, double absolute_tolerance,
+                         long int max_num_steps, std::ostream* msgs,
+                         const T_Args&... args)
+      : vari(NOT_A_NUMBER),
+        N_(y0.size()),
+        returned_(false),
+        memory(NULL),
+        msgs_(msgs),
+        relative_tolerance_(relative_tolerance),
+        absolute_tolerance_(absolute_tolerance),
+        max_num_steps_(max_num_steps),
+        t0_vars_(count_vars(t0)),
+        ts_vars_(count_vars(ts)),
+        y0_vars_(count_vars(y0)),
+        args_vars_(count_vars(args...)),
+        t0_varis_(
+            ChainableStack::instance_->memalloc_.alloc_array<vari*>(t0_vars_)),
+        ts_varis_(
+            ChainableStack::instance_->memalloc_.alloc_array<vari*>(ts_vars_)),
+        y0_varis_(
+            ChainableStack::instance_->memalloc_.alloc_array<vari*>(y0_vars_)),
+        args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
+            args_vars_)) {
+    const char* fun = "cvodes_integrator::integrate";
+
+    memory = new cvodes_integrator_adjoint_memory<Lmm, F, T_y0, T_t0, T_ts, T_Args...>(
+        f, y0, t0, ts, args...);
+
+    save_varis(t0_varis_, t0);
+    save_varis(ts_varis_, ts);
+    save_varis(y0_varis_, y0);
+    save_varis(args_varis_, args...);
+
+    check_finite(fun, "initial state", y0);
+    check_finite(fun, "initial time", t0);
+    check_finite(fun, "times", ts);
+
+    // Code from: https://stackoverflow.com/a/17340003 . Should probably do
+    // something better
+    apply(
+        [&](auto&&... args) {
+          std::vector<int> unused_temp{
+              0, (check_finite(fun, "ode parameters and data", args), 0)...};
+        },
+        memory->args_tuple_);
+
+    check_nonzero_size(fun, "times", ts);
+    check_nonzero_size(fun, "initial state", y0);
+    check_ordered(fun, "times", ts);
+    check_less(fun, "initial time", t0, ts[0]);
+    check_positive_finite(fun, "relative_tolerance", relative_tolerance_);
+    check_positive_finite(fun, "absolute_tolerance", absolute_tolerance_);
+    check_positive(fun, "max_num_steps", max_num_steps_);
+  }
+
+  ~cvodes_integrator_adjoint_vari() {}
+
+  /**
+   * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
+   * times, { t1, t2, t3, ... } using the stiff backward differentiation formula
+   * (BDF) solver in CVODES.
+   *
+   * @return std::vector of Eigen::Matrix of the states of the ODE, one for each
+   *   solution time (excluding the initial state)
+   */
+  std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> operator()() {
+    const double t0_dbl = value_of(memory->t0_);
+    const std::vector<double> ts_dbl = value_of(memory->ts_);
+
+    check_flag_sundials(
+        CVodeInit(memory->cvodes_mem_, &cvodes_integrator_adjoint_vari::cv_rhs, t0_dbl,
+                  memory->nv_state_),
+        "CVodeInit");
+
+    // Assign pointer to this as user data
+    check_flag_sundials(
+        CVodeSetUserData(memory->cvodes_mem_, reinterpret_cast<void*>(this)),
+        "CVodeSetUserData");
+
+    cvodes_set_options(memory->cvodes_mem_, relative_tolerance_,
+                       absolute_tolerance_, max_num_steps_);
+
+    // for the stiff solvers we need to reserve additional memory
+    // and provide a Jacobian function call. new API since 3.0.0:
+    // create matrix object and linear solver object; resource
+    // (de-)allocation is handled in the cvodes_ode_data
+    check_flag_sundials(
+        CVodeSetLinearSolver(memory->cvodes_mem_, memory->LS_, memory->A_),
+        "CVodeSetLinearSolver");
+
+    check_flag_sundials(
+        CVodeSetJacFn(memory->cvodes_mem_,
+                      &cvodes_integrator_adjoint_vari::cv_jacobian_states),
+        "CVodeSetJacFn");
+
+    // initialize forward sensitivity system of CVODES as needed
+    if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
+      check_flag_sundials(CVodeAdjInit(memory->cvodes_mem_, 25, CV_HERMITE),
+                          "CVodeAdjInit");
+    }
+
+    double t_init = t0_dbl;
+    for (size_t n = 0; n < ts_dbl.size(); ++n) {
+      double t_final = ts_dbl[n];
+
+      if (t_final != t_init) {
+        if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
+          int ncheck;
+          check_flag_sundials(
+              CVodeF(memory->cvodes_mem_, t_final, memory->nv_state_, &t_init,
+                     CV_NORMAL, &ncheck),
+              "CVodeF");
+        } else {
+          check_flag_sundials(CVode(memory->cvodes_mem_, t_final,
+                                    memory->nv_state_, &t_init, CV_NORMAL),
+                              "CVode");
+        }
+      }
+
+      memory->y_[n] = memory->state;
+
+      t_init = t_final;
+    }
+
+    returned_ = true;
+    return build_varis<T_Return>(this, non_chaining_varis_, memory->y_);
+  }
+
+  virtual void chain() {
+    // std::cout << "chain" << std::endl; <-- Good way to verify it's only
+    //  being called once
+    if (memory == NULL)
+      return;
+
+    if (memory->cvodes_mem_ == NULL)
+      return;
+
+    if (returned_ == false)
+      return;
+
+    if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ == 0) {
+      return;
+    }
+
+    Eigen::VectorXd state_sens(N_);
+    Eigen::VectorXd quad(args_vars_);
+    N_Vector nv_state_sens
+        = N_VMake_Serial(state_sens.size(), state_sens.data());
+    N_Vector nv_quad = N_VMake_Serial(quad.size(), quad.data());
+    N_VConst(0.0, nv_state_sens);
+    N_VConst(0.0, nv_quad);
+
+    SUNMatrix AB_ = SUNDenseMatrix(N_, N_);
+    SUNLinearSolver LSB_ = SUNDenseLinearSolver(nv_state_sens, AB_);
+
+    try {
+      int indexB;
+
+      // This is all boilerplate CVODES setting up the adjoint ODE to solve
+      check_flag_sundials(CVodeCreateB(memory->cvodes_mem_, Lmm, &indexB),
+                          "CVodeCreateB");
+
+      check_flag_sundials(CVodeSetUserDataB(memory->cvodes_mem_, indexB,
+                                            reinterpret_cast<void*>(this)),
+                          "CVodeSetUserDataB");
+
+      // The ode_rhs_adj_sense functions passed in here cause problems with
+      // the autodiff stack (they can cause reallocations of the internal
+      // vectors and cause segfaults)
+      check_flag_sundials(
+          CVodeInitB(memory->cvodes_mem_, indexB,
+                     &cvodes_integrator_adjoint_vari::cv_rhs_adj_sens,
+                     value_of(memory->ts_.back()), nv_state_sens),
+          "CVodeInitB");
+
+      check_flag_sundials(
+          CVodeSStolerancesB(memory->cvodes_mem_, indexB, relative_tolerance_,
+                             absolute_tolerance_),
+          "CVodeSStolerancesB");
+
+      check_flag_sundials(
+          CVodeSetLinearSolverB(memory->cvodes_mem_, indexB, LSB_, AB_),
+          "CVodeSetLinearSolverB");
+
+      // The same autodiff issue that applies to ode_rhs_adj_sense applies
+      // here
+      check_flag_sundials(
+          CVodeSetJacFnB(memory->cvodes_mem_, indexB,
+                         &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
+          "CVodeSetJacFnB");
+
+      // Allocate space for backwards quadrature
+      if (args_vars_ > 0) {
+        check_flag_sundials(
+            CVodeQuadInitB(memory->cvodes_mem_, indexB,
+                           &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj, nv_quad),
+            "CVodeQuadInitB");
+
+        check_flag_sundials(
+            CVodeQuadSStolerancesB(memory->cvodes_mem_, indexB,
+                                   relative_tolerance_, absolute_tolerance_),
+            "CVodeQuadSStolerancesB");
+
+        check_flag_sundials(
+            CVodeSetQuadErrConB(memory->cvodes_mem_, indexB, SUNTRUE),
+            "CVodeSetQuadErrConB");
+      }
+
+      // At every time step, collect the adjoints from the output
+      // variables and re-initialize the solver
+      double t_init = value_of(memory->ts_.back());
+      for (int i = memory->ts_.size() - 1; i >= 0; --i) {
+        // Take in the adjoints from all the output variables at this point
+        // in time
+        Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
+        for (int j = 0; j < N_; j++) {
+          // std::cout << "i: " << i << ", j: " << j << std::endl;
+          state_sens(j) += non_chaining_varis_[i * N_ + j]->adj_;
+          step_sens(j) += non_chaining_varis_[i * N_ + j]->adj_;
+        }
+
+        if (ts_vars_ > 0 && i >= 0) {
+          ts_varis_[i]->adj_ += apply(
+              [&](auto&&... args) {
+                double adj = step_sens.dot(
+                    memory->f_(t_init, memory->y_[i], msgs_, args...));
+                // std::cout << "adj: " << adj << ", i: " << i << std::endl;
+                return adj;
+              },
+              memory->value_of_args_tuple_);
+        }
+
+        double t_final = value_of((i > 0) ? memory->ts_[i - 1] : memory->t0_);
+        if (t_final != t_init) {
+          check_flag_sundials(
+              CVodeReInitB(memory->cvodes_mem_, indexB, t_init, nv_state_sens),
+              "CVodeReInitB");
+
+          if (args_vars_ > 0) {
+            check_flag_sundials(
+                CVodeQuadReInitB(memory->cvodes_mem_, indexB, nv_quad),
+                "CVodeQuadReInitB");
+          }
+
+          check_flag_sundials(CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL),
+                              "CVodeB");
+
+          check_flag_sundials(
+              CVodeGetB(memory->cvodes_mem_, indexB, &t_init, nv_state_sens),
+              "CVodeGetB");
+
+          if (args_vars_ > 0) {
+            check_flag_sundials(
+                CVodeGetQuadB(memory->cvodes_mem_, indexB, &t_init, nv_quad),
+                "CVodeGetQuadB");
+          }
+        }
+      }
+
+      if (t0_vars_ > 0) {
+        Eigen::VectorXd y0d = value_of(memory->y0_);
+        t0_varis_[0]->adj_ += apply(
+            [&](auto&&... args) {
+              return -state_sens.dot(memory->f_(t_init, y0d, msgs_, args...));
+            },
+            memory->value_of_args_tuple_);
+      }
+
+      if (args_vars_ > 0) {
+        check_flag_sundials(
+            CVodeGetQuadB(memory->cvodes_mem_, indexB, &t_init, nv_quad),
+            "CVodeGetQuadB");
+      }
+
+      // After integrating all the way back to t0, we finally have the
+      // the adjoints we wanted
+      // These are the dlog_density / d(initial_conditions[s]) adjoints
+      for (size_t s = 0; s < y0_vars_; s++) {
+        y0_varis_[s]->adj_ += state_sens(s);
+      }
+
+      // These are the dlog_density / d(parameters[s]) adjoints
+      for (size_t s = 0; s < args_vars_; s++) {
+        args_varis_[s]->adj_ += quad(s);
+      }
+    } catch (const std::exception& e) {
+      SUNLinSolFree(LSB_);
+      SUNMatDestroy(AB_);
+      N_VDestroy_Serial(nv_state_sens);
+      N_VDestroy_Serial(nv_quad);
+      throw;
+    }
+
+    SUNLinSolFree(LSB_);
+    SUNMatDestroy(AB_);
+    N_VDestroy_Serial(nv_state_sens);
+    N_VDestroy_Serial(nv_quad);
+  }
+};  // cvodes integrator
+
+}  // namespace math
+}  // namespace stan
+#endif
diff --git a/stan/math/rev/functor/ode_bdf_adjoint.hpp b/stan/math/rev/functor/ode_bdf_adjoint.hpp
new file mode 100644
index 00000000000..9d95a415f10
--- /dev/null
+++ b/stan/math/rev/functor/ode_bdf_adjoint.hpp
@@ -0,0 +1,113 @@
+#ifndef STAN_MATH_REV_FUNCTOR_ODE_BDF_ADJOINT_HPP
+#define STAN_MATH_REV_FUNCTOR_ODE_BDF_ADJOINT_HPP
+
+#include <stan/math/rev/meta.hpp>
+#include <stan/math/rev/functor/cvodes_integrator_adjoint.hpp>
+#include <ostream>
+#include <vector>
+
+namespace stan {
+namespace math {
+
+/**
+ * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
+ * times, { t1, t2, t3, ... } using the stiff backward differentiation formula
+ * BDF solver from CVODES.
+ *
+ * \p f must define an operator() with the signature as:
+ *   template<typename T_t, typename T_y, typename... T_Args>
+ *   Eigen::Matrix<stan::return_type_t<T_t, T_y, T_Args...>, Eigen::Dynamic, 1>
+ *     operator()(const T_t& t, const Eigen::Matrix<T_y, Eigen::Dynamic, 1>& y,
+ *     std::ostream* msgs, const T_Args&... args);
+ *
+ * t is the time, y is the state, msgs is a stream for error messages, and args
+ * are optional arguments passed to the ODE solve function (which are passed
+ * through to \p f without modification).
+ *
+ * @tparam F Type of ODE right hand side
+ * @tparam T_0 Type of initial time
+ * @tparam T_ts Type of output times
+ * @tparam T_Args Types of pass-through parameters
+ *
+ * @param f Right hand side of the ODE
+ * @param y0 Initial state
+ * @param t0 Initial time
+ * @param ts Times at which to solve the ODE at. All values must be sorted and
+ *   not less than t0.
+ * @param relative_tolerance Relative tolerance passed to CVODES
+ * @param absolute_tolerance Absolute tolerance passed to CVODES
+ * @param max_num_steps Upper limit on the number of integration steps to
+ *   take between each output (error if exceeded)
+ * @param[in, out] msgs the print stream for warning messages
+ * @param args Extra arguments passed unmodified through to ODE right hand side
+ * @return Solution to ODE at times \p ts
+ */
+template <typename F, typename T_initial, typename T_t0, typename T_ts,
+          typename... T_Args>
+std::vector<Eigen::Matrix<stan::return_type_t<T_initial, T_t0, T_ts, T_Args...>,
+                          Eigen::Dynamic, 1>>
+ode_bdf_adjoint_tol(const F& f, const Eigen::Matrix<T_initial, Eigen::Dynamic, 1>& y0,
+            const T_t0& t0, const std::vector<T_ts>& ts,
+            double relative_tolerance, double absolute_tolerance,
+            long int max_num_steps, std::ostream* msgs, const T_Args&... args) {
+  auto integrator
+      = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_initial, T_t0, T_ts,
+                                               T_Args...>(
+          f, y0, t0, ts, relative_tolerance, absolute_tolerance, max_num_steps,
+          msgs, args...);
+
+  return (*integrator)();
+}
+
+/**
+ * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
+ * times, { t1, t2, t3, ... } using the stiff backward differentiation formula
+ * (BDF) solver in CVODES with a relative tolerance of 1e-10, an absolute
+ * tolerance of 1e-10, and taking a maximum of 1e8 steps.
+ *
+ * \p f must define an operator() with the signature as:
+ *   template<typename T_t, typename T_y, typename... T_Args>
+ *   Eigen::Matrix<stan::return_type_t<T_t, T_y, T_Args...>, Eigen::Dynamic, 1>
+ *     operator()(const T_t& t, const Eigen::Matrix<T_y, Eigen::Dynamic, 1>& y,
+ *     std::ostream* msgs, const T_Args&... args);
+ *
+ * t is the time, y is the state, msgs is a stream for error messages, and args
+ * are optional arguments passed to the ODE solve function (which are passed
+ * through to \p f without modification).
+ *
+ * @tparam F Type of ODE right hand side
+ * @tparam T_0 Type of initial time
+ * @tparam T_ts Type of output times
+ * @tparam T_Args Types of pass-through parameters
+ *
+ * @param f Right hand side of the ODE
+ * @param y0 Initial state
+ * @param t0 Initial time
+ * @param ts Times at which to solve the ODE at. All values must be sorted and
+ *   not less than t0.
+ * @param relative_tolerance Relative tolerance passed to CVODES
+ * @param absolute_tolerance Absolute tolerance passed to CVODES
+ * @param max_num_steps Upper limit on the number of integration steps to
+ *   take between each output (error if exceeded)
+ * @param[in, out] msgs the print stream for warning messages
+ * @param args Extra arguments passed unmodified through to ODE right hand side
+ * @return Solution to ODE at times \p ts
+ */
+template <typename F, typename T_initial, typename T_t0, typename T_ts,
+          typename... T_Args>
+std::vector<Eigen::Matrix<stan::return_type_t<T_initial, T_t0, T_ts, T_Args...>,
+                          Eigen::Dynamic, 1>>
+ode_bdf_adjoint(const F& f, const Eigen::Matrix<T_initial, Eigen::Dynamic, 1>& y0,
+        const T_t0& t0, const std::vector<T_ts>& ts, std::ostream* msgs,
+        const T_Args&... args) {
+  double relative_tolerance = 1e-10;
+  double absolute_tolerance = 1e-10;
+  long int max_num_steps = 1e8;
+
+  return ode_bdf_adjoint_tol(f, y0, t0, ts, relative_tolerance, absolute_tolerance,
+                     max_num_steps, msgs, args...);
+}
+
+}  // namespace math
+}  // namespace stan
+#endif
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp
new file mode 100644
index 00000000000..bb82b7d06ca
--- /dev/null
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp
@@ -0,0 +1,522 @@
+#include <stan/math/rev.hpp>
+#include <gtest/gtest.h>
+#include <test/unit/math/prim/functor/ode_test_functors.hpp>
+#include <iostream>
+#include <vector>
+
+TEST(StanMathOde_ode_bdf_adjoint, int_t0) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  int t0 = 0;
+  std::vector<double> ts = {0.45, 1.1};
+
+  double a = 1.5;
+
+  std::vector<Eigen::Matrix<double, Eigen::Dynamic, 1>> output
+      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr, a);
+
+  EXPECT_NEAR(output[0][0], 0.4165982112, 1e-5);
+  EXPECT_NEAR(output[1][0], 0.66457668563, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, int_ts) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  double t0 = 0.0;
+  std::vector<double> ts = {1, 2};
+
+  double a = 1.5;
+
+  std::vector<Eigen::Matrix<double, Eigen::Dynamic, 1>> output
+      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr, a);
+
+  EXPECT_NEAR(output[0][0], 0.6649966577, 1e-5);
+  EXPECT_NEAR(output[1][0], 0.09408000537, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, t0) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  var t0 = 0.0;
+  std::vector<double> ts = {0.45, 1.1};
+
+  double a = 1.5;
+
+  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> output
+      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr, a);
+
+  output[0][0].grad();
+
+  EXPECT_NEAR(output[0][0].val(), 0.4165982112, 1e-5);
+  EXPECT_NEAR(t0.adj(), -1.0, 1e-5);
+
+  stan::math::set_zero_all_adjoints();
+
+  output[1][0].grad();
+
+  EXPECT_NEAR(output[1][0].val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(t0.adj(), -1.0, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, ts) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  double t0 = 0.0;
+  std::vector<var> ts = {0.45, 1.1};
+
+  double a = 1.5;
+
+  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> output
+      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr, a);
+
+  output[0][0].grad();
+
+  EXPECT_NEAR(output[0][0].val(), 0.4165982112, 1e-5);
+  EXPECT_NEAR(ts[0].adj(), 0.78070695113, 1e-5);
+
+  stan::math::set_zero_all_adjoints();
+
+  output[1][0].grad();
+
+  EXPECT_NEAR(output[1][0].val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(ts[1].adj(), -0.0791208888, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, ts_repeat) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  double t0 = 0.0;
+  std::vector<var> ts = {0.45, 0.45, 1.1, 1.1};
+
+  double a = 1.5;
+
+  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> output
+      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr, a);
+
+  EXPECT_EQ(output.size(), ts.size());
+
+  output[0][0].grad();
+
+  EXPECT_NEAR(output[0][0].val(), 0.4165982112, 1e-5);
+  EXPECT_NEAR(ts[0].adj(), 0.78070695113, 1e-5);
+
+  stan::math::set_zero_all_adjoints();
+
+  output[1][0].grad();
+
+  EXPECT_NEAR(output[1][0].val(), 0.4165982112, 1e-5);
+  EXPECT_NEAR(ts[1].adj(), 0.78070695113, 1e-5);
+
+  stan::math::set_zero_all_adjoints();
+
+  output[2][0].grad();
+
+  EXPECT_NEAR(output[2][0].val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(ts[2].adj(), -0.0791208888, 1e-5);
+
+  stan::math::set_zero_all_adjoints();
+
+  output[3][0].grad();
+
+  EXPECT_NEAR(output[3][0].val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(ts[3].adj(), -0.0791208888, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, scalar_arg) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  double t0 = 0.0;
+  std::vector<double> ts = {1.1};
+
+  var a = 1.5;
+
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                   a)[0][0];
+
+  output.grad();
+
+  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(a.adj(), -0.50107310888, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, scalar_arg_multi_time) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  double t0 = 0.0;
+  std::vector<double> ts = {0.45, 1.1};
+
+  var a = 1.5;
+
+  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> output
+      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr, a);
+
+  output[0](0).grad();
+
+  EXPECT_NEAR(output[0](0).val(), 0.4165982112, 1e-5);
+  EXPECT_NEAR(a.adj(), -0.04352005542, 1e-5);
+
+  stan::math::set_zero_all_adjoints();
+
+  output[1](0).grad();
+
+  EXPECT_NEAR(output[1](0).val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(a.adj(), -0.50107310888, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, std_vector_arg) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  double t0 = 0.0;
+  std::vector<double> ts = {1.1};
+
+  std::vector<var> a = {1.5};
+
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                   a)[0][0];
+
+  output.grad();
+
+  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(a[0].adj(), -0.50107310888, 1e-5);
+}
+/*
+TEST(StanMathOde_ode_bdf_adjoint, vector_arg) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  double t0 = 0.0;
+  std::vector<double> ts = {1.1};
+
+  Eigen::Matrix<var, Eigen::Dynamic, 1> a(1);
+  a << 1.5;
+
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                   a)[0][0];
+
+  output.grad();
+
+  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(a(0).adj(), -0.50107310888, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, row_vector_arg) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  double t0 = 0.0;
+  std::vector<double> ts = {1.1};
+
+  Eigen::Matrix<var, 1, Eigen::Dynamic> a(1);
+  a << 1.5;
+
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                   a)[0][0];
+
+  output.grad();
+
+  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(a(0).adj(), -0.50107310888, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, matrix_arg) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  double t0 = 0.0;
+  std::vector<double> ts = {1.1};
+
+  Eigen::Matrix<var, Eigen::Dynamic, Eigen::Dynamic> a(1, 1);
+  a << 1.5;
+
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                   a)[0][0];
+
+  output.grad();
+
+  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(a(0, 0).adj(), -0.50107310888, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, scalar_std_vector_args) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  double t0 = 0.0;
+  std::vector<double> ts = {1.1};
+
+  var a0 = 0.75;
+  std::vector<var> a1 = {0.75};
+
+  var output = stan::math::ode_bdf_adjoint(stan::test::Cos2Arg(), y0, t0, ts, nullptr,
+                                   a0, a1)[0][0];
+
+  output.grad();
+
+  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(a0.adj(), -0.50107310888, 1e-5);
+  EXPECT_NEAR(a1[0].adj(), -0.50107310888, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, std_vector_std_vector_args) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  double t0 = 0.0;
+  std::vector<double> ts = {1.1};
+
+  var a0 = 1.5;
+  std::vector<var> a1(1, a0);
+  std::vector<std::vector<var>> a2(1, a1);
+
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                   a2)[0][0];
+
+  output.grad();
+
+  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(a2[0][0].adj(), -0.50107310888, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, std_vector_vector_args) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  double t0 = 0.0;
+  std::vector<double> ts = {1.1};
+
+  var a0 = 1.5;
+  Eigen::Matrix<var, Eigen::Dynamic, 1> a1(1);
+  a1 << a0;
+  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> a2(1, a1);
+
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                   a2)[0][0];
+
+  output.grad();
+
+  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(a2[0](0).adj(), -0.50107310888, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, std_vector_row_vector_args) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  double t0 = 0.0;
+  std::vector<double> ts = {1.1};
+
+  var a0 = 1.5;
+  Eigen::Matrix<var, 1, Eigen::Dynamic> a1(1);
+  a1 << a0;
+  std::vector<Eigen::Matrix<var, 1, Eigen::Dynamic>> a2(1, a1);
+
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                   a2)[0][0];
+
+  output.grad();
+
+  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(a2[0](0).adj(), -0.50107310888, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, std_vector_matrix_args) {
+  using stan::math::var;
+
+  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
+  double t0 = 0.0;
+  std::vector<double> ts = {1.1};
+
+  var a0 = 1.5;
+  Eigen::Matrix<var, Eigen::Dynamic, Eigen::Dynamic> a1(1, 1);
+  a1 << a0;
+  std::vector<Eigen::Matrix<var, Eigen::Dynamic, Eigen::Dynamic>> a2(1, a1);
+
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                   a2)[0][0];
+
+  output.grad();
+
+  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
+  EXPECT_NEAR(a2[0](0).adj(), -0.50107310888, 1e-5);
+}
+
+TEST(StanMathOde_ode_bdf_adjoint, arg_combos_test) {
+  using stan::math::var;
+
+  var t0 = 0.5;
+  var a = 0.2;
+  std::vector<var> ts = {1.25};
+  Eigen::Matrix<var, Eigen::Dynamic, 1> y0(1);
+  y0 << 0.75;
+
+  double t0d = stan::math::value_of(t0);
+  double ad = stan::math::value_of(a);
+  std::vector<double> tsd = stan::math::value_of(ts);
+  Eigen::VectorXd y0d = stan::math::value_of(y0);
+
+  auto check_yT = [&](auto yT) {
+    EXPECT_NEAR(stan::math::value_of(yT),
+                y0d(0) * exp(-0.5 * ad * (tsd[0] * tsd[0] - t0d * t0d)), 1e-5);
+  };
+
+  auto check_t0 = [&](var t0) {
+    EXPECT_NEAR(
+        t0.adj(),
+        ad * t0d * y0d(0) * exp(-0.5 * ad * (tsd[0] * tsd[0] - t0d * t0d)),
+        1e-5);
+  };
+
+  auto check_a = [&](var a) {
+    EXPECT_NEAR(a.adj(),
+                -0.5 * (tsd[0] * tsd[0] - t0d * t0d) * y0d(0)
+                    * exp(-0.5 * ad * (tsd[0] * tsd[0] - t0d * t0d)),
+                1e-5);
+  };
+
+  auto check_ts = [&](std::vector<var> ts) {
+    EXPECT_NEAR(
+        ts[0].adj(),
+        -ad * tsd[0] * y0d(0) * exp(-0.5 * ad * (tsd[0] * tsd[0] - t0d * t0d)),
+        1e-5);
+  };
+
+  auto check_y0 = [&](Eigen::Matrix<var, Eigen::Dynamic, 1> y0) {
+    EXPECT_NEAR(y0(0).adj(), exp(-0.5 * ad * (tsd[0] * tsd[0] - t0d * t0d)),
+                1e-5);
+  };
+
+  double yT1 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, tsd, nullptr,
+                                   ad)[0](0);
+  check_yT(yT1);
+
+  var yT2
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, tsd, nullptr, a)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT2.grad();
+  check_yT(yT2);
+  check_a(a);
+
+  var yT3
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, ts, nullptr, ad)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT3.grad();
+  check_yT(yT3);
+  check_ts(ts);
+
+  var yT4
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, ts, nullptr, a)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT4.grad();
+  check_yT(yT4);
+  check_ts(ts);
+  check_a(a);
+
+  var yT5
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, tsd, nullptr, ad)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT5.grad();
+  check_yT(yT5);
+  check_t0(t0);
+
+  var yT6
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, tsd, nullptr, a)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT6.grad();
+  check_yT(yT6);
+  check_t0(t0);
+  check_a(a);
+
+  var yT7
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, ts, nullptr, ad)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT7.grad();
+  check_yT(yT7);
+  check_t0(t0);
+  check_ts(ts);
+
+  var yT8
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, ts, nullptr, a)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT8.grad();
+  check_yT(yT8);
+  check_t0(t0);
+  check_ts(ts);
+  check_a(a);
+
+  var yT9
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, tsd, nullptr, ad)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT9.grad();
+  check_yT(yT9);
+  check_y0(y0);
+
+  var yT10
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, tsd, nullptr, a)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT10.grad();
+  check_yT(yT10);
+  check_y0(y0);
+  check_a(a);
+
+  var yT11
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, ts, nullptr, ad)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT11.grad();
+  check_yT(yT11);
+  check_y0(y0);
+  check_ts(ts);
+
+  var yT12
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, ts, nullptr, a)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT12.grad();
+  check_yT(yT12);
+  check_y0(y0);
+  check_ts(ts);
+  check_a(a);
+
+  var yT13
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, tsd, nullptr, ad)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT13.grad();
+  check_yT(yT13);
+  check_y0(y0);
+  check_t0(t0);
+
+  var yT14
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, tsd, nullptr, a)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT14.grad();
+  check_yT(yT14);
+  check_y0(y0);
+  check_t0(t0);
+  check_a(a);
+
+  var yT15
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, ts, nullptr, ad)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT15.grad();
+  check_yT(yT15);
+  check_y0(y0);
+  check_t0(t0);
+  check_ts(ts);
+
+  var yT16
+      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, ts, nullptr, a)[0](0);
+  stan::math::set_zero_all_adjoints();
+  yT16.grad();
+  check_yT(yT16);
+  check_y0(y0);
+  check_t0(t0);
+  check_ts(ts);
+  check_a(a);
+}
+*/

From c48021257cadb7a054494f021b5eaca2ca71de1b Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sat, 2 Jan 2021 18:02:00 +0100
Subject: [PATCH 003/157] adding sho tests

---
 .../rev/functor/cvodes_integrator_adjoint.hpp |   2 +-
 .../functor/ode_bdf_adjoint_grad_rev_test.cpp | 257 ++++++++++++++++++
 2 files changed, 258 insertions(+), 1 deletion(-)
 create mode 100644 test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 2cbed8af12c..735d0da578c 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -451,7 +451,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     check_nonzero_size(fun, "times", ts);
     check_nonzero_size(fun, "initial state", y0);
-    check_ordered(fun, "times", ts);
+    check_sorted(fun, "times", ts);
     check_less(fun, "initial time", t0, ts[0]);
     check_positive_finite(fun, "relative_tolerance", relative_tolerance_);
     check_positive_finite(fun, "absolute_tolerance", absolute_tolerance_);
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
new file mode 100644
index 00000000000..51deaf4e59a
--- /dev/null
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
@@ -0,0 +1,257 @@
+#include <stan/math/rev.hpp>
+#include <gtest/gtest.h>
+#include <stan/math/rev/functor/gradient.hpp>
+#include <iostream>
+#include <sstream>
+#include <vector>
+#include <stdexcept>
+
+#define TEST_CVODES_ADAMS 1
+#define TEST_CVODES_BDF 2
+
+using std::cos;
+using std::sin;
+
+double y1(double t, double omega, double chi) { return chi * cos(omega * t); }
+
+double dy1_domega(double t, double omega, double chi) {
+  return -t * chi * sin(omega * t);
+}
+
+double dy1_dchi(double t, double omega, double chi) { return cos(omega * t); }
+
+double y2(double t, double omega, double chi) {
+  return -omega * chi * sin(omega * t);
+}
+
+double dy2_domega(double t, double omega, double chi) {
+  return -chi * (sin(omega * t) + omega * t * cos(omega * t));
+}
+
+double dy2_dchi(double t, double omega, double chi) {
+  return -omega * sin(omega * t);
+}
+
+struct sho_functor {
+  template<typename T0, typename T1, typename T2>
+  inline Eigen::Matrix<stan::return_type_t<T1, T2>, Eigen::Dynamic, 1>
+  operator()(const T0& t, const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y,
+             std::ostream* msgs, const T2& omega) const {
+    Eigen::Matrix<stan::return_type_t<T1, T2>, Eigen::Dynamic, 1> out(2);
+    out << y.coeff(1), - omega * omega * y.coeff(0);
+    return out;
+  }
+};
+
+
+template <int state>
+class test_functor_double_var {
+  int lmm_;
+
+ public:
+  explicit test_functor_double_var(int lmm) : lmm_(lmm) {}
+
+  template <typename T>
+  inline T operator()(Eigen::Matrix<T, Eigen::Dynamic, 1>& x) const {
+    sho_functor sho;
+
+    T omega = x(0);
+
+    Eigen::VectorXd y0(2);
+    y0 << 1.25, 0.0;
+
+    double t0 = 0.0;
+    std::vector<double> ts{5.0};
+
+    std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
+        = (lmm_ == TEST_CVODES_ADAMS)
+        ? stan::math::ode_adams(sho, y0, t0, ts, nullptr, omega)
+        : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
+
+    return ys[0](state);
+  }
+};
+
+
+TEST(StanMathOdeIntegrateODEGradMat, double_var) {
+  double omega = 0.5;
+  double chi = 1.25;
+  double t = 5;
+
+  Eigen::VectorXd x(1);
+  x(0) = omega;
+
+  double f;
+  Eigen::VectorXd grad(1);
+
+  {  // Adams
+    test_functor_double_var<0> func1(TEST_CVODES_ADAMS);
+    stan::math::gradient(func1, x, f, grad);
+
+    EXPECT_NEAR(y1(t, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy1_domega(t, omega, chi), grad(0), 1e-7);
+
+    test_functor_double_var<1> func2(TEST_CVODES_ADAMS);
+    stan::math::gradient(func2, x, f, grad);
+
+    EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-7);
+  }
+
+  {  // bdf
+    test_functor_double_var<0> func1(TEST_CVODES_BDF);
+    stan::math::gradient(func1, x, f, grad);
+
+    EXPECT_NEAR(y1(t, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy1_domega(t, omega, chi), grad(0), 1e-7);
+
+    test_functor_double_var<1> func2(TEST_CVODES_BDF);
+    stan::math::gradient(func2, x, f, grad);
+
+    EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-7);
+  }
+}
+
+template <int state>
+class test_functor_var_double {
+  int lmm_;
+
+ public:
+  explicit test_functor_var_double(int lmm) : lmm_(lmm) {}
+
+  template <typename T>
+  inline T operator()(Eigen::Matrix<T, Eigen::Dynamic, 1>& x) const {
+    sho_functor sho;
+
+    double omega = 0.5;
+
+    Eigen::Matrix<T, Eigen::Dynamic, 1> y0(2);
+    y0 << x(0), 0.0;
+
+    double t0 = 0.0;
+    std::vector<double> ts{5.0};
+
+    std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
+        = (lmm_ == TEST_CVODES_ADAMS)
+        ? stan::math::ode_adams(sho, y0, t0, ts, nullptr, omega)
+        : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
+
+    return ys[0](state);
+  }
+};
+
+
+TEST(StanMathOdeIntegrateODEGradMat, var_double) {
+  double omega = 0.5;
+  double chi = 1.25;
+  double t = 5;
+
+  Eigen::VectorXd x(1);
+  x(0) = chi;
+
+  double f;
+  Eigen::VectorXd grad(1);
+
+  {  // adams
+    test_functor_var_double<0> func1(TEST_CVODES_ADAMS);
+    stan::math::gradient(func1, x, f, grad);
+
+    EXPECT_NEAR(y1(t, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy1_dchi(t, omega, chi), grad(0), 1e-7);
+
+    test_functor_var_double<1> func2(TEST_CVODES_ADAMS);
+    stan::math::gradient(func2, x, f, grad);
+
+    EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy2_dchi(t, omega, chi), grad(0), 1e-7);
+  }
+
+  {  // bdf
+    test_functor_var_double<0> func1(TEST_CVODES_BDF);
+    stan::math::gradient(func1, x, f, grad);
+
+    EXPECT_NEAR(y1(t, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy1_dchi(t, omega, chi), grad(0), 1e-7);
+
+    test_functor_var_double<1> func2(TEST_CVODES_BDF);
+    stan::math::gradient(func2, x, f, grad);
+
+    EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy2_dchi(t, omega, chi), grad(0), 1e-7);
+  }
+}
+
+template <int state>
+class test_functor_var_var {
+  int lmm_;
+
+ public:
+  explicit test_functor_var_var(int lmm) : lmm_(lmm) {}
+
+  template <typename T>
+  inline T operator()(Eigen::Matrix<T, Eigen::Dynamic, 1>& x) const {
+    sho_functor sho;
+
+    T omega = x(0);
+
+    Eigen::Matrix<T, Eigen::Dynamic, 1> y0(2);
+    y0 << x(1), 0.0;
+
+    double t0 = 0.0;
+    std::vector<double> ts{5.0};
+
+    std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
+        = (lmm_ == TEST_CVODES_ADAMS)
+        ? stan::math::ode_adams(sho, y0, t0, ts, nullptr, omega)
+        : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
+
+    return ys[0](state);
+  }
+};
+
+TEST(StanMathOdeIntegrateODEGradMat, var_var) {
+  double omega = 0.5;
+  double chi = 1.25;
+  double t = 5;
+
+  Eigen::VectorXd x(2);
+  x(0) = omega;
+  x(1) = chi;
+
+  double f;
+  Eigen::VectorXd grad(2);
+
+  {
+    test_functor_var_var<0> func1(TEST_CVODES_ADAMS);
+    stan::math::gradient(func1, x, f, grad);
+
+    EXPECT_NEAR(y1(t, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy1_domega(t, omega, chi), grad(0), 1e-7);
+    EXPECT_NEAR(dy1_dchi(t, omega, chi), grad(1), 1e-7);
+
+    test_functor_var_var<1> func2(TEST_CVODES_ADAMS);
+    stan::math::gradient(func2, x, f, grad);
+
+    EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-7);
+    EXPECT_NEAR(dy2_dchi(t, omega, chi), grad(1), 1e-7);
+  }
+
+  {
+    test_functor_var_var<0> func1(TEST_CVODES_BDF);
+    stan::math::gradient(func1, x, f, grad);
+
+    EXPECT_NEAR(y1(t, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy1_domega(t, omega, chi), grad(0), 1e-7);
+    EXPECT_NEAR(dy1_dchi(t, omega, chi), grad(1), 1e-7);
+
+    test_functor_var_var<1> func2(TEST_CVODES_BDF);
+    stan::math::gradient(func2, x, f, grad);
+
+    EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-7);
+    EXPECT_NEAR(dy2_dchi(t, omega, chi), grad(1), 1e-7);
+  }
+}
+

From 942dd5122c53d20fcb5bef9e845619ea68565bac Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sat, 2 Jan 2021 18:09:53 +0100
Subject: [PATCH 004/157] reroute ode_adams to ode_bdf_adjoint

---
 stan/math/rev/functor/ode_adams.hpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/stan/math/rev/functor/ode_adams.hpp b/stan/math/rev/functor/ode_adams.hpp
index ee0bdafbbd5..d4a3c2ffd33 100644
--- a/stan/math/rev/functor/ode_adams.hpp
+++ b/stan/math/rev/functor/ode_adams.hpp
@@ -53,6 +53,7 @@ ode_adams_tol_impl(const char* function_name, const F& f, const T_y0& y0,
                    double relative_tolerance, double absolute_tolerance,
                    long int max_num_steps,  // NOLINT(runtime/int)
                    std::ostream* msgs, const T_Args&... args) {
+  /*
   const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
   return apply(
       [&](const auto&... args_refs) {
@@ -63,6 +64,9 @@ ode_adams_tol_impl(const char* function_name, const F& f, const T_y0& y0,
         return integrator();
       },
       args_ref_tuple);
+  */
+  return ode_bdf_adjoint_tol(f, y0, t0, ts, relative_tolerance, absolute_tolerance,
+                             max_num_steps, msgs, args...);
 }
 
 /**

From d86cbc9c00dad55b27050d98fa27fe3f0cea410e Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sat, 2 Jan 2021 18:57:19 +0100
Subject: [PATCH 005/157] bump

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 735d0da578c..2cd04bbd710 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -77,8 +77,8 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   const T_t0 t0_;
   const std::vector<T_ts> ts_;
   std::tuple<T_Args...> args_tuple_;
-  //std::tuple<decltype(value_of(T_Args()))...> value_of_args_tuple_;
-  std::tuple<decltype(value_of(std::declval<T_Args>()))...> value_of_args_tuple_;
+  std::tuple<decltype(value_of(T_Args()))...> value_of_args_tuple_;
+  //std::tuple<decltype(value_of(std::declval<T_Args>()))...> value_of_args_tuple_;
   std::vector<Eigen::VectorXd> y_;
   void* cvodes_mem_;
   Eigen::VectorXd state;

From 6383fa3229419a3ae150448a826c52fe0776d2ad Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sun, 3 Jan 2021 07:07:01 +0100
Subject: [PATCH 006/157] introduce plain_type_t

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp     | 4 ++--
 test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 2cd04bbd710..0f6cda7f424 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -77,8 +77,8 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   const T_t0 t0_;
   const std::vector<T_ts> ts_;
   std::tuple<T_Args...> args_tuple_;
-  std::tuple<decltype(value_of(T_Args()))...> value_of_args_tuple_;
-  //std::tuple<decltype(value_of(std::declval<T_Args>()))...> value_of_args_tuple_;
+  std::tuple<plain_type_t<decltype(value_of(std::declval<const T_Args&>()))>...>
+value_of_args_tuple_;
   std::vector<Eigen::VectorXd> y_;
   void* cvodes_mem_;
   Eigen::VectorXd state;
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp
index bb82b7d06ca..0ec8a5ac3b8 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp
@@ -244,7 +244,7 @@ TEST(StanMathOde_ode_bdf_adjoint, matrix_arg) {
   EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
   EXPECT_NEAR(a(0, 0).adj(), -0.50107310888, 1e-5);
 }
-
+*/
 TEST(StanMathOde_ode_bdf_adjoint, scalar_std_vector_args) {
   using stan::math::var;
 
@@ -264,7 +264,7 @@ TEST(StanMathOde_ode_bdf_adjoint, scalar_std_vector_args) {
   EXPECT_NEAR(a0.adj(), -0.50107310888, 1e-5);
   EXPECT_NEAR(a1[0].adj(), -0.50107310888, 1e-5);
 }
-
+/*
 TEST(StanMathOde_ode_bdf_adjoint, std_vector_std_vector_args) {
   using stan::math::var;
 

From f97844066e8e00184085fe4c641193fa79351baf Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sun, 3 Jan 2021 07:47:51 +0100
Subject: [PATCH 007/157] fix eigen expression stuff for
 cvodes_integrator_adjoint

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 20 ++++--
 stan/math/rev/functor/ode_bdf_adjoint.hpp     | 67 ++++++++++++++++---
 2 files changed, 72 insertions(+), 15 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 0f6cda7f424..6b448a78387 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -71,9 +71,12 @@ class cvodes_integrator_adjoint_vari;
 template <int Lmm, typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args>
 class cvodes_integrator_adjoint_memory : public chainable_alloc {
+  using T_Return = return_type_t<T_y0, T_t0, T_ts, T_Args...>;
+  using T_y0_t0 = return_type_t<T_y0, T_t0>;
+
   const size_t N_;
   const F f_;
-  const Eigen::Matrix<T_y0, Eigen::Dynamic, 1> y0_;
+  const Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1> y0_;
   const T_t0 t0_;
   const std::vector<T_ts> ts_;
   std::tuple<T_Args...> args_tuple_;
@@ -86,8 +89,9 @@ value_of_args_tuple_;
   SUNMatrix A_;
   SUNLinearSolver LS_;
 
+  template <require_eigen_col_vector_t<T_y0>* = nullptr>
   cvodes_integrator_adjoint_memory(const F& f,
-                           const Eigen::Matrix<T_y0, Eigen::Dynamic, 1>& y0,
+                           const T_y0& y0,
                            const T_t0& t0, const std::vector<T_ts>& ts,
                            const T_Args&... args)
       : N_(y0.size()),
@@ -144,6 +148,8 @@ template <int Lmm, typename F, typename T_y0, typename T_t0, typename T_ts,
 class cvodes_integrator_adjoint_vari : public vari {
   using T_Return = return_type_t<T_y0, T_t0, T_ts, T_Args...>;
 
+  const char* function_name_;
+
   const size_t N_;
   bool returned_;
   std::ostream* msgs_;
@@ -384,6 +390,8 @@ class cvodes_integrator_adjoint_vari : public vari {
   /**
    * Construct cvodes_integrator object
    *
+   * @param function_name Calling function name (for printing debugging
+   * messages)
    * @param f Right hand side of the ODE
    * @param y0 Initial state
    * @param t0 Initial time
@@ -400,13 +408,15 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @return a vector of states, each state being a vector of the
    * same size as the state variable, corresponding to a time in ts.
    */
-  cvodes_integrator_adjoint_vari(const F& f,
-                         const Eigen::Matrix<T_y0, Eigen::Dynamic, 1>& y0,
+  template<require_eigen_col_vector_t<T_y0>* = nullptr>
+  cvodes_integrator_adjoint_vari(const char* function_name, const F& f,
+                                 const T_y0& y0,
                          const T_t0& t0, const std::vector<T_ts>& ts,
                          double relative_tolerance, double absolute_tolerance,
                          long int max_num_steps, std::ostream* msgs,
                          const T_Args&... args)
-      : vari(NOT_A_NUMBER),
+      : function_name_(function_name),
+        vari(NOT_A_NUMBER),
         N_(y0.size()),
         returned_(false),
         memory(NULL),
diff --git a/stan/math/rev/functor/ode_bdf_adjoint.hpp b/stan/math/rev/functor/ode_bdf_adjoint.hpp
index 9d95a415f10..1c038fc93a2 100644
--- a/stan/math/rev/functor/ode_bdf_adjoint.hpp
+++ b/stan/math/rev/functor/ode_bdf_adjoint.hpp
@@ -29,6 +29,7 @@ namespace math {
  * @tparam T_ts Type of output times
  * @tparam T_Args Types of pass-through parameters
  *
+ * @param function_name Calling function name (for printing debugging messages)
  * @param f Right hand side of the ODE
  * @param y0 Initial state
  * @param t0 Initial time
@@ -42,23 +43,69 @@ namespace math {
  * @param args Extra arguments passed unmodified through to ODE right hand side
  * @return Solution to ODE at times \p ts
  */
-template <typename F, typename T_initial, typename T_t0, typename T_ts,
+template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args>
-std::vector<Eigen::Matrix<stan::return_type_t<T_initial, T_t0, T_ts, T_Args...>,
+std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
-ode_bdf_adjoint_tol(const F& f, const Eigen::Matrix<T_initial, Eigen::Dynamic, 1>& y0,
+ode_bdf_adjoint_tol_impl(const char* function_name,
+                         const F& f, const T_y0& y0,
             const T_t0& t0, const std::vector<T_ts>& ts,
             double relative_tolerance, double absolute_tolerance,
             long int max_num_steps, std::ostream* msgs, const T_Args&... args) {
   auto integrator
-      = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_initial, T_t0, T_ts,
+      = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0, T_t0, T_ts,
                                                T_Args...>(
-          f, y0, t0, ts, relative_tolerance, absolute_tolerance, max_num_steps,
+                                                   function_name, f, y0, t0, ts, relative_tolerance, absolute_tolerance, max_num_steps,
           msgs, args...);
 
   return (*integrator)();
 }
 
+/**
+ * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
+ * times, { t1, t2, t3, ... } using the stiff backward differentiation formula
+ * BDF solver from CVODES.
+ *
+ * \p f must define an operator() with the signature as:
+ *   template<typename T_t, typename T_y, typename... T_Args>
+ *   Eigen::Matrix<stan::return_type_t<T_t, T_y, T_Args...>, Eigen::Dynamic, 1>
+ *     operator()(const T_t& t, const Eigen::Matrix<T_y, Eigen::Dynamic, 1>& y,
+ *     std::ostream* msgs, const T_Args&... args);
+ *
+ * t is the time, y is the state, msgs is a stream for error messages, and args
+ * are optional arguments passed to the ODE solve function (which are passed
+ * through to \p f without modification).
+ *
+ * @tparam F Type of ODE right hand side
+ * @tparam T_0 Type of initial time
+ * @tparam T_ts Type of output times
+ * @tparam T_Args Types of pass-through parameters
+ *
+ * @param f Right hand side of the ODE
+ * @param y0 Initial state
+ * @param t0 Initial time
+ * @param ts Times at which to solve the ODE at. All values must be sorted and
+ *   not less than t0.
+ * @param relative_tolerance Relative tolerance passed to CVODES
+ * @param absolute_tolerance Absolute tolerance passed to CVODES
+ * @param max_num_steps Upper limit on the number of integration steps to
+ *   take between each output (error if exceeded)
+ * @param[in, out] msgs the print stream for warning messages
+ * @param args Extra arguments passed unmodified through to ODE right hand side
+ * @return Solution to ODE at times \p ts
+ */
+template <typename F, typename T_y0, typename T_t0, typename T_ts,
+          typename... T_Args>
+std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
+                          Eigen::Dynamic, 1>>
+ode_bdf_adjoint_tol(const F& f, const T_y0& y0,
+            const T_t0& t0, const std::vector<T_ts>& ts,
+            double relative_tolerance, double absolute_tolerance,
+            long int max_num_steps, std::ostream* msgs, const T_Args&... args) {
+  return ode_bdf_tol_impl("ode_bdf_adjoint_tol", f, y0, t0, ts, relative_tolerance,
+                          absolute_tolerance, max_num_steps, msgs, args...);
+}
+
 /**
  * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
  * times, { t1, t2, t3, ... } using the stiff backward differentiation formula
@@ -93,19 +140,19 @@ ode_bdf_adjoint_tol(const F& f, const Eigen::Matrix<T_initial, Eigen::Dynamic, 1
  * @param args Extra arguments passed unmodified through to ODE right hand side
  * @return Solution to ODE at times \p ts
  */
-template <typename F, typename T_initial, typename T_t0, typename T_ts,
+template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args>
-std::vector<Eigen::Matrix<stan::return_type_t<T_initial, T_t0, T_ts, T_Args...>,
+std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
-ode_bdf_adjoint(const F& f, const Eigen::Matrix<T_initial, Eigen::Dynamic, 1>& y0,
+ode_bdf_adjoint(const F& f, const T_y0& y0,
         const T_t0& t0, const std::vector<T_ts>& ts, std::ostream* msgs,
         const T_Args&... args) {
   double relative_tolerance = 1e-10;
   double absolute_tolerance = 1e-10;
   long int max_num_steps = 1e8;
 
-  return ode_bdf_adjoint_tol(f, y0, t0, ts, relative_tolerance, absolute_tolerance,
-                     max_num_steps, msgs, args...);
+  return ode_bdf_adjoint_tol_impl("ode_bdf_adjoint", f, y0, t0, ts, relative_tolerance, absolute_tolerance,
+                                  max_num_steps, msgs, args...);
 }
 
 }  // namespace math

From 79cfeca55ae25d90a32f98d64883dc63ecb404b1 Mon Sep 17 00:00:00 2001
From: Ben <bbbales2@gmail.com>
Date: Sun, 3 Jan 2021 14:42:15 -0500
Subject: [PATCH 008/157] Changed how tuples initialized in ODE adjoint memory

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp     | 2 +-
 test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 6b448a78387..8bbd18a90d6 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -100,7 +100,7 @@ value_of_args_tuple_;
         t0_(t0),
         ts_(ts),
         args_tuple_(std::make_tuple(args...)),
-        value_of_args_tuple_(std::make_tuple(value_of(args)...)),
+        value_of_args_tuple_(value_of(args)...),
         y_(ts_.size()),
         cvodes_mem_(nullptr),
         state(value_of(y0)) {
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp
index 0ec8a5ac3b8..a4f60782804 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp
@@ -187,7 +187,7 @@ TEST(StanMathOde_ode_bdf_adjoint, std_vector_arg) {
   EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
   EXPECT_NEAR(a[0].adj(), -0.50107310888, 1e-5);
 }
-/*
+
 TEST(StanMathOde_ode_bdf_adjoint, vector_arg) {
   using stan::math::var;
 
@@ -244,7 +244,7 @@ TEST(StanMathOde_ode_bdf_adjoint, matrix_arg) {
   EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
   EXPECT_NEAR(a(0, 0).adj(), -0.50107310888, 1e-5);
 }
-*/
+
 TEST(StanMathOde_ode_bdf_adjoint, scalar_std_vector_args) {
   using stan::math::var;
 
@@ -264,7 +264,7 @@ TEST(StanMathOde_ode_bdf_adjoint, scalar_std_vector_args) {
   EXPECT_NEAR(a0.adj(), -0.50107310888, 1e-5);
   EXPECT_NEAR(a1[0].adj(), -0.50107310888, 1e-5);
 }
-/*
+
 TEST(StanMathOde_ode_bdf_adjoint, std_vector_std_vector_args) {
   using stan::math::var;
 
@@ -519,4 +519,4 @@ TEST(StanMathOde_ode_bdf_adjoint, arg_combos_test) {
   check_ts(ts);
   check_a(a);
 }
-*/
+

From cdddc433a6c750eb7984ea84c22c897993b47e9a Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sun, 3 Jan 2021 23:02:14 +0100
Subject: [PATCH 009/157] more expression things

---
 stan/math/rev/functor/ode_bdf_adjoint.hpp | 25 ++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/stan/math/rev/functor/ode_bdf_adjoint.hpp b/stan/math/rev/functor/ode_bdf_adjoint.hpp
index 1c038fc93a2..d8efeac37ae 100644
--- a/stan/math/rev/functor/ode_bdf_adjoint.hpp
+++ b/stan/math/rev/functor/ode_bdf_adjoint.hpp
@@ -44,7 +44,7 @@ namespace math {
  * @return Solution to ODE at times \p ts
  */
 template <typename F, typename T_y0, typename T_t0, typename T_ts,
-          typename... T_Args>
+          typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_bdf_adjoint_tol_impl(const char* function_name,
@@ -52,12 +52,23 @@ ode_bdf_adjoint_tol_impl(const char* function_name,
             const T_t0& t0, const std::vector<T_ts>& ts,
             double relative_tolerance, double absolute_tolerance,
             long int max_num_steps, std::ostream* msgs, const T_Args&... args) {
-  auto integrator
-      = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0, T_t0, T_ts,
+  /*
+  const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
+  return apply(
+      [&](const auto&... args_refs) {
+        auto integrator
+            = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0, T_t0, T_ts,
                                                T_Args...>(
                                                    function_name, f, y0, t0, ts, relative_tolerance, absolute_tolerance, max_num_steps,
-          msgs, args...);
-
+                                                   msgs, args_refs...);
+        return (*integrator)();
+      }, args_ref_tuple);
+  */
+  auto integrator
+      = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0, T_t0, T_ts,
+                                                       T_Args...>(
+                                                           function_name, f, y0, t0, ts, relative_tolerance, absolute_tolerance, max_num_steps,
+                                                           msgs, args...);
   return (*integrator)();
 }
 
@@ -95,7 +106,7 @@ ode_bdf_adjoint_tol_impl(const char* function_name,
  * @return Solution to ODE at times \p ts
  */
 template <typename F, typename T_y0, typename T_t0, typename T_ts,
-          typename... T_Args>
+          typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_bdf_adjoint_tol(const F& f, const T_y0& y0,
@@ -141,7 +152,7 @@ ode_bdf_adjoint_tol(const F& f, const T_y0& y0,
  * @return Solution to ODE at times \p ts
  */
 template <typename F, typename T_y0, typename T_t0, typename T_ts,
-          typename... T_Args>
+          typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_bdf_adjoint(const F& f, const T_y0& y0,

From c4f530abb9e0ac4b27829c8299c97d4501850943 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Sun, 3 Jan 2021 22:06:27 +0000
Subject: [PATCH 010/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp |  37 +++---
 stan/math/rev/functor/ode_adams.hpp           |   4 +-
 stan/math/rev/functor/ode_bdf_adjoint.hpp     |  49 +++----
 .../functor/ode_bdf_adjoint_grad_rev_test.cpp |  20 ++-
 .../rev/functor/ode_bdf_adjoint_rev_test.cpp  | 123 +++++++++---------
 5 files changed, 119 insertions(+), 114 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 8bbd18a90d6..9fa8c809240 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -81,7 +81,7 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   const std::vector<T_ts> ts_;
   std::tuple<T_Args...> args_tuple_;
   std::tuple<plain_type_t<decltype(value_of(std::declval<const T_Args&>()))>...>
-value_of_args_tuple_;
+      value_of_args_tuple_;
   std::vector<Eigen::VectorXd> y_;
   void* cvodes_mem_;
   Eigen::VectorXd state;
@@ -90,10 +90,9 @@ value_of_args_tuple_;
   SUNLinearSolver LS_;
 
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
-  cvodes_integrator_adjoint_memory(const F& f,
-                           const T_y0& y0,
-                           const T_t0& t0, const std::vector<T_ts>& ts,
-                           const T_Args&... args)
+  cvodes_integrator_adjoint_memory(const F& f, const T_y0& y0, const T_t0& t0,
+                                   const std::vector<T_ts>& ts,
+                                   const T_Args&... args)
       : N_(y0.size()),
         f_(f),
         y0_(y0),
@@ -129,7 +128,8 @@ value_of_args_tuple_;
     }
   }
 
-  friend class cvodes_integrator_adjoint_vari<Lmm, F, T_y0, T_t0, T_ts, T_Args...>;
+  friend class cvodes_integrator_adjoint_vari<Lmm, F, T_y0, T_t0, T_ts,
+                                              T_Args...>;
 };
 
 /**
@@ -408,13 +408,14 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @return a vector of states, each state being a vector of the
    * same size as the state variable, corresponding to a time in ts.
    */
-  template<require_eigen_col_vector_t<T_y0>* = nullptr>
+  template <require_eigen_col_vector_t<T_y0>* = nullptr>
   cvodes_integrator_adjoint_vari(const char* function_name, const F& f,
-                                 const T_y0& y0,
-                         const T_t0& t0, const std::vector<T_ts>& ts,
-                         double relative_tolerance, double absolute_tolerance,
-                         long int max_num_steps, std::ostream* msgs,
-                         const T_Args&... args)
+                                 const T_y0& y0, const T_t0& t0,
+                                 const std::vector<T_ts>& ts,
+                                 double relative_tolerance,
+                                 double absolute_tolerance,
+                                 long int max_num_steps, std::ostream* msgs,
+                                 const T_Args&... args)
       : function_name_(function_name),
         vari(NOT_A_NUMBER),
         N_(y0.size()),
@@ -438,8 +439,9 @@ class cvodes_integrator_adjoint_vari : public vari {
             args_vars_)) {
     const char* fun = "cvodes_integrator::integrate";
 
-    memory = new cvodes_integrator_adjoint_memory<Lmm, F, T_y0, T_t0, T_ts, T_Args...>(
-        f, y0, t0, ts, args...);
+    memory = new cvodes_integrator_adjoint_memory<Lmm, F, T_y0, T_t0, T_ts,
+                                                  T_Args...>(f, y0, t0, ts,
+                                                             args...);
 
     save_varis(t0_varis_, t0);
     save_varis(ts_varis_, ts);
@@ -483,8 +485,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     const std::vector<double> ts_dbl = value_of(memory->ts_);
 
     check_flag_sundials(
-        CVodeInit(memory->cvodes_mem_, &cvodes_integrator_adjoint_vari::cv_rhs, t0_dbl,
-                  memory->nv_state_),
+        CVodeInit(memory->cvodes_mem_, &cvodes_integrator_adjoint_vari::cv_rhs,
+                  t0_dbl, memory->nv_state_),
         "CVodeInit");
 
     // Assign pointer to this as user data
@@ -608,7 +610,8 @@ class cvodes_integrator_adjoint_vari : public vari {
       if (args_vars_ > 0) {
         check_flag_sundials(
             CVodeQuadInitB(memory->cvodes_mem_, indexB,
-                           &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj, nv_quad),
+                           &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj,
+                           nv_quad),
             "CVodeQuadInitB");
 
         check_flag_sundials(
diff --git a/stan/math/rev/functor/ode_adams.hpp b/stan/math/rev/functor/ode_adams.hpp
index d4a3c2ffd33..894d07c5a90 100644
--- a/stan/math/rev/functor/ode_adams.hpp
+++ b/stan/math/rev/functor/ode_adams.hpp
@@ -65,8 +65,8 @@ ode_adams_tol_impl(const char* function_name, const F& f, const T_y0& y0,
       },
       args_ref_tuple);
   */
-  return ode_bdf_adjoint_tol(f, y0, t0, ts, relative_tolerance, absolute_tolerance,
-                             max_num_steps, msgs, args...);
+  return ode_bdf_adjoint_tol(f, y0, t0, ts, relative_tolerance,
+                             absolute_tolerance, max_num_steps, msgs, args...);
 }
 
 /**
diff --git a/stan/math/rev/functor/ode_bdf_adjoint.hpp b/stan/math/rev/functor/ode_bdf_adjoint.hpp
index d8efeac37ae..c93d978d57d 100644
--- a/stan/math/rev/functor/ode_bdf_adjoint.hpp
+++ b/stan/math/rev/functor/ode_bdf_adjoint.hpp
@@ -47,28 +47,27 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
-ode_bdf_adjoint_tol_impl(const char* function_name,
-                         const F& f, const T_y0& y0,
-            const T_t0& t0, const std::vector<T_ts>& ts,
-            double relative_tolerance, double absolute_tolerance,
-            long int max_num_steps, std::ostream* msgs, const T_Args&... args) {
+ode_bdf_adjoint_tol_impl(const char* function_name, const F& f, const T_y0& y0,
+                         const T_t0& t0, const std::vector<T_ts>& ts,
+                         double relative_tolerance, double absolute_tolerance,
+                         long int max_num_steps, std::ostream* msgs,
+                         const T_Args&... args) {
   /*
   const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
   return apply(
       [&](const auto&... args_refs) {
         auto integrator
-            = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0, T_t0, T_ts,
-                                               T_Args...>(
-                                                   function_name, f, y0, t0, ts, relative_tolerance, absolute_tolerance, max_num_steps,
-                                                   msgs, args_refs...);
-        return (*integrator)();
+            = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0,
+  T_t0, T_ts, T_Args...>( function_name, f, y0, t0, ts, relative_tolerance,
+  absolute_tolerance, max_num_steps, msgs, args_refs...); return
+  (*integrator)();
       }, args_ref_tuple);
   */
   auto integrator
-      = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0, T_t0, T_ts,
-                                                       T_Args...>(
-                                                           function_name, f, y0, t0, ts, relative_tolerance, absolute_tolerance, max_num_steps,
-                                                           msgs, args...);
+      = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0, T_t0,
+                                                       T_ts, T_Args...>(
+          function_name, f, y0, t0, ts, relative_tolerance, absolute_tolerance,
+          max_num_steps, msgs, args...);
   return (*integrator)();
 }
 
@@ -109,12 +108,13 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
-ode_bdf_adjoint_tol(const F& f, const T_y0& y0,
-            const T_t0& t0, const std::vector<T_ts>& ts,
-            double relative_tolerance, double absolute_tolerance,
-            long int max_num_steps, std::ostream* msgs, const T_Args&... args) {
-  return ode_bdf_tol_impl("ode_bdf_adjoint_tol", f, y0, t0, ts, relative_tolerance,
-                          absolute_tolerance, max_num_steps, msgs, args...);
+ode_bdf_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
+                    const std::vector<T_ts>& ts, double relative_tolerance,
+                    double absolute_tolerance, long int max_num_steps,
+                    std::ostream* msgs, const T_Args&... args) {
+  return ode_bdf_tol_impl("ode_bdf_adjoint_tol", f, y0, t0, ts,
+                          relative_tolerance, absolute_tolerance, max_num_steps,
+                          msgs, args...);
 }
 
 /**
@@ -155,14 +155,15 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
-ode_bdf_adjoint(const F& f, const T_y0& y0,
-        const T_t0& t0, const std::vector<T_ts>& ts, std::ostream* msgs,
-        const T_Args&... args) {
+ode_bdf_adjoint(const F& f, const T_y0& y0, const T_t0& t0,
+                const std::vector<T_ts>& ts, std::ostream* msgs,
+                const T_Args&... args) {
   double relative_tolerance = 1e-10;
   double absolute_tolerance = 1e-10;
   long int max_num_steps = 1e8;
 
-  return ode_bdf_adjoint_tol_impl("ode_bdf_adjoint", f, y0, t0, ts, relative_tolerance, absolute_tolerance,
+  return ode_bdf_adjoint_tol_impl("ode_bdf_adjoint", f, y0, t0, ts,
+                                  relative_tolerance, absolute_tolerance,
                                   max_num_steps, msgs, args...);
 }
 
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
index 51deaf4e59a..77252b08890 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
@@ -33,17 +33,16 @@ double dy2_dchi(double t, double omega, double chi) {
 }
 
 struct sho_functor {
-  template<typename T0, typename T1, typename T2>
+  template <typename T0, typename T1, typename T2>
   inline Eigen::Matrix<stan::return_type_t<T1, T2>, Eigen::Dynamic, 1>
   operator()(const T0& t, const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y,
              std::ostream* msgs, const T2& omega) const {
     Eigen::Matrix<stan::return_type_t<T1, T2>, Eigen::Dynamic, 1> out(2);
-    out << y.coeff(1), - omega * omega * y.coeff(0);
+    out << y.coeff(1), -omega * omega * y.coeff(0);
     return out;
   }
 };
 
-
 template <int state>
 class test_functor_double_var {
   int lmm_;
@@ -65,14 +64,13 @@ class test_functor_double_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-        ? stan::math::ode_adams(sho, y0, t0, ts, nullptr, omega)
-        : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
+              ? stan::math::ode_adams(sho, y0, t0, ts, nullptr, omega)
+              : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return ys[0](state);
   }
 };
 
-
 TEST(StanMathOdeIntegrateODEGradMat, double_var) {
   double omega = 0.5;
   double chi = 1.25;
@@ -134,14 +132,13 @@ class test_functor_var_double {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-        ? stan::math::ode_adams(sho, y0, t0, ts, nullptr, omega)
-        : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
+              ? stan::math::ode_adams(sho, y0, t0, ts, nullptr, omega)
+              : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return ys[0](state);
   }
 };
 
-
 TEST(StanMathOdeIntegrateODEGradMat, var_double) {
   double omega = 0.5;
   double chi = 1.25;
@@ -203,8 +200,8 @@ class test_functor_var_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-        ? stan::math::ode_adams(sho, y0, t0, ts, nullptr, omega)
-        : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
+              ? stan::math::ode_adams(sho, y0, t0, ts, nullptr, omega)
+              : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return ys[0](state);
   }
@@ -254,4 +251,3 @@ TEST(StanMathOdeIntegrateODEGradMat, var_var) {
     EXPECT_NEAR(dy2_dchi(t, omega, chi), grad(1), 1e-7);
   }
 }
-
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp
index a4f60782804..76d86d2e222 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp
@@ -14,7 +14,8 @@ TEST(StanMathOde_ode_bdf_adjoint, int_t0) {
   double a = 1.5;
 
   std::vector<Eigen::Matrix<double, Eigen::Dynamic, 1>> output
-      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr, a);
+      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                    a);
 
   EXPECT_NEAR(output[0][0], 0.4165982112, 1e-5);
   EXPECT_NEAR(output[1][0], 0.66457668563, 1e-5);
@@ -30,7 +31,8 @@ TEST(StanMathOde_ode_bdf_adjoint, int_ts) {
   double a = 1.5;
 
   std::vector<Eigen::Matrix<double, Eigen::Dynamic, 1>> output
-      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr, a);
+      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                    a);
 
   EXPECT_NEAR(output[0][0], 0.6649966577, 1e-5);
   EXPECT_NEAR(output[1][0], 0.09408000537, 1e-5);
@@ -46,7 +48,8 @@ TEST(StanMathOde_ode_bdf_adjoint, t0) {
   double a = 1.5;
 
   std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> output
-      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr, a);
+      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                    a);
 
   output[0][0].grad();
 
@@ -71,7 +74,8 @@ TEST(StanMathOde_ode_bdf_adjoint, ts) {
   double a = 1.5;
 
   std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> output
-      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr, a);
+      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                    a);
 
   output[0][0].grad();
 
@@ -96,7 +100,8 @@ TEST(StanMathOde_ode_bdf_adjoint, ts_repeat) {
   double a = 1.5;
 
   std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> output
-      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr, a);
+      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                    a);
 
   EXPECT_EQ(output.size(), ts.size());
 
@@ -136,8 +141,8 @@ TEST(StanMathOde_ode_bdf_adjoint, scalar_arg) {
 
   var a = 1.5;
 
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                   a)[0][0];
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
+                                           nullptr, a)[0][0];
 
   output.grad();
 
@@ -155,7 +160,8 @@ TEST(StanMathOde_ode_bdf_adjoint, scalar_arg_multi_time) {
   var a = 1.5;
 
   std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> output
-      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr, a);
+      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
+                                    a);
 
   output[0](0).grad();
 
@@ -179,8 +185,8 @@ TEST(StanMathOde_ode_bdf_adjoint, std_vector_arg) {
 
   std::vector<var> a = {1.5};
 
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                   a)[0][0];
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
+                                           nullptr, a)[0][0];
 
   output.grad();
 
@@ -198,8 +204,8 @@ TEST(StanMathOde_ode_bdf_adjoint, vector_arg) {
   Eigen::Matrix<var, Eigen::Dynamic, 1> a(1);
   a << 1.5;
 
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                   a)[0][0];
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
+                                           nullptr, a)[0][0];
 
   output.grad();
 
@@ -217,8 +223,8 @@ TEST(StanMathOde_ode_bdf_adjoint, row_vector_arg) {
   Eigen::Matrix<var, 1, Eigen::Dynamic> a(1);
   a << 1.5;
 
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                   a)[0][0];
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
+                                           nullptr, a)[0][0];
 
   output.grad();
 
@@ -236,8 +242,8 @@ TEST(StanMathOde_ode_bdf_adjoint, matrix_arg) {
   Eigen::Matrix<var, Eigen::Dynamic, Eigen::Dynamic> a(1, 1);
   a << 1.5;
 
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                   a)[0][0];
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
+                                           nullptr, a)[0][0];
 
   output.grad();
 
@@ -255,8 +261,8 @@ TEST(StanMathOde_ode_bdf_adjoint, scalar_std_vector_args) {
   var a0 = 0.75;
   std::vector<var> a1 = {0.75};
 
-  var output = stan::math::ode_bdf_adjoint(stan::test::Cos2Arg(), y0, t0, ts, nullptr,
-                                   a0, a1)[0][0];
+  var output = stan::math::ode_bdf_adjoint(stan::test::Cos2Arg(), y0, t0, ts,
+                                           nullptr, a0, a1)[0][0];
 
   output.grad();
 
@@ -276,8 +282,8 @@ TEST(StanMathOde_ode_bdf_adjoint, std_vector_std_vector_args) {
   std::vector<var> a1(1, a0);
   std::vector<std::vector<var>> a2(1, a1);
 
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                   a2)[0][0];
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
+                                           nullptr, a2)[0][0];
 
   output.grad();
 
@@ -297,8 +303,8 @@ TEST(StanMathOde_ode_bdf_adjoint, std_vector_vector_args) {
   a1 << a0;
   std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> a2(1, a1);
 
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                   a2)[0][0];
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
+                                           nullptr, a2)[0][0];
 
   output.grad();
 
@@ -318,8 +324,8 @@ TEST(StanMathOde_ode_bdf_adjoint, std_vector_row_vector_args) {
   a1 << a0;
   std::vector<Eigen::Matrix<var, 1, Eigen::Dynamic>> a2(1, a1);
 
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                   a2)[0][0];
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
+                                           nullptr, a2)[0][0];
 
   output.grad();
 
@@ -339,8 +345,8 @@ TEST(StanMathOde_ode_bdf_adjoint, std_vector_matrix_args) {
   a1 << a0;
   std::vector<Eigen::Matrix<var, Eigen::Dynamic, Eigen::Dynamic>> a2(1, a1);
 
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                   a2)[0][0];
+  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
+                                           nullptr, a2)[0][0];
 
   output.grad();
 
@@ -393,57 +399,57 @@ TEST(StanMathOde_ode_bdf_adjoint, arg_combos_test) {
                 1e-5);
   };
 
-  double yT1 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, tsd, nullptr,
-                                   ad)[0](0);
+  double yT1 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, tsd,
+                                           nullptr, ad)[0](0);
   check_yT(yT1);
 
-  var yT2
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, tsd, nullptr, a)[0](0);
+  var yT2 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, tsd,
+                                        nullptr, a)[0](0);
   stan::math::set_zero_all_adjoints();
   yT2.grad();
   check_yT(yT2);
   check_a(a);
 
-  var yT3
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, ts, nullptr, ad)[0](0);
+  var yT3 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, ts,
+                                        nullptr, ad)[0](0);
   stan::math::set_zero_all_adjoints();
   yT3.grad();
   check_yT(yT3);
   check_ts(ts);
 
-  var yT4
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, ts, nullptr, a)[0](0);
+  var yT4 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, ts,
+                                        nullptr, a)[0](0);
   stan::math::set_zero_all_adjoints();
   yT4.grad();
   check_yT(yT4);
   check_ts(ts);
   check_a(a);
 
-  var yT5
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, tsd, nullptr, ad)[0](0);
+  var yT5 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, tsd,
+                                        nullptr, ad)[0](0);
   stan::math::set_zero_all_adjoints();
   yT5.grad();
   check_yT(yT5);
   check_t0(t0);
 
-  var yT6
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, tsd, nullptr, a)[0](0);
+  var yT6 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, tsd,
+                                        nullptr, a)[0](0);
   stan::math::set_zero_all_adjoints();
   yT6.grad();
   check_yT(yT6);
   check_t0(t0);
   check_a(a);
 
-  var yT7
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, ts, nullptr, ad)[0](0);
+  var yT7 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, ts, nullptr,
+                                        ad)[0](0);
   stan::math::set_zero_all_adjoints();
   yT7.grad();
   check_yT(yT7);
   check_t0(t0);
   check_ts(ts);
 
-  var yT8
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, ts, nullptr, a)[0](0);
+  var yT8 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, ts, nullptr,
+                                        a)[0](0);
   stan::math::set_zero_all_adjoints();
   yT8.grad();
   check_yT(yT8);
@@ -451,31 +457,31 @@ TEST(StanMathOde_ode_bdf_adjoint, arg_combos_test) {
   check_ts(ts);
   check_a(a);
 
-  var yT9
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, tsd, nullptr, ad)[0](0);
+  var yT9 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, tsd,
+                                        nullptr, ad)[0](0);
   stan::math::set_zero_all_adjoints();
   yT9.grad();
   check_yT(yT9);
   check_y0(y0);
 
-  var yT10
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, tsd, nullptr, a)[0](0);
+  var yT10 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, tsd,
+                                         nullptr, a)[0](0);
   stan::math::set_zero_all_adjoints();
   yT10.grad();
   check_yT(yT10);
   check_y0(y0);
   check_a(a);
 
-  var yT11
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, ts, nullptr, ad)[0](0);
+  var yT11 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, ts,
+                                         nullptr, ad)[0](0);
   stan::math::set_zero_all_adjoints();
   yT11.grad();
   check_yT(yT11);
   check_y0(y0);
   check_ts(ts);
 
-  var yT12
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, ts, nullptr, a)[0](0);
+  var yT12 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, ts,
+                                         nullptr, a)[0](0);
   stan::math::set_zero_all_adjoints();
   yT12.grad();
   check_yT(yT12);
@@ -483,16 +489,16 @@ TEST(StanMathOde_ode_bdf_adjoint, arg_combos_test) {
   check_ts(ts);
   check_a(a);
 
-  var yT13
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, tsd, nullptr, ad)[0](0);
+  var yT13 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, tsd,
+                                         nullptr, ad)[0](0);
   stan::math::set_zero_all_adjoints();
   yT13.grad();
   check_yT(yT13);
   check_y0(y0);
   check_t0(t0);
 
-  var yT14
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, tsd, nullptr, a)[0](0);
+  var yT14 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, tsd,
+                                         nullptr, a)[0](0);
   stan::math::set_zero_all_adjoints();
   yT14.grad();
   check_yT(yT14);
@@ -500,8 +506,8 @@ TEST(StanMathOde_ode_bdf_adjoint, arg_combos_test) {
   check_t0(t0);
   check_a(a);
 
-  var yT15
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, ts, nullptr, ad)[0](0);
+  var yT15 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, ts, nullptr,
+                                         ad)[0](0);
   stan::math::set_zero_all_adjoints();
   yT15.grad();
   check_yT(yT15);
@@ -509,8 +515,8 @@ TEST(StanMathOde_ode_bdf_adjoint, arg_combos_test) {
   check_t0(t0);
   check_ts(ts);
 
-  var yT16
-      = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, ts, nullptr, a)[0](0);
+  var yT16 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, ts, nullptr,
+                                         a)[0](0);
   stan::math::set_zero_all_adjoints();
   yT16.grad();
   check_yT(yT16);
@@ -519,4 +525,3 @@ TEST(StanMathOde_ode_bdf_adjoint, arg_combos_test) {
   check_ts(ts);
   check_a(a);
 }
-

From 37ec72c32a390be63fa5b0d478c8bd501ffbd20b Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sun, 3 Jan 2021 23:07:59 +0100
Subject: [PATCH 011/157] add simple benchmark in the form of a test

---
 .../functor/ode_bdf_adjoint_bench_test.cpp    | 134 ++++++++++++++++++
 1 file changed, 134 insertions(+)
 create mode 100644 test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
new file mode 100644
index 00000000000..b7369a27ee5
--- /dev/null
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
@@ -0,0 +1,134 @@
+
+#include <stan/math/rev.hpp>
+#include <gtest/gtest.h>
+#include <stan/math/rev/functor/gradient.hpp>
+#include <iostream>
+#include <sstream>
+#include <vector>
+#include <stdexcept>
+
+using stan::math::var;
+using stan::math::lognormal_rng;
+
+struct pkpd_rhs {
+  template<typename T0, typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8>
+  inline Eigen::Matrix<stan::return_type_t<T1, T2, T3, T4, T5, T6, T7, T8>, Eigen::Dynamic, 1>
+  operator()(const T0& t,
+             const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y,
+             std::ostream* msgs,
+             const T2& ka,
+             const T3& ke,
+             const T4& k12,
+             const T5& k21,
+             const T6& kin,
+             const T7& kout,
+             const T8& ea50) const {
+    Eigen::Matrix<stan::return_type_t<T1, T2, T3, T4, T5, T6, T7, T8>, Eigen::Dynamic, 1> dydt(4);
+
+    dydt <<
+        -ka * y(0),
+        +ka * y(0) - ke * y(1) - k12 * y(1) + k21 * y(2),
+        +k12 * y(1) - k21 * y(2),
+        +kin - kout * (1.0 - y(1) / (y(1) + ea50) ) * y(3);
+    
+    return dydt;
+  }  
+};
+
+TEST(StanMathOdeBench, bdf) {
+
+  double true_CL = 8.0;
+  double true_Q = 18.0;
+  double true_V1 = 10.0;
+  double true_V2 = 14.0;
+  double true_ka = log(2.0)/2.0;
+  double true_pd0 = 1.0;
+  double true_kin = 4.0;
+  double true_kout = true_kin/true_pd0;
+  double true_ec50 = 0.01;
+  std::vector<double> ts{1., 2.0, 4.0, 8.0, 16.0, 32.0, 64.0};
+  std::size_t ts_size = ts.size();
+
+  pkpd_rhs ode;
+
+  boost::ecuyer1988 base_rng(563646);
+
+  for(std::size_t i = 0; i != 200; i++) {
+    stan::math::nested_rev_autodiff nested;
+    
+    var CL = lognormal_rng(true_CL, 4.0, base_rng);
+    var Q = lognormal_rng(true_Q, 4.0, base_rng);
+    var V1 = lognormal_rng(true_V1, 2.0, base_rng);
+    var V2 = lognormal_rng(true_V2, 2.0, base_rng);
+    var ka = true_ka;
+    var pd0 = true_pd0;
+    var kin = true_kin;
+    var kout =  kin/ pd0;
+    var ec50 = true_ec50;
+    var ea50 = ec50 * V1;
+    
+    var ke = CL / V1;
+    var k12 = Q / V2;
+    var k21 = k12 * V1 / V2;
+    
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(4);
+    y0 << 4.0, 0.0, 0.0, pd0;
+
+    double t0 = 0.0;
+
+    std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
+        = ode_bdf_tol(ode, y0, t0, ts, 1E-8, 1E-8, 10000, nullptr, ka, ke, k12, k21, kin, kout, ea50);
+
+    stan::math::grad();
+  }
+
+}
+
+TEST(StanMathOdeBench, bdf_adjoint) {
+
+  double true_CL = 8.0;
+  double true_Q = 18.0;
+  double true_V1 = 10.0;
+  double true_V2 = 14.0;
+  double true_ka = log(2.0)/2.0;
+  double true_pd0 = 1.0;
+  double true_kin = 4.0;
+  double true_kout = true_kin/true_pd0;
+  double true_ec50 = 0.01;
+  std::vector<double> ts{1., 2.0, 4.0, 8.0, 16.0, 32.0, 64.0};
+  std::size_t ts_size = ts.size();
+
+  pkpd_rhs ode;
+
+  boost::ecuyer1988 base_rng(563646);
+
+  for(std::size_t i = 0; i != 200; i++) {
+    stan::math::nested_rev_autodiff nested;
+    
+    var CL = lognormal_rng(true_CL, 4.0, base_rng);
+    var Q = lognormal_rng(true_Q, 4.0, base_rng);
+    var V1 = lognormal_rng(true_V1, 2.0, base_rng);
+    var V2 = lognormal_rng(true_V2, 2.0, base_rng);
+    var ka = true_ka;
+    var pd0 = true_pd0;
+    var kin = true_kin;
+    var kout =  kin/ pd0;
+    var ec50 = true_ec50;
+    var ea50 = ec50 * V1;
+    
+    var ke = CL / V1;
+    var k12 = Q / V2;
+    var k21 = k12 * V1 / V2;
+    
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(4);
+    y0 << 4.0, 0.0, 0.0, pd0;
+
+    double t0 = 0.0;
+
+    std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
+        = ode_bdf_adjoint_tol(ode, y0, t0, ts, 1E-8, 1E-8, 10000, nullptr, ka, ke, k12, k21, kin, kout, ea50);
+
+    stan::math::grad();
+  }
+
+}

From 64acf4a01c0de294a18f2ae2e5c4a0524d6a24b6 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Sun, 3 Jan 2021 22:09:22 +0000
Subject: [PATCH 012/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../functor/ode_bdf_adjoint_bench_test.cpp    | 72 +++++++++----------
 1 file changed, 33 insertions(+), 39 deletions(-)

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
index b7369a27ee5..e9d0389ca18 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
@@ -7,44 +7,39 @@
 #include <vector>
 #include <stdexcept>
 
-using stan::math::var;
 using stan::math::lognormal_rng;
+using stan::math::var;
 
 struct pkpd_rhs {
-  template<typename T0, typename T1, typename T2, typename T3, typename T4, typename T5, typename T6, typename T7, typename T8>
-  inline Eigen::Matrix<stan::return_type_t<T1, T2, T3, T4, T5, T6, T7, T8>, Eigen::Dynamic, 1>
-  operator()(const T0& t,
-             const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y,
-             std::ostream* msgs,
-             const T2& ka,
-             const T3& ke,
-             const T4& k12,
-             const T5& k21,
-             const T6& kin,
-             const T7& kout,
+  template <typename T0, typename T1, typename T2, typename T3, typename T4,
+            typename T5, typename T6, typename T7, typename T8>
+  inline Eigen::Matrix<stan::return_type_t<T1, T2, T3, T4, T5, T6, T7, T8>,
+                       Eigen::Dynamic, 1>
+  operator()(const T0& t, const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y,
+             std::ostream* msgs, const T2& ka, const T3& ke, const T4& k12,
+             const T5& k21, const T6& kin, const T7& kout,
              const T8& ea50) const {
-    Eigen::Matrix<stan::return_type_t<T1, T2, T3, T4, T5, T6, T7, T8>, Eigen::Dynamic, 1> dydt(4);
+    Eigen::Matrix<stan::return_type_t<T1, T2, T3, T4, T5, T6, T7, T8>,
+                  Eigen::Dynamic, 1>
+        dydt(4);
 
-    dydt <<
-        -ka * y(0),
-        +ka * y(0) - ke * y(1) - k12 * y(1) + k21 * y(2),
+    dydt << -ka * y(0), +ka * y(0) - ke * y(1) - k12 * y(1) + k21 * y(2),
         +k12 * y(1) - k21 * y(2),
-        +kin - kout * (1.0 - y(1) / (y(1) + ea50) ) * y(3);
-    
+        +kin - kout * (1.0 - y(1) / (y(1) + ea50)) * y(3);
+
     return dydt;
-  }  
+  }
 };
 
 TEST(StanMathOdeBench, bdf) {
-
   double true_CL = 8.0;
   double true_Q = 18.0;
   double true_V1 = 10.0;
   double true_V2 = 14.0;
-  double true_ka = log(2.0)/2.0;
+  double true_ka = log(2.0) / 2.0;
   double true_pd0 = 1.0;
   double true_kin = 4.0;
-  double true_kout = true_kin/true_pd0;
+  double true_kout = true_kin / true_pd0;
   double true_ec50 = 0.01;
   std::vector<double> ts{1., 2.0, 4.0, 8.0, 16.0, 32.0, 64.0};
   std::size_t ts_size = ts.size();
@@ -53,9 +48,9 @@ TEST(StanMathOdeBench, bdf) {
 
   boost::ecuyer1988 base_rng(563646);
 
-  for(std::size_t i = 0; i != 200; i++) {
+  for (std::size_t i = 0; i != 200; i++) {
     stan::math::nested_rev_autodiff nested;
-    
+
     var CL = lognormal_rng(true_CL, 4.0, base_rng);
     var Q = lognormal_rng(true_Q, 4.0, base_rng);
     var V1 = lognormal_rng(true_V1, 2.0, base_rng);
@@ -63,37 +58,36 @@ TEST(StanMathOdeBench, bdf) {
     var ka = true_ka;
     var pd0 = true_pd0;
     var kin = true_kin;
-    var kout =  kin/ pd0;
+    var kout = kin / pd0;
     var ec50 = true_ec50;
     var ea50 = ec50 * V1;
-    
+
     var ke = CL / V1;
     var k12 = Q / V2;
     var k21 = k12 * V1 / V2;
-    
+
     Eigen::Matrix<var, Eigen::Dynamic, 1> y0(4);
     y0 << 4.0, 0.0, 0.0, pd0;
 
     double t0 = 0.0;
 
     std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-        = ode_bdf_tol(ode, y0, t0, ts, 1E-8, 1E-8, 10000, nullptr, ka, ke, k12, k21, kin, kout, ea50);
+        = ode_bdf_tol(ode, y0, t0, ts, 1E-8, 1E-8, 10000, nullptr, ka, ke, k12,
+                      k21, kin, kout, ea50);
 
     stan::math::grad();
   }
-
 }
 
 TEST(StanMathOdeBench, bdf_adjoint) {
-
   double true_CL = 8.0;
   double true_Q = 18.0;
   double true_V1 = 10.0;
   double true_V2 = 14.0;
-  double true_ka = log(2.0)/2.0;
+  double true_ka = log(2.0) / 2.0;
   double true_pd0 = 1.0;
   double true_kin = 4.0;
-  double true_kout = true_kin/true_pd0;
+  double true_kout = true_kin / true_pd0;
   double true_ec50 = 0.01;
   std::vector<double> ts{1., 2.0, 4.0, 8.0, 16.0, 32.0, 64.0};
   std::size_t ts_size = ts.size();
@@ -102,9 +96,9 @@ TEST(StanMathOdeBench, bdf_adjoint) {
 
   boost::ecuyer1988 base_rng(563646);
 
-  for(std::size_t i = 0; i != 200; i++) {
+  for (std::size_t i = 0; i != 200; i++) {
     stan::math::nested_rev_autodiff nested;
-    
+
     var CL = lognormal_rng(true_CL, 4.0, base_rng);
     var Q = lognormal_rng(true_Q, 4.0, base_rng);
     var V1 = lognormal_rng(true_V1, 2.0, base_rng);
@@ -112,23 +106,23 @@ TEST(StanMathOdeBench, bdf_adjoint) {
     var ka = true_ka;
     var pd0 = true_pd0;
     var kin = true_kin;
-    var kout =  kin/ pd0;
+    var kout = kin / pd0;
     var ec50 = true_ec50;
     var ea50 = ec50 * V1;
-    
+
     var ke = CL / V1;
     var k12 = Q / V2;
     var k21 = k12 * V1 / V2;
-    
+
     Eigen::Matrix<var, Eigen::Dynamic, 1> y0(4);
     y0 << 4.0, 0.0, 0.0, pd0;
 
     double t0 = 0.0;
 
     std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-        = ode_bdf_adjoint_tol(ode, y0, t0, ts, 1E-8, 1E-8, 10000, nullptr, ka, ke, k12, k21, kin, kout, ea50);
+        = ode_bdf_adjoint_tol(ode, y0, t0, ts, 1E-8, 1E-8, 10000, nullptr, ka,
+                              ke, k12, k21, kin, kout, ea50);
 
     stan::math::grad();
   }
-
 }

From edea9b05ee643fe5c905d8fc475da4dc4f2927dd Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sun, 3 Jan 2021 23:34:39 +0100
Subject: [PATCH 013/157] recycle argument vari's

---
 .../rev/functor/cvodes_integrator_adjoint.hpp   | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 9fa8c809240..6a642975de3 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -79,7 +79,9 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   const Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1> y0_;
   const T_t0 t0_;
   const std::vector<T_ts> ts_;
-  std::tuple<T_Args...> args_tuple_;
+  //std::tuple<T_Args...> args_tuple_;
+  std::tuple<decltype(deep_copy_vars(std::declval<const T_Args&>()))...>
+  local_args_tuple_;
   std::tuple<plain_type_t<decltype(value_of(std::declval<const T_Args&>()))>...>
       value_of_args_tuple_;
   std::vector<Eigen::VectorXd> y_;
@@ -98,7 +100,8 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
         y0_(y0),
         t0_(t0),
         ts_(ts),
-        args_tuple_(std::make_tuple(args...)),
+        //args_tuple_(std::make_tuple(args...)),
+        local_args_tuple_(deep_copy_vars(args)...),
         value_of_args_tuple_(value_of(args)...),
         y_(ts_.size()),
         cvodes_mem_(nullptr),
@@ -310,16 +313,22 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     nested_rev_autodiff nested;
 
+    /*
     auto local_args_tuple = apply(
         [&](auto&&... args) {
           return std::tuple<decltype(deep_copy_vars(args))...>(
               deep_copy_vars(args)...);
         },
         memory->args_tuple_);
+    */
+
+    // The vars here do not live on the nested stack so must be zero'd
+    // separately
+    apply([&](auto&&... args) { zero_adjoints(args...); }, memory->local_args_tuple_);
 
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars = apply(
         [&](auto&&... args) { return memory->f_(t, y_vec, msgs_, args...); },
-        local_args_tuple);
+        memory->local_args_tuple_);
 
     check_size_match("coupled_ode_system2", "dy_dt", f_y_t_vars.size(),
                      "states", N_);
@@ -331,7 +340,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     grad();
 
     apply([&](auto&&... args) { accumulate_adjoints(mu_dot.data(), args...); },
-          local_args_tuple);
+          memory->local_args_tuple_);
   }
 
   /**

From c81e6aa4e66749b87a875353e5080970c3a91237 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Sun, 3 Jan 2021 22:35:39 +0000
Subject: [PATCH 014/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 6a642975de3..4671b7fe535 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -79,9 +79,9 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   const Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1> y0_;
   const T_t0 t0_;
   const std::vector<T_ts> ts_;
-  //std::tuple<T_Args...> args_tuple_;
+  // std::tuple<T_Args...> args_tuple_;
   std::tuple<decltype(deep_copy_vars(std::declval<const T_Args&>()))...>
-  local_args_tuple_;
+      local_args_tuple_;
   std::tuple<plain_type_t<decltype(value_of(std::declval<const T_Args&>()))>...>
       value_of_args_tuple_;
   std::vector<Eigen::VectorXd> y_;
@@ -100,7 +100,7 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
         y0_(y0),
         t0_(t0),
         ts_(ts),
-        //args_tuple_(std::make_tuple(args...)),
+        // args_tuple_(std::make_tuple(args...)),
         local_args_tuple_(deep_copy_vars(args)...),
         value_of_args_tuple_(value_of(args)...),
         y_(ts_.size()),
@@ -324,7 +324,8 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     // The vars here do not live on the nested stack so must be zero'd
     // separately
-    apply([&](auto&&... args) { zero_adjoints(args...); }, memory->local_args_tuple_);
+    apply([&](auto&&... args) { zero_adjoints(args...); },
+          memory->local_args_tuple_);
 
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars = apply(
         [&](auto&&... args) { return memory->f_(t, y_vec, msgs_, args...); },

From fd9d5314d0e950b65dcfcedc1a4828b4bf503d13 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 4 Jan 2021 21:18:29 +0100
Subject: [PATCH 015/157] fix

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp         | 2 +-
 .../unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 4671b7fe535..81d1dc07fd5 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -469,7 +469,7 @@ class cvodes_integrator_adjoint_vari : public vari {
           std::vector<int> unused_temp{
               0, (check_finite(fun, "ode parameters and data", args), 0)...};
         },
-        memory->args_tuple_);
+        memory->local_args_tuple_);
 
     check_nonzero_size(fun, "times", ts);
     check_nonzero_size(fun, "initial state", y0);
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
index 77252b08890..d60bbe75d2e 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
@@ -64,7 +64,7 @@ class test_functor_double_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_adams(sho, y0, t0, ts, nullptr, omega)
+              ? stan::math::ode_bdf_adjoint(sho, y0, t0, ts, nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return ys[0](state);
@@ -132,7 +132,7 @@ class test_functor_var_double {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_adams(sho, y0, t0, ts, nullptr, omega)
+              ? stan::math::ode_bdf_adjoint(sho, y0, t0, ts, nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return ys[0](state);
@@ -200,7 +200,7 @@ class test_functor_var_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_adams(sho, y0, t0, ts, nullptr, omega)
+              ? stan::math::ode_bdf_adjoint(sho, y0, t0, ts, nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return ys[0](state);

From bd35bd9953cc791ddae473fc5e7f536d9a41ca42 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 4 Jan 2021 21:21:53 +0100
Subject: [PATCH 016/157] add plain_type_t

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 81d1dc07fd5..bb4c7aca5e7 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -80,7 +80,7 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   const T_t0 t0_;
   const std::vector<T_ts> ts_;
   // std::tuple<T_Args...> args_tuple_;
-  std::tuple<decltype(deep_copy_vars(std::declval<const T_Args&>()))...>
+  std::tuple<plain_type_t<decltype(deep_copy_vars(std::declval<const T_Args&>()))>...>
       local_args_tuple_;
   std::tuple<plain_type_t<decltype(value_of(std::declval<const T_Args&>()))>...>
       value_of_args_tuple_;

From e0a4e40f77480ed94eeb2e8031244bb67737dddb Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Mon, 4 Jan 2021 20:28:27 +0000
Subject: [PATCH 017/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index bb4c7aca5e7..cefad13ee07 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -80,7 +80,8 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   const T_t0 t0_;
   const std::vector<T_ts> ts_;
   // std::tuple<T_Args...> args_tuple_;
-  std::tuple<plain_type_t<decltype(deep_copy_vars(std::declval<const T_Args&>()))>...>
+  std::tuple<
+      plain_type_t<decltype(deep_copy_vars(std::declval<const T_Args&>()))>...>
       local_args_tuple_;
   std::tuple<plain_type_t<decltype(value_of(std::declval<const T_Args&>()))>...>
       value_of_args_tuple_;

From a15e58a3f02b763bb7a652254e5f9ea864d10514 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 4 Jan 2021 21:54:20 +0100
Subject: [PATCH 018/157] bump

---
 .../rev/functor/cvodes_integrator_adjoint.hpp |  2 +-
 .../functor/ode_bdf_adjoint_bench_test.cpp    | 41 +++++++++++--------
 2 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index cefad13ee07..ac7e94ea990 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -555,7 +555,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   }
 
   virtual void chain() {
-    // std::cout << "chain" << std::endl; <-- Good way to verify it's only
+    //std::cout << "chain" << std::endl; //<-- Good way to verify it's only
     //  being called once
     if (memory == NULL)
       return;
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
index e9d0389ca18..0cce87685b0 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
@@ -21,11 +21,13 @@ struct pkpd_rhs {
              const T8& ea50) const {
     Eigen::Matrix<stan::return_type_t<T1, T2, T3, T4, T5, T6, T7, T8>,
                   Eigen::Dynamic, 1>
-        dydt(4);
+        dydt(5);
 
-    dydt << -ka * y(0), +ka * y(0) - ke * y(1) - k12 * y(1) + k21 * y(2),
+    dydt << -ka * y(0),
+        +ka * y(0) - ke * y(1) - k12 * y(1) + k21 * y(2),
         +k12 * y(1) - k21 * y(2),
-        +kin - kout * (1.0 - y(1) / (y(1) + ea50)) * y(3);
+        +kin - kout * (1.0 - y(1) / (y(1) + ea50)) * y(3),
+        +kin - kout * (1.0 - y(2) / (y(2) + ea50)) * y(4); // pseudo pd
 
     return dydt;
   }
@@ -55,19 +57,19 @@ TEST(StanMathOdeBench, bdf) {
     var Q = lognormal_rng(true_Q, 4.0, base_rng);
     var V1 = lognormal_rng(true_V1, 2.0, base_rng);
     var V2 = lognormal_rng(true_V2, 2.0, base_rng);
-    var ka = true_ka;
-    var pd0 = true_pd0;
-    var kin = true_kin;
+    var ka = lognormal_rng(true_ka, 4.0, base_rng);
+    var pd0 = lognormal_rng(true_pd0, 2.0, base_rng);
+    var kin = lognormal_rng(true_kin, 4.0, base_rng);
     var kout = kin / pd0;
-    var ec50 = true_ec50;
+    var ec50 = lognormal_rng(true_ec50, 2.0, base_rng);
     var ea50 = ec50 * V1;
 
     var ke = CL / V1;
     var k12 = Q / V2;
     var k21 = k12 * V1 / V2;
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(4);
-    y0 << 4.0, 0.0, 0.0, pd0;
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(5);
+    y0 << 4.0, 0.0, 0.0, pd0, pd0 * 2.0;
 
     double t0 = 0.0;
 
@@ -77,8 +79,11 @@ TEST(StanMathOdeBench, bdf) {
 
     stan::math::grad();
   }
+
+  stan::math::recover_memory();
 }
 
+
 TEST(StanMathOdeBench, bdf_adjoint) {
   double true_CL = 8.0;
   double true_Q = 18.0;
@@ -103,26 +108,28 @@ TEST(StanMathOdeBench, bdf_adjoint) {
     var Q = lognormal_rng(true_Q, 4.0, base_rng);
     var V1 = lognormal_rng(true_V1, 2.0, base_rng);
     var V2 = lognormal_rng(true_V2, 2.0, base_rng);
-    var ka = true_ka;
-    var pd0 = true_pd0;
-    var kin = true_kin;
+    var ka = lognormal_rng(true_ka, 4.0, base_rng);
+    var pd0 = lognormal_rng(true_pd0, 2.0, base_rng);
+    var kin = lognormal_rng(true_kin, 4.0, base_rng);
     var kout = kin / pd0;
-    var ec50 = true_ec50;
+    var ec50 = lognormal_rng(true_ec50, 2.0, base_rng);
     var ea50 = ec50 * V1;
 
     var ke = CL / V1;
     var k12 = Q / V2;
     var k21 = k12 * V1 / V2;
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(4);
-    y0 << 4.0, 0.0, 0.0, pd0;
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(5);
+    y0 << 4.0, 0.0, 0.0, pd0, pd0 * 2.0;
 
     double t0 = 0.0;
 
     std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-        = ode_bdf_adjoint_tol(ode, y0, t0, ts, 1E-8, 1E-8, 10000, nullptr, ka,
-                              ke, k12, k21, kin, kout, ea50);
+        = ode_bdf_adjoint_tol(ode, y0, t0, ts, 1E-8, 1E-8, 10000, nullptr, ka, ke, k12,
+                      k21, kin, kout, ea50);
 
     stan::math::grad();
   }
+  stan::math::recover_memory();
 }
+

From 2adfcc00f01e15692e38de5dc42c52dc5d35e1ac Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 4 Jan 2021 22:16:40 +0100
Subject: [PATCH 019/157] route calls correctly

---
 stan/math/rev/functor/ode_adams.hpp       | 1 +
 stan/math/rev/functor/ode_bdf_adjoint.hpp | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/stan/math/rev/functor/ode_adams.hpp b/stan/math/rev/functor/ode_adams.hpp
index 894d07c5a90..fd35c4dceb8 100644
--- a/stan/math/rev/functor/ode_adams.hpp
+++ b/stan/math/rev/functor/ode_adams.hpp
@@ -3,6 +3,7 @@
 
 #include <stan/math/rev/meta.hpp>
 #include <stan/math/rev/functor/cvodes_integrator.hpp>
+#include <stan/math/rev/functor/ode_bdf_adjoint.hpp>
 #include <stan/math/prim/fun/Eigen.hpp>
 #include <ostream>
 #include <vector>
diff --git a/stan/math/rev/functor/ode_bdf_adjoint.hpp b/stan/math/rev/functor/ode_bdf_adjoint.hpp
index c93d978d57d..289b881a614 100644
--- a/stan/math/rev/functor/ode_bdf_adjoint.hpp
+++ b/stan/math/rev/functor/ode_bdf_adjoint.hpp
@@ -112,9 +112,9 @@ ode_bdf_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
                     const std::vector<T_ts>& ts, double relative_tolerance,
                     double absolute_tolerance, long int max_num_steps,
                     std::ostream* msgs, const T_Args&... args) {
-  return ode_bdf_tol_impl("ode_bdf_adjoint_tol", f, y0, t0, ts,
-                          relative_tolerance, absolute_tolerance, max_num_steps,
-                          msgs, args...);
+  return ode_bdf_adjoint_tol_impl("ode_bdf_adjoint_tol", f, y0, t0, ts,
+                                  relative_tolerance, absolute_tolerance, max_num_steps,
+                                  msgs, args...);
 }
 
 /**

From b15c7b1283084d37f589bf6066561a0ad46afda8 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 4 Jan 2021 22:23:11 +0100
Subject: [PATCH 020/157] bump

---
 .../math/rev/functor/ode_bdf_adjoint_bench_test.cpp | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
index 0cce87685b0..821b3afff71 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
@@ -21,13 +21,12 @@ struct pkpd_rhs {
              const T8& ea50) const {
     Eigen::Matrix<stan::return_type_t<T1, T2, T3, T4, T5, T6, T7, T8>,
                   Eigen::Dynamic, 1>
-        dydt(5);
+        dydt(4);
 
     dydt << -ka * y(0),
         +ka * y(0) - ke * y(1) - k12 * y(1) + k21 * y(2),
         +k12 * y(1) - k21 * y(2),
-        +kin - kout * (1.0 - y(1) / (y(1) + ea50)) * y(3),
-        +kin - kout * (1.0 - y(2) / (y(2) + ea50)) * y(4); // pseudo pd
+        +kin - kout * (1.0 - y(1) / (y(1) + ea50)) * y(3);
 
     return dydt;
   }
@@ -68,8 +67,8 @@ TEST(StanMathOdeBench, bdf) {
     var k12 = Q / V2;
     var k21 = k12 * V1 / V2;
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(5);
-    y0 << 4.0, 0.0, 0.0, pd0, pd0 * 2.0;
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(4);
+    y0 << 4.0, 0.0, 0.0, pd0;
 
     double t0 = 0.0;
 
@@ -119,8 +118,8 @@ TEST(StanMathOdeBench, bdf_adjoint) {
     var k12 = Q / V2;
     var k21 = k12 * V1 / V2;
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(5);
-    y0 << 4.0, 0.0, 0.0, pd0, pd0 * 2.0;
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(4);
+    y0 << 4.0, 0.0, 0.0, pd0;
 
     double t0 = 0.0;
 

From 47abeb8ede31667d45bd111b4b89cfc7b0544587 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Mon, 4 Jan 2021 21:35:16 +0000
Subject: [PATCH 021/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp   |  2 +-
 stan/math/rev/functor/ode_bdf_adjoint.hpp             |  4 ++--
 .../math/rev/functor/ode_bdf_adjoint_bench_test.cpp   | 11 ++++-------
 3 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index ac7e94ea990..88281938023 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -555,7 +555,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   }
 
   virtual void chain() {
-    //std::cout << "chain" << std::endl; //<-- Good way to verify it's only
+    // std::cout << "chain" << std::endl; //<-- Good way to verify it's only
     //  being called once
     if (memory == NULL)
       return;
diff --git a/stan/math/rev/functor/ode_bdf_adjoint.hpp b/stan/math/rev/functor/ode_bdf_adjoint.hpp
index 289b881a614..7659695688e 100644
--- a/stan/math/rev/functor/ode_bdf_adjoint.hpp
+++ b/stan/math/rev/functor/ode_bdf_adjoint.hpp
@@ -113,8 +113,8 @@ ode_bdf_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
                     double absolute_tolerance, long int max_num_steps,
                     std::ostream* msgs, const T_Args&... args) {
   return ode_bdf_adjoint_tol_impl("ode_bdf_adjoint_tol", f, y0, t0, ts,
-                                  relative_tolerance, absolute_tolerance, max_num_steps,
-                                  msgs, args...);
+                                  relative_tolerance, absolute_tolerance,
+                                  max_num_steps, msgs, args...);
 }
 
 /**
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
index 0cce87685b0..75511b6d839 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
@@ -23,11 +23,10 @@ struct pkpd_rhs {
                   Eigen::Dynamic, 1>
         dydt(5);
 
-    dydt << -ka * y(0),
-        +ka * y(0) - ke * y(1) - k12 * y(1) + k21 * y(2),
+    dydt << -ka * y(0), +ka * y(0) - ke * y(1) - k12 * y(1) + k21 * y(2),
         +k12 * y(1) - k21 * y(2),
         +kin - kout * (1.0 - y(1) / (y(1) + ea50)) * y(3),
-        +kin - kout * (1.0 - y(2) / (y(2) + ea50)) * y(4); // pseudo pd
+        +kin - kout * (1.0 - y(2) / (y(2) + ea50)) * y(4);  // pseudo pd
 
     return dydt;
   }
@@ -83,7 +82,6 @@ TEST(StanMathOdeBench, bdf) {
   stan::math::recover_memory();
 }
 
-
 TEST(StanMathOdeBench, bdf_adjoint) {
   double true_CL = 8.0;
   double true_Q = 18.0;
@@ -125,11 +123,10 @@ TEST(StanMathOdeBench, bdf_adjoint) {
     double t0 = 0.0;
 
     std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-        = ode_bdf_adjoint_tol(ode, y0, t0, ts, 1E-8, 1E-8, 10000, nullptr, ka, ke, k12,
-                      k21, kin, kout, ea50);
+        = ode_bdf_adjoint_tol(ode, y0, t0, ts, 1E-8, 1E-8, 10000, nullptr, ka,
+                              ke, k12, k21, kin, kout, ea50);
 
     stan::math::grad();
   }
   stan::math::recover_memory();
 }
-

From cfcc616a3120ca64aa60bb1b7baae8e415df2873 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 5 Jan 2021 09:39:54 +0100
Subject: [PATCH 022/157] too much work error message

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 48 +++++++++++++++----
 .../functor/ode_bdf_adjoint_bench_test.cpp    |  2 +-
 2 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 88281938023..bf392104035 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -534,14 +534,33 @@ class cvodes_integrator_adjoint_vari : public vari {
       if (t_final != t_init) {
         if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
           int ncheck;
-          check_flag_sundials(
-              CVodeF(memory->cvodes_mem_, t_final, memory->nv_state_, &t_init,
-                     CV_NORMAL, &ncheck),
-              "CVodeF");
+          
+          int error_code
+              = CVodeF(memory->cvodes_mem_, t_final, memory->nv_state_, &t_init,
+                       CV_NORMAL, &ncheck);
+
+          if (error_code == CV_TOO_MUCH_WORK) {
+            throw_domain_error(function_name_, "", t_final,
+                               "Failed to integrate to next output time (",
+                               ") in less than max_num_steps steps");
+          } else {
+            check_flag_sundials(error_code, "CVodeF");
+          }
+          
         } else {
-          check_flag_sundials(CVode(memory->cvodes_mem_, t_final,
-                                    memory->nv_state_, &t_init, CV_NORMAL),
-                              "CVode");
+
+          int error_code
+              = CVode(memory->cvodes_mem_, t_final, memory->nv_state_, &t_init,
+                       CV_NORMAL);
+
+          if (error_code == CV_TOO_MUCH_WORK) {
+            throw_domain_error(function_name_, "", t_final,
+                               "Failed to integrate to next output time (",
+                               ") in less than max_num_steps steps");
+          } else {
+            check_flag_sundials(error_code, "CVode");
+          }
+          
         }
       }
 
@@ -671,8 +690,19 @@ class cvodes_integrator_adjoint_vari : public vari {
                 "CVodeQuadReInitB");
           }
 
-          check_flag_sundials(CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL),
-                              "CVodeB");
+          int error_code
+              = CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL);
+
+          if (error_code == CV_TOO_MUCH_WORK) {
+            throw_domain_error(function_name_, "", t_final,
+                               "Failed to integrate backward to output time (",
+                               ") in less than max_num_steps steps");
+          } else {
+            check_flag_sundials(error_code, "CVodeB");
+          }
+
+          //check_flag_sundials(CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL),
+          //                    "CVodeB");
 
           check_flag_sundials(
               CVodeGetB(memory->cvodes_mem_, indexB, &t_init, nv_state_sens),
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
index a7315fc7ccc..603221c053e 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
@@ -122,7 +122,7 @@ TEST(StanMathOdeBench, bdf_adjoint) {
     double t0 = 0.0;
 
     std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-        = ode_bdf_adjoint_tol(ode, y0, t0, ts, 1E-8, 1E-8, 10000, nullptr, ka,
+        = ode_bdf_adjoint_tol(ode, y0, t0, ts, 1E-12, 1E-12, 10000, nullptr, ka,
                               ke, k12, k21, kin, kout, ea50);
 
     stan::math::grad();

From 83d7c41a50c1149ccab85c8f2a3d7b562a729170 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Tue, 5 Jan 2021 08:45:42 +0000
Subject: [PATCH 023/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp   | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index bf392104035..d252071b4e7 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -534,7 +534,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       if (t_final != t_init) {
         if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
           int ncheck;
-          
+
           int error_code
               = CVodeF(memory->cvodes_mem_, t_final, memory->nv_state_, &t_init,
                        CV_NORMAL, &ncheck);
@@ -546,12 +546,10 @@ class cvodes_integrator_adjoint_vari : public vari {
           } else {
             check_flag_sundials(error_code, "CVodeF");
           }
-          
-        } else {
 
-          int error_code
-              = CVode(memory->cvodes_mem_, t_final, memory->nv_state_, &t_init,
-                       CV_NORMAL);
+        } else {
+          int error_code = CVode(memory->cvodes_mem_, t_final,
+                                 memory->nv_state_, &t_init, CV_NORMAL);
 
           if (error_code == CV_TOO_MUCH_WORK) {
             throw_domain_error(function_name_, "", t_final,
@@ -560,7 +558,6 @@ class cvodes_integrator_adjoint_vari : public vari {
           } else {
             check_flag_sundials(error_code, "CVode");
           }
-          
         }
       }
 
@@ -690,8 +687,7 @@ class cvodes_integrator_adjoint_vari : public vari {
                 "CVodeQuadReInitB");
           }
 
-          int error_code
-              = CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL);
+          int error_code = CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL);
 
           if (error_code == CV_TOO_MUCH_WORK) {
             throw_domain_error(function_name_, "", t_final,
@@ -701,7 +697,8 @@ class cvodes_integrator_adjoint_vari : public vari {
             check_flag_sundials(error_code, "CVodeB");
           }
 
-          //check_flag_sundials(CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL),
+          // check_flag_sundials(CVodeB(memory->cvodes_mem_, t_final,
+          // CV_NORMAL),
           //                    "CVodeB");
 
           check_flag_sundials(

From 78f87e052ad58d2e34f9f0bee2233febbe020879 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 5 Jan 2021 21:03:53 +0100
Subject: [PATCH 024/157] bump

---
 .../rev/functor/cvodes_integrator_adjoint.hpp |  28 ++++-
 stan/math/rev/functor/ode_adams.hpp           |  18 +++-
 stan/math/rev/functor/ode_bdf_adjoint.hpp     |  25 ++++-
 .../functor/ode_bdf_adjoint_bench_test.cpp    | 101 ++++++++----------
 4 files changed, 101 insertions(+), 71 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index bf392104035..6a79cda1e93 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -159,6 +159,9 @@ class cvodes_integrator_adjoint_vari : public vari {
   std::ostream* msgs_;
   double relative_tolerance_;
   double absolute_tolerance_;
+  double absolute_tolerance_B_;
+  double absolute_tolerance_QB_;
+  long int steps_checkpoint_;
   long int max_num_steps_;
 
   const size_t t0_vars_;
@@ -425,6 +428,9 @@ class cvodes_integrator_adjoint_vari : public vari {
                                  const std::vector<T_ts>& ts,
                                  double relative_tolerance,
                                  double absolute_tolerance,
+                                 double absolute_tolerance_B,
+                                 double absolute_tolerance_QB,
+                                 long int steps_checkpoint,
                                  long int max_num_steps, std::ostream* msgs,
                                  const T_Args&... args)
       : function_name_(function_name),
@@ -435,6 +441,9 @@ class cvodes_integrator_adjoint_vari : public vari {
         msgs_(msgs),
         relative_tolerance_(relative_tolerance),
         absolute_tolerance_(absolute_tolerance),
+        absolute_tolerance_B_(absolute_tolerance_B),
+        absolute_tolerance_QB_(absolute_tolerance_QB),
+        steps_checkpoint_(steps_checkpoint),
         max_num_steps_(max_num_steps),
         t0_vars_(count_vars(t0)),
         ts_vars_(count_vars(ts)),
@@ -478,7 +487,19 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_less(fun, "initial time", t0, ts[0]);
     check_positive_finite(fun, "relative_tolerance", relative_tolerance_);
     check_positive_finite(fun, "absolute_tolerance", absolute_tolerance_);
+    check_positive_finite(fun, "absolute_tolerance_B", absolute_tolerance_B_);
+    check_positive_finite(fun, "absolute_tolerance_QB", absolute_tolerance_QB_);
     check_positive(fun, "max_num_steps", max_num_steps_);
+    check_positive(fun, "steps_checkpoint", steps_checkpoint_);
+
+    /*
+    std::cout << "relative_tolerance = " << relative_tolerance << std::endl;
+    std::cout << "absolute_tolerance = " << absolute_tolerance << std::endl;
+    std::cout << "absolute_tolerance_B = " << absolute_tolerance_B << std::endl;
+    std::cout << "absolute_tolerance_QB = " << absolute_tolerance_QB << std::endl;
+    std::cout << "max_num_steps = " << max_num_steps << std::endl;
+    std::cout << "steps_checkpoint = " << steps_checkpoint << std::endl;
+    */
   }
 
   ~cvodes_integrator_adjoint_vari() {}
@@ -523,7 +544,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     // initialize forward sensitivity system of CVODES as needed
     if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
-      check_flag_sundials(CVodeAdjInit(memory->cvodes_mem_, 25, CV_HERMITE),
+      check_flag_sundials(CVodeAdjInit(memory->cvodes_mem_, steps_checkpoint_, CV_HERMITE),
                           "CVodeAdjInit");
     }
 
@@ -622,7 +643,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
       check_flag_sundials(
           CVodeSStolerancesB(memory->cvodes_mem_, indexB, relative_tolerance_,
-                             absolute_tolerance_),
+                             absolute_tolerance_B_),
           "CVodeSStolerancesB");
 
       check_flag_sundials(
@@ -646,12 +667,13 @@ class cvodes_integrator_adjoint_vari : public vari {
 
         check_flag_sundials(
             CVodeQuadSStolerancesB(memory->cvodes_mem_, indexB,
-                                   relative_tolerance_, absolute_tolerance_),
+                                   relative_tolerance_, absolute_tolerance_QB_),
             "CVodeQuadSStolerancesB");
 
         check_flag_sundials(
             CVodeSetQuadErrConB(memory->cvodes_mem_, indexB, SUNTRUE),
             "CVodeSetQuadErrConB");
+
       }
 
       // At every time step, collect the adjoints from the output
diff --git a/stan/math/rev/functor/ode_adams.hpp b/stan/math/rev/functor/ode_adams.hpp
index fd35c4dceb8..0f30ba279ba 100644
--- a/stan/math/rev/functor/ode_adams.hpp
+++ b/stan/math/rev/functor/ode_adams.hpp
@@ -53,7 +53,10 @@ ode_adams_tol_impl(const char* function_name, const F& f, const T_y0& y0,
                    const T_t0& t0, const std::vector<T_ts>& ts,
                    double relative_tolerance, double absolute_tolerance,
                    long int max_num_steps,  // NOLINT(runtime/int)
-                   std::ostream* msgs, const T_Args&... args) {
+                   std::ostream* msgs,
+                   double absolute_tolerance_B, double absolute_tolerance_QB,
+                   long int steps_checkpoint,
+                   const T_Args&... args) {
   /*
   const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
   return apply(
@@ -67,7 +70,7 @@ ode_adams_tol_impl(const char* function_name, const F& f, const T_y0& y0,
       args_ref_tuple);
   */
   return ode_bdf_adjoint_tol(f, y0, t0, ts, relative_tolerance,
-                             absolute_tolerance, max_num_steps, msgs, args...);
+                             absolute_tolerance, max_num_steps, msgs, absolute_tolerance_B, absolute_tolerance_QB, steps_checkpoint, args...);
 }
 
 /**
@@ -112,8 +115,12 @@ ode_adams_tol(const F& f, const T_y0& y0, const T_t0& t0,
               double absolute_tolerance,
               long int max_num_steps,  // NOLINT(runtime/int)
               std::ostream* msgs, const T_Args&... args) {
+  double absolute_tolerance_B = absolute_tolerance * 100.0;
+  double absolute_tolerance_QB = absolute_tolerance_B * 10.0;
+  long int steps_checkpoint = 100;
+
   return ode_adams_tol_impl("ode_adams_tol", f, y0, t0, ts, relative_tolerance,
-                            absolute_tolerance, max_num_steps, msgs, args...);
+                            absolute_tolerance, max_num_steps, msgs, absolute_tolerance_B, absolute_tolerance_QB, steps_checkpoint, args...);
 }
 
 /**
@@ -156,9 +163,12 @@ ode_adams(const F& f, const T_y0& y0, const T_t0& t0,
   double relative_tolerance = 1e-10;
   double absolute_tolerance = 1e-10;
   long int max_num_steps = 1e8;  // NOLINT(runtime/int)
+  double absolute_tolerance_B = absolute_tolerance * 100.0;
+  double absolute_tolerance_QB = absolute_tolerance_B * 10.0;
+  long int steps_checkpoint = 100;
 
   return ode_adams_tol_impl("ode_adams", f, y0, t0, ts, relative_tolerance,
-                            absolute_tolerance, max_num_steps, msgs, args...);
+                            absolute_tolerance, max_num_steps, msgs, absolute_tolerance_B, absolute_tolerance_QB, steps_checkpoint, args...);
 }
 
 }  // namespace math
diff --git a/stan/math/rev/functor/ode_bdf_adjoint.hpp b/stan/math/rev/functor/ode_bdf_adjoint.hpp
index 7659695688e..cda12d85c54 100644
--- a/stan/math/rev/functor/ode_bdf_adjoint.hpp
+++ b/stan/math/rev/functor/ode_bdf_adjoint.hpp
@@ -51,6 +51,8 @@ ode_bdf_adjoint_tol_impl(const char* function_name, const F& f, const T_y0& y0,
                          const T_t0& t0, const std::vector<T_ts>& ts,
                          double relative_tolerance, double absolute_tolerance,
                          long int max_num_steps, std::ostream* msgs,
+                         double absolute_tolerance_B, double absolute_tolerance_QB,
+                         long int steps_checkpoint,
                          const T_Args&... args) {
   /*
   const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
@@ -66,8 +68,9 @@ ode_bdf_adjoint_tol_impl(const char* function_name, const F& f, const T_y0& y0,
   auto integrator
       = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0, T_t0,
                                                        T_ts, T_Args...>(
-          function_name, f, y0, t0, ts, relative_tolerance, absolute_tolerance,
-          max_num_steps, msgs, args...);
+          function_name, f, y0, t0, ts,
+          relative_tolerance, absolute_tolerance, absolute_tolerance_B, absolute_tolerance_QB,
+          steps_checkpoint, max_num_steps, msgs, args...);
   return (*integrator)();
 }
 
@@ -111,10 +114,16 @@ std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
 ode_bdf_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
                     const std::vector<T_ts>& ts, double relative_tolerance,
                     double absolute_tolerance, long int max_num_steps,
-                    std::ostream* msgs, const T_Args&... args) {
+                    std::ostream* msgs,
+                    double absolute_tolerance_B, double absolute_tolerance_QB,
+                    long int steps_checkpoint,
+                    const T_Args&... args) {
   return ode_bdf_adjoint_tol_impl("ode_bdf_adjoint_tol", f, y0, t0, ts,
                                   relative_tolerance, absolute_tolerance,
-                                  max_num_steps, msgs, args...);
+                                  max_num_steps, msgs,
+                                  absolute_tolerance_B, absolute_tolerance_QB,
+                                  steps_checkpoint,
+                                  args...);
 }
 
 /**
@@ -160,11 +169,17 @@ ode_bdf_adjoint(const F& f, const T_y0& y0, const T_t0& t0,
                 const T_Args&... args) {
   double relative_tolerance = 1e-10;
   double absolute_tolerance = 1e-10;
+  double absolute_tolerance_B = 1e-8;
+  double absolute_tolerance_QB = 1e-6;
+  long int steps_checkpoint = 150;
   long int max_num_steps = 1e8;
 
   return ode_bdf_adjoint_tol_impl("ode_bdf_adjoint", f, y0, t0, ts,
                                   relative_tolerance, absolute_tolerance,
-                                  max_num_steps, msgs, args...);
+                                  max_num_steps, msgs,
+                                  absolute_tolerance_B, absolute_tolerance_QB,
+                                  steps_checkpoint,
+                                  args...);
 }
 
 }  // namespace math
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
index 603221c053e..9e5e984f9e3 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
@@ -31,7 +31,8 @@ struct pkpd_rhs {
   }
 };
 
-TEST(StanMathOdeBench, bdf) {
+
+void run_benchmark(int adjoint_integrator) {
   double true_CL = 8.0;
   double true_Q = 18.0;
   double true_V1 = 10.0;
@@ -48,18 +49,26 @@ TEST(StanMathOdeBench, bdf) {
 
   boost::ecuyer1988 base_rng(563646);
 
+  double log_sigma = 10.0;
+  double abs_tol = 1e-8;
+  double abs_tol_B = abs_tol * 100.0;
+  double abs_tol_QB = abs_tol_B * 10.0;
+  double rel_tol = 1e-6;
+  int steps_checkpoint = 100;
+  int max_num_steps = 100000;
+
   for (std::size_t i = 0; i != 200; i++) {
     stan::math::nested_rev_autodiff nested;
 
-    var CL = lognormal_rng(true_CL, 4.0, base_rng);
-    var Q = lognormal_rng(true_Q, 4.0, base_rng);
-    var V1 = lognormal_rng(true_V1, 2.0, base_rng);
-    var V2 = lognormal_rng(true_V2, 2.0, base_rng);
-    var ka = lognormal_rng(true_ka, 4.0, base_rng);
-    var pd0 = lognormal_rng(true_pd0, 2.0, base_rng);
-    var kin = lognormal_rng(true_kin, 4.0, base_rng);
+    var CL = lognormal_rng(true_CL, log_sigma, base_rng);
+    var Q = lognormal_rng(true_Q, log_sigma, base_rng);
+    var V1 = lognormal_rng(true_V1, log_sigma, base_rng);
+    var V2 = lognormal_rng(true_V2, log_sigma, base_rng);
+    var ka = lognormal_rng(true_ka, log_sigma, base_rng);
+    var pd0 = lognormal_rng(true_pd0, log_sigma, base_rng);
+    var kin = lognormal_rng(true_kin, log_sigma, base_rng);
     var kout = kin / pd0;
-    var ec50 = lognormal_rng(true_ec50, 2.0, base_rng);
+    var ec50 = lognormal_rng(true_ec50, log_sigma, base_rng);
     var ea50 = ec50 * V1;
 
     var ke = CL / V1;
@@ -71,61 +80,35 @@ TEST(StanMathOdeBench, bdf) {
 
     double t0 = 0.0;
 
-    std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-        = ode_bdf_tol(ode, y0, t0, ts, 1E-8, 1E-8, 10000, nullptr, ka, ke, k12,
-                      k21, kin, kout, ea50);
-
-    stan::math::grad();
+    try {
+      if(adjoint_integrator) {
+        std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
+            = ode_bdf_adjoint_tol(ode, y0, t0, ts, rel_tol, abs_tol, max_num_steps, nullptr,
+                                  abs_tol_B, abs_tol_QB, steps_checkpoint,
+                                  ka, ke, k12, k21, kin, kout, ea50);
+
+        stan::math::grad();
+      } else {
+        std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
+            = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol, max_num_steps, nullptr, ka,
+                        ke, k12, k21, kin, kout, ea50);
+        
+        stan::math::grad();
+      }
+    } catch(std::exception& exc) {
+      std::cout << "oops, keep going please!" << std::endl;
+      std::cerr << exc.what() << std::endl;
+    }
   }
-
   stan::math::recover_memory();
 }
 
-TEST(StanMathOdeBench, bdf_adjoint) {
-  double true_CL = 8.0;
-  double true_Q = 18.0;
-  double true_V1 = 10.0;
-  double true_V2 = 14.0;
-  double true_ka = log(2.0) / 2.0;
-  double true_pd0 = 1.0;
-  double true_kin = 4.0;
-  double true_kout = true_kin / true_pd0;
-  double true_ec50 = 0.01;
-  std::vector<double> ts{1., 2.0, 4.0, 8.0, 16.0, 32.0, 64.0};
-  std::size_t ts_size = ts.size();
-
-  pkpd_rhs ode;
-
-  boost::ecuyer1988 base_rng(563646);
-
-  for (std::size_t i = 0; i != 200; i++) {
-    stan::math::nested_rev_autodiff nested;
-
-    var CL = lognormal_rng(true_CL, 4.0, base_rng);
-    var Q = lognormal_rng(true_Q, 4.0, base_rng);
-    var V1 = lognormal_rng(true_V1, 2.0, base_rng);
-    var V2 = lognormal_rng(true_V2, 2.0, base_rng);
-    var ka = lognormal_rng(true_ka, 4.0, base_rng);
-    var pd0 = lognormal_rng(true_pd0, 2.0, base_rng);
-    var kin = lognormal_rng(true_kin, 4.0, base_rng);
-    var kout = kin / pd0;
-    var ec50 = lognormal_rng(true_ec50, 2.0, base_rng);
-    var ea50 = ec50 * V1;
-
-    var ke = CL / V1;
-    var k12 = Q / V2;
-    var k21 = k12 * V1 / V2;
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(4);
-    y0 << 4.0, 0.0, 0.0, pd0;
-
-    double t0 = 0.0;
+TEST(StanMathOdeBench, bdf) {
+  run_benchmark(0);
+}
 
-    std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-        = ode_bdf_adjoint_tol(ode, y0, t0, ts, 1E-12, 1E-12, 10000, nullptr, ka,
-                              ke, k12, k21, kin, kout, ea50);
 
-    stan::math::grad();
-  }
-  stan::math::recover_memory();
+TEST(StanMathOdeBench, bdf_adjoint) {
+  run_benchmark(1);
 }

From 6be8f54d0372a704531aa42bf8bc45b868db9e63 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Tue, 5 Jan 2021 20:11:07 +0000
Subject: [PATCH 025/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 26 ++++++-------
 stan/math/rev/functor/ode_adams.hpp           | 17 ++++++---
 stan/math/rev/functor/ode_bdf_adjoint.hpp     | 37 ++++++++-----------
 .../functor/ode_bdf_adjoint_bench_test.cpp    | 28 ++++++--------
 4 files changed, 49 insertions(+), 59 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 234ae83dea8..eda16a10fc8 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -423,16 +423,12 @@ class cvodes_integrator_adjoint_vari : public vari {
    * same size as the state variable, corresponding to a time in ts.
    */
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
-  cvodes_integrator_adjoint_vari(const char* function_name, const F& f,
-                                 const T_y0& y0, const T_t0& t0,
-                                 const std::vector<T_ts>& ts,
-                                 double relative_tolerance,
-                                 double absolute_tolerance,
-                                 double absolute_tolerance_B,
-                                 double absolute_tolerance_QB,
-                                 long int steps_checkpoint,
-                                 long int max_num_steps, std::ostream* msgs,
-                                 const T_Args&... args)
+  cvodes_integrator_adjoint_vari(
+      const char* function_name, const F& f, const T_y0& y0, const T_t0& t0,
+      const std::vector<T_ts>& ts, double relative_tolerance,
+      double absolute_tolerance, double absolute_tolerance_B,
+      double absolute_tolerance_QB, long int steps_checkpoint,
+      long int max_num_steps, std::ostream* msgs, const T_Args&... args)
       : function_name_(function_name),
         vari(NOT_A_NUMBER),
         N_(y0.size()),
@@ -496,8 +492,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     std::cout << "relative_tolerance = " << relative_tolerance << std::endl;
     std::cout << "absolute_tolerance = " << absolute_tolerance << std::endl;
     std::cout << "absolute_tolerance_B = " << absolute_tolerance_B << std::endl;
-    std::cout << "absolute_tolerance_QB = " << absolute_tolerance_QB << std::endl;
-    std::cout << "max_num_steps = " << max_num_steps << std::endl;
+    std::cout << "absolute_tolerance_QB = " << absolute_tolerance_QB <<
+    std::endl; std::cout << "max_num_steps = " << max_num_steps << std::endl;
     std::cout << "steps_checkpoint = " << steps_checkpoint << std::endl;
     */
   }
@@ -544,8 +540,9 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     // initialize forward sensitivity system of CVODES as needed
     if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
-      check_flag_sundials(CVodeAdjInit(memory->cvodes_mem_, steps_checkpoint_, CV_HERMITE),
-                          "CVodeAdjInit");
+      check_flag_sundials(
+          CVodeAdjInit(memory->cvodes_mem_, steps_checkpoint_, CV_HERMITE),
+          "CVodeAdjInit");
     }
 
     double t_init = t0_dbl;
@@ -670,7 +667,6 @@ class cvodes_integrator_adjoint_vari : public vari {
         check_flag_sundials(
             CVodeSetQuadErrConB(memory->cvodes_mem_, indexB, SUNTRUE),
             "CVodeSetQuadErrConB");
-
       }
 
       // At every time step, collect the adjoints from the output
diff --git a/stan/math/rev/functor/ode_adams.hpp b/stan/math/rev/functor/ode_adams.hpp
index 0f30ba279ba..622bebbf63f 100644
--- a/stan/math/rev/functor/ode_adams.hpp
+++ b/stan/math/rev/functor/ode_adams.hpp
@@ -53,9 +53,8 @@ ode_adams_tol_impl(const char* function_name, const F& f, const T_y0& y0,
                    const T_t0& t0, const std::vector<T_ts>& ts,
                    double relative_tolerance, double absolute_tolerance,
                    long int max_num_steps,  // NOLINT(runtime/int)
-                   std::ostream* msgs,
-                   double absolute_tolerance_B, double absolute_tolerance_QB,
-                   long int steps_checkpoint,
+                   std::ostream* msgs, double absolute_tolerance_B,
+                   double absolute_tolerance_QB, long int steps_checkpoint,
                    const T_Args&... args) {
   /*
   const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
@@ -70,7 +69,9 @@ ode_adams_tol_impl(const char* function_name, const F& f, const T_y0& y0,
       args_ref_tuple);
   */
   return ode_bdf_adjoint_tol(f, y0, t0, ts, relative_tolerance,
-                             absolute_tolerance, max_num_steps, msgs, absolute_tolerance_B, absolute_tolerance_QB, steps_checkpoint, args...);
+                             absolute_tolerance, max_num_steps, msgs,
+                             absolute_tolerance_B, absolute_tolerance_QB,
+                             steps_checkpoint, args...);
 }
 
 /**
@@ -120,7 +121,9 @@ ode_adams_tol(const F& f, const T_y0& y0, const T_t0& t0,
   long int steps_checkpoint = 100;
 
   return ode_adams_tol_impl("ode_adams_tol", f, y0, t0, ts, relative_tolerance,
-                            absolute_tolerance, max_num_steps, msgs, absolute_tolerance_B, absolute_tolerance_QB, steps_checkpoint, args...);
+                            absolute_tolerance, max_num_steps, msgs,
+                            absolute_tolerance_B, absolute_tolerance_QB,
+                            steps_checkpoint, args...);
 }
 
 /**
@@ -168,7 +171,9 @@ ode_adams(const F& f, const T_y0& y0, const T_t0& t0,
   long int steps_checkpoint = 100;
 
   return ode_adams_tol_impl("ode_adams", f, y0, t0, ts, relative_tolerance,
-                            absolute_tolerance, max_num_steps, msgs, absolute_tolerance_B, absolute_tolerance_QB, steps_checkpoint, args...);
+                            absolute_tolerance, max_num_steps, msgs,
+                            absolute_tolerance_B, absolute_tolerance_QB,
+                            steps_checkpoint, args...);
 }
 
 }  // namespace math
diff --git a/stan/math/rev/functor/ode_bdf_adjoint.hpp b/stan/math/rev/functor/ode_bdf_adjoint.hpp
index cda12d85c54..0352753c3ce 100644
--- a/stan/math/rev/functor/ode_bdf_adjoint.hpp
+++ b/stan/math/rev/functor/ode_bdf_adjoint.hpp
@@ -51,9 +51,9 @@ ode_bdf_adjoint_tol_impl(const char* function_name, const F& f, const T_y0& y0,
                          const T_t0& t0, const std::vector<T_ts>& ts,
                          double relative_tolerance, double absolute_tolerance,
                          long int max_num_steps, std::ostream* msgs,
-                         double absolute_tolerance_B, double absolute_tolerance_QB,
-                         long int steps_checkpoint,
-                         const T_Args&... args) {
+                         double absolute_tolerance_B,
+                         double absolute_tolerance_QB,
+                         long int steps_checkpoint, const T_Args&... args) {
   /*
   const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
   return apply(
@@ -68,9 +68,9 @@ ode_bdf_adjoint_tol_impl(const char* function_name, const F& f, const T_y0& y0,
   auto integrator
       = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0, T_t0,
                                                        T_ts, T_Args...>(
-          function_name, f, y0, t0, ts,
-          relative_tolerance, absolute_tolerance, absolute_tolerance_B, absolute_tolerance_QB,
-          steps_checkpoint, max_num_steps, msgs, args...);
+          function_name, f, y0, t0, ts, relative_tolerance, absolute_tolerance,
+          absolute_tolerance_B, absolute_tolerance_QB, steps_checkpoint,
+          max_num_steps, msgs, args...);
   return (*integrator)();
 }
 
@@ -114,16 +114,13 @@ std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
 ode_bdf_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
                     const std::vector<T_ts>& ts, double relative_tolerance,
                     double absolute_tolerance, long int max_num_steps,
-                    std::ostream* msgs,
-                    double absolute_tolerance_B, double absolute_tolerance_QB,
-                    long int steps_checkpoint,
+                    std::ostream* msgs, double absolute_tolerance_B,
+                    double absolute_tolerance_QB, long int steps_checkpoint,
                     const T_Args&... args) {
-  return ode_bdf_adjoint_tol_impl("ode_bdf_adjoint_tol", f, y0, t0, ts,
-                                  relative_tolerance, absolute_tolerance,
-                                  max_num_steps, msgs,
-                                  absolute_tolerance_B, absolute_tolerance_QB,
-                                  steps_checkpoint,
-                                  args...);
+  return ode_bdf_adjoint_tol_impl(
+      "ode_bdf_adjoint_tol", f, y0, t0, ts, relative_tolerance,
+      absolute_tolerance, max_num_steps, msgs, absolute_tolerance_B,
+      absolute_tolerance_QB, steps_checkpoint, args...);
 }
 
 /**
@@ -174,12 +171,10 @@ ode_bdf_adjoint(const F& f, const T_y0& y0, const T_t0& t0,
   long int steps_checkpoint = 150;
   long int max_num_steps = 1e8;
 
-  return ode_bdf_adjoint_tol_impl("ode_bdf_adjoint", f, y0, t0, ts,
-                                  relative_tolerance, absolute_tolerance,
-                                  max_num_steps, msgs,
-                                  absolute_tolerance_B, absolute_tolerance_QB,
-                                  steps_checkpoint,
-                                  args...);
+  return ode_bdf_adjoint_tol_impl(
+      "ode_bdf_adjoint", f, y0, t0, ts, relative_tolerance, absolute_tolerance,
+      max_num_steps, msgs, absolute_tolerance_B, absolute_tolerance_QB,
+      steps_checkpoint, args...);
 }
 
 }  // namespace math
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
index 9e5e984f9e3..3184ff35179 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
@@ -31,7 +31,6 @@ struct pkpd_rhs {
   }
 };
 
-
 void run_benchmark(int adjoint_integrator) {
   double true_CL = 8.0;
   double true_Q = 18.0;
@@ -81,21 +80,22 @@ void run_benchmark(int adjoint_integrator) {
     double t0 = 0.0;
 
     try {
-      if(adjoint_integrator) {
+      if (adjoint_integrator) {
         std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-            = ode_bdf_adjoint_tol(ode, y0, t0, ts, rel_tol, abs_tol, max_num_steps, nullptr,
-                                  abs_tol_B, abs_tol_QB, steps_checkpoint,
-                                  ka, ke, k12, k21, kin, kout, ea50);
+            = ode_bdf_adjoint_tol(ode, y0, t0, ts, rel_tol, abs_tol,
+                                  max_num_steps, nullptr, abs_tol_B, abs_tol_QB,
+                                  steps_checkpoint, ka, ke, k12, k21, kin, kout,
+                                  ea50);
 
         stan::math::grad();
       } else {
         std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-            = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol, max_num_steps, nullptr, ka,
-                        ke, k12, k21, kin, kout, ea50);
-        
+            = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol, max_num_steps,
+                          nullptr, ka, ke, k12, k21, kin, kout, ea50);
+
         stan::math::grad();
       }
-    } catch(std::exception& exc) {
+    } catch (std::exception& exc) {
       std::cout << "oops, keep going please!" << std::endl;
       std::cerr << exc.what() << std::endl;
     }
@@ -103,12 +103,6 @@ void run_benchmark(int adjoint_integrator) {
   stan::math::recover_memory();
 }
 
+TEST(StanMathOdeBench, bdf) { run_benchmark(0); }
 
-TEST(StanMathOdeBench, bdf) {
-  run_benchmark(0);
-}
-
-
-TEST(StanMathOdeBench, bdf_adjoint) {
-  run_benchmark(1);
-}
+TEST(StanMathOdeBench, bdf_adjoint) { run_benchmark(1); }

From 6fff468618e728ce7982a64ad2f4a9671e497ead Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Wed, 6 Jan 2021 13:37:41 +0100
Subject: [PATCH 026/157] add maximal number of steps option to backward solve

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 31 +++++++++++++++----
 stan/math/rev/functor/ode_adams.hpp           |  4 +--
 2 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 234ae83dea8..113f198d3c1 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -618,6 +618,18 @@ class cvodes_integrator_adjoint_vari : public vari {
     SUNMatrix AB_ = SUNDenseMatrix(N_, N_);
     SUNLinearSolver LSB_ = SUNDenseLinearSolver(nv_state_sens, AB_);
 
+    /* check these if needed 
+      flag = CVodeSetUserDataB(cvode_mem, which, user_dataB);
+      flag = CVodeSetMaxOrdB(cvode_mem, which, maxordB);
+      flag = CVodeSetMaxNumStepsB(cvode_mem, which, mxstepsB);  ** WE
+      NEED THIS **
+      flag = CVodeSetInitStepB(cvode_mem, which, hinB);
+      flag = CVodeSetMinStepB(cvode_mem, which, hminB);
+      flag = CVodeSetMaxStepB(cvode_mem, which, hmaxB);
+      flag = CVodeSetStabLimDetB(cvode_mem, which, stldetB); ** MAYBE? **
+      flag = CVodeSetConstraintsB(cvode_mem, which, constraintsB);
+     */
+
     try {
       int indexB;
 
@@ -643,6 +655,10 @@ class cvodes_integrator_adjoint_vari : public vari {
                              absolute_tolerance_B_),
           "CVodeSStolerancesB");
 
+      check_flag_sundials(
+          CVodeSetMaxNumStepsB(memory->cvodes_mem_, indexB, max_num_steps_),
+          "CVodeSetMaxNumStepsB");
+
       check_flag_sundials(
           CVodeSetLinearSolverB(memory->cvodes_mem_, indexB, LSB_, AB_),
           "CVodeSetLinearSolverB");
@@ -699,14 +715,17 @@ class cvodes_integrator_adjoint_vari : public vari {
 
         double t_final = value_of((i > 0) ? memory->ts_[i - 1] : memory->t0_);
         if (t_final != t_init) {
-          check_flag_sundials(
-              CVodeReInitB(memory->cvodes_mem_, indexB, t_init, nv_state_sens),
-              "CVodeReInitB");
 
-          if (args_vars_ > 0) {
+          if (i != memory->ts_.size() - 1) {
             check_flag_sundials(
-                CVodeQuadReInitB(memory->cvodes_mem_, indexB, nv_quad),
-                "CVodeQuadReInitB");
+                CVodeReInitB(memory->cvodes_mem_, indexB, t_init, nv_state_sens),
+                "CVodeReInitB");
+
+            if (args_vars_ > 0) {
+              check_flag_sundials(
+                  CVodeQuadReInitB(memory->cvodes_mem_, indexB, nv_quad),
+                  "CVodeQuadReInitB");
+            }
           }
 
           int error_code = CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL);
diff --git a/stan/math/rev/functor/ode_adams.hpp b/stan/math/rev/functor/ode_adams.hpp
index 0f30ba279ba..d09e029166f 100644
--- a/stan/math/rev/functor/ode_adams.hpp
+++ b/stan/math/rev/functor/ode_adams.hpp
@@ -115,7 +115,7 @@ ode_adams_tol(const F& f, const T_y0& y0, const T_t0& t0,
               double absolute_tolerance,
               long int max_num_steps,  // NOLINT(runtime/int)
               std::ostream* msgs, const T_Args&... args) {
-  double absolute_tolerance_B = absolute_tolerance * 100.0;
+  double absolute_tolerance_B = absolute_tolerance * 10.0;
   double absolute_tolerance_QB = absolute_tolerance_B * 10.0;
   long int steps_checkpoint = 100;
 
@@ -163,7 +163,7 @@ ode_adams(const F& f, const T_y0& y0, const T_t0& t0,
   double relative_tolerance = 1e-10;
   double absolute_tolerance = 1e-10;
   long int max_num_steps = 1e8;  // NOLINT(runtime/int)
-  double absolute_tolerance_B = absolute_tolerance * 100.0;
+  double absolute_tolerance_B = absolute_tolerance * 10.0;
   double absolute_tolerance_QB = absolute_tolerance_B * 10.0;
   long int steps_checkpoint = 100;
 

From a09696bc11eec0f7737744e0739108fe0eddb6a2 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Wed, 6 Jan 2021 12:45:22 +0000
Subject: [PATCH 027/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index f6fec645a5d..92fb4a05e27 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -615,7 +615,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     SUNMatrix AB_ = SUNDenseMatrix(N_, N_);
     SUNLinearSolver LSB_ = SUNDenseLinearSolver(nv_state_sens, AB_);
 
-    /* check these if needed 
+    /* check these if needed
       flag = CVodeSetUserDataB(cvode_mem, which, user_dataB);
       flag = CVodeSetMaxOrdB(cvode_mem, which, maxordB);
       flag = CVodeSetMaxNumStepsB(cvode_mem, which, mxstepsB);  ** WE
@@ -711,11 +711,10 @@ class cvodes_integrator_adjoint_vari : public vari {
 
         double t_final = value_of((i > 0) ? memory->ts_[i - 1] : memory->t0_);
         if (t_final != t_init) {
-
           if (i != memory->ts_.size() - 1) {
-            check_flag_sundials(
-                CVodeReInitB(memory->cvodes_mem_, indexB, t_init, nv_state_sens),
-                "CVodeReInitB");
+            check_flag_sundials(CVodeReInitB(memory->cvodes_mem_, indexB,
+                                             t_init, nv_state_sens),
+                                "CVodeReInitB");
 
             if (args_vars_ > 0) {
               check_flag_sundials(

From 7e78900bff554db62050ac246f4e101077e7aa79 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 7 Jan 2021 17:09:01 +0100
Subject: [PATCH 028/157] add scaling test

---
 .../ode_bdf_adjoint_scale_bench_test.cpp      | 136 ++++++++++++++++++
 1 file changed, 136 insertions(+)
 create mode 100644 test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
new file mode 100644
index 00000000000..d01d17556ef
--- /dev/null
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
@@ -0,0 +1,136 @@
+
+
+#include <stan/math/rev.hpp>
+#include <gtest/gtest.h>
+#include <stan/math/rev/functor/gradient.hpp>
+#include <iostream>
+#include <sstream>
+#include <vector>
+#include <stdexcept>
+
+using stan::math::lognormal_rng;
+using stan::math::var;
+
+struct scaling_rhs {
+  template <typename T0, typename T1, typename T2, typename T3, typename T4,
+            typename T5>
+  inline Eigen::Matrix<stan::return_type_t<T1, T2, T3, T4, T5>,
+                       Eigen::Dynamic, 1>
+  operator()(const T0& t, const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y,
+             std::ostream* msgs,
+             const std::vector<T2>& kt,
+             const std::vector<T3>& e50,
+             const std::vector<T4>& k12,
+             const std::vector<T5>& k21
+             ) const {
+    std::size_t num_main_states = kt.size();
+    std::size_t num_states = 2 * num_main_states;
+
+    using return_t = stan::return_type_t<T1, T2, T3, T4, T5>;
+    
+    Eigen::Matrix<return_t,
+                  Eigen::Dynamic, 1>
+        dydt(num_states);
+
+    for(std::size_t i = 0; i != num_main_states; ++i) {
+      std::size_t m = 2*i;   // main state
+      std::size_t a = m+1; // auxilary state
+      return_t ksat = kt[i] * y(m) / ( y(m) + e50[i] );
+      
+      dydt(m) = - kt[i] * y(m) - k12[i] * y(m) + k21[i] * y(a);
+      dydt(a) =                + k12[i] * y(m) - k21[i] * y(a);
+
+      if (i != 0) {
+        return_t ksat_prev = kt[i-1] * y(2*(i-1)) / ( y(2*(i-1)) + e50[i-i] );
+        dydt(m) += ksat_prev * y(2*(i-1));
+      }
+
+    }
+    return dydt;
+  }
+};
+
+void run_benchmark(std::size_t system_size, int adjoint_integrator) {
+
+  scaling_rhs ode;
+  
+  std::vector<double> ts{1., 2.0, 4.0, 8.0, 16.0, 32.0, 64.0};
+  std::size_t ts_size = ts.size();
+
+  boost::ecuyer1988 base_rng(563646);
+
+  double log_sigma = 10.0;
+  double abs_tol = 1e-8;
+  double abs_tol_B = abs_tol * 100.0;
+  double abs_tol_QB = abs_tol_B * 10.0;
+  double rel_tol = 1e-6;
+  int steps_checkpoint = 100;
+  int max_num_steps = 100000;
+
+  for (std::size_t i = 0; i != 10; i++) {
+    stan::math::nested_rev_autodiff nested;
+
+    std::vector<var> kt(system_size);
+    std::vector<var> e50(system_size);
+    std::vector<var> k12(system_size);
+    std::vector<var> k21(system_size);
+
+    for(std::size_t j = 0; j != system_size; ++j) {
+      kt[j] = lognormal_rng(0.0, log_sigma, base_rng);
+      e50[j] = lognormal_rng(0.0, log_sigma, base_rng);
+      k12[j] = lognormal_rng(0.0, log_sigma, base_rng);
+      k21[j] = lognormal_rng(0.0, log_sigma, base_rng);
+    }
+
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(2*system_size);
+    for(std::size_t j = 0; j != 2*system_size; ++j)
+      y0(j) = lognormal_rng(0.0, log_sigma, base_rng);
+
+    double t0 = 0.0;
+
+    try {
+      if (adjoint_integrator) {
+        std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
+            = ode_bdf_adjoint_tol(ode, y0, t0, ts, rel_tol, abs_tol,
+                                  max_num_steps, nullptr, abs_tol_B, abs_tol_QB,
+                                  steps_checkpoint,
+                                  kt, e50, k12, k21
+                                  );
+
+        stan::math::grad();
+      } else {
+        std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
+            = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol, max_num_steps,
+                          nullptr,
+                          kt, e50, k12, k21);
+
+        stan::math::grad();
+      }
+    } catch (std::exception& exc) {
+      std::cout << "oops, keep going please!" << std::endl;
+      std::cerr << exc.what() << std::endl;
+    }
+    
+  }
+  stan::math::recover_memory();
+}
+
+TEST(StanMathOdeBench, bdf_2) { run_benchmark(2, 0); }
+
+TEST(StanMathOdeBench, bdf_adjoint_2) { run_benchmark(2, 1); }
+
+TEST(StanMathOdeBench, bdf_4) { run_benchmark(4, 0); }
+
+TEST(StanMathOdeBench, bdf_adjoint_4) { run_benchmark(4, 1); }
+
+TEST(StanMathOdeBench, bdf_8) { run_benchmark(8, 0); }
+
+TEST(StanMathOdeBench, bdf_adjoint_8) { run_benchmark(8, 1); }
+
+TEST(StanMathOdeBench, bdf_16) { run_benchmark(16, 0); }
+
+TEST(StanMathOdeBench, bdf_adjoint_16) { run_benchmark(16, 1); }
+
+TEST(StanMathOdeBench, bdf_32) { run_benchmark(32, 0); }
+
+TEST(StanMathOdeBench, bdf_adjoint_32) { run_benchmark(32, 1); }

From 4bb948283d2982cc037d7ad0cc1cd1458f7184a6 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Thu, 7 Jan 2021 16:17:06 +0000
Subject: [PATCH 029/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../ode_bdf_adjoint_scale_bench_test.cpp      | 58 ++++++++-----------
 1 file changed, 24 insertions(+), 34 deletions(-)

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
index d01d17556ef..8392e6d3468 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
@@ -14,46 +14,40 @@ using stan::math::var;
 struct scaling_rhs {
   template <typename T0, typename T1, typename T2, typename T3, typename T4,
             typename T5>
-  inline Eigen::Matrix<stan::return_type_t<T1, T2, T3, T4, T5>,
-                       Eigen::Dynamic, 1>
+  inline Eigen::Matrix<stan::return_type_t<T1, T2, T3, T4, T5>, Eigen::Dynamic,
+                       1>
   operator()(const T0& t, const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y,
-             std::ostream* msgs,
-             const std::vector<T2>& kt,
-             const std::vector<T3>& e50,
-             const std::vector<T4>& k12,
-             const std::vector<T5>& k21
-             ) const {
+             std::ostream* msgs, const std::vector<T2>& kt,
+             const std::vector<T3>& e50, const std::vector<T4>& k12,
+             const std::vector<T5>& k21) const {
     std::size_t num_main_states = kt.size();
     std::size_t num_states = 2 * num_main_states;
 
     using return_t = stan::return_type_t<T1, T2, T3, T4, T5>;
-    
-    Eigen::Matrix<return_t,
-                  Eigen::Dynamic, 1>
-        dydt(num_states);
-
-    for(std::size_t i = 0; i != num_main_states; ++i) {
-      std::size_t m = 2*i;   // main state
-      std::size_t a = m+1; // auxilary state
-      return_t ksat = kt[i] * y(m) / ( y(m) + e50[i] );
-      
-      dydt(m) = - kt[i] * y(m) - k12[i] * y(m) + k21[i] * y(a);
-      dydt(a) =                + k12[i] * y(m) - k21[i] * y(a);
+
+    Eigen::Matrix<return_t, Eigen::Dynamic, 1> dydt(num_states);
+
+    for (std::size_t i = 0; i != num_main_states; ++i) {
+      std::size_t m = 2 * i;  // main state
+      std::size_t a = m + 1;  // auxilary state
+      return_t ksat = kt[i] * y(m) / (y(m) + e50[i]);
+
+      dydt(m) = -kt[i] * y(m) - k12[i] * y(m) + k21[i] * y(a);
+      dydt(a) = +k12[i] * y(m) - k21[i] * y(a);
 
       if (i != 0) {
-        return_t ksat_prev = kt[i-1] * y(2*(i-1)) / ( y(2*(i-1)) + e50[i-i] );
-        dydt(m) += ksat_prev * y(2*(i-1));
+        return_t ksat_prev
+            = kt[i - 1] * y(2 * (i - 1)) / (y(2 * (i - 1)) + e50[i - i]);
+        dydt(m) += ksat_prev * y(2 * (i - 1));
       }
-
     }
     return dydt;
   }
 };
 
 void run_benchmark(std::size_t system_size, int adjoint_integrator) {
-
   scaling_rhs ode;
-  
+
   std::vector<double> ts{1., 2.0, 4.0, 8.0, 16.0, 32.0, 64.0};
   std::size_t ts_size = ts.size();
 
@@ -75,15 +69,15 @@ void run_benchmark(std::size_t system_size, int adjoint_integrator) {
     std::vector<var> k12(system_size);
     std::vector<var> k21(system_size);
 
-    for(std::size_t j = 0; j != system_size; ++j) {
+    for (std::size_t j = 0; j != system_size; ++j) {
       kt[j] = lognormal_rng(0.0, log_sigma, base_rng);
       e50[j] = lognormal_rng(0.0, log_sigma, base_rng);
       k12[j] = lognormal_rng(0.0, log_sigma, base_rng);
       k21[j] = lognormal_rng(0.0, log_sigma, base_rng);
     }
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(2*system_size);
-    for(std::size_t j = 0; j != 2*system_size; ++j)
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(2 * system_size);
+    for (std::size_t j = 0; j != 2 * system_size; ++j)
       y0(j) = lognormal_rng(0.0, log_sigma, base_rng);
 
     double t0 = 0.0;
@@ -93,16 +87,13 @@ void run_benchmark(std::size_t system_size, int adjoint_integrator) {
         std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
             = ode_bdf_adjoint_tol(ode, y0, t0, ts, rel_tol, abs_tol,
                                   max_num_steps, nullptr, abs_tol_B, abs_tol_QB,
-                                  steps_checkpoint,
-                                  kt, e50, k12, k21
-                                  );
+                                  steps_checkpoint, kt, e50, k12, k21);
 
         stan::math::grad();
       } else {
         std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
             = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol, max_num_steps,
-                          nullptr,
-                          kt, e50, k12, k21);
+                          nullptr, kt, e50, k12, k21);
 
         stan::math::grad();
       }
@@ -110,7 +101,6 @@ void run_benchmark(std::size_t system_size, int adjoint_integrator) {
       std::cout << "oops, keep going please!" << std::endl;
       std::cerr << exc.what() << std::endl;
     }
-    
   }
   stan::math::recover_memory();
 }

From a1fd530d585868b529feb99be3e77606538dced8 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 7 Jan 2021 17:20:23 +0100
Subject: [PATCH 030/157] align tolerances

---
 test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
index 3184ff35179..24bf616a0e2 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
@@ -90,7 +90,7 @@ void run_benchmark(int adjoint_integrator) {
         stan::math::grad();
       } else {
         std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-            = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol, max_num_steps,
+            = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol_QB, max_num_steps,
                           nullptr, ka, ke, k12, k21, kin, kout, ea50);
 
         stan::math::grad();

From 72cbc3f145d762471fde8773b4d0c03a94eea516 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 11 Jan 2021 14:04:05 +0100
Subject: [PATCH 031/157] fix scaling benchmark

---
 .../functor/ode_bdf_adjoint_scale_bench_test.cpp    | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
index 8392e6d3468..c41f916f64a 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
@@ -26,19 +26,18 @@ struct scaling_rhs {
     using return_t = stan::return_type_t<T1, T2, T3, T4, T5>;
 
     Eigen::Matrix<return_t, Eigen::Dynamic, 1> dydt(num_states);
+    std::vector<return_t> ksat(num_main_states);
 
     for (std::size_t i = 0; i != num_main_states; ++i) {
       std::size_t m = 2 * i;  // main state
       std::size_t a = m + 1;  // auxilary state
-      return_t ksat = kt[i] * y(m) / (y(m) + e50[i]);
+      ksat[i] = kt[i] * y(m) / (y(m) + e50[i]);
 
-      dydt(m) = -kt[i] * y(m) - k12[i] * y(m) + k21[i] * y(a);
+      dydt(m) = -ksat[i] * y(m) - k12[i] * y(m) + k21[i] * y(a);
       dydt(a) = +k12[i] * y(m) - k21[i] * y(a);
 
       if (i != 0) {
-        return_t ksat_prev
-            = kt[i - 1] * y(2 * (i - 1)) / (y(2 * (i - 1)) + e50[i - i]);
-        dydt(m) += ksat_prev * y(2 * (i - 1));
+        dydt(m) += ksat[i-1] * y(2 * (i - 1));
       }
     }
     return dydt;
@@ -55,7 +54,7 @@ void run_benchmark(std::size_t system_size, int adjoint_integrator) {
 
   double log_sigma = 10.0;
   double abs_tol = 1e-8;
-  double abs_tol_B = abs_tol * 100.0;
+  double abs_tol_B = abs_tol * 10.0;
   double abs_tol_QB = abs_tol_B * 10.0;
   double rel_tol = 1e-6;
   int steps_checkpoint = 100;
@@ -92,7 +91,7 @@ void run_benchmark(std::size_t system_size, int adjoint_integrator) {
         stan::math::grad();
       } else {
         std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-            = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol, max_num_steps,
+            = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol_QB, max_num_steps,
                           nullptr, kt, e50, k12, k21);
 
         stan::math::grad();

From 4d680491705bc58161331d3c7c1ed05cbd0c3288 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Mon, 11 Jan 2021 13:11:01 +0000
Subject: [PATCH 032/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
index c41f916f64a..b4e8ff9d3d7 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
@@ -37,7 +37,7 @@ struct scaling_rhs {
       dydt(a) = +k12[i] * y(m) - k21[i] * y(a);
 
       if (i != 0) {
-        dydt(m) += ksat[i-1] * y(2 * (i - 1));
+        dydt(m) += ksat[i - 1] * y(2 * (i - 1));
       }
     }
     return dydt;

From 94012ea4d6f8e6086551da1cf30188ac063319e0 Mon Sep 17 00:00:00 2001
From: Charles Margossian <charlesm@metrumrg.com>
Date: Mon, 11 Jan 2021 12:04:31 -0500
Subject: [PATCH 033/157] Set adjoints to 1 in benchmark test.

---
 .../ode_bdf_adjoint_scale_bench_test.cpp      | 24 ++++++++++---------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
index b4e8ff9d3d7..a45823bb80e 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
@@ -82,20 +82,22 @@ void run_benchmark(std::size_t system_size, int adjoint_integrator) {
     double t0 = 0.0;
 
     try {
+      std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y;
       if (adjoint_integrator) {
-        std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-            = ode_bdf_adjoint_tol(ode, y0, t0, ts, rel_tol, abs_tol,
-                                  max_num_steps, nullptr, abs_tol_B, abs_tol_QB,
-                                  steps_checkpoint, kt, e50, k12, k21);
-
-        stan::math::grad();
+        y = ode_bdf_adjoint_tol(ode, y0, t0, ts, rel_tol, abs_tol,
+                                max_num_steps, nullptr, abs_tol_B, abs_tol_QB,
+                                steps_checkpoint, kt, e50, k12, k21);
       } else {
-        std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-            = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol_QB, max_num_steps,
-                          nullptr, kt, e50, k12, k21);
-
-        stan::math::grad();
+        y = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol_QB, max_num_steps,
+                        nullptr, kt, e50, k12, k21);
       }
+
+      // Essentially sets the adjoint for all states to 1.
+      var target = stan::math::sum(y[0]);
+      for (int k = 1; k < ts_size; k++) target += stan::math::sum(y[k]);
+
+      target.grad();
+
     } catch (std::exception& exc) {
       std::cout << "oops, keep going please!" << std::endl;
       std::cerr << exc.what() << std::endl;

From 14dc05d69877113aece0939741e48ad68460937c Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Mon, 11 Jan 2021 17:43:03 +0000
Subject: [PATCH 034/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
index a45823bb80e..7fe9ee6e1f3 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
@@ -94,7 +94,8 @@ void run_benchmark(std::size_t system_size, int adjoint_integrator) {
 
       // Essentially sets the adjoint for all states to 1.
       var target = stan::math::sum(y[0]);
-      for (int k = 1; k < ts_size; k++) target += stan::math::sum(y[k]);
+      for (int k = 1; k < ts_size; k++)
+        target += stan::math::sum(y[k]);
 
       target.grad();
 

From 9a874209ced7702aa959a9532575d0b54bc2110f Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Wed, 13 Jan 2021 08:37:27 +0100
Subject: [PATCH 035/157] add toy model

---
 ode-adjoint/linked-mass-flow.R                |  82 ++++++++++
 ode-adjoint/linked-mass-flow.stan             | 154 ++++++++++++++++++
 .../rev/functor/cvodes_integrator_adjoint.hpp |   1 +
 stan/math/rev/functor/ode_adams.hpp           |  11 +-
 4 files changed, 245 insertions(+), 3 deletions(-)
 create mode 100644 ode-adjoint/linked-mass-flow.R
 create mode 100644 ode-adjoint/linked-mass-flow.stan

diff --git a/ode-adjoint/linked-mass-flow.R b/ode-adjoint/linked-mass-flow.R
new file mode 100644
index 00000000000..3896ec5c64e
--- /dev/null
+++ b/ode-adjoint/linked-mass-flow.R
@@ -0,0 +1,82 @@
+##install.packages("cmdstanr", repos = c("https://mc-stan.org/r-packages/", getOption("repos")))
+
+library(cmdstanr)
+library(posterior)
+library(bayesplot)
+color_scheme_set("brightblue")
+
+set_cmdstan_path("~/work/cmdstan")
+
+stan_model_file <- "linked-mass-flow.stan"
+
+mod <- cmdstan_model(stan_model_file)
+
+base_data <- list(rel_tol=1e-6,
+                  abs_tol=1e-6,
+                  max_num_steps=10000,
+                  adjoint_integrator=0,
+                  system_size=2,
+                  num_obs=8,
+                  sigma_sim=2.0,
+                  sigma_y=0.2)
+
+run_benchmark <- function(..., num_iter=250, adapt_delta=0.8) {
+    fit_args <- list(...)
+    fit_data <- modifyList(base_data, fit_args)
+    print(str(fit_data))
+    fit <- mod$sample(
+                   data = fit_data,
+                   iter_warmup = num_iter,
+                   iter_sampling = num_iter,
+                   seed = 123,
+                   chains = 1,
+                   parallel_chains = 1,
+                   refresh = round(num_iter/5),
+                   save_warmup=TRUE,
+                   adapt_delta=adapt_delta
+                   ##verbose=TRUE
+               )
+    fit
+}
+
+bdf_fit <- run_benchmark(adjoint_integrator=0)
+adjoint_fit <- run_benchmark(adjoint_integrator=1)
+
+adjoint_fit_2 <- run_benchmark(adjoint_integrator=1, adapt_delta=0.65)
+
+adjoint_fit_3 <- run_benchmark(adjoint_integrator=1, adapt_delta=0.95)
+
+adjoint_fit_4 <- run_benchmark(adjoint_integrator=1, abs_tol=1E-10, rel_tol=1E-10)
+
+summarize_benchmark <- function(fit) {
+    sdiag_all  <- fit$sampler_diagnostics(inc_warmup=TRUE)
+    sdiag_main  <- fit$sampler_diagnostics(inc_warmup=FALSE)
+
+    num_divergent <- sum(sdiag_main[,,"divergent__"])
+    accept_stat <- mean(sdiag_main[,,"accept_stat__"])
+    treedepth <- mean(sdiag_main[,,"treedepth__"])
+    num_leaps <- sum(sdiag_all[,,"n_leapfrog__"])
+    fit_time <- fit$time()$total
+
+    ess_speed <- fit$summary("lp__")[1,c("ess_bulk", "ess_tail")] / fit_time
+
+    list(num_leaps=num_leaps, time=fit_time, time_per_leap=1E3 * fit_time / num_leaps,
+         step_size=fit$metadata()$step_size_adaptation,
+         num_divergent=num_divergent, accept_stat=accept_stat, treedepth=treedepth,
+         ess_lp_speed=ess_speed)
+}
+
+mcmc_rhat(rhat = rhat(bdf_fit))  + yaxis_text(hjust = 0)
+
+str(summarize_benchmark(bdf_fit))
+str(summarize_benchmark(adjoint_fit))
+
+
+str(summarize_benchmark(adjoint_fit_2))
+str(summarize_benchmark(adjoint_fit_3))
+
+bdf_fit$metadata()
+
+bdf_fit$time()
+
+
diff --git a/ode-adjoint/linked-mass-flow.stan b/ode-adjoint/linked-mass-flow.stan
new file mode 100644
index 00000000000..cfbe138daf2
--- /dev/null
+++ b/ode-adjoint/linked-mass-flow.stan
@@ -0,0 +1,154 @@
+functions {
+
+  vector linked_mass_flow(real t, vector y,
+                          vector kt, vector e50, vector k12, vector k21) {
+
+    int num_main_states = num_elements(kt);
+    int num_states = 2 * num_main_states;
+
+    vector[num_states] dydt;
+    vector[num_main_states] ksat;
+
+    for (i in 1:num_main_states) {
+      int m = 2 * (i-1) + 1;  // main state
+      int a = m + 1;  // auxilary state
+      ksat[i] = kt[i] * y[m] / (y[m] + e50[i]);
+
+      dydt[m] = -ksat[i] * y[m] - k12[i] * y[m] + k21[i] * y[a];
+      dydt[a] = +k12[i] * y[m] - k21[i] * y[a];
+
+      if (i != 1) {
+        dydt[m] += ksat[i-1] * y[2 * (i - 1) + 1];
+      }
+    }
+    return dydt;
+
+  }
+
+  vector sample_vector_rng(real m, real s, int N) {
+    vector[N] sample;
+    for(i in 1:N)
+      sample[i] = normal_rng(m, s);
+    return sample;
+  }
+
+  vector[] simulate_mean(real t0, vector log_a0, real[] ts, int adjoint_integrator,
+                         real rel_tol, real abs_tol, int max_num_steps,
+                         vector log_kt, vector log_e50, vector log_k12, vector log_k21) {
+    int num_sim = num_elements(ts);
+    int num_states = num_elements(log_a0);
+    vector[num_states] y[num_sim];
+
+    if(adjoint_integrator) {
+      y = ode_adams_tol(linked_mass_flow, exp(log_a0), t0, ts, rel_tol, abs_tol, max_num_steps,
+                        exp(log_kt), exp(log_e50), exp(log_k12), exp(log_k21));
+    } else {
+      y = ode_bdf_tol(linked_mass_flow, exp(log_a0), t0, ts, rel_tol, abs_tol, max_num_steps,
+                      exp(log_kt), exp(log_e50), exp(log_k12), exp(log_k21));
+    }
+
+    return y;
+  }
+  
+}
+data {
+  real<lower=0> rel_tol;
+  real<lower=0> abs_tol;
+  int<lower=0,upper=1> adjoint_integrator;
+  int<lower=1> max_num_steps;
+  int<lower=1> system_size;
+  int<lower=1> num_obs;
+  real<lower=0> sigma_sim;
+  real<lower=0> sigma_y;
+}
+transformed data {
+  int num_states = 2 * system_size;
+
+  vector[system_size] log_kt_ = sample_vector_rng(0.0, sigma_sim, system_size);
+  vector[system_size] log_e50_ = sample_vector_rng(0.0, sigma_sim, system_size);
+  vector[system_size] log_k12_ = sample_vector_rng(0.0, sigma_sim, system_size);
+  vector[system_size] log_k21_ = sample_vector_rng(0.0, sigma_sim, system_size);
+  vector[system_size] log_sigma_y_ = sample_vector_rng(0.0, sigma_y, system_size);
+  vector[num_states] log_a0_ = sample_vector_rng(0.0, sigma_sim, num_states);
+  vector[num_states] y_[num_obs];
+  real ts[num_obs];
+  real t0 = 0.0;
+
+  ts[1] = 1.0;
+  for(i in 2:num_obs) {
+    ts[i] = 1.5 * ts[i-1];
+  }
+
+  y_ = simulate_mean(t0, log_a0_, ts, 0, rel_tol, abs_tol, max_num_steps,
+                     log_kt_, log_e50_, log_k12_, log_k21_);
+  
+  for(i in 1:num_obs) {
+    for(j in 1:system_size) {
+      int m = 2*(j-1) + 1;
+      y_[i,m] = lognormal_rng(log(y_[i,m]+1E-3), exp(log_sigma_y_[j]));
+    }
+  }
+
+  if(adjoint_integrator) {
+    print("Using bdf_adjoint integrator.");
+  } else {
+    print("Using bdf integrator.");
+  }
+  print("relative tolerance: ", rel_tol);
+  print("absolute tolerance: ", abs_tol);
+  print("maximum number of steps: ", max_num_steps);
+  print("number of time points: ", num_obs);
+  print("system size: ", system_size);
+  print("time points: ", ts);
+  print("y_: ", y_);
+}
+parameters {
+  vector[system_size] log_kt;
+  vector[system_size] log_e50;
+  vector[system_size] log_k12;
+  vector[system_size] log_k21;
+  vector[system_size] log_sigma_y;
+  vector[num_states] log_a0;
+}
+transformed parameters {
+}
+model {
+  vector[num_states] mu[num_obs] = simulate_mean(t0, log_a0, ts, adjoint_integrator, rel_tol, abs_tol, max_num_steps,
+                                                 log_kt, log_e50, log_k12, log_k21);
+  
+  target += normal_lpdf(log_kt| 0.0, sigma_sim);
+  target += normal_lpdf(log_e50| 0.0, sigma_sim);
+  target += normal_lpdf(log_k12| 0.0, sigma_sim);
+  target += normal_lpdf(log_k21| 0.0, sigma_sim);
+  target += normal_lpdf(log_a0| 0.0, sigma_sim);
+  target += normal_lpdf(log_sigma_y| 0.0, sigma_y);
+
+  for(j in 1:system_size) {
+    int m = 2*(j-1) + 1;
+    target += lognormal_lpdf(to_vector(y_[:,m])| log(to_vector(mu[:,m])+1E-3), exp(log_sigma_y[j]));
+  }
+}
+generated quantities {
+  int rank_log_kt[system_size];
+  int rank_log_e50[system_size];
+  int rank_log_k12[system_size];
+  int rank_log_k21[system_size];
+  int rank_log_sigma_y[system_size];
+  int rank_log_a0[system_size];
+  vector[system_size] bias_log_kt = log_kt - log_kt_;
+  vector[system_size] bias_log_e50 = log_e50 - log_e50_;
+  vector[system_size] bias_log_k12 = log_k12 - log_k12_;
+  vector[system_size] bias_log_k21 = log_k21 - log_k21_;
+  vector[system_size] bias_log_sigma_y = log_sigma_y - log_sigma_y_;
+  vector[system_size] bias_log_a0 = log_a0 - log_a0_;
+
+  for(i in 1:system_size) {
+    rank_log_kt[i] = (log_kt[i] > log_kt_[i] ? 1 : 0);
+    rank_log_e50[i] = (log_e50[i] > log_e50_[i] ? 1 : 0);
+    rank_log_k12[i] = (log_k12[i] > log_k12_[i] ? 1 : 0);
+    rank_log_k21[i] = (log_k21[i] > log_k21_[i] ? 1 : 0);
+    rank_log_sigma_y[i] = (log_sigma_y[i] > log_sigma_y_[i] ? 1 : 0);
+    rank_log_a0[i] = (log_a0[i] > log_a0_[i] ? 1 : 0);
+  }
+
+}
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 92fb4a05e27..272acab43da 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -541,6 +541,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     // initialize forward sensitivity system of CVODES as needed
     if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
       check_flag_sundials(
+          //CVodeAdjInit(memory->cvodes_mem_, steps_checkpoint_, CV_POLYNOMIAL),
           CVodeAdjInit(memory->cvodes_mem_, steps_checkpoint_, CV_HERMITE),
           "CVodeAdjInit");
     }
diff --git a/stan/math/rev/functor/ode_adams.hpp b/stan/math/rev/functor/ode_adams.hpp
index 55bd84ae6a8..9b2b85ed774 100644
--- a/stan/math/rev/functor/ode_adams.hpp
+++ b/stan/math/rev/functor/ode_adams.hpp
@@ -116,12 +116,17 @@ ode_adams_tol(const F& f, const T_y0& y0, const T_t0& t0,
               double absolute_tolerance,
               long int max_num_steps,  // NOLINT(runtime/int)
               std::ostream* msgs, const T_Args&... args) {
-  double absolute_tolerance_B = absolute_tolerance * 10.0;
-  double absolute_tolerance_QB = absolute_tolerance_B * 10.0;
+  // sensitivities are output with absolute_tolerance
+  double absolute_tolerance_QB = absolute_tolerance;
+  // backward solve is 10x more precise than quadrature
+  double absolute_tolerance_B = absolute_tolerance_QB / 10.0;
+  // forward solve is 10x more precise that backward solve
+  double absolute_tolerance_F = absolute_tolerance_B / 10.0;
+
   long int steps_checkpoint = 100;
 
   return ode_adams_tol_impl("ode_adams_tol", f, y0, t0, ts, relative_tolerance,
-                            absolute_tolerance, max_num_steps, msgs,
+                            absolute_tolerance_F, max_num_steps, msgs,
                             absolute_tolerance_B, absolute_tolerance_QB,
                             steps_checkpoint, args...);
 }

From c5509e70caa44273f0fb2ca22054813aa8047275 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Wed, 13 Jan 2021 07:40:38 +0000
Subject: [PATCH 036/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 272acab43da..5e7f967a8c4 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -541,7 +541,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     // initialize forward sensitivity system of CVODES as needed
     if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
       check_flag_sundials(
-          //CVodeAdjInit(memory->cvodes_mem_, steps_checkpoint_, CV_POLYNOMIAL),
+          // CVodeAdjInit(memory->cvodes_mem_, steps_checkpoint_,
+          // CV_POLYNOMIAL),
           CVodeAdjInit(memory->cvodes_mem_, steps_checkpoint_, CV_HERMITE),
           "CVodeAdjInit");
     }

From 49349952ff30b97f04fbf6144235ec0e8e4ad23a Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Wed, 13 Jan 2021 09:30:37 +0100
Subject: [PATCH 037/157] fix

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 86 ++++++++++++++++++-
 stan/math/rev/functor/ode_bdf_adjoint.hpp     |  6 +-
 2 files changed, 88 insertions(+), 4 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 272acab43da..4699ea539f8 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -685,7 +685,7 @@ class cvodes_integrator_adjoint_vari : public vari {
             CVodeSetQuadErrConB(memory->cvodes_mem_, indexB, SUNTRUE),
             "CVodeSetQuadErrConB");
       }
-
+      /* newer, but buggy code
       // At every time step, collect the adjoints from the output
       // variables and re-initialize the solver
       double t_init = value_of(memory->ts_.back());
@@ -776,6 +776,90 @@ class cvodes_integrator_adjoint_vari : public vari {
       for (size_t s = 0; s < args_vars_; s++) {
         args_varis_[s]->adj_ += quad(s);
       }
+
+*/
+      // old code without nice error messages, but works according to
+      // grad test
+      
+      // At every time step, collect the adjoints from the output
+      // variables and re-initialize the solver
+      double t_init = value_of(memory->ts_.back());
+      for (int i = memory->ts_.size() - 1; i >= 0; --i) {
+        // Take in the adjoints from all the output variables at this point
+        // in time
+        Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
+        for (int j = 0; j < N_; j++) {
+          // std::cout << "i: " << i << ", j: " << j << std::endl;
+          state_sens(j) += non_chaining_varis_[i * N_ + j]->adj_;
+          step_sens(j) += non_chaining_varis_[i * N_ + j]->adj_;
+        }
+
+        if (ts_vars_ > 0 && i >= 0) {
+          ts_varis_[i]->adj_ += apply(
+              [&](auto&&... args) {
+                double adj = step_sens.dot(
+                    memory->f_(t_init, memory->y_[i], msgs_, args...));
+                // std::cout << "adj: " << adj << ", i: " << i << std::endl;
+                return adj;
+              },
+              memory->value_of_args_tuple_);
+        }
+
+        double t_final = value_of((i > 0) ? memory->ts_[i - 1] : memory->t0_);
+        if (t_final != t_init) {
+          check_flag_sundials(
+              CVodeReInitB(memory->cvodes_mem_, indexB, t_init, nv_state_sens),
+              "CVodeReInitB");
+
+          if (args_vars_ > 0) {
+            check_flag_sundials(
+                CVodeQuadReInitB(memory->cvodes_mem_, indexB, nv_quad),
+                "CVodeQuadReInitB");
+          }
+
+          check_flag_sundials(CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL),
+                              "CVodeB");
+
+          check_flag_sundials(
+              CVodeGetB(memory->cvodes_mem_, indexB, &t_init, nv_state_sens),
+              "CVodeGetB");
+
+          if (args_vars_ > 0) {
+            check_flag_sundials(
+                CVodeGetQuadB(memory->cvodes_mem_, indexB, &t_init, nv_quad),
+                "CVodeGetQuadB");
+          }
+        }
+      }
+
+      if (t0_vars_ > 0) {
+        Eigen::VectorXd y0d = value_of(memory->y0_);
+        t0_varis_[0]->adj_ += apply(
+            [&](auto&&... args) {
+              return -state_sens.dot(memory->f_(t_init, y0d, msgs_, args...));
+            },
+            memory->value_of_args_tuple_);
+      }
+
+      if (args_vars_ > 0) {
+        check_flag_sundials(
+            CVodeGetQuadB(memory->cvodes_mem_, indexB, &t_init, nv_quad),
+            "CVodeGetQuadB");
+      }
+
+      // After integrating all the way back to t0, we finally have the
+      // the adjoints we wanted
+      // These are the dlog_density / d(initial_conditions[s]) adjoints
+      for (size_t s = 0; s < y0_vars_; s++) {
+        y0_varis_[s]->adj_ += state_sens(s);
+      }
+
+      // These are the dlog_density / d(parameters[s]) adjoints
+      for (size_t s = 0; s < args_vars_; s++) {
+        args_varis_[s]->adj_ += quad(s);
+      }
+
+      
     } catch (const std::exception& e) {
       SUNLinSolFree(LSB_);
       SUNMatDestroy(AB_);
diff --git a/stan/math/rev/functor/ode_bdf_adjoint.hpp b/stan/math/rev/functor/ode_bdf_adjoint.hpp
index 0352753c3ce..c644eb192cd 100644
--- a/stan/math/rev/functor/ode_bdf_adjoint.hpp
+++ b/stan/math/rev/functor/ode_bdf_adjoint.hpp
@@ -166,9 +166,9 @@ ode_bdf_adjoint(const F& f, const T_y0& y0, const T_t0& t0,
                 const T_Args&... args) {
   double relative_tolerance = 1e-10;
   double absolute_tolerance = 1e-10;
-  double absolute_tolerance_B = 1e-8;
-  double absolute_tolerance_QB = 1e-6;
-  long int steps_checkpoint = 150;
+  double absolute_tolerance_B = 1e-9;
+  double absolute_tolerance_QB = 1e-8;
+  long int steps_checkpoint = 100;
   long int max_num_steps = 1e8;
 
   return ode_bdf_adjoint_tol_impl(

From 992624651e7d06d4b4594c8db061c767289272bc Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Wed, 13 Jan 2021 08:36:47 +0000
Subject: [PATCH 038/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 314add96beb..9f6435aa426 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -781,7 +781,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 */
       // old code without nice error messages, but works according to
       // grad test
-      
+
       // At every time step, collect the adjoints from the output
       // variables and re-initialize the solver
       double t_init = value_of(memory->ts_.back());
@@ -860,7 +860,6 @@ class cvodes_integrator_adjoint_vari : public vari {
         args_varis_[s]->adj_ += quad(s);
       }
 
-      
     } catch (const std::exception& e) {
       SUNLinSolFree(LSB_);
       SUNMatDestroy(AB_);

From 999f76663d2ece2178c423a49fd19f8f19b2b4e6 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Wed, 13 Jan 2021 22:26:52 +0100
Subject: [PATCH 039/157] fix cvodes init / reinit logic

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 178 +++++-------------
 stan/math/rev/functor/ode_adams.hpp           |   8 +-
 .../functor/ode_bdf_adjoint_grad_rev_test.cpp |  22 ++-
 3 files changed, 65 insertions(+), 143 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 314add96beb..55456cd8940 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -640,53 +640,8 @@ class cvodes_integrator_adjoint_vari : public vari {
                                             reinterpret_cast<void*>(this)),
                           "CVodeSetUserDataB");
 
-      // The ode_rhs_adj_sense functions passed in here cause problems with
-      // the autodiff stack (they can cause reallocations of the internal
-      // vectors and cause segfaults)
-      check_flag_sundials(
-          CVodeInitB(memory->cvodes_mem_, indexB,
-                     &cvodes_integrator_adjoint_vari::cv_rhs_adj_sens,
-                     value_of(memory->ts_.back()), nv_state_sens),
-          "CVodeInitB");
-
-      check_flag_sundials(
-          CVodeSStolerancesB(memory->cvodes_mem_, indexB, relative_tolerance_,
-                             absolute_tolerance_B_),
-          "CVodeSStolerancesB");
-
-      check_flag_sundials(
-          CVodeSetMaxNumStepsB(memory->cvodes_mem_, indexB, max_num_steps_),
-          "CVodeSetMaxNumStepsB");
-
-      check_flag_sundials(
-          CVodeSetLinearSolverB(memory->cvodes_mem_, indexB, LSB_, AB_),
-          "CVodeSetLinearSolverB");
-
-      // The same autodiff issue that applies to ode_rhs_adj_sense applies
-      // here
-      check_flag_sundials(
-          CVodeSetJacFnB(memory->cvodes_mem_, indexB,
-                         &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
-          "CVodeSetJacFnB");
-
-      // Allocate space for backwards quadrature
-      if (args_vars_ > 0) {
-        check_flag_sundials(
-            CVodeQuadInitB(memory->cvodes_mem_, indexB,
-                           &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj,
-                           nv_quad),
-            "CVodeQuadInitB");
-
-        check_flag_sundials(
-            CVodeQuadSStolerancesB(memory->cvodes_mem_, indexB,
-                                   relative_tolerance_, absolute_tolerance_QB_),
-            "CVodeQuadSStolerancesB");
+      bool cvodes_backward_initialized = false;
 
-        check_flag_sundials(
-            CVodeSetQuadErrConB(memory->cvodes_mem_, indexB, SUNTRUE),
-            "CVodeSetQuadErrConB");
-      }
-      /* newer, but buggy code
       // At every time step, collect the adjoints from the output
       // variables and re-initialize the solver
       double t_init = value_of(memory->ts_.back());
@@ -713,7 +668,53 @@ class cvodes_integrator_adjoint_vari : public vari {
 
         double t_final = value_of((i > 0) ? memory->ts_[i - 1] : memory->t0_);
         if (t_final != t_init) {
-          if (i != memory->ts_.size() - 1) {
+          if (!cvodes_backward_initialized) {
+            // initialize CVODES backward machinery
+            check_flag_sundials(
+                CVodeInitB(memory->cvodes_mem_, indexB,
+                           &cvodes_integrator_adjoint_vari::cv_rhs_adj_sens,
+                           t_init, nv_state_sens),
+                "CVodeInitB");
+
+            check_flag_sundials(
+                CVodeSStolerancesB(memory->cvodes_mem_, indexB, relative_tolerance_,
+                                   absolute_tolerance_B_),
+                "CVodeSStolerancesB");
+
+            check_flag_sundials(
+                CVodeSetMaxNumStepsB(memory->cvodes_mem_, indexB, max_num_steps_),
+                "CVodeSetMaxNumStepsB");
+
+            check_flag_sundials(
+                CVodeSetLinearSolverB(memory->cvodes_mem_, indexB, LSB_, AB_),
+                "CVodeSetLinearSolverB");
+
+            check_flag_sundials(
+                CVodeSetJacFnB(memory->cvodes_mem_, indexB,
+                               &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
+                "CVodeSetJacFnB");
+
+            // Allocate space for backwards quadrature
+            if (args_vars_ > 0) {
+              check_flag_sundials(
+                  CVodeQuadInitB(memory->cvodes_mem_, indexB,
+                                 &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj,
+                                 nv_quad),
+                  "CVodeQuadInitB");
+
+              check_flag_sundials(
+                  CVodeQuadSStolerancesB(memory->cvodes_mem_, indexB,
+                                         relative_tolerance_, absolute_tolerance_QB_),
+                  "CVodeQuadSStolerancesB");
+
+              check_flag_sundials(
+                  CVodeSetQuadErrConB(memory->cvodes_mem_, indexB, SUNTRUE),
+                  "CVodeSetQuadErrConB");
+            }
+            
+            cvodes_backward_initialized = true;
+          } else {
+            // just re-initialize the solver
             check_flag_sundials(CVodeReInitB(memory->cvodes_mem_, indexB,
                                              t_init, nv_state_sens),
                                 "CVodeReInitB");
@@ -777,89 +778,6 @@ class cvodes_integrator_adjoint_vari : public vari {
       for (size_t s = 0; s < args_vars_; s++) {
         args_varis_[s]->adj_ += quad(s);
       }
-
-*/
-      // old code without nice error messages, but works according to
-      // grad test
-      
-      // At every time step, collect the adjoints from the output
-      // variables and re-initialize the solver
-      double t_init = value_of(memory->ts_.back());
-      for (int i = memory->ts_.size() - 1; i >= 0; --i) {
-        // Take in the adjoints from all the output variables at this point
-        // in time
-        Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
-        for (int j = 0; j < N_; j++) {
-          // std::cout << "i: " << i << ", j: " << j << std::endl;
-          state_sens(j) += non_chaining_varis_[i * N_ + j]->adj_;
-          step_sens(j) += non_chaining_varis_[i * N_ + j]->adj_;
-        }
-
-        if (ts_vars_ > 0 && i >= 0) {
-          ts_varis_[i]->adj_ += apply(
-              [&](auto&&... args) {
-                double adj = step_sens.dot(
-                    memory->f_(t_init, memory->y_[i], msgs_, args...));
-                // std::cout << "adj: " << adj << ", i: " << i << std::endl;
-                return adj;
-              },
-              memory->value_of_args_tuple_);
-        }
-
-        double t_final = value_of((i > 0) ? memory->ts_[i - 1] : memory->t0_);
-        if (t_final != t_init) {
-          check_flag_sundials(
-              CVodeReInitB(memory->cvodes_mem_, indexB, t_init, nv_state_sens),
-              "CVodeReInitB");
-
-          if (args_vars_ > 0) {
-            check_flag_sundials(
-                CVodeQuadReInitB(memory->cvodes_mem_, indexB, nv_quad),
-                "CVodeQuadReInitB");
-          }
-
-          check_flag_sundials(CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL),
-                              "CVodeB");
-
-          check_flag_sundials(
-              CVodeGetB(memory->cvodes_mem_, indexB, &t_init, nv_state_sens),
-              "CVodeGetB");
-
-          if (args_vars_ > 0) {
-            check_flag_sundials(
-                CVodeGetQuadB(memory->cvodes_mem_, indexB, &t_init, nv_quad),
-                "CVodeGetQuadB");
-          }
-        }
-      }
-
-      if (t0_vars_ > 0) {
-        Eigen::VectorXd y0d = value_of(memory->y0_);
-        t0_varis_[0]->adj_ += apply(
-            [&](auto&&... args) {
-              return -state_sens.dot(memory->f_(t_init, y0d, msgs_, args...));
-            },
-            memory->value_of_args_tuple_);
-      }
-
-      if (args_vars_ > 0) {
-        check_flag_sundials(
-            CVodeGetQuadB(memory->cvodes_mem_, indexB, &t_init, nv_quad),
-            "CVodeGetQuadB");
-      }
-
-      // After integrating all the way back to t0, we finally have the
-      // the adjoints we wanted
-      // These are the dlog_density / d(initial_conditions[s]) adjoints
-      for (size_t s = 0; s < y0_vars_; s++) {
-        y0_varis_[s]->adj_ += state_sens(s);
-      }
-
-      // These are the dlog_density / d(parameters[s]) adjoints
-      for (size_t s = 0; s < args_vars_; s++) {
-        args_varis_[s]->adj_ += quad(s);
-      }
-
       
     } catch (const std::exception& e) {
       SUNLinSolFree(LSB_);
diff --git a/stan/math/rev/functor/ode_adams.hpp b/stan/math/rev/functor/ode_adams.hpp
index 9b2b85ed774..f66037f98e1 100644
--- a/stan/math/rev/functor/ode_adams.hpp
+++ b/stan/math/rev/functor/ode_adams.hpp
@@ -169,14 +169,14 @@ ode_adams(const F& f, const T_y0& y0, const T_t0& t0,
           const std::vector<T_ts>& ts, std::ostream* msgs,
           const T_Args&... args) {
   double relative_tolerance = 1e-10;
-  double absolute_tolerance = 1e-10;
+  double absolute_tolerance_QB = 1e-10;
   long int max_num_steps = 1e8;  // NOLINT(runtime/int)
-  double absolute_tolerance_B = absolute_tolerance * 10.0;
-  double absolute_tolerance_QB = absolute_tolerance_B * 10.0;
+  double absolute_tolerance_B = absolute_tolerance_QB / 10.0;
+  double absolute_tolerance_F = absolute_tolerance_B / 10.0;
   long int steps_checkpoint = 100;
 
   return ode_adams_tol_impl("ode_adams", f, y0, t0, ts, relative_tolerance,
-                            absolute_tolerance, max_num_steps, msgs,
+                            absolute_tolerance_F, max_num_steps, msgs,
                             absolute_tolerance_B, absolute_tolerance_QB,
                             steps_checkpoint, args...);
 }
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
index d60bbe75d2e..b620d3859e9 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
@@ -60,14 +60,14 @@ class test_functor_double_var {
     y0 << 1.25, 0.0;
 
     double t0 = 0.0;
-    std::vector<double> ts{5.0};
+    std::vector<double> ts{2.5, 5.0};
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
               ? stan::math::ode_bdf_adjoint(sho, y0, t0, ts, nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
-    return ys[0](state);
+    return ys[1](state);
   }
 };
 
@@ -87,13 +87,13 @@ TEST(StanMathOdeIntegrateODEGradMat, double_var) {
     stan::math::gradient(func1, x, f, grad);
 
     EXPECT_NEAR(y1(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy1_domega(t, omega, chi), grad(0), 1e-7);
+    EXPECT_NEAR(dy1_domega(t, omega, chi), grad(0), 1e-6);
 
     test_functor_double_var<1> func2(TEST_CVODES_ADAMS);
     stan::math::gradient(func2, x, f, grad);
 
     EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-7);
+    EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-6);
   }
 
   {  // bdf
@@ -196,14 +196,14 @@ class test_functor_var_var {
     y0 << x(1), 0.0;
 
     double t0 = 0.0;
-    std::vector<double> ts{5.0};
+    std::vector<double> ts{2.0, 5.0};
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_bdf_adjoint(sho, y0, t0, ts, nullptr, omega)
+        ? stan::math::ode_adams_tol(sho, y0, t0, ts, 1E-10, 1E-10, 10000, nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
-    return ys[0](state);
+    return ys[1](state);
   }
 };
 
@@ -220,6 +220,8 @@ TEST(StanMathOdeIntegrateODEGradMat, var_var) {
   Eigen::VectorXd grad(2);
 
   {
+    stan::math::nested_rev_autodiff nested;
+    
     test_functor_var_var<0> func1(TEST_CVODES_ADAMS);
     stan::math::gradient(func1, x, f, grad);
 
@@ -231,11 +233,13 @@ TEST(StanMathOdeIntegrateODEGradMat, var_var) {
     stan::math::gradient(func2, x, f, grad);
 
     EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-7);
+    EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-6);
     EXPECT_NEAR(dy2_dchi(t, omega, chi), grad(1), 1e-7);
   }
 
   {
+    stan::math::nested_rev_autodiff nested;
+    
     test_functor_var_var<0> func1(TEST_CVODES_BDF);
     stan::math::gradient(func1, x, f, grad);
 
@@ -247,7 +251,7 @@ TEST(StanMathOdeIntegrateODEGradMat, var_var) {
     stan::math::gradient(func2, x, f, grad);
 
     EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-7);
+    EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-6);
     EXPECT_NEAR(dy2_dchi(t, omega, chi), grad(1), 1e-7);
   }
 }

From 12b8408b7d319d9b90d259991424e312aec0cb09 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Wed, 13 Jan 2021 21:35:29 +0000
Subject: [PATCH 040/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 27 ++++++++++---------
 .../functor/ode_bdf_adjoint_grad_rev_test.cpp |  7 ++---
 2 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 1904ff96356..03bb076459c 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -677,41 +677,44 @@ class cvodes_integrator_adjoint_vari : public vari {
                 "CVodeInitB");
 
             check_flag_sundials(
-                CVodeSStolerancesB(memory->cvodes_mem_, indexB, relative_tolerance_,
-                                   absolute_tolerance_B_),
+                CVodeSStolerancesB(memory->cvodes_mem_, indexB,
+                                   relative_tolerance_, absolute_tolerance_B_),
                 "CVodeSStolerancesB");
 
-            check_flag_sundials(
-                CVodeSetMaxNumStepsB(memory->cvodes_mem_, indexB, max_num_steps_),
-                "CVodeSetMaxNumStepsB");
+            check_flag_sundials(CVodeSetMaxNumStepsB(memory->cvodes_mem_,
+                                                     indexB, max_num_steps_),
+                                "CVodeSetMaxNumStepsB");
 
             check_flag_sundials(
                 CVodeSetLinearSolverB(memory->cvodes_mem_, indexB, LSB_, AB_),
                 "CVodeSetLinearSolverB");
 
             check_flag_sundials(
-                CVodeSetJacFnB(memory->cvodes_mem_, indexB,
-                               &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
+                CVodeSetJacFnB(
+                    memory->cvodes_mem_, indexB,
+                    &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
                 "CVodeSetJacFnB");
 
             // Allocate space for backwards quadrature
             if (args_vars_ > 0) {
               check_flag_sundials(
-                  CVodeQuadInitB(memory->cvodes_mem_, indexB,
-                                 &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj,
-                                 nv_quad),
+                  CVodeQuadInitB(
+                      memory->cvodes_mem_, indexB,
+                      &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj,
+                      nv_quad),
                   "CVodeQuadInitB");
 
               check_flag_sundials(
                   CVodeQuadSStolerancesB(memory->cvodes_mem_, indexB,
-                                         relative_tolerance_, absolute_tolerance_QB_),
+                                         relative_tolerance_,
+                                         absolute_tolerance_QB_),
                   "CVodeQuadSStolerancesB");
 
               check_flag_sundials(
                   CVodeSetQuadErrConB(memory->cvodes_mem_, indexB, SUNTRUE),
                   "CVodeSetQuadErrConB");
             }
-            
+
             cvodes_backward_initialized = true;
           } else {
             // just re-initialize the solver
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
index b620d3859e9..edc81f77f30 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
@@ -200,7 +200,8 @@ class test_functor_var_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-        ? stan::math::ode_adams_tol(sho, y0, t0, ts, 1E-10, 1E-10, 10000, nullptr, omega)
+              ? stan::math::ode_adams_tol(sho, y0, t0, ts, 1E-10, 1E-10, 10000,
+                                          nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return ys[1](state);
@@ -221,7 +222,7 @@ TEST(StanMathOdeIntegrateODEGradMat, var_var) {
 
   {
     stan::math::nested_rev_autodiff nested;
-    
+
     test_functor_var_var<0> func1(TEST_CVODES_ADAMS);
     stan::math::gradient(func1, x, f, grad);
 
@@ -239,7 +240,7 @@ TEST(StanMathOdeIntegrateODEGradMat, var_var) {
 
   {
     stan::math::nested_rev_autodiff nested;
-    
+
     test_functor_var_var<0> func1(TEST_CVODES_BDF);
     stan::math::gradient(func1, x, f, grad);
 

From 18ff5b3e5b1a4aecf049ba9a7e82471d21732a03 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 14 Jan 2021 09:47:43 +0100
Subject: [PATCH 041/157] add sum test

---
 .../functor/ode_bdf_adjoint_grad_rev_test.cpp | 84 +++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
index edc81f77f30..280dc4f259c 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
@@ -82,6 +82,8 @@ TEST(StanMathOdeIntegrateODEGradMat, double_var) {
   double f;
   Eigen::VectorXd grad(1);
 
+  stan::math::nested_rev_autodiff nested;
+
   {  // Adams
     test_functor_double_var<0> func1(TEST_CVODES_ADAMS);
     stan::math::gradient(func1, x, f, grad);
@@ -150,6 +152,7 @@ TEST(StanMathOdeIntegrateODEGradMat, var_double) {
   double f;
   Eigen::VectorXd grad(1);
 
+  stan::math::nested_rev_autodiff nested;
   {  // adams
     test_functor_var_double<0> func1(TEST_CVODES_ADAMS);
     stan::math::gradient(func1, x, f, grad);
@@ -256,3 +259,84 @@ TEST(StanMathOdeIntegrateODEGradMat, var_var) {
     EXPECT_NEAR(dy2_dchi(t, omega, chi), grad(1), 1e-7);
   }
 }
+
+
+
+template <int state>
+class test_functor_sum_var_var {
+  int lmm_;
+
+ public:
+  explicit test_functor_sum_var_var(int lmm) : lmm_(lmm) {}
+
+  template <typename T>
+  inline T operator()(Eigen::Matrix<T, Eigen::Dynamic, 1>& x) const {
+    sho_functor sho;
+
+    T omega = x(0);
+
+    Eigen::Matrix<T, Eigen::Dynamic, 1> y0(2);
+    y0 << x(1), 0.0;
+
+    double t0 = 0.0;
+    std::vector<double> ts{2.0, 5.0};
+
+    std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
+        = (lmm_ == TEST_CVODES_ADAMS)
+              ? stan::math::ode_adams_tol(sho, y0, t0, ts, 1E-10, 1E-10, 10000,
+                                          nullptr, omega)
+              : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
+
+    return stan::math::sum(ys[0](state) + ys[1](state));
+  }
+};
+
+TEST(StanMathOdeIntegrateODEGradMat, sum_var_var) {
+  double omega = 0.5;
+  double chi = 1.25;
+  double t1 = 2.0;
+  double t2 = 5;
+
+  Eigen::VectorXd x(2);
+  x(0) = omega;
+  x(1) = chi;
+
+  double f;
+  Eigen::VectorXd grad(2);
+
+  {
+    stan::math::nested_rev_autodiff nested;
+
+    test_functor_sum_var_var<0> func1(TEST_CVODES_ADAMS);
+    stan::math::gradient(func1, x, f, grad);
+
+    EXPECT_NEAR(y1(t1, omega, chi) + y1(t2, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy1_domega(t1, omega, chi) + dy1_domega(t2, omega, chi), grad(0), 1e-7);
+    EXPECT_NEAR(dy1_dchi(t1, omega, chi) + dy1_dchi(t2, omega, chi), grad(1), 1e-7);
+
+    test_functor_sum_var_var<1> func2(TEST_CVODES_ADAMS);
+    stan::math::gradient(func2, x, f, grad);
+
+    EXPECT_NEAR(y2(t1, omega, chi) + y2(t2, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy2_domega(t1, omega, chi) + dy2_domega(t2, omega, chi), grad(0), 1e-6);
+    EXPECT_NEAR(dy2_dchi(t1, omega, chi) + dy2_dchi(t2, omega, chi), grad(1), 1e-7);
+  }
+
+  {
+    stan::math::nested_rev_autodiff nested;
+
+    test_functor_sum_var_var<0> func1(TEST_CVODES_BDF);
+    stan::math::gradient(func1, x, f, grad);
+
+    EXPECT_NEAR(y1(t1, omega, chi) + y1(t2, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy1_domega(t1, omega, chi) + dy1_domega(t2, omega, chi), grad(0), 1e-7);
+    EXPECT_NEAR(dy1_dchi(t1, omega, chi) + dy1_dchi(t2, omega, chi), grad(1), 1e-7);
+
+    test_functor_sum_var_var<1> func2(TEST_CVODES_BDF);
+    stan::math::gradient(func2, x, f, grad);
+
+    EXPECT_NEAR(y2(t1, omega, chi) + y2(t2, omega, chi), f, 1e-8);
+    EXPECT_NEAR(dy2_domega(t1, omega, chi) + dy2_domega(t2, omega, chi), grad(0), 1e-6);
+    EXPECT_NEAR(dy2_dchi(t1, omega, chi) + dy2_dchi(t2, omega, chi), grad(1), 1e-7);
+  }
+}

From 71cefa144c7f8f064262dd1851b18da8e7de59f7 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Thu, 14 Jan 2021 08:52:25 +0000
Subject: [PATCH 042/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../functor/ode_bdf_adjoint_grad_rev_test.cpp | 26 ++++++++++++-------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
index 280dc4f259c..07d1ae4e907 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
@@ -260,8 +260,6 @@ TEST(StanMathOdeIntegrateODEGradMat, var_var) {
   }
 }
 
-
-
 template <int state>
 class test_functor_sum_var_var {
   int lmm_;
@@ -311,15 +309,19 @@ TEST(StanMathOdeIntegrateODEGradMat, sum_var_var) {
     stan::math::gradient(func1, x, f, grad);
 
     EXPECT_NEAR(y1(t1, omega, chi) + y1(t2, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy1_domega(t1, omega, chi) + dy1_domega(t2, omega, chi), grad(0), 1e-7);
-    EXPECT_NEAR(dy1_dchi(t1, omega, chi) + dy1_dchi(t2, omega, chi), grad(1), 1e-7);
+    EXPECT_NEAR(dy1_domega(t1, omega, chi) + dy1_domega(t2, omega, chi),
+                grad(0), 1e-7);
+    EXPECT_NEAR(dy1_dchi(t1, omega, chi) + dy1_dchi(t2, omega, chi), grad(1),
+                1e-7);
 
     test_functor_sum_var_var<1> func2(TEST_CVODES_ADAMS);
     stan::math::gradient(func2, x, f, grad);
 
     EXPECT_NEAR(y2(t1, omega, chi) + y2(t2, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy2_domega(t1, omega, chi) + dy2_domega(t2, omega, chi), grad(0), 1e-6);
-    EXPECT_NEAR(dy2_dchi(t1, omega, chi) + dy2_dchi(t2, omega, chi), grad(1), 1e-7);
+    EXPECT_NEAR(dy2_domega(t1, omega, chi) + dy2_domega(t2, omega, chi),
+                grad(0), 1e-6);
+    EXPECT_NEAR(dy2_dchi(t1, omega, chi) + dy2_dchi(t2, omega, chi), grad(1),
+                1e-7);
   }
 
   {
@@ -329,14 +331,18 @@ TEST(StanMathOdeIntegrateODEGradMat, sum_var_var) {
     stan::math::gradient(func1, x, f, grad);
 
     EXPECT_NEAR(y1(t1, omega, chi) + y1(t2, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy1_domega(t1, omega, chi) + dy1_domega(t2, omega, chi), grad(0), 1e-7);
-    EXPECT_NEAR(dy1_dchi(t1, omega, chi) + dy1_dchi(t2, omega, chi), grad(1), 1e-7);
+    EXPECT_NEAR(dy1_domega(t1, omega, chi) + dy1_domega(t2, omega, chi),
+                grad(0), 1e-7);
+    EXPECT_NEAR(dy1_dchi(t1, omega, chi) + dy1_dchi(t2, omega, chi), grad(1),
+                1e-7);
 
     test_functor_sum_var_var<1> func2(TEST_CVODES_BDF);
     stan::math::gradient(func2, x, f, grad);
 
     EXPECT_NEAR(y2(t1, omega, chi) + y2(t2, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy2_domega(t1, omega, chi) + dy2_domega(t2, omega, chi), grad(0), 1e-6);
-    EXPECT_NEAR(dy2_dchi(t1, omega, chi) + dy2_dchi(t2, omega, chi), grad(1), 1e-7);
+    EXPECT_NEAR(dy2_domega(t1, omega, chi) + dy2_domega(t2, omega, chi),
+                grad(0), 1e-6);
+    EXPECT_NEAR(dy2_dchi(t1, omega, chi) + dy2_dchi(t2, omega, chi), grad(1),
+                1e-7);
   }
 }

From d1280cff354c0bb9d555cc0a117140e70ec6750b Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 14 Jan 2021 10:08:28 +0100
Subject: [PATCH 043/157] fix linked mass flow example

---
 ode-adjoint/linked-mass-flow-buggy.stan | 154 ++++++++++++++++++++++++
 ode-adjoint/linked-mass-flow.R          |  16 +--
 ode-adjoint/linked-mass-flow.stan       |  36 +++++-
 3 files changed, 193 insertions(+), 13 deletions(-)
 create mode 100644 ode-adjoint/linked-mass-flow-buggy.stan

diff --git a/ode-adjoint/linked-mass-flow-buggy.stan b/ode-adjoint/linked-mass-flow-buggy.stan
new file mode 100644
index 00000000000..cfbe138daf2
--- /dev/null
+++ b/ode-adjoint/linked-mass-flow-buggy.stan
@@ -0,0 +1,154 @@
+functions {
+
+  vector linked_mass_flow(real t, vector y,
+                          vector kt, vector e50, vector k12, vector k21) {
+
+    int num_main_states = num_elements(kt);
+    int num_states = 2 * num_main_states;
+
+    vector[num_states] dydt;
+    vector[num_main_states] ksat;
+
+    for (i in 1:num_main_states) {
+      int m = 2 * (i-1) + 1;  // main state
+      int a = m + 1;  // auxilary state
+      ksat[i] = kt[i] * y[m] / (y[m] + e50[i]);
+
+      dydt[m] = -ksat[i] * y[m] - k12[i] * y[m] + k21[i] * y[a];
+      dydt[a] = +k12[i] * y[m] - k21[i] * y[a];
+
+      if (i != 1) {
+        dydt[m] += ksat[i-1] * y[2 * (i - 1) + 1];
+      }
+    }
+    return dydt;
+
+  }
+
+  vector sample_vector_rng(real m, real s, int N) {
+    vector[N] sample;
+    for(i in 1:N)
+      sample[i] = normal_rng(m, s);
+    return sample;
+  }
+
+  vector[] simulate_mean(real t0, vector log_a0, real[] ts, int adjoint_integrator,
+                         real rel_tol, real abs_tol, int max_num_steps,
+                         vector log_kt, vector log_e50, vector log_k12, vector log_k21) {
+    int num_sim = num_elements(ts);
+    int num_states = num_elements(log_a0);
+    vector[num_states] y[num_sim];
+
+    if(adjoint_integrator) {
+      y = ode_adams_tol(linked_mass_flow, exp(log_a0), t0, ts, rel_tol, abs_tol, max_num_steps,
+                        exp(log_kt), exp(log_e50), exp(log_k12), exp(log_k21));
+    } else {
+      y = ode_bdf_tol(linked_mass_flow, exp(log_a0), t0, ts, rel_tol, abs_tol, max_num_steps,
+                      exp(log_kt), exp(log_e50), exp(log_k12), exp(log_k21));
+    }
+
+    return y;
+  }
+  
+}
+data {
+  real<lower=0> rel_tol;
+  real<lower=0> abs_tol;
+  int<lower=0,upper=1> adjoint_integrator;
+  int<lower=1> max_num_steps;
+  int<lower=1> system_size;
+  int<lower=1> num_obs;
+  real<lower=0> sigma_sim;
+  real<lower=0> sigma_y;
+}
+transformed data {
+  int num_states = 2 * system_size;
+
+  vector[system_size] log_kt_ = sample_vector_rng(0.0, sigma_sim, system_size);
+  vector[system_size] log_e50_ = sample_vector_rng(0.0, sigma_sim, system_size);
+  vector[system_size] log_k12_ = sample_vector_rng(0.0, sigma_sim, system_size);
+  vector[system_size] log_k21_ = sample_vector_rng(0.0, sigma_sim, system_size);
+  vector[system_size] log_sigma_y_ = sample_vector_rng(0.0, sigma_y, system_size);
+  vector[num_states] log_a0_ = sample_vector_rng(0.0, sigma_sim, num_states);
+  vector[num_states] y_[num_obs];
+  real ts[num_obs];
+  real t0 = 0.0;
+
+  ts[1] = 1.0;
+  for(i in 2:num_obs) {
+    ts[i] = 1.5 * ts[i-1];
+  }
+
+  y_ = simulate_mean(t0, log_a0_, ts, 0, rel_tol, abs_tol, max_num_steps,
+                     log_kt_, log_e50_, log_k12_, log_k21_);
+  
+  for(i in 1:num_obs) {
+    for(j in 1:system_size) {
+      int m = 2*(j-1) + 1;
+      y_[i,m] = lognormal_rng(log(y_[i,m]+1E-3), exp(log_sigma_y_[j]));
+    }
+  }
+
+  if(adjoint_integrator) {
+    print("Using bdf_adjoint integrator.");
+  } else {
+    print("Using bdf integrator.");
+  }
+  print("relative tolerance: ", rel_tol);
+  print("absolute tolerance: ", abs_tol);
+  print("maximum number of steps: ", max_num_steps);
+  print("number of time points: ", num_obs);
+  print("system size: ", system_size);
+  print("time points: ", ts);
+  print("y_: ", y_);
+}
+parameters {
+  vector[system_size] log_kt;
+  vector[system_size] log_e50;
+  vector[system_size] log_k12;
+  vector[system_size] log_k21;
+  vector[system_size] log_sigma_y;
+  vector[num_states] log_a0;
+}
+transformed parameters {
+}
+model {
+  vector[num_states] mu[num_obs] = simulate_mean(t0, log_a0, ts, adjoint_integrator, rel_tol, abs_tol, max_num_steps,
+                                                 log_kt, log_e50, log_k12, log_k21);
+  
+  target += normal_lpdf(log_kt| 0.0, sigma_sim);
+  target += normal_lpdf(log_e50| 0.0, sigma_sim);
+  target += normal_lpdf(log_k12| 0.0, sigma_sim);
+  target += normal_lpdf(log_k21| 0.0, sigma_sim);
+  target += normal_lpdf(log_a0| 0.0, sigma_sim);
+  target += normal_lpdf(log_sigma_y| 0.0, sigma_y);
+
+  for(j in 1:system_size) {
+    int m = 2*(j-1) + 1;
+    target += lognormal_lpdf(to_vector(y_[:,m])| log(to_vector(mu[:,m])+1E-3), exp(log_sigma_y[j]));
+  }
+}
+generated quantities {
+  int rank_log_kt[system_size];
+  int rank_log_e50[system_size];
+  int rank_log_k12[system_size];
+  int rank_log_k21[system_size];
+  int rank_log_sigma_y[system_size];
+  int rank_log_a0[system_size];
+  vector[system_size] bias_log_kt = log_kt - log_kt_;
+  vector[system_size] bias_log_e50 = log_e50 - log_e50_;
+  vector[system_size] bias_log_k12 = log_k12 - log_k12_;
+  vector[system_size] bias_log_k21 = log_k21 - log_k21_;
+  vector[system_size] bias_log_sigma_y = log_sigma_y - log_sigma_y_;
+  vector[system_size] bias_log_a0 = log_a0 - log_a0_;
+
+  for(i in 1:system_size) {
+    rank_log_kt[i] = (log_kt[i] > log_kt_[i] ? 1 : 0);
+    rank_log_e50[i] = (log_e50[i] > log_e50_[i] ? 1 : 0);
+    rank_log_k12[i] = (log_k12[i] > log_k12_[i] ? 1 : 0);
+    rank_log_k21[i] = (log_k21[i] > log_k21_[i] ? 1 : 0);
+    rank_log_sigma_y[i] = (log_sigma_y[i] > log_sigma_y_[i] ? 1 : 0);
+    rank_log_a0[i] = (log_a0[i] > log_a0_[i] ? 1 : 0);
+  }
+
+}
diff --git a/ode-adjoint/linked-mass-flow.R b/ode-adjoint/linked-mass-flow.R
index 3896ec5c64e..020d1ab6b16 100644
--- a/ode-adjoint/linked-mass-flow.R
+++ b/ode-adjoint/linked-mass-flow.R
@@ -42,11 +42,11 @@ run_benchmark <- function(..., num_iter=250, adapt_delta=0.8) {
 bdf_fit <- run_benchmark(adjoint_integrator=0)
 adjoint_fit <- run_benchmark(adjoint_integrator=1)
 
-adjoint_fit_2 <- run_benchmark(adjoint_integrator=1, adapt_delta=0.65)
-
-adjoint_fit_3 <- run_benchmark(adjoint_integrator=1, adapt_delta=0.95)
-
-adjoint_fit_4 <- run_benchmark(adjoint_integrator=1, abs_tol=1E-10, rel_tol=1E-10)
+if(FALSE) {
+    adjoint_fit_2 <- run_benchmark(adjoint_integrator=1, adapt_delta=0.65)
+    adjoint_fit_3 <- run_benchmark(adjoint_integrator=1, adapt_delta=0.95)
+    adjoint_fit_4 <- run_benchmark(adjoint_integrator=1, abs_tol=1E-10, rel_tol=1E-10)
+}
 
 summarize_benchmark <- function(fit) {
     sdiag_all  <- fit$sampler_diagnostics(inc_warmup=TRUE)
@@ -58,12 +58,13 @@ summarize_benchmark <- function(fit) {
     num_leaps <- sum(sdiag_all[,,"n_leapfrog__"])
     fit_time <- fit$time()$total
 
-    ess_speed <- fit$summary("lp__")[1,c("ess_bulk", "ess_tail")] / fit_time
+    ess_speed <- as.vector(fit$summary("lp__")[1,c("ess_bulk", "ess_tail")]) / fit_time
+    lp_est <- as.vector(fit$summary("lp__"))
 
     list(num_leaps=num_leaps, time=fit_time, time_per_leap=1E3 * fit_time / num_leaps,
          step_size=fit$metadata()$step_size_adaptation,
          num_divergent=num_divergent, accept_stat=accept_stat, treedepth=treedepth,
-         ess_lp_speed=ess_speed)
+         ess_lp_speed=ess_speed, lp_est=lp_est)
 }
 
 mcmc_rhat(rhat = rhat(bdf_fit))  + yaxis_text(hjust = 0)
@@ -80,3 +81,4 @@ bdf_fit$metadata()
 bdf_fit$time()
 
 
+bdf_fit$summary("lp__")
diff --git a/ode-adjoint/linked-mass-flow.stan b/ode-adjoint/linked-mass-flow.stan
index cfbe138daf2..dac9ae9e7f0 100644
--- a/ode-adjoint/linked-mass-flow.stan
+++ b/ode-adjoint/linked-mass-flow.stan
@@ -1,7 +1,7 @@
 functions {
 
   vector linked_mass_flow(real t, vector y,
-                          vector kt, vector e50, vector k12, vector k21) {
+                          real[] kt, real[] e50, real[] k12, real[] k21) {
 
     int num_main_states = num_elements(kt);
     int num_states = 2 * num_main_states;
@@ -37,14 +37,31 @@ functions {
                          vector log_kt, vector log_e50, vector log_k12, vector log_k21) {
     int num_sim = num_elements(ts);
     int num_states = num_elements(log_a0);
+    int system_size = num_elements(log_kt);
     vector[num_states] y[num_sim];
+    real kt[system_size];
+    real e50[system_size];
+    real k12[system_size];
+    real k21[system_size];
+    vector[num_states] a0;
+
+    for(i in 1:system_size) {
+      kt[i] = exp(log_kt[i]);
+      e50[i] = exp(log_e50[i]);
+      k12[i] = exp(log_k12[i]);
+      k21[i] = exp(log_k21[i]);
+    }
+
+    for(i in 1:num_states) {
+        a0[i] = exp(log_a0[i]);
+    }
 
     if(adjoint_integrator) {
-      y = ode_adams_tol(linked_mass_flow, exp(log_a0), t0, ts, rel_tol, abs_tol, max_num_steps,
-                        exp(log_kt), exp(log_e50), exp(log_k12), exp(log_k21));
+      y = ode_adams_tol(linked_mass_flow, a0, t0, ts, rel_tol, abs_tol, max_num_steps,
+                        kt, e50, k12, k21);
     } else {
-      y = ode_bdf_tol(linked_mass_flow, exp(log_a0), t0, ts, rel_tol, abs_tol, max_num_steps,
-                      exp(log_kt), exp(log_e50), exp(log_k12), exp(log_k21));
+      y = ode_bdf_tol(linked_mass_flow, a0, t0, ts, rel_tol, abs_tol, max_num_steps,
+                      kt, e50, k12, k21);
     }
 
     return y;
@@ -123,10 +140,17 @@ model {
   target += normal_lpdf(log_a0| 0.0, sigma_sim);
   target += normal_lpdf(log_sigma_y| 0.0, sigma_y);
 
+  /*
   for(j in 1:system_size) {
     int m = 2*(j-1) + 1;
     target += lognormal_lpdf(to_vector(y_[:,m])| log(to_vector(mu[:,m])+1E-3), exp(log_sigma_y[j]));
   }
+  */
+  
+  for(j in 1:system_size) {
+    int m = 2*(j-1) + 1;
+    target += lognormal_lpdf(y_[num_obs,m]| log(mu[num_obs,m]+1E-3), exp(log_sigma_y[j]));
+  }
 }
 generated quantities {
   int rank_log_kt[system_size];
@@ -150,5 +174,5 @@ generated quantities {
     rank_log_sigma_y[i] = (log_sigma_y[i] > log_sigma_y_[i] ? 1 : 0);
     rank_log_a0[i] = (log_a0[i] > log_a0_[i] ? 1 : 0);
   }
-
+  
 }

From 61aa6f801338924ae3eefb4e6c3c1acd321dbe7b Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sebastian.weber@novartis.com>
Date: Sun, 17 Jan 2021 10:26:03 +0100
Subject: [PATCH 044/157] add SBC codes

---
 ode-adjoint/README.md                         |  10 ++
 ...buggy.stan => linked-mass-flow-eigen.stan} |   0
 ode-adjoint/linked-mass-flow.stan             |  20 +--
 ode-adjoint/sbc.R                             | 102 ++++++++++++
 ode-adjoint/sbc_report.R                      | 116 ++++++++++++++
 ode-adjoint/sbc_tools.R                       | 145 ++++++++++++++++++
 ode-adjoint/scale-benchmark.R                 | 138 +++++++++++++++++
 7 files changed, 518 insertions(+), 13 deletions(-)
 create mode 100644 ode-adjoint/README.md
 rename ode-adjoint/{linked-mass-flow-buggy.stan => linked-mass-flow-eigen.stan} (100%)
 create mode 100644 ode-adjoint/sbc.R
 create mode 100644 ode-adjoint/sbc_report.R
 create mode 100644 ode-adjoint/sbc_tools.R
 create mode 100644 ode-adjoint/scale-benchmark.R

diff --git a/ode-adjoint/README.md b/ode-adjoint/README.md
new file mode 100644
index 00000000000..7acb7af419b
--- /dev/null
+++ b/ode-adjoint/README.md
@@ -0,0 +1,10 @@
+Codes to run SBC & Speed Benchmarks for adjoint ODE
+
+The work is dispatched using batchtools (see https://mllg.github.io/batchtools/articles/batchtools.html ). Batchtools allows to use a cluster as backend or just your laptop.
+
+Files:
+
+sbc.R runs SBC
+sbc_report.R plots SBC results
+scale-benchmark.R Speed benchmark which varries the system size
+sbc_tools.R shared R functions
diff --git a/ode-adjoint/linked-mass-flow-buggy.stan b/ode-adjoint/linked-mass-flow-eigen.stan
similarity index 100%
rename from ode-adjoint/linked-mass-flow-buggy.stan
rename to ode-adjoint/linked-mass-flow-eigen.stan
diff --git a/ode-adjoint/linked-mass-flow.stan b/ode-adjoint/linked-mass-flow.stan
index dac9ae9e7f0..5889d4eab22 100644
--- a/ode-adjoint/linked-mass-flow.stan
+++ b/ode-adjoint/linked-mass-flow.stan
@@ -80,6 +80,7 @@ data {
 }
 transformed data {
   int num_states = 2 * system_size;
+  real param_scale = sigma_sim / sqrt(num_obs/4);
 
   vector[system_size] log_kt_ = sample_vector_rng(0.0, sigma_sim, system_size);
   vector[system_size] log_e50_ = sample_vector_rng(0.0, sigma_sim, system_size);
@@ -120,12 +121,12 @@ transformed data {
   print("y_: ", y_);
 }
 parameters {
-  vector[system_size] log_kt;
-  vector[system_size] log_e50;
-  vector[system_size] log_k12;
-  vector[system_size] log_k21;
-  vector[system_size] log_sigma_y;
-  vector[num_states] log_a0;
+  vector<multiplier=param_scale,offset=log_kt_>[system_size] log_kt;
+  vector<multiplier=param_scale,offset=log_e50_>[system_size] log_e50;
+  vector<multiplier=param_scale,offset=log_k12_>[system_size] log_k12;
+  vector<multiplier=param_scale,offset=log_k21_>[system_size] log_k21;
+  vector<offset=log_sigma_y_>[system_size] log_sigma_y;
+  vector<multiplier=param_scale,offset=log_a0_>[num_states] log_a0;
 }
 transformed parameters {
 }
@@ -140,17 +141,10 @@ model {
   target += normal_lpdf(log_a0| 0.0, sigma_sim);
   target += normal_lpdf(log_sigma_y| 0.0, sigma_y);
 
-  /*
   for(j in 1:system_size) {
     int m = 2*(j-1) + 1;
     target += lognormal_lpdf(to_vector(y_[:,m])| log(to_vector(mu[:,m])+1E-3), exp(log_sigma_y[j]));
   }
-  */
-  
-  for(j in 1:system_size) {
-    int m = 2*(j-1) + 1;
-    target += lognormal_lpdf(y_[num_obs,m]| log(mu[num_obs,m]+1E-3), exp(log_sigma_y[j]));
-  }
 }
 generated quantities {
   int rank_log_kt[system_size];
diff --git a/ode-adjoint/sbc.R b/ode-adjoint/sbc.R
new file mode 100644
index 00000000000..550edaba257
--- /dev/null
+++ b/ode-adjoint/sbc.R
@@ -0,0 +1,102 @@
+library(cmdstanr)
+library(posterior)
+library(batchtools)
+set.seed(453453)
+## load utilities and current dev version of OncoBayes2
+source("sbc_tools.R")
+
+#' according to the docs this speeds up the reduce step
+options(batchtools.progress = FALSE)
+
+registry_tmp <- Sys.getenv("TMP_NFS", tempdir())
+
+reg <- makeExperimentRegistry(
+    file.dir = tempfile("sbc-", registry_tmp),
+    ## use the default configured batchtools configuration batchtools.conf
+    ## - found in the environemnt variable R_BATCHTOOLS_SEARCH_PATH
+    ## - current working directory
+    ## - $(HOME)/.batchtools.conf
+    ## conf.file = NA,
+    seed = 47845854,
+    ## our worker functions and package loading
+    source="sbc_tools.R")
+
+## resources of each job: Less than 55min, 2000MB RAM and 1 core
+job_resources <- list(walltime=55, memory=2000, ncpus=1, max.concurrent.jobs=300)
+
+## choose number of sample as being dividable by 2 minus one. This gives us exactly by 2 divisable number of bins.
+base_data  <- list(num_warmup=2^8 - 1, num_sampling=2^8 - 1, adapt_delta=0.9, stan_model=stan_model)
+
+addProblem("linked_mass_flow",
+           data = base_data,
+           fun = setup_system,
+           seed = 2345,
+           ## caching speeds up the reduce step
+           cache = TRUE
+)
+
+addAlgorithm("stan_cvodes", run_benchmark)
+
+pdes <- list(linked_mass_flow = data.frame(system_size=c(1,4)))
+ades <- list(stan_cvodes= data.frame(adjoint_integrator=c(0,1)))
+
+##addExperiments(pdes, ades, repls = 1000)
+addExperiments(pdes, ades, repls = 2)
+
+summarizeExperiments()
+
+## approximate per job runtime in s
+target_runtime <- 100
+## for 30 min runtime we then need
+chunk_size <- round(30 * 60 / target_runtime)
+chunk_size
+
+##chunk_size <- 10
+
+ids <- unwrap(getJobPars())
+
+ids[,chunk:=chunk(job.id, chunk.size=chunk_size)]
+
+auto_submit(ids, reg, job_resources)
+
+
+if(FALSE) {
+job1 <- testJob(4000)
+
+names(job1)
+
+
+job1$summary("lp__")
+job1$draws("lp__")
+
+job1$metadata()$iter_sampling
+
+as.list(job1$rank)
+
+job1_def <- makeJob(4000)
+
+job1_def$instance
+
+job1_def$repl
+
+names(job1_def$problem$data)
+}
+
+calibration <- ijoin(
+  ## grab job parameters
+  unwrap(getJobPars()),
+    unwrap(reduceResultsDataTable(fun=function(x) c(as.list(x$rank),
+                                      ess_speed=as.list(x$ess_lp_speed),
+                                      x[c("num_leaps", "time", "time_per_leap", "num_divergent")]
+                                                    )) )
+    )
+
+
+
+head(calibration)
+
+saveRDS(calibration, file = "calibration.rds")
+
+removeRegistry(0)
+
+sessionInfo()
diff --git a/ode-adjoint/sbc_report.R b/ode-adjoint/sbc_report.R
new file mode 100644
index 00000000000..caa3a99a7cc
--- /dev/null
+++ b/ode-adjoint/sbc_report.R
@@ -0,0 +1,116 @@
+#' ---
+#' title: Simulation based calibration for CVODES solvers
+#' author: ""
+#' date: "`r date()`"
+#' output: html_vignette
+#' params:
+#'   include_plots: FALSE
+#' ---
+#'
+#+ include=FALSE
+source("sbc_tools.R")
+library(knitr)
+library(tools)
+library(assertthat)
+library(dplyr)
+library(tidyr)
+library(broom)
+library(ggplot2)
+theme_set(theme_bw())
+
+knitr::opts_chunk$set(
+  fig.width = 1.62*4,
+  fig.height = 4,
+  cache=FALSE,
+  echo=FALSE
+    )
+
+calibration <- readRDS("calibration.rds")
+
+plot_binned <- function(cal_df) {
+
+    pl <- NULL
+
+    if(!include_plots)
+        return(pl)
+    
+    if(!all(cal_df$count == 0)) {
+        
+        S <- calibration$S
+        B <- calibration$B
+        
+        c95 <- qbinom(c(0.025, 0.5, 0.975), S, 1 / B)
+        
+        dd <- cal_df %>%
+            arrange(param, bin) %>%
+                group_by(param) %>%
+                    mutate(ecdf = cumsum(count) / S, ecdf_ref = (bin + 1) / B) %>%
+                        filter(!all(ecdf == 0))
+        
+        nparam <- length(unique(dd$param))
+        if(unique(dd$partype) %in% c("mu_eta", "tau_eta")){
+            nc <- nparam
+        } else{
+            nc <- 2
+        }
+        
+        nr <- max(1, ceiling(nparam / nc))
+
+        pl <- list()
+        pl[["hist"]] <- ggplot(dd, aes(bin, count)) +
+            facet_wrap(~ param, nrow = nr, ncol = nc) +
+                geom_col() +
+                    geom_hline(yintercept=c95[c(1,3)], linetype=I(2)) +
+                        geom_hline(yintercept=c95[c(2)], linetype=I(3))
+        pl[["ecdf_diff"]] <- ggplot(dd, aes(bin, ecdf-ecdf_ref)) +
+            facet_wrap(~ param, nrow = nr, ncol = nc) +
+                geom_step() +
+                    geom_hline(yintercept=0, linetype=I(3))
+        pl
+    }
+
+    return(pl)
+}
+
+
+dim(calibration)
+rank_params <- names(calibration)[grepl(names(calibration), pattern = "rank")]
+
+# calibration_data_binned <- calibration_data[, scale64(.SD), by=c("problem", "model", params)]
+
+sampled_ranks <- 2^8
+B <- sampled_ranks/ 2^3
+B
+
+calibration_binned <- calibration %>%
+    mutate_at(.vars = rank_params, .funs = function(x) ceiling((x + 1) / (sampled_ranks/ B) - 1)) %>%
+        mutate(adjoint_integrator=factor(adjoint_integrator))
+
+
+long <- calibration_binned[c("system_size", "adjoint_integrator", rank_params)] %>%
+    tidyr::gather(key = "param", value = "rank", -system_size, -adjoint_integrator) %>%
+        group_by(system_size, adjoint_integrator, param)  %>%
+            mutate(all_NA=all(is.na(rank))) %>%
+                filter(!all_NA)
+
+metrics_long <- calibration_binned[!(names(calibration_binned) %in% rank_params)] %>%
+  tidyr::gather(key = "metric", value = "value", -system_size, -adjoint_integrator, -job.id, -problem, -algorithm) 
+
+long_split <- split(long, long$system_size)
+
+pl_rank <- lapply(long_split, function(d) ggplot(d, aes(rank)) + facet_grid(param~adjoint_integrator*system_size, labeller=label_both) + geom_bar() + theme(strip.text.y = element_text(angle = 0)))
+
+pl_rank[[1]] + theme(strip.text.y = element_text(angle = 0))
+pl_rank[[2]]
+
+ggsave("rank_size_1.png", pl_rank[[1]], width=6, height=6)
+ggsave("rank_size_4.png", pl_rank[[2]], width=6, height=14)
+
+metrics_split <- split(metrics_long, metrics_long$system_size)
+
+pl_metrics <- lapply(metrics_split, function(d) ggplot(d, aes(adjoint_integrator, value)) +
+    facet_wrap(~ metric*system_size, labeller=label_both, scales="free_y") + geom_boxplot()+
+        theme(strip.text.y = element_text(angle = 0)))
+
+ggsave("metrics_1.png", pl_metrics[[1]], width=10, height=10)
+ggsave("metrics_4.png", pl_metrics[[2]], width=10, height=10)
diff --git a/ode-adjoint/sbc_tools.R b/ode-adjoint/sbc_tools.R
new file mode 100644
index 00000000000..c07e829c52e
--- /dev/null
+++ b/ode-adjoint/sbc_tools.R
@@ -0,0 +1,145 @@
+library(cmdstanr)
+library(posterior)
+library(bayesplot)
+color_scheme_set("brightblue")
+
+if(!("sbc_dir" %in%  ls())) {
+    sbc_dir <- getwd()
+}
+
+stan_model_file <- normalizePath(file.path(sbc_dir, "linked-mass-flow.stan"))
+
+stan_model <- cmdstan_model(stan_model_file)
+
+setup_system <- function(data, job, ...) {
+    args <- list(...)
+    base_stan_data <- list(rel_tol=1e-6,
+                           abs_tol=1e-6,
+                           max_num_steps=10000,
+                           adjoint_integrator=0,
+                           system_size=1,
+                           num_obs=8,
+                           sigma_sim=1.0,
+                           sigma_y=0.2)
+    modifyList(base_stan_data, args)
+}
+
+run_benchmark <- function(data, job, instance, adjoint_integrator, ...) {
+    fit_data <- modifyList(instance, list(adjoint_integrator=adjoint_integrator))
+    r <- job$repl
+    fit <- data$stan_model$sample(
+        data = fit_data,
+        iter_warmup = data$num_warmup,
+        iter_sampling = data$num_sampling,
+        seed = 123 + r,
+        init=0,
+        chains = 1,
+        parallel_chains = 1,
+        refresh = 100,
+        save_warmup=TRUE,
+        adapt_delta=data$adapt_delta
+        ##verbose=TRUE
+        )
+    summarize_benchmark(fit)
+}
+
+
+summarize_benchmark <- function(fit) {
+    sdiag_all  <- fit$sampler_diagnostics(inc_warmup=TRUE)
+    sdiag_main  <- fit$sampler_diagnostics(inc_warmup=FALSE)
+
+    num_divergent <- sum(sdiag_main[,,"divergent__"])
+    accept_stat <- mean(sdiag_main[,,"accept_stat__"])
+    treedepth <- mean(sdiag_main[,,"treedepth__"])
+    num_leaps <- sum(sdiag_all[,,"n_leapfrog__"])
+    fit_time <- fit$time()$total
+
+    ess_speed <- as.vector(unlist(fit$summary("lp__")[1,c("ess_bulk", "ess_tail")])) / fit_time
+    names(ess_speed) <- c("bulk", "tail")
+    lp_est <- as.vector(fit$summary("lp__"))
+
+    rank_vars <- grep("^rank", fit$metadata()$stan_variables, value=TRUE)
+    ranks <- fit$summary(rank_vars)
+    num_iter <- fit$metadata()$iter_sampling
+
+    count_ranks <- ranks$mean * num_iter
+    names(count_ranks) <- ranks$variable
+
+    list(num_leaps=num_leaps, time=fit_time, time_per_leap=1E3 * fit_time / num_leaps,
+         step_size=fit$metadata()$step_size_adaptation,
+         num_divergent=num_divergent, accept_stat=accept_stat, treedepth=treedepth,
+         ess_lp_speed=ess_speed, lp_est=lp_est, ranks=count_ranks)
+}
+
+
+## Submits to batchtools cluster with fault tolerance, i.e.
+## resubmitting failed jobs max_num_tries times
+auto_submit <- function(jobs, registry, resources=list(), max_num_tries = 10) {
+  all_unfinished_jobs <- jobs
+
+  num_unfinished_jobs <- nrow(all_unfinished_jobs)
+  num_all_jobs <- num_unfinished_jobs
+  remaining_tries <- max_num_tries
+  all_jobs_finished <- FALSE
+  while (remaining_tries > 0 && !all_jobs_finished) {
+    remaining_tries <- remaining_tries - 1
+
+    message("Submitting jobs at ", Sys.time())
+    # Once things run fine let's submit this work to the cluster.
+    submitJobs(all_unfinished_jobs, resources=resources)
+    # Wait for results.
+    waitForJobs()
+    message("Finished waiting for jobs at ", Sys.time())
+
+    # Check status:
+    print(getStatus())
+
+    # Ensure that all jobs are done
+    if (nrow(findNotDone()) != 0) {
+      not_done_jobs <- findNotDone()
+      print(getErrorMessages(not_done_jobs))
+      ##browser()
+      ##invisible(readline(prompt="Press [enter] to continue"))
+
+      message("Some jobs did not complete. Please check the batchtools registry ", registry$file.dir)
+      all_unfinished_jobs <- inner_join(not_done_jobs, all_unfinished_jobs)
+
+      if (num_unfinished_jobs == nrow(all_unfinished_jobs) &&  nrow(all_unfinished_jobs) > 0.25 * num_all_jobs)
+      {
+        # Unfinished job count did not change -> retrying will probably not help. Abort!
+        warning("Error: unfinished job count is not decreasing. Aborting job retries.")
+        remaining_tries <- 0
+      }
+
+      if (num_unfinished_jobs == nrow(jobs))
+      {
+        # All jobs errored -> retrying will probably not help. Abort!
+        warning("Error: all jobs errored. Aborting job retries.")
+        remaining_tries <- 0
+      }
+
+      num_unfinished_jobs <- nrow(all_unfinished_jobs)
+      message("Trying to resubmit jobs. Remaining tries: ", remaining_tries, " / ", max_num_tries)
+    } else {
+      all_jobs_finished <- TRUE
+    }
+  }
+
+  invisible(all_jobs_finished)
+}
+
+
+reduce_run <- function(x) {
+    c(as.list(x$rank),
+      ess_speed=as.list(x$ess_lp_speed),
+      x[c("num_leaps", "time", "time_per_leap", "num_divergent")]
+      )
+}
+
+reduce_run_bench <- function(x) {
+    c(
+      ess_speed=as.list(x$ess_lp_speed),
+      x[c("num_leaps", "time", "time_per_leap", "num_divergent")]
+      )
+}
+
diff --git a/ode-adjoint/scale-benchmark.R b/ode-adjoint/scale-benchmark.R
new file mode 100644
index 00000000000..f35c98ec215
--- /dev/null
+++ b/ode-adjoint/scale-benchmark.R
@@ -0,0 +1,138 @@
+library(cmdstanr)
+library(posterior)
+library(batchtools)
+set.seed(453453)
+## load utilities and current dev version of OncoBayes2
+source("sbc_tools.R")
+
+#' according to the docs this speeds up the reduce step
+options(batchtools.progress = FALSE)
+
+registry_tmp <- Sys.getenv("TMP_NFS", tempdir())
+
+reg <- makeExperimentRegistry(
+    file.dir = tempfile("scale-bench-", registry_tmp),
+    ## use the default configured batchtools configuration batchtools.conf
+    ## - found in the environemnt variable R_BATCHTOOLS_SEARCH_PATH
+    ## - current working directory
+    ## - $(HOME)/.batchtools.conf
+    ## conf.file = NA,
+    seed = 47845854,
+    ## our worker functions and package loading
+    source="sbc_tools.R")
+
+## resources of each job: Less than 4h, 2000MB RAM and 1 core
+job_resources <- list(walltime=4*60, memory=2000, ncpus=1, max.concurrent.jobs=300)
+
+## choose number of sample as being dividable by 2 minus one. This gives us exactly by 2 divisable number of bins.
+base_data  <- list(num_warmup=20, num_sampling=20, adapt_delta=0.9, stan_model=stan_model)
+
+addProblem("linked_mass_flow",
+           data = base_data,
+           fun = setup_system,
+           seed = 2345,
+           ## caching speeds up the reduce step
+           cache = TRUE
+)
+
+addAlgorithm("stan_cvodes", run_benchmark)
+
+pdes <- list(linked_mass_flow = data.frame(system_size=c(1,2,4,8,16)))
+ades <- list(stan_cvodes= data.frame(adjoint_integrator=c(0,1)))
+
+##addExperiments(pdes, ades, repls = 1000)
+addExperiments(pdes, ades, repls = 2)
+
+summarizeExperiments()
+
+## approximate per job runtime in s
+target_runtime <- 100
+## for 30 min runtime we then need
+chunk_size <- round(30 * 60 / target_runtime)
+chunk_size
+
+##chunk_size <- 10
+
+ids <- unwrap(getJobPars())
+
+##ids[,chunk:=chunk(job.id, chunk.size=chunk_size)]
+
+##auto_submit(ids, reg, job_resources)
+
+submitJobs(ids)
+
+getStatus()
+
+getLog(3)
+
+if(FALSE) {
+
+    job1 <- testJob(3)
+
+    names(job1)
+
+    job1
+
+
+job1$summary("lp__")
+job1$draws("lp__")
+
+job1$metadata()$iter_sampling
+
+as.list(job1$rank)
+
+job1_def <- makeJob(4)
+
+job1_def$instance
+
+job1_def$repl
+
+names(job1_def$problem$data)
+}
+
+
+library(ggplot2)
+library(dplyr)
+
+scale_benchmark <- ijoin(
+  ## grab job parameters
+  unwrap(getJobPars()),
+  unwrap(reduceResultsDataTable(fun=reduce_run_bench))
+)
+
+head(scale_benchmark)
+
+scale_benchmark <- mutate(scale_benchmark,
+                          adjoint_integrator=factor(adjoint_integrator))
+
+saveRDS(scale_benchmark, file = "scale_benchmark.rds")
+
+removeRegistry(0)
+
+sessionInfo()
+
+scale_benchmark <- as.data.frame(scale_benchmark)
+scale_benchmark <- mutate(scale_benchmark, repl=factor(rep(1:2, 10)), set=paste(repl,adjoint_integrator,sep="-"))
+
+
+scale_benchmark_ref <- left_join(scale_benchmark, filter(scale_benchmark, adjoint_integrator==0)[c("repl", "system_size", "time_per_leap")], suffix=c("", "_ref"), by=c("repl", "system_size"))
+
+scale_benchmark_ref
+
+theme_set(theme_bw())
+ggplot(scale_benchmark, aes(system_size, time_per_leap, group=set, colour=adjoint_integrator, shape=repl)) +
+   geom_point()  + geom_line() +
+    scale_x_log10(breaks=c(1,2,4,8,16)) +
+    scale_y_log10() +
+    ggtitle("Time per leapfrog of adjoint and forward per replication")
+
+ggsave("scale_time_per_leap.png", width=1.6 *4, height=4, dpi=120)
+
+ggplot(filter(scale_benchmark_ref, adjoint_integrator==1), aes(system_size, time_per_leap_ref/time_per_leap, group=set, shape=repl)) +
+   geom_point()  + geom_line() +
+    scale_x_log10(breaks=c(1,2,4,8,16)) +
+    scale_y_log10() +
+    geom_hline(yintercept=1.0, linetype=2) +
+    ggtitle("Speedup of adjoint vs forward per replication")
+
+ggsave("scale_speedup.png", width=1.6 *4, height=4, dpi=120)

From 4cbdb043ea12bcd965d9db1a9faf2a5e0170a384 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sebastian.weber@novartis.com>
Date: Sun, 17 Jan 2021 10:29:04 +0100
Subject: [PATCH 045/157] bump

---
 ode-adjoint/README.md | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/ode-adjoint/README.md b/ode-adjoint/README.md
index 7acb7af419b..f7b9e740fed 100644
--- a/ode-adjoint/README.md
+++ b/ode-adjoint/README.md
@@ -1,6 +1,13 @@
-Codes to run SBC & Speed Benchmarks for adjoint ODE
+# Codes to run SBC & Speed Benchmarks for adjoint ODE
 
-The work is dispatched using batchtools (see https://mllg.github.io/batchtools/articles/batchtools.html ). Batchtools allows to use a cluster as backend or just your laptop.
+The work is dispatched using batchtools (see
+https://mllg.github.io/batchtools/articles/batchtools.html
+). Batchtools allows to use a cluster as backend or just your laptop.
+
+The R codes rely on cmdstanr. So please ensure that the environment
+variable CMDSTAN points to a properly setup cmdstan checkout which
+itself is configured to use a math checkout of the branch
+feature/adjoint-odes .
 
 Files:
 

From ff1571f8b6532b6f250454cd067593be1c96af4c Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 18 Jan 2021 21:16:12 +0100
Subject: [PATCH 046/157] adding eval and some notes on how the math works out

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 25 +++++++++++++--
 stan/math/rev/functor/ode_bdf_adjoint.hpp     | 31 ++++++++++++++++---
 2 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 03bb076459c..79d8b679b57 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -262,6 +262,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Calculate the adjoint sensitivity RHS for varying initial conditions
    * and parameters
    *
+   * Equation 2.23 in the cvs_guide.
+   *
    * @param[in] initial var vector
    * @param[in] param var vector
    * @param[in] t time
@@ -300,7 +302,10 @@ class cvodes_integrator_adjoint_vari : public vari {
   }
 
   /*
-   * Calculate the RHS for the quadrature part of the adjoint ODE problem.
+   * Calculate the RHS for the quadrature part of the adjoint ODE
+   * problem.
+   *
+   * This is the integrand of equation 2.22 in the cvs_guide.
    *
    * @param[in] initial var vector
    * @param[in] param var vector
@@ -667,9 +672,12 @@ class cvodes_integrator_adjoint_vari : public vari {
         }
 
         double t_final = value_of((i > 0) ? memory->ts_[i - 1] : memory->t0_);
+        //std::cout << "time-point " << i << "; t_init = " << t_init << "; t_final = " << t_final << std::endl;
         if (t_final != t_init) {
           if (!cvodes_backward_initialized) {
-            // initialize CVODES backward machinery
+            // initialize CVODES backward machinery.
+            // the states of the backward problem *are* the adjoints
+            // of the ode states
             check_flag_sundials(
                 CVodeInitB(memory->cvodes_mem_, indexB,
                            &cvodes_integrator_adjoint_vari::cv_rhs_adj_sens,
@@ -695,7 +703,13 @@ class cvodes_integrator_adjoint_vari : public vari {
                     &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
                 "CVodeSetJacFnB");
 
-            // Allocate space for backwards quadrature
+            // Allocate space for backwards quadrature needed when
+            // parameters vary. The function for which we seek the
+            // derivative wrt to all parameters p is (in notation of
+            // the CVODES manual)
+            // g(t,y,p) = sum_i=1^N y_i * a^y_i * delta(t-T)
+            // this is the sum over each state y_i multiplied with
+            // it's adjoint and a delta function which peaks at t=T
             if (args_vars_ > 0) {
               check_flag_sundials(
                   CVodeQuadInitB(
@@ -743,6 +757,8 @@ class cvodes_integrator_adjoint_vari : public vari {
           // CV_NORMAL),
           //                    "CVodeB");
 
+          // obtain adjoint states and update t_init to time point
+          // reached of t_final
           check_flag_sundials(
               CVodeGetB(memory->cvodes_mem_, indexB, &t_init, nv_state_sens),
               "CVodeGetB");
@@ -764,11 +780,14 @@ class cvodes_integrator_adjoint_vari : public vari {
             memory->value_of_args_tuple_);
       }
 
+      // do we need this a 2nd time? Don't think so.
+      /*
       if (args_vars_ > 0) {
         check_flag_sundials(
             CVodeGetQuadB(memory->cvodes_mem_, indexB, &t_init, nv_quad),
             "CVodeGetQuadB");
       }
+      */
 
       // After integrating all the way back to t0, we finally have the
       // the adjoints we wanted
diff --git a/stan/math/rev/functor/ode_bdf_adjoint.hpp b/stan/math/rev/functor/ode_bdf_adjoint.hpp
index c644eb192cd..5c817d3786d 100644
--- a/stan/math/rev/functor/ode_bdf_adjoint.hpp
+++ b/stan/math/rev/functor/ode_bdf_adjoint.hpp
@@ -2,6 +2,7 @@
 #define STAN_MATH_REV_FUNCTOR_ODE_BDF_ADJOINT_HPP
 
 #include <stan/math/rev/meta.hpp>
+#include <stan/math/prim/fun/eval.hpp>
 #include <stan/math/rev/functor/cvodes_integrator_adjoint.hpp>
 #include <ostream>
 #include <vector>
@@ -61,16 +62,38 @@ ode_bdf_adjoint_tol_impl(const char* function_name, const F& f, const T_y0& y0,
         auto integrator
             = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0,
   T_t0, T_ts, T_Args...>( function_name, f, y0, t0, ts, relative_tolerance,
-  absolute_tolerance, max_num_steps, msgs, args_refs...); return
-  (*integrator)();
+                          absolute_tolerance, absolute_tolerance_B, absolute_tolerance_QB,
+                          steps_checkpoint, max_num_steps, msgs, args_refs...);
+        return (*integrator)();
       }, args_ref_tuple);
-  */
+      */
+  /*
+  const auto& args_eval_tuple = std::make_tuple(eval(args)...);
+  return apply(
+      [&](const auto&... args_eval) {
+        auto integrator
+            = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0,
+  T_t0, T_ts, T_Args...>( function_name, f, y0, t0, ts, relative_tolerance,
+                          absolute_tolerance, absolute_tolerance_B, absolute_tolerance_QB,
+                          steps_checkpoint, max_num_steps, msgs, args_eval...);
+        return (*integrator)();
+      }, args_eval_tuple);
+      */
+  /*
   auto integrator
       = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0, T_t0,
                                                        T_ts, T_Args...>(
           function_name, f, y0, t0, ts, relative_tolerance, absolute_tolerance,
           absolute_tolerance_B, absolute_tolerance_QB, steps_checkpoint,
-          max_num_steps, msgs, args...);
+          max_num_steps, msgs, eval(args)...);
+  return (*integrator)();
+  */
+  auto integrator
+      = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, plain_type_t<T_y0>, T_t0,
+                                                       T_ts, plain_type_t<T_Args>...>(
+                                                           function_name, f, eval(y0), t0, ts, relative_tolerance, absolute_tolerance,
+          absolute_tolerance_B, absolute_tolerance_QB, steps_checkpoint,
+          max_num_steps, msgs, eval(args)...);
   return (*integrator)();
 }
 

From ed3ce6efe01f3fbdd603d5500181aa75d2484c29 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Mon, 18 Jan 2021 20:17:36 +0000
Subject: [PATCH 047/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp |  3 ++-
 stan/math/rev/functor/ode_bdf_adjoint.hpp     | 19 +++++++++----------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 79d8b679b57..7fad717e04c 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -672,7 +672,8 @@ class cvodes_integrator_adjoint_vari : public vari {
         }
 
         double t_final = value_of((i > 0) ? memory->ts_[i - 1] : memory->t0_);
-        //std::cout << "time-point " << i << "; t_init = " << t_init << "; t_final = " << t_final << std::endl;
+        // std::cout << "time-point " << i << "; t_init = " << t_init << ";
+        // t_final = " << t_final << std::endl;
         if (t_final != t_init) {
           if (!cvodes_backward_initialized) {
             // initialize CVODES backward machinery.
diff --git a/stan/math/rev/functor/ode_bdf_adjoint.hpp b/stan/math/rev/functor/ode_bdf_adjoint.hpp
index 5c817d3786d..bd973f4d00b 100644
--- a/stan/math/rev/functor/ode_bdf_adjoint.hpp
+++ b/stan/math/rev/functor/ode_bdf_adjoint.hpp
@@ -62,8 +62,8 @@ ode_bdf_adjoint_tol_impl(const char* function_name, const F& f, const T_y0& y0,
         auto integrator
             = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0,
   T_t0, T_ts, T_Args...>( function_name, f, y0, t0, ts, relative_tolerance,
-                          absolute_tolerance, absolute_tolerance_B, absolute_tolerance_QB,
-                          steps_checkpoint, max_num_steps, msgs, args_refs...);
+                          absolute_tolerance, absolute_tolerance_B,
+  absolute_tolerance_QB, steps_checkpoint, max_num_steps, msgs, args_refs...);
         return (*integrator)();
       }, args_ref_tuple);
       */
@@ -74,8 +74,8 @@ ode_bdf_adjoint_tol_impl(const char* function_name, const F& f, const T_y0& y0,
         auto integrator
             = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0,
   T_t0, T_ts, T_Args...>( function_name, f, y0, t0, ts, relative_tolerance,
-                          absolute_tolerance, absolute_tolerance_B, absolute_tolerance_QB,
-                          steps_checkpoint, max_num_steps, msgs, args_eval...);
+                          absolute_tolerance, absolute_tolerance_B,
+  absolute_tolerance_QB, steps_checkpoint, max_num_steps, msgs, args_eval...);
         return (*integrator)();
       }, args_eval_tuple);
       */
@@ -88,12 +88,11 @@ ode_bdf_adjoint_tol_impl(const char* function_name, const F& f, const T_y0& y0,
           max_num_steps, msgs, eval(args)...);
   return (*integrator)();
   */
-  auto integrator
-      = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, plain_type_t<T_y0>, T_t0,
-                                                       T_ts, plain_type_t<T_Args>...>(
-                                                           function_name, f, eval(y0), t0, ts, relative_tolerance, absolute_tolerance,
-          absolute_tolerance_B, absolute_tolerance_QB, steps_checkpoint,
-          max_num_steps, msgs, eval(args)...);
+  auto integrator = new stan::math::cvodes_integrator_adjoint_vari<
+      CV_BDF, F, plain_type_t<T_y0>, T_t0, T_ts, plain_type_t<T_Args>...>(
+      function_name, f, eval(y0), t0, ts, relative_tolerance,
+      absolute_tolerance, absolute_tolerance_B, absolute_tolerance_QB,
+      steps_checkpoint, max_num_steps, msgs, eval(args)...);
   return (*integrator)();
 }
 

From 9735ffdc26b1d82f81294911ad388808baf1f41a Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 22 Feb 2021 21:28:10 +0100
Subject: [PATCH 048/157] revert adams back to actual adams

---
 stan/math/rev/functor/ode_adams.hpp | 33 ++++-------------------------
 1 file changed, 4 insertions(+), 29 deletions(-)

diff --git a/stan/math/rev/functor/ode_adams.hpp b/stan/math/rev/functor/ode_adams.hpp
index f66037f98e1..ee0bdafbbd5 100644
--- a/stan/math/rev/functor/ode_adams.hpp
+++ b/stan/math/rev/functor/ode_adams.hpp
@@ -3,7 +3,6 @@
 
 #include <stan/math/rev/meta.hpp>
 #include <stan/math/rev/functor/cvodes_integrator.hpp>
-#include <stan/math/rev/functor/ode_bdf_adjoint.hpp>
 #include <stan/math/prim/fun/Eigen.hpp>
 #include <ostream>
 #include <vector>
@@ -53,10 +52,7 @@ ode_adams_tol_impl(const char* function_name, const F& f, const T_y0& y0,
                    const T_t0& t0, const std::vector<T_ts>& ts,
                    double relative_tolerance, double absolute_tolerance,
                    long int max_num_steps,  // NOLINT(runtime/int)
-                   std::ostream* msgs, double absolute_tolerance_B,
-                   double absolute_tolerance_QB, long int steps_checkpoint,
-                   const T_Args&... args) {
-  /*
+                   std::ostream* msgs, const T_Args&... args) {
   const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
   return apply(
       [&](const auto&... args_refs) {
@@ -67,11 +63,6 @@ ode_adams_tol_impl(const char* function_name, const F& f, const T_y0& y0,
         return integrator();
       },
       args_ref_tuple);
-  */
-  return ode_bdf_adjoint_tol(f, y0, t0, ts, relative_tolerance,
-                             absolute_tolerance, max_num_steps, msgs,
-                             absolute_tolerance_B, absolute_tolerance_QB,
-                             steps_checkpoint, args...);
 }
 
 /**
@@ -116,19 +107,8 @@ ode_adams_tol(const F& f, const T_y0& y0, const T_t0& t0,
               double absolute_tolerance,
               long int max_num_steps,  // NOLINT(runtime/int)
               std::ostream* msgs, const T_Args&... args) {
-  // sensitivities are output with absolute_tolerance
-  double absolute_tolerance_QB = absolute_tolerance;
-  // backward solve is 10x more precise than quadrature
-  double absolute_tolerance_B = absolute_tolerance_QB / 10.0;
-  // forward solve is 10x more precise that backward solve
-  double absolute_tolerance_F = absolute_tolerance_B / 10.0;
-
-  long int steps_checkpoint = 100;
-
   return ode_adams_tol_impl("ode_adams_tol", f, y0, t0, ts, relative_tolerance,
-                            absolute_tolerance_F, max_num_steps, msgs,
-                            absolute_tolerance_B, absolute_tolerance_QB,
-                            steps_checkpoint, args...);
+                            absolute_tolerance, max_num_steps, msgs, args...);
 }
 
 /**
@@ -169,16 +149,11 @@ ode_adams(const F& f, const T_y0& y0, const T_t0& t0,
           const std::vector<T_ts>& ts, std::ostream* msgs,
           const T_Args&... args) {
   double relative_tolerance = 1e-10;
-  double absolute_tolerance_QB = 1e-10;
+  double absolute_tolerance = 1e-10;
   long int max_num_steps = 1e8;  // NOLINT(runtime/int)
-  double absolute_tolerance_B = absolute_tolerance_QB / 10.0;
-  double absolute_tolerance_F = absolute_tolerance_B / 10.0;
-  long int steps_checkpoint = 100;
 
   return ode_adams_tol_impl("ode_adams", f, y0, t0, ts, relative_tolerance,
-                            absolute_tolerance_F, max_num_steps, msgs,
-                            absolute_tolerance_B, absolute_tolerance_QB,
-                            steps_checkpoint, args...);
+                            absolute_tolerance, max_num_steps, msgs, args...);
 }
 
 }  // namespace math

From 920fc7c241930e55ec4c3dd79360834c81718558 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 22 Feb 2021 22:20:56 +0100
Subject: [PATCH 049/157] make signature work as decribed in design doc

---
 stan/math/rev/functor.hpp                     |   2 +-
 .../rev/functor/cvodes_integrator_adjoint.hpp |  90 +++++++-----
 .../{ode_bdf_adjoint.hpp => ode_adjoint.hpp}  | 130 +++++++-----------
 3 files changed, 113 insertions(+), 109 deletions(-)
 rename stan/math/rev/functor/{ode_bdf_adjoint.hpp => ode_adjoint.hpp} (57%)

diff --git a/stan/math/rev/functor.hpp b/stan/math/rev/functor.hpp
index 45720c3a7fc..4e36d2e7b14 100644
--- a/stan/math/rev/functor.hpp
+++ b/stan/math/rev/functor.hpp
@@ -17,7 +17,7 @@
 #include <stan/math/rev/functor/integrate_ode_bdf.hpp>
 #include <stan/math/rev/functor/ode_adams.hpp>
 #include <stan/math/rev/functor/ode_bdf.hpp>
-#include <stan/math/rev/functor/ode_bdf_adjoint.hpp>
+#include <stan/math/rev/functor/ode_adjoint.hpp>
 #include <stan/math/rev/functor/ode_store_sensitivities.hpp>
 #include <stan/math/rev/functor/jacobian.hpp>
 #include <stan/math/rev/functor/kinsol_data.hpp>
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 7fad717e04c..2388186b2a5 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -157,13 +157,18 @@ class cvodes_integrator_adjoint_vari : public vari {
   const size_t N_;
   bool returned_;
   std::ostream* msgs_;
-  double relative_tolerance_;
-  double absolute_tolerance_;
-  double absolute_tolerance_B_;
-  double absolute_tolerance_QB_;
-  long int steps_checkpoint_;
+  double rel_tol_f_;
+  Eigen::VectorXd abs_tol_f_;
+  double rel_tol_b_;
+  double abs_tol_b_;
+  double rel_tol_q_;
+  double abs_tol_q_;
   long int max_num_steps_;
-
+  long int num_checkpoints_;
+  int interpolation_polynomial_;
+  int solver_f_;
+  int solver_b_;
+  
   const size_t t0_vars_;
   const size_t ts_vars_;
   const size_t y0_vars_;
@@ -416,10 +421,19 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param t0 Initial time
    * @param ts Times at which to solve the ODE at. All values must be sorted and
    *   not less than t0.
-   * @param relative_tolerance Relative tolerance passed to CVODES
-   * @param absolute_tolerance Absolute tolerance passed to CVODES
+   * @param rel_tol_f Relative tolerance for forward problem passed to CVODES
+   * @param abs_tol_f Absolute tolerance for forward problem passed to CVODES
+   * @param rel_tol_b Relative tolerance for backward problem passed to CVODES
+   * @param abs_tol_b Absolute tolerance for backward problem passed to CVODES
+   * @param rel_tol_q Relative tolerance for quadrature problem passed to CVODES
+   * @param abs_tol_q Absolute tolerance for quadrature problem passed to CVODES
    * @param max_num_steps Upper limit on the number of integration steps to
    *   take between each output (error if exceeded)
+   * @param num_checkpoints Number of integrator steps after which a checkpoint is stored for the backward pass
+   * @param interpolation_polynomial type of polynomial used for interpolation
+   * @param solver_f solver used for forward pass
+   * @param solver_b solver used for backward pass
+   *   take between each output (error if exceeded)
    * @param[in, out] msgs the print stream for warning messages
    * @param args Extra arguments passed unmodified through to ODE right hand
    * side function
@@ -430,22 +444,31 @@ class cvodes_integrator_adjoint_vari : public vari {
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
   cvodes_integrator_adjoint_vari(
       const char* function_name, const F& f, const T_y0& y0, const T_t0& t0,
-      const std::vector<T_ts>& ts, double relative_tolerance,
-      double absolute_tolerance, double absolute_tolerance_B,
-      double absolute_tolerance_QB, long int steps_checkpoint,
-      long int max_num_steps, std::ostream* msgs, const T_Args&... args)
+      const std::vector<T_ts>& ts,
+      double rel_tol_f, Eigen::VectorXd abs_tol_f,
+      double rel_tol_b, double abs_tol_b,
+      double rel_tol_q, double abs_tol_q,
+      long int max_num_steps, long int num_checkpoints,
+      int interpolation_polynomial,
+      int solver_f, int solver_b,
+      std::ostream* msgs, const T_Args&... args)
       : function_name_(function_name),
         vari(NOT_A_NUMBER),
         N_(y0.size()),
         returned_(false),
         memory(NULL),
-        msgs_(msgs),
-        relative_tolerance_(relative_tolerance),
-        absolute_tolerance_(absolute_tolerance),
-        absolute_tolerance_B_(absolute_tolerance_B),
-        absolute_tolerance_QB_(absolute_tolerance_QB),
-        steps_checkpoint_(steps_checkpoint),
+        rel_tol_f_(rel_tol_f),
+        abs_tol_f_(abs_tol_f),
+        rel_tol_b_(rel_tol_b),
+        abs_tol_b_(abs_tol_b),
+        rel_tol_q_(rel_tol_q),
+        abs_tol_q_(abs_tol_q),
         max_num_steps_(max_num_steps),
+        num_checkpoints_(num_checkpoints),
+        interpolation_polynomial_(interpolation_polynomial),
+        solver_f_(solver_f),
+        solver_b_(solver_b),
+        msgs_(msgs),        
         t0_vars_(count_vars(t0)),
         ts_vars_(count_vars(ts)),
         y0_vars_(count_vars(y0)),
@@ -486,12 +509,18 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_nonzero_size(fun, "initial state", y0);
     check_sorted(fun, "times", ts);
     check_less(fun, "initial time", t0, ts[0]);
-    check_positive_finite(fun, "relative_tolerance", relative_tolerance_);
-    check_positive_finite(fun, "absolute_tolerance", absolute_tolerance_);
-    check_positive_finite(fun, "absolute_tolerance_B", absolute_tolerance_B_);
-    check_positive_finite(fun, "absolute_tolerance_QB", absolute_tolerance_QB_);
+    check_positive_finite(fun, "rel_tol_f", rel_tol_f_);
+    check_positive_finite(fun, "abs_tol_f", abs_tol_f_);
+    check_positive_finite(fun, "rel_tol_b", rel_tol_b_);
+    check_positive_finite(fun, "abs_tol_b", abs_tol_b_);
+    check_positive_finite(fun, "rel_tol_q", rel_tol_q_);
+    check_positive_finite(fun, "abs_tol_q", abs_tol_q_);
     check_positive(fun, "max_num_steps", max_num_steps_);
-    check_positive(fun, "steps_checkpoint", steps_checkpoint_);
+    check_positive(fun, "num_checkpoints", num_checkpoints_);
+    // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
+    check_range(fun, "interpolation_polynomial", 2, interpolation_polynomial_);
+    check_range(fun, "solver_f", 3, solver_f_);
+    check_range(fun, "solver_b", 3, solver_b_);
 
     /*
     std::cout << "relative_tolerance = " << relative_tolerance << std::endl;
@@ -527,8 +556,9 @@ class cvodes_integrator_adjoint_vari : public vari {
         CVodeSetUserData(memory->cvodes_mem_, reinterpret_cast<void*>(this)),
         "CVodeSetUserData");
 
-    cvodes_set_options(memory->cvodes_mem_, relative_tolerance_,
-                       absolute_tolerance_, max_num_steps_);
+    cvodes_set_options(memory->cvodes_mem_, rel_tol_f_,
+                       abs_tol_f_(0), // MAKE THIS USE THE VECTOR
+                       max_num_steps_);
 
     // for the stiff solvers we need to reserve additional memory
     // and provide a Jacobian function call. new API since 3.0.0:
@@ -546,9 +576,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     // initialize forward sensitivity system of CVODES as needed
     if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
       check_flag_sundials(
-          // CVodeAdjInit(memory->cvodes_mem_, steps_checkpoint_,
-          // CV_POLYNOMIAL),
-          CVodeAdjInit(memory->cvodes_mem_, steps_checkpoint_, CV_HERMITE),
+          CVodeAdjInit(memory->cvodes_mem_, num_checkpoints_, interpolation_polynomial_),
           "CVodeAdjInit");
     }
 
@@ -687,7 +715,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
             check_flag_sundials(
                 CVodeSStolerancesB(memory->cvodes_mem_, indexB,
-                                   relative_tolerance_, absolute_tolerance_B_),
+                                   rel_tol_b_, abs_tol_b_),
                 "CVodeSStolerancesB");
 
             check_flag_sundials(CVodeSetMaxNumStepsB(memory->cvodes_mem_,
@@ -721,8 +749,8 @@ class cvodes_integrator_adjoint_vari : public vari {
 
               check_flag_sundials(
                   CVodeQuadSStolerancesB(memory->cvodes_mem_, indexB,
-                                         relative_tolerance_,
-                                         absolute_tolerance_QB_),
+                                         rel_tol_q_,
+                                         abs_tol_q_),
                   "CVodeQuadSStolerancesB");
 
               check_flag_sundials(
diff --git a/stan/math/rev/functor/ode_bdf_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
similarity index 57%
rename from stan/math/rev/functor/ode_bdf_adjoint.hpp
rename to stan/math/rev/functor/ode_adjoint.hpp
index bd973f4d00b..b9af94bcbfa 100644
--- a/stan/math/rev/functor/ode_bdf_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -1,5 +1,5 @@
-#ifndef STAN_MATH_REV_FUNCTOR_ODE_BDF_ADJOINT_HPP
-#define STAN_MATH_REV_FUNCTOR_ODE_BDF_ADJOINT_HPP
+#ifndef STAN_MATH_REV_FUNCTOR_ODE_ADJOINT_HPP
+#define STAN_MATH_REV_FUNCTOR_ODE_ADJOINT_HPP
 
 #include <stan/math/rev/meta.hpp>
 #include <stan/math/prim/fun/eval.hpp>
@@ -36,10 +36,18 @@ namespace math {
  * @param t0 Initial time
  * @param ts Times at which to solve the ODE at. All values must be sorted and
  *   not less than t0.
- * @param relative_tolerance Relative tolerance passed to CVODES
- * @param absolute_tolerance Absolute tolerance passed to CVODES
+ * @param rel_tol_f Relative tolerance for forward problem passed to CVODES
+ * @param abs_tol_f Absolute tolerance for forward problem passed to CVODES
+ * @param rel_tol_b Relative tolerance for backward problem passed to CVODES
+ * @param abs_tol_b Absolute tolerance for backward problem passed to CVODES
+ * @param rel_tol_q Relative tolerance for quadrature problem passed to CVODES
+ * @param abs_tol_q Absolute tolerance for quadrature problem passed to CVODES
  * @param max_num_steps Upper limit on the number of integration steps to
  *   take between each output (error if exceeded)
+ * @param num_checkpoints Number of integrator steps after which a checkpoint is stored for the backward pass
+ * @param interpolation_polynomial type of polynomial used for interpolation
+ * @param solver_f solver used for forward pass
+ * @param solver_b solver used for backward pass
  * @param[in, out] msgs the print stream for warning messages
  * @param args Extra arguments passed unmodified through to ODE right hand side
  * @return Solution to ODE at times \p ts
@@ -48,13 +56,17 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
-ode_bdf_adjoint_tol_impl(const char* function_name, const F& f, const T_y0& y0,
-                         const T_t0& t0, const std::vector<T_ts>& ts,
-                         double relative_tolerance, double absolute_tolerance,
-                         long int max_num_steps, std::ostream* msgs,
-                         double absolute_tolerance_B,
-                         double absolute_tolerance_QB,
-                         long int steps_checkpoint, const T_Args&... args) {
+ode_adjoint_impl(const char* function_name, const F& f,
+                 const T_y0& y0,
+                 const T_t0& t0, const std::vector<T_ts>& ts,
+                 double rel_tol_f, Eigen::VectorXd abs_tol_f,
+                 double rel_tol_b, double abs_tol_b,
+                 double rel_tol_q, double abs_tol_q,
+                 long int max_num_steps, long int num_checkpoints,
+                 int interpolation_polynomial,
+                 int solver_f, int solver_b,
+                 std::ostream* msgs,
+                 const T_Args&... args) {
   /*
   const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
   return apply(
@@ -90,9 +102,13 @@ ode_bdf_adjoint_tol_impl(const char* function_name, const F& f, const T_y0& y0,
   */
   auto integrator = new stan::math::cvodes_integrator_adjoint_vari<
       CV_BDF, F, plain_type_t<T_y0>, T_t0, T_ts, plain_type_t<T_Args>...>(
-      function_name, f, eval(y0), t0, ts, relative_tolerance,
-      absolute_tolerance, absolute_tolerance_B, absolute_tolerance_QB,
-      steps_checkpoint, max_num_steps, msgs, eval(args)...);
+      function_name, f, eval(y0), t0, ts,
+      rel_tol_f, abs_tol_f,
+      rel_tol_b, abs_tol_b,
+      rel_tol_q, abs_tol_q,
+      max_num_steps, num_checkpoints, interpolation_polynomial, solver_f, solver_b,
+      msgs,
+      eval(args)...);
   return (*integrator)();
 }
 
@@ -133,71 +149,31 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
-ode_bdf_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
-                    const std::vector<T_ts>& ts, double relative_tolerance,
-                    double absolute_tolerance, long int max_num_steps,
-                    std::ostream* msgs, double absolute_tolerance_B,
-                    double absolute_tolerance_QB, long int steps_checkpoint,
-                    const T_Args&... args) {
-  return ode_bdf_adjoint_tol_impl(
-      "ode_bdf_adjoint_tol", f, y0, t0, ts, relative_tolerance,
-      absolute_tolerance, max_num_steps, msgs, absolute_tolerance_B,
-      absolute_tolerance_QB, steps_checkpoint, args...);
+ode_adjoint(const F& f, const T_y0& y0, const T_t0& t0,
+            const std::vector<T_ts>& ts,
+            std::ostream* msgs, // @rok: should be moved to end like
+                                // for bdf??
+            double rel_tol_f, Eigen::VectorXd abs_tol_f,
+            double rel_tol_b, double abs_tol_b,
+            double rel_tol_q, double abs_tol_q,
+            long int max_num_steps, long int num_checkpoints,
+            int interpolation_polynomial,
+            int solver_f, int solver_b,
+            //std::ostream* msgs,
+            const T_Args&... args
+            ) {
+  return ode_adjoint_impl(
+      "ode_adjoint",
+      f, y0, t0, ts,
+      rel_tol_f, abs_tol_f,
+      rel_tol_b, abs_tol_b,
+      rel_tol_q, abs_tol_q,
+      max_num_steps, num_checkpoints, interpolation_polynomial, solver_f, solver_b,
+      msgs,
+      args...);
 }
 
-/**
- * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
- * times, { t1, t2, t3, ... } using the stiff backward differentiation formula
- * (BDF) solver in CVODES with a relative tolerance of 1e-10, an absolute
- * tolerance of 1e-10, and taking a maximum of 1e8 steps.
- *
- * \p f must define an operator() with the signature as:
- *   template<typename T_t, typename T_y, typename... T_Args>
- *   Eigen::Matrix<stan::return_type_t<T_t, T_y, T_Args...>, Eigen::Dynamic, 1>
- *     operator()(const T_t& t, const Eigen::Matrix<T_y, Eigen::Dynamic, 1>& y,
- *     std::ostream* msgs, const T_Args&... args);
- *
- * t is the time, y is the state, msgs is a stream for error messages, and args
- * are optional arguments passed to the ODE solve function (which are passed
- * through to \p f without modification).
- *
- * @tparam F Type of ODE right hand side
- * @tparam T_0 Type of initial time
- * @tparam T_ts Type of output times
- * @tparam T_Args Types of pass-through parameters
- *
- * @param f Right hand side of the ODE
- * @param y0 Initial state
- * @param t0 Initial time
- * @param ts Times at which to solve the ODE at. All values must be sorted and
- *   not less than t0.
- * @param relative_tolerance Relative tolerance passed to CVODES
- * @param absolute_tolerance Absolute tolerance passed to CVODES
- * @param max_num_steps Upper limit on the number of integration steps to
- *   take between each output (error if exceeded)
- * @param[in, out] msgs the print stream for warning messages
- * @param args Extra arguments passed unmodified through to ODE right hand side
- * @return Solution to ODE at times \p ts
- */
-template <typename F, typename T_y0, typename T_t0, typename T_ts,
-          typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
-std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
-                          Eigen::Dynamic, 1>>
-ode_bdf_adjoint(const F& f, const T_y0& y0, const T_t0& t0,
-                const std::vector<T_ts>& ts, std::ostream* msgs,
-                const T_Args&... args) {
-  double relative_tolerance = 1e-10;
-  double absolute_tolerance = 1e-10;
-  double absolute_tolerance_B = 1e-9;
-  double absolute_tolerance_QB = 1e-8;
-  long int steps_checkpoint = 100;
-  long int max_num_steps = 1e8;
-
-  return ode_bdf_adjoint_tol_impl(
-      "ode_bdf_adjoint", f, y0, t0, ts, relative_tolerance, absolute_tolerance,
-      max_num_steps, msgs, absolute_tolerance_B, absolute_tolerance_QB,
-      steps_checkpoint, args...);
-}
+// TODO(wds15): ask @rok/all to name this ode_adjoint_tol ?
 
 }  // namespace math
 }  // namespace stan

From 59eb1d82a1194fca7f99fe250c24f5f0308201bd Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 22 Feb 2021 22:31:10 +0100
Subject: [PATCH 050/157] make first test work with new function

---
 ...test.cpp => ode_adjoint_grad_rev_test.cpp} | 23 ++++++++++++++-----
 1 file changed, 17 insertions(+), 6 deletions(-)
 rename test/unit/math/rev/functor/{ode_bdf_adjoint_grad_rev_test.cpp => ode_adjoint_grad_rev_test.cpp} (93%)

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_adjoint_grad_rev_test.cpp
similarity index 93%
rename from test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
rename to test/unit/math/rev/functor/ode_adjoint_grad_rev_test.cpp
index 07d1ae4e907..e0ab80832f6 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_grad_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_adjoint_grad_rev_test.cpp
@@ -42,7 +42,7 @@ struct sho_functor {
     return out;
   }
 };
-
+/*
 template <int state>
 class test_functor_double_var {
   int lmm_;
@@ -64,7 +64,7 @@ class test_functor_double_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_bdf_adjoint(sho, y0, t0, ts, nullptr, omega)
+              ? stan::math::ode_adjoint(sho, y0, t0, ts, nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return ys[1](state);
@@ -134,7 +134,7 @@ class test_functor_var_double {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_bdf_adjoint(sho, y0, t0, ts, nullptr, omega)
+              ? stan::math::ode_adjoint(sho, y0, t0, ts, nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return ys[0](state);
@@ -203,7 +203,7 @@ class test_functor_var_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_adams_tol(sho, y0, t0, ts, 1E-10, 1E-10, 10000,
+              ? stan::math::ode_adjoint(sho, y0, t0, ts, 1E-10, 1E-10, 10000,
                                           nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
@@ -259,6 +259,7 @@ TEST(StanMathOdeIntegrateODEGradMat, var_var) {
     EXPECT_NEAR(dy2_dchi(t, omega, chi), grad(1), 1e-7);
   }
 }
+*/
 
 template <int state>
 class test_functor_sum_var_var {
@@ -279,16 +280,26 @@ class test_functor_sum_var_var {
     double t0 = 0.0;
     std::vector<double> ts{2.0, 5.0};
 
+    Eigen::VectorXd abs_tol_f(2);
+    abs_tol_f << 1E-10, 1E-10;
+
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_adams_tol(sho, y0, t0, ts, 1E-10, 1E-10, 10000,
-                                          nullptr, omega)
+        ? stan::math::ode_adjoint(sho, y0, t0, ts, nullptr,
+                                  1E-10, abs_tol_f,
+                                  1E-10, 1E-10,
+                                  1E-10, 1E-10,
+                                  10000, 50,
+                                  1, 1, 1,
+                                  //nullptr,
+                                  omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return stan::math::sum(ys[0](state) + ys[1](state));
   }
 };
 
+
 TEST(StanMathOdeIntegrateODEGradMat, sum_var_var) {
   double omega = 0.5;
   double chi = 1.25;

From 1c4482c72678389ae72216f1bad522ccc8a1e477 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Mon, 22 Feb 2021 21:34:27 +0000
Subject: [PATCH 051/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 34 +++++------
 stan/math/rev/functor/ode_adjoint.hpp         | 57 +++++++------------
 .../rev/functor/ode_adjoint_grad_rev_test.cpp | 14 ++---
 3 files changed, 41 insertions(+), 64 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 2388186b2a5..6ab0c985e65 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -168,7 +168,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   int interpolation_polynomial_;
   int solver_f_;
   int solver_b_;
-  
+
   const size_t t0_vars_;
   const size_t ts_vars_;
   const size_t y0_vars_;
@@ -429,7 +429,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param abs_tol_q Absolute tolerance for quadrature problem passed to CVODES
    * @param max_num_steps Upper limit on the number of integration steps to
    *   take between each output (error if exceeded)
-   * @param num_checkpoints Number of integrator steps after which a checkpoint is stored for the backward pass
+   * @param num_checkpoints Number of integrator steps after which a checkpoint
+   * is stored for the backward pass
    * @param interpolation_polynomial type of polynomial used for interpolation
    * @param solver_f solver used for forward pass
    * @param solver_b solver used for backward pass
@@ -444,13 +445,10 @@ class cvodes_integrator_adjoint_vari : public vari {
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
   cvodes_integrator_adjoint_vari(
       const char* function_name, const F& f, const T_y0& y0, const T_t0& t0,
-      const std::vector<T_ts>& ts,
-      double rel_tol_f, Eigen::VectorXd abs_tol_f,
-      double rel_tol_b, double abs_tol_b,
-      double rel_tol_q, double abs_tol_q,
+      const std::vector<T_ts>& ts, double rel_tol_f, Eigen::VectorXd abs_tol_f,
+      double rel_tol_b, double abs_tol_b, double rel_tol_q, double abs_tol_q,
       long int max_num_steps, long int num_checkpoints,
-      int interpolation_polynomial,
-      int solver_f, int solver_b,
+      int interpolation_polynomial, int solver_f, int solver_b,
       std::ostream* msgs, const T_Args&... args)
       : function_name_(function_name),
         vari(NOT_A_NUMBER),
@@ -468,7 +466,7 @@ class cvodes_integrator_adjoint_vari : public vari {
         interpolation_polynomial_(interpolation_polynomial),
         solver_f_(solver_f),
         solver_b_(solver_b),
-        msgs_(msgs),        
+        msgs_(msgs),
         t0_vars_(count_vars(t0)),
         ts_vars_(count_vars(ts)),
         y0_vars_(count_vars(y0)),
@@ -557,7 +555,7 @@ class cvodes_integrator_adjoint_vari : public vari {
         "CVodeSetUserData");
 
     cvodes_set_options(memory->cvodes_mem_, rel_tol_f_,
-                       abs_tol_f_(0), // MAKE THIS USE THE VECTOR
+                       abs_tol_f_(0),  // MAKE THIS USE THE VECTOR
                        max_num_steps_);
 
     // for the stiff solvers we need to reserve additional memory
@@ -575,9 +573,9 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     // initialize forward sensitivity system of CVODES as needed
     if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
-      check_flag_sundials(
-          CVodeAdjInit(memory->cvodes_mem_, num_checkpoints_, interpolation_polynomial_),
-          "CVodeAdjInit");
+      check_flag_sundials(CVodeAdjInit(memory->cvodes_mem_, num_checkpoints_,
+                                       interpolation_polynomial_),
+                          "CVodeAdjInit");
     }
 
     double t_init = t0_dbl;
@@ -713,10 +711,9 @@ class cvodes_integrator_adjoint_vari : public vari {
                            t_init, nv_state_sens),
                 "CVodeInitB");
 
-            check_flag_sundials(
-                CVodeSStolerancesB(memory->cvodes_mem_, indexB,
-                                   rel_tol_b_, abs_tol_b_),
-                "CVodeSStolerancesB");
+            check_flag_sundials(CVodeSStolerancesB(memory->cvodes_mem_, indexB,
+                                                   rel_tol_b_, abs_tol_b_),
+                                "CVodeSStolerancesB");
 
             check_flag_sundials(CVodeSetMaxNumStepsB(memory->cvodes_mem_,
                                                      indexB, max_num_steps_),
@@ -749,8 +746,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
               check_flag_sundials(
                   CVodeQuadSStolerancesB(memory->cvodes_mem_, indexB,
-                                         rel_tol_q_,
-                                         abs_tol_q_),
+                                         rel_tol_q_, abs_tol_q_),
                   "CVodeQuadSStolerancesB");
 
               check_flag_sundials(
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index b9af94bcbfa..3e92d17446a 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -44,7 +44,8 @@ namespace math {
  * @param abs_tol_q Absolute tolerance for quadrature problem passed to CVODES
  * @param max_num_steps Upper limit on the number of integration steps to
  *   take between each output (error if exceeded)
- * @param num_checkpoints Number of integrator steps after which a checkpoint is stored for the backward pass
+ * @param num_checkpoints Number of integrator steps after which a checkpoint is
+ * stored for the backward pass
  * @param interpolation_polynomial type of polynomial used for interpolation
  * @param solver_f solver used for forward pass
  * @param solver_b solver used for backward pass
@@ -56,16 +57,12 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
-ode_adjoint_impl(const char* function_name, const F& f,
-                 const T_y0& y0,
-                 const T_t0& t0, const std::vector<T_ts>& ts,
-                 double rel_tol_f, Eigen::VectorXd abs_tol_f,
-                 double rel_tol_b, double abs_tol_b,
-                 double rel_tol_q, double abs_tol_q,
-                 long int max_num_steps, long int num_checkpoints,
-                 int interpolation_polynomial,
-                 int solver_f, int solver_b,
-                 std::ostream* msgs,
+ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
+                 const T_t0& t0, const std::vector<T_ts>& ts, double rel_tol_f,
+                 Eigen::VectorXd abs_tol_f, double rel_tol_b, double abs_tol_b,
+                 double rel_tol_q, double abs_tol_q, long int max_num_steps,
+                 long int num_checkpoints, int interpolation_polynomial,
+                 int solver_f, int solver_b, std::ostream* msgs,
                  const T_Args&... args) {
   /*
   const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
@@ -102,13 +99,9 @@ ode_adjoint_impl(const char* function_name, const F& f,
   */
   auto integrator = new stan::math::cvodes_integrator_adjoint_vari<
       CV_BDF, F, plain_type_t<T_y0>, T_t0, T_ts, plain_type_t<T_Args>...>(
-      function_name, f, eval(y0), t0, ts,
-      rel_tol_f, abs_tol_f,
-      rel_tol_b, abs_tol_b,
-      rel_tol_q, abs_tol_q,
-      max_num_steps, num_checkpoints, interpolation_polynomial, solver_f, solver_b,
-      msgs,
-      eval(args)...);
+      function_name, f, eval(y0), t0, ts, rel_tol_f, abs_tol_f, rel_tol_b,
+      abs_tol_b, rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,
+      interpolation_polynomial, solver_f, solver_b, msgs, eval(args)...);
   return (*integrator)();
 }
 
@@ -151,26 +144,18 @@ std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint(const F& f, const T_y0& y0, const T_t0& t0,
             const std::vector<T_ts>& ts,
-            std::ostream* msgs, // @rok: should be moved to end like
-                                // for bdf??
-            double rel_tol_f, Eigen::VectorXd abs_tol_f,
-            double rel_tol_b, double abs_tol_b,
-            double rel_tol_q, double abs_tol_q,
+            std::ostream* msgs,  // @rok: should be moved to end like
+                                 // for bdf??
+            double rel_tol_f, Eigen::VectorXd abs_tol_f, double rel_tol_b,
+            double abs_tol_b, double rel_tol_q, double abs_tol_q,
             long int max_num_steps, long int num_checkpoints,
-            int interpolation_polynomial,
-            int solver_f, int solver_b,
-            //std::ostream* msgs,
-            const T_Args&... args
-            ) {
+            int interpolation_polynomial, int solver_f, int solver_b,
+            // std::ostream* msgs,
+            const T_Args&... args) {
   return ode_adjoint_impl(
-      "ode_adjoint",
-      f, y0, t0, ts,
-      rel_tol_f, abs_tol_f,
-      rel_tol_b, abs_tol_b,
-      rel_tol_q, abs_tol_q,
-      max_num_steps, num_checkpoints, interpolation_polynomial, solver_f, solver_b,
-      msgs,
-      args...);
+      "ode_adjoint", f, y0, t0, ts, rel_tol_f, abs_tol_f, rel_tol_b, abs_tol_b,
+      rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,
+      interpolation_polynomial, solver_f, solver_b, msgs, args...);
 }
 
 // TODO(wds15): ask @rok/all to name this ode_adjoint_tol ?
diff --git a/test/unit/math/rev/functor/ode_adjoint_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_adjoint_grad_rev_test.cpp
index e0ab80832f6..4cb0d0cacf7 100644
--- a/test/unit/math/rev/functor/ode_adjoint_grad_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_adjoint_grad_rev_test.cpp
@@ -285,21 +285,17 @@ class test_functor_sum_var_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-        ? stan::math::ode_adjoint(sho, y0, t0, ts, nullptr,
-                                  1E-10, abs_tol_f,
-                                  1E-10, 1E-10,
-                                  1E-10, 1E-10,
-                                  10000, 50,
-                                  1, 1, 1,
-                                  //nullptr,
-                                  omega)
+              ? stan::math::ode_adjoint(sho, y0, t0, ts, nullptr, 1E-10,
+                                        abs_tol_f, 1E-10, 1E-10, 1E-10, 1E-10,
+                                        10000, 50, 1, 1, 1,
+                                        // nullptr,
+                                        omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return stan::math::sum(ys[0](state) + ys[1](state));
   }
 };
 
-
 TEST(StanMathOdeIntegrateODEGradMat, sum_var_var) {
   double omega = 0.5;
   double chi = 1.25;

From 3157e49d3bfc6e49c15e4f4dfeed7f7d9e8da68e Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 23 Feb 2021 11:23:07 +0100
Subject: [PATCH 052/157] make vector absolute tolerance work and expose solver
 choice

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 45 +++++++++++++------
 stan/math/rev/functor/ode_adjoint.hpp         |  2 +-
 ...p => ode_adjoint_cvodes_grad_rev_test.cpp} |  0
 3 files changed, 32 insertions(+), 15 deletions(-)
 rename test/unit/math/rev/functor/{ode_adjoint_grad_rev_test.cpp => ode_adjoint_cvodes_grad_rev_test.cpp} (100%)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 2388186b2a5..4400571f3c5 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -64,17 +64,19 @@ inline std::vector<Eigen::VectorXd> build_varis<double>(
   return y;
 }
 
-template <int Lmm, typename F, typename T_y0, typename T_t0, typename T_ts,
+template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args>
 class cvodes_integrator_adjoint_vari;
 
-template <int Lmm, typename F, typename T_y0, typename T_t0, typename T_ts,
+template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args>
 class cvodes_integrator_adjoint_memory : public chainable_alloc {
   using T_Return = return_type_t<T_y0, T_t0, T_ts, T_Args...>;
   using T_y0_t0 = return_type_t<T_y0, T_t0>;
 
   const size_t N_;
+  Eigen::VectorXd abs_tol_f_;
+  const int lmm_f_;
   const F f_;
   const Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1> y0_;
   const T_t0 t0_;
@@ -89,14 +91,19 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   void* cvodes_mem_;
   Eigen::VectorXd state;
   N_Vector nv_state_;
+  N_Vector nv_abs_tol_f_;
   SUNMatrix A_;
   SUNLinearSolver LS_;
 
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
-  cvodes_integrator_adjoint_memory(const F& f, const T_y0& y0, const T_t0& t0,
+  cvodes_integrator_adjoint_memory(Eigen::VectorXd abs_tol_f,
+                                   int lmm_f,
+                                   const F& f, const T_y0& y0, const T_t0& t0,
                                    const std::vector<T_ts>& ts,
                                    const T_Args&... args)
       : N_(y0.size()),
+        abs_tol_f_(abs_tol_f),
+        lmm_f_(lmm_f),
         f_(f),
         y0_(y0),
         t0_(t0),
@@ -109,10 +116,11 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
         state(value_of(y0)) {
     if (N_ > 0) {
       nv_state_ = N_VMake_Serial(N_, state.data());
+      nv_abs_tol_f_ = N_VMake_Serial(N_, &abs_tol_f_(0));
       A_ = SUNDenseMatrix(N_, N_);
       LS_ = SUNDenseLinearSolver(nv_state_, A_);
 
-      cvodes_mem_ = CVodeCreate(Lmm);
+      cvodes_mem_ = CVodeCreate(lmm_f_);
       if (cvodes_mem_ == nullptr) {
         throw std::runtime_error("CVodeCreate failed to allocate memory");
       }
@@ -125,6 +133,7 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
       SUNMatDestroy(A_);
 
       N_VDestroy_Serial(nv_state_);
+      N_VDestroy_Serial(nv_abs_tol_f_);
 
       if (cvodes_mem_) {
         CVodeFree(&cvodes_mem_);
@@ -132,7 +141,7 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
     }
   }
 
-  friend class cvodes_integrator_adjoint_vari<Lmm, F, T_y0, T_t0, T_ts,
+  friend class cvodes_integrator_adjoint_vari<F, T_y0, T_t0, T_ts,
                                               T_Args...>;
 };
 
@@ -140,14 +149,13 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
  * Integrator interface for CVODES' ODE solvers (Adams & BDF
  * methods).
  *
- * @tparam Lmm ID of ODE solver (1: ADAMS, 2: BDF)
  * @tparam F Type of ODE right hand side
  * @tparam T_y0 Type of scalars for initial state
  * @tparam T_param Type of scalars for parameters
  * @tparam T_t0 Type of scalar of initial time point
  * @tparam T_ts Type of time-points where ODE solution is returned
  */
-template <int Lmm, typename F, typename T_y0, typename T_t0, typename T_ts,
+template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args>
 class cvodes_integrator_adjoint_vari : public vari {
   using T_Return = return_type_t<T_y0, T_t0, T_ts, T_Args...>;
@@ -168,6 +176,8 @@ class cvodes_integrator_adjoint_vari : public vari {
   int interpolation_polynomial_;
   int solver_f_;
   int solver_b_;
+  int lmm_f_;
+  int lmm_b_;
   
   const size_t t0_vars_;
   const size_t ts_vars_;
@@ -181,7 +191,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   vari** y0_varis_;
   vari** args_varis_;
 
-  cvodes_integrator_adjoint_memory<Lmm, F, T_y0, T_t0, T_ts, T_Args...>* memory;
+  cvodes_integrator_adjoint_memory<F, T_y0, T_t0, T_ts, T_Args...>* memory;
 
   /**
    * Implements the function of type CVRhsFn which is the user-defined
@@ -468,6 +478,8 @@ class cvodes_integrator_adjoint_vari : public vari {
         interpolation_polynomial_(interpolation_polynomial),
         solver_f_(solver_f),
         solver_b_(solver_b),
+        lmm_f_(solver_f_ == 1 ? CV_ADAMS : CV_BDF),
+        lmm_b_(solver_b_ == 1 ? CV_ADAMS : CV_BDF),
         msgs_(msgs),        
         t0_vars_(count_vars(t0)),
         ts_vars_(count_vars(ts)),
@@ -483,8 +495,8 @@ class cvodes_integrator_adjoint_vari : public vari {
             args_vars_)) {
     const char* fun = "cvodes_integrator::integrate";
 
-    memory = new cvodes_integrator_adjoint_memory<Lmm, F, T_y0, T_t0, T_ts,
-                                                  T_Args...>(f, y0, t0, ts,
+    memory = new cvodes_integrator_adjoint_memory<F, T_y0, T_t0, T_ts,
+                                                  T_Args...>(abs_tol_f_, lmm_f_, f, y0, t0, ts,
                                                              args...);
 
     save_varis(t0_varis_, t0);
@@ -511,6 +523,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_less(fun, "initial time", t0, ts[0]);
     check_positive_finite(fun, "rel_tol_f", rel_tol_f_);
     check_positive_finite(fun, "abs_tol_f", abs_tol_f_);
+    check_size_match(fun, "abs_tol_f", abs_tol_f_.size(), "states", N_);
     check_positive_finite(fun, "rel_tol_b", rel_tol_b_);
     check_positive_finite(fun, "abs_tol_b", abs_tol_b_);
     check_positive_finite(fun, "rel_tol_q", rel_tol_q_);
@@ -519,8 +532,9 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_positive(fun, "num_checkpoints", num_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
     check_range(fun, "interpolation_polynomial", 2, interpolation_polynomial_);
-    check_range(fun, "solver_f", 3, solver_f_);
-    check_range(fun, "solver_b", 3, solver_b_);
+    // 1=Adams, 2=BDF, 3=BDF_iterated (todo)
+    check_range(fun, "solver_f", 2, solver_f_);
+    check_range(fun, "solver_b", 2, solver_b_);
 
     /*
     std::cout << "relative_tolerance = " << relative_tolerance << std::endl;
@@ -557,9 +571,12 @@ class cvodes_integrator_adjoint_vari : public vari {
         "CVodeSetUserData");
 
     cvodes_set_options(memory->cvodes_mem_, rel_tol_f_,
-                       abs_tol_f_(0), // MAKE THIS USE THE VECTOR
+                       abs_tol_f_(0),
                        max_num_steps_);
 
+    check_flag_sundials(CVodeSVtolerances(memory->cvodes_mem_, rel_tol_f_, memory->nv_abs_tol_f_),
+                        "CVodeSVtolerances");
+
     // for the stiff solvers we need to reserve additional memory
     // and provide a Jacobian function call. new API since 3.0.0:
     // create matrix object and linear solver object; resource
@@ -666,7 +683,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       int indexB;
 
       // This is all boilerplate CVODES setting up the adjoint ODE to solve
-      check_flag_sundials(CVodeCreateB(memory->cvodes_mem_, Lmm, &indexB),
+      check_flag_sundials(CVodeCreateB(memory->cvodes_mem_, lmm_b_, &indexB),
                           "CVodeCreateB");
 
       check_flag_sundials(CVodeSetUserDataB(memory->cvodes_mem_, indexB,
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index b9af94bcbfa..3202bb6b4f0 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -101,7 +101,7 @@ ode_adjoint_impl(const char* function_name, const F& f,
   return (*integrator)();
   */
   auto integrator = new stan::math::cvodes_integrator_adjoint_vari<
-      CV_BDF, F, plain_type_t<T_y0>, T_t0, T_ts, plain_type_t<T_Args>...>(
+      F, plain_type_t<T_y0>, T_t0, T_ts, plain_type_t<T_Args>...>(
       function_name, f, eval(y0), t0, ts,
       rel_tol_f, abs_tol_f,
       rel_tol_b, abs_tol_b,
diff --git a/test/unit/math/rev/functor/ode_adjoint_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
similarity index 100%
rename from test/unit/math/rev/functor/ode_adjoint_grad_rev_test.cpp
rename to test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp

From 1eda2f98e902c722e67e15c5dc039354147a91dd Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Tue, 23 Feb 2021 10:34:51 +0000
Subject: [PATCH 053/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 22 +++++++++----------
 stan/math/rev/functor/ode_adjoint.hpp         | 10 +++------
 2 files changed, 13 insertions(+), 19 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 2f3646e588d..6b5103b5d9d 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -96,8 +96,7 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   SUNLinearSolver LS_;
 
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
-  cvodes_integrator_adjoint_memory(Eigen::VectorXd abs_tol_f,
-                                   int lmm_f,
+  cvodes_integrator_adjoint_memory(Eigen::VectorXd abs_tol_f, int lmm_f,
                                    const F& f, const T_y0& y0, const T_t0& t0,
                                    const std::vector<T_ts>& ts,
                                    const T_Args&... args)
@@ -141,8 +140,7 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
     }
   }
 
-  friend class cvodes_integrator_adjoint_vari<F, T_y0, T_t0, T_ts,
-                                              T_Args...>;
+  friend class cvodes_integrator_adjoint_vari<F, T_y0, T_t0, T_ts, T_Args...>;
 };
 
 /**
@@ -178,7 +176,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   int solver_b_;
   int lmm_f_;
   int lmm_b_;
-  
+
   const size_t t0_vars_;
   const size_t ts_vars_;
   const size_t y0_vars_;
@@ -478,7 +476,7 @@ class cvodes_integrator_adjoint_vari : public vari {
         solver_b_(solver_b),
         lmm_f_(solver_f_ == 1 ? CV_ADAMS : CV_BDF),
         lmm_b_(solver_b_ == 1 ? CV_ADAMS : CV_BDF),
-        msgs_(msgs),        
+        msgs_(msgs),
 
         t0_vars_(count_vars(t0)),
         ts_vars_(count_vars(ts)),
@@ -494,9 +492,9 @@ class cvodes_integrator_adjoint_vari : public vari {
             args_vars_)) {
     const char* fun = "cvodes_integrator::integrate";
 
-    memory = new cvodes_integrator_adjoint_memory<F, T_y0, T_t0, T_ts,
-                                                  T_Args...>(abs_tol_f_, lmm_f_, f, y0, t0, ts,
-                                                             args...);
+    memory
+        = new cvodes_integrator_adjoint_memory<F, T_y0, T_t0, T_ts, T_Args...>(
+            abs_tol_f_, lmm_f_, f, y0, t0, ts, args...);
 
     save_varis(t0_varis_, t0);
     save_varis(ts_varis_, ts);
@@ -569,11 +567,11 @@ class cvodes_integrator_adjoint_vari : public vari {
         CVodeSetUserData(memory->cvodes_mem_, reinterpret_cast<void*>(this)),
         "CVodeSetUserData");
 
-    cvodes_set_options(memory->cvodes_mem_, rel_tol_f_,
-                       abs_tol_f_(0),
+    cvodes_set_options(memory->cvodes_mem_, rel_tol_f_, abs_tol_f_(0),
                        max_num_steps_);
 
-    check_flag_sundials(CVodeSVtolerances(memory->cvodes_mem_, rel_tol_f_, memory->nv_abs_tol_f_),
+    check_flag_sundials(CVodeSVtolerances(memory->cvodes_mem_, rel_tol_f_,
+                                          memory->nv_abs_tol_f_),
                         "CVodeSVtolerances");
 
     // for the stiff solvers we need to reserve additional memory
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index f05cb1b8b32..9170a3ad26b 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -99,13 +99,9 @@ ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
   */
   auto integrator = new stan::math::cvodes_integrator_adjoint_vari<
       F, plain_type_t<T_y0>, T_t0, T_ts, plain_type_t<T_Args>...>(
-      function_name, f, eval(y0), t0, ts,
-      rel_tol_f, abs_tol_f,
-      rel_tol_b, abs_tol_b,
-      rel_tol_q, abs_tol_q,
-      max_num_steps, num_checkpoints, interpolation_polynomial, solver_f, solver_b,
-      msgs,
-      eval(args)...);
+      function_name, f, eval(y0), t0, ts, rel_tol_f, abs_tol_f, rel_tol_b,
+      abs_tol_b, rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,
+      interpolation_polynomial, solver_f, solver_b, msgs, eval(args)...);
   return (*integrator)();
 }
 

From 1faf260eceefa67948cc52dc483d33e93c9fd52d Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Wed, 24 Feb 2021 08:46:03 +0100
Subject: [PATCH 054/157] add tol call

---
 stan/math/rev/functor/ode_adjoint.hpp | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 9170a3ad26b..51eb4ea140a 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -158,6 +158,25 @@ ode_adjoint(const F& f, const T_y0& y0, const T_t0& t0,
       interpolation_polynomial, solver_f, solver_b, msgs, args...);
 }
 
+template <typename F, typename T_y0, typename T_t0, typename T_ts,
+          typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
+std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
+                          Eigen::Dynamic, 1>>
+ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
+                const std::vector<T_ts>& ts,
+                //std::ostream* msgs,
+                double rel_tol_f, Eigen::VectorXd abs_tol_f, double rel_tol_b,
+                double abs_tol_b, double rel_tol_q, double abs_tol_q,
+                long int max_num_steps, long int num_checkpoints,
+                int interpolation_polynomial, int solver_f, int solver_b,
+                std::ostream* msgs,
+                const T_Args&... args) {
+  return ode_adjoint_impl(
+      "ode_adjoint_tol", f, y0, t0, ts, rel_tol_f, abs_tol_f, rel_tol_b, abs_tol_b,
+      rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,
+      interpolation_polynomial, solver_f, solver_b, msgs, args...);
+}
+
 // TODO(wds15): ask @rok/all to name this ode_adjoint_tol ?
 
 }  // namespace math

From 5d74d6870913bfd5ff394a54b1945218dfdc6394 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Wed, 24 Feb 2021 08:30:02 +0000
Subject: [PATCH 055/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/ode_adjoint.hpp | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 51eb4ea140a..0bd5cf5f66d 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -164,16 +164,15 @@ std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
                 const std::vector<T_ts>& ts,
-                //std::ostream* msgs,
+                // std::ostream* msgs,
                 double rel_tol_f, Eigen::VectorXd abs_tol_f, double rel_tol_b,
                 double abs_tol_b, double rel_tol_q, double abs_tol_q,
                 long int max_num_steps, long int num_checkpoints,
                 int interpolation_polynomial, int solver_f, int solver_b,
-                std::ostream* msgs,
-                const T_Args&... args) {
+                std::ostream* msgs, const T_Args&... args) {
   return ode_adjoint_impl(
-      "ode_adjoint_tol", f, y0, t0, ts, rel_tol_f, abs_tol_f, rel_tol_b, abs_tol_b,
-      rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,
+      "ode_adjoint_tol", f, y0, t0, ts, rel_tol_f, abs_tol_f, rel_tol_b,
+      abs_tol_b, rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,
       interpolation_polynomial, solver_f, solver_b, msgs, args...);
 }
 

From 3413734e0ad511c0f33ed69500d7f71d8ccda1f9 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Fri, 26 Feb 2021 21:27:20 +0100
Subject: [PATCH 056/157] add various benchmarks

---
 ode-adjoint/linked-mass-flow-eigen.stan | 10 +++++--
 ode-adjoint/linked-mass-flow.stan       | 24 ++++++++++++++--
 ode-adjoint/sbc_tools.R                 | 17 +++++++----
 ode-adjoint/scale-benchmark.R           | 38 +++++++++++++++++++------
 stan/math/rev/functor/ode_adjoint.hpp   |  5 +---
 5 files changed, 73 insertions(+), 21 deletions(-)

diff --git a/ode-adjoint/linked-mass-flow-eigen.stan b/ode-adjoint/linked-mass-flow-eigen.stan
index cfbe138daf2..d309f236573 100644
--- a/ode-adjoint/linked-mass-flow-eigen.stan
+++ b/ode-adjoint/linked-mass-flow-eigen.stan
@@ -40,8 +40,14 @@ functions {
     vector[num_states] y[num_sim];
 
     if(adjoint_integrator) {
-      y = ode_adams_tol(linked_mass_flow, exp(log_a0), t0, ts, rel_tol, abs_tol, max_num_steps,
-                        exp(log_kt), exp(log_e50), exp(log_k12), exp(log_k21));
+      y = ode_adjoint(linked_mass_flow, exp(log_a0), t0, ts,
+                      rel_tol, rep_vector(abs_tol, num_states), // forward
+                      rel_tol, 10*abs_tol, // backward
+                      rel_tol, 100*abs_tol, // quadrature
+                      max_num_steps, 50, 1,
+                      2, // bdf forward
+                      2, // bdf backward
+                      exp(log_kt), exp(log_e50), exp(log_k12), exp(log_k21));
     } else {
       y = ode_bdf_tol(linked_mass_flow, exp(log_a0), t0, ts, rel_tol, abs_tol, max_num_steps,
                       exp(log_kt), exp(log_e50), exp(log_k12), exp(log_k21));
diff --git a/ode-adjoint/linked-mass-flow.stan b/ode-adjoint/linked-mass-flow.stan
index 5889d4eab22..8e69a76b6d4 100644
--- a/ode-adjoint/linked-mass-flow.stan
+++ b/ode-adjoint/linked-mass-flow.stan
@@ -34,6 +34,8 @@ functions {
 
   vector[] simulate_mean(real t0, vector log_a0, real[] ts, int adjoint_integrator,
                          real rel_tol, real abs_tol, int max_num_steps,
+                         int num_checkpoints,
+                         int solver_f, int solver_b,
                          vector log_kt, vector log_e50, vector log_k12, vector log_k21) {
     int num_sim = num_elements(ts);
     int num_states = num_elements(log_a0);
@@ -57,8 +59,16 @@ functions {
     }
 
     if(adjoint_integrator) {
-      y = ode_adams_tol(linked_mass_flow, a0, t0, ts, rel_tol, abs_tol, max_num_steps,
-                        kt, e50, k12, k21);
+      y = ode_adjoint_tol(linked_mass_flow, a0, t0, ts, 
+                          rel_tol, rep_vector(abs_tol, num_states), // forward
+                          rel_tol, 10*abs_tol, // backward
+                          rel_tol, 100*abs_tol, // quadrature
+                          max_num_steps,
+                          num_checkpoints, // number of steps between checkpoints
+                          1,  // hermite polynomials
+                          solver_f,  // bdf forward
+                          solver_b,  // bdf backward
+                          kt, e50, k12, k21);
     } else {
       y = ode_bdf_tol(linked_mass_flow, a0, t0, ts, rel_tol, abs_tol, max_num_steps,
                       kt, e50, k12, k21);
@@ -73,6 +83,9 @@ data {
   real<lower=0> abs_tol;
   int<lower=0,upper=1> adjoint_integrator;
   int<lower=1> max_num_steps;
+  int<lower=1> num_checkpoints;
+  int<lower=1,upper=2> solver_f;
+  int<lower=1,upper=2> solver_b;
   int<lower=1> system_size;
   int<lower=1> num_obs;
   real<lower=0> sigma_sim;
@@ -98,6 +111,8 @@ transformed data {
   }
 
   y_ = simulate_mean(t0, log_a0_, ts, 0, rel_tol, abs_tol, max_num_steps,
+                     num_checkpoints,
+                     solver_f, solver_b,
                      log_kt_, log_e50_, log_k12_, log_k21_);
   
   for(i in 1:num_obs) {
@@ -115,6 +130,9 @@ transformed data {
   print("relative tolerance: ", rel_tol);
   print("absolute tolerance: ", abs_tol);
   print("maximum number of steps: ", max_num_steps);
+  print("number of checkpoints: ", num_checkpoints);
+  print("solver forward: ", solver_f);
+  print("solver backward: ", solver_b);
   print("number of time points: ", num_obs);
   print("system size: ", system_size);
   print("time points: ", ts);
@@ -132,6 +150,8 @@ transformed parameters {
 }
 model {
   vector[num_states] mu[num_obs] = simulate_mean(t0, log_a0, ts, adjoint_integrator, rel_tol, abs_tol, max_num_steps,
+                                                 num_checkpoints,
+                                                 solver_f, solver_b,
                                                  log_kt, log_e50, log_k12, log_k21);
   
   target += normal_lpdf(log_kt| 0.0, sigma_sim);
diff --git a/ode-adjoint/sbc_tools.R b/ode-adjoint/sbc_tools.R
index c07e829c52e..17d0b78bd5d 100644
--- a/ode-adjoint/sbc_tools.R
+++ b/ode-adjoint/sbc_tools.R
@@ -8,6 +8,7 @@ if(!("sbc_dir" %in%  ls())) {
 }
 
 stan_model_file <- normalizePath(file.path(sbc_dir, "linked-mass-flow.stan"))
+##stan_model_file <- normalizePath(file.path(sbc_dir, "linked-mass-flow-eigen.stan"))
 
 stan_model <- cmdstan_model(stan_model_file)
 
@@ -16,6 +17,9 @@ setup_system <- function(data, job, ...) {
     base_stan_data <- list(rel_tol=1e-6,
                            abs_tol=1e-6,
                            max_num_steps=10000,
+                           num_checkpoints=50,
+                           solver_f=2,
+                           solver_b=2,
                            adjoint_integrator=0,
                            system_size=1,
                            num_obs=8,
@@ -24,8 +28,11 @@ setup_system <- function(data, job, ...) {
     modifyList(base_stan_data, args)
 }
 
-run_benchmark <- function(data, job, instance, adjoint_integrator, ...) {
-    fit_data <- modifyList(instance, list(adjoint_integrator=adjoint_integrator))
+run_benchmark <- function(data, job, instance, adjoint_integrator, num_checkpoints, solver_f, solver_b, ...) {
+    fit_data <- modifyList(instance, list(adjoint_integrator=adjoint_integrator,
+                                          num_checkpoints=num_checkpoints,
+                                          solver_f=solver_f,
+                                          solver_b=solver_b))
     r <- job$repl
     fit <- data$stan_model$sample(
         data = fit_data,
@@ -40,7 +47,7 @@ run_benchmark <- function(data, job, instance, adjoint_integrator, ...) {
         adapt_delta=data$adapt_delta
         ##verbose=TRUE
         )
-    summarize_benchmark(fit)
+    c(summarize_benchmark(fit), list(repl=r))
 }
 
 
@@ -132,14 +139,14 @@ auto_submit <- function(jobs, registry, resources=list(), max_num_tries = 10) {
 reduce_run <- function(x) {
     c(as.list(x$rank),
       ess_speed=as.list(x$ess_lp_speed),
-      x[c("num_leaps", "time", "time_per_leap", "num_divergent")]
+      x[c("num_leaps", "time", "time_per_leap", "num_divergent", "repl")]
       )
 }
 
 reduce_run_bench <- function(x) {
     c(
       ess_speed=as.list(x$ess_lp_speed),
-      x[c("num_leaps", "time", "time_per_leap", "num_divergent")]
+      x[c("num_leaps", "time", "time_per_leap", "num_divergent", "repl")]
       )
 }
 
diff --git a/ode-adjoint/scale-benchmark.R b/ode-adjoint/scale-benchmark.R
index f35c98ec215..3b9e04033c6 100644
--- a/ode-adjoint/scale-benchmark.R
+++ b/ode-adjoint/scale-benchmark.R
@@ -37,11 +37,14 @@ addProblem("linked_mass_flow",
 
 addAlgorithm("stan_cvodes", run_benchmark)
 
-pdes <- list(linked_mass_flow = data.frame(system_size=c(1,2,4,8,16)))
-ades <- list(stan_cvodes= data.frame(adjoint_integrator=c(0,1)))
+##pdes <- list(linked_mass_flow = data.frame(system_size=c(1,2,4,8,16)))
+##pdes <- list(linked_mass_flow = data.frame(system_size=c(1,4)))
+
+pdes <- list(linked_mass_flow = data.frame(system_size=c(1,2,4,8)))
+ades <- list(stan_cvodes= expand.grid(adjoint_integrator=c(1), num_checkpoints=c(10, 100, 250), solver_f=1:2, solver_b=1:2))
 
 ##addExperiments(pdes, ades, repls = 1000)
-addExperiments(pdes, ades, repls = 2)
+addExperiments(pdes, ades, repls = 4)
 
 summarizeExperiments()
 
@@ -63,11 +66,13 @@ submitJobs(ids)
 
 getStatus()
 
-getLog(3)
+findErrors()
+
+getLog(47)
 
 if(FALSE) {
 
-    job1 <- testJob(3)
+    job1 <- testJob(47)
 
     names(job1)
 
@@ -98,12 +103,16 @@ scale_benchmark <- ijoin(
   ## grab job parameters
   unwrap(getJobPars()),
   unwrap(reduceResultsDataTable(fun=reduce_run_bench))
-)
+) %>% filter(!(system_size==8 & num_checkpoints==100 & solver_f==2 & solver_b==2))
 
 head(scale_benchmark)
 
 scale_benchmark <- mutate(scale_benchmark,
-                          adjoint_integrator=factor(adjoint_integrator))
+                          adjoint_integrator=factor(adjoint_integrator),
+                          ##num_checkpoints=factor(num_checkpoints),
+                          solver_f=factor(solver_f),
+                          solver_b=factor(solver_b)
+                          )
 
 saveRDS(scale_benchmark, file = "scale_benchmark.rds")
 
@@ -112,13 +121,26 @@ removeRegistry(0)
 sessionInfo()
 
 scale_benchmark <- as.data.frame(scale_benchmark)
-scale_benchmark <- mutate(scale_benchmark, repl=factor(rep(1:2, 10)), set=paste(repl,adjoint_integrator,sep="-"))
+scale_benchmark <- mutate(scale_benchmark, repl=factor(repl), set=paste(repl,adjoint_integrator,sep="-"), solver=factor(paste(solver_f, solver_b, sep="-")))
 
 
 scale_benchmark_ref <- left_join(scale_benchmark, filter(scale_benchmark, adjoint_integrator==0)[c("repl", "system_size", "time_per_leap")], suffix=c("", "_ref"), by=c("repl", "system_size"))
 
 scale_benchmark_ref
 
+lf <- lm(log(time_per_leap) ~ 1 + solver_f + solver_b + factor(num_checkpoints), data=subset(scale_benchmark_ref, system_size==8))
+summary(lf) ## bdf for backward makes things by 13% slower... change polyonmial?
+
+theme_set(theme_bw())
+ggplot(scale_benchmark, aes(num_checkpoints, time_per_leap, colour=solver, shape=repl)) +
+    facet_wrap(~system_size, scales="free_y") +
+   geom_point()  + geom_line() +
+    ##scale_x_log10(breaks=c(1,2,4,8,16)) +
+    scale_y_log10() +
+    ggtitle("Time per leapfrog of adjoint and forward per replication")
+
+ggsave("tuning_scale_time_per_leap.png", width=1.6 *4, height=4, dpi=120)
+
 theme_set(theme_bw())
 ggplot(scale_benchmark, aes(system_size, time_per_leap, group=set, colour=adjoint_integrator, shape=repl)) +
    geom_point()  + geom_line() +
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 0bd5cf5f66d..d22302d6faf 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -144,13 +144,11 @@ std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint(const F& f, const T_y0& y0, const T_t0& t0,
             const std::vector<T_ts>& ts,
-            std::ostream* msgs,  // @rok: should be moved to end like
-                                 // for bdf??
             double rel_tol_f, Eigen::VectorXd abs_tol_f, double rel_tol_b,
             double abs_tol_b, double rel_tol_q, double abs_tol_q,
             long int max_num_steps, long int num_checkpoints,
             int interpolation_polynomial, int solver_f, int solver_b,
-            // std::ostream* msgs,
+            std::ostream* msgs,
             const T_Args&... args) {
   return ode_adjoint_impl(
       "ode_adjoint", f, y0, t0, ts, rel_tol_f, abs_tol_f, rel_tol_b, abs_tol_b,
@@ -164,7 +162,6 @@ std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
                 const std::vector<T_ts>& ts,
-                // std::ostream* msgs,
                 double rel_tol_f, Eigen::VectorXd abs_tol_f, double rel_tol_b,
                 double abs_tol_b, double rel_tol_q, double abs_tol_q,
                 long int max_num_steps, long int num_checkpoints,

From ff87fd5f087760557e2277e19b370aa626aa3c73 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 2 Mar 2021 22:47:44 +0100
Subject: [PATCH 057/157] tune sparse matrix support

---
 ode-adjoint/linked-mass-flow.stan             |   4 +-
 ode-adjoint/scale-benchmark.R                 |  32 ++-
 .../rev/functor/cvodes_integrator_adjoint.hpp | 218 ++++++++++++++----
 .../ode_adjoint_cvodes_grad_rev_test.cpp      |   6 +-
 4 files changed, 201 insertions(+), 59 deletions(-)

diff --git a/ode-adjoint/linked-mass-flow.stan b/ode-adjoint/linked-mass-flow.stan
index 8e69a76b6d4..1f39b843041 100644
--- a/ode-adjoint/linked-mass-flow.stan
+++ b/ode-adjoint/linked-mass-flow.stan
@@ -84,8 +84,8 @@ data {
   int<lower=0,upper=1> adjoint_integrator;
   int<lower=1> max_num_steps;
   int<lower=1> num_checkpoints;
-  int<lower=1,upper=2> solver_f;
-  int<lower=1,upper=2> solver_b;
+  int<lower=1,upper=4> solver_f;
+  int<lower=1,upper=4> solver_b;
   int<lower=1> system_size;
   int<lower=1> num_obs;
   real<lower=0> sigma_sim;
diff --git a/ode-adjoint/scale-benchmark.R b/ode-adjoint/scale-benchmark.R
index 3b9e04033c6..bab13dc5ea3 100644
--- a/ode-adjoint/scale-benchmark.R
+++ b/ode-adjoint/scale-benchmark.R
@@ -26,6 +26,7 @@ job_resources <- list(walltime=4*60, memory=2000, ncpus=1, max.concurrent.jobs=3
 
 ## choose number of sample as being dividable by 2 minus one. This gives us exactly by 2 divisable number of bins.
 base_data  <- list(num_warmup=20, num_sampling=20, adapt_delta=0.9, stan_model=stan_model)
+base_data  <- list(num_warmup=10, num_sampling=10, adapt_delta=0.9, stan_model=stan_model)
 
 addProblem("linked_mass_flow",
            data = base_data,
@@ -43,8 +44,20 @@ addAlgorithm("stan_cvodes", run_benchmark)
 pdes <- list(linked_mass_flow = data.frame(system_size=c(1,2,4,8)))
 ades <- list(stan_cvodes= expand.grid(adjoint_integrator=c(1), num_checkpoints=c(10, 100, 250), solver_f=1:2, solver_b=1:2))
 
+
+pdes <- list(linked_mass_flow = data.frame(system_size=c(128)))
+
+ades <- list(stan_cvodes= rbind(expand.grid(adjoint_integrator=c(1), num_checkpoints=c(250), solver_f=2, solver_b=2),
+                                expand.grid(adjoint_integrator=c(1), num_checkpoints=c(250), solver_f=4, solver_b=4))
+             )
+
+
+##ades <- list(stan_cvodes= rbind(expand.grid(adjoint_integrator=c(1), num_checkpoints=c(250), solver_f=1:2, solver_b=1:2),
+##                                expand.grid(adjoint_integrator=c(1), num_checkpoints=c(250)##, solver_f=3:4, solver_b=3:4))
+##             )
+
 ##addExperiments(pdes, ades, repls = 1000)
-addExperiments(pdes, ades, repls = 4)
+addExperiments(pdes, ades, repls = 1)
 
 summarizeExperiments()
 
@@ -109,9 +122,13 @@ head(scale_benchmark)
 
 scale_benchmark <- mutate(scale_benchmark,
                           adjoint_integrator=factor(adjoint_integrator),
+                          sparse_f=solver_f > 2,
+                          sparse_b=solver_b > 2,
+                          lmm_f=case_when(solver_f %% 2 == 0 ~ "BDF", TRUE ~ "Adams"),
+                          lmm_b=case_when(solver_b %% 2 == 0 ~ "BDF", TRUE ~ "Adams")
                           ##num_checkpoints=factor(num_checkpoints),
-                          solver_f=factor(solver_f),
-                          solver_b=factor(solver_b)
+                          ##solver_f=factor(solver_f),
+                          ##solver_b=factor(solver_b)
                           )
 
 saveRDS(scale_benchmark, file = "scale_benchmark.rds")
@@ -141,6 +158,15 @@ ggplot(scale_benchmark, aes(num_checkpoints, time_per_leap, colour=solver, shape
 
 ggsave("tuning_scale_time_per_leap.png", width=1.6 *4, height=4, dpi=120)
 
+
+ggplot(scale_benchmark, aes(sparse_f, time_per_leap, colour=sparse_b, linetype=lmm_f, shape=repl)) +
+    facet_grid(system_size~lmm_b, scales="free_y") +
+   geom_point()  + geom_line() +
+    ##scale_x_log10(breaks=c(1,2,4,8,16)) +
+    scale_y_log10() +
+    ggtitle("Time per leapfrog of adjoint and forward per replication")
+
+
 theme_set(theme_bw())
 ggplot(scale_benchmark, aes(system_size, time_per_leap, group=set, colour=adjoint_integrator, shape=repl)) +
    geom_point()  + geom_line() +
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 6b5103b5d9d..3bf6e8065e4 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -9,7 +9,9 @@
 #include <stan/math/prim/err.hpp>
 #include <stan/math/prim/fun/value_of.hpp>
 #include <cvodes/cvodes.h>
+#include <sunmatrix/sunmatrix_sparse.h>
 #include <sunlinsol/sunlinsol_dense.h>
+#include <sunlinsol/sunlinsol_klu.h>
 #include <algorithm>
 #include <ostream>
 #include <utility>
@@ -77,6 +79,7 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   const size_t N_;
   Eigen::VectorXd abs_tol_f_;
   const int lmm_f_;
+  const int ls_sparse_f_;
   const F f_;
   const Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1> y0_;
   const T_t0 t0_;
@@ -92,17 +95,20 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   Eigen::VectorXd state;
   N_Vector nv_state_;
   N_Vector nv_abs_tol_f_;
-  SUNMatrix A_;
-  SUNLinearSolver LS_;
+  SUNMatrix J_dense_states_;
+  SUNMatrix A_f_;
+  SUNLinearSolver LS_f_;
+  const int nz_;
 
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
-  cvodes_integrator_adjoint_memory(Eigen::VectorXd abs_tol_f, int lmm_f,
+  cvodes_integrator_adjoint_memory(Eigen::VectorXd abs_tol_f, int lmm_f, int ls_sparse_f,
                                    const F& f, const T_y0& y0, const T_t0& t0,
                                    const std::vector<T_ts>& ts,
                                    const T_Args&... args)
       : N_(y0.size()),
         abs_tol_f_(abs_tol_f),
         lmm_f_(lmm_f),
+        ls_sparse_f_(ls_sparse_f),
         f_(f),
         y0_(y0),
         t0_(t0),
@@ -112,13 +118,19 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
         value_of_args_tuple_(value_of(args)...),
         y_(ts_.size()),
         cvodes_mem_(nullptr),
-        state(value_of(y0)) {
+        state(value_of(y0)),
+        nz_(N_*N_) {
     if (N_ > 0) {
       nv_state_ = N_VMake_Serial(N_, state.data());
       nv_abs_tol_f_ = N_VMake_Serial(N_, &abs_tol_f_(0));
-      A_ = SUNDenseMatrix(N_, N_);
-      LS_ = SUNDenseLinearSolver(nv_state_, A_);
-
+      J_dense_states_ = SUNDenseMatrix(N_, N_);
+      if (ls_sparse_f) {
+        A_f_ = SUNSparseMatrix(N_, N_, nz_, CSC_MAT);
+        LS_f_ = SUNLinSol_KLU(nv_state_, A_f_);
+      } else {
+        A_f_ = SUNDenseMatrix(N_, N_); 
+        LS_f_ = SUNDenseLinearSolver(nv_state_, A_f_);
+      }
       cvodes_mem_ = CVodeCreate(lmm_f_);
       if (cvodes_mem_ == nullptr) {
         throw std::runtime_error("CVodeCreate failed to allocate memory");
@@ -128,8 +140,9 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
 
   ~cvodes_integrator_adjoint_memory() {
     if (N_ > 0) {
-      SUNLinSolFree(LS_);
-      SUNMatDestroy(A_);
+      SUNLinSolFree(LS_f_);
+      SUNMatDestroy(J_dense_states_);
+      SUNMatDestroy(A_f_);
 
       N_VDestroy_Serial(nv_state_);
       N_VDestroy_Serial(nv_abs_tol_f_);
@@ -176,6 +189,8 @@ class cvodes_integrator_adjoint_vari : public vari {
   int solver_b_;
   int lmm_f_;
   int lmm_b_;
+  int ls_sparse_f_;
+  int ls_sparse_b_;
 
   const size_t t0_vars_;
   const size_t ts_vars_;
@@ -240,6 +255,14 @@ class cvodes_integrator_adjoint_vari : public vari {
     integrator->jacobian_states(t, y, J);
     return 0;
   }
+  static int cv_jacobian_states_sparse(realtype t, N_Vector y, N_Vector fy,
+                                       SUNMatrix J, void* user_data, N_Vector tmp1,
+                                       N_Vector tmp2, N_Vector tmp3) {
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+    integrator->jacobian_states_sparse(t, y, J);
+    return 0;
+  }
 
   /**
    * Implements the CVLsJacFnB function for evaluating the jacobian of
@@ -254,6 +277,15 @@ class cvodes_integrator_adjoint_vari : public vari {
     return 0;
   }
 
+  static int cv_jacobian_adj_sparse(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
+                                    SUNMatrix J, void* user_data, N_Vector tmp1,
+                                    N_Vector tmp2, N_Vector tmp3) {
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+    integrator->jacobian_adj_sparse(t, y, J);
+    return 0;
+  }
+
   /**
    * Calculates the ODE RHS, dy_dt, using the user-supplied functor at
    * the given time t and state y.
@@ -371,8 +403,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * given time-point t and state y.
    */
   inline void jacobian_states(double t, N_Vector y, SUNMatrix J) const {
-    Eigen::VectorXd fy;
-    Eigen::MatrixXd Jfy;
+    Eigen::Map<Eigen::MatrixXd> Jfy(SM_DATA_D(J), N_, N_);
+    Eigen::Map<const Eigen::VectorXd> x(NV_DATA_S(y), N_);
 
     auto f_wrapped = [&](const Eigen::Matrix<var, Eigen::Dynamic, 1>& y) {
       return apply(
@@ -380,16 +412,35 @@ class cvodes_integrator_adjoint_vari : public vari {
           memory->value_of_args_tuple_);
     };
 
-    jacobian(f_wrapped, Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_), fy,
-             Jfy);
+    nested_rev_autodiff nested;
 
-    for (size_t j = 0; j < Jfy.cols(); ++j) {
-      for (size_t i = 0; i < Jfy.rows(); ++i) {
-        SM_ELEMENT_D(J, i, j) = Jfy(i, j);
-      }
+    Eigen::Matrix<var, Eigen::Dynamic, 1> x_var(x);
+    Eigen::Matrix<var, Eigen::Dynamic, 1> fx_var = f_wrapped(x_var);
+    grad(fx_var(0).vi_);
+    for (size_t j = 0; j < N_; ++j) {
+      //SM_ELEMENT_D(Jfy, 0, j) = x_var(j).adj();
+      Jfy(0, j) = x_var(j).adj();
+    } 
+    for (int i = 1; i < fx_var.size(); ++i) {
+      nested.set_zero_all_adjoints();
+      grad(fx_var(i).vi_);
+      for (size_t j = 0; j < N_; ++j) {
+        //SM_ELEMENT_D(Jfy, i, j) = x_var(j).adj();
+        Jfy(i, j) = x_var(j).adj();
+      } 
     }
   }
 
+  inline void jacobian_states_sparse(double t, N_Vector y, SUNMatrix J) const {
+    //SUNMatrix J_dense = SUNDenseMatrix(N_, N_);
+
+    jacobian_states(t, y, memory->J_dense_states_);
+    
+    J = SUNSparseFromDenseMatrix(memory->J_dense_states_, 1E-8, CSC_MAT);
+
+    //SUNMatDestroy(J_dense);
+  }
+
   /*
    * Calculate the Jacobian of the RHS of the adjoint ODE (see rhs_adj_sens
    * below for citation for how this is done)
@@ -399,8 +450,29 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[out] J CVode structure where output is to be stored
    */
   inline void jacobian_adj(double t, N_Vector y, SUNMatrix J) const {
-    Eigen::VectorXd fy;
-    Eigen::MatrixXd Jfy;
+    Eigen::Map<Eigen::MatrixXd> J_adj_y(SM_DATA_D(J), N_, N_);
+
+    // J_adj_y = -1 * transpose(J_y)
+    jacobian_states(t, y, J);
+
+    J_adj_y.transposeInPlace();
+    J_adj_y.array() *= -1.0;    
+  }
+
+  inline void jacobian_adj_sparse(double t, N_Vector y, SUNMatrix J) const {
+    //SUNMatrix J_adj_y_dense = SUNDenseMatrix(N_, N_);
+
+    jacobian_adj(t, y, memory->J_dense_states_);
+    
+    J = SUNSparseFromDenseMatrix(memory->J_dense_states_, 1E-8, CSC_MAT);
+
+    //SUNMatDestroy(J_adj_y_dense);
+
+    /*    
+    //Eigen::VectorXd fy;
+    //Eigen::MatrixXd Jfy;
+    SUNMatrix Jfy = SUNDenseMatrix(N_, N_);
+    Eigen::Map<const Eigen::VectorXd> x(NV_DATA_S(y), N_);
 
     auto f_wrapped = [&](const Eigen::Matrix<var, Eigen::Dynamic, 1>& y) {
       return apply(
@@ -408,16 +480,40 @@ class cvodes_integrator_adjoint_vari : public vari {
           memory->value_of_args_tuple_);
     };
 
-    jacobian(f_wrapped, Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_), fy,
-             Jfy);
 
+    nested_rev_autodiff nested;
+
+    Eigen::Matrix<var, Eigen::Dynamic, 1> x_var(x);
+    Eigen::Matrix<var, Eigen::Dynamic, 1> fx_var = f_wrapped(x_var);
+    grad(fx_var(0).vi_);
+    for (size_t j = 0; j < N_; ++j) {
+      //SM_ELEMENT_D(Jfy_dense, j, 0) = -1 *  Jfy(0, j);
+      SM_ELEMENT_D(Jfy, j, 0) = -1 *  x_var(j).adj();
+    } 
+    for (int i = 1; i < fx_var.size(); ++i) {
+      nested.set_zero_all_adjoints();
+      grad(fx_var(i).vi_);
+      for (size_t j = 0; j < N_; ++j) {
+        //SM_ELEMENT_D(Jfy_dense, j, 0) = -1 *  Jfy(0, j);
+        SM_ELEMENT_D(Jfy, j, i) = -1 *  x_var(j).adj();
+      } 
+    }
+
+    //SUNMatZero(J);
+    J = SUNSparseFromDenseMatrix(Jfy, 1E-8, CSC_MAT);
+
+    SUNMatDestroy(Jfy);
+    */
+    /*
     for (size_t j = 0; j < Jfy.cols(); ++j) {
       for (size_t i = 0; i < Jfy.rows(); ++i) {
         SM_ELEMENT_D(J, j, i) = -Jfy(i, j);
       }
     }
+    */
   }
 
+  
  public:
   /**
    * Construct cvodes_integrator object
@@ -474,8 +570,10 @@ class cvodes_integrator_adjoint_vari : public vari {
         interpolation_polynomial_(interpolation_polynomial),
         solver_f_(solver_f),
         solver_b_(solver_b),
-        lmm_f_(solver_f_ == 1 ? CV_ADAMS : CV_BDF),
-        lmm_b_(solver_b_ == 1 ? CV_ADAMS : CV_BDF),
+        lmm_f_(solver_f_ % 2 ? CV_ADAMS : CV_BDF),
+        lmm_b_(solver_b_ % 2 ? CV_ADAMS : CV_BDF),
+        ls_sparse_f_(solver_f_ > 2 ? 1 : 0),
+        ls_sparse_b_(solver_b_ > 2 ? 1 : 0),
         msgs_(msgs),
 
         t0_vars_(count_vars(t0)),
@@ -489,12 +587,13 @@ class cvodes_integrator_adjoint_vari : public vari {
         y0_varis_(
             ChainableStack::instance_->memalloc_.alloc_array<vari*>(y0_vars_)),
         args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
-            args_vars_)) {
+            args_vars_))
+  {
     const char* fun = "cvodes_integrator::integrate";
 
     memory
         = new cvodes_integrator_adjoint_memory<F, T_y0, T_t0, T_ts, T_Args...>(
-            abs_tol_f_, lmm_f_, f, y0, t0, ts, args...);
+            abs_tol_f_, lmm_f_, ls_sparse_f_, f, y0, t0, ts, args...);
 
     save_varis(t0_varis_, t0);
     save_varis(ts_varis_, ts);
@@ -529,9 +628,9 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_positive(fun, "num_checkpoints", num_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
     check_range(fun, "interpolation_polynomial", 2, interpolation_polynomial_);
-    // 1=Adams, 2=BDF, 3=BDF_iterated (todo)
-    check_range(fun, "solver_f", 2, solver_f_);
-    check_range(fun, "solver_b", 2, solver_b_);
+    // 1=Adams, 2=BDF, 3=Adams_sparse, 4=BDF_sparse 
+    check_range(fun, "solver_f", 4, solver_f_);
+    check_range(fun, "solver_b", 4, solver_b_);
 
     /*
     std::cout << "relative_tolerance = " << relative_tolerance << std::endl;
@@ -579,13 +678,20 @@ class cvodes_integrator_adjoint_vari : public vari {
     // create matrix object and linear solver object; resource
     // (de-)allocation is handled in the cvodes_ode_data
     check_flag_sundials(
-        CVodeSetLinearSolver(memory->cvodes_mem_, memory->LS_, memory->A_),
+        CVodeSetLinearSolver(memory->cvodes_mem_, memory->LS_f_, memory->A_f_),
         "CVodeSetLinearSolver");
 
-    check_flag_sundials(
-        CVodeSetJacFn(memory->cvodes_mem_,
-                      &cvodes_integrator_adjoint_vari::cv_jacobian_states),
-        "CVodeSetJacFn");
+    if(ls_sparse_f_) {
+      check_flag_sundials(
+          CVodeSetJacFn(memory->cvodes_mem_,
+                        &cvodes_integrator_adjoint_vari::cv_jacobian_states_sparse),
+          "CVodeSetJacFn");
+    } else {
+      check_flag_sundials(
+          CVodeSetJacFn(memory->cvodes_mem_,
+                        &cvodes_integrator_adjoint_vari::cv_jacobian_states),
+          "CVodeSetJacFn");
+    }
 
     // initialize forward sensitivity system of CVODES as needed
     if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
@@ -661,8 +767,15 @@ class cvodes_integrator_adjoint_vari : public vari {
     N_VConst(0.0, nv_state_sens);
     N_VConst(0.0, nv_quad);
 
-    SUNMatrix AB_ = SUNDenseMatrix(N_, N_);
-    SUNLinearSolver LSB_ = SUNDenseLinearSolver(nv_state_sens, AB_);
+    SUNMatrix A_b;
+    SUNLinearSolver LS_b;
+    if (ls_sparse_b_) {
+      A_b = SUNSparseMatrix(N_, N_, N_*N_, CSC_MAT);
+      LS_b = SUNLinSol_KLU(nv_state_sens, A_b);
+    } else {
+      A_b = SUNDenseMatrix(N_, N_); 
+      LS_b = SUNDenseLinearSolver(nv_state_sens, A_b);
+    }
 
     /* check these if needed
       flag = CVodeSetUserDataB(cvode_mem, which, user_dataB);
@@ -736,22 +849,25 @@ class cvodes_integrator_adjoint_vari : public vari {
                                 "CVodeSetMaxNumStepsB");
 
             check_flag_sundials(
-                CVodeSetLinearSolverB(memory->cvodes_mem_, indexB, LSB_, AB_),
+                CVodeSetLinearSolverB(memory->cvodes_mem_, indexB, LS_b, A_b),
                 "CVodeSetLinearSolverB");
 
-            check_flag_sundials(
-                CVodeSetJacFnB(
-                    memory->cvodes_mem_, indexB,
-                    &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
-                "CVodeSetJacFnB");
+            if(ls_sparse_b_) {
+              check_flag_sundials(
+                  CVodeSetJacFnB(
+                      memory->cvodes_mem_, indexB,
+                      &cvodes_integrator_adjoint_vari::cv_jacobian_adj_sparse),
+                  "CVodeSetJacFnB");
+            } else {
+              check_flag_sundials(
+                  CVodeSetJacFnB(
+                      memory->cvodes_mem_, indexB,
+                      &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
+                  "CVodeSetJacFnB");
+            }
 
             // Allocate space for backwards quadrature needed when
-            // parameters vary. The function for which we seek the
-            // derivative wrt to all parameters p is (in notation of
-            // the CVODES manual)
-            // g(t,y,p) = sum_i=1^N y_i * a^y_i * delta(t-T)
-            // this is the sum over each state y_i multiplied with
-            // it's adjoint and a delta function which peaks at t=T
+            // parameters vary.
             if (args_vars_ > 0) {
               check_flag_sundials(
                   CVodeQuadInitB(
@@ -843,15 +959,15 @@ class cvodes_integrator_adjoint_vari : public vari {
       }
 
     } catch (const std::exception& e) {
-      SUNLinSolFree(LSB_);
-      SUNMatDestroy(AB_);
+      SUNLinSolFree(LS_b);
+      SUNMatDestroy(A_b);
       N_VDestroy_Serial(nv_state_sens);
       N_VDestroy_Serial(nv_quad);
       throw;
     }
 
-    SUNLinSolFree(LSB_);
-    SUNMatDestroy(AB_);
+    SUNLinSolFree(LS_b);
+    SUNMatDestroy(A_b);
     N_VDestroy_Serial(nv_state_sens);
     N_VDestroy_Serial(nv_quad);
   }
diff --git a/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
index 4cb0d0cacf7..5f01da36a1c 100644
--- a/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
@@ -285,10 +285,10 @@ class test_functor_sum_var_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_adjoint(sho, y0, t0, ts, nullptr, 1E-10,
+              ? stan::math::ode_adjoint(sho, y0, t0, ts, 1E-10,
                                         abs_tol_f, 1E-10, 1E-10, 1E-10, 1E-10,
-                                        10000, 50, 1, 1, 1,
-                                        // nullptr,
+                                        10000, 50, 1, 1, 1, 
+                                        nullptr,
                                         omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 

From 69e2e913b1c0d2209388a24d4a61e16b30f557c1 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Tue, 2 Mar 2021 21:51:30 +0000
Subject: [PATCH 058/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 69 ++++++++++---------
 stan/math/rev/functor/ode_adjoint.hpp         | 23 +++----
 .../ode_adjoint_cvodes_grad_rev_test.cpp      |  8 +--
 3 files changed, 49 insertions(+), 51 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 3bf6e8065e4..3a5f1bcfe56 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -101,9 +101,9 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   const int nz_;
 
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
-  cvodes_integrator_adjoint_memory(Eigen::VectorXd abs_tol_f, int lmm_f, int ls_sparse_f,
-                                   const F& f, const T_y0& y0, const T_t0& t0,
-                                   const std::vector<T_ts>& ts,
+  cvodes_integrator_adjoint_memory(Eigen::VectorXd abs_tol_f, int lmm_f,
+                                   int ls_sparse_f, const F& f, const T_y0& y0,
+                                   const T_t0& t0, const std::vector<T_ts>& ts,
                                    const T_Args&... args)
       : N_(y0.size()),
         abs_tol_f_(abs_tol_f),
@@ -119,7 +119,7 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
         y_(ts_.size()),
         cvodes_mem_(nullptr),
         state(value_of(y0)),
-        nz_(N_*N_) {
+        nz_(N_ * N_) {
     if (N_ > 0) {
       nv_state_ = N_VMake_Serial(N_, state.data());
       nv_abs_tol_f_ = N_VMake_Serial(N_, &abs_tol_f_(0));
@@ -128,7 +128,7 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
         A_f_ = SUNSparseMatrix(N_, N_, nz_, CSC_MAT);
         LS_f_ = SUNLinSol_KLU(nv_state_, A_f_);
       } else {
-        A_f_ = SUNDenseMatrix(N_, N_); 
+        A_f_ = SUNDenseMatrix(N_, N_);
         LS_f_ = SUNDenseLinearSolver(nv_state_, A_f_);
       }
       cvodes_mem_ = CVodeCreate(lmm_f_);
@@ -256,8 +256,9 @@ class cvodes_integrator_adjoint_vari : public vari {
     return 0;
   }
   static int cv_jacobian_states_sparse(realtype t, N_Vector y, N_Vector fy,
-                                       SUNMatrix J, void* user_data, N_Vector tmp1,
-                                       N_Vector tmp2, N_Vector tmp3) {
+                                       SUNMatrix J, void* user_data,
+                                       N_Vector tmp1, N_Vector tmp2,
+                                       N_Vector tmp3) {
     const cvodes_integrator_adjoint_vari* integrator
         = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_states_sparse(t, y, J);
@@ -277,9 +278,10 @@ class cvodes_integrator_adjoint_vari : public vari {
     return 0;
   }
 
-  static int cv_jacobian_adj_sparse(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
-                                    SUNMatrix J, void* user_data, N_Vector tmp1,
-                                    N_Vector tmp2, N_Vector tmp3) {
+  static int cv_jacobian_adj_sparse(realtype t, N_Vector y, N_Vector yB,
+                                    N_Vector fyB, SUNMatrix J, void* user_data,
+                                    N_Vector tmp1, N_Vector tmp2,
+                                    N_Vector tmp3) {
     const cvodes_integrator_adjoint_vari* integrator
         = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_adj_sparse(t, y, J);
@@ -418,27 +420,27 @@ class cvodes_integrator_adjoint_vari : public vari {
     Eigen::Matrix<var, Eigen::Dynamic, 1> fx_var = f_wrapped(x_var);
     grad(fx_var(0).vi_);
     for (size_t j = 0; j < N_; ++j) {
-      //SM_ELEMENT_D(Jfy, 0, j) = x_var(j).adj();
+      // SM_ELEMENT_D(Jfy, 0, j) = x_var(j).adj();
       Jfy(0, j) = x_var(j).adj();
-    } 
+    }
     for (int i = 1; i < fx_var.size(); ++i) {
       nested.set_zero_all_adjoints();
       grad(fx_var(i).vi_);
       for (size_t j = 0; j < N_; ++j) {
-        //SM_ELEMENT_D(Jfy, i, j) = x_var(j).adj();
+        // SM_ELEMENT_D(Jfy, i, j) = x_var(j).adj();
         Jfy(i, j) = x_var(j).adj();
-      } 
+      }
     }
   }
 
   inline void jacobian_states_sparse(double t, N_Vector y, SUNMatrix J) const {
-    //SUNMatrix J_dense = SUNDenseMatrix(N_, N_);
+    // SUNMatrix J_dense = SUNDenseMatrix(N_, N_);
 
     jacobian_states(t, y, memory->J_dense_states_);
-    
+
     J = SUNSparseFromDenseMatrix(memory->J_dense_states_, 1E-8, CSC_MAT);
 
-    //SUNMatDestroy(J_dense);
+    // SUNMatDestroy(J_dense);
   }
 
   /*
@@ -456,19 +458,19 @@ class cvodes_integrator_adjoint_vari : public vari {
     jacobian_states(t, y, J);
 
     J_adj_y.transposeInPlace();
-    J_adj_y.array() *= -1.0;    
+    J_adj_y.array() *= -1.0;
   }
 
   inline void jacobian_adj_sparse(double t, N_Vector y, SUNMatrix J) const {
-    //SUNMatrix J_adj_y_dense = SUNDenseMatrix(N_, N_);
+    // SUNMatrix J_adj_y_dense = SUNDenseMatrix(N_, N_);
 
     jacobian_adj(t, y, memory->J_dense_states_);
-    
+
     J = SUNSparseFromDenseMatrix(memory->J_dense_states_, 1E-8, CSC_MAT);
 
-    //SUNMatDestroy(J_adj_y_dense);
+    // SUNMatDestroy(J_adj_y_dense);
 
-    /*    
+    /*
     //Eigen::VectorXd fy;
     //Eigen::MatrixXd Jfy;
     SUNMatrix Jfy = SUNDenseMatrix(N_, N_);
@@ -489,14 +491,14 @@ class cvodes_integrator_adjoint_vari : public vari {
     for (size_t j = 0; j < N_; ++j) {
       //SM_ELEMENT_D(Jfy_dense, j, 0) = -1 *  Jfy(0, j);
       SM_ELEMENT_D(Jfy, j, 0) = -1 *  x_var(j).adj();
-    } 
+    }
     for (int i = 1; i < fx_var.size(); ++i) {
       nested.set_zero_all_adjoints();
       grad(fx_var(i).vi_);
       for (size_t j = 0; j < N_; ++j) {
         //SM_ELEMENT_D(Jfy_dense, j, 0) = -1 *  Jfy(0, j);
         SM_ELEMENT_D(Jfy, j, i) = -1 *  x_var(j).adj();
-      } 
+      }
     }
 
     //SUNMatZero(J);
@@ -513,7 +515,6 @@ class cvodes_integrator_adjoint_vari : public vari {
     */
   }
 
-  
  public:
   /**
    * Construct cvodes_integrator object
@@ -587,8 +588,7 @@ class cvodes_integrator_adjoint_vari : public vari {
         y0_varis_(
             ChainableStack::instance_->memalloc_.alloc_array<vari*>(y0_vars_)),
         args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
-            args_vars_))
-  {
+            args_vars_)) {
     const char* fun = "cvodes_integrator::integrate";
 
     memory
@@ -628,7 +628,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_positive(fun, "num_checkpoints", num_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
     check_range(fun, "interpolation_polynomial", 2, interpolation_polynomial_);
-    // 1=Adams, 2=BDF, 3=Adams_sparse, 4=BDF_sparse 
+    // 1=Adams, 2=BDF, 3=Adams_sparse, 4=BDF_sparse
     check_range(fun, "solver_f", 4, solver_f_);
     check_range(fun, "solver_b", 4, solver_b_);
 
@@ -681,10 +681,11 @@ class cvodes_integrator_adjoint_vari : public vari {
         CVodeSetLinearSolver(memory->cvodes_mem_, memory->LS_f_, memory->A_f_),
         "CVodeSetLinearSolver");
 
-    if(ls_sparse_f_) {
+    if (ls_sparse_f_) {
       check_flag_sundials(
-          CVodeSetJacFn(memory->cvodes_mem_,
-                        &cvodes_integrator_adjoint_vari::cv_jacobian_states_sparse),
+          CVodeSetJacFn(
+              memory->cvodes_mem_,
+              &cvodes_integrator_adjoint_vari::cv_jacobian_states_sparse),
           "CVodeSetJacFn");
     } else {
       check_flag_sundials(
@@ -770,10 +771,10 @@ class cvodes_integrator_adjoint_vari : public vari {
     SUNMatrix A_b;
     SUNLinearSolver LS_b;
     if (ls_sparse_b_) {
-      A_b = SUNSparseMatrix(N_, N_, N_*N_, CSC_MAT);
+      A_b = SUNSparseMatrix(N_, N_, N_ * N_, CSC_MAT);
       LS_b = SUNLinSol_KLU(nv_state_sens, A_b);
     } else {
-      A_b = SUNDenseMatrix(N_, N_); 
+      A_b = SUNDenseMatrix(N_, N_);
       LS_b = SUNDenseLinearSolver(nv_state_sens, A_b);
     }
 
@@ -852,7 +853,7 @@ class cvodes_integrator_adjoint_vari : public vari {
                 CVodeSetLinearSolverB(memory->cvodes_mem_, indexB, LS_b, A_b),
                 "CVodeSetLinearSolverB");
 
-            if(ls_sparse_b_) {
+            if (ls_sparse_b_) {
               check_flag_sundials(
                   CVodeSetJacFnB(
                       memory->cvodes_mem_, indexB,
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index d22302d6faf..17d3d1f3ff7 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -143,12 +143,11 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint(const F& f, const T_y0& y0, const T_t0& t0,
-            const std::vector<T_ts>& ts,
-            double rel_tol_f, Eigen::VectorXd abs_tol_f, double rel_tol_b,
-            double abs_tol_b, double rel_tol_q, double abs_tol_q,
-            long int max_num_steps, long int num_checkpoints,
-            int interpolation_polynomial, int solver_f, int solver_b,
-            std::ostream* msgs,
+            const std::vector<T_ts>& ts, double rel_tol_f,
+            Eigen::VectorXd abs_tol_f, double rel_tol_b, double abs_tol_b,
+            double rel_tol_q, double abs_tol_q, long int max_num_steps,
+            long int num_checkpoints, int interpolation_polynomial,
+            int solver_f, int solver_b, std::ostream* msgs,
             const T_Args&... args) {
   return ode_adjoint_impl(
       "ode_adjoint", f, y0, t0, ts, rel_tol_f, abs_tol_f, rel_tol_b, abs_tol_b,
@@ -161,12 +160,12 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
-                const std::vector<T_ts>& ts,
-                double rel_tol_f, Eigen::VectorXd abs_tol_f, double rel_tol_b,
-                double abs_tol_b, double rel_tol_q, double abs_tol_q,
-                long int max_num_steps, long int num_checkpoints,
-                int interpolation_polynomial, int solver_f, int solver_b,
-                std::ostream* msgs, const T_Args&... args) {
+                const std::vector<T_ts>& ts, double rel_tol_f,
+                Eigen::VectorXd abs_tol_f, double rel_tol_b, double abs_tol_b,
+                double rel_tol_q, double abs_tol_q, long int max_num_steps,
+                long int num_checkpoints, int interpolation_polynomial,
+                int solver_f, int solver_b, std::ostream* msgs,
+                const T_Args&... args) {
   return ode_adjoint_impl(
       "ode_adjoint_tol", f, y0, t0, ts, rel_tol_f, abs_tol_f, rel_tol_b,
       abs_tol_b, rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,
diff --git a/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
index 5f01da36a1c..17750585e0d 100644
--- a/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
@@ -285,11 +285,9 @@ class test_functor_sum_var_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_adjoint(sho, y0, t0, ts, 1E-10,
-                                        abs_tol_f, 1E-10, 1E-10, 1E-10, 1E-10,
-                                        10000, 50, 1, 1, 1, 
-                                        nullptr,
-                                        omega)
+              ? stan::math::ode_adjoint(sho, y0, t0, ts, 1E-10, abs_tol_f,
+                                        1E-10, 1E-10, 1E-10, 1E-10, 10000, 50,
+                                        1, 1, 1, nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return stan::math::sum(ys[0](state) + ys[1](state));

From 4a8afdf161466bb1700f07be6f0edd388afce753 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 4 Mar 2021 21:38:17 +0100
Subject: [PATCH 059/157] disable sparse matrix stuff on CVODES and switch to
 new function names for ode_adjoint_tol_ctl names

---
 ode-adjoint/linked-mass-flow.stan             |  27 ++-
 .../rev/functor/cvodes_integrator_adjoint.hpp | 175 ++++--------------
 stan/math/rev/functor/ode_adjoint.hpp         |  40 ++--
 .../ode_adjoint_cvodes_grad_rev_test.cpp      |   8 +-
 4 files changed, 76 insertions(+), 174 deletions(-)

diff --git a/ode-adjoint/linked-mass-flow.stan b/ode-adjoint/linked-mass-flow.stan
index 1f39b843041..85b210e307d 100644
--- a/ode-adjoint/linked-mass-flow.stan
+++ b/ode-adjoint/linked-mass-flow.stan
@@ -59,16 +59,25 @@ functions {
     }
 
     if(adjoint_integrator) {
+      /* parser does not accept vector backward for now... */
+      y = ode_adjoint_tol_ctl(linked_mass_flow, a0, t0, ts, 
+                              rel_tol, rep_vector(abs_tol, num_states), // forward
+                              //rel_tol, rep_vector(10*abs_tol, num_states), // backward
+                              rel_tol, 10*abs_tol, // backward
+                              rel_tol, 100*abs_tol, // quadrature
+                              max_num_steps,
+                              num_checkpoints, // number of steps between checkpoints
+                              1,  // hermite polynomials
+                              solver_f,  // bdf forward
+                              solver_b,  // bdf backward
+                              kt, e50, k12, k21);
+      /**/
+      /*
       y = ode_adjoint_tol(linked_mass_flow, a0, t0, ts, 
-                          rel_tol, rep_vector(abs_tol, num_states), // forward
-                          rel_tol, 10*abs_tol, // backward
-                          rel_tol, 100*abs_tol, // quadrature
-                          max_num_steps,
-                          num_checkpoints, // number of steps between checkpoints
-                          1,  // hermite polynomials
-                          solver_f,  // bdf forward
-                          solver_b,  // bdf backward
-                          kt, e50, k12, k21);
+                              rel_tol, abs_tol,
+                              max_num_steps,
+                              kt, e50, k12, k21);
+      */
     } else {
       y = ode_bdf_tol(linked_mass_flow, a0, t0, ts, rel_tol, abs_tol, max_num_steps,
                       kt, e50, k12, k21);
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 3bf6e8065e4..cbef021b351 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -9,9 +9,8 @@
 #include <stan/math/prim/err.hpp>
 #include <stan/math/prim/fun/value_of.hpp>
 #include <cvodes/cvodes.h>
-#include <sunmatrix/sunmatrix_sparse.h>
+#include <sunmatrix/sunmatrix_dense.h>
 #include <sunlinsol/sunlinsol_dense.h>
-#include <sunlinsol/sunlinsol_klu.h>
 #include <algorithm>
 #include <ostream>
 #include <utility>
@@ -78,8 +77,8 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
 
   const size_t N_;
   Eigen::VectorXd abs_tol_f_;
+  Eigen::VectorXd abs_tol_b_;
   const int lmm_f_;
-  const int ls_sparse_f_;
   const F f_;
   const Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1> y0_;
   const T_t0 t0_;
@@ -95,20 +94,19 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   Eigen::VectorXd state;
   N_Vector nv_state_;
   N_Vector nv_abs_tol_f_;
-  SUNMatrix J_dense_states_;
+  N_Vector nv_abs_tol_b_;
   SUNMatrix A_f_;
   SUNLinearSolver LS_f_;
-  const int nz_;
 
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
-  cvodes_integrator_adjoint_memory(Eigen::VectorXd abs_tol_f, int lmm_f, int ls_sparse_f,
+  cvodes_integrator_adjoint_memory(Eigen::VectorXd abs_tol_f, Eigen::VectorXd abs_tol_b, int lmm_f, 
                                    const F& f, const T_y0& y0, const T_t0& t0,
                                    const std::vector<T_ts>& ts,
                                    const T_Args&... args)
       : N_(y0.size()),
         abs_tol_f_(abs_tol_f),
+        abs_tol_b_(abs_tol_b),
         lmm_f_(lmm_f),
-        ls_sparse_f_(ls_sparse_f),
         f_(f),
         y0_(y0),
         t0_(t0),
@@ -118,19 +116,14 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
         value_of_args_tuple_(value_of(args)...),
         y_(ts_.size()),
         cvodes_mem_(nullptr),
-        state(value_of(y0)),
-        nz_(N_*N_) {
+        state(value_of(y0))
+  {
     if (N_ > 0) {
       nv_state_ = N_VMake_Serial(N_, state.data());
       nv_abs_tol_f_ = N_VMake_Serial(N_, &abs_tol_f_(0));
-      J_dense_states_ = SUNDenseMatrix(N_, N_);
-      if (ls_sparse_f) {
-        A_f_ = SUNSparseMatrix(N_, N_, nz_, CSC_MAT);
-        LS_f_ = SUNLinSol_KLU(nv_state_, A_f_);
-      } else {
-        A_f_ = SUNDenseMatrix(N_, N_); 
-        LS_f_ = SUNDenseLinearSolver(nv_state_, A_f_);
-      }
+      nv_abs_tol_b_ = N_VMake_Serial(N_, &abs_tol_b_(0));
+      A_f_ = SUNDenseMatrix(N_, N_); 
+      LS_f_ = SUNDenseLinearSolver(nv_state_, A_f_);
       cvodes_mem_ = CVodeCreate(lmm_f_);
       if (cvodes_mem_ == nullptr) {
         throw std::runtime_error("CVodeCreate failed to allocate memory");
@@ -141,7 +134,6 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   ~cvodes_integrator_adjoint_memory() {
     if (N_ > 0) {
       SUNLinSolFree(LS_f_);
-      SUNMatDestroy(J_dense_states_);
       SUNMatDestroy(A_f_);
 
       N_VDestroy_Serial(nv_state_);
@@ -179,7 +171,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   double rel_tol_f_;
   Eigen::VectorXd abs_tol_f_;
   double rel_tol_b_;
-  double abs_tol_b_;
+  Eigen::VectorXd abs_tol_b_;
   double rel_tol_q_;
   double abs_tol_q_;
   long int max_num_steps_;
@@ -189,8 +181,6 @@ class cvodes_integrator_adjoint_vari : public vari {
   int solver_b_;
   int lmm_f_;
   int lmm_b_;
-  int ls_sparse_f_;
-  int ls_sparse_b_;
 
   const size_t t0_vars_;
   const size_t ts_vars_;
@@ -255,14 +245,6 @@ class cvodes_integrator_adjoint_vari : public vari {
     integrator->jacobian_states(t, y, J);
     return 0;
   }
-  static int cv_jacobian_states_sparse(realtype t, N_Vector y, N_Vector fy,
-                                       SUNMatrix J, void* user_data, N_Vector tmp1,
-                                       N_Vector tmp2, N_Vector tmp3) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
-    integrator->jacobian_states_sparse(t, y, J);
-    return 0;
-  }
 
   /**
    * Implements the CVLsJacFnB function for evaluating the jacobian of
@@ -277,15 +259,6 @@ class cvodes_integrator_adjoint_vari : public vari {
     return 0;
   }
 
-  static int cv_jacobian_adj_sparse(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
-                                    SUNMatrix J, void* user_data, N_Vector tmp1,
-                                    N_Vector tmp2, N_Vector tmp3) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
-    integrator->jacobian_adj_sparse(t, y, J);
-    return 0;
-  }
-
   /**
    * Calculates the ODE RHS, dy_dt, using the user-supplied functor at
    * the given time t and state y.
@@ -431,16 +404,6 @@ class cvodes_integrator_adjoint_vari : public vari {
     }
   }
 
-  inline void jacobian_states_sparse(double t, N_Vector y, SUNMatrix J) const {
-    //SUNMatrix J_dense = SUNDenseMatrix(N_, N_);
-
-    jacobian_states(t, y, memory->J_dense_states_);
-    
-    J = SUNSparseFromDenseMatrix(memory->J_dense_states_, 1E-8, CSC_MAT);
-
-    //SUNMatDestroy(J_dense);
-  }
-
   /*
    * Calculate the Jacobian of the RHS of the adjoint ODE (see rhs_adj_sens
    * below for citation for how this is done)
@@ -458,61 +421,6 @@ class cvodes_integrator_adjoint_vari : public vari {
     J_adj_y.transposeInPlace();
     J_adj_y.array() *= -1.0;    
   }
-
-  inline void jacobian_adj_sparse(double t, N_Vector y, SUNMatrix J) const {
-    //SUNMatrix J_adj_y_dense = SUNDenseMatrix(N_, N_);
-
-    jacobian_adj(t, y, memory->J_dense_states_);
-    
-    J = SUNSparseFromDenseMatrix(memory->J_dense_states_, 1E-8, CSC_MAT);
-
-    //SUNMatDestroy(J_adj_y_dense);
-
-    /*    
-    //Eigen::VectorXd fy;
-    //Eigen::MatrixXd Jfy;
-    SUNMatrix Jfy = SUNDenseMatrix(N_, N_);
-    Eigen::Map<const Eigen::VectorXd> x(NV_DATA_S(y), N_);
-
-    auto f_wrapped = [&](const Eigen::Matrix<var, Eigen::Dynamic, 1>& y) {
-      return apply(
-          [&](auto&&... args) { return memory->f_(t, y, msgs_, args...); },
-          memory->value_of_args_tuple_);
-    };
-
-
-    nested_rev_autodiff nested;
-
-    Eigen::Matrix<var, Eigen::Dynamic, 1> x_var(x);
-    Eigen::Matrix<var, Eigen::Dynamic, 1> fx_var = f_wrapped(x_var);
-    grad(fx_var(0).vi_);
-    for (size_t j = 0; j < N_; ++j) {
-      //SM_ELEMENT_D(Jfy_dense, j, 0) = -1 *  Jfy(0, j);
-      SM_ELEMENT_D(Jfy, j, 0) = -1 *  x_var(j).adj();
-    } 
-    for (int i = 1; i < fx_var.size(); ++i) {
-      nested.set_zero_all_adjoints();
-      grad(fx_var(i).vi_);
-      for (size_t j = 0; j < N_; ++j) {
-        //SM_ELEMENT_D(Jfy_dense, j, 0) = -1 *  Jfy(0, j);
-        SM_ELEMENT_D(Jfy, j, i) = -1 *  x_var(j).adj();
-      } 
-    }
-
-    //SUNMatZero(J);
-    J = SUNSparseFromDenseMatrix(Jfy, 1E-8, CSC_MAT);
-
-    SUNMatDestroy(Jfy);
-    */
-    /*
-    for (size_t j = 0; j < Jfy.cols(); ++j) {
-      for (size_t i = 0; i < Jfy.rows(); ++i) {
-        SM_ELEMENT_D(J, j, i) = -Jfy(i, j);
-      }
-    }
-    */
-  }
-
   
  public:
   /**
@@ -550,7 +458,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   cvodes_integrator_adjoint_vari(
       const char* function_name, const F& f, const T_y0& y0, const T_t0& t0,
       const std::vector<T_ts>& ts, double rel_tol_f, Eigen::VectorXd abs_tol_f,
-      double rel_tol_b, double abs_tol_b, double rel_tol_q, double abs_tol_q,
+      double rel_tol_b, Eigen::VectorXd abs_tol_b, double rel_tol_q, double abs_tol_q,
       long int max_num_steps, long int num_checkpoints,
       int interpolation_polynomial, int solver_f, int solver_b,
       std::ostream* msgs, const T_Args&... args)
@@ -572,8 +480,6 @@ class cvodes_integrator_adjoint_vari : public vari {
         solver_b_(solver_b),
         lmm_f_(solver_f_ % 2 ? CV_ADAMS : CV_BDF),
         lmm_b_(solver_b_ % 2 ? CV_ADAMS : CV_BDF),
-        ls_sparse_f_(solver_f_ > 2 ? 1 : 0),
-        ls_sparse_b_(solver_b_ > 2 ? 1 : 0),
         msgs_(msgs),
 
         t0_vars_(count_vars(t0)),
@@ -593,7 +499,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     memory
         = new cvodes_integrator_adjoint_memory<F, T_y0, T_t0, T_ts, T_Args...>(
-            abs_tol_f_, lmm_f_, ls_sparse_f_, f, y0, t0, ts, args...);
+            abs_tol_f_, abs_tol_b_, lmm_f_, f, y0, t0, ts, args...);
 
     save_varis(t0_varis_, t0);
     save_varis(ts_varis_, ts);
@@ -622,15 +528,16 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_size_match(fun, "abs_tol_f", abs_tol_f_.size(), "states", N_);
     check_positive_finite(fun, "rel_tol_b", rel_tol_b_);
     check_positive_finite(fun, "abs_tol_b", abs_tol_b_);
+    check_size_match(fun, "abs_tol_b", abs_tol_b_.size(), "states", N_);
     check_positive_finite(fun, "rel_tol_q", rel_tol_q_);
     check_positive_finite(fun, "abs_tol_q", abs_tol_q_);
     check_positive(fun, "max_num_steps", max_num_steps_);
     check_positive(fun, "num_checkpoints", num_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
     check_range(fun, "interpolation_polynomial", 2, interpolation_polynomial_);
-    // 1=Adams, 2=BDF, 3=Adams_sparse, 4=BDF_sparse 
-    check_range(fun, "solver_f", 4, solver_f_);
-    check_range(fun, "solver_b", 4, solver_b_);
+    // 1=Adams, 2=BDF
+    check_range(fun, "solver_f", 2, solver_f_);
+    check_range(fun, "solver_b", 2, solver_b_);
 
     /*
     std::cout << "relative_tolerance = " << relative_tolerance << std::endl;
@@ -681,17 +588,10 @@ class cvodes_integrator_adjoint_vari : public vari {
         CVodeSetLinearSolver(memory->cvodes_mem_, memory->LS_f_, memory->A_f_),
         "CVodeSetLinearSolver");
 
-    if(ls_sparse_f_) {
-      check_flag_sundials(
-          CVodeSetJacFn(memory->cvodes_mem_,
-                        &cvodes_integrator_adjoint_vari::cv_jacobian_states_sparse),
-          "CVodeSetJacFn");
-    } else {
-      check_flag_sundials(
-          CVodeSetJacFn(memory->cvodes_mem_,
-                        &cvodes_integrator_adjoint_vari::cv_jacobian_states),
-          "CVodeSetJacFn");
-    }
+    check_flag_sundials(
+        CVodeSetJacFn(memory->cvodes_mem_,
+                      &cvodes_integrator_adjoint_vari::cv_jacobian_states),
+        "CVodeSetJacFn");
 
     // initialize forward sensitivity system of CVODES as needed
     if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
@@ -769,13 +669,8 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     SUNMatrix A_b;
     SUNLinearSolver LS_b;
-    if (ls_sparse_b_) {
-      A_b = SUNSparseMatrix(N_, N_, N_*N_, CSC_MAT);
-      LS_b = SUNLinSol_KLU(nv_state_sens, A_b);
-    } else {
-      A_b = SUNDenseMatrix(N_, N_); 
-      LS_b = SUNDenseLinearSolver(nv_state_sens, A_b);
-    }
+    A_b = SUNDenseMatrix(N_, N_); 
+    LS_b = SUNDenseLinearSolver(nv_state_sens, A_b);
 
     /* check these if needed
       flag = CVodeSetUserDataB(cvode_mem, which, user_dataB);
@@ -840,9 +735,9 @@ class cvodes_integrator_adjoint_vari : public vari {
                            t_init, nv_state_sens),
                 "CVodeInitB");
 
-            check_flag_sundials(CVodeSStolerancesB(memory->cvodes_mem_, indexB,
-                                                   rel_tol_b_, abs_tol_b_),
-                                "CVodeSStolerancesB");
+            check_flag_sundials(CVodeSVtolerancesB(memory->cvodes_mem_, indexB,
+                                                   rel_tol_b_, memory->nv_abs_tol_b_),
+                                "CVodeSVtolerancesB");
 
             check_flag_sundials(CVodeSetMaxNumStepsB(memory->cvodes_mem_,
                                                      indexB, max_num_steps_),
@@ -852,19 +747,11 @@ class cvodes_integrator_adjoint_vari : public vari {
                 CVodeSetLinearSolverB(memory->cvodes_mem_, indexB, LS_b, A_b),
                 "CVodeSetLinearSolverB");
 
-            if(ls_sparse_b_) {
-              check_flag_sundials(
-                  CVodeSetJacFnB(
-                      memory->cvodes_mem_, indexB,
-                      &cvodes_integrator_adjoint_vari::cv_jacobian_adj_sparse),
-                  "CVodeSetJacFnB");
-            } else {
-              check_flag_sundials(
-                  CVodeSetJacFnB(
-                      memory->cvodes_mem_, indexB,
-                      &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
-                  "CVodeSetJacFnB");
-            }
+            check_flag_sundials(
+                CVodeSetJacFnB(
+                    memory->cvodes_mem_, indexB,
+                    &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
+                "CVodeSetJacFnB");
 
             // Allocate space for backwards quadrature needed when
             // parameters vary.
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index d22302d6faf..1ececb942c0 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -59,7 +59,7 @@ std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
                  const T_t0& t0, const std::vector<T_ts>& ts, double rel_tol_f,
-                 Eigen::VectorXd abs_tol_f, double rel_tol_b, double abs_tol_b,
+                 Eigen::VectorXd abs_tol_f, double rel_tol_b, Eigen::VectorXd abs_tol_b,
                  double rel_tol_q, double abs_tol_q, long int max_num_steps,
                  long int num_checkpoints, int interpolation_polynomial,
                  int solver_f, int solver_b, std::ostream* msgs,
@@ -142,16 +142,19 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
-ode_adjoint(const F& f, const T_y0& y0, const T_t0& t0,
-            const std::vector<T_ts>& ts,
-            double rel_tol_f, Eigen::VectorXd abs_tol_f, double rel_tol_b,
-            double abs_tol_b, double rel_tol_q, double abs_tol_q,
-            long int max_num_steps, long int num_checkpoints,
-            int interpolation_polynomial, int solver_f, int solver_b,
-            std::ostream* msgs,
-            const T_Args&... args) {
+ode_adjoint_tol_ctl(const F& f, const T_y0& y0, const T_t0& t0,
+                    const std::vector<T_ts>& ts,
+                    double rel_tol_f, Eigen::VectorXd abs_tol_f,
+                    double rel_tol_b, double abs_tol_b_scalar, //Eigen::VectorXd abs_tol_b,
+                    double rel_tol_q, double abs_tol_q,
+                    long int max_num_steps, long int num_checkpoints,
+                    int interpolation_polynomial, int solver_f, int solver_b,
+                    std::ostream* msgs,
+                    const T_Args&... args) {
+  const int N = y0.size();
+  const Eigen::VectorXd abs_tol_b = Eigen::VectorXd::Constant(N, abs_tol_b_scalar);
   return ode_adjoint_impl(
-      "ode_adjoint", f, y0, t0, ts, rel_tol_f, abs_tol_f, rel_tol_b, abs_tol_b,
+      "ode_adjoint_tol_ctl", f, y0, t0, ts, rel_tol_f, abs_tol_f, rel_tol_b, abs_tol_b,
       rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,
       interpolation_polynomial, solver_f, solver_b, msgs, args...);
 }
@@ -162,14 +165,19 @@ std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
                 const std::vector<T_ts>& ts,
-                double rel_tol_f, Eigen::VectorXd abs_tol_f, double rel_tol_b,
-                double abs_tol_b, double rel_tol_q, double abs_tol_q,
-                long int max_num_steps, long int num_checkpoints,
-                int interpolation_polynomial, int solver_f, int solver_b,
+                double rel_tol, double abs_tol,
+                long int max_num_steps,
                 std::ostream* msgs, const T_Args&... args) {
+  const int N = y0.size();
+  const Eigen::VectorXd abs_tol_f = Eigen::VectorXd::Constant(N, abs_tol/100.0);
+  const Eigen::VectorXd abs_tol_b = Eigen::VectorXd::Constant(N, abs_tol/10.0);
+  const long int num_checkpoints = 250;
+  const int interpolation_polynomial = 1;
+  const int solver_f = 2;
+  const int solver_b = 2;
   return ode_adjoint_impl(
-      "ode_adjoint_tol", f, y0, t0, ts, rel_tol_f, abs_tol_f, rel_tol_b,
-      abs_tol_b, rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,
+      "ode_adjoint_tol", f, y0, t0, ts, rel_tol, abs_tol_f, rel_tol,
+      abs_tol_b, rel_tol, abs_tol, max_num_steps, num_checkpoints,
       interpolation_polynomial, solver_f, solver_b, msgs, args...);
 }
 
diff --git a/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
index 5f01da36a1c..cd7041ed2bf 100644
--- a/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
@@ -285,11 +285,9 @@ class test_functor_sum_var_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_adjoint(sho, y0, t0, ts, 1E-10,
-                                        abs_tol_f, 1E-10, 1E-10, 1E-10, 1E-10,
-                                        10000, 50, 1, 1, 1, 
-                                        nullptr,
-                                        omega)
+        ? stan::math::ode_adjoint_tol(sho, y0, t0, ts, 1E-10, 1E-10, 10000,
+                                      nullptr,
+                                      omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return stan::math::sum(ys[0](state) + ys[1](state));

From 73041c1b6abb68f79740e04cdf710f99290673c0 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 4 Mar 2021 21:45:02 +0100
Subject: [PATCH 060/157] enable more tests

---
 .../math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
index 8cda66d967e..0f3112552f6 100644
--- a/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
@@ -42,7 +42,7 @@ struct sho_functor {
     return out;
   }
 };
-/*
+
 template <int state>
 class test_functor_double_var {
   int lmm_;
@@ -64,7 +64,7 @@ class test_functor_double_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_adjoint(sho, y0, t0, ts, nullptr, omega)
+        ? stan::math::ode_adjoint_tol(sho, y0, t0, ts, 1E-10, 1E-10, 10000, nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return ys[1](state);
@@ -112,7 +112,7 @@ TEST(StanMathOdeIntegrateODEGradMat, double_var) {
     EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-7);
   }
 }
-*/
+
 template <int state>
 class test_functor_var_double {
   int lmm_;

From 6e238d7631afefe2d438e04905748fc371f978ab Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Thu, 4 Mar 2021 20:54:41 +0000
Subject: [PATCH 061/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 21 ++++----
 stan/math/rev/functor/ode_adjoint.hpp         | 51 ++++++++++---------
 .../ode_adjoint_cvodes_grad_rev_test.cpp      | 16 +++---
 3 files changed, 45 insertions(+), 43 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index e777f5218c5..45f047296ac 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -99,7 +99,8 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   SUNLinearSolver LS_f_;
 
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
-  cvodes_integrator_adjoint_memory(Eigen::VectorXd abs_tol_f, Eigen::VectorXd abs_tol_b, int lmm_f, 
+  cvodes_integrator_adjoint_memory(Eigen::VectorXd abs_tol_f,
+                                   Eigen::VectorXd abs_tol_b, int lmm_f,
                                    const F& f, const T_y0& y0, const T_t0& t0,
                                    const std::vector<T_ts>& ts,
                                    const T_Args&... args)
@@ -116,13 +117,12 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
         value_of_args_tuple_(value_of(args)...),
         y_(ts_.size()),
         cvodes_mem_(nullptr),
-        state(value_of(y0))
-  {
+        state(value_of(y0)) {
     if (N_ > 0) {
       nv_state_ = N_VMake_Serial(N_, state.data());
       nv_abs_tol_f_ = N_VMake_Serial(N_, &abs_tol_f_(0));
       nv_abs_tol_b_ = N_VMake_Serial(N_, &abs_tol_b_(0));
-      A_f_ = SUNDenseMatrix(N_, N_); 
+      A_f_ = SUNDenseMatrix(N_, N_);
       LS_f_ = SUNDenseLinearSolver(nv_state_, A_f_);
 
       cvodes_mem_ = CVodeCreate(lmm_f_);
@@ -459,8 +459,8 @@ class cvodes_integrator_adjoint_vari : public vari {
   cvodes_integrator_adjoint_vari(
       const char* function_name, const F& f, const T_y0& y0, const T_t0& t0,
       const std::vector<T_ts>& ts, double rel_tol_f, Eigen::VectorXd abs_tol_f,
-      double rel_tol_b, Eigen::VectorXd abs_tol_b, double rel_tol_q, double abs_tol_q,
-      long int max_num_steps, long int num_checkpoints,
+      double rel_tol_b, Eigen::VectorXd abs_tol_b, double rel_tol_q,
+      double abs_tol_q, long int max_num_steps, long int num_checkpoints,
       int interpolation_polynomial, int solver_f, int solver_b,
       std::ostream* msgs, const T_Args&... args)
       : function_name_(function_name),
@@ -669,7 +669,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     SUNMatrix A_b;
     SUNLinearSolver LS_b;
-    A_b = SUNDenseMatrix(N_, N_); 
+    A_b = SUNDenseMatrix(N_, N_);
     LS_b = SUNDenseLinearSolver(nv_state_sens, A_b);
 
     /* check these if needed
@@ -735,9 +735,10 @@ class cvodes_integrator_adjoint_vari : public vari {
                            t_init, nv_state_sens),
                 "CVodeInitB");
 
-            check_flag_sundials(CVodeSVtolerancesB(memory->cvodes_mem_, indexB,
-                                                   rel_tol_b_, memory->nv_abs_tol_b_),
-                                "CVodeSVtolerancesB");
+            check_flag_sundials(
+                CVodeSVtolerancesB(memory->cvodes_mem_, indexB, rel_tol_b_,
+                                   memory->nv_abs_tol_b_),
+                "CVodeSVtolerancesB");
 
             check_flag_sundials(CVodeSetMaxNumStepsB(memory->cvodes_mem_,
                                                      indexB, max_num_steps_),
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 153db6341a2..1d31e3b43c9 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -59,11 +59,11 @@ std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
                  const T_t0& t0, const std::vector<T_ts>& ts, double rel_tol_f,
-                 Eigen::VectorXd abs_tol_f, double rel_tol_b, Eigen::VectorXd abs_tol_b,
-                 double rel_tol_q, double abs_tol_q, long int max_num_steps,
-                 long int num_checkpoints, int interpolation_polynomial,
-                 int solver_f, int solver_b, std::ostream* msgs,
-                 const T_Args&... args) {
+                 Eigen::VectorXd abs_tol_f, double rel_tol_b,
+                 Eigen::VectorXd abs_tol_b, double rel_tol_q, double abs_tol_q,
+                 long int max_num_steps, long int num_checkpoints,
+                 int interpolation_polynomial, int solver_f, int solver_b,
+                 std::ostream* msgs, const T_Args&... args) {
   /*
   const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
   return apply(
@@ -143,19 +143,19 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_tol_ctl(const F& f, const T_y0& y0, const T_t0& t0,
-                    const std::vector<T_ts>& ts,
-                    double rel_tol_f, Eigen::VectorXd abs_tol_f,
-                    double rel_tol_b, double abs_tol_b_scalar, //Eigen::VectorXd abs_tol_b,
-                    double rel_tol_q, double abs_tol_q,
-                    long int max_num_steps, long int num_checkpoints,
-                    int interpolation_polynomial, int solver_f, int solver_b,
-                    std::ostream* msgs,
+                    const std::vector<T_ts>& ts, double rel_tol_f,
+                    Eigen::VectorXd abs_tol_f, double rel_tol_b,
+                    double abs_tol_b_scalar,  // Eigen::VectorXd abs_tol_b,
+                    double rel_tol_q, double abs_tol_q, long int max_num_steps,
+                    long int num_checkpoints, int interpolation_polynomial,
+                    int solver_f, int solver_b, std::ostream* msgs,
                     const T_Args&... args) {
   const int N = y0.size();
-  const Eigen::VectorXd abs_tol_b = Eigen::VectorXd::Constant(N, abs_tol_b_scalar);
+  const Eigen::VectorXd abs_tol_b
+      = Eigen::VectorXd::Constant(N, abs_tol_b_scalar);
   return ode_adjoint_impl(
-      "ode_adjoint_tol_ctl", f, y0, t0, ts, rel_tol_f, abs_tol_f, rel_tol_b, abs_tol_b,
-      rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,
+      "ode_adjoint_tol_ctl", f, y0, t0, ts, rel_tol_f, abs_tol_f, rel_tol_b,
+      abs_tol_b, rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,
       interpolation_polynomial, solver_f, solver_b, msgs, args...);
 }
 
@@ -164,21 +164,22 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
-                const std::vector<T_ts>& ts,
-                double rel_tol, double abs_tol,
-                long int max_num_steps,
-                std::ostream* msgs, const T_Args&... args) {
+                const std::vector<T_ts>& ts, double rel_tol, double abs_tol,
+                long int max_num_steps, std::ostream* msgs,
+                const T_Args&... args) {
   const int N = y0.size();
-  const Eigen::VectorXd abs_tol_f = Eigen::VectorXd::Constant(N, abs_tol/100.0);
-  const Eigen::VectorXd abs_tol_b = Eigen::VectorXd::Constant(N, abs_tol/10.0);
+  const Eigen::VectorXd abs_tol_f
+      = Eigen::VectorXd::Constant(N, abs_tol / 100.0);
+  const Eigen::VectorXd abs_tol_b
+      = Eigen::VectorXd::Constant(N, abs_tol / 10.0);
   const long int num_checkpoints = 250;
   const int interpolation_polynomial = 1;
   const int solver_f = 2;
   const int solver_b = 2;
-  return ode_adjoint_impl(
-      "ode_adjoint_tol", f, y0, t0, ts, rel_tol, abs_tol_f, rel_tol,
-      abs_tol_b, rel_tol, abs_tol, max_num_steps, num_checkpoints,
-      interpolation_polynomial, solver_f, solver_b, msgs, args...);
+  return ode_adjoint_impl("ode_adjoint_tol", f, y0, t0, ts, rel_tol, abs_tol_f,
+                          rel_tol, abs_tol_b, rel_tol, abs_tol, max_num_steps,
+                          num_checkpoints, interpolation_polynomial, solver_f,
+                          solver_b, msgs, args...);
 }
 
 }  // namespace math
diff --git a/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
index 0f3112552f6..1cf19ed1e83 100644
--- a/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
+++ b/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
@@ -64,7 +64,8 @@ class test_functor_double_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-        ? stan::math::ode_adjoint_tol(sho, y0, t0, ts, 1E-10, 1E-10, 10000, nullptr, omega)
+              ? stan::math::ode_adjoint_tol(sho, y0, t0, ts, 1E-10, 1E-10,
+                                            10000, nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return ys[1](state);
@@ -134,7 +135,8 @@ class test_functor_var_double {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-        ? stan::math::ode_adjoint_tol(sho, y0, t0, ts, 1E-10, 1E-10, 10000, nullptr, omega)
+              ? stan::math::ode_adjoint_tol(sho, y0, t0, ts, 1E-10, 1E-10,
+                                            10000, nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return ys[0](state);
@@ -203,8 +205,8 @@ class test_functor_var_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-        ? stan::math::ode_adjoint_tol(sho, y0, t0, ts, 1E-10, 1E-10, 10000,
-                                      nullptr, omega)
+              ? stan::math::ode_adjoint_tol(sho, y0, t0, ts, 1E-10, 1E-10,
+                                            10000, nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return ys[1](state);
@@ -260,7 +262,6 @@ TEST(StanMathOdeIntegrateODEGradMat, var_var) {
   }
 }
 
-
 template <int state>
 class test_functor_sum_var_var {
   int lmm_;
@@ -285,9 +286,8 @@ class test_functor_sum_var_var {
 
     std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
         = (lmm_ == TEST_CVODES_ADAMS)
-        ? stan::math::ode_adjoint_tol(sho, y0, t0, ts, 1E-10, 1E-10, 10000,
-                                      nullptr,
-                                      omega)
+              ? stan::math::ode_adjoint_tol(sho, y0, t0, ts, 1E-10, 1E-10,
+                                            10000, nullptr, omega)
               : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
 
     return stan::math::sum(ys[0](state) + ys[1](state));

From efba64e6d2cb5539ad4295a7eda8c90edac745b3 Mon Sep 17 00:00:00 2001
From: Rok Cesnovar <rok.cesnovar@fri.uni-lj.si>
Date: Fri, 5 Mar 2021 10:02:28 +0100
Subject: [PATCH 062/157] use vector for abs_tol_b

---
 stan/math/rev/functor/ode_adjoint.hpp | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 1d31e3b43c9..b7efb2d1e24 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -145,14 +145,11 @@ std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
 ode_adjoint_tol_ctl(const F& f, const T_y0& y0, const T_t0& t0,
                     const std::vector<T_ts>& ts, double rel_tol_f,
                     Eigen::VectorXd abs_tol_f, double rel_tol_b,
-                    double abs_tol_b_scalar,  // Eigen::VectorXd abs_tol_b,
+                    Eigen::VectorXd abs_tol_b,
                     double rel_tol_q, double abs_tol_q, long int max_num_steps,
                     long int num_checkpoints, int interpolation_polynomial,
                     int solver_f, int solver_b, std::ostream* msgs,
                     const T_Args&... args) {
-  const int N = y0.size();
-  const Eigen::VectorXd abs_tol_b
-      = Eigen::VectorXd::Constant(N, abs_tol_b_scalar);
   return ode_adjoint_impl(
       "ode_adjoint_tol_ctl", f, y0, t0, ts, rel_tol_f, abs_tol_f, rel_tol_b,
       abs_tol_b, rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,

From 6db3dad5ab57efcba1135763c475531438f44764 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Fri, 5 Mar 2021 09:03:07 +0000
Subject: [PATCH 063/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/ode_adjoint.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index b7efb2d1e24..88ba737a718 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -145,8 +145,8 @@ std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
 ode_adjoint_tol_ctl(const F& f, const T_y0& y0, const T_t0& t0,
                     const std::vector<T_ts>& ts, double rel_tol_f,
                     Eigen::VectorXd abs_tol_f, double rel_tol_b,
-                    Eigen::VectorXd abs_tol_b,
-                    double rel_tol_q, double abs_tol_q, long int max_num_steps,
+                    Eigen::VectorXd abs_tol_b, double rel_tol_q,
+                    double abs_tol_q, long int max_num_steps,
                     long int num_checkpoints, int interpolation_polynomial,
                     int solver_f, int solver_b, std::ostream* msgs,
                     const T_Args&... args) {

From 024e94a1a33cb4865e241d0880273d0931607ed3 Mon Sep 17 00:00:00 2001
From: Rok Cesnovar <rok.cesnovar@fri.uni-lj.si>
Date: Fri, 5 Mar 2021 10:05:47 +0100
Subject: [PATCH 064/157] fix test model

---
 ode-adjoint/linked-mass-flow.stan | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ode-adjoint/linked-mass-flow.stan b/ode-adjoint/linked-mass-flow.stan
index 85b210e307d..cb2cfb0eaa3 100644
--- a/ode-adjoint/linked-mass-flow.stan
+++ b/ode-adjoint/linked-mass-flow.stan
@@ -62,8 +62,7 @@ functions {
       /* parser does not accept vector backward for now... */
       y = ode_adjoint_tol_ctl(linked_mass_flow, a0, t0, ts, 
                               rel_tol, rep_vector(abs_tol, num_states), // forward
-                              //rel_tol, rep_vector(10*abs_tol, num_states), // backward
-                              rel_tol, 10*abs_tol, // backward
+                              rel_tol, rep_vector(10*abs_tol, num_states), // backward
                               rel_tol, 100*abs_tol, // quadrature
                               max_num_steps,
                               num_checkpoints, // number of steps between checkpoints

From 4450cca1d47d10401dffc3f2d1baf5f561d25046 Mon Sep 17 00:00:00 2001
From: Ben <bbbales2@gmail.com>
Date: Sat, 6 Mar 2021 17:47:19 -0500
Subject: [PATCH 065/157] Switch to reference types

---
 stan/math/rev/functor/ode_adjoint.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 88ba737a718..858697df994 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -98,10 +98,10 @@ ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
   return (*integrator)();
   */
   auto integrator = new stan::math::cvodes_integrator_adjoint_vari<
-      F, plain_type_t<T_y0>, T_t0, T_ts, plain_type_t<T_Args>...>(
+      F, plain_type_t<T_y0>, T_t0, T_ts, ref_type_t<T_Args>...>(
       function_name, f, eval(y0), t0, ts, rel_tol_f, abs_tol_f, rel_tol_b,
       abs_tol_b, rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,
-      interpolation_polynomial, solver_f, solver_b, msgs, eval(args)...);
+      interpolation_polynomial, solver_f, solver_b, msgs, to_ref(args)...);
   return (*integrator)();
 }
 

From 149c649848bdaf8f5bb3c575db2bb2a37c10d6b5 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sebastian.weber@novartis.com>
Date: Thu, 11 Mar 2021 10:17:35 +0100
Subject: [PATCH 066/157] align defaults of simplified ode_adjoint_tol
 interface with doc

---
 stan/math/rev/functor/ode_adjoint.hpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 858697df994..5db7f9d1036 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -170,9 +170,9 @@ ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
   const Eigen::VectorXd abs_tol_b
       = Eigen::VectorXd::Constant(N, abs_tol / 10.0);
   const long int num_checkpoints = 250;
-  const int interpolation_polynomial = 1;
-  const int solver_f = 2;
-  const int solver_b = 2;
+  const int interpolation_polynomial = CV_POLYNOMIAL;
+  const int solver_f = CV_BDF;
+  const int solver_b = CV_BDF;
   return ode_adjoint_impl("ode_adjoint_tol", f, y0, t0, ts, rel_tol, abs_tol_f,
                           rel_tol, abs_tol_b, rel_tol, abs_tol, max_num_steps,
                           num_checkpoints, interpolation_polynomial, solver_f,

From b31a72819ce3eec74732b475af0115cc8ce4ad28 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sebastian.weber@novartis.com>
Date: Thu, 11 Mar 2021 13:28:24 +0100
Subject: [PATCH 067/157] revert to hermite polynomials which appear more
 stable

---
 stan/math/rev/functor/ode_adjoint.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 5db7f9d1036..c65e14abf10 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -170,7 +170,7 @@ ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
   const Eigen::VectorXd abs_tol_b
       = Eigen::VectorXd::Constant(N, abs_tol / 10.0);
   const long int num_checkpoints = 250;
-  const int interpolation_polynomial = CV_POLYNOMIAL;
+  const int interpolation_polynomial = CV_HERMITE;
   const int solver_f = CV_BDF;
   const int solver_b = CV_BDF;
   return ode_adjoint_impl("ode_adjoint_tol", f, y0, t0, ts, rel_tol, abs_tol_f,

From fd74b602c863180cbeca9ddf19e01befb7265bdf Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sebastian.weber@novartis.com>
Date: Thu, 11 Mar 2021 13:56:52 +0100
Subject: [PATCH 068/157] bump examples

---
 ode-adjoint/linked-mass-flow-fixed.stan | 206 ++++++++++++++++++++++++
 ode-adjoint/linked-mass-flow.R          |  64 +++++---
 ode-adjoint/linked-mass-flow.stan       |  85 ++++++----
 3 files changed, 296 insertions(+), 59 deletions(-)
 create mode 100644 ode-adjoint/linked-mass-flow-fixed.stan

diff --git a/ode-adjoint/linked-mass-flow-fixed.stan b/ode-adjoint/linked-mass-flow-fixed.stan
new file mode 100644
index 00000000000..16f9ca0cfac
--- /dev/null
+++ b/ode-adjoint/linked-mass-flow-fixed.stan
@@ -0,0 +1,206 @@
+functions {
+
+  vector linked_mass_flow(real t, vector y,
+                          real kp, real ks, real e50, real k12, real k21) {
+
+    int num_main_states = num_elements(y)/2;
+    int num_states = 2 * num_main_states;
+
+    vector[num_states] dydt;
+    vector[num_main_states] mass_flow;
+
+    for (i in 1:num_main_states) {
+      int m = 2 * (i-1) + 1;  // main state
+      int a = m + 1;  // auxilary state
+      mass_flow[i] = (kp + ks  / (y[m] + e50)) * y[m];
+
+      dydt[m] = -mass_flow[i] - k12 * y[m] + k21 * y[a];
+      dydt[a] = +k12 * y[m] - k21 * y[a];
+
+      if (i != 1) {
+        dydt[m] += mass_flow[i-1];
+      }
+    }
+    return dydt;
+
+  }
+
+  vector sample_vector_rng(real m, real s, int N) {
+    vector[N] sample;
+    for(i in 1:N)
+      sample[i] = normal_rng(m, s);
+    return sample;
+  }
+
+  vector[] simulate_mean(real t0, vector log_a0, real[] ts, int adjoint_integrator,
+                         real rel_tol, real abs_tol, int max_num_steps,
+                         int num_checkpoints,
+                         int interpolation_polynomial,
+                         int solver_f, int solver_b,
+                         real log_kp, real log_ks, real log_e50, real log_k12, real log_k21) {
+    int num_sim = num_elements(ts);
+    int num_states = num_elements(log_a0);
+    int system_size = num_states/2;
+    vector[num_states] y[num_sim];
+    real kp = exp(log_kp);
+    real ks = exp(log_ks);
+    real e50 = exp(log_e50);
+    real k12 = exp(log_k12);
+    real k21 = exp(log_k21);
+    vector[num_states] a0 = exp(log_a0);
+
+    if(adjoint_integrator) {
+      y = ode_adjoint_tol_ctl(linked_mass_flow, a0, t0, ts, 
+                              rel_tol, rep_vector(abs_tol/100.0, num_states), // forward
+                              rel_tol, rep_vector(abs_tol/10.0, num_states), // backward
+                              rel_tol, abs_tol, // quadrature
+                              max_num_steps,
+                              num_checkpoints, // number of steps between checkpoints
+                              interpolation_polynomial,  // polynomials
+                              solver_f,  // bdf forward
+                              solver_b,  // bdf backward
+                              kp, ks, e50, k12, k21);
+      /* simplified interface
+      y = ode_adjoint_tol(linked_mass_flow, a0, t0, ts, 
+                              rel_tol, abs_tol,
+                              max_num_steps,
+                              kp, ks, e50, k12, k21);
+      */
+    } else {
+      y = ode_bdf_tol(linked_mass_flow, a0, t0, ts, rel_tol, abs_tol, max_num_steps,
+                      kp, ks, e50, k12, k21);
+    }
+
+    return y;
+  }
+  
+}
+data {
+  real<lower=0> rel_tol;
+  real<lower=0> abs_tol;
+  int<lower=0,upper=1> adjoint_integrator;
+  int<lower=1> max_num_steps;
+  int<lower=1> num_checkpoints;
+  int<lower=1,upper=2> interpolation_polynomial;
+  int<lower=1,upper=2> solver_f;
+  int<lower=1,upper=2> solver_b;
+  int<lower=1> system_size;
+  int<lower=1> num_obs;
+  real<lower=0> sigma_sim;
+  real<lower=0> sigma_y;
+}
+transformed data {
+  int num_states = 2 * system_size;
+  real param_scale = sigma_sim / sqrt(num_obs/4);
+
+  real log_kp_ = normal_rng(0.0, sigma_sim);
+  real log_ks_ = normal_rng(0.0, sigma_sim);
+  real log_e50_ = normal_rng(0.0, sigma_sim);
+  real log_k12_ = normal_rng(0.0, sigma_sim);
+  real log_k21_ = normal_rng(0.0, sigma_sim);
+  vector[system_size] log_sigma_y_ = sample_vector_rng(0.0, sigma_y, system_size);
+  vector[num_states] log_a0_ = sample_vector_rng(0.0, sigma_sim, num_states);
+  vector[num_states] y_[num_obs];
+  real ts[num_obs];
+  real t0 = 0.0;
+
+  ts[1] = 1.0;
+  for(i in 2:num_obs) {
+    ts[i] = 1.5 * ts[i-1];
+  }
+
+  y_ = simulate_mean(t0, log_a0_, ts, 0, rel_tol, abs_tol, max_num_steps,
+                     num_checkpoints,
+                     interpolation_polynomial,
+                     solver_f, solver_b,
+                     log_kp_, log_ks_, log_e50_, log_k12_, log_k21_);
+  
+  for(i in 1:num_obs) {
+    for(j in 1:system_size) {
+      int m = 2*(j-1) + 1;
+      y_[i,m] = lognormal_rng(log(y_[i,m]+1E-3), exp(log_sigma_y_[j]));
+    }
+  }
+
+  if(adjoint_integrator) {
+    print("Using adjoint integrator.");
+  } else {
+    print("Using bdf integrator.");
+  }
+  print("relative tolerance: ", rel_tol);
+  print("absolute tolerance: ", abs_tol);
+  print("maximum number of steps: ", max_num_steps);
+  print("number of checkpoints: ", num_checkpoints);
+  print("interpolation polynomial: ", interpolation_polynomial);
+  print("solver forward: ", solver_f);
+  print("solver backward: ", solver_b);
+  print("number of time points: ", num_obs);
+  print("system size: ", system_size);
+  print("time points: ", ts);
+  print("y_: ", y_);
+  print("ODE states N: ", num_states);
+  print("ODE parameters varying M: ", 5 + num_states);
+}
+parameters {
+  real<multiplier=param_scale,offset=log_kp_> log_kp;
+  real<multiplier=param_scale,offset=log_ks_> log_ks;
+  real<multiplier=param_scale,offset=log_e50_> log_e50;
+  real<multiplier=param_scale,offset=log_k12_> log_k12;
+  real<multiplier=param_scale,offset=log_k21_> log_k21;
+  vector<offset=log_sigma_y_>[system_size] log_sigma_y;
+  vector<multiplier=param_scale,offset=log_a0_>[num_states] log_a0;
+}
+transformed parameters {
+}
+model {
+  vector[num_states] mu[num_obs];
+
+  profile("ode") {
+    mu = simulate_mean(t0, log_a0, ts, adjoint_integrator, rel_tol, abs_tol, max_num_steps,
+                       num_checkpoints,
+                       interpolation_polynomial,
+                       solver_f, solver_b,
+                       log_kp, log_ks, log_e50, log_k12, log_k21);
+  }
+  
+  target += normal_lpdf(log_kp| 0.0, sigma_sim);
+  target += normal_lpdf(log_ks| 0.0, sigma_sim);
+  target += normal_lpdf(log_e50| 0.0, sigma_sim);
+  target += normal_lpdf(log_k12| 0.0, sigma_sim);
+  target += normal_lpdf(log_k21| 0.0, sigma_sim);
+  target += normal_lpdf(log_a0| 0.0, sigma_sim);
+  target += normal_lpdf(log_sigma_y| 0.0, sigma_y);
+
+  for(j in 1:system_size) {
+    int m = 2*(j-1) + 1;
+    target += lognormal_lpdf(to_vector(y_[:,m])| log(to_vector(mu[:,m])+1E-3), exp(log_sigma_y[j]));
+  }
+}
+generated quantities {
+  int rank_log_kp;
+  int rank_log_ks;
+  int rank_log_e50;
+  int rank_log_k12;
+  int rank_log_k21;
+  int rank_log_sigma_y[system_size];
+  int rank_log_a0[system_size];
+  real bias_log_kp = log_kp - log_kp_;
+  real bias_log_ks = log_ks - log_ks_;
+  real bias_log_e50 = log_e50 - log_e50_;
+  real bias_log_k12 = log_k12 - log_k12_;
+  real bias_log_k21 = log_k21 - log_k21_;
+  vector[system_size] bias_log_sigma_y = log_sigma_y - log_sigma_y_;
+  vector[system_size] bias_log_a0 = log_a0 - log_a0_;
+
+  rank_log_kp = (log_kp > log_kp_ ? 1 : 0);
+  rank_log_ks = (log_ks > log_ks_ ? 1 : 0);
+  rank_log_e50 = (log_e50 > log_e50_ ? 1 : 0);
+  rank_log_k12 = (log_k12 > log_k12_ ? 1 : 0);
+  rank_log_k21 = (log_k21 > log_k21_ ? 1 : 0);
+  
+  for(i in 1:system_size) {
+    rank_log_sigma_y[i] = (log_sigma_y[i] > log_sigma_y_[i] ? 1 : 0);
+    rank_log_a0[i] = (log_a0[i] > log_a0_[i] ? 1 : 0);
+  }
+  
+}
diff --git a/ode-adjoint/linked-mass-flow.R b/ode-adjoint/linked-mass-flow.R
index 020d1ab6b16..bdd56ca309b 100644
--- a/ode-adjoint/linked-mass-flow.R
+++ b/ode-adjoint/linked-mass-flow.R
@@ -1,30 +1,43 @@
-##install.packages("cmdstanr", repos = c("https://mc-stan.org/r-packages/", getOption("repos")))
+
+if(FALSE) {
+    ## run as needed to get latest cmdstanr
+    install.packages("cmdstanr", repos = c("https://mc-stan.org/r-packages/", getOption("repos")))
+    ## or even install the latest dev version with profiling support
+    Sys.setenv(R_REMOTES_NO_ERRORS_FROM_WARNINGS="true")
+    remotes::install_github("stan-dev/cmdstanr")
+}
 
 library(cmdstanr)
-library(posterior)
-library(bayesplot)
-color_scheme_set("brightblue")
 
 set_cmdstan_path("~/work/cmdstan")
 
+## model which scales number of states and parameters
 stan_model_file <- "linked-mass-flow.stan"
+## model which scales only number of states, but recycles parameters
+stan_model_fixed_file <- "linked-mass-flow-fixed.stan"
 
 mod <- cmdstan_model(stan_model_file)
+mod_fixed <- cmdstan_model(stan_model_fixed_file)
 
 base_data <- list(rel_tol=1e-6,
-                  abs_tol=1e-6,
-                  max_num_steps=10000,
+                  abs_tol=1e-4,
                   adjoint_integrator=0,
+                  max_num_steps=10000,
+                  num_checkpoints=150,
+                  interpolation_polynomial=1,
+                  solver_f=2,
+                  solver_b=2,
                   system_size=2,
                   num_obs=8,
                   sigma_sim=2.0,
-                  sigma_y=0.2)
+                  sigma_y=0.2
+                  )
 
-run_benchmark <- function(..., num_iter=250, adapt_delta=0.8) {
+run_benchmark <- function(..., model, num_iter=250, adapt_delta=0.8) {
     fit_args <- list(...)
     fit_data <- modifyList(base_data, fit_args)
     print(str(fit_data))
-    fit <- mod$sample(
+    fit <- model$sample(
                    data = fit_data,
                    iter_warmup = num_iter,
                    iter_sampling = num_iter,
@@ -33,20 +46,13 @@ run_benchmark <- function(..., num_iter=250, adapt_delta=0.8) {
                    parallel_chains = 1,
                    refresh = round(num_iter/5),
                    save_warmup=TRUE,
-                   adapt_delta=adapt_delta
+                   adapt_delta=adapt_delta,
+                   init=0.1
                    ##verbose=TRUE
                )
     fit
 }
 
-bdf_fit <- run_benchmark(adjoint_integrator=0)
-adjoint_fit <- run_benchmark(adjoint_integrator=1)
-
-if(FALSE) {
-    adjoint_fit_2 <- run_benchmark(adjoint_integrator=1, adapt_delta=0.65)
-    adjoint_fit_3 <- run_benchmark(adjoint_integrator=1, adapt_delta=0.95)
-    adjoint_fit_4 <- run_benchmark(adjoint_integrator=1, abs_tol=1E-10, rel_tol=1E-10)
-}
 
 summarize_benchmark <- function(fit) {
     sdiag_all  <- fit$sampler_diagnostics(inc_warmup=TRUE)
@@ -67,18 +73,26 @@ summarize_benchmark <- function(fit) {
          ess_lp_speed=ess_speed, lp_est=lp_est)
 }
 
-mcmc_rhat(rhat = rhat(bdf_fit))  + yaxis_text(hjust = 0)
+
+bdf_fit <- run_benchmark(adjoint_integrator=0, num_iter=50)
+adjoint_fit <- run_benchmark(adjoint_integrator=1, num_iter=50)
+## vary solvers, e.g.
+##adjoint_fit <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=1, num_iter=50)
 
 str(summarize_benchmark(bdf_fit))
 str(summarize_benchmark(adjoint_fit))
 
+Lbdf_fit <- run_benchmark(adjoint_integrator=0, system_size=5, model=mod, num_iter=50)
+Ladjoint_fit <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=2, system_size=5, interpolation_polynomial=1, model=mod, num_iter=50)
+Ladjoint_fit_1 <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=1, system_size=5, interpolation_polynomial=1, model=mod, num_iter=50)
 
-str(summarize_benchmark(adjoint_fit_2))
-str(summarize_benchmark(adjoint_fit_3))
-
-bdf_fit$metadata()
+str(summarize_benchmark(Lbdf_fit))
+str(summarize_benchmark(Ladjoint_fit))
+str(summarize_benchmark(Ladjoint_fit_1))
 
-bdf_fit$time()
+Lbdf_fit_fixed <- run_benchmark(adjoint_integrator=0, system_size=5, model=mod_fixed, num_iter=50)
+Ladjoint_fit_fixed <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=2, system_size=5, model=mod_fixed, interpolation_polynomial=1, num_iter=50)
 
+str(summarize_benchmark(Lbdf_fit_fixed))
+str(summarize_benchmark(Ladjoint_fit_fixed))
 
-bdf_fit$summary("lp__")
diff --git a/ode-adjoint/linked-mass-flow.stan b/ode-adjoint/linked-mass-flow.stan
index cb2cfb0eaa3..299ac025e6d 100644
--- a/ode-adjoint/linked-mass-flow.stan
+++ b/ode-adjoint/linked-mass-flow.stan
@@ -1,24 +1,24 @@
 functions {
 
   vector linked_mass_flow(real t, vector y,
-                          real[] kt, real[] e50, real[] k12, real[] k21) {
+                          real[] kp, real[] ks, real[] e50, real[] k12, real[] k21) {
 
-    int num_main_states = num_elements(kt);
+    int num_main_states = num_elements(kp);
     int num_states = 2 * num_main_states;
 
     vector[num_states] dydt;
-    vector[num_main_states] ksat;
+    vector[num_main_states] mass_flow;
 
     for (i in 1:num_main_states) {
       int m = 2 * (i-1) + 1;  // main state
       int a = m + 1;  // auxilary state
-      ksat[i] = kt[i] * y[m] / (y[m] + e50[i]);
+      mass_flow[i] = (kp[i] + ks[i]  / (y[m] + e50[i])) * y[m];
 
-      dydt[m] = -ksat[i] * y[m] - k12[i] * y[m] + k21[i] * y[a];
+      dydt[m] = -mass_flow[i] - k12[i] * y[m] + k21[i] * y[a];
       dydt[a] = +k12[i] * y[m] - k21[i] * y[a];
 
       if (i != 1) {
-        dydt[m] += ksat[i-1] * y[2 * (i - 1) + 1];
+        dydt[m] += mass_flow[i-1];
       }
     }
     return dydt;
@@ -35,20 +35,23 @@ functions {
   vector[] simulate_mean(real t0, vector log_a0, real[] ts, int adjoint_integrator,
                          real rel_tol, real abs_tol, int max_num_steps,
                          int num_checkpoints,
+                         int interpolation_polynomial,
                          int solver_f, int solver_b,
-                         vector log_kt, vector log_e50, vector log_k12, vector log_k21) {
+                         vector log_kp, vector log_ks, vector log_e50, vector log_k12, vector log_k21) {
     int num_sim = num_elements(ts);
     int num_states = num_elements(log_a0);
-    int system_size = num_elements(log_kt);
+    int system_size = num_elements(log_kp);
     vector[num_states] y[num_sim];
-    real kt[system_size];
+    real kp[system_size];
+    real ks[system_size];
     real e50[system_size];
     real k12[system_size];
     real k21[system_size];
     vector[num_states] a0;
 
     for(i in 1:system_size) {
-      kt[i] = exp(log_kt[i]);
+      kp[i] = exp(log_kp[i]);
+      ks[i] = exp(log_ks[i]);
       e50[i] = exp(log_e50[i]);
       k12[i] = exp(log_k12[i]);
       k21[i] = exp(log_k21[i]);
@@ -59,27 +62,25 @@ functions {
     }
 
     if(adjoint_integrator) {
-      /* parser does not accept vector backward for now... */
       y = ode_adjoint_tol_ctl(linked_mass_flow, a0, t0, ts, 
-                              rel_tol, rep_vector(abs_tol, num_states), // forward
-                              rel_tol, rep_vector(10*abs_tol, num_states), // backward
-                              rel_tol, 100*abs_tol, // quadrature
+                              rel_tol, rep_vector(abs_tol/100.0, num_states), // forward
+                              rel_tol, rep_vector(abs_tol/10.0, num_states), // backward
+                              rel_tol, abs_tol, // quadrature
                               max_num_steps,
                               num_checkpoints, // number of steps between checkpoints
-                              1,  // hermite polynomials
+                              interpolation_polynomial,  // polynomials
                               solver_f,  // bdf forward
                               solver_b,  // bdf backward
-                              kt, e50, k12, k21);
-      /**/
-      /*
+                              kp, ks, e50, k12, k21);
+      /* simplified interface
       y = ode_adjoint_tol(linked_mass_flow, a0, t0, ts, 
                               rel_tol, abs_tol,
                               max_num_steps,
-                              kt, e50, k12, k21);
+                              kp, ks, e50, k12, k21);
       */
     } else {
       y = ode_bdf_tol(linked_mass_flow, a0, t0, ts, rel_tol, abs_tol, max_num_steps,
-                      kt, e50, k12, k21);
+                      kp, ks, e50, k12, k21);
     }
 
     return y;
@@ -92,8 +93,9 @@ data {
   int<lower=0,upper=1> adjoint_integrator;
   int<lower=1> max_num_steps;
   int<lower=1> num_checkpoints;
-  int<lower=1,upper=4> solver_f;
-  int<lower=1,upper=4> solver_b;
+  int<lower=1,upper=2> interpolation_polynomial;
+  int<lower=1,upper=2> solver_f;
+  int<lower=1,upper=2> solver_b;
   int<lower=1> system_size;
   int<lower=1> num_obs;
   real<lower=0> sigma_sim;
@@ -103,7 +105,8 @@ transformed data {
   int num_states = 2 * system_size;
   real param_scale = sigma_sim / sqrt(num_obs/4);
 
-  vector[system_size] log_kt_ = sample_vector_rng(0.0, sigma_sim, system_size);
+  vector[system_size] log_kp_ = sample_vector_rng(0.0, sigma_sim, system_size);
+  vector[system_size] log_ks_ = sample_vector_rng(0.0, sigma_sim, system_size);
   vector[system_size] log_e50_ = sample_vector_rng(0.0, sigma_sim, system_size);
   vector[system_size] log_k12_ = sample_vector_rng(0.0, sigma_sim, system_size);
   vector[system_size] log_k21_ = sample_vector_rng(0.0, sigma_sim, system_size);
@@ -120,8 +123,9 @@ transformed data {
 
   y_ = simulate_mean(t0, log_a0_, ts, 0, rel_tol, abs_tol, max_num_steps,
                      num_checkpoints,
+                     interpolation_polynomial,
                      solver_f, solver_b,
-                     log_kt_, log_e50_, log_k12_, log_k21_);
+                     log_kp_, log_ks_, log_e50_, log_k12_, log_k21_);
   
   for(i in 1:num_obs) {
     for(j in 1:system_size) {
@@ -131,7 +135,7 @@ transformed data {
   }
 
   if(adjoint_integrator) {
-    print("Using bdf_adjoint integrator.");
+    print("Using adjoint integrator.");
   } else {
     print("Using bdf integrator.");
   }
@@ -139,15 +143,19 @@ transformed data {
   print("absolute tolerance: ", abs_tol);
   print("maximum number of steps: ", max_num_steps);
   print("number of checkpoints: ", num_checkpoints);
+  print("interpolation polynomial: ", interpolation_polynomial);
   print("solver forward: ", solver_f);
   print("solver backward: ", solver_b);
   print("number of time points: ", num_obs);
   print("system size: ", system_size);
   print("time points: ", ts);
   print("y_: ", y_);
+  print("ODE states N: ", num_states);
+  print("ODE parameters varying M: ", 5*system_size + num_states);
 }
 parameters {
-  vector<multiplier=param_scale,offset=log_kt_>[system_size] log_kt;
+  vector<multiplier=param_scale,offset=log_kp_>[system_size] log_kp;
+  vector<multiplier=param_scale,offset=log_ks_>[system_size] log_ks;
   vector<multiplier=param_scale,offset=log_e50_>[system_size] log_e50;
   vector<multiplier=param_scale,offset=log_k12_>[system_size] log_k12;
   vector<multiplier=param_scale,offset=log_k21_>[system_size] log_k21;
@@ -157,12 +165,18 @@ parameters {
 transformed parameters {
 }
 model {
-  vector[num_states] mu[num_obs] = simulate_mean(t0, log_a0, ts, adjoint_integrator, rel_tol, abs_tol, max_num_steps,
-                                                 num_checkpoints,
-                                                 solver_f, solver_b,
-                                                 log_kt, log_e50, log_k12, log_k21);
+  vector[num_states] mu[num_obs];
+
+  profile("ode") {
+    mu = simulate_mean(t0, log_a0, ts, adjoint_integrator, rel_tol, abs_tol, max_num_steps,
+                       num_checkpoints,
+                       interpolation_polynomial,
+                       solver_f, solver_b,
+                       log_kp, log_ks, log_e50, log_k12, log_k21);
+  }
   
-  target += normal_lpdf(log_kt| 0.0, sigma_sim);
+  target += normal_lpdf(log_kp| 0.0, sigma_sim);
+  target += normal_lpdf(log_ks| 0.0, sigma_sim);
   target += normal_lpdf(log_e50| 0.0, sigma_sim);
   target += normal_lpdf(log_k12| 0.0, sigma_sim);
   target += normal_lpdf(log_k21| 0.0, sigma_sim);
@@ -175,13 +189,15 @@ model {
   }
 }
 generated quantities {
-  int rank_log_kt[system_size];
+  int rank_log_kp[system_size];
+  int rank_log_ks[system_size];
   int rank_log_e50[system_size];
   int rank_log_k12[system_size];
   int rank_log_k21[system_size];
   int rank_log_sigma_y[system_size];
   int rank_log_a0[system_size];
-  vector[system_size] bias_log_kt = log_kt - log_kt_;
+  vector[system_size] bias_log_kp = log_kp - log_kp_;
+  vector[system_size] bias_log_ks = log_ks - log_ks_;
   vector[system_size] bias_log_e50 = log_e50 - log_e50_;
   vector[system_size] bias_log_k12 = log_k12 - log_k12_;
   vector[system_size] bias_log_k21 = log_k21 - log_k21_;
@@ -189,7 +205,8 @@ generated quantities {
   vector[system_size] bias_log_a0 = log_a0 - log_a0_;
 
   for(i in 1:system_size) {
-    rank_log_kt[i] = (log_kt[i] > log_kt_[i] ? 1 : 0);
+    rank_log_kp[i] = (log_kp[i] > log_kp_[i] ? 1 : 0);
+    rank_log_ks[i] = (log_ks[i] > log_ks_[i] ? 1 : 0);
     rank_log_e50[i] = (log_e50[i] > log_e50_[i] ? 1 : 0);
     rank_log_k12[i] = (log_k12[i] > log_k12_[i] ? 1 : 0);
     rank_log_k21[i] = (log_k21[i] > log_k21_[i] ? 1 : 0);

From a382feec146f391ec942d09882116706397de7a5 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sebastian.weber@novartis.com>
Date: Thu, 11 Mar 2021 14:16:01 +0100
Subject: [PATCH 069/157] fix

---
 ode-adjoint/linked-mass-flow.R | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/ode-adjoint/linked-mass-flow.R b/ode-adjoint/linked-mass-flow.R
index bdd56ca309b..0ba9766198e 100644
--- a/ode-adjoint/linked-mass-flow.R
+++ b/ode-adjoint/linked-mass-flow.R
@@ -74,22 +74,22 @@ summarize_benchmark <- function(fit) {
 }
 
 
-bdf_fit <- run_benchmark(adjoint_integrator=0, num_iter=50)
-adjoint_fit <- run_benchmark(adjoint_integrator=1, num_iter=50)
+bdf_fit <- run_benchmark(adjoint_integrator=0, model=mod, num_iter=50)
+adjoint_fit <- run_benchmark(adjoint_integrator=1, model=mod, num_iter=50)
 ## vary solvers, e.g.
 ##adjoint_fit <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=1, num_iter=50)
 
 str(summarize_benchmark(bdf_fit))
 str(summarize_benchmark(adjoint_fit))
 
-Lbdf_fit <- run_benchmark(adjoint_integrator=0, system_size=5, model=mod, num_iter=50)
-Ladjoint_fit <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=2, system_size=5, interpolation_polynomial=1, model=mod, num_iter=50)
-Ladjoint_fit_1 <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=1, system_size=5, interpolation_polynomial=1, model=mod, num_iter=50)
+Lbdf_fit <- run_benchmark(adjoint_integrator=0, system_size=6, model=mod, num_iter=50)
+Ladjoint_fit <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=2, system_size=6, model=mod, num_iter=50)
 
 str(summarize_benchmark(Lbdf_fit))
 str(summarize_benchmark(Ladjoint_fit))
-str(summarize_benchmark(Ladjoint_fit_1))
 
+## there is no gain by adjoint method if there are not more parameters
+## than states
 Lbdf_fit_fixed <- run_benchmark(adjoint_integrator=0, system_size=5, model=mod_fixed, num_iter=50)
 Ladjoint_fit_fixed <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=2, system_size=5, model=mod_fixed, interpolation_polynomial=1, num_iter=50)
 

From d72dd8a7f85d402009ff994ce384e199b7d56773 Mon Sep 17 00:00:00 2001
From: wds15 <sebastian.weber@novartis.com>
Date: Fri, 12 Mar 2021 11:18:42 +0100
Subject: [PATCH 070/157] align notation with user guide

---
 .../math/rev/functor/cvodes_integrator_adjoint.hpp | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 45f047296ac..81f9670a747 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -292,9 +292,9 @@ class cvodes_integrator_adjoint_vari : public vari {
    */
   void rhs_adj_sens(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
     Eigen::Map<Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
-    Eigen::Map<Eigen::VectorXd> lambda(NV_DATA_S(yB), N_);
-    Eigen::Map<Eigen::VectorXd> lambda_dot(NV_DATA_S(yBdot), N_);
-    lambda_dot = Eigen::VectorXd::Zero(N_);
+    Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
+    Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(yBdot), N_);
+    mu_dot = Eigen::VectorXd::Zero(N_);
 
     const nested_rev_autodiff nested;
 
@@ -310,13 +310,13 @@ class cvodes_integrator_adjoint_vari : public vari {
                      "states", N_);
 
     for (size_t i = 0; i < f_y_t_vars.size(); ++i) {
-      f_y_t_vars(i).vi_->adj_ = -lambda(i);
+      f_y_t_vars(i).vi_->adj_ = -mu(i);
     }
 
     grad();
 
     for (size_t i = 0; i < y_vars.size(); ++i) {
-      lambda_dot(i) = y_vars(i).adj();
+      mu_dot(i) = y_vars(i).adj();
     }
   }
 
@@ -335,7 +335,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    */
   void quad_rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector qBdot) const {
     Eigen::VectorXd y_vec = Eigen::Map<Eigen::VectorXd>(NV_DATA_S(y), N_);
-    Eigen::Map<Eigen::VectorXd> lambda(NV_DATA_S(yB), N_);
+    Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(qBdot), args_vars_);
     mu_dot = Eigen::VectorXd::Zero(args_vars_);
 
@@ -363,7 +363,7 @@ class cvodes_integrator_adjoint_vari : public vari {
                      "states", N_);
 
     for (size_t i = 0; i < f_y_t_vars.size(); ++i) {
-      f_y_t_vars(i).vi_->adj_ = -lambda(i);
+      f_y_t_vars(i).vi_->adj_ = -mu(i);
     }
 
     grad();

From 217f0777530a7e0e6d6addcc5e1920cd54a7240c Mon Sep 17 00:00:00 2001
From: wds15 <sebastian.weber@novartis.com>
Date: Mon, 15 Mar 2021 09:43:41 +0100
Subject: [PATCH 071/157] export all tolerances for example

---
 ode-adjoint/linked-mass-flow-fixed.stan | 12 +++----
 ode-adjoint/linked-mass-flow.R          |  8 +++--
 ode-adjoint/linked-mass-flow.stan       | 43 ++++++++++++++++++-------
 3 files changed, 44 insertions(+), 19 deletions(-)

diff --git a/ode-adjoint/linked-mass-flow-fixed.stan b/ode-adjoint/linked-mass-flow-fixed.stan
index 16f9ca0cfac..d59a508750f 100644
--- a/ode-adjoint/linked-mass-flow-fixed.stan
+++ b/ode-adjoint/linked-mass-flow-fixed.stan
@@ -76,8 +76,8 @@ functions {
   
 }
 data {
-  real<lower=0> rel_tol;
-  real<lower=0> abs_tol;
+  real<lower=0> rel_tol_f;
+  real<lower=0> abs_tol_f;
   int<lower=0,upper=1> adjoint_integrator;
   int<lower=1> max_num_steps;
   int<lower=1> num_checkpoints;
@@ -109,7 +109,7 @@ transformed data {
     ts[i] = 1.5 * ts[i-1];
   }
 
-  y_ = simulate_mean(t0, log_a0_, ts, 0, rel_tol, abs_tol, max_num_steps,
+  y_ = simulate_mean(t0, log_a0_, ts, 0, rel_tol_f, abs_tol_f, max_num_steps,
                      num_checkpoints,
                      interpolation_polynomial,
                      solver_f, solver_b,
@@ -127,8 +127,8 @@ transformed data {
   } else {
     print("Using bdf integrator.");
   }
-  print("relative tolerance: ", rel_tol);
-  print("absolute tolerance: ", abs_tol);
+  print("relative tolerance: ", rel_tol_f);
+  print("absolute tolerance: ", abs_tol_f);
   print("maximum number of steps: ", max_num_steps);
   print("number of checkpoints: ", num_checkpoints);
   print("interpolation polynomial: ", interpolation_polynomial);
@@ -156,7 +156,7 @@ model {
   vector[num_states] mu[num_obs];
 
   profile("ode") {
-    mu = simulate_mean(t0, log_a0, ts, adjoint_integrator, rel_tol, abs_tol, max_num_steps,
+    mu = simulate_mean(t0, log_a0, ts, adjoint_integrator, rel_tol_f, abs_tol_f, max_num_steps,
                        num_checkpoints,
                        interpolation_polynomial,
                        solver_f, solver_b,
diff --git a/ode-adjoint/linked-mass-flow.R b/ode-adjoint/linked-mass-flow.R
index 0ba9766198e..b021eb43da8 100644
--- a/ode-adjoint/linked-mass-flow.R
+++ b/ode-adjoint/linked-mass-flow.R
@@ -19,8 +19,12 @@ stan_model_fixed_file <- "linked-mass-flow-fixed.stan"
 mod <- cmdstan_model(stan_model_file)
 mod_fixed <- cmdstan_model(stan_model_fixed_file)
 
-base_data <- list(rel_tol=1e-6,
-                  abs_tol=1e-4,
+base_data <- list(rel_tol_f=1e-8,
+                  abs_tol_f=1e-8,
+                  rel_tol_b=1e-6,
+                  abs_tol_b=1e-6,
+                  rel_tol_q=1e-5,
+                  abs_tol_q=1e-5,
                   adjoint_integrator=0,
                   max_num_steps=10000,
                   num_checkpoints=150,
diff --git a/ode-adjoint/linked-mass-flow.stan b/ode-adjoint/linked-mass-flow.stan
index 299ac025e6d..9c192e0fc09 100644
--- a/ode-adjoint/linked-mass-flow.stan
+++ b/ode-adjoint/linked-mass-flow.stan
@@ -33,7 +33,10 @@ functions {
   }
 
   vector[] simulate_mean(real t0, vector log_a0, real[] ts, int adjoint_integrator,
-                         real rel_tol, real abs_tol, int max_num_steps,
+                         real rel_tol_f, real abs_tol_f,
+                         real rel_tol_b, real abs_tol_b,
+                         real rel_tol_q, real abs_tol_q,
+                         int max_num_steps,
                          int num_checkpoints,
                          int interpolation_polynomial,
                          int solver_f, int solver_b,
@@ -63,9 +66,9 @@ functions {
 
     if(adjoint_integrator) {
       y = ode_adjoint_tol_ctl(linked_mass_flow, a0, t0, ts, 
-                              rel_tol, rep_vector(abs_tol/100.0, num_states), // forward
-                              rel_tol, rep_vector(abs_tol/10.0, num_states), // backward
-                              rel_tol, abs_tol, // quadrature
+                              rel_tol_f, rep_vector(abs_tol_f, num_states), // forward
+                              rel_tol_b, rep_vector(abs_tol_b, num_states), // backward
+                              rel_tol_q, abs_tol_q, // quadrature
                               max_num_steps,
                               num_checkpoints, // number of steps between checkpoints
                               interpolation_polynomial,  // polynomials
@@ -79,7 +82,9 @@ functions {
                               kp, ks, e50, k12, k21);
       */
     } else {
-      y = ode_bdf_tol(linked_mass_flow, a0, t0, ts, rel_tol, abs_tol, max_num_steps,
+      y = ode_bdf_tol(linked_mass_flow, a0, t0, ts,
+                      rel_tol_f, abs_tol_f,
+                      max_num_steps,
                       kp, ks, e50, k12, k21);
     }
 
@@ -88,8 +93,12 @@ functions {
   
 }
 data {
-  real<lower=0> rel_tol;
-  real<lower=0> abs_tol;
+  real<lower=0> rel_tol_f;
+  real<lower=0> abs_tol_f;
+  real<lower=0> rel_tol_b;
+  real<lower=0> abs_tol_b;
+  real<lower=0> rel_tol_q;
+  real<lower=0> abs_tol_q;
   int<lower=0,upper=1> adjoint_integrator;
   int<lower=1> max_num_steps;
   int<lower=1> num_checkpoints;
@@ -121,7 +130,11 @@ transformed data {
     ts[i] = 1.5 * ts[i-1];
   }
 
-  y_ = simulate_mean(t0, log_a0_, ts, 0, rel_tol, abs_tol, max_num_steps,
+  y_ = simulate_mean(t0, log_a0_, ts, 0,
+                     rel_tol_f, abs_tol_f,
+                     rel_tol_b, abs_tol_b,
+                     rel_tol_q, abs_tol_q,
+                     max_num_steps,
                      num_checkpoints,
                      interpolation_polynomial,
                      solver_f, solver_b,
@@ -139,8 +152,12 @@ transformed data {
   } else {
     print("Using bdf integrator.");
   }
-  print("relative tolerance: ", rel_tol);
-  print("absolute tolerance: ", abs_tol);
+  print("relative tolerance forward: ", rel_tol_f);
+  print("absolute tolerance forward: ", abs_tol_f);
+  print("relative tolerance backward: ", rel_tol_b);
+  print("absolute tolerance backward: ", abs_tol_b);
+  print("relative tolerance quadrature: ", rel_tol_q);
+  print("absolute tolerance quadrature: ", abs_tol_q);
   print("maximum number of steps: ", max_num_steps);
   print("number of checkpoints: ", num_checkpoints);
   print("interpolation polynomial: ", interpolation_polynomial);
@@ -168,7 +185,11 @@ model {
   vector[num_states] mu[num_obs];
 
   profile("ode") {
-    mu = simulate_mean(t0, log_a0, ts, adjoint_integrator, rel_tol, abs_tol, max_num_steps,
+    mu = simulate_mean(t0, log_a0, ts, adjoint_integrator,
+                       rel_tol_f, abs_tol_f,
+                       rel_tol_b, abs_tol_b,
+                       rel_tol_q, abs_tol_q,
+                       max_num_steps,
                        num_checkpoints,
                        interpolation_polynomial,
                        solver_f, solver_b,

From 5f0f2b7489aa2d9443337a91a8da83b182bec19f Mon Sep 17 00:00:00 2001
From: wds15 <sebastian.weber@novartis.com>
Date: Mon, 15 Mar 2021 10:41:47 +0100
Subject: [PATCH 072/157] make use of Eigen vector expressions and use coeff
 indexing

---
 ode-adjoint/linked-mass-flow.R                | 16 +++--
 ode-adjoint/linked-mass-flow.stan             |  4 +-
 .../rev/functor/cvodes_integrator_adjoint.hpp | 68 +++++++------------
 3 files changed, 35 insertions(+), 53 deletions(-)

diff --git a/ode-adjoint/linked-mass-flow.R b/ode-adjoint/linked-mass-flow.R
index b021eb43da8..eab37ff9b41 100644
--- a/ode-adjoint/linked-mass-flow.R
+++ b/ode-adjoint/linked-mass-flow.R
@@ -19,12 +19,14 @@ stan_model_fixed_file <- "linked-mass-flow-fixed.stan"
 mod <- cmdstan_model(stan_model_file)
 mod_fixed <- cmdstan_model(stan_model_fixed_file)
 
-base_data <- list(rel_tol_f=1e-8,
-                  abs_tol_f=1e-8,
-                  rel_tol_b=1e-6,
-                  abs_tol_b=1e-6,
-                  rel_tol_q=1e-5,
-                  abs_tol_q=1e-5,
+rel_tol_f <- 1E-8
+abs_tol_f <- 1E-8
+base_data <- list(rel_tol_f=rel_tol_f,
+                  abs_tol_f=abs_tol_f,
+                  rel_tol_b=rel_tol_f * 100,
+                  abs_tol_b=abs_tol_f * 100,
+                  rel_tol_q=rel_tol_f * 1000,
+                  abs_tol_q=abs_tol_f * 1000,
                   adjoint_integrator=0,
                   max_num_steps=10000,
                   num_checkpoints=150,
@@ -88,9 +90,11 @@ str(summarize_benchmark(adjoint_fit))
 
 Lbdf_fit <- run_benchmark(adjoint_integrator=0, system_size=6, model=mod, num_iter=50)
 Ladjoint_fit <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=2, system_size=6, model=mod, num_iter=50)
+Ladjoint_fit_2 <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=2, system_size=6, model=mod, num_iter=50)
 
 str(summarize_benchmark(Lbdf_fit))
 str(summarize_benchmark(Ladjoint_fit))
+str(summarize_benchmark(Ladjoint_fit_2))
 
 ## there is no gain by adjoint method if there are not more parameters
 ## than states
diff --git a/ode-adjoint/linked-mass-flow.stan b/ode-adjoint/linked-mass-flow.stan
index 9c192e0fc09..96090fca189 100644
--- a/ode-adjoint/linked-mass-flow.stan
+++ b/ode-adjoint/linked-mass-flow.stan
@@ -69,13 +69,13 @@ functions {
                               rel_tol_f, rep_vector(abs_tol_f, num_states), // forward
                               rel_tol_b, rep_vector(abs_tol_b, num_states), // backward
                               rel_tol_q, abs_tol_q, // quadrature
-                              max_num_steps,
+                              max_num_steps, 
                               num_checkpoints, // number of steps between checkpoints
                               interpolation_polynomial,  // polynomials
                               solver_f,  // bdf forward
                               solver_b,  // bdf backward
                               kp, ks, e50, k12, k21);
-      /* simplified interface
+      /* simplified interface 
       y = ode_adjoint_tol(linked_mass_flow, a0, t0, ts, 
                               rel_tol, abs_tol,
                               max_num_steps,
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 81f9670a747..955b3971ee7 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -267,14 +267,14 @@ class cvodes_integrator_adjoint_vari : public vari {
   inline void rhs(double t, const double y[], double dy_dt[]) const {
     const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
 
-    Eigen::VectorXd dy_dt_vec = apply(
+    const Eigen::VectorXd dy_dt_vec = apply(
         [&](auto&&... args) { return memory->f_(t, y_vec, msgs_, args...); },
         memory->value_of_args_tuple_);
 
     check_size_match("cvodes_integrator::rhs", "dy_dt", dy_dt_vec.size(),
                      "states", N_);
 
-    std::copy(dy_dt_vec.data(), dy_dt_vec.data() + dy_dt_vec.size(), dy_dt);
+    Eigen::Map<Eigen::VectorXd>(dy_dt, N_) = dy_dt_vec;
   }
 
   /*
@@ -298,9 +298,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     const nested_rev_autodiff nested;
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y_vars(y_vec.size());
-    for (size_t i = 0; i < y_vars.size(); ++i)
-      y_vars(i) = new vari(y_vec(i));
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y_vars(y_vec);
 
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars = apply(
         [&](auto&&... args) { return memory->f_(t, y_vars, msgs_, args...); },
@@ -309,15 +307,11 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_size_match("coupled_ode_system1", "dy_dt", f_y_t_vars.size(),
                      "states", N_);
 
-    for (size_t i = 0; i < f_y_t_vars.size(); ++i) {
-      f_y_t_vars(i).vi_->adj_ = -mu(i);
-    }
+    f_y_t_vars.adj() = -mu;
 
     grad();
 
-    for (size_t i = 0; i < y_vars.size(); ++i) {
-      mu_dot(i) = y_vars(i).adj();
-    }
+    mu_dot = y_vars.adj();
   }
 
   /*
@@ -334,22 +328,13 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[out] qBdot evaluation of adjoint ODE quadrature RHS
    */
   void quad_rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector qBdot) const {
-    Eigen::VectorXd y_vec = Eigen::Map<Eigen::VectorXd>(NV_DATA_S(y), N_);
+    const Eigen::VectorXd y_vec = Eigen::Map<Eigen::VectorXd>(NV_DATA_S(y), N_);
     Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(qBdot), args_vars_);
     mu_dot = Eigen::VectorXd::Zero(args_vars_);
 
     nested_rev_autodiff nested;
 
-    /*
-    auto local_args_tuple = apply(
-        [&](auto&&... args) {
-          return std::tuple<decltype(deep_copy_vars(args))...>(
-              deep_copy_vars(args)...);
-        },
-        memory->args_tuple_);
-    */
-
     // The vars here do not live on the nested stack so must be zero'd
     // separately
     apply([&](auto&&... args) { zero_adjoints(args...); },
@@ -362,9 +347,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_size_match("coupled_ode_system2", "dy_dt", f_y_t_vars.size(),
                      "states", N_);
 
-    for (size_t i = 0; i < f_y_t_vars.size(); ++i) {
-      f_y_t_vars(i).vi_->adj_ = -mu(i);
-    }
+    f_y_t_vars.adj() = -mu;
 
     grad();
 
@@ -380,29 +363,24 @@ class cvodes_integrator_adjoint_vari : public vari {
     Eigen::Map<Eigen::MatrixXd> Jfy(SM_DATA_D(J), N_, N_);
     Eigen::Map<const Eigen::VectorXd> x(NV_DATA_S(y), N_);
 
-    auto f_wrapped = [&](const Eigen::Matrix<var, Eigen::Dynamic, 1>& y) {
-      return apply(
-          [&](auto&&... args) { return memory->f_(t, y, msgs_, args...); },
-          memory->value_of_args_tuple_);
-    };
-
     nested_rev_autodiff nested;
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> x_var(x);
-    Eigen::Matrix<var, Eigen::Dynamic, 1> fx_var = f_wrapped(x_var);
-    grad(fx_var(0).vi_);
-    for (size_t j = 0; j < N_; ++j) {
-      // SM_ELEMENT_D(Jfy, 0, j) = x_var(j).adj();
-      Jfy(0, j) = x_var(j).adj();
-    }
-    for (int i = 1; i < fx_var.size(); ++i) {
+    const Eigen::Matrix<var, Eigen::Dynamic, 1> y_var(x);
+    Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var = apply(
+        [&](auto&&... args) { return memory->f_(t, y_var, msgs_, args...); },
+        memory->value_of_args_tuple_);
+    
+    check_size_match("coupled_ode_system2", "dy_dt", fy_var.size(),
+                     "states", N_);
+    
+    grad(fy_var.coeffRef(0).vi_);
+    Jfy.col(0) = y_var.adj();
+    for (int i = 1; i < fy_var.size(); ++i) {
       nested.set_zero_all_adjoints();
-      grad(fx_var(i).vi_);
-      for (size_t j = 0; j < N_; ++j) {
-        // SM_ELEMENT_D(Jfy, i, j) = x_var(j).adj();
-        Jfy(i, j) = x_var(j).adj();
-      }
+      grad(fy_var.coeffRef(i).vi_);
+      Jfy.col(i) = y_var.adj();
     }
+    Jfy.transposeInPlace();
   }
 
   /*
@@ -838,12 +816,12 @@ class cvodes_integrator_adjoint_vari : public vari {
       // the adjoints we wanted
       // These are the dlog_density / d(initial_conditions[s]) adjoints
       for (size_t s = 0; s < y0_vars_; s++) {
-        y0_varis_[s]->adj_ += state_sens(s);
+        y0_varis_[s]->adj_ += state_sens.coeff(s);
       }
 
       // These are the dlog_density / d(parameters[s]) adjoints
       for (size_t s = 0; s < args_vars_; s++) {
-        args_varis_[s]->adj_ += quad(s);
+        args_varis_[s]->adj_ += quad.coeff(s);
       }
 
     } catch (const std::exception& e) {

From 0e90995306447d655533c780c456aca413b7092c Mon Sep 17 00:00:00 2001
From: wds15 <sebastian.weber@novartis.com>
Date: Mon, 15 Mar 2021 11:15:36 +0100
Subject: [PATCH 073/157] put apply call in separate function to clean code

---
 ode-adjoint/linked-mass-flow.R                |  2 -
 ode-adjoint/linked-mass-flow.stan             |  4 +-
 .../rev/functor/cvodes_integrator_adjoint.hpp | 56 +++++++++++--------
 3 files changed, 36 insertions(+), 26 deletions(-)

diff --git a/ode-adjoint/linked-mass-flow.R b/ode-adjoint/linked-mass-flow.R
index eab37ff9b41..e24cd5a61f5 100644
--- a/ode-adjoint/linked-mass-flow.R
+++ b/ode-adjoint/linked-mass-flow.R
@@ -90,11 +90,9 @@ str(summarize_benchmark(adjoint_fit))
 
 Lbdf_fit <- run_benchmark(adjoint_integrator=0, system_size=6, model=mod, num_iter=50)
 Ladjoint_fit <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=2, system_size=6, model=mod, num_iter=50)
-Ladjoint_fit_2 <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=2, system_size=6, model=mod, num_iter=50)
 
 str(summarize_benchmark(Lbdf_fit))
 str(summarize_benchmark(Ladjoint_fit))
-str(summarize_benchmark(Ladjoint_fit_2))
 
 ## there is no gain by adjoint method if there are not more parameters
 ## than states
diff --git a/ode-adjoint/linked-mass-flow.stan b/ode-adjoint/linked-mass-flow.stan
index 96090fca189..f2795451593 100644
--- a/ode-adjoint/linked-mass-flow.stan
+++ b/ode-adjoint/linked-mass-flow.stan
@@ -65,11 +65,11 @@ functions {
     }
 
     if(adjoint_integrator) {
-      y = ode_adjoint_tol_ctl(linked_mass_flow, a0, t0, ts, 
+      y = ode_adjoint_tol_ctl(linked_mass_flow, a0, t0, ts,
                               rel_tol_f, rep_vector(abs_tol_f, num_states), // forward
                               rel_tol_b, rep_vector(abs_tol_b, num_states), // backward
                               rel_tol_q, abs_tol_q, // quadrature
-                              max_num_steps, 
+                              max_num_steps,
                               num_checkpoints, // number of steps between checkpoints
                               interpolation_polynomial,  // polynomials
                               solver_f,  // bdf forward
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 955b3971ee7..6cde7a04df2 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -146,6 +146,16 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
     }
   }
 
+  template <typename yT, typename... ArgsT>
+  constexpr
+  auto rhs(double t, const yT& y, std::ostream* msgs, const std::tuple<ArgsT...>& args_tuple) const {
+    return apply(
+        [&](auto&&... args) {
+          return f_(t, y, msgs, args...);
+        },
+        args_tuple);
+  }
+  
   friend class cvodes_integrator_adjoint_vari<F, T_y0, T_t0, T_ts, T_Args...>;
 };
 
@@ -267,9 +277,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   inline void rhs(double t, const double y[], double dy_dt[]) const {
     const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
 
-    const Eigen::VectorXd dy_dt_vec = apply(
-        [&](auto&&... args) { return memory->f_(t, y_vec, msgs_, args...); },
-        memory->value_of_args_tuple_);
+    const Eigen::VectorXd dy_dt_vec = memory->rhs(t, y_vec, msgs_, memory->value_of_args_tuple_);
 
     check_size_match("cvodes_integrator::rhs", "dy_dt", dy_dt_vec.size(),
                      "states", N_);
@@ -300,9 +308,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     Eigen::Matrix<var, Eigen::Dynamic, 1> y_vars(y_vec);
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars = apply(
-        [&](auto&&... args) { return memory->f_(t, y_vars, msgs_, args...); },
-        memory->value_of_args_tuple_);
+    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars = memory->rhs(t, y_vars, msgs_, memory->value_of_args_tuple_);
 
     check_size_match("coupled_ode_system1", "dy_dt", f_y_t_vars.size(),
                      "states", N_);
@@ -340,9 +346,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     apply([&](auto&&... args) { zero_adjoints(args...); },
           memory->local_args_tuple_);
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars = apply(
-        [&](auto&&... args) { return memory->f_(t, y_vec, msgs_, args...); },
-        memory->local_args_tuple_);
+    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars = memory->rhs(t, y_vec, msgs_, memory->local_args_tuple_);
 
     check_size_match("coupled_ode_system2", "dy_dt", f_y_t_vars.size(),
                      "states", N_);
@@ -366,10 +370,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     nested_rev_autodiff nested;
 
     const Eigen::Matrix<var, Eigen::Dynamic, 1> y_var(x);
-    Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var = apply(
-        [&](auto&&... args) { return memory->f_(t, y_var, msgs_, args...); },
-        memory->value_of_args_tuple_);
-    
+    Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var = memory->rhs(t, y_var, msgs_, memory->value_of_args_tuple_);
+
     check_size_match("coupled_ode_system2", "dy_dt", fy_var.size(),
                      "states", N_);
     
@@ -689,14 +691,19 @@ class cvodes_integrator_adjoint_vari : public vari {
         }
 
         if (ts_vars_ > 0 && i >= 0) {
-          ts_varis_[i]->adj_ += apply(
-              [&](auto&&... args) {
-                double adj = step_sens.dot(
-                    memory->f_(t_init, memory->y_[i], msgs_, args...));
-                // std::cout << "adj: " << adj << ", i: " << i << std::endl;
-                return adj;
-              },
-              memory->value_of_args_tuple_);
+          ts_varis_[i]->adj_ += step_sens.dot(
+              memory->rhs(t_init, memory->y_[i], msgs_, memory->value_of_args_tuple_)
+                                              );
+          /*
+            apply(
+            [&](auto&&... args) {
+            double adj = step_sens.dot(
+            memory->f_(t_init, memory->y_[i], msgs_, args...));
+            // std::cout << "adj: " << adj << ", i: " << i << std::endl;
+            return adj;
+            },
+            memory->value_of_args_tuple_);
+          */
         }
 
         double t_final = value_of((i > 0) ? memory->ts_[i - 1] : memory->t0_);
@@ -796,11 +803,16 @@ class cvodes_integrator_adjoint_vari : public vari {
 
       if (t0_vars_ > 0) {
         Eigen::VectorXd y0d = value_of(memory->y0_);
-        t0_varis_[0]->adj_ += apply(
+        t0_varis_[0]->adj_ += -state_sens.dot(
+              memory->rhs(t_init, y0d, msgs_, memory->value_of_args_tuple_)
+                                              );
+        /*
+          apply(
             [&](auto&&... args) {
               return -state_sens.dot(memory->f_(t_init, y0d, msgs_, args...));
             },
             memory->value_of_args_tuple_);
+        */
       }
 
       // do we need this a 2nd time? Don't think so.

From f4bde7e0b9516fa9f9fd5a5f7025a3d05ad05a8f Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Mon, 15 Mar 2021 10:22:12 +0000
Subject: [PATCH 074/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 39 +++++++++----------
 1 file changed, 19 insertions(+), 20 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 6cde7a04df2..5b7338f5f77 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -147,15 +147,12 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   }
 
   template <typename yT, typename... ArgsT>
-  constexpr
-  auto rhs(double t, const yT& y, std::ostream* msgs, const std::tuple<ArgsT...>& args_tuple) const {
-    return apply(
-        [&](auto&&... args) {
-          return f_(t, y, msgs, args...);
-        },
-        args_tuple);
+  constexpr auto rhs(double t, const yT& y, std::ostream* msgs,
+                     const std::tuple<ArgsT...>& args_tuple) const {
+    return apply([&](auto&&... args) { return f_(t, y, msgs, args...); },
+                 args_tuple);
   }
-  
+
   friend class cvodes_integrator_adjoint_vari<F, T_y0, T_t0, T_ts, T_Args...>;
 };
 
@@ -277,7 +274,8 @@ class cvodes_integrator_adjoint_vari : public vari {
   inline void rhs(double t, const double y[], double dy_dt[]) const {
     const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
 
-    const Eigen::VectorXd dy_dt_vec = memory->rhs(t, y_vec, msgs_, memory->value_of_args_tuple_);
+    const Eigen::VectorXd dy_dt_vec
+        = memory->rhs(t, y_vec, msgs_, memory->value_of_args_tuple_);
 
     check_size_match("cvodes_integrator::rhs", "dy_dt", dy_dt_vec.size(),
                      "states", N_);
@@ -308,7 +306,8 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     Eigen::Matrix<var, Eigen::Dynamic, 1> y_vars(y_vec);
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars = memory->rhs(t, y_vars, msgs_, memory->value_of_args_tuple_);
+    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
+        = memory->rhs(t, y_vars, msgs_, memory->value_of_args_tuple_);
 
     check_size_match("coupled_ode_system1", "dy_dt", f_y_t_vars.size(),
                      "states", N_);
@@ -346,7 +345,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     apply([&](auto&&... args) { zero_adjoints(args...); },
           memory->local_args_tuple_);
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars = memory->rhs(t, y_vec, msgs_, memory->local_args_tuple_);
+    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
+        = memory->rhs(t, y_vec, msgs_, memory->local_args_tuple_);
 
     check_size_match("coupled_ode_system2", "dy_dt", f_y_t_vars.size(),
                      "states", N_);
@@ -370,11 +370,12 @@ class cvodes_integrator_adjoint_vari : public vari {
     nested_rev_autodiff nested;
 
     const Eigen::Matrix<var, Eigen::Dynamic, 1> y_var(x);
-    Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var = memory->rhs(t, y_var, msgs_, memory->value_of_args_tuple_);
+    Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var
+        = memory->rhs(t, y_var, msgs_, memory->value_of_args_tuple_);
+
+    check_size_match("coupled_ode_system2", "dy_dt", fy_var.size(), "states",
+                     N_);
 
-    check_size_match("coupled_ode_system2", "dy_dt", fy_var.size(),
-                     "states", N_);
-    
     grad(fy_var.coeffRef(0).vi_);
     Jfy.col(0) = y_var.adj();
     for (int i = 1; i < fy_var.size(); ++i) {
@@ -691,9 +692,8 @@ class cvodes_integrator_adjoint_vari : public vari {
         }
 
         if (ts_vars_ > 0 && i >= 0) {
-          ts_varis_[i]->adj_ += step_sens.dot(
-              memory->rhs(t_init, memory->y_[i], msgs_, memory->value_of_args_tuple_)
-                                              );
+          ts_varis_[i]->adj_ += step_sens.dot(memory->rhs(
+              t_init, memory->y_[i], msgs_, memory->value_of_args_tuple_));
           /*
             apply(
             [&](auto&&... args) {
@@ -804,8 +804,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       if (t0_vars_ > 0) {
         Eigen::VectorXd y0d = value_of(memory->y0_);
         t0_varis_[0]->adj_ += -state_sens.dot(
-              memory->rhs(t_init, y0d, msgs_, memory->value_of_args_tuple_)
-                                              );
+            memory->rhs(t_init, y0d, msgs_, memory->value_of_args_tuple_));
         /*
           apply(
             [&](auto&&... args) {

From 338de15223b3e51a81147e3c60144aad1d8896ae Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 15 Apr 2021 21:53:24 +0200
Subject: [PATCH 075/157] backport adjoint code and integrate into testing
 framework in a first step

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 546 +++++++++---------
 stan/math/rev/functor/ode_adjoint.hpp         |  76 +--
 .../math/rev/functor/cos_ode_typed_test.cpp   |   3 +-
 .../rev/functor/fho_ode_typed_ts_test.cpp     |   2 +-
 .../rev/functor/lorenz_ode_typed_fd_test.cpp  |   3 +-
 .../math/rev/functor/ode_test_functors.hpp    |  28 +
 .../rev/functor/sho_ode_typed_error_test.cpp  |   2 +-
 .../math/rev/functor/sho_ode_typed_test.cpp   |  13 +-
 8 files changed, 369 insertions(+), 304 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 5b7338f5f77..49206e47467 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -76,8 +76,8 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
   using T_y0_t0 = return_type_t<T_y0, T_t0>;
 
   const size_t N_;
-  Eigen::VectorXd abs_tol_f_;
-  Eigen::VectorXd abs_tol_b_;
+  Eigen::VectorXd absolute_tolerance_forward_;
+  Eigen::VectorXd absolute_tolerance_backward_;
   const int lmm_f_;
   const F f_;
   const Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1> y0_;
@@ -91,22 +91,30 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
       value_of_args_tuple_;
   std::vector<Eigen::VectorXd> y_;
   void* cvodes_mem_;
-  Eigen::VectorXd state;
+  Eigen::VectorXd state_;
+  Eigen::VectorXd state_sens_;
+  Eigen::VectorXd quad_;
   N_Vector nv_state_;
-  N_Vector nv_abs_tol_f_;
-  N_Vector nv_abs_tol_b_;
+  N_Vector nv_state_sens_;
+  N_Vector nv_quad_;
+  N_Vector nv_absolute_tolerance_forward_;
+  N_Vector nv_absolute_tolerance_backward_;
   SUNMatrix A_f_;
   SUNLinearSolver LS_f_;
+  SUNMatrix A_b_;
+  SUNLinearSolver LS_b_;
+  int index_b_;
+  bool backward_is_initialized_;
 
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
-  cvodes_integrator_adjoint_memory(Eigen::VectorXd abs_tol_f,
-                                   Eigen::VectorXd abs_tol_b, int lmm_f,
+  cvodes_integrator_adjoint_memory(Eigen::VectorXd absolute_tolerance_forward,
+                                   Eigen::VectorXd absolute_tolerance_backward, int lmm_f,
                                    const F& f, const T_y0& y0, const T_t0& t0,
                                    const std::vector<T_ts>& ts,
                                    const T_Args&... args)
       : N_(y0.size()),
-        abs_tol_f_(abs_tol_f),
-        abs_tol_b_(abs_tol_b),
+        absolute_tolerance_forward_(absolute_tolerance_forward),
+        absolute_tolerance_backward_(absolute_tolerance_backward),
         lmm_f_(lmm_f),
         f_(f),
         y0_(y0),
@@ -117,15 +125,24 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
         value_of_args_tuple_(value_of(args)...),
         y_(ts_.size()),
         cvodes_mem_(nullptr),
-        state(value_of(y0)) {
+        state_(value_of(y0)),
+        state_sens_(Eigen::VectorXd::Zero(N_)),
+        quad_(Eigen::VectorXd::Zero(count_vars(args...))),
+        backward_is_initialized_(false)
+  {
     if (N_ > 0) {
-      nv_state_ = N_VMake_Serial(N_, state.data());
-      nv_abs_tol_f_ = N_VMake_Serial(N_, &abs_tol_f_(0));
-      nv_abs_tol_b_ = N_VMake_Serial(N_, &abs_tol_b_(0));
+      nv_state_ = N_VMake_Serial(N_, state_.data());
+      nv_state_sens_ = N_VMake_Serial(N_, state_sens_.data());
+      nv_quad_ = N_VMake_Serial(quad_.size(), quad_.data());
+      nv_absolute_tolerance_forward_ = N_VMake_Serial(N_, absolute_tolerance_forward_.data());
+      nv_absolute_tolerance_backward_ = N_VMake_Serial(N_, absolute_tolerance_backward_.data());
       A_f_ = SUNDenseMatrix(N_, N_);
       LS_f_ = SUNDenseLinearSolver(nv_state_, A_f_);
+      A_b_ = SUNDenseMatrix(N_, N_);
+      LS_b_ = SUNDenseLinearSolver(nv_state_sens_, A_b_);
 
       cvodes_mem_ = CVodeCreate(lmm_f_);
+
       if (cvodes_mem_ == nullptr) {
         throw std::runtime_error("CVodeCreate failed to allocate memory");
       }
@@ -136,9 +153,14 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
     if (N_ > 0) {
       SUNLinSolFree(LS_f_);
       SUNMatDestroy(A_f_);
+      SUNLinSolFree(LS_b_);
+      SUNMatDestroy(A_b_);
 
       N_VDestroy_Serial(nv_state_);
-      N_VDestroy_Serial(nv_abs_tol_f_);
+      N_VDestroy_Serial(nv_state_sens_);
+      N_VDestroy_Serial(nv_quad_);
+      N_VDestroy_Serial(nv_absolute_tolerance_forward_);
+      N_VDestroy_Serial(nv_absolute_tolerance_backward_);
 
       if (cvodes_mem_) {
         CVodeFree(&cvodes_mem_);
@@ -175,18 +197,19 @@ class cvodes_integrator_adjoint_vari : public vari {
 
   const size_t N_;
   bool returned_;
+  std::size_t chain_count_;
   std::ostream* msgs_;
-  double rel_tol_f_;
-  Eigen::VectorXd abs_tol_f_;
-  double rel_tol_b_;
-  Eigen::VectorXd abs_tol_b_;
-  double rel_tol_q_;
-  double abs_tol_q_;
+  double relative_tolerance_forward_;
+  Eigen::VectorXd absolute_tolerance_forward_;
+  double relative_tolerance_backward_;
+  Eigen::VectorXd absolute_tolerance_backward_;
+  double relative_tolerance_quadrature_;
+  double absolute_tolerance_quadrature_;
   long int max_num_steps_;
-  long int num_checkpoints_;
+  long int num_steps_between_checkpoints_;
   int interpolation_polynomial_;
-  int solver_f_;
-  int solver_b_;
+  int solver_forward_;
+  int solver_backward_;
   int lmm_f_;
   int lmm_b_;
 
@@ -338,7 +361,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(qBdot), args_vars_);
     mu_dot = Eigen::VectorXd::Zero(args_vars_);
 
-    nested_rev_autodiff nested;
+    const nested_rev_autodiff nested;
 
     // The vars here do not live on the nested stack so must be zero'd
     // separately
@@ -358,7 +381,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     apply([&](auto&&... args) { accumulate_adjoints(mu_dot.data(), args...); },
           memory->local_args_tuple_);
   }
-
+  
   /**
    * Calculates the jacobian of the ODE RHS wrt to its states y at the
    * given time-point t and state y.
@@ -406,7 +429,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
  public:
   /**
-   * Construct cvodes_integrator object
+   * Construct cvodes_integrator object. Note: All arguments must be stored as copies if in doubt. The reason is that the references can go out of scope, since the work done from the integrator is in the chain method.
    *
    * @param function_name Calling function name (for printing debugging
    * messages)
@@ -415,19 +438,19 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param t0 Initial time
    * @param ts Times at which to solve the ODE at. All values must be sorted and
    *   not less than t0.
-   * @param rel_tol_f Relative tolerance for forward problem passed to CVODES
-   * @param abs_tol_f Absolute tolerance for forward problem passed to CVODES
-   * @param rel_tol_b Relative tolerance for backward problem passed to CVODES
-   * @param abs_tol_b Absolute tolerance for backward problem passed to CVODES
-   * @param rel_tol_q Relative tolerance for quadrature problem passed to CVODES
-   * @param abs_tol_q Absolute tolerance for quadrature problem passed to CVODES
+   * @param relative_tolerance_forward Relative tolerance for forward problem passed to CVODES
+   * @param absolute_tolerance_forward Absolute tolerance for forward problem passed to CVODES
+   * @param relative_tolerance_backward Relative tolerance for backward problem passed to CVODES
+   * @param absolute_tolerance_backward Absolute tolerance for backward problem passed to CVODES
+   * @param relative_tolerance_quadrature Relative tolerance for quadrature problem passed to CVODES
+   * @param absolute_tolerance_quadrature Absolute tolerance for quadrature problem passed to CVODES
    * @param max_num_steps Upper limit on the number of integration steps to
    *   take between each output (error if exceeded)
-   * @param num_checkpoints Number of integrator steps after which a checkpoint
+   * @param num_steps_between_checkpoints Number of integrator steps after which a checkpoint
    * is stored for the backward pass
    * @param interpolation_polynomial type of polynomial used for interpolation
-   * @param solver_f solver used for forward pass
-   * @param solver_b solver used for backward pass
+   * @param solver_forward solver used for forward pass
+   * @param solver_backward solver used for backward pass
    *   take between each output (error if exceeded)
    * @param[in, out] msgs the print stream for warning messages
    * @param args Extra arguments passed unmodified through to ODE right hand
@@ -439,29 +462,30 @@ class cvodes_integrator_adjoint_vari : public vari {
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
   cvodes_integrator_adjoint_vari(
       const char* function_name, const F& f, const T_y0& y0, const T_t0& t0,
-      const std::vector<T_ts>& ts, double rel_tol_f, Eigen::VectorXd abs_tol_f,
-      double rel_tol_b, Eigen::VectorXd abs_tol_b, double rel_tol_q,
-      double abs_tol_q, long int max_num_steps, long int num_checkpoints,
-      int interpolation_polynomial, int solver_f, int solver_b,
+      const std::vector<T_ts>& ts, double relative_tolerance_forward, Eigen::VectorXd absolute_tolerance_forward,
+      double relative_tolerance_backward, Eigen::VectorXd absolute_tolerance_backward, double relative_tolerance_quadrature,
+      double absolute_tolerance_quadrature, long int max_num_steps, long int num_steps_between_checkpoints,
+      int interpolation_polynomial, int solver_forward, int solver_backward,
       std::ostream* msgs, const T_Args&... args)
       : function_name_(function_name),
         vari(NOT_A_NUMBER),
         N_(y0.size()),
         returned_(false),
+        chain_count_(0),
         memory(NULL),
-        rel_tol_f_(rel_tol_f),
-        abs_tol_f_(abs_tol_f),
-        rel_tol_b_(rel_tol_b),
-        abs_tol_b_(abs_tol_b),
-        rel_tol_q_(rel_tol_q),
-        abs_tol_q_(abs_tol_q),
+        relative_tolerance_forward_(relative_tolerance_forward),
+        absolute_tolerance_forward_(absolute_tolerance_forward),
+        relative_tolerance_backward_(relative_tolerance_backward),
+        absolute_tolerance_backward_(absolute_tolerance_backward),
+        relative_tolerance_quadrature_(relative_tolerance_quadrature),
+        absolute_tolerance_quadrature_(absolute_tolerance_quadrature),
         max_num_steps_(max_num_steps),
-        num_checkpoints_(num_checkpoints),
+        num_steps_between_checkpoints_(num_steps_between_checkpoints),
         interpolation_polynomial_(interpolation_polynomial),
-        solver_f_(solver_f),
-        solver_b_(solver_b),
-        lmm_f_(solver_f_ % 2 ? CV_ADAMS : CV_BDF),
-        lmm_b_(solver_b_ % 2 ? CV_ADAMS : CV_BDF),
+        solver_forward_(solver_forward),
+        solver_backward_(solver_backward),
+        lmm_f_(solver_forward_ % 2 ? CV_ADAMS : CV_BDF),
+        lmm_b_(solver_backward_ % 2 ? CV_ADAMS : CV_BDF),
         msgs_(msgs),
 
         t0_vars_(count_vars(t0)),
@@ -476,11 +500,11 @@ class cvodes_integrator_adjoint_vari : public vari {
             ChainableStack::instance_->memalloc_.alloc_array<vari*>(y0_vars_)),
         args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             args_vars_)) {
-    const char* fun = "cvodes_integrator::integrate";
+    const char* fun = "cvodes_integrator_adjoint";
 
     memory
         = new cvodes_integrator_adjoint_memory<F, T_y0, T_t0, T_ts, T_Args...>(
-            abs_tol_f_, abs_tol_b_, lmm_f_, f, y0, t0, ts, args...);
+            absolute_tolerance_forward_, absolute_tolerance_backward_, lmm_f_, f, y0, t0, ts, args...);
 
     save_varis(t0_varis_, t0);
     save_varis(ts_varis_, ts);
@@ -504,22 +528,31 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_nonzero_size(fun, "initial state", y0);
     check_sorted(fun, "times", ts);
     check_less(fun, "initial time", t0, ts[0]);
-    check_positive_finite(fun, "rel_tol_f", rel_tol_f_);
-    check_positive_finite(fun, "abs_tol_f", abs_tol_f_);
-    check_size_match(fun, "abs_tol_f", abs_tol_f_.size(), "states", N_);
-    check_positive_finite(fun, "rel_tol_b", rel_tol_b_);
-    check_positive_finite(fun, "abs_tol_b", abs_tol_b_);
-    check_size_match(fun, "abs_tol_b", abs_tol_b_.size(), "states", N_);
-    check_positive_finite(fun, "rel_tol_q", rel_tol_q_);
-    check_positive_finite(fun, "abs_tol_q", abs_tol_q_);
+    // TODO (SW): make names verbose
+    check_positive_finite(fun, "relative_tolerance_f", relative_tolerance_forward_);
+    check_positive_finite(fun, "absolute_tolerance_f", absolute_tolerance_forward_);
+    check_size_match(fun, "absolute_tolerance_forward", absolute_tolerance_forward_.size(), "states", N_);
+    check_positive_finite(fun, "relative_tolerance_backward", relative_tolerance_backward_);
+    check_positive_finite(fun, "absolute_tolerance_backward", absolute_tolerance_backward_);
+    check_size_match(fun, "absolute_tolerance_backward", absolute_tolerance_backward_.size(), "states", N_);
+    check_positive_finite(fun, "relative_tolerance_quadrature", relative_tolerance_quadrature_);
+    check_positive_finite(fun, "absolute_tolerance_quadrature", absolute_tolerance_quadrature_);
     check_positive(fun, "max_num_steps", max_num_steps_);
-    check_positive(fun, "num_checkpoints", num_checkpoints_);
+    // TODO (SW): rename according to design
+    check_positive(fun, "num_steps_between_checkpoints", num_steps_between_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
     check_range(fun, "interpolation_polynomial", 2, interpolation_polynomial_);
     // 1=Adams, 2=BDF
-    check_range(fun, "solver_f", 2, solver_f_);
-    check_range(fun, "solver_b", 2, solver_b_);
+    check_range(fun, "solver_forward", 2, solver_forward_);
+    check_range(fun, "solver_backward", 2, solver_backward_);
 
+    /*
+    std::cout << "ts = ";
+    for(std::size_t i=0; i != ts.size(); i++)
+      std::cout << value_of(ts[i]) << ", ";
+    std::cout << std::endl;
+    */
+    
     /*
     std::cout << "relative_tolerance = " << relative_tolerance << std::endl;
     std::cout << "absolute_tolerance = " << absolute_tolerance << std::endl;
@@ -528,25 +561,13 @@ class cvodes_integrator_adjoint_vari : public vari {
     std::endl; std::cout << "max_num_steps = " << max_num_steps << std::endl;
     std::cout << "steps_checkpoint = " << steps_checkpoint << std::endl;
     */
-  }
 
-  ~cvodes_integrator_adjoint_vari() {}
-
-  /**
-   * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
-   * times, { t1, t2, t3, ... } using the stiff backward differentiation formula
-   * (BDF) solver in CVODES.
-   *
-   * @return std::vector of Eigen::Matrix of the states of the ODE, one for each
-   *   solution time (excluding the initial state)
-   */
-  std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> operator()() {
-    const double t0_dbl = value_of(memory->t0_);
-    const std::vector<double> ts_dbl = value_of(memory->ts_);
 
+    // initializes CVODES solver
+    
     check_flag_sundials(
         CVodeInit(memory->cvodes_mem_, &cvodes_integrator_adjoint_vari::cv_rhs,
-                  t0_dbl, memory->nv_state_),
+                  value_of(memory->t0_), memory->nv_state_),
         "CVodeInit");
 
     // Assign pointer to this as user data
@@ -554,13 +575,13 @@ class cvodes_integrator_adjoint_vari : public vari {
         CVodeSetUserData(memory->cvodes_mem_, reinterpret_cast<void*>(this)),
         "CVodeSetUserData");
 
-    cvodes_set_options(memory->cvodes_mem_, rel_tol_f_, abs_tol_f_(0),
+    cvodes_set_options(memory->cvodes_mem_, relative_tolerance_forward_, absolute_tolerance_forward_(0),
                        max_num_steps_);
 
-    check_flag_sundials(CVodeSVtolerances(memory->cvodes_mem_, rel_tol_f_,
-                                          memory->nv_abs_tol_f_),
+    check_flag_sundials(CVodeSVtolerances(memory->cvodes_mem_, relative_tolerance_forward_,
+                                          memory->nv_absolute_tolerance_forward_),
                         "CVodeSVtolerances");
-
+    
     // for the stiff solvers we need to reserve additional memory
     // and provide a Jacobian function call. new API since 3.0.0:
     // create matrix object and linear solver object; resource
@@ -574,16 +595,34 @@ class cvodes_integrator_adjoint_vari : public vari {
                       &cvodes_integrator_adjoint_vari::cv_jacobian_states),
         "CVodeSetJacFn");
 
-    // initialize forward sensitivity system of CVODES as needed
+    // initialize backward sensitivity system of CVODES as needed
     if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
-      check_flag_sundials(CVodeAdjInit(memory->cvodes_mem_, num_checkpoints_,
+      check_flag_sundials(CVodeAdjInit(memory->cvodes_mem_, num_steps_between_checkpoints_,
                                        interpolation_polynomial_),
                           "CVodeAdjInit");
     }
 
+  }
+
+  ~cvodes_integrator_adjoint_vari() {}
+
+  /**
+   * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
+   * times, { t1, t2, t3, ... } using the stiff backward differentiation formula
+   * (BDF) solver in CVODES.
+   *
+   * @return std::vector of Eigen::Matrix of the states of the ODE, one for each
+   *   solution time (excluding the initial state)
+   */
+  std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> operator()() {
+    //std::cout << "forward integrate..." << std::endl;
+    const double t0_dbl = value_of(memory->t0_);
+    const std::vector<double> ts_dbl = value_of(memory->ts_);
+
     double t_init = t0_dbl;
     for (size_t n = 0; n < ts_dbl.size(); ++n) {
       double t_final = ts_dbl[n];
+      //std::cout << "n = " << n << ": t_init = " << t_init << ", t_final = " << t_final << std::endl;
 
       if (t_final != t_init) {
         if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
@@ -615,43 +654,43 @@ class cvodes_integrator_adjoint_vari : public vari {
         }
       }
 
-      memory->y_[n] = memory->state;
+      memory->y_[n] = memory->state_;
 
       t_init = t_final;
     }
 
     returned_ = true;
+    //std::cout << "forward integrate...done" << std::endl;
     return build_varis<T_Return>(this, non_chaining_varis_, memory->y_);
   }
-
+  
   virtual void chain() {
+    //std::cout << "backward v2 integrate..." << std::endl;
+    //std::cout << "backward v2 chain_count_ = " << chain_count_++ << std::endl;
     // std::cout << "chain" << std::endl; //<-- Good way to verify it's only
     //  being called once
-    if (memory == NULL)
+    if (memory == NULL) {
       return;
+    }
 
-    if (memory->cvodes_mem_ == NULL)
+    if (memory->cvodes_mem_ == NULL) {
       return;
+    }
 
-    if (returned_ == false)
+    if (returned_ == false) {
       return;
+    }
 
     if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ == 0) {
       return;
     }
 
-    Eigen::VectorXd state_sens(N_);
-    Eigen::VectorXd quad(args_vars_);
-    N_Vector nv_state_sens
-        = N_VMake_Serial(state_sens.size(), state_sens.data());
-    N_Vector nv_quad = N_VMake_Serial(quad.size(), quad.data());
-    N_VConst(0.0, nv_state_sens);
-    N_VConst(0.0, nv_quad);
-
-    SUNMatrix A_b;
-    SUNLinearSolver LS_b;
-    A_b = SUNDenseMatrix(N_, N_);
-    LS_b = SUNDenseLinearSolver(nv_state_sens, A_b);
+    Eigen::VectorXd& state_sens = memory->state_sens_;
+    Eigen::VectorXd& quad = memory->quad_;
+    state_sens.setZero();
+    quad.setZero();
+    //N_VConst(0.0, memory->nv_state_sens_);
+    //N_VConst(0.0, memory->nv_quad_);
 
     /* check these if needed
       flag = CVodeSetUserDataB(cvode_mem, which, user_dataB);
@@ -665,189 +704,180 @@ class cvodes_integrator_adjoint_vari : public vari {
       flag = CVodeSetConstraintsB(cvode_mem, which, constraintsB);
      */
 
-    try {
-      int indexB;
-
-      // This is all boilerplate CVODES setting up the adjoint ODE to solve
-      check_flag_sundials(CVodeCreateB(memory->cvodes_mem_, lmm_b_, &indexB),
-                          "CVodeCreateB");
-
-      check_flag_sundials(CVodeSetUserDataB(memory->cvodes_mem_, indexB,
-                                            reinterpret_cast<void*>(this)),
-                          "CVodeSetUserDataB");
-
-      bool cvodes_backward_initialized = false;
-
-      // At every time step, collect the adjoints from the output
-      // variables and re-initialize the solver
-      double t_init = value_of(memory->ts_.back());
-      for (int i = memory->ts_.size() - 1; i >= 0; --i) {
-        // Take in the adjoints from all the output variables at this point
-        // in time
-        Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
-        for (int j = 0; j < N_; j++) {
-          // std::cout << "i: " << i << ", j: " << j << std::endl;
-          state_sens(j) += non_chaining_varis_[i * N_ + j]->adj_;
-          step_sens(j) += non_chaining_varis_[i * N_ + j]->adj_;
-        }
+    /*
+    if (memory->backward_is_initialized_) {
+      std::cout << "backward integrate: index_b_ = " << memory->index_b_ << std::endl;
+    } else {
+      std::cout << "backward not yet initialized" << std::endl;
+    }
+    */
 
-        if (ts_vars_ > 0 && i >= 0) {
-          ts_varis_[i]->adj_ += step_sens.dot(memory->rhs(
-              t_init, memory->y_[i], msgs_, memory->value_of_args_tuple_));
-          /*
-            apply(
-            [&](auto&&... args) {
-            double adj = step_sens.dot(
-            memory->f_(t_init, memory->y_[i], msgs_, args...));
-            // std::cout << "adj: " << adj << ", i: " << i << std::endl;
-            return adj;
-            },
-            memory->value_of_args_tuple_);
-          */
-        }
+    // At every time step, collect the adjoints from the output
+    // variables and re-initialize the solver
+    double t_init = value_of(memory->ts_.back());
+    for (int i = memory->ts_.size() - 1; i >= 0; --i) {
+      // Take in the adjoints from all the output variables at this point
+      // in time
+      Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
+      for (int j = 0; j < N_; j++) {
+        // std::cout << "i: " << i << ", j: " << j << std::endl;
+        state_sens(j) += non_chaining_varis_[i * N_ + j]->adj_;
+        step_sens(j) += non_chaining_varis_[i * N_ + j]->adj_;
+      }
 
-        double t_final = value_of((i > 0) ? memory->ts_[i - 1] : memory->t0_);
-        // std::cout << "time-point " << i << "; t_init = " << t_init << ";
-        // t_final = " << t_final << std::endl;
-        if (t_final != t_init) {
-          if (!cvodes_backward_initialized) {
-            // initialize CVODES backward machinery.
-            // the states of the backward problem *are* the adjoints
-            // of the ode states
-            check_flag_sundials(
-                CVodeInitB(memory->cvodes_mem_, indexB,
-                           &cvodes_integrator_adjoint_vari::cv_rhs_adj_sens,
-                           t_init, nv_state_sens),
-                "CVodeInitB");
+      if (ts_vars_ > 0 && i >= 0) {
+        ts_varis_[i]->adj_ += step_sens.dot(memory->rhs(
+            t_init, memory->y_[i], msgs_, memory->value_of_args_tuple_));
+        /*
+          apply(
+          [&](auto&&... args) {
+          double adj = step_sens.dot(
+          memory->f_(t_init, memory->y_[i], msgs_, args...));
+          // std::cout << "adj: " << adj << ", i: " << i << std::endl;
+          return adj;
+          },
+          memory->value_of_args_tuple_);
+        */
+      }
 
-            check_flag_sundials(
-                CVodeSVtolerancesB(memory->cvodes_mem_, indexB, rel_tol_b_,
-                                   memory->nv_abs_tol_b_),
-                "CVodeSVtolerancesB");
+      double t_final = value_of((i > 0) ? memory->ts_[i - 1] : memory->t0_);
+      //std::cout << "backward: time-point " << i << "; t_init = " << t_init << "; t_final = " << t_final << std::endl;
+      if (t_final != t_init) {
+        if (!memory->backward_is_initialized_) {
+          check_flag_sundials(CVodeCreateB(memory->cvodes_mem_, lmm_b_, &memory->index_b_),
+                              "CVodeCreateB");
 
-            check_flag_sundials(CVodeSetMaxNumStepsB(memory->cvodes_mem_,
-                                                     indexB, max_num_steps_),
-                                "CVodeSetMaxNumStepsB");
+          check_flag_sundials(CVodeSetUserDataB(memory->cvodes_mem_, memory->index_b_,
+                                                reinterpret_cast<void*>(this)),
+                              "CVodeSetUserDataB");
 
-            check_flag_sundials(
-                CVodeSetLinearSolverB(memory->cvodes_mem_, indexB, LS_b, A_b),
-                "CVodeSetLinearSolverB");
+          // initialize CVODES backward machinery.
+          // the states of the backward problem *are* the adjoints
+          // of the ode states
+          check_flag_sundials(
+              CVodeInitB(memory->cvodes_mem_, memory->index_b_,
+                         &cvodes_integrator_adjoint_vari::cv_rhs_adj_sens,
+                         t_init, memory->nv_state_sens_),
+              "CVodeInitB");
 
-            check_flag_sundials(
-                CVodeSetJacFnB(
-                    memory->cvodes_mem_, indexB,
-                    &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
-                "CVodeSetJacFnB");
-
-            // Allocate space for backwards quadrature needed when
-            // parameters vary.
-            if (args_vars_ > 0) {
-              check_flag_sundials(
-                  CVodeQuadInitB(
-                      memory->cvodes_mem_, indexB,
-                      &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj,
-                      nv_quad),
-                  "CVodeQuadInitB");
-
-              check_flag_sundials(
-                  CVodeQuadSStolerancesB(memory->cvodes_mem_, indexB,
-                                         rel_tol_q_, abs_tol_q_),
-                  "CVodeQuadSStolerancesB");
-
-              check_flag_sundials(
-                  CVodeSetQuadErrConB(memory->cvodes_mem_, indexB, SUNTRUE),
-                  "CVodeSetQuadErrConB");
-            }
-
-            cvodes_backward_initialized = true;
-          } else {
-            // just re-initialize the solver
-            check_flag_sundials(CVodeReInitB(memory->cvodes_mem_, indexB,
-                                             t_init, nv_state_sens),
-                                "CVodeReInitB");
-
-            if (args_vars_ > 0) {
-              check_flag_sundials(
-                  CVodeQuadReInitB(memory->cvodes_mem_, indexB, nv_quad),
-                  "CVodeQuadReInitB");
-            }
-          }
+          check_flag_sundials(
+              CVodeSVtolerancesB(memory->cvodes_mem_, memory->index_b_, relative_tolerance_backward_,
+                                 memory->nv_absolute_tolerance_backward_),
+              "CVodeSVtolerancesB");
 
-          int error_code = CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL);
+          check_flag_sundials(CVodeSetMaxNumStepsB(memory->cvodes_mem_,
+                                                   memory->index_b_, max_num_steps_),
+                              "CVodeSetMaxNumStepsB");
 
-          if (error_code == CV_TOO_MUCH_WORK) {
-            throw_domain_error(function_name_, "", t_final,
-                               "Failed to integrate backward to output time (",
-                               ") in less than max_num_steps steps");
-          } else {
-            check_flag_sundials(error_code, "CVodeB");
-          }
+          check_flag_sundials(
+              CVodeSetLinearSolverB(memory->cvodes_mem_, memory->index_b_, memory->LS_b_, memory->A_b_),
+              "CVodeSetLinearSolverB");
+          
+          check_flag_sundials(
+              CVodeSetJacFnB(
+                  memory->cvodes_mem_, memory->index_b_,
+                  &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
+              "CVodeSetJacFnB");
 
-          // check_flag_sundials(CVodeB(memory->cvodes_mem_, t_final,
-          // CV_NORMAL),
-          //                    "CVodeB");
+          // Allocate space for backwards quadrature needed when
+          // parameters vary.
+          if (args_vars_ > 0) {
+            check_flag_sundials(
+                CVodeQuadInitB(
+                    memory->cvodes_mem_, memory->index_b_,
+                    &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj,
+                    memory->nv_quad_),
+                "CVodeQuadInitB");
+            
+            check_flag_sundials(
+                CVodeQuadSStolerancesB(memory->cvodes_mem_, memory->index_b_,
+                                       relative_tolerance_quadrature_, absolute_tolerance_quadrature_),
+                "CVodeQuadSStolerancesB");
+            
+            check_flag_sundials(
+                CVodeSetQuadErrConB(memory->cvodes_mem_, memory->index_b_, SUNTRUE),
+                "CVodeSetQuadErrConB");
+          }
 
-          // obtain adjoint states and update t_init to time point
-          // reached of t_final
-          check_flag_sundials(
-              CVodeGetB(memory->cvodes_mem_, indexB, &t_init, nv_state_sens),
-              "CVodeGetB");
+          memory->backward_is_initialized_ = true;
+        } else {
+          // just re-initialize the solver
+          check_flag_sundials(CVodeReInitB(memory->cvodes_mem_, memory->index_b_,
+                                           t_init, memory->nv_state_sens_),
+                              "CVodeReInitB");
 
           if (args_vars_ > 0) {
             check_flag_sundials(
-                CVodeGetQuadB(memory->cvodes_mem_, indexB, &t_init, nv_quad),
-                "CVodeGetQuadB");
+                CVodeQuadReInitB(memory->cvodes_mem_, memory->index_b_, memory->nv_quad_),
+                "CVodeQuadReInitB");
           }
         }
-      }
 
-      if (t0_vars_ > 0) {
-        Eigen::VectorXd y0d = value_of(memory->y0_);
-        t0_varis_[0]->adj_ += -state_sens.dot(
-            memory->rhs(t_init, y0d, msgs_, memory->value_of_args_tuple_));
-        /*
-          apply(
-            [&](auto&&... args) {
-              return -state_sens.dot(memory->f_(t_init, y0d, msgs_, args...));
-            },
-            memory->value_of_args_tuple_);
-        */
-      }
+        int error_code = CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL);
+        
+        if (error_code == CV_TOO_MUCH_WORK) {
+          throw_domain_error(function_name_, "", t_final,
+                             "Failed to integrate backward to output time (",
+                             ") in less than max_num_steps steps");
+        } else {
+          check_flag_sundials(error_code, "CVodeB");
+        }
 
-      // do we need this a 2nd time? Don't think so.
-      /*
-      if (args_vars_ > 0) {
+        // check_flag_sundials(CVodeB(memory->cvodes_mem_, t_final,
+        // CV_NORMAL),
+        //                    "CVodeB");
+        
+        // obtain adjoint states and update t_init to time point
+        // reached of t_final
         check_flag_sundials(
-            CVodeGetQuadB(memory->cvodes_mem_, indexB, &t_init, nv_quad),
-            "CVodeGetQuadB");
-      }
-      */
+            CVodeGetB(memory->cvodes_mem_, memory->index_b_, &t_init, memory->nv_state_sens_),
+            "CVodeGetB");
 
-      // After integrating all the way back to t0, we finally have the
-      // the adjoints we wanted
-      // These are the dlog_density / d(initial_conditions[s]) adjoints
-      for (size_t s = 0; s < y0_vars_; s++) {
-        y0_varis_[s]->adj_ += state_sens.coeff(s);
+        if (args_vars_ > 0) {
+          check_flag_sundials(
+              CVodeGetQuadB(memory->cvodes_mem_, memory->index_b_, &t_init, memory->nv_quad_),
+              "CVodeGetQuadB");
+        }
       }
+    }
 
-      // These are the dlog_density / d(parameters[s]) adjoints
-      for (size_t s = 0; s < args_vars_; s++) {
-        args_varis_[s]->adj_ += quad.coeff(s);
+    if (t0_vars_ > 0) {
+      Eigen::VectorXd y0d = value_of(memory->y0_);
+      t0_varis_[0]->adj_ += -state_sens.dot(
+          memory->rhs(t_init, y0d, msgs_, memory->value_of_args_tuple_));
+      /*
+        apply(
+        [&](auto&&... args) {
+        return -state_sens.dot(memory->f_(t_init, y0d, msgs_, args...));
+        },
+        memory->value_of_args_tuple_);
+      */
+    }
+    
+    // do we need this a 2nd time? Don't think so.
+    /*
+      if (args_vars_ > 0) {
+      check_flag_sundials(
+      CVodeGetQuadB(memory->cvodes_mem_, indexB, &t_init, nv_quad),
+      "CVodeGetQuadB");
       }
+    */
 
-    } catch (const std::exception& e) {
-      SUNLinSolFree(LS_b);
-      SUNMatDestroy(A_b);
-      N_VDestroy_Serial(nv_state_sens);
-      N_VDestroy_Serial(nv_quad);
-      throw;
+    // After integrating all the way back to t0, we finally have the
+    // the adjoints we wanted
+    // These are the dlog_density / d(initial_conditions[s]) adjoints
+    for (size_t s = 0; s < y0_vars_; s++) {
+      y0_varis_[s]->adj_ += state_sens.coeff(s);
+    }
+    
+    // These are the dlog_density / d(parameters[s]) adjoints
+    for (size_t s = 0; s < args_vars_; s++) {
+      args_varis_[s]->adj_ += quad.coeff(s);
     }
 
-    SUNLinSolFree(LS_b);
-    SUNMatDestroy(A_b);
-    N_VDestroy_Serial(nv_state_sens);
-    N_VDestroy_Serial(nv_quad);
+    //std::cout << "backward v2 integrate...done" << std::endl;
   }
+  
 };  // cvodes integrator
 
 }  // namespace math
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index c65e14abf10..b05b6567f46 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -36,19 +36,19 @@ namespace math {
  * @param t0 Initial time
  * @param ts Times at which to solve the ODE at. All values must be sorted and
  *   not less than t0.
- * @param rel_tol_f Relative tolerance for forward problem passed to CVODES
- * @param abs_tol_f Absolute tolerance for forward problem passed to CVODES
- * @param rel_tol_b Relative tolerance for backward problem passed to CVODES
- * @param abs_tol_b Absolute tolerance for backward problem passed to CVODES
- * @param rel_tol_q Relative tolerance for quadrature problem passed to CVODES
- * @param abs_tol_q Absolute tolerance for quadrature problem passed to CVODES
+ * @param relative_tolerance_forward Relative tolerance for forward problem passed to CVODES
+ * @param absolute_tolerance_forward Absolute tolerance for forward problem passed to CVODES
+ * @param relative_tolerance_backward Relative tolerance for backward problem passed to CVODES
+ * @param absolute_tolerance_backward Absolute tolerance for backward problem passed to CVODES
+ * @param relative_tolerance_quadrature Relative tolerance for quadrature problem passed to CVODES
+ * @param absolute_tolerance_quadrature Absolute tolerance for quadrature problem passed to CVODES
  * @param max_num_steps Upper limit on the number of integration steps to
  *   take between each output (error if exceeded)
- * @param num_checkpoints Number of integrator steps after which a checkpoint is
+ * @param num_steps_between_checkpoints Number of integrator steps after which a checkpoint is
  * stored for the backward pass
  * @param interpolation_polynomial type of polynomial used for interpolation
- * @param solver_f solver used for forward pass
- * @param solver_b solver used for backward pass
+ * @param solver_forward solver used for forward pass
+ * @param solver_backward solver used for backward pass
  * @param[in, out] msgs the print stream for warning messages
  * @param args Extra arguments passed unmodified through to ODE right hand side
  * @return Solution to ODE at times \p ts
@@ -58,11 +58,11 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
-                 const T_t0& t0, const std::vector<T_ts>& ts, double rel_tol_f,
-                 Eigen::VectorXd abs_tol_f, double rel_tol_b,
-                 Eigen::VectorXd abs_tol_b, double rel_tol_q, double abs_tol_q,
-                 long int max_num_steps, long int num_checkpoints,
-                 int interpolation_polynomial, int solver_f, int solver_b,
+                 const T_t0& t0, const std::vector<T_ts>& ts, double relative_tolerance_forward,
+                 Eigen::VectorXd absolute_tolerance_forward, double relative_tolerance_backward,
+                 Eigen::VectorXd absolute_tolerance_backward, double relative_tolerance_quadrature, double absolute_tolerance_quadrature,
+                 long int max_num_steps, long int num_steps_between_checkpoints,
+                 int interpolation_polynomial, int solver_forward, int solver_backward,
                  std::ostream* msgs, const T_Args&... args) {
   /*
   const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
@@ -99,9 +99,9 @@ ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
   */
   auto integrator = new stan::math::cvodes_integrator_adjoint_vari<
       F, plain_type_t<T_y0>, T_t0, T_ts, ref_type_t<T_Args>...>(
-      function_name, f, eval(y0), t0, ts, rel_tol_f, abs_tol_f, rel_tol_b,
-      abs_tol_b, rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,
-      interpolation_polynomial, solver_f, solver_b, msgs, to_ref(args)...);
+      function_name, f, eval(y0), t0, ts, relative_tolerance_forward, absolute_tolerance_forward, relative_tolerance_backward,
+      absolute_tolerance_backward, relative_tolerance_quadrature, absolute_tolerance_quadrature, max_num_steps, num_steps_between_checkpoints,
+      interpolation_polynomial, solver_forward, solver_backward, msgs, to_ref(args)...);
   return (*integrator)();
 }
 
@@ -143,17 +143,17 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_tol_ctl(const F& f, const T_y0& y0, const T_t0& t0,
-                    const std::vector<T_ts>& ts, double rel_tol_f,
-                    Eigen::VectorXd abs_tol_f, double rel_tol_b,
-                    Eigen::VectorXd abs_tol_b, double rel_tol_q,
-                    double abs_tol_q, long int max_num_steps,
-                    long int num_checkpoints, int interpolation_polynomial,
-                    int solver_f, int solver_b, std::ostream* msgs,
+                    const std::vector<T_ts>& ts, double relative_tolerance_forward,
+                    Eigen::VectorXd absolute_tolerance_forward, double relative_tolerance_backward,
+                    Eigen::VectorXd absolute_tolerance_backward, double relative_tolerance_quadrature,
+                    double absolute_tolerance_quadrature, long int max_num_steps,
+                    long int num_steps_between_checkpoints, int interpolation_polynomial,
+                    int solver_forward, int solver_backward, std::ostream* msgs,
                     const T_Args&... args) {
   return ode_adjoint_impl(
-      "ode_adjoint_tol_ctl", f, y0, t0, ts, rel_tol_f, abs_tol_f, rel_tol_b,
-      abs_tol_b, rel_tol_q, abs_tol_q, max_num_steps, num_checkpoints,
-      interpolation_polynomial, solver_f, solver_b, msgs, args...);
+      "ode_adjoint_tol_ctl", f, y0, t0, ts, relative_tolerance_forward, absolute_tolerance_forward, relative_tolerance_backward,
+      absolute_tolerance_backward, relative_tolerance_quadrature, absolute_tolerance_quadrature, max_num_steps, num_steps_between_checkpoints,
+      interpolation_polynomial, solver_forward, solver_backward, msgs, args...);
 }
 
 template <typename F, typename T_y0, typename T_t0, typename T_ts,
@@ -161,22 +161,22 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
-                const std::vector<T_ts>& ts, double rel_tol, double abs_tol,
+                const std::vector<T_ts>& ts, double relative_tolerance, double absolute_tolerance,
                 long int max_num_steps, std::ostream* msgs,
                 const T_Args&... args) {
   const int N = y0.size();
-  const Eigen::VectorXd abs_tol_f
-      = Eigen::VectorXd::Constant(N, abs_tol / 100.0);
-  const Eigen::VectorXd abs_tol_b
-      = Eigen::VectorXd::Constant(N, abs_tol / 10.0);
-  const long int num_checkpoints = 250;
+  const Eigen::VectorXd absolute_tolerance_forward
+      = Eigen::VectorXd::Constant(N, absolute_tolerance / 100.0);
+  const Eigen::VectorXd absolute_tolerance_backward
+      = Eigen::VectorXd::Constant(N, absolute_tolerance / 10.0);
+  const long int num_steps_between_checkpoints = 250;
   const int interpolation_polynomial = CV_HERMITE;
-  const int solver_f = CV_BDF;
-  const int solver_b = CV_BDF;
-  return ode_adjoint_impl("ode_adjoint_tol", f, y0, t0, ts, rel_tol, abs_tol_f,
-                          rel_tol, abs_tol_b, rel_tol, abs_tol, max_num_steps,
-                          num_checkpoints, interpolation_polynomial, solver_f,
-                          solver_b, msgs, args...);
+  const int solver_forward = CV_BDF;
+  const int solver_backward = CV_BDF;
+  return ode_adjoint_impl("ode_adjoint_tol", f, y0, t0, ts, relative_tolerance, absolute_tolerance_forward,
+                          relative_tolerance, absolute_tolerance_backward, relative_tolerance, absolute_tolerance, max_num_steps,
+                          num_steps_between_checkpoints, interpolation_polynomial, solver_forward,
+                          solver_backward, msgs, args...);
 }
 
 }  // namespace math
diff --git a/test/unit/math/rev/functor/cos_ode_typed_test.cpp b/test/unit/math/rev/functor/cos_ode_typed_test.cpp
index 1327745c2e3..a0e32d2e01c 100644
--- a/test/unit/math/rev/functor/cos_ode_typed_test.cpp
+++ b/test/unit/math/rev/functor/cos_ode_typed_test.cpp
@@ -17,7 +17,8 @@ using ode_test_tuple = std::tuple<solve_type, solve_type, Ts...>;
  */
 using cos_arg_test_types = boost::mp11::mp_product<
     ode_test_tuple, ::testing::Types<ode_adams_functor, ode_bdf_functor,
-                                     ode_ckrk_functor, ode_rk45_functor> >;
+                                     ode_ckrk_functor, ode_rk45_functor,
+                                     ode_adjoint_functor> >;
 
 TYPED_TEST_SUITE_P(cos_arg_test);
 TYPED_TEST_P(cos_arg_test, y0_error) {
diff --git a/test/unit/math/rev/functor/fho_ode_typed_ts_test.cpp b/test/unit/math/rev/functor/fho_ode_typed_ts_test.cpp
index 17ed94dc23e..f405cad4952 100644
--- a/test/unit/math/rev/functor/fho_ode_typed_ts_test.cpp
+++ b/test/unit/math/rev/functor/fho_ode_typed_ts_test.cpp
@@ -18,7 +18,7 @@ using ode_test_tuple = std::tuple<solve_type, solve_type, Ts...>;
 using forced_harm_osc_ts_test_types = boost::mp11::mp_product<
     ode_test_tuple,
     ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
-                     ode_rk45_functor>,
+                     ode_rk45_functor, ode_adjoint_functor>,
     ::testing::Types<stan::math::var_value<double> >,          // time
     ::testing::Types<double, stan::math::var_value<double> >,  // y0
     ::testing::Types<double, stan::math::var_value<double> >   // theta
diff --git a/test/unit/math/rev/functor/lorenz_ode_typed_fd_test.cpp b/test/unit/math/rev/functor/lorenz_ode_typed_fd_test.cpp
index e882930d38e..bf805630178 100644
--- a/test/unit/math/rev/functor/lorenz_ode_typed_fd_test.cpp
+++ b/test/unit/math/rev/functor/lorenz_ode_typed_fd_test.cpp
@@ -17,7 +17,8 @@ using ode_test_tuple = std::tuple<solve_type, solve_type, Ts...>;
  */
 using lorenz_test_types = boost::mp11::mp_product<
     ode_test_tuple, ::testing::Types<ode_adams_functor, ode_bdf_functor,
-                                     ode_ckrk_functor, ode_rk45_functor>>;
+                                     ode_ckrk_functor, ode_rk45_functor,
+                                     ode_adjoint_functor>>;
 
 TYPED_TEST_SUITE_P(lorenz_test);
 TYPED_TEST_P(lorenz_test, param_and_data_finite_diff) {
diff --git a/test/unit/math/rev/functor/ode_test_functors.hpp b/test/unit/math/rev/functor/ode_test_functors.hpp
index 1abf22f1138..dab4cef68ad 100644
--- a/test/unit/math/rev/functor/ode_test_functors.hpp
+++ b/test/unit/math/rev/functor/ode_test_functors.hpp
@@ -5,6 +5,7 @@
 #include <stan/math/prim/functor/ode_rk45.hpp>
 #include <stan/math/rev/functor/ode_bdf.hpp>
 #include <stan/math/rev/functor/ode_adams.hpp>
+#include <stan/math/rev/functor/ode_adjoint.hpp>
 #include <stan/math/prim/functor/integrate_ode_rk45.hpp>
 
 #define STAN_DEF_ODE_SOLVER_FUNCTOR(solver_name, solver_func)                  \
@@ -61,4 +62,31 @@ STAN_DEF_STD_ODE_SOLVER_FUNCTOR(integrate_ode_bdf,
 STAN_DEF_STD_ODE_SOLVER_FUNCTOR(integrate_ode_rk45,
                                 stan::math::integrate_ode_rk45);
 
+struct ode_adjoint_functor {
+    const std::string functor_name = "ode_adjoint";
+
+    template <typename F, typename T_y0, typename T_t0, typename T_ts,
+              typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr>
+    std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,
+                              Eigen::Dynamic, 1>>
+    operator()(const F& f, const T_y0& y0, T_t0 t0,
+               const std::vector<T_ts>& ts, std::ostream* msgs,
+               const Args&... args) {
+      return stan::math::ode_adjoint_tol(f, y0, t0, ts, 1E-13, 1E-13, 1000000, msgs, args...);
+    }
+
+    template <typename F, typename T_y0, typename T_t0, typename T_ts,
+              typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr>
+    std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,
+                              Eigen::Dynamic, 1>>
+    operator()(const F& f, const T_y0& y0_arg, T_t0 t0,
+               const std::vector<T_ts>& ts, double rtol, double atol,
+               size_t max_num_steps, std::ostream* msgs,
+               const Args&... args) {
+      return stan::math::ode_adjoint_tol(f, y0_arg, t0, ts, rtol, atol, max_num_steps,
+                               msgs, args...);
+    }
+};
+
+
 #endif
diff --git a/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp b/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
index c85ce707482..aebf9b1d216 100644
--- a/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
@@ -18,7 +18,7 @@ using ode_test_tuple = std::tuple<solve_type, solve_type, Ts...>;
 using harmonic_oscillator_test_types = boost::mp11::mp_product<
     ode_test_tuple,
     ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
-                     ode_rk45_functor>,
+                     ode_rk45_functor, ode_adjoint_functor>,
     ::testing::Types<double, stan::math::var_value<double>>,  // t
     ::testing::Types<double, stan::math::var_value<double>>,  // y0
     ::testing::Types<double, stan::math::var_value<double>>   // theta
diff --git a/test/unit/math/rev/functor/sho_ode_typed_test.cpp b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
index fc57ce6a489..084408780c5 100644
--- a/test/unit/math/rev/functor/sho_ode_typed_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
@@ -18,8 +18,8 @@ using ode_test_tuple = std::tuple<solve_type, solve_type, Ts...>;
  */
 using harmonic_oscillator_fd_test_types = boost::mp11::mp_product<
     ode_test_tuple,
-    ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
-                     ode_rk45_functor>,
+  ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
+                   ode_rk45_functor, ode_adjoint_functor>,
     ::testing::Types<double>,  // t
     ::testing::Types<double>,  // y0
     ::testing::Types<double>   // theta
@@ -89,8 +89,9 @@ INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_data_test,
 
 using harmonic_oscillator_test_types = boost::mp11::mp_product<
     ode_test_tuple,
-    ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
-                     ode_rk45_functor>,
+  ::testing::Types<//ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
+    //ode_rk45_functor,
+                     ode_adjoint_functor>,
     ::testing::Types<double>,                                  // t
     ::testing::Types<double, stan::math::var_value<double> >,  // y0
     ::testing::Types<double, stan::math::var_value<double> >   // theta
@@ -114,6 +115,10 @@ TYPED_TEST_P(harmonic_oscillator_t0_ad_test, t0_ad) {
                    ode_bdf_functor>::value) {
     this->test_t0_ad(1e-7);
   }
+  if (std::is_same<std::tuple_element_t<0, TypeParam>,
+                   ode_adjoint_functor>::value) {
+    this->test_t0_ad(1e-7);
+  }
 }
 REGISTER_TYPED_TEST_SUITE_P(harmonic_oscillator_t0_ad_test, t0_ad);
 INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_t0_ad_test,

From 4abf989d0e3dbca450e1ed87d821141887cb98e5 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Thu, 15 Apr 2021 19:58:38 +0000
Subject: [PATCH 076/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 223 ++++++++++--------
 stan/math/rev/functor/ode_adjoint.hpp         |  91 ++++---
 .../math/rev/functor/cos_ode_typed_test.cpp   |   6 +-
 .../rev/functor/lorenz_ode_typed_fd_test.cpp  |   6 +-
 .../math/rev/functor/ode_test_functors.hpp    |  42 ++--
 .../math/rev/functor/sho_ode_typed_test.cpp   |  10 +-
 6 files changed, 215 insertions(+), 163 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 49206e47467..e8459fb9a07 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -108,9 +108,9 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
 
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
   cvodes_integrator_adjoint_memory(Eigen::VectorXd absolute_tolerance_forward,
-                                   Eigen::VectorXd absolute_tolerance_backward, int lmm_f,
-                                   const F& f, const T_y0& y0, const T_t0& t0,
-                                   const std::vector<T_ts>& ts,
+                                   Eigen::VectorXd absolute_tolerance_backward,
+                                   int lmm_f, const F& f, const T_y0& y0,
+                                   const T_t0& t0, const std::vector<T_ts>& ts,
                                    const T_Args&... args)
       : N_(y0.size()),
         absolute_tolerance_forward_(absolute_tolerance_forward),
@@ -128,14 +128,15 @@ class cvodes_integrator_adjoint_memory : public chainable_alloc {
         state_(value_of(y0)),
         state_sens_(Eigen::VectorXd::Zero(N_)),
         quad_(Eigen::VectorXd::Zero(count_vars(args...))),
-        backward_is_initialized_(false)
-  {
+        backward_is_initialized_(false) {
     if (N_ > 0) {
       nv_state_ = N_VMake_Serial(N_, state_.data());
       nv_state_sens_ = N_VMake_Serial(N_, state_sens_.data());
       nv_quad_ = N_VMake_Serial(quad_.size(), quad_.data());
-      nv_absolute_tolerance_forward_ = N_VMake_Serial(N_, absolute_tolerance_forward_.data());
-      nv_absolute_tolerance_backward_ = N_VMake_Serial(N_, absolute_tolerance_backward_.data());
+      nv_absolute_tolerance_forward_
+          = N_VMake_Serial(N_, absolute_tolerance_forward_.data());
+      nv_absolute_tolerance_backward_
+          = N_VMake_Serial(N_, absolute_tolerance_backward_.data());
       A_f_ = SUNDenseMatrix(N_, N_);
       LS_f_ = SUNDenseLinearSolver(nv_state_, A_f_);
       A_b_ = SUNDenseMatrix(N_, N_);
@@ -381,7 +382,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     apply([&](auto&&... args) { accumulate_adjoints(mu_dot.data(), args...); },
           memory->local_args_tuple_);
   }
-  
+
   /**
    * Calculates the jacobian of the ODE RHS wrt to its states y at the
    * given time-point t and state y.
@@ -429,7 +430,9 @@ class cvodes_integrator_adjoint_vari : public vari {
 
  public:
   /**
-   * Construct cvodes_integrator object. Note: All arguments must be stored as copies if in doubt. The reason is that the references can go out of scope, since the work done from the integrator is in the chain method.
+   * Construct cvodes_integrator object. Note: All arguments must be stored as
+   * copies if in doubt. The reason is that the references can go out of scope,
+   * since the work done from the integrator is in the chain method.
    *
    * @param function_name Calling function name (for printing debugging
    * messages)
@@ -438,16 +441,22 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param t0 Initial time
    * @param ts Times at which to solve the ODE at. All values must be sorted and
    *   not less than t0.
-   * @param relative_tolerance_forward Relative tolerance for forward problem passed to CVODES
-   * @param absolute_tolerance_forward Absolute tolerance for forward problem passed to CVODES
-   * @param relative_tolerance_backward Relative tolerance for backward problem passed to CVODES
-   * @param absolute_tolerance_backward Absolute tolerance for backward problem passed to CVODES
-   * @param relative_tolerance_quadrature Relative tolerance for quadrature problem passed to CVODES
-   * @param absolute_tolerance_quadrature Absolute tolerance for quadrature problem passed to CVODES
+   * @param relative_tolerance_forward Relative tolerance for forward problem
+   * passed to CVODES
+   * @param absolute_tolerance_forward Absolute tolerance for forward problem
+   * passed to CVODES
+   * @param relative_tolerance_backward Relative tolerance for backward problem
+   * passed to CVODES
+   * @param absolute_tolerance_backward Absolute tolerance for backward problem
+   * passed to CVODES
+   * @param relative_tolerance_quadrature Relative tolerance for quadrature
+   * problem passed to CVODES
+   * @param absolute_tolerance_quadrature Absolute tolerance for quadrature
+   * problem passed to CVODES
    * @param max_num_steps Upper limit on the number of integration steps to
    *   take between each output (error if exceeded)
-   * @param num_steps_between_checkpoints Number of integrator steps after which a checkpoint
-   * is stored for the backward pass
+   * @param num_steps_between_checkpoints Number of integrator steps after which
+   * a checkpoint is stored for the backward pass
    * @param interpolation_polynomial type of polynomial used for interpolation
    * @param solver_forward solver used for forward pass
    * @param solver_backward solver used for backward pass
@@ -462,11 +471,15 @@ class cvodes_integrator_adjoint_vari : public vari {
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
   cvodes_integrator_adjoint_vari(
       const char* function_name, const F& f, const T_y0& y0, const T_t0& t0,
-      const std::vector<T_ts>& ts, double relative_tolerance_forward, Eigen::VectorXd absolute_tolerance_forward,
-      double relative_tolerance_backward, Eigen::VectorXd absolute_tolerance_backward, double relative_tolerance_quadrature,
-      double absolute_tolerance_quadrature, long int max_num_steps, long int num_steps_between_checkpoints,
-      int interpolation_polynomial, int solver_forward, int solver_backward,
-      std::ostream* msgs, const T_Args&... args)
+      const std::vector<T_ts>& ts, double relative_tolerance_forward,
+      Eigen::VectorXd absolute_tolerance_forward,
+      double relative_tolerance_backward,
+      Eigen::VectorXd absolute_tolerance_backward,
+      double relative_tolerance_quadrature,
+      double absolute_tolerance_quadrature, long int max_num_steps,
+      long int num_steps_between_checkpoints, int interpolation_polynomial,
+      int solver_forward, int solver_backward, std::ostream* msgs,
+      const T_Args&... args)
       : function_name_(function_name),
         vari(NOT_A_NUMBER),
         N_(y0.size()),
@@ -504,7 +517,8 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     memory
         = new cvodes_integrator_adjoint_memory<F, T_y0, T_t0, T_ts, T_Args...>(
-            absolute_tolerance_forward_, absolute_tolerance_backward_, lmm_f_, f, y0, t0, ts, args...);
+            absolute_tolerance_forward_, absolute_tolerance_backward_, lmm_f_,
+            f, y0, t0, ts, args...);
 
     save_varis(t0_varis_, t0);
     save_varis(ts_varis_, ts);
@@ -529,17 +543,26 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_sorted(fun, "times", ts);
     check_less(fun, "initial time", t0, ts[0]);
     // TODO (SW): make names verbose
-    check_positive_finite(fun, "relative_tolerance_f", relative_tolerance_forward_);
-    check_positive_finite(fun, "absolute_tolerance_f", absolute_tolerance_forward_);
-    check_size_match(fun, "absolute_tolerance_forward", absolute_tolerance_forward_.size(), "states", N_);
-    check_positive_finite(fun, "relative_tolerance_backward", relative_tolerance_backward_);
-    check_positive_finite(fun, "absolute_tolerance_backward", absolute_tolerance_backward_);
-    check_size_match(fun, "absolute_tolerance_backward", absolute_tolerance_backward_.size(), "states", N_);
-    check_positive_finite(fun, "relative_tolerance_quadrature", relative_tolerance_quadrature_);
-    check_positive_finite(fun, "absolute_tolerance_quadrature", absolute_tolerance_quadrature_);
+    check_positive_finite(fun, "relative_tolerance_f",
+                          relative_tolerance_forward_);
+    check_positive_finite(fun, "absolute_tolerance_f",
+                          absolute_tolerance_forward_);
+    check_size_match(fun, "absolute_tolerance_forward",
+                     absolute_tolerance_forward_.size(), "states", N_);
+    check_positive_finite(fun, "relative_tolerance_backward",
+                          relative_tolerance_backward_);
+    check_positive_finite(fun, "absolute_tolerance_backward",
+                          absolute_tolerance_backward_);
+    check_size_match(fun, "absolute_tolerance_backward",
+                     absolute_tolerance_backward_.size(), "states", N_);
+    check_positive_finite(fun, "relative_tolerance_quadrature",
+                          relative_tolerance_quadrature_);
+    check_positive_finite(fun, "absolute_tolerance_quadrature",
+                          absolute_tolerance_quadrature_);
     check_positive(fun, "max_num_steps", max_num_steps_);
     // TODO (SW): rename according to design
-    check_positive(fun, "num_steps_between_checkpoints", num_steps_between_checkpoints_);
+    check_positive(fun, "num_steps_between_checkpoints",
+                   num_steps_between_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
     check_range(fun, "interpolation_polynomial", 2, interpolation_polynomial_);
     // 1=Adams, 2=BDF
@@ -552,7 +575,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       std::cout << value_of(ts[i]) << ", ";
     std::cout << std::endl;
     */
-    
+
     /*
     std::cout << "relative_tolerance = " << relative_tolerance << std::endl;
     std::cout << "absolute_tolerance = " << absolute_tolerance << std::endl;
@@ -562,9 +585,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     std::cout << "steps_checkpoint = " << steps_checkpoint << std::endl;
     */
 
-
     // initializes CVODES solver
-    
+
     check_flag_sundials(
         CVodeInit(memory->cvodes_mem_, &cvodes_integrator_adjoint_vari::cv_rhs,
                   value_of(memory->t0_), memory->nv_state_),
@@ -575,13 +597,14 @@ class cvodes_integrator_adjoint_vari : public vari {
         CVodeSetUserData(memory->cvodes_mem_, reinterpret_cast<void*>(this)),
         "CVodeSetUserData");
 
-    cvodes_set_options(memory->cvodes_mem_, relative_tolerance_forward_, absolute_tolerance_forward_(0),
-                       max_num_steps_);
+    cvodes_set_options(memory->cvodes_mem_, relative_tolerance_forward_,
+                       absolute_tolerance_forward_(0), max_num_steps_);
+
+    check_flag_sundials(
+        CVodeSVtolerances(memory->cvodes_mem_, relative_tolerance_forward_,
+                          memory->nv_absolute_tolerance_forward_),
+        "CVodeSVtolerances");
 
-    check_flag_sundials(CVodeSVtolerances(memory->cvodes_mem_, relative_tolerance_forward_,
-                                          memory->nv_absolute_tolerance_forward_),
-                        "CVodeSVtolerances");
-    
     // for the stiff solvers we need to reserve additional memory
     // and provide a Jacobian function call. new API since 3.0.0:
     // create matrix object and linear solver object; resource
@@ -597,11 +620,11 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     // initialize backward sensitivity system of CVODES as needed
     if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
-      check_flag_sundials(CVodeAdjInit(memory->cvodes_mem_, num_steps_between_checkpoints_,
-                                       interpolation_polynomial_),
-                          "CVodeAdjInit");
+      check_flag_sundials(
+          CVodeAdjInit(memory->cvodes_mem_, num_steps_between_checkpoints_,
+                       interpolation_polynomial_),
+          "CVodeAdjInit");
     }
-
   }
 
   ~cvodes_integrator_adjoint_vari() {}
@@ -615,14 +638,15 @@ class cvodes_integrator_adjoint_vari : public vari {
    *   solution time (excluding the initial state)
    */
   std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> operator()() {
-    //std::cout << "forward integrate..." << std::endl;
+    // std::cout << "forward integrate..." << std::endl;
     const double t0_dbl = value_of(memory->t0_);
     const std::vector<double> ts_dbl = value_of(memory->ts_);
 
     double t_init = t0_dbl;
     for (size_t n = 0; n < ts_dbl.size(); ++n) {
       double t_final = ts_dbl[n];
-      //std::cout << "n = " << n << ": t_init = " << t_init << ", t_final = " << t_final << std::endl;
+      // std::cout << "n = " << n << ": t_init = " << t_init << ", t_final = "
+      // << t_final << std::endl;
 
       if (t_final != t_init) {
         if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
@@ -660,13 +684,14 @@ class cvodes_integrator_adjoint_vari : public vari {
     }
 
     returned_ = true;
-    //std::cout << "forward integrate...done" << std::endl;
+    // std::cout << "forward integrate...done" << std::endl;
     return build_varis<T_Return>(this, non_chaining_varis_, memory->y_);
   }
-  
+
   virtual void chain() {
-    //std::cout << "backward v2 integrate..." << std::endl;
-    //std::cout << "backward v2 chain_count_ = " << chain_count_++ << std::endl;
+    // std::cout << "backward v2 integrate..." << std::endl;
+    // std::cout << "backward v2 chain_count_ = " << chain_count_++ <<
+    // std::endl;
     // std::cout << "chain" << std::endl; //<-- Good way to verify it's only
     //  being called once
     if (memory == NULL) {
@@ -689,8 +714,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     Eigen::VectorXd& quad = memory->quad_;
     state_sens.setZero();
     quad.setZero();
-    //N_VConst(0.0, memory->nv_state_sens_);
-    //N_VConst(0.0, memory->nv_quad_);
+    // N_VConst(0.0, memory->nv_state_sens_);
+    // N_VConst(0.0, memory->nv_quad_);
 
     /* check these if needed
       flag = CVodeSetUserDataB(cvode_mem, which, user_dataB);
@@ -706,9 +731,9 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     /*
     if (memory->backward_is_initialized_) {
-      std::cout << "backward integrate: index_b_ = " << memory->index_b_ << std::endl;
-    } else {
-      std::cout << "backward not yet initialized" << std::endl;
+      std::cout << "backward integrate: index_b_ = " << memory->index_b_ <<
+    std::endl; } else { std::cout << "backward not yet initialized" <<
+    std::endl;
     }
     */
 
@@ -741,15 +766,18 @@ class cvodes_integrator_adjoint_vari : public vari {
       }
 
       double t_final = value_of((i > 0) ? memory->ts_[i - 1] : memory->t0_);
-      //std::cout << "backward: time-point " << i << "; t_init = " << t_init << "; t_final = " << t_final << std::endl;
+      // std::cout << "backward: time-point " << i << "; t_init = " << t_init <<
+      // "; t_final = " << t_final << std::endl;
       if (t_final != t_init) {
         if (!memory->backward_is_initialized_) {
-          check_flag_sundials(CVodeCreateB(memory->cvodes_mem_, lmm_b_, &memory->index_b_),
-                              "CVodeCreateB");
+          check_flag_sundials(
+              CVodeCreateB(memory->cvodes_mem_, lmm_b_, &memory->index_b_),
+              "CVodeCreateB");
 
-          check_flag_sundials(CVodeSetUserDataB(memory->cvodes_mem_, memory->index_b_,
-                                                reinterpret_cast<void*>(this)),
-                              "CVodeSetUserDataB");
+          check_flag_sundials(
+              CVodeSetUserDataB(memory->cvodes_mem_, memory->index_b_,
+                                reinterpret_cast<void*>(this)),
+              "CVodeSetUserDataB");
 
           // initialize CVODES backward machinery.
           // the states of the backward problem *are* the adjoints
@@ -761,60 +789,64 @@ class cvodes_integrator_adjoint_vari : public vari {
               "CVodeInitB");
 
           check_flag_sundials(
-              CVodeSVtolerancesB(memory->cvodes_mem_, memory->index_b_, relative_tolerance_backward_,
+              CVodeSVtolerancesB(memory->cvodes_mem_, memory->index_b_,
+                                 relative_tolerance_backward_,
                                  memory->nv_absolute_tolerance_backward_),
               "CVodeSVtolerancesB");
 
-          check_flag_sundials(CVodeSetMaxNumStepsB(memory->cvodes_mem_,
-                                                   memory->index_b_, max_num_steps_),
-                              "CVodeSetMaxNumStepsB");
+          check_flag_sundials(
+              CVodeSetMaxNumStepsB(memory->cvodes_mem_, memory->index_b_,
+                                   max_num_steps_),
+              "CVodeSetMaxNumStepsB");
 
           check_flag_sundials(
-              CVodeSetLinearSolverB(memory->cvodes_mem_, memory->index_b_, memory->LS_b_, memory->A_b_),
+              CVodeSetLinearSolverB(memory->cvodes_mem_, memory->index_b_,
+                                    memory->LS_b_, memory->A_b_),
               "CVodeSetLinearSolverB");
-          
+
           check_flag_sundials(
-              CVodeSetJacFnB(
-                  memory->cvodes_mem_, memory->index_b_,
-                  &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
+              CVodeSetJacFnB(memory->cvodes_mem_, memory->index_b_,
+                             &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
               "CVodeSetJacFnB");
 
           // Allocate space for backwards quadrature needed when
           // parameters vary.
           if (args_vars_ > 0) {
             check_flag_sundials(
-                CVodeQuadInitB(
-                    memory->cvodes_mem_, memory->index_b_,
-                    &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj,
-                    memory->nv_quad_),
+                CVodeQuadInitB(memory->cvodes_mem_, memory->index_b_,
+                               &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj,
+                               memory->nv_quad_),
                 "CVodeQuadInitB");
-            
+
             check_flag_sundials(
                 CVodeQuadSStolerancesB(memory->cvodes_mem_, memory->index_b_,
-                                       relative_tolerance_quadrature_, absolute_tolerance_quadrature_),
+                                       relative_tolerance_quadrature_,
+                                       absolute_tolerance_quadrature_),
                 "CVodeQuadSStolerancesB");
-            
-            check_flag_sundials(
-                CVodeSetQuadErrConB(memory->cvodes_mem_, memory->index_b_, SUNTRUE),
-                "CVodeSetQuadErrConB");
+
+            check_flag_sundials(CVodeSetQuadErrConB(memory->cvodes_mem_,
+                                                    memory->index_b_, SUNTRUE),
+                                "CVodeSetQuadErrConB");
           }
 
           memory->backward_is_initialized_ = true;
         } else {
           // just re-initialize the solver
-          check_flag_sundials(CVodeReInitB(memory->cvodes_mem_, memory->index_b_,
-                                           t_init, memory->nv_state_sens_),
-                              "CVodeReInitB");
+          check_flag_sundials(
+              CVodeReInitB(memory->cvodes_mem_, memory->index_b_, t_init,
+                           memory->nv_state_sens_),
+              "CVodeReInitB");
 
           if (args_vars_ > 0) {
             check_flag_sundials(
-                CVodeQuadReInitB(memory->cvodes_mem_, memory->index_b_, memory->nv_quad_),
+                CVodeQuadReInitB(memory->cvodes_mem_, memory->index_b_,
+                                 memory->nv_quad_),
                 "CVodeQuadReInitB");
           }
         }
 
         int error_code = CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL);
-        
+
         if (error_code == CV_TOO_MUCH_WORK) {
           throw_domain_error(function_name_, "", t_final,
                              "Failed to integrate backward to output time (",
@@ -826,16 +858,17 @@ class cvodes_integrator_adjoint_vari : public vari {
         // check_flag_sundials(CVodeB(memory->cvodes_mem_, t_final,
         // CV_NORMAL),
         //                    "CVodeB");
-        
+
         // obtain adjoint states and update t_init to time point
         // reached of t_final
-        check_flag_sundials(
-            CVodeGetB(memory->cvodes_mem_, memory->index_b_, &t_init, memory->nv_state_sens_),
-            "CVodeGetB");
+        check_flag_sundials(CVodeGetB(memory->cvodes_mem_, memory->index_b_,
+                                      &t_init, memory->nv_state_sens_),
+                            "CVodeGetB");
 
         if (args_vars_ > 0) {
           check_flag_sundials(
-              CVodeGetQuadB(memory->cvodes_mem_, memory->index_b_, &t_init, memory->nv_quad_),
+              CVodeGetQuadB(memory->cvodes_mem_, memory->index_b_, &t_init,
+                            memory->nv_quad_),
               "CVodeGetQuadB");
         }
       }
@@ -853,7 +886,7 @@ class cvodes_integrator_adjoint_vari : public vari {
         memory->value_of_args_tuple_);
       */
     }
-    
+
     // do we need this a 2nd time? Don't think so.
     /*
       if (args_vars_ > 0) {
@@ -869,15 +902,15 @@ class cvodes_integrator_adjoint_vari : public vari {
     for (size_t s = 0; s < y0_vars_; s++) {
       y0_varis_[s]->adj_ += state_sens.coeff(s);
     }
-    
+
     // These are the dlog_density / d(parameters[s]) adjoints
     for (size_t s = 0; s < args_vars_; s++) {
       args_varis_[s]->adj_ += quad.coeff(s);
     }
 
-    //std::cout << "backward v2 integrate...done" << std::endl;
+    // std::cout << "backward v2 integrate...done" << std::endl;
   }
-  
+
 };  // cvodes integrator
 
 }  // namespace math
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index b05b6567f46..dee72acc80d 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -36,16 +36,22 @@ namespace math {
  * @param t0 Initial time
  * @param ts Times at which to solve the ODE at. All values must be sorted and
  *   not less than t0.
- * @param relative_tolerance_forward Relative tolerance for forward problem passed to CVODES
- * @param absolute_tolerance_forward Absolute tolerance for forward problem passed to CVODES
- * @param relative_tolerance_backward Relative tolerance for backward problem passed to CVODES
- * @param absolute_tolerance_backward Absolute tolerance for backward problem passed to CVODES
- * @param relative_tolerance_quadrature Relative tolerance for quadrature problem passed to CVODES
- * @param absolute_tolerance_quadrature Absolute tolerance for quadrature problem passed to CVODES
+ * @param relative_tolerance_forward Relative tolerance for forward problem
+ * passed to CVODES
+ * @param absolute_tolerance_forward Absolute tolerance for forward problem
+ * passed to CVODES
+ * @param relative_tolerance_backward Relative tolerance for backward problem
+ * passed to CVODES
+ * @param absolute_tolerance_backward Absolute tolerance for backward problem
+ * passed to CVODES
+ * @param relative_tolerance_quadrature Relative tolerance for quadrature
+ * problem passed to CVODES
+ * @param absolute_tolerance_quadrature Absolute tolerance for quadrature
+ * problem passed to CVODES
  * @param max_num_steps Upper limit on the number of integration steps to
  *   take between each output (error if exceeded)
- * @param num_steps_between_checkpoints Number of integrator steps after which a checkpoint is
- * stored for the backward pass
+ * @param num_steps_between_checkpoints Number of integrator steps after which a
+ * checkpoint is stored for the backward pass
  * @param interpolation_polynomial type of polynomial used for interpolation
  * @param solver_forward solver used for forward pass
  * @param solver_backward solver used for backward pass
@@ -58,12 +64,17 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
-                 const T_t0& t0, const std::vector<T_ts>& ts, double relative_tolerance_forward,
-                 Eigen::VectorXd absolute_tolerance_forward, double relative_tolerance_backward,
-                 Eigen::VectorXd absolute_tolerance_backward, double relative_tolerance_quadrature, double absolute_tolerance_quadrature,
-                 long int max_num_steps, long int num_steps_between_checkpoints,
-                 int interpolation_polynomial, int solver_forward, int solver_backward,
-                 std::ostream* msgs, const T_Args&... args) {
+                 const T_t0& t0, const std::vector<T_ts>& ts,
+                 double relative_tolerance_forward,
+                 Eigen::VectorXd absolute_tolerance_forward,
+                 double relative_tolerance_backward,
+                 Eigen::VectorXd absolute_tolerance_backward,
+                 double relative_tolerance_quadrature,
+                 double absolute_tolerance_quadrature, long int max_num_steps,
+                 long int num_steps_between_checkpoints,
+                 int interpolation_polynomial, int solver_forward,
+                 int solver_backward, std::ostream* msgs,
+                 const T_Args&... args) {
   /*
   const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
   return apply(
@@ -99,9 +110,12 @@ ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
   */
   auto integrator = new stan::math::cvodes_integrator_adjoint_vari<
       F, plain_type_t<T_y0>, T_t0, T_ts, ref_type_t<T_Args>...>(
-      function_name, f, eval(y0), t0, ts, relative_tolerance_forward, absolute_tolerance_forward, relative_tolerance_backward,
-      absolute_tolerance_backward, relative_tolerance_quadrature, absolute_tolerance_quadrature, max_num_steps, num_steps_between_checkpoints,
-      interpolation_polynomial, solver_forward, solver_backward, msgs, to_ref(args)...);
+      function_name, f, eval(y0), t0, ts, relative_tolerance_forward,
+      absolute_tolerance_forward, relative_tolerance_backward,
+      absolute_tolerance_backward, relative_tolerance_quadrature,
+      absolute_tolerance_quadrature, max_num_steps,
+      num_steps_between_checkpoints, interpolation_polynomial, solver_forward,
+      solver_backward, msgs, to_ref(args)...);
   return (*integrator)();
 }
 
@@ -142,18 +156,23 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
-ode_adjoint_tol_ctl(const F& f, const T_y0& y0, const T_t0& t0,
-                    const std::vector<T_ts>& ts, double relative_tolerance_forward,
-                    Eigen::VectorXd absolute_tolerance_forward, double relative_tolerance_backward,
-                    Eigen::VectorXd absolute_tolerance_backward, double relative_tolerance_quadrature,
-                    double absolute_tolerance_quadrature, long int max_num_steps,
-                    long int num_steps_between_checkpoints, int interpolation_polynomial,
-                    int solver_forward, int solver_backward, std::ostream* msgs,
-                    const T_Args&... args) {
+ode_adjoint_tol_ctl(
+    const F& f, const T_y0& y0, const T_t0& t0, const std::vector<T_ts>& ts,
+    double relative_tolerance_forward,
+    Eigen::VectorXd absolute_tolerance_forward,
+    double relative_tolerance_backward,
+    Eigen::VectorXd absolute_tolerance_backward,
+    double relative_tolerance_quadrature, double absolute_tolerance_quadrature,
+    long int max_num_steps, long int num_steps_between_checkpoints,
+    int interpolation_polynomial, int solver_forward, int solver_backward,
+    std::ostream* msgs, const T_Args&... args) {
   return ode_adjoint_impl(
-      "ode_adjoint_tol_ctl", f, y0, t0, ts, relative_tolerance_forward, absolute_tolerance_forward, relative_tolerance_backward,
-      absolute_tolerance_backward, relative_tolerance_quadrature, absolute_tolerance_quadrature, max_num_steps, num_steps_between_checkpoints,
-      interpolation_polynomial, solver_forward, solver_backward, msgs, args...);
+      "ode_adjoint_tol_ctl", f, y0, t0, ts, relative_tolerance_forward,
+      absolute_tolerance_forward, relative_tolerance_backward,
+      absolute_tolerance_backward, relative_tolerance_quadrature,
+      absolute_tolerance_quadrature, max_num_steps,
+      num_steps_between_checkpoints, interpolation_polynomial, solver_forward,
+      solver_backward, msgs, args...);
 }
 
 template <typename F, typename T_y0, typename T_t0, typename T_ts,
@@ -161,9 +180,9 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
-                const std::vector<T_ts>& ts, double relative_tolerance, double absolute_tolerance,
-                long int max_num_steps, std::ostream* msgs,
-                const T_Args&... args) {
+                const std::vector<T_ts>& ts, double relative_tolerance,
+                double absolute_tolerance, long int max_num_steps,
+                std::ostream* msgs, const T_Args&... args) {
   const int N = y0.size();
   const Eigen::VectorXd absolute_tolerance_forward
       = Eigen::VectorXd::Constant(N, absolute_tolerance / 100.0);
@@ -173,10 +192,12 @@ ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
   const int interpolation_polynomial = CV_HERMITE;
   const int solver_forward = CV_BDF;
   const int solver_backward = CV_BDF;
-  return ode_adjoint_impl("ode_adjoint_tol", f, y0, t0, ts, relative_tolerance, absolute_tolerance_forward,
-                          relative_tolerance, absolute_tolerance_backward, relative_tolerance, absolute_tolerance, max_num_steps,
-                          num_steps_between_checkpoints, interpolation_polynomial, solver_forward,
-                          solver_backward, msgs, args...);
+  return ode_adjoint_impl(
+      "ode_adjoint_tol", f, y0, t0, ts, relative_tolerance,
+      absolute_tolerance_forward, relative_tolerance,
+      absolute_tolerance_backward, relative_tolerance, absolute_tolerance,
+      max_num_steps, num_steps_between_checkpoints, interpolation_polynomial,
+      solver_forward, solver_backward, msgs, args...);
 }
 
 }  // namespace math
diff --git a/test/unit/math/rev/functor/cos_ode_typed_test.cpp b/test/unit/math/rev/functor/cos_ode_typed_test.cpp
index a0e32d2e01c..e72f749e277 100644
--- a/test/unit/math/rev/functor/cos_ode_typed_test.cpp
+++ b/test/unit/math/rev/functor/cos_ode_typed_test.cpp
@@ -16,9 +16,9 @@ using ode_test_tuple = std::tuple<solve_type, solve_type, Ts...>;
  * Outer product of test types
  */
 using cos_arg_test_types = boost::mp11::mp_product<
-    ode_test_tuple, ::testing::Types<ode_adams_functor, ode_bdf_functor,
-                                     ode_ckrk_functor, ode_rk45_functor,
-                                     ode_adjoint_functor> >;
+    ode_test_tuple,
+    ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
+                     ode_rk45_functor, ode_adjoint_functor> >;
 
 TYPED_TEST_SUITE_P(cos_arg_test);
 TYPED_TEST_P(cos_arg_test, y0_error) {
diff --git a/test/unit/math/rev/functor/lorenz_ode_typed_fd_test.cpp b/test/unit/math/rev/functor/lorenz_ode_typed_fd_test.cpp
index bf805630178..a979aeb0e42 100644
--- a/test/unit/math/rev/functor/lorenz_ode_typed_fd_test.cpp
+++ b/test/unit/math/rev/functor/lorenz_ode_typed_fd_test.cpp
@@ -16,9 +16,9 @@ using ode_test_tuple = std::tuple<solve_type, solve_type, Ts...>;
  * Outer product of test types
  */
 using lorenz_test_types = boost::mp11::mp_product<
-    ode_test_tuple, ::testing::Types<ode_adams_functor, ode_bdf_functor,
-                                     ode_ckrk_functor, ode_rk45_functor,
-                                     ode_adjoint_functor>>;
+    ode_test_tuple,
+    ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
+                     ode_rk45_functor, ode_adjoint_functor>>;
 
 TYPED_TEST_SUITE_P(lorenz_test);
 TYPED_TEST_P(lorenz_test, param_and_data_finite_diff) {
diff --git a/test/unit/math/rev/functor/ode_test_functors.hpp b/test/unit/math/rev/functor/ode_test_functors.hpp
index dab4cef68ad..9963b4a0ad5 100644
--- a/test/unit/math/rev/functor/ode_test_functors.hpp
+++ b/test/unit/math/rev/functor/ode_test_functors.hpp
@@ -63,30 +63,28 @@ STAN_DEF_STD_ODE_SOLVER_FUNCTOR(integrate_ode_rk45,
                                 stan::math::integrate_ode_rk45);
 
 struct ode_adjoint_functor {
-    const std::string functor_name = "ode_adjoint";
+  const std::string functor_name = "ode_adjoint";
 
-    template <typename F, typename T_y0, typename T_t0, typename T_ts,
-              typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr>
-    std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,
-                              Eigen::Dynamic, 1>>
-    operator()(const F& f, const T_y0& y0, T_t0 t0,
-               const std::vector<T_ts>& ts, std::ostream* msgs,
-               const Args&... args) {
-      return stan::math::ode_adjoint_tol(f, y0, t0, ts, 1E-13, 1E-13, 1000000, msgs, args...);
-    }
+  template <typename F, typename T_y0, typename T_t0, typename T_ts,
+            typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr>
+  std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,
+                            Eigen::Dynamic, 1>>
+  operator()(const F& f, const T_y0& y0, T_t0 t0, const std::vector<T_ts>& ts,
+             std::ostream* msgs, const Args&... args) {
+    return stan::math::ode_adjoint_tol(f, y0, t0, ts, 1E-13, 1E-13, 1000000,
+                                       msgs, args...);
+  }
 
-    template <typename F, typename T_y0, typename T_t0, typename T_ts,
-              typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr>
-    std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,
-                              Eigen::Dynamic, 1>>
-    operator()(const F& f, const T_y0& y0_arg, T_t0 t0,
-               const std::vector<T_ts>& ts, double rtol, double atol,
-               size_t max_num_steps, std::ostream* msgs,
-               const Args&... args) {
-      return stan::math::ode_adjoint_tol(f, y0_arg, t0, ts, rtol, atol, max_num_steps,
-                               msgs, args...);
-    }
+  template <typename F, typename T_y0, typename T_t0, typename T_ts,
+            typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr>
+  std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,
+                            Eigen::Dynamic, 1>>
+  operator()(const F& f, const T_y0& y0_arg, T_t0 t0,
+             const std::vector<T_ts>& ts, double rtol, double atol,
+             size_t max_num_steps, std::ostream* msgs, const Args&... args) {
+    return stan::math::ode_adjoint_tol(f, y0_arg, t0, ts, rtol, atol,
+                                       max_num_steps, msgs, args...);
+  }
 };
 
-
 #endif
diff --git a/test/unit/math/rev/functor/sho_ode_typed_test.cpp b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
index 084408780c5..ed2eb069b9f 100644
--- a/test/unit/math/rev/functor/sho_ode_typed_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
@@ -18,8 +18,8 @@ using ode_test_tuple = std::tuple<solve_type, solve_type, Ts...>;
  */
 using harmonic_oscillator_fd_test_types = boost::mp11::mp_product<
     ode_test_tuple,
-  ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
-                   ode_rk45_functor, ode_adjoint_functor>,
+    ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
+                     ode_rk45_functor, ode_adjoint_functor>,
     ::testing::Types<double>,  // t
     ::testing::Types<double>,  // y0
     ::testing::Types<double>   // theta
@@ -89,9 +89,9 @@ INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_data_test,
 
 using harmonic_oscillator_test_types = boost::mp11::mp_product<
     ode_test_tuple,
-  ::testing::Types<//ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
-    //ode_rk45_functor,
-                     ode_adjoint_functor>,
+    ::testing::Types<  // ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
+                       // ode_rk45_functor,
+        ode_adjoint_functor>,
     ::testing::Types<double>,                                  // t
     ::testing::Types<double, stan::math::var_value<double> >,  // y0
     ::testing::Types<double, stan::math::var_value<double> >   // theta

From c2e9e3990275a1aeab5e8b7a3212ddd5759cb10e Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Fri, 16 Apr 2021 14:17:28 +0200
Subject: [PATCH 077/157] cpplint

---
 .../math/rev/functor/cvodes_integrator_adjoint.hpp | 14 ++++++++------
 stan/math/rev/functor/ode_adjoint.hpp              | 12 ++++++++----
 test/unit/math/rev/functor/ode_test_functors.hpp   |  2 +-
 test/unit/math/rev/functor/sho_ode_typed_test.cpp  |  5 ++---
 4 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 49206e47467..3f724759702 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -9,6 +9,7 @@
 #include <stan/math/prim/err.hpp>
 #include <stan/math/prim/fun/value_of.hpp>
 #include <cvodes/cvodes.h>
+#include <nvector/nvector_serial.h>
 #include <sunmatrix/sunmatrix_dense.h>
 #include <sunlinsol/sunlinsol_dense.h>
 #include <algorithm>
@@ -205,8 +206,8 @@ class cvodes_integrator_adjoint_vari : public vari {
   Eigen::VectorXd absolute_tolerance_backward_;
   double relative_tolerance_quadrature_;
   double absolute_tolerance_quadrature_;
-  long int max_num_steps_;
-  long int num_steps_between_checkpoints_;
+  long int max_num_steps_; // NOLINT(runtime/int)
+  long int num_steps_between_checkpoints_; // NOLINT(runtime/int)
   int interpolation_polynomial_;
   int solver_forward_;
   int solver_backward_;
@@ -464,7 +465,9 @@ class cvodes_integrator_adjoint_vari : public vari {
       const char* function_name, const F& f, const T_y0& y0, const T_t0& t0,
       const std::vector<T_ts>& ts, double relative_tolerance_forward, Eigen::VectorXd absolute_tolerance_forward,
       double relative_tolerance_backward, Eigen::VectorXd absolute_tolerance_backward, double relative_tolerance_quadrature,
-      double absolute_tolerance_quadrature, long int max_num_steps, long int num_steps_between_checkpoints,
+      double absolute_tolerance_quadrature,
+      long int max_num_steps, // NOLINT(runtime/int)
+      long int num_steps_between_checkpoints, // NOLINT(runtime/int)
       int interpolation_polynomial, int solver_forward, int solver_backward,
       std::ostream* msgs, const T_Args&... args)
       : function_name_(function_name),
@@ -528,7 +531,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_nonzero_size(fun, "initial state", y0);
     check_sorted(fun, "times", ts);
     check_less(fun, "initial time", t0, ts[0]);
-    // TODO (SW): make names verbose
+    //TODO(wds15): make names verbose
     check_positive_finite(fun, "relative_tolerance_f", relative_tolerance_forward_);
     check_positive_finite(fun, "absolute_tolerance_f", absolute_tolerance_forward_);
     check_size_match(fun, "absolute_tolerance_forward", absolute_tolerance_forward_.size(), "states", N_);
@@ -538,7 +541,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_positive_finite(fun, "relative_tolerance_quadrature", relative_tolerance_quadrature_);
     check_positive_finite(fun, "absolute_tolerance_quadrature", absolute_tolerance_quadrature_);
     check_positive(fun, "max_num_steps", max_num_steps_);
-    // TODO (SW): rename according to design
+    // TODO(wds15): rename according to design
     check_positive(fun, "num_steps_between_checkpoints", num_steps_between_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
     check_range(fun, "interpolation_polynomial", 2, interpolation_polynomial_);
@@ -877,7 +880,6 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     //std::cout << "backward v2 integrate...done" << std::endl;
   }
-  
 };  // cvodes integrator
 
 }  // namespace math
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index b05b6567f46..957475305b4 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -61,7 +61,8 @@ ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
                  const T_t0& t0, const std::vector<T_ts>& ts, double relative_tolerance_forward,
                  Eigen::VectorXd absolute_tolerance_forward, double relative_tolerance_backward,
                  Eigen::VectorXd absolute_tolerance_backward, double relative_tolerance_quadrature, double absolute_tolerance_quadrature,
-                 long int max_num_steps, long int num_steps_between_checkpoints,
+                 long int max_num_steps, // NOLINT(runtime/int)
+                 long int num_steps_between_checkpoints, // NOLINT(runtime/int)
                  int interpolation_polynomial, int solver_forward, int solver_backward,
                  std::ostream* msgs, const T_Args&... args) {
   /*
@@ -146,8 +147,10 @@ ode_adjoint_tol_ctl(const F& f, const T_y0& y0, const T_t0& t0,
                     const std::vector<T_ts>& ts, double relative_tolerance_forward,
                     Eigen::VectorXd absolute_tolerance_forward, double relative_tolerance_backward,
                     Eigen::VectorXd absolute_tolerance_backward, double relative_tolerance_quadrature,
-                    double absolute_tolerance_quadrature, long int max_num_steps,
-                    long int num_steps_between_checkpoints, int interpolation_polynomial,
+                    double absolute_tolerance_quadrature,
+                    long int max_num_steps, // NOLINT(runtime/int)
+                    long int num_steps_between_checkpoints, // NOLINT(runtime/int)
+                    int interpolation_polynomial,
                     int solver_forward, int solver_backward, std::ostream* msgs,
                     const T_Args&... args) {
   return ode_adjoint_impl(
@@ -162,7 +165,8 @@ std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
                 const std::vector<T_ts>& ts, double relative_tolerance, double absolute_tolerance,
-                long int max_num_steps, std::ostream* msgs,
+                long int max_num_steps, // NOLINT(runtime/int)
+                std::ostream* msgs,
                 const T_Args&... args) {
   const int N = y0.size();
   const Eigen::VectorXd absolute_tolerance_forward
diff --git a/test/unit/math/rev/functor/ode_test_functors.hpp b/test/unit/math/rev/functor/ode_test_functors.hpp
index dab4cef68ad..7a8c2e15ca8 100644
--- a/test/unit/math/rev/functor/ode_test_functors.hpp
+++ b/test/unit/math/rev/functor/ode_test_functors.hpp
@@ -72,7 +72,7 @@ struct ode_adjoint_functor {
     operator()(const F& f, const T_y0& y0, T_t0 t0,
                const std::vector<T_ts>& ts, std::ostream* msgs,
                const Args&... args) {
-      return stan::math::ode_adjoint_tol(f, y0, t0, ts, 1E-13, 1E-13, 1000000, msgs, args...);
+      return stan::math::ode_adjoint_tol(f, y0, t0, ts, 1E-11, 1E-10, 1000000, msgs, args...);
     }
 
     template <typename F, typename T_y0, typename T_t0, typename T_ts,
diff --git a/test/unit/math/rev/functor/sho_ode_typed_test.cpp b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
index 084408780c5..cf86347bd6c 100644
--- a/test/unit/math/rev/functor/sho_ode_typed_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
@@ -89,9 +89,8 @@ INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_data_test,
 
 using harmonic_oscillator_test_types = boost::mp11::mp_product<
     ode_test_tuple,
-  ::testing::Types<//ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
-    //ode_rk45_functor,
-                     ode_adjoint_functor>,
+  ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
+                   ode_rk45_functor, ode_adjoint_functor>,
     ::testing::Types<double>,                                  // t
     ::testing::Types<double, stan::math::var_value<double> >,  // y0
     ::testing::Types<double, stan::math::var_value<double> >   // theta

From 9390361a9a28ed31338842d5ebd7f1fc37696eda Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Fri, 16 Apr 2021 12:30:41 +0000
Subject: [PATCH 078/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 44 ++++++++++-------
 stan/math/rev/functor/ode_adjoint.hpp         | 48 +++++++++++--------
 .../math/rev/functor/ode_test_functors.hpp    | 18 +++----
 .../math/rev/functor/sho_ode_typed_test.cpp   |  4 +-
 4 files changed, 66 insertions(+), 48 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 6f4fb984a09..0c71dd6bfd4 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -207,8 +207,8 @@ class cvodes_integrator_adjoint_vari : public vari {
   Eigen::VectorXd absolute_tolerance_backward_;
   double relative_tolerance_quadrature_;
   double absolute_tolerance_quadrature_;
-  long int max_num_steps_; // NOLINT(runtime/int)
-  long int num_steps_between_checkpoints_; // NOLINT(runtime/int)
+  long int max_num_steps_;                  // NOLINT(runtime/int)
+  long int num_steps_between_checkpoints_;  // NOLINT(runtime/int)
   int interpolation_polynomial_;
   int solver_forward_;
   int solver_backward_;
@@ -472,11 +472,14 @@ class cvodes_integrator_adjoint_vari : public vari {
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
   cvodes_integrator_adjoint_vari(
       const char* function_name, const F& f, const T_y0& y0, const T_t0& t0,
-      const std::vector<T_ts>& ts, double relative_tolerance_forward, Eigen::VectorXd absolute_tolerance_forward,
-      double relative_tolerance_backward, Eigen::VectorXd absolute_tolerance_backward, double relative_tolerance_quadrature,
+      const std::vector<T_ts>& ts, double relative_tolerance_forward,
+      Eigen::VectorXd absolute_tolerance_forward,
+      double relative_tolerance_backward,
+      Eigen::VectorXd absolute_tolerance_backward,
+      double relative_tolerance_quadrature,
       double absolute_tolerance_quadrature,
-      long int max_num_steps, // NOLINT(runtime/int)
-      long int num_steps_between_checkpoints, // NOLINT(runtime/int)
+      long int max_num_steps,                  // NOLINT(runtime/int)
+      long int num_steps_between_checkpoints,  // NOLINT(runtime/int)
       int interpolation_polynomial, int solver_forward, int solver_backward,
       std::ostream* msgs, const T_Args&... args)
       : function_name_(function_name),
@@ -541,18 +544,27 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_nonzero_size(fun, "initial state", y0);
     check_sorted(fun, "times", ts);
     check_less(fun, "initial time", t0, ts[0]);
-    //TODO(wds15): make names verbose
-    check_positive_finite(fun, "relative_tolerance_f", relative_tolerance_forward_);
-    check_positive_finite(fun, "absolute_tolerance_f", absolute_tolerance_forward_);
-    check_size_match(fun, "absolute_tolerance_forward", absolute_tolerance_forward_.size(), "states", N_);
-    check_positive_finite(fun, "relative_tolerance_backward", relative_tolerance_backward_);
-    check_positive_finite(fun, "absolute_tolerance_backward", absolute_tolerance_backward_);
-    check_size_match(fun, "absolute_tolerance_backward", absolute_tolerance_backward_.size(), "states", N_);
-    check_positive_finite(fun, "relative_tolerance_quadrature", relative_tolerance_quadrature_);
-    check_positive_finite(fun, "absolute_tolerance_quadrature", absolute_tolerance_quadrature_);
+    // TODO(wds15): make names verbose
+    check_positive_finite(fun, "relative_tolerance_f",
+                          relative_tolerance_forward_);
+    check_positive_finite(fun, "absolute_tolerance_f",
+                          absolute_tolerance_forward_);
+    check_size_match(fun, "absolute_tolerance_forward",
+                     absolute_tolerance_forward_.size(), "states", N_);
+    check_positive_finite(fun, "relative_tolerance_backward",
+                          relative_tolerance_backward_);
+    check_positive_finite(fun, "absolute_tolerance_backward",
+                          absolute_tolerance_backward_);
+    check_size_match(fun, "absolute_tolerance_backward",
+                     absolute_tolerance_backward_.size(), "states", N_);
+    check_positive_finite(fun, "relative_tolerance_quadrature",
+                          relative_tolerance_quadrature_);
+    check_positive_finite(fun, "absolute_tolerance_quadrature",
+                          absolute_tolerance_quadrature_);
     check_positive(fun, "max_num_steps", max_num_steps_);
     // TODO(wds15): rename according to design
-    check_positive(fun, "num_steps_between_checkpoints", num_steps_between_checkpoints_);
+    check_positive(fun, "num_steps_between_checkpoints",
+                   num_steps_between_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
     check_range(fun, "interpolation_polynomial", 2, interpolation_polynomial_);
     // 1=Adams, 2=BDF
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index f51a386a4b6..8bed5b7433f 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -64,13 +64,18 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
-                 const T_t0& t0, const std::vector<T_ts>& ts, double relative_tolerance_forward,
-                 Eigen::VectorXd absolute_tolerance_forward, double relative_tolerance_backward,
-                 Eigen::VectorXd absolute_tolerance_backward, double relative_tolerance_quadrature, double absolute_tolerance_quadrature,
-                 long int max_num_steps, // NOLINT(runtime/int)
-                 long int num_steps_between_checkpoints, // NOLINT(runtime/int)
-                 int interpolation_polynomial, int solver_forward, int solver_backward,
-                 std::ostream* msgs, const T_Args&... args) {
+                 const T_t0& t0, const std::vector<T_ts>& ts,
+                 double relative_tolerance_forward,
+                 Eigen::VectorXd absolute_tolerance_forward,
+                 double relative_tolerance_backward,
+                 Eigen::VectorXd absolute_tolerance_backward,
+                 double relative_tolerance_quadrature,
+                 double absolute_tolerance_quadrature,
+                 long int max_num_steps,                  // NOLINT(runtime/int)
+                 long int num_steps_between_checkpoints,  // NOLINT(runtime/int)
+                 int interpolation_polynomial, int solver_forward,
+                 int solver_backward, std::ostream* msgs,
+                 const T_Args&... args) {
   /*
   const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
   return apply(
@@ -152,16 +157,17 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
-ode_adjoint_tol_ctl(const F& f, const T_y0& y0, const T_t0& t0,
-                    const std::vector<T_ts>& ts, double relative_tolerance_forward,
-                    Eigen::VectorXd absolute_tolerance_forward, double relative_tolerance_backward,
-                    Eigen::VectorXd absolute_tolerance_backward, double relative_tolerance_quadrature,
-                    double absolute_tolerance_quadrature,
-                    long int max_num_steps, // NOLINT(runtime/int)
-                    long int num_steps_between_checkpoints, // NOLINT(runtime/int)
-                    int interpolation_polynomial,
-                    int solver_forward, int solver_backward, std::ostream* msgs,
-                    const T_Args&... args) {
+ode_adjoint_tol_ctl(
+    const F& f, const T_y0& y0, const T_t0& t0, const std::vector<T_ts>& ts,
+    double relative_tolerance_forward,
+    Eigen::VectorXd absolute_tolerance_forward,
+    double relative_tolerance_backward,
+    Eigen::VectorXd absolute_tolerance_backward,
+    double relative_tolerance_quadrature, double absolute_tolerance_quadrature,
+    long int max_num_steps,                  // NOLINT(runtime/int)
+    long int num_steps_between_checkpoints,  // NOLINT(runtime/int)
+    int interpolation_polynomial, int solver_forward, int solver_backward,
+    std::ostream* msgs, const T_Args&... args) {
   return ode_adjoint_impl(
       "ode_adjoint_tol_ctl", f, y0, t0, ts, relative_tolerance_forward,
       absolute_tolerance_forward, relative_tolerance_backward,
@@ -176,10 +182,10 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
 std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                           Eigen::Dynamic, 1>>
 ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
-                const std::vector<T_ts>& ts, double relative_tolerance, double absolute_tolerance,
-                long int max_num_steps, // NOLINT(runtime/int)
-                std::ostream* msgs,
-                const T_Args&... args) {
+                const std::vector<T_ts>& ts, double relative_tolerance,
+                double absolute_tolerance,
+                long int max_num_steps,  // NOLINT(runtime/int)
+                std::ostream* msgs, const T_Args&... args) {
   const int N = y0.size();
   const Eigen::VectorXd absolute_tolerance_forward
       = Eigen::VectorXd::Constant(N, absolute_tolerance / 100.0);
diff --git a/test/unit/math/rev/functor/ode_test_functors.hpp b/test/unit/math/rev/functor/ode_test_functors.hpp
index 3a3c932176a..01cc7dd4e68 100644
--- a/test/unit/math/rev/functor/ode_test_functors.hpp
+++ b/test/unit/math/rev/functor/ode_test_functors.hpp
@@ -65,15 +65,15 @@ STAN_DEF_STD_ODE_SOLVER_FUNCTOR(integrate_ode_rk45,
 struct ode_adjoint_functor {
   const std::string functor_name = "ode_adjoint";
 
-    template <typename F, typename T_y0, typename T_t0, typename T_ts,
-              typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr>
-    std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,
-                              Eigen::Dynamic, 1>>
-    operator()(const F& f, const T_y0& y0, T_t0 t0,
-               const std::vector<T_ts>& ts, std::ostream* msgs,
-               const Args&... args) {
-      return stan::math::ode_adjoint_tol(f, y0, t0, ts, 1E-11, 1E-10, 1000000, msgs, args...);
-    }
+  template <typename F, typename T_y0, typename T_t0, typename T_ts,
+            typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr>
+  std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,
+                            Eigen::Dynamic, 1>>
+  operator()(const F& f, const T_y0& y0, T_t0 t0, const std::vector<T_ts>& ts,
+             std::ostream* msgs, const Args&... args) {
+    return stan::math::ode_adjoint_tol(f, y0, t0, ts, 1E-11, 1E-10, 1000000,
+                                       msgs, args...);
+  }
 
   template <typename F, typename T_y0, typename T_t0, typename T_ts,
             typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr>
diff --git a/test/unit/math/rev/functor/sho_ode_typed_test.cpp b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
index 197bdac430a..9d8b56f4855 100644
--- a/test/unit/math/rev/functor/sho_ode_typed_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
@@ -89,8 +89,8 @@ INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_data_test,
 
 using harmonic_oscillator_test_types = boost::mp11::mp_product<
     ode_test_tuple,
-  ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
-                   ode_rk45_functor, ode_adjoint_functor>,
+    ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
+                     ode_rk45_functor, ode_adjoint_functor>,
     ::testing::Types<double>,                                  // t
     ::testing::Types<double, stan::math::var_value<double> >,  // y0
     ::testing::Types<double, stan::math::var_value<double> >   // theta

From 527fa2e33742f48d50a344e98b17014e8c19a8ae Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Fri, 16 Apr 2021 16:22:53 +0200
Subject: [PATCH 079/157] cpplint

---
 stan/math/rev/functor/ode_adjoint.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 8bed5b7433f..69822730d6e 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -191,7 +191,7 @@ ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
       = Eigen::VectorXd::Constant(N, absolute_tolerance / 100.0);
   const Eigen::VectorXd absolute_tolerance_backward
       = Eigen::VectorXd::Constant(N, absolute_tolerance / 10.0);
-  const long int num_steps_between_checkpoints = 250;
+  const long int num_steps_between_checkpoints = 250; // NOLINT(runtime/int)
   const int interpolation_polynomial = CV_HERMITE;
   const int solver_forward = CV_BDF;
   const int solver_backward = CV_BDF;

From 437741141656bf0c3071a17df3e313a81c649041 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Fri, 16 Apr 2021 16:29:22 +0000
Subject: [PATCH 080/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/ode_adjoint.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 69822730d6e..4f575b014e2 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -191,7 +191,7 @@ ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
       = Eigen::VectorXd::Constant(N, absolute_tolerance / 100.0);
   const Eigen::VectorXd absolute_tolerance_backward
       = Eigen::VectorXd::Constant(N, absolute_tolerance / 10.0);
-  const long int num_steps_between_checkpoints = 250; // NOLINT(runtime/int)
+  const long int num_steps_between_checkpoints = 250;  // NOLINT(runtime/int)
   const int interpolation_polynomial = CV_HERMITE;
   const int solver_forward = CV_BDF;
   const int solver_backward = CV_BDF;

From 3e2e653ea7464385d2ad62b162185b83a96a753c Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 19 Apr 2021 21:18:44 +0200
Subject: [PATCH 081/157] lint

---
 test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
index 7fe9ee6e1f3..43d760d0d3e 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
@@ -98,7 +98,6 @@ void run_benchmark(std::size_t system_size, int adjoint_integrator) {
         target += stan::math::sum(y[k]);
 
       target.grad();
-
     } catch (std::exception& exc) {
       std::cout << "oops, keep going please!" << std::endl;
       std::cerr << exc.what() << std::endl;

From 8f4aa86da3b687c28097153e8eb3c0c731982414 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 22 Apr 2021 09:36:07 +0200
Subject: [PATCH 082/157] refactor adjoint solver to not use extra memory
 object

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 604 +++++++++---------
 stan/math/rev/functor/ode_adjoint.hpp         |  35 +-
 .../math/rev/functor/ode_test_functors.hpp    |  38 +-
 .../math/rev/functor/sho_ode_typed_test.cpp   |  10 +-
 4 files changed, 326 insertions(+), 361 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 0c71dd6bfd4..76cd29fc5f5 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -20,6 +20,7 @@
 namespace stan {
 namespace math {
 
+namespace internal {
 template <typename T_Return>
 inline std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> build_varis(
     vari* ode_vari, vari**& non_chaining_varis,
@@ -35,22 +36,26 @@ inline std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> build_varis<var>(
     return y_return;
   }
 
-  int N = y[0].size();
+  const int N = y[0].size();
 
   non_chaining_varis
       = ChainableStack::instance_->memalloc_.alloc_array<vari*>(y.size() * N);
 
   for (size_t i = 0; i < y.size(); ++i) {
+    y_return[i].resize(N);
     for (size_t j = 0; j < N; j++) {
-      non_chaining_varis[i * N + j] = new vari(y[i][j], false);
+      non_chaining_varis[i * N + j] = new vari(y[i].coeff(j), false);
+      y_return[i].coeffRef(j) = var(non_chaining_varis[i * N + j]);
     }
   }
 
+  /*
   for (size_t i = 0; i < y.size(); ++i) {
     y_return[i].resize(N);
     for (size_t j = 0; j < N; j++)
       y_return[i][j] = var(non_chaining_varis[i * N + j]);
   }
+  */
 
   return y_return;
 }
@@ -66,119 +71,8 @@ inline std::vector<Eigen::VectorXd> build_varis<double>(
   return y;
 }
 
-template <typename F, typename T_y0, typename T_t0, typename T_ts,
-          typename... T_Args>
-class cvodes_integrator_adjoint_vari;
-
-template <typename F, typename T_y0, typename T_t0, typename T_ts,
-          typename... T_Args>
-class cvodes_integrator_adjoint_memory : public chainable_alloc {
-  using T_Return = return_type_t<T_y0, T_t0, T_ts, T_Args...>;
-  using T_y0_t0 = return_type_t<T_y0, T_t0>;
-
-  const size_t N_;
-  Eigen::VectorXd absolute_tolerance_forward_;
-  Eigen::VectorXd absolute_tolerance_backward_;
-  const int lmm_f_;
-  const F f_;
-  const Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1> y0_;
-  const T_t0 t0_;
-  const std::vector<T_ts> ts_;
-  // std::tuple<T_Args...> args_tuple_;
-  std::tuple<
-      plain_type_t<decltype(deep_copy_vars(std::declval<const T_Args&>()))>...>
-      local_args_tuple_;
-  std::tuple<plain_type_t<decltype(value_of(std::declval<const T_Args&>()))>...>
-      value_of_args_tuple_;
-  std::vector<Eigen::VectorXd> y_;
-  void* cvodes_mem_;
-  Eigen::VectorXd state_;
-  Eigen::VectorXd state_sens_;
-  Eigen::VectorXd quad_;
-  N_Vector nv_state_;
-  N_Vector nv_state_sens_;
-  N_Vector nv_quad_;
-  N_Vector nv_absolute_tolerance_forward_;
-  N_Vector nv_absolute_tolerance_backward_;
-  SUNMatrix A_f_;
-  SUNLinearSolver LS_f_;
-  SUNMatrix A_b_;
-  SUNLinearSolver LS_b_;
-  int index_b_;
-  bool backward_is_initialized_;
-
-  template <require_eigen_col_vector_t<T_y0>* = nullptr>
-  cvodes_integrator_adjoint_memory(Eigen::VectorXd absolute_tolerance_forward,
-                                   Eigen::VectorXd absolute_tolerance_backward,
-                                   int lmm_f, const F& f, const T_y0& y0,
-                                   const T_t0& t0, const std::vector<T_ts>& ts,
-                                   const T_Args&... args)
-      : N_(y0.size()),
-        absolute_tolerance_forward_(absolute_tolerance_forward),
-        absolute_tolerance_backward_(absolute_tolerance_backward),
-        lmm_f_(lmm_f),
-        f_(f),
-        y0_(y0),
-        t0_(t0),
-        ts_(ts),
-        // args_tuple_(std::make_tuple(args...)),
-        local_args_tuple_(deep_copy_vars(args)...),
-        value_of_args_tuple_(value_of(args)...),
-        y_(ts_.size()),
-        cvodes_mem_(nullptr),
-        state_(value_of(y0)),
-        state_sens_(Eigen::VectorXd::Zero(N_)),
-        quad_(Eigen::VectorXd::Zero(count_vars(args...))),
-        backward_is_initialized_(false) {
-    if (N_ > 0) {
-      nv_state_ = N_VMake_Serial(N_, state_.data());
-      nv_state_sens_ = N_VMake_Serial(N_, state_sens_.data());
-      nv_quad_ = N_VMake_Serial(quad_.size(), quad_.data());
-      nv_absolute_tolerance_forward_
-          = N_VMake_Serial(N_, absolute_tolerance_forward_.data());
-      nv_absolute_tolerance_backward_
-          = N_VMake_Serial(N_, absolute_tolerance_backward_.data());
-      A_f_ = SUNDenseMatrix(N_, N_);
-      LS_f_ = SUNDenseLinearSolver(nv_state_, A_f_);
-      A_b_ = SUNDenseMatrix(N_, N_);
-      LS_b_ = SUNDenseLinearSolver(nv_state_sens_, A_b_);
-
-      cvodes_mem_ = CVodeCreate(lmm_f_);
-
-      if (cvodes_mem_ == nullptr) {
-        throw std::runtime_error("CVodeCreate failed to allocate memory");
-      }
-    }
-  }
-
-  ~cvodes_integrator_adjoint_memory() {
-    if (N_ > 0) {
-      SUNLinSolFree(LS_f_);
-      SUNMatDestroy(A_f_);
-      SUNLinSolFree(LS_b_);
-      SUNMatDestroy(A_b_);
-
-      N_VDestroy_Serial(nv_state_);
-      N_VDestroy_Serial(nv_state_sens_);
-      N_VDestroy_Serial(nv_quad_);
-      N_VDestroy_Serial(nv_absolute_tolerance_forward_);
-      N_VDestroy_Serial(nv_absolute_tolerance_backward_);
-
-      if (cvodes_mem_) {
-        CVodeFree(&cvodes_mem_);
-      }
-    }
-  }
-
-  template <typename yT, typename... ArgsT>
-  constexpr auto rhs(double t, const yT& y, std::ostream* msgs,
-                     const std::tuple<ArgsT...>& args_tuple) const {
-    return apply([&](auto&&... args) { return f_(t, y, msgs, args...); },
-                 args_tuple);
-  }
+}
 
-  friend class cvodes_integrator_adjoint_vari<F, T_y0, T_t0, T_ts, T_Args...>;
-};
 
 /**
  * Integrator interface for CVODES' ODE solvers (Adams & BDF
@@ -194,31 +88,50 @@ template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args>
 class cvodes_integrator_adjoint_vari : public vari {
   using T_Return = return_type_t<T_y0, T_t0, T_ts, T_Args...>;
+  using T_y0_t0 = return_type_t<T_y0, T_t0>;
 
   const char* function_name_;
 
-  const size_t N_;
-  bool returned_;
-  std::size_t chain_count_;
-  std::ostream* msgs_;
-  double relative_tolerance_forward_;
+  const F f_;
+  const Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1> y0_;
+  const T_t0 t0_;
+  const std::vector<T_ts> ts_;
+
+  const double relative_tolerance_forward_;
   Eigen::VectorXd absolute_tolerance_forward_;
-  double relative_tolerance_backward_;
+  const double relative_tolerance_backward_;
   Eigen::VectorXd absolute_tolerance_backward_;
-  double relative_tolerance_quadrature_;
-  double absolute_tolerance_quadrature_;
-  long int max_num_steps_;                  // NOLINT(runtime/int)
-  long int num_steps_between_checkpoints_;  // NOLINT(runtime/int)
-  int interpolation_polynomial_;
-  int solver_forward_;
-  int solver_backward_;
-  int lmm_f_;
-  int lmm_b_;
-
-  const size_t t0_vars_;
-  const size_t ts_vars_;
-  const size_t y0_vars_;
-  const size_t args_vars_;
+  const double relative_tolerance_quadrature_;
+  const double absolute_tolerance_quadrature_;
+  const long int max_num_steps_;                  // NOLINT(runtime/int)
+  const long int num_steps_between_checkpoints_;  // NOLINT(runtime/int)
+  const int interpolation_polynomial_;
+  const int solver_forward_;
+  const int solver_backward_;
+  
+  std::ostream* msgs_;
+
+  std::tuple<
+      plain_type_t<decltype(deep_copy_vars(std::declval<const T_Args&>()))>...>
+      local_args_tuple_;
+  std::tuple<plain_type_t<decltype(value_of(std::declval<const T_Args&>()))>...>
+      value_of_args_tuple_;
+
+  const size_t N_;
+  bool forward_returned_;
+  bool backward_is_initialized_;
+
+  const size_t num_t0_vars_;
+  const size_t num_ts_vars_;
+  const size_t num_y0_vars_;
+  const size_t num_args_vars_;
+  const size_t num_vars_;
+
+  Eigen::VectorXd state_forward_;
+  Eigen::VectorXd state_backward_;
+  Eigen::VectorXd quad_;
+
+  std::vector<Eigen::VectorXd> y_;
 
   vari** non_chaining_varis_;
 
@@ -227,15 +140,38 @@ class cvodes_integrator_adjoint_vari : public vari {
   vari** y0_varis_;
   vari** args_varis_;
 
-  cvodes_integrator_adjoint_memory<F, T_y0, T_t0, T_ts, T_Args...>* memory;
+  void* cvodes_mem_;
+  N_Vector nv_state_forward_;
+  N_Vector nv_state_backward_;
+  N_Vector nv_quad_;
+  N_Vector nv_absolute_tolerance_forward_;
+  N_Vector nv_absolute_tolerance_backward_;
+  
+  SUNMatrix A_forward_;
+  SUNLinearSolver LS_forward_;
+  
+  SUNMatrix A_backward_;
+  SUNLinearSolver LS_backward_;
+  
+  int index_backward_;
+
+  /**
+   * Call the ODE RHS with given tuple.
+   */
+  template <typename yT, typename... ArgsT>
+  constexpr auto rhs(double t, const yT& y, 
+                     const std::tuple<ArgsT...>& args_tuple) const {
+    return apply([&](auto&&... args) { return f_(t, y, msgs_, args...); },
+                 args_tuple);
+  }
 
   /**
    * Implements the function of type CVRhsFn which is the user-defined
    * ODE RHS passed to CVODES.
    */
   static int cv_rhs(realtype t, N_Vector y, N_Vector ydot, void* user_data) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+    cvodes_integrator_adjoint_vari* integrator
+        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
     integrator->rhs(t, NV_DATA_S(y), NV_DATA_S(ydot));
     return 0;
   }
@@ -246,8 +182,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    */
   static int cv_rhs_adj_sens(realtype t, N_Vector y, N_Vector yB,
                              N_Vector yBdot, void* user_data) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+    cvodes_integrator_adjoint_vari* integrator
+        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
     integrator->rhs_adj_sens(t, y, yB, yBdot);
     return 0;
   }
@@ -258,8 +194,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    */
   static int cv_quad_rhs_adj(realtype t, N_Vector y, N_Vector yB,
                              N_Vector qBdot, void* user_data) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+    cvodes_integrator_adjoint_vari* integrator
+        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
     integrator->quad_rhs_adj(t, y, yB, qBdot);
     return 0;
   }
@@ -273,8 +209,8 @@ class cvodes_integrator_adjoint_vari : public vari {
   static int cv_jacobian_states(realtype t, N_Vector y, N_Vector fy,
                                 SUNMatrix J, void* user_data, N_Vector tmp1,
                                 N_Vector tmp2, N_Vector tmp3) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+    cvodes_integrator_adjoint_vari* integrator
+        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_states(t, y, J);
     return 0;
   }
@@ -286,8 +222,8 @@ class cvodes_integrator_adjoint_vari : public vari {
   static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
                              SUNMatrix J, void* user_data, N_Vector tmp1,
                              N_Vector tmp2, N_Vector tmp3) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+     cvodes_integrator_adjoint_vari* integrator
+        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_adj(t, y, J);
     return 0;
   }
@@ -296,13 +232,13 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Calculates the ODE RHS, dy_dt, using the user-supplied functor at
    * the given time t and state y.
    */
-  inline void rhs(double t, const double y[], double dy_dt[]) const {
+  inline void rhs(double t, const double y[], double dy_dt[]) {
     const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
 
     const Eigen::VectorXd dy_dt_vec
-        = memory->rhs(t, y_vec, msgs_, memory->value_of_args_tuple_);
+        = rhs(t, y_vec, value_of_args_tuple_);
 
-    check_size_match("cvodes_integrator::rhs", "dy_dt", dy_dt_vec.size(),
+    check_size_match(function_name_, "dy_dt", dy_dt_vec.size(),
                      "states", N_);
 
     Eigen::Map<Eigen::VectorXd>(dy_dt, N_) = dy_dt_vec;
@@ -321,7 +257,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] yB state of the adjoint ODE system
    * @param[out] yBdot evaluation of adjoint ODE RHS
    */
-  void rhs_adj_sens(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
+  void rhs_adj_sens(double t, N_Vector y, N_Vector yB, N_Vector yBdot) {
     Eigen::Map<Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
     Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(yBdot), N_);
@@ -332,9 +268,9 @@ class cvodes_integrator_adjoint_vari : public vari {
     Eigen::Matrix<var, Eigen::Dynamic, 1> y_vars(y_vec);
 
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
-        = memory->rhs(t, y_vars, msgs_, memory->value_of_args_tuple_);
+        = rhs(t, y_vars, value_of_args_tuple_);
 
-    check_size_match("coupled_ode_system1", "dy_dt", f_y_t_vars.size(),
+    check_size_match(function_name_, "dy_dt", f_y_t_vars.size(),
                      "states", N_);
 
     f_y_t_vars.adj() = -mu;
@@ -357,23 +293,23 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] yB state of the adjoint ODE system
    * @param[out] qBdot evaluation of adjoint ODE quadrature RHS
    */
-  void quad_rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector qBdot) const {
+  void quad_rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector qBdot) {
     const Eigen::VectorXd y_vec = Eigen::Map<Eigen::VectorXd>(NV_DATA_S(y), N_);
     Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
-    Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(qBdot), args_vars_);
-    mu_dot = Eigen::VectorXd::Zero(args_vars_);
+    Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(qBdot), num_args_vars_);
+    mu_dot = Eigen::VectorXd::Zero(num_args_vars_);
 
     const nested_rev_autodiff nested;
 
     // The vars here do not live on the nested stack so must be zero'd
     // separately
     apply([&](auto&&... args) { zero_adjoints(args...); },
-          memory->local_args_tuple_);
+          local_args_tuple_);
 
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
-        = memory->rhs(t, y_vec, msgs_, memory->local_args_tuple_);
+        = rhs(t, y_vec, local_args_tuple_);
 
-    check_size_match("coupled_ode_system2", "dy_dt", f_y_t_vars.size(),
+    check_size_match(function_name_, "dy_dt", f_y_t_vars.size(),
                      "states", N_);
 
     f_y_t_vars.adj() = -mu;
@@ -381,14 +317,14 @@ class cvodes_integrator_adjoint_vari : public vari {
     grad();
 
     apply([&](auto&&... args) { accumulate_adjoints(mu_dot.data(), args...); },
-          memory->local_args_tuple_);
+          local_args_tuple_);
   }
 
   /**
    * Calculates the jacobian of the ODE RHS wrt to its states y at the
    * given time-point t and state y.
    */
-  inline void jacobian_states(double t, N_Vector y, SUNMatrix J) const {
+  inline void jacobian_states(double t, N_Vector y, SUNMatrix J) {
     Eigen::Map<Eigen::MatrixXd> Jfy(SM_DATA_D(J), N_, N_);
     Eigen::Map<const Eigen::VectorXd> x(NV_DATA_S(y), N_);
 
@@ -396,9 +332,9 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     const Eigen::Matrix<var, Eigen::Dynamic, 1> y_var(x);
     Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var
-        = memory->rhs(t, y_var, msgs_, memory->value_of_args_tuple_);
+        = rhs(t, y_var, value_of_args_tuple_);
 
-    check_size_match("coupled_ode_system2", "dy_dt", fy_var.size(), "states",
+    check_size_match(function_name_, "dy_dt", fy_var.size(), "states",
                      N_);
 
     grad(fy_var.coeffRef(0).vi_);
@@ -419,7 +355,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] t Time
    * @param[out] J CVode structure where output is to be stored
    */
-  inline void jacobian_adj(double t, N_Vector y, SUNMatrix J) const {
+  inline void jacobian_adj(double t, N_Vector y, SUNMatrix J) {
     Eigen::Map<Eigen::MatrixXd> J_adj_y(SM_DATA_D(J), N_, N_);
 
     // J_adj_y = -1 * transpose(J_y)
@@ -472,111 +408,110 @@ class cvodes_integrator_adjoint_vari : public vari {
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
   cvodes_integrator_adjoint_vari(
       const char* function_name, const F& f, const T_y0& y0, const T_t0& t0,
-      const std::vector<T_ts>& ts, double relative_tolerance_forward,
-      Eigen::VectorXd absolute_tolerance_forward,
+      const std::vector<T_ts>& ts,
+      double relative_tolerance_forward,
+      const Eigen::VectorXd& absolute_tolerance_forward,
       double relative_tolerance_backward,
-      Eigen::VectorXd absolute_tolerance_backward,
+      const Eigen::VectorXd& absolute_tolerance_backward,
       double relative_tolerance_quadrature,
       double absolute_tolerance_quadrature,
       long int max_num_steps,                  // NOLINT(runtime/int)
       long int num_steps_between_checkpoints,  // NOLINT(runtime/int)
       int interpolation_polynomial, int solver_forward, int solver_backward,
       std::ostream* msgs, const T_Args&... args)
-      : function_name_(function_name),
-        vari(NOT_A_NUMBER),
-        N_(y0.size()),
-        returned_(false),
-        chain_count_(0),
-        memory(NULL),
-        relative_tolerance_forward_(relative_tolerance_forward),
-        absolute_tolerance_forward_(absolute_tolerance_forward),
-        relative_tolerance_backward_(relative_tolerance_backward),
-        absolute_tolerance_backward_(absolute_tolerance_backward),
-        relative_tolerance_quadrature_(relative_tolerance_quadrature),
-        absolute_tolerance_quadrature_(absolute_tolerance_quadrature),
-        max_num_steps_(max_num_steps),
-        num_steps_between_checkpoints_(num_steps_between_checkpoints),
-        interpolation_polynomial_(interpolation_polynomial),
-        solver_forward_(solver_forward),
-        solver_backward_(solver_backward),
-        lmm_f_(solver_forward_ % 2 ? CV_ADAMS : CV_BDF),
-        lmm_b_(solver_backward_ % 2 ? CV_ADAMS : CV_BDF),
-        msgs_(msgs),
-
-        t0_vars_(count_vars(t0)),
-        ts_vars_(count_vars(ts)),
-        y0_vars_(count_vars(y0)),
-        args_vars_(count_vars(args...)),
-        t0_varis_(
-            ChainableStack::instance_->memalloc_.alloc_array<vari*>(t0_vars_)),
-        ts_varis_(
-            ChainableStack::instance_->memalloc_.alloc_array<vari*>(ts_vars_)),
-        y0_varis_(
-            ChainableStack::instance_->memalloc_.alloc_array<vari*>(y0_vars_)),
-        args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
-            args_vars_)) {
-    const char* fun = "cvodes_integrator_adjoint";
-
-    memory
-        = new cvodes_integrator_adjoint_memory<F, T_y0, T_t0, T_ts, T_Args...>(
-            absolute_tolerance_forward_, absolute_tolerance_backward_, lmm_f_,
-            f, y0, t0, ts, args...);
+  : vari(NOT_A_NUMBER),
+    function_name_(function_name),
+    f_(f),
+    y0_(y0),
+    t0_(t0),
+    ts_(ts),
+    relative_tolerance_forward_(relative_tolerance_forward),
+    absolute_tolerance_forward_(absolute_tolerance_forward),
+    relative_tolerance_backward_(relative_tolerance_backward),
+    absolute_tolerance_backward_(absolute_tolerance_backward),
+    relative_tolerance_quadrature_(relative_tolerance_quadrature),
+    absolute_tolerance_quadrature_(absolute_tolerance_quadrature),
+    max_num_steps_(max_num_steps),
+    num_steps_between_checkpoints_(num_steps_between_checkpoints),
+    interpolation_polynomial_(interpolation_polynomial),
+    solver_forward_(solver_forward),
+    solver_backward_(solver_backward),
+    msgs_(msgs),
+    local_args_tuple_(deep_copy_vars(args)...),
+    value_of_args_tuple_(value_of(args)...),
+
+    N_(y0.size()),
+    forward_returned_(false),
+    backward_is_initialized_(false),
+
+    num_t0_vars_(count_vars(t0)),
+    num_ts_vars_(count_vars(ts)),
+    num_y0_vars_(count_vars(y0)),
+    num_args_vars_(count_vars(args...)),
+    num_vars_(num_t0_vars_ + num_ts_vars_ + num_y0_vars_ + num_args_vars_),
+
+    state_forward_(value_of(y0)),
+    state_backward_(Eigen::VectorXd::Zero(N_)),
+    quad_(Eigen::VectorXd::Zero(num_args_vars_)),
+
+    y_(ts_.size()),
+
+    non_chaining_varis_(nullptr),
+    t0_varis_(
+        ChainableStack::instance_->memalloc_.alloc_array<vari*>(num_t0_vars_)),
+    ts_varis_(
+        ChainableStack::instance_->memalloc_.alloc_array<vari*>(num_ts_vars_)),
+    y0_varis_(
+        ChainableStack::instance_->memalloc_.alloc_array<vari*>(num_y0_vars_)),
+    args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
+        num_args_vars_)) {
 
     save_varis(t0_varis_, t0);
     save_varis(ts_varis_, ts);
     save_varis(y0_varis_, y0);
     save_varis(args_varis_, args...);
 
-    check_finite(fun, "initial state", y0);
-    check_finite(fun, "initial time", t0);
-    check_finite(fun, "times", ts);
+    check_finite(function_name_, "initial state", y0);
+    check_finite(function_name_, "initial time", t0);
+    check_finite(function_name_, "times", ts);
 
     // Code from: https://stackoverflow.com/a/17340003 . Should probably do
     // something better
     apply(
         [&](auto&&... args) {
           std::vector<int> unused_temp{
-              0, (check_finite(fun, "ode parameters and data", args), 0)...};
+              0, (check_finite(function_name_, "ode parameters and data", args), 0)...};
         },
-        memory->local_args_tuple_);
-
-    check_nonzero_size(fun, "times", ts);
-    check_nonzero_size(fun, "initial state", y0);
-    check_sorted(fun, "times", ts);
-    check_less(fun, "initial time", t0, ts[0]);
-    // TODO(wds15): make names verbose
-    check_positive_finite(fun, "relative_tolerance_f",
+        local_args_tuple_);
+
+    check_nonzero_size(function_name_, "times", ts);
+    check_nonzero_size(function_name_, "initial state", y0);
+    check_sorted(function_name_, "times", ts);
+    check_less(function_name_, "initial time", t0, ts[0]);
+    check_positive_finite(function_name_, "relative_tolerance_forward",
                           relative_tolerance_forward_);
-    check_positive_finite(fun, "absolute_tolerance_f",
+    check_positive_finite(function_name_, "absolute_tolerance_forward",
                           absolute_tolerance_forward_);
-    check_size_match(fun, "absolute_tolerance_forward",
+    check_size_match(function_name_, "absolute_tolerance_forward",
                      absolute_tolerance_forward_.size(), "states", N_);
-    check_positive_finite(fun, "relative_tolerance_backward",
+    check_positive_finite(function_name_, "relative_tolerance_backward",
                           relative_tolerance_backward_);
-    check_positive_finite(fun, "absolute_tolerance_backward",
+    check_positive_finite(function_name_, "absolute_tolerance_backward",
                           absolute_tolerance_backward_);
-    check_size_match(fun, "absolute_tolerance_backward",
+    check_size_match(function_name_, "absolute_tolerance_backward",
                      absolute_tolerance_backward_.size(), "states", N_);
-    check_positive_finite(fun, "relative_tolerance_quadrature",
+    check_positive_finite(function_name_, "relative_tolerance_quadrature",
                           relative_tolerance_quadrature_);
-    check_positive_finite(fun, "absolute_tolerance_quadrature",
+    check_positive_finite(function_name_, "absolute_tolerance_quadrature",
                           absolute_tolerance_quadrature_);
-    check_positive(fun, "max_num_steps", max_num_steps_);
-    // TODO(wds15): rename according to design
-    check_positive(fun, "num_steps_between_checkpoints",
+    check_positive(function_name_, "max_num_steps", max_num_steps_);
+    check_positive(function_name_, "num_steps_between_checkpoints",
                    num_steps_between_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
-    check_range(fun, "interpolation_polynomial", 2, interpolation_polynomial_);
-    // 1=Adams, 2=BDF
-    check_range(fun, "solver_forward", 2, solver_forward_);
-    check_range(fun, "solver_backward", 2, solver_backward_);
-
-    /*
-    std::cout << "ts = ";
-    for(std::size_t i=0; i != ts.size(); i++)
-      std::cout << value_of(ts[i]) << ", ";
-    std::cout << std::endl;
-    */
+    check_range(function_name_, "interpolation_polynomial", 2, interpolation_polynomial_);
+    // 1=Adams=CV_ADAMS, 2=BDF=CV_BDF
+    check_range(function_name_, "solver_forward", 2, solver_forward_);
+    check_range(function_name_, "solver_backward", 2, solver_backward_);
 
     /*
     std::cout << "relative_tolerance = " << relative_tolerance << std::endl;
@@ -589,22 +524,44 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     // initializes CVODES solver
 
+    if (N_ > 0) {
+      nv_state_forward_ = N_VMake_Serial(N_, state_forward_.data());
+      nv_state_backward_ = N_VMake_Serial(N_, state_backward_.data());
+      nv_quad_ = N_VMake_Serial(num_args_vars_, quad_.data());
+      nv_absolute_tolerance_forward_
+          = N_VMake_Serial(N_, absolute_tolerance_forward_.data());
+      nv_absolute_tolerance_backward_
+          = N_VMake_Serial(N_, absolute_tolerance_backward_.data());
+      
+      A_forward_ = SUNDenseMatrix(N_, N_);
+      LS_forward_ = SUNDenseLinearSolver(nv_state_forward_, A_forward_);
+      
+      A_backward_ = SUNDenseMatrix(N_, N_);
+      LS_backward_ = SUNDenseLinearSolver(nv_state_backward_, A_backward_);
+
+      cvodes_mem_ = CVodeCreate(solver_forward_);
+
+      if (cvodes_mem_ == nullptr) {
+        throw std::runtime_error("CVodeCreate failed to allocate memory");
+      }
+    }
+
     check_flag_sundials(
-        CVodeInit(memory->cvodes_mem_, &cvodes_integrator_adjoint_vari::cv_rhs,
-                  value_of(memory->t0_), memory->nv_state_),
+        CVodeInit(cvodes_mem_, &cvodes_integrator_adjoint_vari::cv_rhs,
+                  value_of(t0_), nv_state_forward_),
         "CVodeInit");
 
     // Assign pointer to this as user data
     check_flag_sundials(
-        CVodeSetUserData(memory->cvodes_mem_, reinterpret_cast<void*>(this)),
+        CVodeSetUserData(cvodes_mem_, reinterpret_cast<void*>(this)),
         "CVodeSetUserData");
 
-    cvodes_set_options(memory->cvodes_mem_, relative_tolerance_forward_,
+    cvodes_set_options(cvodes_mem_, relative_tolerance_forward_,
                        absolute_tolerance_forward_(0), max_num_steps_);
 
     check_flag_sundials(
-        CVodeSVtolerances(memory->cvodes_mem_, relative_tolerance_forward_,
-                          memory->nv_absolute_tolerance_forward_),
+        CVodeSVtolerances(cvodes_mem_, relative_tolerance_forward_,
+                          nv_absolute_tolerance_forward_),
         "CVodeSVtolerances");
 
     // for the stiff solvers we need to reserve additional memory
@@ -612,24 +569,41 @@ class cvodes_integrator_adjoint_vari : public vari {
     // create matrix object and linear solver object; resource
     // (de-)allocation is handled in the cvodes_ode_data
     check_flag_sundials(
-        CVodeSetLinearSolver(memory->cvodes_mem_, memory->LS_f_, memory->A_f_),
+        CVodeSetLinearSolver(cvodes_mem_, LS_forward_, A_forward_),
         "CVodeSetLinearSolver");
 
     check_flag_sundials(
-        CVodeSetJacFn(memory->cvodes_mem_,
+        CVodeSetJacFn(cvodes_mem_,
                       &cvodes_integrator_adjoint_vari::cv_jacobian_states),
         "CVodeSetJacFn");
 
     // initialize backward sensitivity system of CVODES as needed
-    if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
+    if (num_vars_ != 0) {
       check_flag_sundials(
-          CVodeAdjInit(memory->cvodes_mem_, num_steps_between_checkpoints_,
+          CVodeAdjInit(cvodes_mem_, num_steps_between_checkpoints_,
                        interpolation_polynomial_),
           "CVodeAdjInit");
     }
   }
 
-  ~cvodes_integrator_adjoint_vari() {}
+  ~cvodes_integrator_adjoint_vari() {
+    if (N_ > 0) {
+      SUNLinSolFree(LS_forward_);
+      SUNMatDestroy(A_forward_);
+      SUNLinSolFree(LS_backward_);
+      SUNMatDestroy(A_backward_);
+
+      N_VDestroy_Serial(nv_state_forward_);
+      N_VDestroy_Serial(nv_state_backward_);
+      N_VDestroy_Serial(nv_quad_);
+      N_VDestroy_Serial(nv_absolute_tolerance_forward_);
+      N_VDestroy_Serial(nv_absolute_tolerance_backward_);
+
+      if (cvodes_mem_) {
+        CVodeFree(&cvodes_mem_);
+      }
+    }
+  }
 
   /**
    * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
@@ -640,22 +614,21 @@ class cvodes_integrator_adjoint_vari : public vari {
    *   solution time (excluding the initial state)
    */
   std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> operator()() {
-    // std::cout << "forward integrate..." << std::endl;
-    const double t0_dbl = value_of(memory->t0_);
-    const std::vector<double> ts_dbl = value_of(memory->ts_);
+    //std::cout << "forward integrate..." << std::endl;
+    const double t0_dbl = value_of(t0_);
+    const std::vector<double> ts_dbl = value_of(ts_);
 
     double t_init = t0_dbl;
     for (size_t n = 0; n < ts_dbl.size(); ++n) {
       double t_final = ts_dbl[n];
-      // std::cout << "n = " << n << ": t_init = " << t_init << ", t_final = "
-      // << t_final << std::endl;
+      //std::cout << "n = " << n << ": t_init = " << t_init << ", t_final = " << t_final << std::endl;
 
       if (t_final != t_init) {
-        if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ > 0) {
+        if (num_vars_ != 0) {
           int ncheck;
 
           int error_code
-              = CVodeF(memory->cvodes_mem_, t_final, memory->nv_state_, &t_init,
+              = CVodeF(cvodes_mem_, t_final, nv_state_forward_, &t_init,
                        CV_NORMAL, &ncheck);
 
           if (error_code == CV_TOO_MUCH_WORK) {
@@ -667,8 +640,8 @@ class cvodes_integrator_adjoint_vari : public vari {
           }
 
         } else {
-          int error_code = CVode(memory->cvodes_mem_, t_final,
-                                 memory->nv_state_, &t_init, CV_NORMAL);
+          int error_code = CVode(cvodes_mem_, t_final,
+                                 nv_state_forward_, &t_init, CV_NORMAL);
 
           if (error_code == CV_TOO_MUCH_WORK) {
             throw_domain_error(function_name_, "", t_final,
@@ -680,42 +653,37 @@ class cvodes_integrator_adjoint_vari : public vari {
         }
       }
 
-      memory->y_[n] = memory->state_;
+      y_[n] = state_forward_;
 
       t_init = t_final;
     }
 
-    returned_ = true;
-    // std::cout << "forward integrate...done" << std::endl;
-    return build_varis<T_Return>(this, non_chaining_varis_, memory->y_);
+    forward_returned_ = true;
+    //std::cout << "forward integrate...done" << std::endl;
+    return internal::build_varis<T_Return>(this, non_chaining_varis_, y_);
   }
 
   virtual void chain() {
-    // std::cout << "backward v2 integrate..." << std::endl;
+    //std::cout << "backward v2 integrate..." << std::endl;
     // std::cout << "backward v2 chain_count_ = " << chain_count_++ <<
     // std::endl;
     // std::cout << "chain" << std::endl; //<-- Good way to verify it's only
     //  being called once
-    if (memory == NULL) {
-      return;
-    }
 
-    if (memory->cvodes_mem_ == NULL) {
+    if (cvodes_mem_ == nullptr) {
       return;
     }
 
-    if (returned_ == false) {
+    if (!forward_returned_) {
       return;
     }
 
-    if (t0_vars_ + ts_vars_ + y0_vars_ + args_vars_ == 0) {
+    if (num_vars_ == 0) {
       return;
     }
 
-    Eigen::VectorXd& state_sens = memory->state_sens_;
-    Eigen::VectorXd& quad = memory->quad_;
-    state_sens.setZero();
-    quad.setZero();
+    state_backward_.setZero();
+    quad_.setZero();
     // N_VConst(0.0, memory->nv_state_sens_);
     // N_VConst(0.0, memory->nv_quad_);
 
@@ -741,20 +709,20 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     // At every time step, collect the adjoints from the output
     // variables and re-initialize the solver
-    double t_init = value_of(memory->ts_.back());
-    for (int i = memory->ts_.size() - 1; i >= 0; --i) {
+    double t_init = value_of(ts_.back());
+    for (int i = ts_.size() - 1; i >= 0; --i) {
       // Take in the adjoints from all the output variables at this point
       // in time
       Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
       for (int j = 0; j < N_; j++) {
         // std::cout << "i: " << i << ", j: " << j << std::endl;
-        state_sens(j) += non_chaining_varis_[i * N_ + j]->adj_;
-        step_sens(j) += non_chaining_varis_[i * N_ + j]->adj_;
+        state_backward_.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
+        step_sens.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
       }
 
-      if (ts_vars_ > 0 && i >= 0) {
-        ts_varis_[i]->adj_ += step_sens.dot(memory->rhs(
-            t_init, memory->y_[i], msgs_, memory->value_of_args_tuple_));
+      if (num_ts_vars_ > 0 && i >= 0) {
+        ts_varis_[i]->adj_ += step_sens.dot(rhs(
+            t_init, y_[i], value_of_args_tuple_));
         /*
           apply(
           [&](auto&&... args) {
@@ -767,17 +735,17 @@ class cvodes_integrator_adjoint_vari : public vari {
         */
       }
 
-      double t_final = value_of((i > 0) ? memory->ts_[i - 1] : memory->t0_);
+      double t_final = value_of((i > 0) ? ts_[i - 1] : t0_);
       // std::cout << "backward: time-point " << i << "; t_init = " << t_init <<
       // "; t_final = " << t_final << std::endl;
       if (t_final != t_init) {
-        if (!memory->backward_is_initialized_) {
+        if (!backward_is_initialized_) {
           check_flag_sundials(
-              CVodeCreateB(memory->cvodes_mem_, lmm_b_, &memory->index_b_),
+              CVodeCreateB(cvodes_mem_, solver_backward_, &index_backward_),
               "CVodeCreateB");
 
           check_flag_sundials(
-              CVodeSetUserDataB(memory->cvodes_mem_, memory->index_b_,
+              CVodeSetUserDataB(cvodes_mem_, index_backward_,
                                 reinterpret_cast<void*>(this)),
               "CVodeSetUserDataB");
 
@@ -785,69 +753,69 @@ class cvodes_integrator_adjoint_vari : public vari {
           // the states of the backward problem *are* the adjoints
           // of the ode states
           check_flag_sundials(
-              CVodeInitB(memory->cvodes_mem_, memory->index_b_,
+              CVodeInitB(cvodes_mem_, index_backward_,
                          &cvodes_integrator_adjoint_vari::cv_rhs_adj_sens,
-                         t_init, memory->nv_state_sens_),
+                         t_init, nv_state_backward_),
               "CVodeInitB");
 
           check_flag_sundials(
-              CVodeSVtolerancesB(memory->cvodes_mem_, memory->index_b_,
+              CVodeSVtolerancesB(cvodes_mem_, index_backward_,
                                  relative_tolerance_backward_,
-                                 memory->nv_absolute_tolerance_backward_),
+                                 nv_absolute_tolerance_backward_),
               "CVodeSVtolerancesB");
 
           check_flag_sundials(
-              CVodeSetMaxNumStepsB(memory->cvodes_mem_, memory->index_b_,
+              CVodeSetMaxNumStepsB(cvodes_mem_, index_backward_,
                                    max_num_steps_),
               "CVodeSetMaxNumStepsB");
 
           check_flag_sundials(
-              CVodeSetLinearSolverB(memory->cvodes_mem_, memory->index_b_,
-                                    memory->LS_b_, memory->A_b_),
+              CVodeSetLinearSolverB(cvodes_mem_, index_backward_,
+                                    LS_backward_, A_backward_),
               "CVodeSetLinearSolverB");
 
           check_flag_sundials(
-              CVodeSetJacFnB(memory->cvodes_mem_, memory->index_b_,
+              CVodeSetJacFnB(cvodes_mem_, index_backward_,
                              &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
               "CVodeSetJacFnB");
 
           // Allocate space for backwards quadrature needed when
           // parameters vary.
-          if (args_vars_ > 0) {
+          if (num_args_vars_ > 0) {
             check_flag_sundials(
-                CVodeQuadInitB(memory->cvodes_mem_, memory->index_b_,
+                CVodeQuadInitB(cvodes_mem_, index_backward_,
                                &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj,
-                               memory->nv_quad_),
+                               nv_quad_),
                 "CVodeQuadInitB");
 
             check_flag_sundials(
-                CVodeQuadSStolerancesB(memory->cvodes_mem_, memory->index_b_,
+                CVodeQuadSStolerancesB(cvodes_mem_, index_backward_,
                                        relative_tolerance_quadrature_,
                                        absolute_tolerance_quadrature_),
                 "CVodeQuadSStolerancesB");
 
-            check_flag_sundials(CVodeSetQuadErrConB(memory->cvodes_mem_,
-                                                    memory->index_b_, SUNTRUE),
+            check_flag_sundials(CVodeSetQuadErrConB(cvodes_mem_,
+                                                    index_backward_, SUNTRUE),
                                 "CVodeSetQuadErrConB");
           }
 
-          memory->backward_is_initialized_ = true;
+          backward_is_initialized_ = true;
         } else {
           // just re-initialize the solver
           check_flag_sundials(
-              CVodeReInitB(memory->cvodes_mem_, memory->index_b_, t_init,
-                           memory->nv_state_sens_),
+              CVodeReInitB(cvodes_mem_, index_backward_, t_init,
+                           nv_state_backward_),
               "CVodeReInitB");
 
-          if (args_vars_ > 0) {
+          if (num_args_vars_ > 0) {
             check_flag_sundials(
-                CVodeQuadReInitB(memory->cvodes_mem_, memory->index_b_,
-                                 memory->nv_quad_),
+                CVodeQuadReInitB(cvodes_mem_, index_backward_,
+                                 nv_quad_),
                 "CVodeQuadReInitB");
           }
         }
 
-        int error_code = CVodeB(memory->cvodes_mem_, t_final, CV_NORMAL);
+        int error_code = CVodeB(cvodes_mem_, t_final, CV_NORMAL);
 
         if (error_code == CV_TOO_MUCH_WORK) {
           throw_domain_error(function_name_, "", t_final,
@@ -863,23 +831,23 @@ class cvodes_integrator_adjoint_vari : public vari {
 
         // obtain adjoint states and update t_init to time point
         // reached of t_final
-        check_flag_sundials(CVodeGetB(memory->cvodes_mem_, memory->index_b_,
-                                      &t_init, memory->nv_state_sens_),
+        check_flag_sundials(CVodeGetB(cvodes_mem_, index_backward_,
+                                      &t_init, nv_state_backward_),
                             "CVodeGetB");
 
-        if (args_vars_ > 0) {
+        if (num_args_vars_ > 0) {
           check_flag_sundials(
-              CVodeGetQuadB(memory->cvodes_mem_, memory->index_b_, &t_init,
-                            memory->nv_quad_),
+              CVodeGetQuadB(cvodes_mem_, index_backward_, &t_init,
+                            nv_quad_),
               "CVodeGetQuadB");
         }
       }
     }
 
-    if (t0_vars_ > 0) {
-      Eigen::VectorXd y0d = value_of(memory->y0_);
-      t0_varis_[0]->adj_ += -state_sens.dot(
-          memory->rhs(t_init, y0d, msgs_, memory->value_of_args_tuple_));
+    if (num_t0_vars_ > 0) {
+      Eigen::VectorXd y0d = value_of(y0_);
+      t0_varis_[0]->adj_ += -state_backward_.dot(
+          rhs(t_init, y0d, value_of_args_tuple_));
       /*
         apply(
         [&](auto&&... args) {
@@ -901,16 +869,16 @@ class cvodes_integrator_adjoint_vari : public vari {
     // After integrating all the way back to t0, we finally have the
     // the adjoints we wanted
     // These are the dlog_density / d(initial_conditions[s]) adjoints
-    for (size_t s = 0; s < y0_vars_; s++) {
-      y0_varis_[s]->adj_ += state_sens.coeff(s);
+    for (size_t s = 0; s < num_y0_vars_; s++) {
+      y0_varis_[s]->adj_ += state_backward_.coeff(s);
     }
 
     // These are the dlog_density / d(parameters[s]) adjoints
-    for (size_t s = 0; s < args_vars_; s++) {
-      args_varis_[s]->adj_ += quad.coeff(s);
+    for (size_t s = 0; s < num_args_vars_; s++) {
+      args_varis_[s]->adj_ += quad_.coeff(s);
     }
 
-    // std::cout << "backward v2 integrate...done" << std::endl;
+    //std::cout << "backward v2 integrate...done" << std::endl;
   }
 };  // cvodes integrator
 
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 4f575b014e2..437f92ff50c 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -169,40 +169,15 @@ ode_adjoint_tol_ctl(
     int interpolation_polynomial, int solver_forward, int solver_backward,
     std::ostream* msgs, const T_Args&... args) {
   return ode_adjoint_impl(
-      "ode_adjoint_tol_ctl", f, y0, t0, ts, relative_tolerance_forward,
-      absolute_tolerance_forward, relative_tolerance_backward,
-      absolute_tolerance_backward, relative_tolerance_quadrature,
-      absolute_tolerance_quadrature, max_num_steps,
+      "ode_adjoint_tol_ctl", f, y0, t0, ts,
+      relative_tolerance_forward, absolute_tolerance_forward,
+      relative_tolerance_backward, absolute_tolerance_backward,
+      relative_tolerance_quadrature, absolute_tolerance_quadrature,
+      max_num_steps,
       num_steps_between_checkpoints, interpolation_polynomial, solver_forward,
       solver_backward, msgs, args...);
 }
 
-template <typename F, typename T_y0, typename T_t0, typename T_ts,
-          typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
-std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
-                          Eigen::Dynamic, 1>>
-ode_adjoint_tol(const F& f, const T_y0& y0, const T_t0& t0,
-                const std::vector<T_ts>& ts, double relative_tolerance,
-                double absolute_tolerance,
-                long int max_num_steps,  // NOLINT(runtime/int)
-                std::ostream* msgs, const T_Args&... args) {
-  const int N = y0.size();
-  const Eigen::VectorXd absolute_tolerance_forward
-      = Eigen::VectorXd::Constant(N, absolute_tolerance / 100.0);
-  const Eigen::VectorXd absolute_tolerance_backward
-      = Eigen::VectorXd::Constant(N, absolute_tolerance / 10.0);
-  const long int num_steps_between_checkpoints = 250;  // NOLINT(runtime/int)
-  const int interpolation_polynomial = CV_HERMITE;
-  const int solver_forward = CV_BDF;
-  const int solver_backward = CV_BDF;
-  return ode_adjoint_impl(
-      "ode_adjoint_tol", f, y0, t0, ts, relative_tolerance,
-      absolute_tolerance_forward, relative_tolerance,
-      absolute_tolerance_backward, relative_tolerance, absolute_tolerance,
-      max_num_steps, num_steps_between_checkpoints, interpolation_polynomial,
-      solver_forward, solver_backward, msgs, args...);
-}
-
 }  // namespace math
 }  // namespace stan
 #endif
diff --git a/test/unit/math/rev/functor/ode_test_functors.hpp b/test/unit/math/rev/functor/ode_test_functors.hpp
index 01cc7dd4e68..821f38422ca 100644
--- a/test/unit/math/rev/functor/ode_test_functors.hpp
+++ b/test/unit/math/rev/functor/ode_test_functors.hpp
@@ -16,7 +16,7 @@
               typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr> \
     std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,  \
                               Eigen::Dynamic, 1>>                              \
-    operator()(const F& f, const T_y0& y0, T_t0 t0,                            \
+    operator()(const F& f, const T_y0& y0, const T_t0& t0,                            \
                const std::vector<T_ts>& ts, std::ostream* msgs,                \
                const Args&... args) {                                          \
       return solver_func(f, y0, t0, ts, msgs, args...);                        \
@@ -26,7 +26,7 @@
               typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr> \
     std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,  \
                               Eigen::Dynamic, 1>>                              \
-    operator()(const F& f, const T_y0& y0_arg, T_t0 t0,                        \
+    operator()(const F& f, const T_y0& y0_arg, const T_t0& t0,                        \
                const std::vector<T_ts>& ts, double rtol, double atol,          \
                size_t max_num_steps, std::ostream* msgs,                       \
                const Args&... args) {                                          \
@@ -69,21 +69,41 @@ struct ode_adjoint_functor {
             typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr>
   std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,
                             Eigen::Dynamic, 1>>
-  operator()(const F& f, const T_y0& y0, T_t0 t0, const std::vector<T_ts>& ts,
+  operator()(const F& f, const T_y0& y0, const T_t0& t0, const std::vector<T_ts>& ts,
              std::ostream* msgs, const Args&... args) {
-    return stan::math::ode_adjoint_tol(f, y0, t0, ts, 1E-11, 1E-10, 1000000,
-                                       msgs, args...);
+    return (*this)(f, y0, t0, ts, 1E-10, 1E-10, 1000000,
+                   msgs, args...);
   }
 
   template <typename F, typename T_y0, typename T_t0, typename T_ts,
             typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr>
   std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,
                             Eigen::Dynamic, 1>>
-  operator()(const F& f, const T_y0& y0_arg, T_t0 t0,
-             const std::vector<T_ts>& ts, double rtol, double atol,
+  operator()(const F& f, const T_y0& y0_arg, const T_t0& t0,
+             const std::vector<T_ts>& ts,
+             double relative_tolerance, double absolute_tolerance,
              size_t max_num_steps, std::ostream* msgs, const Args&... args) {
-    return stan::math::ode_adjoint_tol(f, y0_arg, t0, ts, rtol, atol,
-                                       max_num_steps, msgs, args...);
+    const int N = y0_arg.size();
+    const double relative_tolerance_forward = relative_tolerance / 8.0;
+    const double relative_tolerance_backward = relative_tolerance / 4.0;
+    const double relative_tolerance_quadrature = relative_tolerance;
+    const Eigen::VectorXd absolute_tolerance_forward
+        = Eigen::VectorXd::Constant(N, absolute_tolerance / 6.0);
+    const Eigen::VectorXd absolute_tolerance_backward
+        = Eigen::VectorXd::Constant(N, absolute_tolerance / 3.0);
+    const double absolute_tolerance_quadrature = absolute_tolerance;
+    const long int num_steps_between_checkpoints = 150;  // NOLINT(runtime/int)
+    const int interpolation_polynomial = CV_HERMITE;
+    const int solver_forward = CV_BDF;
+    const int solver_backward = CV_ADAMS;
+
+    return stan::math::ode_adjoint_tol_ctl(
+        f, y0_arg, t0, ts,
+        relative_tolerance_forward, absolute_tolerance_forward,
+        relative_tolerance_backward, absolute_tolerance_backward,
+        relative_tolerance_quadrature, absolute_tolerance_quadrature,
+        max_num_steps, num_steps_between_checkpoints, interpolation_polynomial,
+        solver_forward, solver_backward, msgs, args...);
   }
 };
 
diff --git a/test/unit/math/rev/functor/sho_ode_typed_test.cpp b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
index 9d8b56f4855..2e46e42f0c8 100644
--- a/test/unit/math/rev/functor/sho_ode_typed_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
@@ -18,8 +18,9 @@ using ode_test_tuple = std::tuple<solve_type, solve_type, Ts...>;
  */
 using harmonic_oscillator_fd_test_types = boost::mp11::mp_product<
     ode_test_tuple,
-    ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
-                     ode_rk45_functor, ode_adjoint_functor>,
+  ::testing::Types<//ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
+                     //ode_rk45_functor,
+                     ode_adjoint_functor>,
     ::testing::Types<double>,  // t
     ::testing::Types<double>,  // y0
     ::testing::Types<double>   // theta
@@ -89,8 +90,9 @@ INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_data_test,
 
 using harmonic_oscillator_test_types = boost::mp11::mp_product<
     ode_test_tuple,
-    ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
-                     ode_rk45_functor, ode_adjoint_functor>,
+  ::testing::Types<//ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
+    //ode_rk45_functor,
+                     ode_adjoint_functor>,
     ::testing::Types<double>,                                  // t
     ::testing::Types<double, stan::math::var_value<double> >,  // y0
     ::testing::Types<double, stan::math::var_value<double> >   // theta

From b39361509b2ae1a30dcc7bc0b665d1034a68a9e9 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 22 Apr 2021 09:51:26 +0200
Subject: [PATCH 083/157] some more simplifcations

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 29 ++++---------
 stan/math/rev/functor/ode_adjoint.hpp         | 41 ++-----------------
 2 files changed, 12 insertions(+), 58 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 76cd29fc5f5..dbfa3927b45 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -23,13 +23,11 @@ namespace math {
 namespace internal {
 template <typename T_Return>
 inline std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> build_varis(
-    vari* ode_vari, vari**& non_chaining_varis,
-    const std::vector<Eigen::VectorXd>& y);
+    vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y);
 
 template <>
 inline std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> build_varis<var>(
-    vari* ode_vari, vari**& non_chaining_varis,
-    const std::vector<Eigen::VectorXd>& y) {
+    vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y) {
   std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y_return(y.size());
 
   if (y.size() == 0) {
@@ -49,14 +47,6 @@ inline std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> build_varis<var>(
     }
   }
 
-  /*
-  for (size_t i = 0; i < y.size(); ++i) {
-    y_return[i].resize(N);
-    for (size_t j = 0; j < N; j++)
-      y_return[i][j] = var(non_chaining_varis[i * N + j]);
-  }
-  */
-
   return y_return;
 }
 
@@ -66,8 +56,7 @@ inline std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> build_varis<var>(
  */
 template <>
 inline std::vector<Eigen::VectorXd> build_varis<double>(
-    vari* ode_vari, vari**& non_chaining_varis,
-    const std::vector<Eigen::VectorXd>& y) {
+    vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y) {
   return y;
 }
 
@@ -90,6 +79,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   using T_Return = return_type_t<T_y0, T_t0, T_ts, T_Args...>;
   using T_y0_t0 = return_type_t<T_y0, T_t0>;
 
+  const std::string function_name_string_;
   const char* function_name_;
 
   const F f_;
@@ -420,7 +410,8 @@ class cvodes_integrator_adjoint_vari : public vari {
       int interpolation_polynomial, int solver_forward, int solver_backward,
       std::ostream* msgs, const T_Args&... args)
   : vari(NOT_A_NUMBER),
-    function_name_(function_name),
+    function_name_string_(function_name),
+    function_name_(function_name_string_.c_str()),
     f_(f),
     y0_(y0),
     t0_(t0),
@@ -522,7 +513,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     std::cout << "steps_checkpoint = " << steps_checkpoint << std::endl;
     */
 
-    // initializes CVODES solver
+    // initialize CVODES solver
 
     if (N_ > 0) {
       nv_state_forward_ = N_VMake_Serial(N_, state_forward_.data());
@@ -564,10 +555,6 @@ class cvodes_integrator_adjoint_vari : public vari {
                           nv_absolute_tolerance_forward_),
         "CVodeSVtolerances");
 
-    // for the stiff solvers we need to reserve additional memory
-    // and provide a Jacobian function call. new API since 3.0.0:
-    // create matrix object and linear solver object; resource
-    // (de-)allocation is handled in the cvodes_ode_data
     check_flag_sundials(
         CVodeSetLinearSolver(cvodes_mem_, LS_forward_, A_forward_),
         "CVodeSetLinearSolver");
@@ -660,7 +647,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     forward_returned_ = true;
     //std::cout << "forward integrate...done" << std::endl;
-    return internal::build_varis<T_Return>(this, non_chaining_varis_, y_);
+    return internal::build_varis<T_Return>(non_chaining_varis_, y_);
   }
 
   virtual void chain() {
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 437f92ff50c..65b3f1f8553 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -66,9 +66,9 @@ std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
 ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
                  const T_t0& t0, const std::vector<T_ts>& ts,
                  double relative_tolerance_forward,
-                 Eigen::VectorXd absolute_tolerance_forward,
+                 const Eigen::VectorXd& absolute_tolerance_forward,
                  double relative_tolerance_backward,
-                 Eigen::VectorXd absolute_tolerance_backward,
+                 const Eigen::VectorXd& absolute_tolerance_backward,
                  double relative_tolerance_quadrature,
                  double absolute_tolerance_quadrature,
                  long int max_num_steps,                  // NOLINT(runtime/int)
@@ -76,39 +76,6 @@ ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
                  int interpolation_polynomial, int solver_forward,
                  int solver_backward, std::ostream* msgs,
                  const T_Args&... args) {
-  /*
-  const auto& args_ref_tuple = std::make_tuple(to_ref(args)...);
-  return apply(
-      [&](const auto&... args_refs) {
-        auto integrator
-            = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0,
-  T_t0, T_ts, T_Args...>( function_name, f, y0, t0, ts, relative_tolerance,
-                          absolute_tolerance, absolute_tolerance_B,
-  absolute_tolerance_QB, steps_checkpoint, max_num_steps, msgs, args_refs...);
-        return (*integrator)();
-      }, args_ref_tuple);
-      */
-  /*
-  const auto& args_eval_tuple = std::make_tuple(eval(args)...);
-  return apply(
-      [&](const auto&... args_eval) {
-        auto integrator
-            = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0,
-  T_t0, T_ts, T_Args...>( function_name, f, y0, t0, ts, relative_tolerance,
-                          absolute_tolerance, absolute_tolerance_B,
-  absolute_tolerance_QB, steps_checkpoint, max_num_steps, msgs, args_eval...);
-        return (*integrator)();
-      }, args_eval_tuple);
-      */
-  /*
-  auto integrator
-      = new stan::math::cvodes_integrator_adjoint_vari<CV_BDF, F, T_y0, T_t0,
-                                                       T_ts, T_Args...>(
-          function_name, f, y0, t0, ts, relative_tolerance, absolute_tolerance,
-          absolute_tolerance_B, absolute_tolerance_QB, steps_checkpoint,
-          max_num_steps, msgs, eval(args)...);
-  return (*integrator)();
-  */
   auto integrator = new stan::math::cvodes_integrator_adjoint_vari<
       F, plain_type_t<T_y0>, T_t0, T_ts, ref_type_t<T_Args>...>(
       function_name, f, eval(y0), t0, ts, relative_tolerance_forward,
@@ -160,9 +127,9 @@ std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
 ode_adjoint_tol_ctl(
     const F& f, const T_y0& y0, const T_t0& t0, const std::vector<T_ts>& ts,
     double relative_tolerance_forward,
-    Eigen::VectorXd absolute_tolerance_forward,
+    const Eigen::VectorXd& absolute_tolerance_forward,
     double relative_tolerance_backward,
-    Eigen::VectorXd absolute_tolerance_backward,
+    const Eigen::VectorXd& absolute_tolerance_backward,
     double relative_tolerance_quadrature, double absolute_tolerance_quadrature,
     long int max_num_steps,                  // NOLINT(runtime/int)
     long int num_steps_between_checkpoints,  // NOLINT(runtime/int)

From 1de856da5e9d003b2c203960c754865149a9a44a Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 22 Apr 2021 10:02:23 +0200
Subject: [PATCH 084/157] move build_varis function inside
 cvodes_integrator_adjoint

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 42 ++++++++++++++++++-
 1 file changed, 40 insertions(+), 2 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index dbfa3927b45..f9535ee32b5 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -21,6 +21,7 @@ namespace stan {
 namespace math {
 
 namespace internal {
+/*
 template <typename T_Return>
 inline std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> build_varis(
     vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y);
@@ -49,16 +50,18 @@ inline std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> build_varis<var>(
 
   return y_return;
 }
-
+*/
 /*
  * If theta and y are both doubles, just pass the values through (there's
  * no autodiff to handle here).
  */
+/*
 template <>
 inline std::vector<Eigen::VectorXd> build_varis<double>(
     vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y) {
   return y;
 }
+*/
 
 }
 
@@ -355,6 +358,41 @@ class cvodes_integrator_adjoint_vari : public vari {
     J_adj_y.array() *= -1.0;
   }
 
+template <typename T_ReturnArg>
+inline std::vector<Eigen::Matrix<T_ReturnArg, Eigen::Dynamic, 1>> build_varis(
+    vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y);
+
+template <>
+inline std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> build_varis<var>(
+    vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y) {
+  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y_return(y.size());
+
+  if (y.size() == 0) {
+    return y_return;
+  }
+
+  const int N = y[0].size();
+
+  non_chaining_varis
+      = ChainableStack::instance_->memalloc_.alloc_array<vari*>(y.size() * N);
+
+  for (size_t i = 0; i < y.size(); ++i) {
+    y_return[i].resize(N);
+    for (size_t j = 0; j < N; j++) {
+      non_chaining_varis[i * N + j] = new vari(y[i].coeff(j), false);
+      y_return[i].coeffRef(j) = var(non_chaining_varis[i * N + j]);
+    }
+  }
+
+  return y_return;
+}
+
+template <>
+inline std::vector<Eigen::VectorXd> build_varis<double>(
+    vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y) {
+  return y;
+}  
+
  public:
   /**
    * Construct cvodes_integrator object. Note: All arguments must be stored as
@@ -647,7 +685,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     forward_returned_ = true;
     //std::cout << "forward integrate...done" << std::endl;
-    return internal::build_varis<T_Return>(non_chaining_varis_, y_);
+    return build_varis<T_Return>(non_chaining_varis_, y_);
   }
 
   virtual void chain() {

From db528e03f87cdf436c1804422aa24b0d20f88cca Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 22 Apr 2021 10:08:10 +0200
Subject: [PATCH 085/157] undo last change

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 44 ++-----------------
 1 file changed, 4 insertions(+), 40 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index f9535ee32b5..660936cbbe6 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -21,7 +21,7 @@ namespace stan {
 namespace math {
 
 namespace internal {
-/*
+
 template <typename T_Return>
 inline std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> build_varis(
     vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y);
@@ -50,18 +50,17 @@ inline std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> build_varis<var>(
 
   return y_return;
 }
-*/
+
 /*
  * If theta and y are both doubles, just pass the values through (there's
  * no autodiff to handle here).
  */
-/*
 template <>
 inline std::vector<Eigen::VectorXd> build_varis<double>(
     vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y) {
   return y;
 }
-*/
+
 
 }
 
@@ -358,41 +357,6 @@ class cvodes_integrator_adjoint_vari : public vari {
     J_adj_y.array() *= -1.0;
   }
 
-template <typename T_ReturnArg>
-inline std::vector<Eigen::Matrix<T_ReturnArg, Eigen::Dynamic, 1>> build_varis(
-    vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y);
-
-template <>
-inline std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> build_varis<var>(
-    vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y) {
-  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y_return(y.size());
-
-  if (y.size() == 0) {
-    return y_return;
-  }
-
-  const int N = y[0].size();
-
-  non_chaining_varis
-      = ChainableStack::instance_->memalloc_.alloc_array<vari*>(y.size() * N);
-
-  for (size_t i = 0; i < y.size(); ++i) {
-    y_return[i].resize(N);
-    for (size_t j = 0; j < N; j++) {
-      non_chaining_varis[i * N + j] = new vari(y[i].coeff(j), false);
-      y_return[i].coeffRef(j) = var(non_chaining_varis[i * N + j]);
-    }
-  }
-
-  return y_return;
-}
-
-template <>
-inline std::vector<Eigen::VectorXd> build_varis<double>(
-    vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y) {
-  return y;
-}  
-
  public:
   /**
    * Construct cvodes_integrator object. Note: All arguments must be stored as
@@ -685,7 +649,7 @@ inline std::vector<Eigen::VectorXd> build_varis<double>(
 
     forward_returned_ = true;
     //std::cout << "forward integrate...done" << std::endl;
-    return build_varis<T_Return>(non_chaining_varis_, y_);
+    return internal::build_varis<T_Return>(non_chaining_varis_, y_);
   }
 
   virtual void chain() {

From bbb3bd2382a14de6a7869e24b49e0d1f15665ec1 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Thu, 22 Apr 2021 08:19:51 +0000
Subject: [PATCH 086/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 211 ++++++++----------
 stan/math/rev/functor/ode_adjoint.hpp         |   9 +-
 .../math/rev/functor/ode_test_functors.hpp    |  30 +--
 .../math/rev/functor/sho_ode_typed_test.cpp   |  12 +-
 4 files changed, 124 insertions(+), 138 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 660936cbbe6..3d40f03b1dc 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -61,9 +61,7 @@ inline std::vector<Eigen::VectorXd> build_varis<double>(
   return y;
 }
 
-
-}
-
+}  // namespace internal
 
 /**
  * Integrator interface for CVODES' ODE solvers (Adams & BDF
@@ -100,7 +98,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   const int interpolation_polynomial_;
   const int solver_forward_;
   const int solver_backward_;
-  
+
   std::ostream* msgs_;
 
   std::tuple<
@@ -138,20 +136,20 @@ class cvodes_integrator_adjoint_vari : public vari {
   N_Vector nv_quad_;
   N_Vector nv_absolute_tolerance_forward_;
   N_Vector nv_absolute_tolerance_backward_;
-  
+
   SUNMatrix A_forward_;
   SUNLinearSolver LS_forward_;
-  
+
   SUNMatrix A_backward_;
   SUNLinearSolver LS_backward_;
-  
+
   int index_backward_;
 
   /**
    * Call the ODE RHS with given tuple.
    */
   template <typename yT, typename... ArgsT>
-  constexpr auto rhs(double t, const yT& y, 
+  constexpr auto rhs(double t, const yT& y,
                      const std::tuple<ArgsT...>& args_tuple) const {
     return apply([&](auto&&... args) { return f_(t, y, msgs_, args...); },
                  args_tuple);
@@ -214,7 +212,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
                              SUNMatrix J, void* user_data, N_Vector tmp1,
                              N_Vector tmp2, N_Vector tmp3) {
-     cvodes_integrator_adjoint_vari* integrator
+    cvodes_integrator_adjoint_vari* integrator
         = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_adj(t, y, J);
     return 0;
@@ -227,11 +225,9 @@ class cvodes_integrator_adjoint_vari : public vari {
   inline void rhs(double t, const double y[], double dy_dt[]) {
     const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
 
-    const Eigen::VectorXd dy_dt_vec
-        = rhs(t, y_vec, value_of_args_tuple_);
+    const Eigen::VectorXd dy_dt_vec = rhs(t, y_vec, value_of_args_tuple_);
 
-    check_size_match(function_name_, "dy_dt", dy_dt_vec.size(),
-                     "states", N_);
+    check_size_match(function_name_, "dy_dt", dy_dt_vec.size(), "states", N_);
 
     Eigen::Map<Eigen::VectorXd>(dy_dt, N_) = dy_dt_vec;
   }
@@ -262,8 +258,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vars, value_of_args_tuple_);
 
-    check_size_match(function_name_, "dy_dt", f_y_t_vars.size(),
-                     "states", N_);
+    check_size_match(function_name_, "dy_dt", f_y_t_vars.size(), "states", N_);
 
     f_y_t_vars.adj() = -mu;
 
@@ -295,14 +290,12 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     // The vars here do not live on the nested stack so must be zero'd
     // separately
-    apply([&](auto&&... args) { zero_adjoints(args...); },
-          local_args_tuple_);
+    apply([&](auto&&... args) { zero_adjoints(args...); }, local_args_tuple_);
 
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vec, local_args_tuple_);
 
-    check_size_match(function_name_, "dy_dt", f_y_t_vars.size(),
-                     "states", N_);
+    check_size_match(function_name_, "dy_dt", f_y_t_vars.size(), "states", N_);
 
     f_y_t_vars.adj() = -mu;
 
@@ -326,8 +319,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var
         = rhs(t, y_var, value_of_args_tuple_);
 
-    check_size_match(function_name_, "dy_dt", fy_var.size(), "states",
-                     N_);
+    check_size_match(function_name_, "dy_dt", fy_var.size(), "states", N_);
 
     grad(fy_var.coeffRef(0).vi_);
     Jfy.col(0) = y_var.adj();
@@ -400,8 +392,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   template <require_eigen_col_vector_t<T_y0>* = nullptr>
   cvodes_integrator_adjoint_vari(
       const char* function_name, const F& f, const T_y0& y0, const T_t0& t0,
-      const std::vector<T_ts>& ts,
-      double relative_tolerance_forward,
+      const std::vector<T_ts>& ts, double relative_tolerance_forward,
       const Eigen::VectorXd& absolute_tolerance_forward,
       double relative_tolerance_backward,
       const Eigen::VectorXd& absolute_tolerance_backward,
@@ -411,54 +402,53 @@ class cvodes_integrator_adjoint_vari : public vari {
       long int num_steps_between_checkpoints,  // NOLINT(runtime/int)
       int interpolation_polynomial, int solver_forward, int solver_backward,
       std::ostream* msgs, const T_Args&... args)
-  : vari(NOT_A_NUMBER),
-    function_name_string_(function_name),
-    function_name_(function_name_string_.c_str()),
-    f_(f),
-    y0_(y0),
-    t0_(t0),
-    ts_(ts),
-    relative_tolerance_forward_(relative_tolerance_forward),
-    absolute_tolerance_forward_(absolute_tolerance_forward),
-    relative_tolerance_backward_(relative_tolerance_backward),
-    absolute_tolerance_backward_(absolute_tolerance_backward),
-    relative_tolerance_quadrature_(relative_tolerance_quadrature),
-    absolute_tolerance_quadrature_(absolute_tolerance_quadrature),
-    max_num_steps_(max_num_steps),
-    num_steps_between_checkpoints_(num_steps_between_checkpoints),
-    interpolation_polynomial_(interpolation_polynomial),
-    solver_forward_(solver_forward),
-    solver_backward_(solver_backward),
-    msgs_(msgs),
-    local_args_tuple_(deep_copy_vars(args)...),
-    value_of_args_tuple_(value_of(args)...),
-
-    N_(y0.size()),
-    forward_returned_(false),
-    backward_is_initialized_(false),
-
-    num_t0_vars_(count_vars(t0)),
-    num_ts_vars_(count_vars(ts)),
-    num_y0_vars_(count_vars(y0)),
-    num_args_vars_(count_vars(args...)),
-    num_vars_(num_t0_vars_ + num_ts_vars_ + num_y0_vars_ + num_args_vars_),
-
-    state_forward_(value_of(y0)),
-    state_backward_(Eigen::VectorXd::Zero(N_)),
-    quad_(Eigen::VectorXd::Zero(num_args_vars_)),
-
-    y_(ts_.size()),
-
-    non_chaining_varis_(nullptr),
-    t0_varis_(
-        ChainableStack::instance_->memalloc_.alloc_array<vari*>(num_t0_vars_)),
-    ts_varis_(
-        ChainableStack::instance_->memalloc_.alloc_array<vari*>(num_ts_vars_)),
-    y0_varis_(
-        ChainableStack::instance_->memalloc_.alloc_array<vari*>(num_y0_vars_)),
-    args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
-        num_args_vars_)) {
-
+      : vari(NOT_A_NUMBER),
+        function_name_string_(function_name),
+        function_name_(function_name_string_.c_str()),
+        f_(f),
+        y0_(y0),
+        t0_(t0),
+        ts_(ts),
+        relative_tolerance_forward_(relative_tolerance_forward),
+        absolute_tolerance_forward_(absolute_tolerance_forward),
+        relative_tolerance_backward_(relative_tolerance_backward),
+        absolute_tolerance_backward_(absolute_tolerance_backward),
+        relative_tolerance_quadrature_(relative_tolerance_quadrature),
+        absolute_tolerance_quadrature_(absolute_tolerance_quadrature),
+        max_num_steps_(max_num_steps),
+        num_steps_between_checkpoints_(num_steps_between_checkpoints),
+        interpolation_polynomial_(interpolation_polynomial),
+        solver_forward_(solver_forward),
+        solver_backward_(solver_backward),
+        msgs_(msgs),
+        local_args_tuple_(deep_copy_vars(args)...),
+        value_of_args_tuple_(value_of(args)...),
+
+        N_(y0.size()),
+        forward_returned_(false),
+        backward_is_initialized_(false),
+
+        num_t0_vars_(count_vars(t0)),
+        num_ts_vars_(count_vars(ts)),
+        num_y0_vars_(count_vars(y0)),
+        num_args_vars_(count_vars(args...)),
+        num_vars_(num_t0_vars_ + num_ts_vars_ + num_y0_vars_ + num_args_vars_),
+
+        state_forward_(value_of(y0)),
+        state_backward_(Eigen::VectorXd::Zero(N_)),
+        quad_(Eigen::VectorXd::Zero(num_args_vars_)),
+
+        y_(ts_.size()),
+
+        non_chaining_varis_(nullptr),
+        t0_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
+            num_t0_vars_)),
+        ts_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
+            num_ts_vars_)),
+        y0_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
+            num_y0_vars_)),
+        args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
+            num_args_vars_)) {
     save_varis(t0_varis_, t0);
     save_varis(ts_varis_, ts);
     save_varis(y0_varis_, y0);
@@ -473,7 +463,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     apply(
         [&](auto&&... args) {
           std::vector<int> unused_temp{
-              0, (check_finite(function_name_, "ode parameters and data", args), 0)...};
+              0, (check_finite(function_name_, "ode parameters and data", args),
+                  0)...};
         },
         local_args_tuple_);
 
@@ -501,7 +492,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_positive(function_name_, "num_steps_between_checkpoints",
                    num_steps_between_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
-    check_range(function_name_, "interpolation_polynomial", 2, interpolation_polynomial_);
+    check_range(function_name_, "interpolation_polynomial", 2,
+                interpolation_polynomial_);
     // 1=Adams=CV_ADAMS, 2=BDF=CV_BDF
     check_range(function_name_, "solver_forward", 2, solver_forward_);
     check_range(function_name_, "solver_backward", 2, solver_backward_);
@@ -525,10 +517,10 @@ class cvodes_integrator_adjoint_vari : public vari {
           = N_VMake_Serial(N_, absolute_tolerance_forward_.data());
       nv_absolute_tolerance_backward_
           = N_VMake_Serial(N_, absolute_tolerance_backward_.data());
-      
+
       A_forward_ = SUNDenseMatrix(N_, N_);
       LS_forward_ = SUNDenseLinearSolver(nv_state_forward_, A_forward_);
-      
+
       A_backward_ = SUNDenseMatrix(N_, N_);
       LS_backward_ = SUNDenseLinearSolver(nv_state_backward_, A_backward_);
 
@@ -603,22 +595,22 @@ class cvodes_integrator_adjoint_vari : public vari {
    *   solution time (excluding the initial state)
    */
   std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> operator()() {
-    //std::cout << "forward integrate..." << std::endl;
+    // std::cout << "forward integrate..." << std::endl;
     const double t0_dbl = value_of(t0_);
     const std::vector<double> ts_dbl = value_of(ts_);
 
     double t_init = t0_dbl;
     for (size_t n = 0; n < ts_dbl.size(); ++n) {
       double t_final = ts_dbl[n];
-      //std::cout << "n = " << n << ": t_init = " << t_init << ", t_final = " << t_final << std::endl;
+      // std::cout << "n = " << n << ": t_init = " << t_init << ", t_final = "
+      // << t_final << std::endl;
 
       if (t_final != t_init) {
         if (num_vars_ != 0) {
           int ncheck;
 
-          int error_code
-              = CVodeF(cvodes_mem_, t_final, nv_state_forward_, &t_init,
-                       CV_NORMAL, &ncheck);
+          int error_code = CVodeF(cvodes_mem_, t_final, nv_state_forward_,
+                                  &t_init, CV_NORMAL, &ncheck);
 
           if (error_code == CV_TOO_MUCH_WORK) {
             throw_domain_error(function_name_, "", t_final,
@@ -629,8 +621,8 @@ class cvodes_integrator_adjoint_vari : public vari {
           }
 
         } else {
-          int error_code = CVode(cvodes_mem_, t_final,
-                                 nv_state_forward_, &t_init, CV_NORMAL);
+          int error_code = CVode(cvodes_mem_, t_final, nv_state_forward_,
+                                 &t_init, CV_NORMAL);
 
           if (error_code == CV_TOO_MUCH_WORK) {
             throw_domain_error(function_name_, "", t_final,
@@ -648,12 +640,12 @@ class cvodes_integrator_adjoint_vari : public vari {
     }
 
     forward_returned_ = true;
-    //std::cout << "forward integrate...done" << std::endl;
+    // std::cout << "forward integrate...done" << std::endl;
     return internal::build_varis<T_Return>(non_chaining_varis_, y_);
   }
 
   virtual void chain() {
-    //std::cout << "backward v2 integrate..." << std::endl;
+    // std::cout << "backward v2 integrate..." << std::endl;
     // std::cout << "backward v2 chain_count_ = " << chain_count_++ <<
     // std::endl;
     // std::cout << "chain" << std::endl; //<-- Good way to verify it's only
@@ -710,8 +702,8 @@ class cvodes_integrator_adjoint_vari : public vari {
       }
 
       if (num_ts_vars_ > 0 && i >= 0) {
-        ts_varis_[i]->adj_ += step_sens.dot(rhs(
-            t_init, y_[i], value_of_args_tuple_));
+        ts_varis_[i]->adj_
+            += step_sens.dot(rhs(t_init, y_[i], value_of_args_tuple_));
         /*
           apply(
           [&](auto&&... args) {
@@ -733,10 +725,9 @@ class cvodes_integrator_adjoint_vari : public vari {
               CVodeCreateB(cvodes_mem_, solver_backward_, &index_backward_),
               "CVodeCreateB");
 
-          check_flag_sundials(
-              CVodeSetUserDataB(cvodes_mem_, index_backward_,
-                                reinterpret_cast<void*>(this)),
-              "CVodeSetUserDataB");
+          check_flag_sundials(CVodeSetUserDataB(cvodes_mem_, index_backward_,
+                                                reinterpret_cast<void*>(this)),
+                              "CVodeSetUserDataB");
 
           // initialize CVODES backward machinery.
           // the states of the backward problem *are* the adjoints
@@ -753,14 +744,13 @@ class cvodes_integrator_adjoint_vari : public vari {
                                  nv_absolute_tolerance_backward_),
               "CVodeSVtolerancesB");
 
-          check_flag_sundials(
-              CVodeSetMaxNumStepsB(cvodes_mem_, index_backward_,
-                                   max_num_steps_),
-              "CVodeSetMaxNumStepsB");
+          check_flag_sundials(CVodeSetMaxNumStepsB(cvodes_mem_, index_backward_,
+                                                   max_num_steps_),
+                              "CVodeSetMaxNumStepsB");
 
           check_flag_sundials(
-              CVodeSetLinearSolverB(cvodes_mem_, index_backward_,
-                                    LS_backward_, A_backward_),
+              CVodeSetLinearSolverB(cvodes_mem_, index_backward_, LS_backward_,
+                                    A_backward_),
               "CVodeSetLinearSolverB");
 
           check_flag_sundials(
@@ -783,23 +773,21 @@ class cvodes_integrator_adjoint_vari : public vari {
                                        absolute_tolerance_quadrature_),
                 "CVodeQuadSStolerancesB");
 
-            check_flag_sundials(CVodeSetQuadErrConB(cvodes_mem_,
-                                                    index_backward_, SUNTRUE),
-                                "CVodeSetQuadErrConB");
+            check_flag_sundials(
+                CVodeSetQuadErrConB(cvodes_mem_, index_backward_, SUNTRUE),
+                "CVodeSetQuadErrConB");
           }
 
           backward_is_initialized_ = true;
         } else {
           // just re-initialize the solver
-          check_flag_sundials(
-              CVodeReInitB(cvodes_mem_, index_backward_, t_init,
-                           nv_state_backward_),
-              "CVodeReInitB");
+          check_flag_sundials(CVodeReInitB(cvodes_mem_, index_backward_, t_init,
+                                           nv_state_backward_),
+                              "CVodeReInitB");
 
           if (num_args_vars_ > 0) {
             check_flag_sundials(
-                CVodeQuadReInitB(cvodes_mem_, index_backward_,
-                                 nv_quad_),
+                CVodeQuadReInitB(cvodes_mem_, index_backward_, nv_quad_),
                 "CVodeQuadReInitB");
           }
         }
@@ -820,14 +808,13 @@ class cvodes_integrator_adjoint_vari : public vari {
 
         // obtain adjoint states and update t_init to time point
         // reached of t_final
-        check_flag_sundials(CVodeGetB(cvodes_mem_, index_backward_,
-                                      &t_init, nv_state_backward_),
+        check_flag_sundials(CVodeGetB(cvodes_mem_, index_backward_, &t_init,
+                                      nv_state_backward_),
                             "CVodeGetB");
 
         if (num_args_vars_ > 0) {
           check_flag_sundials(
-              CVodeGetQuadB(cvodes_mem_, index_backward_, &t_init,
-                            nv_quad_),
+              CVodeGetQuadB(cvodes_mem_, index_backward_, &t_init, nv_quad_),
               "CVodeGetQuadB");
         }
       }
@@ -835,8 +822,8 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     if (num_t0_vars_ > 0) {
       Eigen::VectorXd y0d = value_of(y0_);
-      t0_varis_[0]->adj_ += -state_backward_.dot(
-          rhs(t_init, y0d, value_of_args_tuple_));
+      t0_varis_[0]->adj_
+          += -state_backward_.dot(rhs(t_init, y0d, value_of_args_tuple_));
       /*
         apply(
         [&](auto&&... args) {
@@ -867,7 +854,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       args_varis_[s]->adj_ += quad_.coeff(s);
     }
 
-    //std::cout << "backward v2 integrate...done" << std::endl;
+    // std::cout << "backward v2 integrate...done" << std::endl;
   }
 };  // cvodes integrator
 
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 65b3f1f8553..82178eccce9 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -136,11 +136,10 @@ ode_adjoint_tol_ctl(
     int interpolation_polynomial, int solver_forward, int solver_backward,
     std::ostream* msgs, const T_Args&... args) {
   return ode_adjoint_impl(
-      "ode_adjoint_tol_ctl", f, y0, t0, ts,
-      relative_tolerance_forward, absolute_tolerance_forward,
-      relative_tolerance_backward, absolute_tolerance_backward,
-      relative_tolerance_quadrature, absolute_tolerance_quadrature,
-      max_num_steps,
+      "ode_adjoint_tol_ctl", f, y0, t0, ts, relative_tolerance_forward,
+      absolute_tolerance_forward, relative_tolerance_backward,
+      absolute_tolerance_backward, relative_tolerance_quadrature,
+      absolute_tolerance_quadrature, max_num_steps,
       num_steps_between_checkpoints, interpolation_polynomial, solver_forward,
       solver_backward, msgs, args...);
 }
diff --git a/test/unit/math/rev/functor/ode_test_functors.hpp b/test/unit/math/rev/functor/ode_test_functors.hpp
index 821f38422ca..d876556656d 100644
--- a/test/unit/math/rev/functor/ode_test_functors.hpp
+++ b/test/unit/math/rev/functor/ode_test_functors.hpp
@@ -16,7 +16,7 @@
               typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr> \
     std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,  \
                               Eigen::Dynamic, 1>>                              \
-    operator()(const F& f, const T_y0& y0, const T_t0& t0,                            \
+    operator()(const F& f, const T_y0& y0, const T_t0& t0,                     \
                const std::vector<T_ts>& ts, std::ostream* msgs,                \
                const Args&... args) {                                          \
       return solver_func(f, y0, t0, ts, msgs, args...);                        \
@@ -26,7 +26,7 @@
               typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr> \
     std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,  \
                               Eigen::Dynamic, 1>>                              \
-    operator()(const F& f, const T_y0& y0_arg, const T_t0& t0,                        \
+    operator()(const F& f, const T_y0& y0_arg, const T_t0& t0,                 \
                const std::vector<T_ts>& ts, double rtol, double atol,          \
                size_t max_num_steps, std::ostream* msgs,                       \
                const Args&... args) {                                          \
@@ -69,10 +69,10 @@ struct ode_adjoint_functor {
             typename... Args, stan::require_eigen_vector_t<T_y0>* = nullptr>
   std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,
                             Eigen::Dynamic, 1>>
-  operator()(const F& f, const T_y0& y0, const T_t0& t0, const std::vector<T_ts>& ts,
-             std::ostream* msgs, const Args&... args) {
-    return (*this)(f, y0, t0, ts, 1E-10, 1E-10, 1000000,
-                   msgs, args...);
+  operator()(const F& f, const T_y0& y0, const T_t0& t0,
+             const std::vector<T_ts>& ts, std::ostream* msgs,
+             const Args&... args) {
+    return (*this)(f, y0, t0, ts, 1E-10, 1E-10, 1000000, msgs, args...);
   }
 
   template <typename F, typename T_y0, typename T_t0, typename T_ts,
@@ -80,9 +80,9 @@ struct ode_adjoint_functor {
   std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, Args...>,
                             Eigen::Dynamic, 1>>
   operator()(const F& f, const T_y0& y0_arg, const T_t0& t0,
-             const std::vector<T_ts>& ts,
-             double relative_tolerance, double absolute_tolerance,
-             size_t max_num_steps, std::ostream* msgs, const Args&... args) {
+             const std::vector<T_ts>& ts, double relative_tolerance,
+             double absolute_tolerance, size_t max_num_steps,
+             std::ostream* msgs, const Args&... args) {
     const int N = y0_arg.size();
     const double relative_tolerance_forward = relative_tolerance / 8.0;
     const double relative_tolerance_backward = relative_tolerance / 4.0;
@@ -98,12 +98,12 @@ struct ode_adjoint_functor {
     const int solver_backward = CV_ADAMS;
 
     return stan::math::ode_adjoint_tol_ctl(
-        f, y0_arg, t0, ts,
-        relative_tolerance_forward, absolute_tolerance_forward,
-        relative_tolerance_backward, absolute_tolerance_backward,
-        relative_tolerance_quadrature, absolute_tolerance_quadrature,
-        max_num_steps, num_steps_between_checkpoints, interpolation_polynomial,
-        solver_forward, solver_backward, msgs, args...);
+        f, y0_arg, t0, ts, relative_tolerance_forward,
+        absolute_tolerance_forward, relative_tolerance_backward,
+        absolute_tolerance_backward, relative_tolerance_quadrature,
+        absolute_tolerance_quadrature, max_num_steps,
+        num_steps_between_checkpoints, interpolation_polynomial, solver_forward,
+        solver_backward, msgs, args...);
   }
 };
 
diff --git a/test/unit/math/rev/functor/sho_ode_typed_test.cpp b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
index 2e46e42f0c8..6056bd3579a 100644
--- a/test/unit/math/rev/functor/sho_ode_typed_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
@@ -18,9 +18,9 @@ using ode_test_tuple = std::tuple<solve_type, solve_type, Ts...>;
  */
 using harmonic_oscillator_fd_test_types = boost::mp11::mp_product<
     ode_test_tuple,
-  ::testing::Types<//ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
-                     //ode_rk45_functor,
-                     ode_adjoint_functor>,
+    ::testing::Types<  // ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
+                       // ode_rk45_functor,
+        ode_adjoint_functor>,
     ::testing::Types<double>,  // t
     ::testing::Types<double>,  // y0
     ::testing::Types<double>   // theta
@@ -90,9 +90,9 @@ INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_data_test,
 
 using harmonic_oscillator_test_types = boost::mp11::mp_product<
     ode_test_tuple,
-  ::testing::Types<//ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
-    //ode_rk45_functor,
-                     ode_adjoint_functor>,
+    ::testing::Types<  // ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
+                       // ode_rk45_functor,
+        ode_adjoint_functor>,
     ::testing::Types<double>,                                  // t
     ::testing::Types<double, stan::math::var_value<double> >,  // y0
     ::testing::Types<double, stan::math::var_value<double> >   // theta

From d0e76a3346a5d7eb6603f734f72aa2f57b0f0024 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 22 Apr 2021 13:51:56 +0200
Subject: [PATCH 087/157] make benchmarks work with newer interface

---
 .../functor/ode_bdf_adjoint_bench_test.cpp    |  14 +-
 .../rev/functor/ode_bdf_adjoint_rev_test.cpp  | 527 ------------------
 .../ode_bdf_adjoint_scale_bench_test.cpp      |  15 +-
 3 files changed, 19 insertions(+), 537 deletions(-)
 delete mode 100644 test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
index 24bf616a0e2..53ee104d265 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
@@ -54,9 +54,9 @@ void run_benchmark(int adjoint_integrator) {
   double abs_tol_QB = abs_tol_B * 10.0;
   double rel_tol = 1e-6;
   int steps_checkpoint = 100;
-  int max_num_steps = 100000;
+  int max_num_steps = 1000000;
 
-  for (std::size_t i = 0; i != 200; i++) {
+  for (std::size_t i = 0; i != 2; i++) {
     stan::math::nested_rev_autodiff nested;
 
     var CL = lognormal_rng(true_CL, log_sigma, base_rng);
@@ -81,10 +81,14 @@ void run_benchmark(int adjoint_integrator) {
 
     try {
       if (adjoint_integrator) {
+        const int N = y0.size();
         std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-            = ode_bdf_adjoint_tol(ode, y0, t0, ts, rel_tol, abs_tol,
-                                  max_num_steps, nullptr, abs_tol_B, abs_tol_QB,
-                                  steps_checkpoint, ka, ke, k12, k21, kin, kout,
+            = ode_adjoint_tol_ctl(ode, y0, t0, ts,
+                                  rel_tol, Eigen::VectorXd::Constant(N, abs_tol),
+                                  rel_tol, Eigen::VectorXd::Constant(N, abs_tol_B),
+                                  rel_tol, abs_tol_QB,
+                                  max_num_steps, steps_checkpoint, 1, 2, 2,
+                                  nullptr, ka, ke, k12, k21, kin, kout,
                                   ea50);
 
         stan::math::grad();
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp
deleted file mode 100644
index 76d86d2e222..00000000000
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_rev_test.cpp
+++ /dev/null
@@ -1,527 +0,0 @@
-#include <stan/math/rev.hpp>
-#include <gtest/gtest.h>
-#include <test/unit/math/prim/functor/ode_test_functors.hpp>
-#include <iostream>
-#include <vector>
-
-TEST(StanMathOde_ode_bdf_adjoint, int_t0) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  int t0 = 0;
-  std::vector<double> ts = {0.45, 1.1};
-
-  double a = 1.5;
-
-  std::vector<Eigen::Matrix<double, Eigen::Dynamic, 1>> output
-      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                    a);
-
-  EXPECT_NEAR(output[0][0], 0.4165982112, 1e-5);
-  EXPECT_NEAR(output[1][0], 0.66457668563, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, int_ts) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  double t0 = 0.0;
-  std::vector<double> ts = {1, 2};
-
-  double a = 1.5;
-
-  std::vector<Eigen::Matrix<double, Eigen::Dynamic, 1>> output
-      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                    a);
-
-  EXPECT_NEAR(output[0][0], 0.6649966577, 1e-5);
-  EXPECT_NEAR(output[1][0], 0.09408000537, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, t0) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  var t0 = 0.0;
-  std::vector<double> ts = {0.45, 1.1};
-
-  double a = 1.5;
-
-  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> output
-      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                    a);
-
-  output[0][0].grad();
-
-  EXPECT_NEAR(output[0][0].val(), 0.4165982112, 1e-5);
-  EXPECT_NEAR(t0.adj(), -1.0, 1e-5);
-
-  stan::math::set_zero_all_adjoints();
-
-  output[1][0].grad();
-
-  EXPECT_NEAR(output[1][0].val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(t0.adj(), -1.0, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, ts) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  double t0 = 0.0;
-  std::vector<var> ts = {0.45, 1.1};
-
-  double a = 1.5;
-
-  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> output
-      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                    a);
-
-  output[0][0].grad();
-
-  EXPECT_NEAR(output[0][0].val(), 0.4165982112, 1e-5);
-  EXPECT_NEAR(ts[0].adj(), 0.78070695113, 1e-5);
-
-  stan::math::set_zero_all_adjoints();
-
-  output[1][0].grad();
-
-  EXPECT_NEAR(output[1][0].val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(ts[1].adj(), -0.0791208888, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, ts_repeat) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  double t0 = 0.0;
-  std::vector<var> ts = {0.45, 0.45, 1.1, 1.1};
-
-  double a = 1.5;
-
-  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> output
-      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                    a);
-
-  EXPECT_EQ(output.size(), ts.size());
-
-  output[0][0].grad();
-
-  EXPECT_NEAR(output[0][0].val(), 0.4165982112, 1e-5);
-  EXPECT_NEAR(ts[0].adj(), 0.78070695113, 1e-5);
-
-  stan::math::set_zero_all_adjoints();
-
-  output[1][0].grad();
-
-  EXPECT_NEAR(output[1][0].val(), 0.4165982112, 1e-5);
-  EXPECT_NEAR(ts[1].adj(), 0.78070695113, 1e-5);
-
-  stan::math::set_zero_all_adjoints();
-
-  output[2][0].grad();
-
-  EXPECT_NEAR(output[2][0].val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(ts[2].adj(), -0.0791208888, 1e-5);
-
-  stan::math::set_zero_all_adjoints();
-
-  output[3][0].grad();
-
-  EXPECT_NEAR(output[3][0].val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(ts[3].adj(), -0.0791208888, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, scalar_arg) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  double t0 = 0.0;
-  std::vector<double> ts = {1.1};
-
-  var a = 1.5;
-
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
-                                           nullptr, a)[0][0];
-
-  output.grad();
-
-  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(a.adj(), -0.50107310888, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, scalar_arg_multi_time) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  double t0 = 0.0;
-  std::vector<double> ts = {0.45, 1.1};
-
-  var a = 1.5;
-
-  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> output
-      = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts, nullptr,
-                                    a);
-
-  output[0](0).grad();
-
-  EXPECT_NEAR(output[0](0).val(), 0.4165982112, 1e-5);
-  EXPECT_NEAR(a.adj(), -0.04352005542, 1e-5);
-
-  stan::math::set_zero_all_adjoints();
-
-  output[1](0).grad();
-
-  EXPECT_NEAR(output[1](0).val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(a.adj(), -0.50107310888, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, std_vector_arg) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  double t0 = 0.0;
-  std::vector<double> ts = {1.1};
-
-  std::vector<var> a = {1.5};
-
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
-                                           nullptr, a)[0][0];
-
-  output.grad();
-
-  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(a[0].adj(), -0.50107310888, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, vector_arg) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  double t0 = 0.0;
-  std::vector<double> ts = {1.1};
-
-  Eigen::Matrix<var, Eigen::Dynamic, 1> a(1);
-  a << 1.5;
-
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
-                                           nullptr, a)[0][0];
-
-  output.grad();
-
-  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(a(0).adj(), -0.50107310888, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, row_vector_arg) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  double t0 = 0.0;
-  std::vector<double> ts = {1.1};
-
-  Eigen::Matrix<var, 1, Eigen::Dynamic> a(1);
-  a << 1.5;
-
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
-                                           nullptr, a)[0][0];
-
-  output.grad();
-
-  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(a(0).adj(), -0.50107310888, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, matrix_arg) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  double t0 = 0.0;
-  std::vector<double> ts = {1.1};
-
-  Eigen::Matrix<var, Eigen::Dynamic, Eigen::Dynamic> a(1, 1);
-  a << 1.5;
-
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
-                                           nullptr, a)[0][0];
-
-  output.grad();
-
-  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(a(0, 0).adj(), -0.50107310888, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, scalar_std_vector_args) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  double t0 = 0.0;
-  std::vector<double> ts = {1.1};
-
-  var a0 = 0.75;
-  std::vector<var> a1 = {0.75};
-
-  var output = stan::math::ode_bdf_adjoint(stan::test::Cos2Arg(), y0, t0, ts,
-                                           nullptr, a0, a1)[0][0];
-
-  output.grad();
-
-  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(a0.adj(), -0.50107310888, 1e-5);
-  EXPECT_NEAR(a1[0].adj(), -0.50107310888, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, std_vector_std_vector_args) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  double t0 = 0.0;
-  std::vector<double> ts = {1.1};
-
-  var a0 = 1.5;
-  std::vector<var> a1(1, a0);
-  std::vector<std::vector<var>> a2(1, a1);
-
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
-                                           nullptr, a2)[0][0];
-
-  output.grad();
-
-  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(a2[0][0].adj(), -0.50107310888, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, std_vector_vector_args) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  double t0 = 0.0;
-  std::vector<double> ts = {1.1};
-
-  var a0 = 1.5;
-  Eigen::Matrix<var, Eigen::Dynamic, 1> a1(1);
-  a1 << a0;
-  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> a2(1, a1);
-
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
-                                           nullptr, a2)[0][0];
-
-  output.grad();
-
-  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(a2[0](0).adj(), -0.50107310888, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, std_vector_row_vector_args) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  double t0 = 0.0;
-  std::vector<double> ts = {1.1};
-
-  var a0 = 1.5;
-  Eigen::Matrix<var, 1, Eigen::Dynamic> a1(1);
-  a1 << a0;
-  std::vector<Eigen::Matrix<var, 1, Eigen::Dynamic>> a2(1, a1);
-
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
-                                           nullptr, a2)[0][0];
-
-  output.grad();
-
-  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(a2[0](0).adj(), -0.50107310888, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, std_vector_matrix_args) {
-  using stan::math::var;
-
-  Eigen::VectorXd y0 = Eigen::VectorXd::Zero(1);
-  double t0 = 0.0;
-  std::vector<double> ts = {1.1};
-
-  var a0 = 1.5;
-  Eigen::Matrix<var, Eigen::Dynamic, Eigen::Dynamic> a1(1, 1);
-  a1 << a0;
-  std::vector<Eigen::Matrix<var, Eigen::Dynamic, Eigen::Dynamic>> a2(1, a1);
-
-  var output = stan::math::ode_bdf_adjoint(stan::test::CosArg1(), y0, t0, ts,
-                                           nullptr, a2)[0][0];
-
-  output.grad();
-
-  EXPECT_NEAR(output.val(), 0.66457668563, 1e-5);
-  EXPECT_NEAR(a2[0](0).adj(), -0.50107310888, 1e-5);
-}
-
-TEST(StanMathOde_ode_bdf_adjoint, arg_combos_test) {
-  using stan::math::var;
-
-  var t0 = 0.5;
-  var a = 0.2;
-  std::vector<var> ts = {1.25};
-  Eigen::Matrix<var, Eigen::Dynamic, 1> y0(1);
-  y0 << 0.75;
-
-  double t0d = stan::math::value_of(t0);
-  double ad = stan::math::value_of(a);
-  std::vector<double> tsd = stan::math::value_of(ts);
-  Eigen::VectorXd y0d = stan::math::value_of(y0);
-
-  auto check_yT = [&](auto yT) {
-    EXPECT_NEAR(stan::math::value_of(yT),
-                y0d(0) * exp(-0.5 * ad * (tsd[0] * tsd[0] - t0d * t0d)), 1e-5);
-  };
-
-  auto check_t0 = [&](var t0) {
-    EXPECT_NEAR(
-        t0.adj(),
-        ad * t0d * y0d(0) * exp(-0.5 * ad * (tsd[0] * tsd[0] - t0d * t0d)),
-        1e-5);
-  };
-
-  auto check_a = [&](var a) {
-    EXPECT_NEAR(a.adj(),
-                -0.5 * (tsd[0] * tsd[0] - t0d * t0d) * y0d(0)
-                    * exp(-0.5 * ad * (tsd[0] * tsd[0] - t0d * t0d)),
-                1e-5);
-  };
-
-  auto check_ts = [&](std::vector<var> ts) {
-    EXPECT_NEAR(
-        ts[0].adj(),
-        -ad * tsd[0] * y0d(0) * exp(-0.5 * ad * (tsd[0] * tsd[0] - t0d * t0d)),
-        1e-5);
-  };
-
-  auto check_y0 = [&](Eigen::Matrix<var, Eigen::Dynamic, 1> y0) {
-    EXPECT_NEAR(y0(0).adj(), exp(-0.5 * ad * (tsd[0] * tsd[0] - t0d * t0d)),
-                1e-5);
-  };
-
-  double yT1 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, tsd,
-                                           nullptr, ad)[0](0);
-  check_yT(yT1);
-
-  var yT2 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, tsd,
-                                        nullptr, a)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT2.grad();
-  check_yT(yT2);
-  check_a(a);
-
-  var yT3 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, ts,
-                                        nullptr, ad)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT3.grad();
-  check_yT(yT3);
-  check_ts(ts);
-
-  var yT4 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0d, ts,
-                                        nullptr, a)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT4.grad();
-  check_yT(yT4);
-  check_ts(ts);
-  check_a(a);
-
-  var yT5 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, tsd,
-                                        nullptr, ad)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT5.grad();
-  check_yT(yT5);
-  check_t0(t0);
-
-  var yT6 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, tsd,
-                                        nullptr, a)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT6.grad();
-  check_yT(yT6);
-  check_t0(t0);
-  check_a(a);
-
-  var yT7 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, ts, nullptr,
-                                        ad)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT7.grad();
-  check_yT(yT7);
-  check_t0(t0);
-  check_ts(ts);
-
-  var yT8 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0d, t0, ts, nullptr,
-                                        a)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT8.grad();
-  check_yT(yT8);
-  check_t0(t0);
-  check_ts(ts);
-  check_a(a);
-
-  var yT9 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, tsd,
-                                        nullptr, ad)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT9.grad();
-  check_yT(yT9);
-  check_y0(y0);
-
-  var yT10 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, tsd,
-                                         nullptr, a)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT10.grad();
-  check_yT(yT10);
-  check_y0(y0);
-  check_a(a);
-
-  var yT11 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, ts,
-                                         nullptr, ad)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT11.grad();
-  check_yT(yT11);
-  check_y0(y0);
-  check_ts(ts);
-
-  var yT12 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0d, ts,
-                                         nullptr, a)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT12.grad();
-  check_yT(yT12);
-  check_y0(y0);
-  check_ts(ts);
-  check_a(a);
-
-  var yT13 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, tsd,
-                                         nullptr, ad)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT13.grad();
-  check_yT(yT13);
-  check_y0(y0);
-  check_t0(t0);
-
-  var yT14 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, tsd,
-                                         nullptr, a)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT14.grad();
-  check_yT(yT14);
-  check_y0(y0);
-  check_t0(t0);
-  check_a(a);
-
-  var yT15 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, ts, nullptr,
-                                         ad)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT15.grad();
-  check_yT(yT15);
-  check_y0(y0);
-  check_t0(t0);
-  check_ts(ts);
-
-  var yT16 = stan::math::ode_bdf_adjoint(stan::test::ayt(), y0, t0, ts, nullptr,
-                                         a)[0](0);
-  stan::math::set_zero_all_adjoints();
-  yT16.grad();
-  check_yT(yT16);
-  check_y0(y0);
-  check_t0(t0);
-  check_ts(ts);
-  check_a(a);
-}
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
index 43d760d0d3e..e7bcb6b4e87 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
@@ -58,9 +58,9 @@ void run_benchmark(std::size_t system_size, int adjoint_integrator) {
   double abs_tol_QB = abs_tol_B * 10.0;
   double rel_tol = 1e-6;
   int steps_checkpoint = 100;
-  int max_num_steps = 100000;
+  int max_num_steps = 1000000;
 
-  for (std::size_t i = 0; i != 10; i++) {
+  for (std::size_t i = 0; i != 2; i++) {
     stan::math::nested_rev_autodiff nested;
 
     std::vector<var> kt(system_size);
@@ -84,9 +84,14 @@ void run_benchmark(std::size_t system_size, int adjoint_integrator) {
     try {
       std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y;
       if (adjoint_integrator) {
-        y = ode_bdf_adjoint_tol(ode, y0, t0, ts, rel_tol, abs_tol,
-                                max_num_steps, nullptr, abs_tol_B, abs_tol_QB,
-                                steps_checkpoint, kt, e50, k12, k21);
+        const int N = y0.size();
+        y = ode_adjoint_tol_ctl(ode, y0, t0, ts,
+                                rel_tol, Eigen::VectorXd::Constant(N, abs_tol),
+                                rel_tol, Eigen::VectorXd::Constant(N, abs_tol_B),
+                                rel_tol, abs_tol_QB,
+                                max_num_steps, steps_checkpoint, 1, 2, 2,
+                                nullptr, 
+                                kt, e50, k12, k21);
       } else {
         y = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol_QB, max_num_steps,
                         nullptr, kt, e50, k12, k21);

From 5ee531ae350caa89fb928f0041f896a6202083a2 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Thu, 22 Apr 2021 11:53:27 +0000
Subject: [PATCH 088/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../math/rev/functor/ode_bdf_adjoint_bench_test.cpp  | 12 +++++-------
 .../rev/functor/ode_bdf_adjoint_scale_bench_test.cpp | 12 +++++-------
 2 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
index 53ee104d265..c6233d2b018 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
@@ -83,13 +83,11 @@ void run_benchmark(int adjoint_integrator) {
       if (adjoint_integrator) {
         const int N = y0.size();
         std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-            = ode_adjoint_tol_ctl(ode, y0, t0, ts,
-                                  rel_tol, Eigen::VectorXd::Constant(N, abs_tol),
-                                  rel_tol, Eigen::VectorXd::Constant(N, abs_tol_B),
-                                  rel_tol, abs_tol_QB,
-                                  max_num_steps, steps_checkpoint, 1, 2, 2,
-                                  nullptr, ka, ke, k12, k21, kin, kout,
-                                  ea50);
+            = ode_adjoint_tol_ctl(
+                ode, y0, t0, ts, rel_tol, Eigen::VectorXd::Constant(N, abs_tol),
+                rel_tol, Eigen::VectorXd::Constant(N, abs_tol_B), rel_tol,
+                abs_tol_QB, max_num_steps, steps_checkpoint, 1, 2, 2, nullptr,
+                ka, ke, k12, k21, kin, kout, ea50);
 
         stan::math::grad();
       } else {
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
index e7bcb6b4e87..35fe7f79882 100644
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
+++ b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
@@ -85,13 +85,11 @@ void run_benchmark(std::size_t system_size, int adjoint_integrator) {
       std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y;
       if (adjoint_integrator) {
         const int N = y0.size();
-        y = ode_adjoint_tol_ctl(ode, y0, t0, ts,
-                                rel_tol, Eigen::VectorXd::Constant(N, abs_tol),
-                                rel_tol, Eigen::VectorXd::Constant(N, abs_tol_B),
-                                rel_tol, abs_tol_QB,
-                                max_num_steps, steps_checkpoint, 1, 2, 2,
-                                nullptr, 
-                                kt, e50, k12, k21);
+        y = ode_adjoint_tol_ctl(
+            ode, y0, t0, ts, rel_tol, Eigen::VectorXd::Constant(N, abs_tol),
+            rel_tol, Eigen::VectorXd::Constant(N, abs_tol_B), rel_tol,
+            abs_tol_QB, max_num_steps, steps_checkpoint, 1, 2, 2, nullptr, kt,
+            e50, k12, k21);
       } else {
         y = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol_QB, max_num_steps,
                         nullptr, kt, e50, k12, k21);

From 2b887ca788c03b2fd52bcb6efb5e72c6dea91a57 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 22 Apr 2021 15:36:40 +0200
Subject: [PATCH 089/157] remove obsolete test

---
 .../ode_adjoint_cvodes_grad_rev_test.cpp      | 353 ------------------
 1 file changed, 353 deletions(-)
 delete mode 100644 test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp

diff --git a/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp b/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
deleted file mode 100644
index 1cf19ed1e83..00000000000
--- a/test/unit/math/rev/functor/ode_adjoint_cvodes_grad_rev_test.cpp
+++ /dev/null
@@ -1,353 +0,0 @@
-#include <stan/math/rev.hpp>
-#include <gtest/gtest.h>
-#include <stan/math/rev/functor/gradient.hpp>
-#include <iostream>
-#include <sstream>
-#include <vector>
-#include <stdexcept>
-
-#define TEST_CVODES_ADAMS 1
-#define TEST_CVODES_BDF 2
-
-using std::cos;
-using std::sin;
-
-double y1(double t, double omega, double chi) { return chi * cos(omega * t); }
-
-double dy1_domega(double t, double omega, double chi) {
-  return -t * chi * sin(omega * t);
-}
-
-double dy1_dchi(double t, double omega, double chi) { return cos(omega * t); }
-
-double y2(double t, double omega, double chi) {
-  return -omega * chi * sin(omega * t);
-}
-
-double dy2_domega(double t, double omega, double chi) {
-  return -chi * (sin(omega * t) + omega * t * cos(omega * t));
-}
-
-double dy2_dchi(double t, double omega, double chi) {
-  return -omega * sin(omega * t);
-}
-
-struct sho_functor {
-  template <typename T0, typename T1, typename T2>
-  inline Eigen::Matrix<stan::return_type_t<T1, T2>, Eigen::Dynamic, 1>
-  operator()(const T0& t, const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y,
-             std::ostream* msgs, const T2& omega) const {
-    Eigen::Matrix<stan::return_type_t<T1, T2>, Eigen::Dynamic, 1> out(2);
-    out << y.coeff(1), -omega * omega * y.coeff(0);
-    return out;
-  }
-};
-
-template <int state>
-class test_functor_double_var {
-  int lmm_;
-
- public:
-  explicit test_functor_double_var(int lmm) : lmm_(lmm) {}
-
-  template <typename T>
-  inline T operator()(Eigen::Matrix<T, Eigen::Dynamic, 1>& x) const {
-    sho_functor sho;
-
-    T omega = x(0);
-
-    Eigen::VectorXd y0(2);
-    y0 << 1.25, 0.0;
-
-    double t0 = 0.0;
-    std::vector<double> ts{2.5, 5.0};
-
-    std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
-        = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_adjoint_tol(sho, y0, t0, ts, 1E-10, 1E-10,
-                                            10000, nullptr, omega)
-              : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
-
-    return ys[1](state);
-  }
-};
-
-TEST(StanMathOdeIntegrateODEGradMat, double_var) {
-  double omega = 0.5;
-  double chi = 1.25;
-  double t = 5;
-
-  Eigen::VectorXd x(1);
-  x(0) = omega;
-
-  double f;
-  Eigen::VectorXd grad(1);
-
-  stan::math::nested_rev_autodiff nested;
-
-  {  // Adams
-    test_functor_double_var<0> func1(TEST_CVODES_ADAMS);
-    stan::math::gradient(func1, x, f, grad);
-
-    EXPECT_NEAR(y1(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy1_domega(t, omega, chi), grad(0), 1e-6);
-
-    test_functor_double_var<1> func2(TEST_CVODES_ADAMS);
-    stan::math::gradient(func2, x, f, grad);
-
-    EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-6);
-  }
-
-  {  // bdf
-    test_functor_double_var<0> func1(TEST_CVODES_BDF);
-    stan::math::gradient(func1, x, f, grad);
-
-    EXPECT_NEAR(y1(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy1_domega(t, omega, chi), grad(0), 1e-7);
-
-    test_functor_double_var<1> func2(TEST_CVODES_BDF);
-    stan::math::gradient(func2, x, f, grad);
-
-    EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-7);
-  }
-}
-
-template <int state>
-class test_functor_var_double {
-  int lmm_;
-
- public:
-  explicit test_functor_var_double(int lmm) : lmm_(lmm) {}
-
-  template <typename T>
-  inline T operator()(Eigen::Matrix<T, Eigen::Dynamic, 1>& x) const {
-    sho_functor sho;
-
-    double omega = 0.5;
-
-    Eigen::Matrix<T, Eigen::Dynamic, 1> y0(2);
-    y0 << x(0), 0.0;
-
-    double t0 = 0.0;
-    std::vector<double> ts{5.0};
-
-    std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
-        = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_adjoint_tol(sho, y0, t0, ts, 1E-10, 1E-10,
-                                            10000, nullptr, omega)
-              : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
-
-    return ys[0](state);
-  }
-};
-
-TEST(StanMathOdeIntegrateODEGradMat, var_double) {
-  double omega = 0.5;
-  double chi = 1.25;
-  double t = 5;
-
-  Eigen::VectorXd x(1);
-  x(0) = chi;
-
-  double f;
-  Eigen::VectorXd grad(1);
-
-  stan::math::nested_rev_autodiff nested;
-  {  // adams
-    test_functor_var_double<0> func1(TEST_CVODES_ADAMS);
-    stan::math::gradient(func1, x, f, grad);
-
-    EXPECT_NEAR(y1(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy1_dchi(t, omega, chi), grad(0), 1e-7);
-
-    test_functor_var_double<1> func2(TEST_CVODES_ADAMS);
-    stan::math::gradient(func2, x, f, grad);
-
-    EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy2_dchi(t, omega, chi), grad(0), 1e-7);
-  }
-
-  {  // bdf
-    test_functor_var_double<0> func1(TEST_CVODES_BDF);
-    stan::math::gradient(func1, x, f, grad);
-
-    EXPECT_NEAR(y1(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy1_dchi(t, omega, chi), grad(0), 1e-7);
-
-    test_functor_var_double<1> func2(TEST_CVODES_BDF);
-    stan::math::gradient(func2, x, f, grad);
-
-    EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy2_dchi(t, omega, chi), grad(0), 1e-7);
-  }
-}
-
-template <int state>
-class test_functor_var_var {
-  int lmm_;
-
- public:
-  explicit test_functor_var_var(int lmm) : lmm_(lmm) {}
-
-  template <typename T>
-  inline T operator()(Eigen::Matrix<T, Eigen::Dynamic, 1>& x) const {
-    sho_functor sho;
-
-    T omega = x(0);
-
-    Eigen::Matrix<T, Eigen::Dynamic, 1> y0(2);
-    y0 << x(1), 0.0;
-
-    double t0 = 0.0;
-    std::vector<double> ts{2.0, 5.0};
-
-    std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
-        = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_adjoint_tol(sho, y0, t0, ts, 1E-10, 1E-10,
-                                            10000, nullptr, omega)
-              : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
-
-    return ys[1](state);
-  }
-};
-
-TEST(StanMathOdeIntegrateODEGradMat, var_var) {
-  double omega = 0.5;
-  double chi = 1.25;
-  double t = 5;
-
-  Eigen::VectorXd x(2);
-  x(0) = omega;
-  x(1) = chi;
-
-  double f;
-  Eigen::VectorXd grad(2);
-
-  {
-    stan::math::nested_rev_autodiff nested;
-
-    test_functor_var_var<0> func1(TEST_CVODES_ADAMS);
-    stan::math::gradient(func1, x, f, grad);
-
-    EXPECT_NEAR(y1(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy1_domega(t, omega, chi), grad(0), 1e-7);
-    EXPECT_NEAR(dy1_dchi(t, omega, chi), grad(1), 1e-7);
-
-    test_functor_var_var<1> func2(TEST_CVODES_ADAMS);
-    stan::math::gradient(func2, x, f, grad);
-
-    EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-6);
-    EXPECT_NEAR(dy2_dchi(t, omega, chi), grad(1), 1e-7);
-  }
-
-  {
-    stan::math::nested_rev_autodiff nested;
-
-    test_functor_var_var<0> func1(TEST_CVODES_BDF);
-    stan::math::gradient(func1, x, f, grad);
-
-    EXPECT_NEAR(y1(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy1_domega(t, omega, chi), grad(0), 1e-7);
-    EXPECT_NEAR(dy1_dchi(t, omega, chi), grad(1), 1e-7);
-
-    test_functor_var_var<1> func2(TEST_CVODES_BDF);
-    stan::math::gradient(func2, x, f, grad);
-
-    EXPECT_NEAR(y2(t, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy2_domega(t, omega, chi), grad(0), 1e-6);
-    EXPECT_NEAR(dy2_dchi(t, omega, chi), grad(1), 1e-7);
-  }
-}
-
-template <int state>
-class test_functor_sum_var_var {
-  int lmm_;
-
- public:
-  explicit test_functor_sum_var_var(int lmm) : lmm_(lmm) {}
-
-  template <typename T>
-  inline T operator()(Eigen::Matrix<T, Eigen::Dynamic, 1>& x) const {
-    sho_functor sho;
-
-    T omega = x(0);
-
-    Eigen::Matrix<T, Eigen::Dynamic, 1> y0(2);
-    y0 << x(1), 0.0;
-
-    double t0 = 0.0;
-    std::vector<double> ts{2.0, 5.0};
-
-    Eigen::VectorXd abs_tol_f(2);
-    abs_tol_f << 1E-10, 1E-10;
-
-    std::vector<Eigen::Matrix<T, Eigen::Dynamic, 1>> ys
-        = (lmm_ == TEST_CVODES_ADAMS)
-              ? stan::math::ode_adjoint_tol(sho, y0, t0, ts, 1E-10, 1E-10,
-                                            10000, nullptr, omega)
-              : stan::math::ode_bdf(sho, y0, t0, ts, nullptr, omega);
-
-    return stan::math::sum(ys[0](state) + ys[1](state));
-  }
-};
-
-TEST(StanMathOdeIntegrateODEGradMat, sum_var_var) {
-  double omega = 0.5;
-  double chi = 1.25;
-  double t1 = 2.0;
-  double t2 = 5;
-
-  Eigen::VectorXd x(2);
-  x(0) = omega;
-  x(1) = chi;
-
-  double f;
-  Eigen::VectorXd grad(2);
-
-  {
-    stan::math::nested_rev_autodiff nested;
-
-    test_functor_sum_var_var<0> func1(TEST_CVODES_ADAMS);
-    stan::math::gradient(func1, x, f, grad);
-
-    EXPECT_NEAR(y1(t1, omega, chi) + y1(t2, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy1_domega(t1, omega, chi) + dy1_domega(t2, omega, chi),
-                grad(0), 1e-7);
-    EXPECT_NEAR(dy1_dchi(t1, omega, chi) + dy1_dchi(t2, omega, chi), grad(1),
-                1e-7);
-
-    test_functor_sum_var_var<1> func2(TEST_CVODES_ADAMS);
-    stan::math::gradient(func2, x, f, grad);
-
-    EXPECT_NEAR(y2(t1, omega, chi) + y2(t2, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy2_domega(t1, omega, chi) + dy2_domega(t2, omega, chi),
-                grad(0), 1e-6);
-    EXPECT_NEAR(dy2_dchi(t1, omega, chi) + dy2_dchi(t2, omega, chi), grad(1),
-                1e-7);
-  }
-
-  {
-    stan::math::nested_rev_autodiff nested;
-
-    test_functor_sum_var_var<0> func1(TEST_CVODES_BDF);
-    stan::math::gradient(func1, x, f, grad);
-
-    EXPECT_NEAR(y1(t1, omega, chi) + y1(t2, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy1_domega(t1, omega, chi) + dy1_domega(t2, omega, chi),
-                grad(0), 1e-7);
-    EXPECT_NEAR(dy1_dchi(t1, omega, chi) + dy1_dchi(t2, omega, chi), grad(1),
-                1e-7);
-
-    test_functor_sum_var_var<1> func2(TEST_CVODES_BDF);
-    stan::math::gradient(func2, x, f, grad);
-
-    EXPECT_NEAR(y2(t1, omega, chi) + y2(t2, omega, chi), f, 1e-8);
-    EXPECT_NEAR(dy2_domega(t1, omega, chi) + dy2_domega(t2, omega, chi),
-                grad(0), 1e-6);
-    EXPECT_NEAR(dy2_dchi(t1, omega, chi) + dy2_dchi(t2, omega, chi), grad(1),
-                1e-7);
-  }
-}

From 7e02e5d2639e91905b0229870ce3e5f33f698b8a Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 22 Apr 2021 21:56:16 +0200
Subject: [PATCH 090/157] get rid of helper build_vari and replace it with
 overload withing vari class

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 72 +++++++------------
 1 file changed, 27 insertions(+), 45 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 3d40f03b1dc..6d846e83d9e 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -20,49 +20,6 @@
 namespace stan {
 namespace math {
 
-namespace internal {
-
-template <typename T_Return>
-inline std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> build_varis(
-    vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y);
-
-template <>
-inline std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> build_varis<var>(
-    vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y) {
-  std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y_return(y.size());
-
-  if (y.size() == 0) {
-    return y_return;
-  }
-
-  const int N = y[0].size();
-
-  non_chaining_varis
-      = ChainableStack::instance_->memalloc_.alloc_array<vari*>(y.size() * N);
-
-  for (size_t i = 0; i < y.size(); ++i) {
-    y_return[i].resize(N);
-    for (size_t j = 0; j < N; j++) {
-      non_chaining_varis[i * N + j] = new vari(y[i].coeff(j), false);
-      y_return[i].coeffRef(j) = var(non_chaining_varis[i * N + j]);
-    }
-  }
-
-  return y_return;
-}
-
-/*
- * If theta and y are both doubles, just pass the values through (there's
- * no autodiff to handle here).
- */
-template <>
-inline std::vector<Eigen::VectorXd> build_varis<double>(
-    vari**& non_chaining_varis, const std::vector<Eigen::VectorXd>& y) {
-  return y;
-}
-
-}  // namespace internal
-
 /**
  * Integrator interface for CVODES' ODE solvers (Adams & BDF
  * methods).
@@ -349,6 +306,25 @@ class cvodes_integrator_adjoint_vari : public vari {
     J_adj_y.array() *= -1.0;
   }
 
+  /**
+   * Overloads which setup the states returned from the forward solve. In case the return type is a double only, then no autodiff is needed. In case of autodiff non-chaining varis are setup accordingly.
+   */
+  void store_state(vari** non_chaining_varis,
+                   const Eigen::VectorXd& state,
+                   Eigen::Matrix<var, Eigen::Dynamic, 1>& state_return) {
+    state_return.resize(N_);
+    for (size_t i = 0; i < N_; i++) {
+      non_chaining_varis[i] = new vari(state.coeff(i), false);
+      state_return.coeffRef(i) = var(non_chaining_varis[i]);
+    }
+  }
+
+  void store_state(vari** non_chaining_varis,
+                   const Eigen::VectorXd& state,
+                   Eigen::Matrix<double, Eigen::Dynamic, 1>& state_return) {
+    state_return = state;
+  }
+
  public:
   /**
    * Construct cvodes_integrator object. Note: All arguments must be stored as
@@ -440,7 +416,9 @@ class cvodes_integrator_adjoint_vari : public vari {
 
         y_(ts_.size()),
 
-        non_chaining_varis_(nullptr),
+        non_chaining_varis_(num_vars_ == 0 ?
+                            nullptr :
+                            ChainableStack::instance_->memalloc_.alloc_array<vari*>(ts_.size() * N_)),
         t0_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             num_t0_vars_)),
         ts_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
@@ -599,6 +577,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     const double t0_dbl = value_of(t0_);
     const std::vector<double> ts_dbl = value_of(ts_);
 
+    std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return(ts_.size());
+
     double t_init = t0_dbl;
     for (size_t n = 0; n < ts_dbl.size(); ++n) {
       double t_final = ts_dbl[n];
@@ -634,6 +614,7 @@ class cvodes_integrator_adjoint_vari : public vari {
         }
       }
 
+      store_state(non_chaining_varis_ + N_ * n, state_forward_, y_return[n]);
       y_[n] = state_forward_;
 
       t_init = t_final;
@@ -641,7 +622,8 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     forward_returned_ = true;
     // std::cout << "forward integrate...done" << std::endl;
-    return internal::build_varis<T_Return>(non_chaining_varis_, y_);
+    //return internal::build_varis<T_Return>(non_chaining_varis_, y_);
+    return y_return;
   }
 
   virtual void chain() {

From dbd07eaaa4afb0c7b6b18f91e6d664a63c8d7223 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 22 Apr 2021 22:27:11 +0200
Subject: [PATCH 091/157] mark methods as const where possible

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 6d846e83d9e..36060de78fd 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -117,8 +117,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * ODE RHS passed to CVODES.
    */
   static int cv_rhs(realtype t, N_Vector y, N_Vector ydot, void* user_data) {
-    cvodes_integrator_adjoint_vari* integrator
-        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->rhs(t, NV_DATA_S(y), NV_DATA_S(ydot));
     return 0;
   }
@@ -129,8 +129,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    */
   static int cv_rhs_adj_sens(realtype t, N_Vector y, N_Vector yB,
                              N_Vector yBdot, void* user_data) {
-    cvodes_integrator_adjoint_vari* integrator
-        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->rhs_adj_sens(t, y, yB, yBdot);
     return 0;
   }
@@ -156,8 +156,8 @@ class cvodes_integrator_adjoint_vari : public vari {
   static int cv_jacobian_states(realtype t, N_Vector y, N_Vector fy,
                                 SUNMatrix J, void* user_data, N_Vector tmp1,
                                 N_Vector tmp2, N_Vector tmp3) {
-    cvodes_integrator_adjoint_vari* integrator
-        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_states(t, y, J);
     return 0;
   }
@@ -169,8 +169,8 @@ class cvodes_integrator_adjoint_vari : public vari {
   static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
                              SUNMatrix J, void* user_data, N_Vector tmp1,
                              N_Vector tmp2, N_Vector tmp3) {
-    cvodes_integrator_adjoint_vari* integrator
-        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_adj(t, y, J);
     return 0;
   }
@@ -179,7 +179,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Calculates the ODE RHS, dy_dt, using the user-supplied functor at
    * the given time t and state y.
    */
-  inline void rhs(double t, const double y[], double dy_dt[]) {
+  inline void rhs(double t, const double y[], double dy_dt[]) const {
     const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
 
     const Eigen::VectorXd dy_dt_vec = rhs(t, y_vec, value_of_args_tuple_);
@@ -202,7 +202,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] yB state of the adjoint ODE system
    * @param[out] yBdot evaluation of adjoint ODE RHS
    */
-  void rhs_adj_sens(double t, N_Vector y, N_Vector yB, N_Vector yBdot) {
+  void rhs_adj_sens(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
     Eigen::Map<Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
     Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(yBdot), N_);
@@ -266,7 +266,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Calculates the jacobian of the ODE RHS wrt to its states y at the
    * given time-point t and state y.
    */
-  inline void jacobian_states(double t, N_Vector y, SUNMatrix J) {
+  inline void jacobian_states(double t, N_Vector y, SUNMatrix J) const {
     Eigen::Map<Eigen::MatrixXd> Jfy(SM_DATA_D(J), N_, N_);
     Eigen::Map<const Eigen::VectorXd> x(NV_DATA_S(y), N_);
 
@@ -296,7 +296,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] t Time
    * @param[out] J CVode structure where output is to be stored
    */
-  inline void jacobian_adj(double t, N_Vector y, SUNMatrix J) {
+  inline void jacobian_adj(double t, N_Vector y, SUNMatrix J) const {
     Eigen::Map<Eigen::MatrixXd> J_adj_y(SM_DATA_D(J), N_, N_);
 
     // J_adj_y = -1 * transpose(J_y)
@@ -411,8 +411,8 @@ class cvodes_integrator_adjoint_vari : public vari {
         num_vars_(num_t0_vars_ + num_ts_vars_ + num_y0_vars_ + num_args_vars_),
 
         state_forward_(value_of(y0)),
-        state_backward_(Eigen::VectorXd::Zero(N_)),
-        quad_(Eigen::VectorXd::Zero(num_args_vars_)),
+        state_backward_(N_),
+        quad_(num_args_vars_),
 
         y_(ts_.size()),
 

From 0d46d0725c195bcc79956c7cff5f89808e7ccc5f Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Thu, 22 Apr 2021 20:39:01 +0000
Subject: [PATCH 092/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 23 +++++++++++--------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 36060de78fd..381a1bbf113 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -307,10 +307,11 @@ class cvodes_integrator_adjoint_vari : public vari {
   }
 
   /**
-   * Overloads which setup the states returned from the forward solve. In case the return type is a double only, then no autodiff is needed. In case of autodiff non-chaining varis are setup accordingly.
+   * Overloads which setup the states returned from the forward solve. In case
+   * the return type is a double only, then no autodiff is needed. In case of
+   * autodiff non-chaining varis are setup accordingly.
    */
-  void store_state(vari** non_chaining_varis,
-                   const Eigen::VectorXd& state,
+  void store_state(vari** non_chaining_varis, const Eigen::VectorXd& state,
                    Eigen::Matrix<var, Eigen::Dynamic, 1>& state_return) {
     state_return.resize(N_);
     for (size_t i = 0; i < N_; i++) {
@@ -319,8 +320,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     }
   }
 
-  void store_state(vari** non_chaining_varis,
-                   const Eigen::VectorXd& state,
+  void store_state(vari** non_chaining_varis, const Eigen::VectorXd& state,
                    Eigen::Matrix<double, Eigen::Dynamic, 1>& state_return) {
     state_return = state;
   }
@@ -416,9 +416,11 @@ class cvodes_integrator_adjoint_vari : public vari {
 
         y_(ts_.size()),
 
-        non_chaining_varis_(num_vars_ == 0 ?
-                            nullptr :
-                            ChainableStack::instance_->memalloc_.alloc_array<vari*>(ts_.size() * N_)),
+        non_chaining_varis_(
+            num_vars_ == 0
+                ? nullptr
+                : ChainableStack::instance_->memalloc_.alloc_array<vari*>(
+                      ts_.size() * N_)),
         t0_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             num_t0_vars_)),
         ts_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
@@ -577,7 +579,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     const double t0_dbl = value_of(t0_);
     const std::vector<double> ts_dbl = value_of(ts_);
 
-    std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return(ts_.size());
+    std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return(
+        ts_.size());
 
     double t_init = t0_dbl;
     for (size_t n = 0; n < ts_dbl.size(); ++n) {
@@ -622,7 +625,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     forward_returned_ = true;
     // std::cout << "forward integrate...done" << std::endl;
-    //return internal::build_varis<T_Return>(non_chaining_varis_, y_);
+    // return internal::build_varis<T_Return>(non_chaining_varis_, y_);
     return y_return;
   }
 

From 643c6029900fcc0585a887f15dd97ecef48d7752 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Thu, 22 Apr 2021 22:44:23 +0200
Subject: [PATCH 093/157] reorder freeing

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 36060de78fd..f72e090c7c4 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -547,6 +547,10 @@ class cvodes_integrator_adjoint_vari : public vari {
 
   ~cvodes_integrator_adjoint_vari() {
     if (N_ > 0) {
+      if (cvodes_mem_) {
+        CVodeFree(&cvodes_mem_);
+      }
+
       SUNLinSolFree(LS_forward_);
       SUNMatDestroy(A_forward_);
       SUNLinSolFree(LS_backward_);
@@ -557,10 +561,6 @@ class cvodes_integrator_adjoint_vari : public vari {
       N_VDestroy_Serial(nv_quad_);
       N_VDestroy_Serial(nv_absolute_tolerance_forward_);
       N_VDestroy_Serial(nv_absolute_tolerance_backward_);
-
-      if (cvodes_mem_) {
-        CVodeFree(&cvodes_mem_);
-      }
     }
   }
 

From fa3dbb2224c750236ae0a53dc9a4f3eff4e6a3b8 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 26 Apr 2021 21:25:35 +0200
Subject: [PATCH 094/157] port over changes from review3 branch

---
 stan/math/rev/core/zero_adjoints.hpp          |  45 +-
 .../rev/functor/cvodes_integrator_adjoint.hpp | 411 +++++++++---------
 .../math/rev/functor/test_fixture_ode_fho.hpp |   4 +-
 .../rev/functor/test_fixture_ode_lorenz.hpp   |   4 +-
 .../math/rev/functor/test_fixture_ode_sho.hpp |   8 +-
 5 files changed, 221 insertions(+), 251 deletions(-)

diff --git a/stan/math/rev/core/zero_adjoints.hpp b/stan/math/rev/core/zero_adjoints.hpp
index 36368d443ee..c49012702e3 100644
--- a/stan/math/rev/core/zero_adjoints.hpp
+++ b/stan/math/rev/core/zero_adjoints.hpp
@@ -8,24 +8,10 @@
 namespace stan {
 namespace math {
 
-inline void zero_adjoints();
-
-template <typename T, typename... Pargs, require_st_arithmetic<T>* = nullptr>
-inline void zero_adjoints(T& x, Pargs&... args);
-
-template <typename... Pargs>
-inline void zero_adjoints(var& x, Pargs&... args);
-
-template <int R, int C, typename... Pargs>
-inline void zero_adjoints(Eigen::Matrix<var, R, C>& x, Pargs&... args);
-
-template <typename T, typename... Pargs, require_st_autodiff<T>* = nullptr>
-inline void zero_adjoints(std::vector<T>& x, Pargs&... args);
-
 /**
  * End of recursion for set_zero_adjoints
  */
-inline void zero_adjoints() {}
+inline void zero_adjoints() noexcept {}
 
 /**
  * Do nothing for non-autodiff arguments. Recursively call zero_adjoints
@@ -37,10 +23,8 @@ inline void zero_adjoints() {}
  * @param x current argument
  * @param args rest of arguments to zero
  */
-template <typename T, typename... Pargs, require_st_arithmetic<T>*>
-inline void zero_adjoints(T& x, Pargs&... args) {
-  zero_adjoints(args...);
-}
+template <typename T, require_st_arithmetic<T>* = nullptr>
+inline void zero_adjoints(T& x) noexcept {}
 
 /**
  * Zero the adjoint of the vari in the first argument. Recursively call
@@ -52,10 +36,8 @@ inline void zero_adjoints(T& x, Pargs&... args) {
  * @param x current argument
  * @param args rest of arguments to zero
  */
-template <typename... Pargs>
-inline void zero_adjoints(var& x, Pargs&... args) {
-  x.vi_->set_zero_adjoint();
-  zero_adjoints(args...);
+inline void zero_adjoints(var& x) {
+  x.adj() = 0;
 }
 
 /**
@@ -68,11 +50,10 @@ inline void zero_adjoints(var& x, Pargs&... args) {
  * @param x current argument
  * @param args rest of arguments to zero
  */
-template <int R, int C, typename... Pargs>
-inline void zero_adjoints(Eigen::Matrix<var, R, C>& x, Pargs&... args) {
+template <typename EigMat, require_eigen_vt<is_autodiff, EigMat>* = nullptr>
+inline void zero_adjoints(EigMat& x) {
   for (size_t i = 0; i < x.size(); ++i)
-    x.coeffRef(i).vi_->set_zero_adjoint();
-  zero_adjoints(args...);
+    x.coeffRef(i).adj() = 0;
 }
 
 /**
@@ -85,11 +66,11 @@ inline void zero_adjoints(Eigen::Matrix<var, R, C>& x, Pargs&... args) {
  * @param x current argument
  * @param args rest of arguments to zero
  */
-template <typename T, typename... Pargs, require_st_autodiff<T>*>
-inline void zero_adjoints(std::vector<T>& x, Pargs&... args) {
-  for (size_t i = 0; i < x.size(); ++i)
-    zero_adjoints(x[i]);
-  zero_adjoints(args...);
+template <typename StdVec, require_std_vector_st<is_autodiff, StdVec>* = nullptr>
+inline void zero_adjoints(StdVec& x) {
+  for (size_t i = 0; i < x.size(); ++i) {
+    zero_adjoints(x[i]);    
+  }
 }
 
 }  // namespace math
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 54581d8d165..8ca4264c6f7 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -20,6 +20,70 @@
 namespace stan {
 namespace math {
 
+class cvodes_integrator_adjoint_mem : public chainable_alloc {
+public:
+  size_t N_;
+  std::vector<Eigen::VectorXd> y_;
+  const char* function_name_;
+  void* cvodes_mem_;
+  N_Vector nv_state_forward_;
+  N_Vector nv_state_backward_;
+  N_Vector nv_quad_;
+  N_Vector nv_absolute_tolerance_forward_;
+  N_Vector nv_absolute_tolerance_backward_;
+
+  SUNMatrix A_forward_;
+  SUNLinearSolver LS_forward_;
+
+  SUNMatrix A_backward_;
+  SUNLinearSolver LS_backward_;
+  template <typename StateFwd, typename StateBwd, typename Quad,
+  typename AbsTolFwd, typename AbsTolBwd>
+  cvodes_integrator_adjoint_mem(const char* function_name, size_t N,
+    size_t num_args_vars, size_t ts_size, int solver_forward, StateFwd& state_forward,
+    StateBwd& state_backward, Quad& quad,
+    AbsTolFwd& absolute_tolerance_forward,
+    AbsTolBwd& absolute_tolerance_backward) : chainable_alloc(), N_(N), y_(ts_size),
+    function_name_(function_name) {
+      nv_state_forward_ = N_VMake_Serial(N, state_forward.data());
+      nv_state_backward_ = N_VMake_Serial(N, state_backward.data());
+      nv_quad_ = N_VMake_Serial(num_args_vars, quad.data());
+      nv_absolute_tolerance_forward_
+          = N_VMake_Serial(N, absolute_tolerance_forward.data());
+      nv_absolute_tolerance_backward_
+          = N_VMake_Serial(N, absolute_tolerance_backward.data());
+
+      A_forward_ = SUNDenseMatrix(N, N);
+      A_backward_ = SUNDenseMatrix(N, N);
+      if (N > 0 ) {
+        LS_forward_ = SUNDenseLinearSolver(nv_state_forward_, A_forward_);
+        LS_backward_ = SUNDenseLinearSolver(nv_state_backward_, A_backward_);
+      }
+      cvodes_mem_ = CVodeCreate(solver_forward);
+
+      if (cvodes_mem_ == nullptr) {
+        throw std::runtime_error("CVodeCreate failed to allocate memory");
+      }
+  }
+  virtual ~cvodes_integrator_adjoint_mem() {
+      SUNMatDestroy(A_forward_);
+      SUNMatDestroy(A_backward_);
+      if (N_ > 0) {
+        SUNLinSolFree(LS_forward_);
+        SUNLinSolFree(LS_backward_);
+      }
+      N_VDestroy_Serial(nv_state_forward_);
+      N_VDestroy_Serial(nv_state_backward_);
+      N_VDestroy_Serial(nv_quad_);
+      N_VDestroy_Serial(nv_absolute_tolerance_forward_);
+      N_VDestroy_Serial(nv_absolute_tolerance_backward_);
+
+      if (cvodes_mem_) {
+        CVodeFree(&cvodes_mem_);
+      }
+  }
+};
+
 /**
  * Integrator interface for CVODES' ODE solvers (Adams & BDF
  * methods).
@@ -36,49 +100,47 @@ class cvodes_integrator_adjoint_vari : public vari {
   using T_Return = return_type_t<T_y0, T_t0, T_ts, T_Args...>;
   using T_y0_t0 = return_type_t<T_y0, T_t0>;
 
-  const std::string function_name_string_;
-  const char* function_name_;
-
-  const F f_;
-  const Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1> y0_;
-  const T_t0 t0_;
-  const std::vector<T_ts> ts_;
-
-  const double relative_tolerance_forward_;
-  Eigen::VectorXd absolute_tolerance_forward_;
-  const double relative_tolerance_backward_;
-  Eigen::VectorXd absolute_tolerance_backward_;
-  const double relative_tolerance_quadrature_;
-  const double absolute_tolerance_quadrature_;
-  const long int max_num_steps_;                  // NOLINT(runtime/int)
-  const long int num_steps_between_checkpoints_;  // NOLINT(runtime/int)
-  const int interpolation_polynomial_;
-  const int solver_forward_;
-  const int solver_backward_;
+  std::decay_t<F> f_;
+  arena_t<Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1>> y0_;
+  arena_t<T_t0> t0_;
+  std::vector<arena_t<T_ts>, arena_allocator<arena_t<T_ts>>> ts_;
+
+  double relative_tolerance_forward_;
+  arena_t<Eigen::VectorXd> absolute_tolerance_forward_;
+  double relative_tolerance_backward_;
+  arena_t<Eigen::VectorXd> absolute_tolerance_backward_;
+  double relative_tolerance_quadrature_;
+  double absolute_tolerance_quadrature_;
+  long int max_num_steps_;                  // NOLINT(runtime/int)
+  long int num_steps_between_checkpoints_;  // NOLINT(runtime/int)
+  int interpolation_polynomial_;
+  int solver_forward_;
+  int solver_backward_;
 
   std::ostream* msgs_;
 
   std::tuple<
-      plain_type_t<decltype(deep_copy_vars(std::declval<const T_Args&>()))>...>
+      arena_t<plain_type_t<decltype(deep_copy_vars(std::declval<const T_Args&>()))>>...>
       local_args_tuple_;
-  std::tuple<plain_type_t<decltype(value_of(std::declval<const T_Args&>()))>...>
+  std::tuple<arena_t<plain_type_t<decltype(value_of(std::declval<const T_Args&>()))>>...>
       value_of_args_tuple_;
 
-  const size_t N_;
+  size_t N_;
   bool forward_returned_;
   bool backward_is_initialized_;
 
-  const size_t num_t0_vars_;
-  const size_t num_ts_vars_;
-  const size_t num_y0_vars_;
-  const size_t num_args_vars_;
-  const size_t num_vars_;
+  size_t num_t0_vars_;
+  size_t num_ts_vars_;
+  size_t num_y0_vars_;
+  size_t num_args_vars_;
+  size_t num_vars_;
 
-  Eigen::VectorXd state_forward_;
-  Eigen::VectorXd state_backward_;
-  Eigen::VectorXd quad_;
+  arena_t<Eigen::VectorXd> state_forward_;
+  arena_t<Eigen::VectorXd> state_backward_;
+  arena_t<Eigen::VectorXd> quad_;
+
+  //std::vector<arena_t<Eigen::VectorXd>, arena_allocator<arena_t<Eigen::VectorXd>>> y_;
 
-  std::vector<Eigen::VectorXd> y_;
 
   vari** non_chaining_varis_;
 
@@ -87,27 +149,13 @@ class cvodes_integrator_adjoint_vari : public vari {
   vari** y0_varis_;
   vari** args_varis_;
 
-  void* cvodes_mem_;
-  N_Vector nv_state_forward_;
-  N_Vector nv_state_backward_;
-  N_Vector nv_quad_;
-  N_Vector nv_absolute_tolerance_forward_;
-  N_Vector nv_absolute_tolerance_backward_;
-
-  SUNMatrix A_forward_;
-  SUNLinearSolver LS_forward_;
-
-  SUNMatrix A_backward_;
-  SUNLinearSolver LS_backward_;
-
   int index_backward_;
-
+  cvodes_integrator_adjoint_mem* mem;
   /**
    * Call the ODE RHS with given tuple.
    */
   template <typename yT, typename... ArgsT>
-  constexpr auto rhs(double t, const yT& y,
-                     const std::tuple<ArgsT...>& args_tuple) const {
+  constexpr auto rhs(double t, const yT& y, const std::tuple<ArgsT...>& args_tuple) const {
     return apply([&](auto&&... args) { return f_(t, y, msgs_, args...); },
                  args_tuple);
   }
@@ -117,8 +165,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * ODE RHS passed to CVODES.
    */
   static int cv_rhs(realtype t, N_Vector y, N_Vector ydot, void* user_data) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+    cvodes_integrator_adjoint_vari* integrator
+        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
     integrator->rhs(t, NV_DATA_S(y), NV_DATA_S(ydot));
     return 0;
   }
@@ -127,10 +175,9 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Implements the function of type CVRhsFnB which is the
    * RHS of the backward ODE system.
    */
-  static int cv_rhs_adj_sens(realtype t, N_Vector y, N_Vector yB,
-                             N_Vector yBdot, void* user_data) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+  static int cv_rhs_adj_sens(realtype t, N_Vector y, N_Vector yB, N_Vector yBdot, void* user_data) {
+    cvodes_integrator_adjoint_vari* integrator
+        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
     integrator->rhs_adj_sens(t, y, yB, yBdot);
     return 0;
   }
@@ -139,8 +186,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Implements the function of type CVQuadRhsFnB which is the
    * RHS of the backward ODE system's quadrature.
    */
-  static int cv_quad_rhs_adj(realtype t, N_Vector y, N_Vector yB,
-                             N_Vector qBdot, void* user_data) {
+  static int cv_quad_rhs_adj(realtype t, N_Vector y, N_Vector yB, N_Vector qBdot, void* user_data) {
     cvodes_integrator_adjoint_vari* integrator
         = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
     integrator->quad_rhs_adj(t, y, yB, qBdot);
@@ -153,11 +199,9 @@ class cvodes_integrator_adjoint_vari : public vari {
    * ode_rhs wrt to the states y. The jacobian is stored in column
    * major format.
    */
-  static int cv_jacobian_states(realtype t, N_Vector y, N_Vector fy,
-                                SUNMatrix J, void* user_data, N_Vector tmp1,
-                                N_Vector tmp2, N_Vector tmp3) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+  static int cv_jacobian_states(realtype t, N_Vector y, N_Vector fy, SUNMatrix J, void* user_data, N_Vector tmp1, N_Vector tmp2, N_Vector tmp3) {
+    cvodes_integrator_adjoint_vari* integrator
+        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_states(t, y, J);
     return 0;
   }
@@ -166,11 +210,9 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Implements the CVLsJacFnB function for evaluating the jacobian of
    * the adjoint problem.
    */
-  static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
-                             SUNMatrix J, void* user_data, N_Vector tmp1,
-                             N_Vector tmp2, N_Vector tmp3) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
+  static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB, N_Vector fyB, SUNMatrix J, void* user_data, N_Vector tmp1, N_Vector tmp2, N_Vector tmp3) {
+    cvodes_integrator_adjoint_vari* integrator
+        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_adj(t, y, J);
     return 0;
   }
@@ -179,12 +221,12 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Calculates the ODE RHS, dy_dt, using the user-supplied functor at
    * the given time t and state y.
    */
-  inline void rhs(double t, const double y[], double dy_dt[]) const {
+  inline void rhs(double t, const double y[], double dy_dt[]) {
     const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
 
     const Eigen::VectorXd dy_dt_vec = rhs(t, y_vec, value_of_args_tuple_);
 
-    check_size_match(function_name_, "dy_dt", dy_dt_vec.size(), "states", N_);
+    check_size_match(mem->function_name_, "dy_dt", dy_dt_vec.size(), "states", N_);
 
     Eigen::Map<Eigen::VectorXd>(dy_dt, N_) = dy_dt_vec;
   }
@@ -202,7 +244,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] yB state of the adjoint ODE system
    * @param[out] yBdot evaluation of adjoint ODE RHS
    */
-  void rhs_adj_sens(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
+  void rhs_adj_sens(double t, N_Vector y, N_Vector yB, N_Vector yBdot) {
     Eigen::Map<Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
     Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(yBdot), N_);
@@ -215,7 +257,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vars, value_of_args_tuple_);
 
-    check_size_match(function_name_, "dy_dt", f_y_t_vars.size(), "states", N_);
+    check_size_match(mem->function_name_, "dy_dt", f_y_t_vars.size(), "states", N_);
 
     f_y_t_vars.adj() = -mu;
 
@@ -252,7 +294,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vec, local_args_tuple_);
 
-    check_size_match(function_name_, "dy_dt", f_y_t_vars.size(), "states", N_);
+    check_size_match(mem->function_name_, "dy_dt", f_y_t_vars.size(), "states", N_);
 
     f_y_t_vars.adj() = -mu;
 
@@ -266,7 +308,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Calculates the jacobian of the ODE RHS wrt to its states y at the
    * given time-point t and state y.
    */
-  inline void jacobian_states(double t, N_Vector y, SUNMatrix J) const {
+  inline void jacobian_states(double t, N_Vector y, SUNMatrix J) {
     Eigen::Map<Eigen::MatrixXd> Jfy(SM_DATA_D(J), N_, N_);
     Eigen::Map<const Eigen::VectorXd> x(NV_DATA_S(y), N_);
 
@@ -276,7 +318,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var
         = rhs(t, y_var, value_of_args_tuple_);
 
-    check_size_match(function_name_, "dy_dt", fy_var.size(), "states", N_);
+    check_size_match(mem->function_name_, "dy_dt", fy_var.size(), "states", N_);
 
     grad(fy_var.coeffRef(0).vi_);
     Jfy.col(0) = y_var.adj();
@@ -296,7 +338,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] t Time
    * @param[out] J CVode structure where output is to be stored
    */
-  inline void jacobian_adj(double t, N_Vector y, SUNMatrix J) const {
+  inline void jacobian_adj(double t, N_Vector y, SUNMatrix J) {
     Eigen::Map<Eigen::MatrixXd> J_adj_y(SM_DATA_D(J), N_, N_);
 
     // J_adj_y = -1 * transpose(J_y)
@@ -307,11 +349,10 @@ class cvodes_integrator_adjoint_vari : public vari {
   }
 
   /**
-   * Overloads which setup the states returned from the forward solve. In case
-   * the return type is a double only, then no autodiff is needed. In case of
-   * autodiff non-chaining varis are setup accordingly.
+   * Overloads which setup the states returned from the forward solve. In case the return type is a double only, then no autodiff is needed. In case of autodiff non-chaining varis are setup accordingly.
    */
-  void store_state(vari** non_chaining_varis, const Eigen::VectorXd& state,
+  void store_state(vari** non_chaining_varis,
+                   const Eigen::VectorXd& state,
                    Eigen::Matrix<var, Eigen::Dynamic, 1>& state_return) {
     state_return.resize(N_);
     for (size_t i = 0; i < N_; i++) {
@@ -320,7 +361,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     }
   }
 
-  void store_state(vari** non_chaining_varis, const Eigen::VectorXd& state,
+  void store_state(vari** non_chaining_varis,
+                   const Eigen::VectorXd& state,
                    Eigen::Matrix<double, Eigen::Dynamic, 1>& state_return) {
     state_return = state;
   }
@@ -365,9 +407,9 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @return a vector of states, each state being a vector of the
    * same size as the state variable, corresponding to a time in ts.
    */
-  template <require_eigen_col_vector_t<T_y0>* = nullptr>
+  template <typename FF, require_eigen_col_vector_t<T_y0>* = nullptr>
   cvodes_integrator_adjoint_vari(
-      const char* function_name, const F& f, const T_y0& y0, const T_t0& t0,
+      const char* function_name, FF&& f, const T_y0& y0, const T_t0& t0,
       const std::vector<T_ts>& ts, double relative_tolerance_forward,
       const Eigen::VectorXd& absolute_tolerance_forward,
       double relative_tolerance_backward,
@@ -379,12 +421,10 @@ class cvodes_integrator_adjoint_vari : public vari {
       int interpolation_polynomial, int solver_forward, int solver_backward,
       std::ostream* msgs, const T_Args&... args)
       : vari(NOT_A_NUMBER),
-        function_name_string_(function_name),
-        function_name_(function_name_string_.c_str()),
         f_(f),
         y0_(y0),
         t0_(t0),
-        ts_(ts),
+        ts_(ts.begin(), ts.end()),
         relative_tolerance_forward_(relative_tolerance_forward),
         absolute_tolerance_forward_(absolute_tolerance_forward),
         relative_tolerance_backward_(relative_tolerance_backward),
@@ -397,8 +437,8 @@ class cvodes_integrator_adjoint_vari : public vari {
         solver_forward_(solver_forward),
         solver_backward_(solver_backward),
         msgs_(msgs),
-        local_args_tuple_(deep_copy_vars(args)...),
-        value_of_args_tuple_(value_of(args)...),
+        local_args_tuple_(to_arena(deep_copy_vars(args))...),
+        value_of_args_tuple_(to_arena(value_of(args))...),
 
         N_(y0.size()),
         forward_returned_(false),
@@ -414,13 +454,9 @@ class cvodes_integrator_adjoint_vari : public vari {
         state_backward_(N_),
         quad_(num_args_vars_),
 
-        y_(ts_.size()),
-
-        non_chaining_varis_(
-            num_vars_ == 0
-                ? nullptr
-                : ChainableStack::instance_->memalloc_.alloc_array<vari*>(
-                      ts_.size() * N_)),
+        non_chaining_varis_(num_vars_ == 0 ?
+                            nullptr :
+                            ChainableStack::instance_->memalloc_.alloc_array<vari*>(ts_.size() * N_)),
         t0_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             num_t0_vars_)),
         ts_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
@@ -428,144 +464,98 @@ class cvodes_integrator_adjoint_vari : public vari {
         y0_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             num_y0_vars_)),
         args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
-            num_args_vars_)) {
+            num_args_vars_)),
+        mem(new cvodes_integrator_adjoint_mem(function_name, N_,
+            num_args_vars_, ts_.size(), solver_forward_, state_forward_,
+            state_backward_, quad_,
+            absolute_tolerance_forward_,
+            absolute_tolerance_backward_)) {
     save_varis(t0_varis_, t0);
     save_varis(ts_varis_, ts);
     save_varis(y0_varis_, y0);
     save_varis(args_varis_, args...);
 
-    check_finite(function_name_, "initial state", y0);
-    check_finite(function_name_, "initial time", t0);
-    check_finite(function_name_, "times", ts);
+    check_finite(mem->function_name_, "initial state", y0);
+    check_finite(mem->function_name_, "initial time", t0);
+    check_finite(mem->function_name_, "times", ts);
 
     // Code from: https://stackoverflow.com/a/17340003 . Should probably do
     // something better
     apply(
         [&](auto&&... args) {
           std::vector<int> unused_temp{
-              0, (check_finite(function_name_, "ode parameters and data", args),
+              0, (check_finite(mem->function_name_, "ode parameters and data", args),
                   0)...};
         },
         local_args_tuple_);
 
-    check_nonzero_size(function_name_, "times", ts);
-    check_nonzero_size(function_name_, "initial state", y0);
-    check_sorted(function_name_, "times", ts);
-    check_less(function_name_, "initial time", t0, ts[0]);
-    check_positive_finite(function_name_, "relative_tolerance_forward",
+    check_nonzero_size(mem->function_name_, "times", ts);
+    check_nonzero_size(mem->function_name_, "initial state", y0);
+    check_sorted(mem->function_name_, "times", ts);
+    check_less(mem->function_name_, "initial time", t0, ts[0]);
+    check_positive_finite(mem->function_name_, "relative_tolerance_forward",
                           relative_tolerance_forward_);
-    check_positive_finite(function_name_, "absolute_tolerance_forward",
+    check_positive_finite(mem->function_name_, "absolute_tolerance_forward",
                           absolute_tolerance_forward_);
-    check_size_match(function_name_, "absolute_tolerance_forward",
+    check_size_match(mem->function_name_, "absolute_tolerance_forward",
                      absolute_tolerance_forward_.size(), "states", N_);
-    check_positive_finite(function_name_, "relative_tolerance_backward",
+    check_positive_finite(mem->function_name_, "relative_tolerance_backward",
                           relative_tolerance_backward_);
-    check_positive_finite(function_name_, "absolute_tolerance_backward",
+    check_positive_finite(mem->function_name_, "absolute_tolerance_backward",
                           absolute_tolerance_backward_);
-    check_size_match(function_name_, "absolute_tolerance_backward",
+    check_size_match(mem->function_name_, "absolute_tolerance_backward",
                      absolute_tolerance_backward_.size(), "states", N_);
-    check_positive_finite(function_name_, "relative_tolerance_quadrature",
+    check_positive_finite(mem->function_name_, "relative_tolerance_quadrature",
                           relative_tolerance_quadrature_);
-    check_positive_finite(function_name_, "absolute_tolerance_quadrature",
+    check_positive_finite(mem->function_name_, "absolute_tolerance_quadrature",
                           absolute_tolerance_quadrature_);
-    check_positive(function_name_, "max_num_steps", max_num_steps_);
-    check_positive(function_name_, "num_steps_between_checkpoints",
+    check_positive(mem->function_name_, "max_num_steps", max_num_steps_);
+    check_positive(mem->function_name_, "num_steps_between_checkpoints",
                    num_steps_between_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
-    check_range(function_name_, "interpolation_polynomial", 2,
+    check_range(mem->function_name_, "interpolation_polynomial", 2,
                 interpolation_polynomial_);
     // 1=Adams=CV_ADAMS, 2=BDF=CV_BDF
-    check_range(function_name_, "solver_forward", 2, solver_forward_);
-    check_range(function_name_, "solver_backward", 2, solver_backward_);
-
-    /*
-    std::cout << "relative_tolerance = " << relative_tolerance << std::endl;
-    std::cout << "absolute_tolerance = " << absolute_tolerance << std::endl;
-    std::cout << "absolute_tolerance_B = " << absolute_tolerance_B << std::endl;
-    std::cout << "absolute_tolerance_QB = " << absolute_tolerance_QB <<
-    std::endl; std::cout << "max_num_steps = " << max_num_steps << std::endl;
-    std::cout << "steps_checkpoint = " << steps_checkpoint << std::endl;
-    */
-
-    // initialize CVODES solver
-
-    if (N_ > 0) {
-      nv_state_forward_ = N_VMake_Serial(N_, state_forward_.data());
-      nv_state_backward_ = N_VMake_Serial(N_, state_backward_.data());
-      nv_quad_ = N_VMake_Serial(num_args_vars_, quad_.data());
-      nv_absolute_tolerance_forward_
-          = N_VMake_Serial(N_, absolute_tolerance_forward_.data());
-      nv_absolute_tolerance_backward_
-          = N_VMake_Serial(N_, absolute_tolerance_backward_.data());
-
-      A_forward_ = SUNDenseMatrix(N_, N_);
-      LS_forward_ = SUNDenseLinearSolver(nv_state_forward_, A_forward_);
-
-      A_backward_ = SUNDenseMatrix(N_, N_);
-      LS_backward_ = SUNDenseLinearSolver(nv_state_backward_, A_backward_);
+    check_range(mem->function_name_, "solver_forward", 2, solver_forward_);
+    check_range(mem->function_name_, "solver_backward", 2, solver_backward_);
 
-      cvodes_mem_ = CVodeCreate(solver_forward_);
-
-      if (cvodes_mem_ == nullptr) {
-        throw std::runtime_error("CVodeCreate failed to allocate memory");
-      }
-    }
 
     check_flag_sundials(
-        CVodeInit(cvodes_mem_, &cvodes_integrator_adjoint_vari::cv_rhs,
-                  value_of(t0_), nv_state_forward_),
+        CVodeInit(mem->cvodes_mem_, &cvodes_integrator_adjoint_vari::cv_rhs,
+                  value_of(t0_), mem->nv_state_forward_),
         "CVodeInit");
 
     // Assign pointer to this as user data
     check_flag_sundials(
-        CVodeSetUserData(cvodes_mem_, reinterpret_cast<void*>(this)),
+        CVodeSetUserData(mem->cvodes_mem_, reinterpret_cast<void*>(this)),
         "CVodeSetUserData");
 
-    cvodes_set_options(cvodes_mem_, relative_tolerance_forward_,
+    cvodes_set_options(mem->cvodes_mem_, relative_tolerance_forward_,
                        absolute_tolerance_forward_(0), max_num_steps_);
 
     check_flag_sundials(
-        CVodeSVtolerances(cvodes_mem_, relative_tolerance_forward_,
-                          nv_absolute_tolerance_forward_),
+        CVodeSVtolerances(mem->cvodes_mem_, relative_tolerance_forward_,
+                          mem->nv_absolute_tolerance_forward_),
         "CVodeSVtolerances");
 
     check_flag_sundials(
-        CVodeSetLinearSolver(cvodes_mem_, LS_forward_, A_forward_),
+        CVodeSetLinearSolver(mem->cvodes_mem_, mem->LS_forward_, mem->A_forward_),
         "CVodeSetLinearSolver");
 
     check_flag_sundials(
-        CVodeSetJacFn(cvodes_mem_,
+        CVodeSetJacFn(mem->cvodes_mem_,
                       &cvodes_integrator_adjoint_vari::cv_jacobian_states),
         "CVodeSetJacFn");
 
     // initialize backward sensitivity system of CVODES as needed
     if (num_vars_ != 0) {
       check_flag_sundials(
-          CVodeAdjInit(cvodes_mem_, num_steps_between_checkpoints_,
+          CVodeAdjInit(mem->cvodes_mem_, num_steps_between_checkpoints_,
                        interpolation_polynomial_),
           "CVodeAdjInit");
     }
   }
 
-  ~cvodes_integrator_adjoint_vari() {
-    if (N_ > 0) {
-      if (cvodes_mem_) {
-        CVodeFree(&cvodes_mem_);
-      }
-
-      SUNLinSolFree(LS_forward_);
-      SUNMatDestroy(A_forward_);
-      SUNLinSolFree(LS_backward_);
-      SUNMatDestroy(A_backward_);
-
-      N_VDestroy_Serial(nv_state_forward_);
-      N_VDestroy_Serial(nv_state_backward_);
-      N_VDestroy_Serial(nv_quad_);
-      N_VDestroy_Serial(nv_absolute_tolerance_forward_);
-      N_VDestroy_Serial(nv_absolute_tolerance_backward_);
-    }
-  }
-
   /**
    * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
    * times, { t1, t2, t3, ... } using the stiff backward differentiation formula
@@ -577,26 +567,23 @@ class cvodes_integrator_adjoint_vari : public vari {
   std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> operator()() {
     // std::cout << "forward integrate..." << std::endl;
     const double t0_dbl = value_of(t0_);
-    const std::vector<double> ts_dbl = value_of(ts_);
-
-    std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return(
-        ts_.size());
+    const auto ts_dbl = value_of(ts_);
+    std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return(ts_.size());
 
     double t_init = t0_dbl;
     for (size_t n = 0; n < ts_dbl.size(); ++n) {
       double t_final = ts_dbl[n];
       // std::cout << "n = " << n << ": t_init = " << t_init << ", t_final = "
       // << t_final << std::endl;
-
       if (t_final != t_init) {
         if (num_vars_ != 0) {
           int ncheck;
 
-          int error_code = CVodeF(cvodes_mem_, t_final, nv_state_forward_,
+          int error_code = CVodeF(mem->cvodes_mem_, t_final, mem->nv_state_forward_,
                                   &t_init, CV_NORMAL, &ncheck);
 
           if (error_code == CV_TOO_MUCH_WORK) {
-            throw_domain_error(function_name_, "", t_final,
+            throw_domain_error(mem->function_name_, "", t_final,
                                "Failed to integrate to next output time (",
                                ") in less than max_num_steps steps");
           } else {
@@ -604,11 +591,11 @@ class cvodes_integrator_adjoint_vari : public vari {
           }
 
         } else {
-          int error_code = CVode(cvodes_mem_, t_final, nv_state_forward_,
+          int error_code = CVode(mem->cvodes_mem_, t_final, mem->nv_state_forward_,
                                  &t_init, CV_NORMAL);
 
           if (error_code == CV_TOO_MUCH_WORK) {
-            throw_domain_error(function_name_, "", t_final,
+            throw_domain_error(mem->function_name_, "", t_final,
                                "Failed to integrate to next output time (",
                                ") in less than max_num_steps steps");
           } else {
@@ -616,16 +603,18 @@ class cvodes_integrator_adjoint_vari : public vari {
           }
         }
       }
-
-      store_state(non_chaining_varis_ + N_ * n, state_forward_, y_return[n]);
-      y_[n] = state_forward_;
+      if (non_chaining_varis_) {
+        store_state(non_chaining_varis_ + N_ * n, state_forward_, y_return[n]);
+      } else {
+        store_state(non_chaining_varis_, state_forward_, y_return[n]);
+      }
+      mem->y_[n] = state_forward_;
 
       t_init = t_final;
     }
 
     forward_returned_ = true;
     // std::cout << "forward integrate...done" << std::endl;
-    // return internal::build_varis<T_Return>(non_chaining_varis_, y_);
     return y_return;
   }
 
@@ -636,7 +625,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     // std::cout << "chain" << std::endl; //<-- Good way to verify it's only
     //  being called once
 
-    if (cvodes_mem_ == nullptr) {
+    if (mem->cvodes_mem_ == nullptr) {
       return;
     }
 
@@ -688,7 +677,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
       if (num_ts_vars_ > 0 && i >= 0) {
         ts_varis_[i]->adj_
-            += step_sens.dot(rhs(t_init, y_[i], value_of_args_tuple_));
+            += step_sens.dot(rhs(t_init, mem->y_[i], value_of_args_tuple_));
         /*
           apply(
           [&](auto&&... args) {
@@ -707,10 +696,10 @@ class cvodes_integrator_adjoint_vari : public vari {
       if (t_final != t_init) {
         if (!backward_is_initialized_) {
           check_flag_sundials(
-              CVodeCreateB(cvodes_mem_, solver_backward_, &index_backward_),
+              CVodeCreateB(mem->cvodes_mem_, solver_backward_, &index_backward_),
               "CVodeCreateB");
 
-          check_flag_sundials(CVodeSetUserDataB(cvodes_mem_, index_backward_,
+          check_flag_sundials(CVodeSetUserDataB(mem->cvodes_mem_, index_backward_,
                                                 reinterpret_cast<void*>(this)),
                               "CVodeSetUserDataB");
 
@@ -718,28 +707,28 @@ class cvodes_integrator_adjoint_vari : public vari {
           // the states of the backward problem *are* the adjoints
           // of the ode states
           check_flag_sundials(
-              CVodeInitB(cvodes_mem_, index_backward_,
+              CVodeInitB(mem->cvodes_mem_, index_backward_,
                          &cvodes_integrator_adjoint_vari::cv_rhs_adj_sens,
-                         t_init, nv_state_backward_),
+                         t_init, mem->nv_state_backward_),
               "CVodeInitB");
 
           check_flag_sundials(
-              CVodeSVtolerancesB(cvodes_mem_, index_backward_,
+              CVodeSVtolerancesB(mem->cvodes_mem_, index_backward_,
                                  relative_tolerance_backward_,
-                                 nv_absolute_tolerance_backward_),
+                                 mem->nv_absolute_tolerance_backward_),
               "CVodeSVtolerancesB");
 
-          check_flag_sundials(CVodeSetMaxNumStepsB(cvodes_mem_, index_backward_,
+          check_flag_sundials(CVodeSetMaxNumStepsB(mem->cvodes_mem_, index_backward_,
                                                    max_num_steps_),
                               "CVodeSetMaxNumStepsB");
 
           check_flag_sundials(
-              CVodeSetLinearSolverB(cvodes_mem_, index_backward_, LS_backward_,
-                                    A_backward_),
+              CVodeSetLinearSolverB(mem->cvodes_mem_, index_backward_, mem->LS_backward_,
+                                    mem->A_backward_),
               "CVodeSetLinearSolverB");
 
           check_flag_sundials(
-              CVodeSetJacFnB(cvodes_mem_, index_backward_,
+              CVodeSetJacFnB(mem->cvodes_mem_, index_backward_,
                              &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
               "CVodeSetJacFnB");
 
@@ -747,40 +736,40 @@ class cvodes_integrator_adjoint_vari : public vari {
           // parameters vary.
           if (num_args_vars_ > 0) {
             check_flag_sundials(
-                CVodeQuadInitB(cvodes_mem_, index_backward_,
+                CVodeQuadInitB(mem->cvodes_mem_, index_backward_,
                                &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj,
-                               nv_quad_),
+                               mem->nv_quad_),
                 "CVodeQuadInitB");
 
             check_flag_sundials(
-                CVodeQuadSStolerancesB(cvodes_mem_, index_backward_,
+                CVodeQuadSStolerancesB(mem->cvodes_mem_, index_backward_,
                                        relative_tolerance_quadrature_,
                                        absolute_tolerance_quadrature_),
                 "CVodeQuadSStolerancesB");
 
             check_flag_sundials(
-                CVodeSetQuadErrConB(cvodes_mem_, index_backward_, SUNTRUE),
+                CVodeSetQuadErrConB(mem->cvodes_mem_, index_backward_, SUNTRUE),
                 "CVodeSetQuadErrConB");
           }
 
           backward_is_initialized_ = true;
         } else {
           // just re-initialize the solver
-          check_flag_sundials(CVodeReInitB(cvodes_mem_, index_backward_, t_init,
-                                           nv_state_backward_),
+          check_flag_sundials(CVodeReInitB(mem->cvodes_mem_, index_backward_, t_init,
+                                           mem->nv_state_backward_),
                               "CVodeReInitB");
 
           if (num_args_vars_ > 0) {
             check_flag_sundials(
-                CVodeQuadReInitB(cvodes_mem_, index_backward_, nv_quad_),
+                CVodeQuadReInitB(mem->cvodes_mem_, index_backward_, mem->nv_quad_),
                 "CVodeQuadReInitB");
           }
         }
 
-        int error_code = CVodeB(cvodes_mem_, t_final, CV_NORMAL);
+        int error_code = CVodeB(mem->cvodes_mem_, t_final, CV_NORMAL);
 
         if (error_code == CV_TOO_MUCH_WORK) {
-          throw_domain_error(function_name_, "", t_final,
+          throw_domain_error(mem->function_name_, "", t_final,
                              "Failed to integrate backward to output time (",
                              ") in less than max_num_steps steps");
         } else {
@@ -793,13 +782,13 @@ class cvodes_integrator_adjoint_vari : public vari {
 
         // obtain adjoint states and update t_init to time point
         // reached of t_final
-        check_flag_sundials(CVodeGetB(cvodes_mem_, index_backward_, &t_init,
-                                      nv_state_backward_),
+        check_flag_sundials(CVodeGetB(mem->cvodes_mem_, index_backward_, &t_init,
+                                      mem->nv_state_backward_),
                             "CVodeGetB");
 
         if (num_args_vars_ > 0) {
           check_flag_sundials(
-              CVodeGetQuadB(cvodes_mem_, index_backward_, &t_init, nv_quad_),
+              CVodeGetQuadB(mem->cvodes_mem_, index_backward_, &t_init, mem->nv_quad_),
               "CVodeGetQuadB");
         }
       }
diff --git a/test/unit/math/rev/functor/test_fixture_ode_fho.hpp b/test/unit/math/rev/functor/test_fixture_ode_fho.hpp
index 9e78a45accf..384221e04be 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_fho.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_fho.hpp
@@ -17,8 +17,8 @@ struct forced_harm_osc_base {
   struct forced_harm_osc {
     template <typename T0, typename T1, typename T2>
     inline Eigen::Matrix<stan::return_type_t<T0, T1, T2>, -1, 1> operator()(
-        const T0& t_in, const Eigen::Matrix<T1, -1, 1>& y_in,
-        std::ostream* msgs, const std::vector<T2>& theta) const {
+        const T0& t_in, const T1& y_in,
+        std::ostream* msgs, const T2& theta) const {
       if (y_in.size() != 2)
         throw std::domain_error(
             "this function was called with inconsistent state");
diff --git a/test/unit/math/rev/functor/test_fixture_ode_lorenz.hpp b/test/unit/math/rev/functor/test_fixture_ode_lorenz.hpp
index c7c1171aed9..ed593f83be2 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_lorenz.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_lorenz.hpp
@@ -17,8 +17,8 @@ struct lorenz_ode_base {
   struct lorenz_rhs {
     template <typename T0, typename T1, typename T2>
     inline Eigen::Matrix<stan::return_type_t<T1, T2>, -1, 1> operator()(
-        const T0& t_in, const Eigen::Matrix<T1, -1, 1>& y_in,
-        std::ostream* msgs, const std::vector<T2>& theta) const {
+        const T0& t_in, const T1& y_in,
+        std::ostream* msgs, const T2& theta) const {
       Eigen::Matrix<stan::return_type_t<T1, T2>, -1, 1> res(3);
       res << theta.at(0) * (y_in(1) - y_in(0)),
           theta.at(1) * y_in(0) - y_in(1) - y_in(0) * y_in(2),
diff --git a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
index 14c25bea139..c537bcabb54 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
@@ -17,11 +17,11 @@
 template <typename T>
 struct harmonic_oscillator_ode_base {
   struct sho_square_fun {
-    template <typename T0, typename T1, typename T2>
+    template <typename T0, typename T1, typename T2, typename T3, typename T4>
     inline Eigen::Matrix<stan::return_type_t<T1, T2>, -1, 1> operator()(
-        const T0& t_in, const Eigen::Matrix<T1, -1, 1>& y_in,
-        std::ostream* msgs, const std::vector<T2>& theta,
-        const std::vector<double>& x, const std::vector<int>& x_int) const {
+        const T0& t_in, const T1& y_in,
+        std::ostream* msgs, const T2& theta,
+        const T3& x, const T4& x_int) const {
       if (y_in.size() != 2)
         throw std::domain_error("Functor called with inconsistent state");
 

From 9d55a24b8511ceb743c79c9971c243fb611e2ecb Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 26 Apr 2021 21:31:16 +0200
Subject: [PATCH 095/157] mark functions const as appropiate

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 24 +++++++++----------
 .../math/prim/functor/ode_test_functors.hpp   |  6 ++---
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 8ca4264c6f7..b1618bbf6f3 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -165,8 +165,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * ODE RHS passed to CVODES.
    */
   static int cv_rhs(realtype t, N_Vector y, N_Vector ydot, void* user_data) {
-    cvodes_integrator_adjoint_vari* integrator
-        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->rhs(t, NV_DATA_S(y), NV_DATA_S(ydot));
     return 0;
   }
@@ -176,8 +176,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * RHS of the backward ODE system.
    */
   static int cv_rhs_adj_sens(realtype t, N_Vector y, N_Vector yB, N_Vector yBdot, void* user_data) {
-    cvodes_integrator_adjoint_vari* integrator
-        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->rhs_adj_sens(t, y, yB, yBdot);
     return 0;
   }
@@ -200,8 +200,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * major format.
    */
   static int cv_jacobian_states(realtype t, N_Vector y, N_Vector fy, SUNMatrix J, void* user_data, N_Vector tmp1, N_Vector tmp2, N_Vector tmp3) {
-    cvodes_integrator_adjoint_vari* integrator
-        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_states(t, y, J);
     return 0;
   }
@@ -211,8 +211,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * the adjoint problem.
    */
   static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB, N_Vector fyB, SUNMatrix J, void* user_data, N_Vector tmp1, N_Vector tmp2, N_Vector tmp3) {
-    cvodes_integrator_adjoint_vari* integrator
-        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
+    const cvodes_integrator_adjoint_vari* integrator
+        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_adj(t, y, J);
     return 0;
   }
@@ -221,7 +221,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Calculates the ODE RHS, dy_dt, using the user-supplied functor at
    * the given time t and state y.
    */
-  inline void rhs(double t, const double y[], double dy_dt[]) {
+  inline void rhs(double t, const double y[], double dy_dt[]) const {
     const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
 
     const Eigen::VectorXd dy_dt_vec = rhs(t, y_vec, value_of_args_tuple_);
@@ -244,7 +244,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] yB state of the adjoint ODE system
    * @param[out] yBdot evaluation of adjoint ODE RHS
    */
-  void rhs_adj_sens(double t, N_Vector y, N_Vector yB, N_Vector yBdot) {
+  void rhs_adj_sens(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
     Eigen::Map<Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
     Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(yBdot), N_);
@@ -308,7 +308,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Calculates the jacobian of the ODE RHS wrt to its states y at the
    * given time-point t and state y.
    */
-  inline void jacobian_states(double t, N_Vector y, SUNMatrix J) {
+  inline void jacobian_states(double t, N_Vector y, SUNMatrix J) const {
     Eigen::Map<Eigen::MatrixXd> Jfy(SM_DATA_D(J), N_, N_);
     Eigen::Map<const Eigen::VectorXd> x(NV_DATA_S(y), N_);
 
@@ -338,7 +338,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] t Time
    * @param[out] J CVode structure where output is to be stored
    */
-  inline void jacobian_adj(double t, N_Vector y, SUNMatrix J) {
+  inline void jacobian_adj(double t, N_Vector y, SUNMatrix J) const {
     Eigen::Map<Eigen::MatrixXd> J_adj_y(SM_DATA_D(J), N_, N_);
 
     // J_adj_y = -1 * transpose(J_y)
diff --git a/test/unit/math/prim/functor/ode_test_functors.hpp b/test/unit/math/prim/functor/ode_test_functors.hpp
index 633f04c1634..fd17d96fee0 100644
--- a/test/unit/math/prim/functor/ode_test_functors.hpp
+++ b/test/unit/math/prim/functor/ode_test_functors.hpp
@@ -28,7 +28,7 @@ auto sum_(Vec&& arg) {
 struct CosArg1 {
   template <typename T0, typename T1, typename... T_Args>
   inline Eigen::Matrix<stan::return_type_t<T1, T_Args...>, Eigen::Dynamic, 1>
-  operator()(const T0& t, const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y,
+  operator()(const T0& t, const T1& y,
              std::ostream* msgs, const T_Args&... a) const {
     std::vector<typename stan::return_type<T0, T_Args...>::type> vec
         = {sum_(a)...};
@@ -41,7 +41,7 @@ struct CosArg1 {
 struct Cos2Arg {
   template <typename T0, typename T1, typename T2, typename T3>
   inline Eigen::Matrix<stan::return_type_t<T1, T2, T3>, Eigen::Dynamic, 1>
-  operator()(const T0& t, const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y,
+  operator()(const T0& t, const T1& y,
              std::ostream* msgs, const T2& a, const T3& b) const {
     Eigen::Matrix<stan::return_type_t<T1, T2, T3>, Eigen::Dynamic, 1> out(1);
     out << stan::math::cos((sum_(a) + sum_(b)) * t);
@@ -52,7 +52,7 @@ struct Cos2Arg {
 struct CosArgWrongSize {
   template <typename T0, typename T1, typename... T_Args>
   inline Eigen::Matrix<stan::return_type_t<T1, T_Args...>, Eigen::Dynamic, 1>
-  operator()(const T0& t, const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y,
+  operator()(const T0& t, const T1& y,
              std::ostream* msgs, const T_Args&... a) const {
     std::vector<typename stan::return_type<T0, T_Args...>::type> vec
         = {sum_(a)...};

From 7b6dd5eb2b045df12d7f89152c64c2aa31f71e6a Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 26 Apr 2021 22:03:29 +0200
Subject: [PATCH 096/157] more merging from review3

---
 stan/math/rev/functor/coupled_ode_system.hpp  |   2 +-
 .../rev/functor/cvodes_integrator_adjoint.hpp | 294 +++++++++---------
 .../math/prim/functor/harmonic_oscillator.hpp |  62 ++--
 3 files changed, 185 insertions(+), 173 deletions(-)

diff --git a/stan/math/rev/functor/coupled_ode_system.hpp b/stan/math/rev/functor/coupled_ode_system.hpp
index 83783d4d03f..08b7dd557fe 100644
--- a/stan/math/rev/functor/coupled_ode_system.hpp
+++ b/stan/math/rev/functor/coupled_ode_system.hpp
@@ -148,7 +148,7 @@ struct coupled_ode_system_impl<false, F, T_y0, Args...> {
 
       // The vars here do not live on the nested stack so must be zero'd
       // separately
-      apply([&](auto&&... args) { zero_adjoints(args...); }, local_args_tuple_);
+      stan::math::for_each([](auto&& arg) { zero_adjoints(arg); }, local_args_tuple_);
 
       // No need to zero adjoints after last sweep
       if (i + 1 < N_) {
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index b1618bbf6f3..93a11e7eca4 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -20,69 +20,6 @@
 namespace stan {
 namespace math {
 
-class cvodes_integrator_adjoint_mem : public chainable_alloc {
-public:
-  size_t N_;
-  std::vector<Eigen::VectorXd> y_;
-  const char* function_name_;
-  void* cvodes_mem_;
-  N_Vector nv_state_forward_;
-  N_Vector nv_state_backward_;
-  N_Vector nv_quad_;
-  N_Vector nv_absolute_tolerance_forward_;
-  N_Vector nv_absolute_tolerance_backward_;
-
-  SUNMatrix A_forward_;
-  SUNLinearSolver LS_forward_;
-
-  SUNMatrix A_backward_;
-  SUNLinearSolver LS_backward_;
-  template <typename StateFwd, typename StateBwd, typename Quad,
-  typename AbsTolFwd, typename AbsTolBwd>
-  cvodes_integrator_adjoint_mem(const char* function_name, size_t N,
-    size_t num_args_vars, size_t ts_size, int solver_forward, StateFwd& state_forward,
-    StateBwd& state_backward, Quad& quad,
-    AbsTolFwd& absolute_tolerance_forward,
-    AbsTolBwd& absolute_tolerance_backward) : chainable_alloc(), N_(N), y_(ts_size),
-    function_name_(function_name) {
-      nv_state_forward_ = N_VMake_Serial(N, state_forward.data());
-      nv_state_backward_ = N_VMake_Serial(N, state_backward.data());
-      nv_quad_ = N_VMake_Serial(num_args_vars, quad.data());
-      nv_absolute_tolerance_forward_
-          = N_VMake_Serial(N, absolute_tolerance_forward.data());
-      nv_absolute_tolerance_backward_
-          = N_VMake_Serial(N, absolute_tolerance_backward.data());
-
-      A_forward_ = SUNDenseMatrix(N, N);
-      A_backward_ = SUNDenseMatrix(N, N);
-      if (N > 0 ) {
-        LS_forward_ = SUNDenseLinearSolver(nv_state_forward_, A_forward_);
-        LS_backward_ = SUNDenseLinearSolver(nv_state_backward_, A_backward_);
-      }
-      cvodes_mem_ = CVodeCreate(solver_forward);
-
-      if (cvodes_mem_ == nullptr) {
-        throw std::runtime_error("CVodeCreate failed to allocate memory");
-      }
-  }
-  virtual ~cvodes_integrator_adjoint_mem() {
-      SUNMatDestroy(A_forward_);
-      SUNMatDestroy(A_backward_);
-      if (N_ > 0) {
-        SUNLinSolFree(LS_forward_);
-        SUNLinSolFree(LS_backward_);
-      }
-      N_VDestroy_Serial(nv_state_forward_);
-      N_VDestroy_Serial(nv_state_backward_);
-      N_VDestroy_Serial(nv_quad_);
-      N_VDestroy_Serial(nv_absolute_tolerance_forward_);
-      N_VDestroy_Serial(nv_absolute_tolerance_backward_);
-
-      if (cvodes_mem_) {
-        CVodeFree(&cvodes_mem_);
-      }
-  }
-};
 
 /**
  * Integrator interface for CVODES' ODE solvers (Adams & BDF
@@ -139,9 +76,6 @@ class cvodes_integrator_adjoint_vari : public vari {
   arena_t<Eigen::VectorXd> state_backward_;
   arena_t<Eigen::VectorXd> quad_;
 
-  //std::vector<arena_t<Eigen::VectorXd>, arena_allocator<arena_t<Eigen::VectorXd>>> y_;
-
-
   vari** non_chaining_varis_;
 
   vari** t0_varis_;
@@ -150,7 +84,17 @@ class cvodes_integrator_adjoint_vari : public vari {
   vari** args_varis_;
 
   int index_backward_;
-  cvodes_integrator_adjoint_mem* mem;
+  
+  struct cvodes_solver;
+  cvodes_solver* solver_;
+
+  static constexpr bool is_var_ts{is_var<T_ts>::value};
+  static constexpr bool is_var_t0{is_var<T_t0>::value};
+  static constexpr bool is_var_y0{is_var<T_y0_t0>::value};
+  //static constexpr std::array<bool, sizeof...(T_Args)> is_var_args{is_var<scalar_type_t<T_Args>>::value...};
+  static constexpr bool is_any_var_args{disjunction<is_var<scalar_type_t<T_Args>>...>::value};
+  static constexpr bool is_var_return{is_var<T_Return>::value};
+  
   /**
    * Call the ODE RHS with given tuple.
    */
@@ -226,7 +170,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     const Eigen::VectorXd dy_dt_vec = rhs(t, y_vec, value_of_args_tuple_);
 
-    check_size_match(mem->function_name_, "dy_dt", dy_dt_vec.size(), "states", N_);
+    check_size_match(solver_->function_name_, "dy_dt", dy_dt_vec.size(), "states", N_);
 
     Eigen::Map<Eigen::VectorXd>(dy_dt, N_) = dy_dt_vec;
   }
@@ -257,7 +201,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vars, value_of_args_tuple_);
 
-    check_size_match(mem->function_name_, "dy_dt", f_y_t_vars.size(), "states", N_);
+    check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(), "states", N_);
 
     f_y_t_vars.adj() = -mu;
 
@@ -289,12 +233,12 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     // The vars here do not live on the nested stack so must be zero'd
     // separately
-    apply([&](auto&&... args) { zero_adjoints(args...); }, local_args_tuple_);
+    stan::math::for_each([](auto&& arg) { zero_adjoints(arg); }, local_args_tuple_);
 
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vec, local_args_tuple_);
 
-    check_size_match(mem->function_name_, "dy_dt", f_y_t_vars.size(), "states", N_);
+    check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(), "states", N_);
 
     f_y_t_vars.adj() = -mu;
 
@@ -318,7 +262,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var
         = rhs(t, y_var, value_of_args_tuple_);
 
-    check_size_match(mem->function_name_, "dy_dt", fy_var.size(), "states", N_);
+    check_size_match(solver_->function_name_, "dy_dt", fy_var.size(), "states", N_);
 
     grad(fy_var.coeffRef(0).vi_);
     Jfy.col(0) = y_var.adj();
@@ -351,22 +295,95 @@ class cvodes_integrator_adjoint_vari : public vari {
   /**
    * Overloads which setup the states returned from the forward solve. In case the return type is a double only, then no autodiff is needed. In case of autodiff non-chaining varis are setup accordingly.
    */
-  void store_state(vari** non_chaining_varis,
+  void store_state(std::size_t n,
                    const Eigen::VectorXd& state,
                    Eigen::Matrix<var, Eigen::Dynamic, 1>& state_return) {
+    solver_->y_[n] = state;
     state_return.resize(N_);
     for (size_t i = 0; i < N_; i++) {
-      non_chaining_varis[i] = new vari(state.coeff(i), false);
-      state_return.coeffRef(i) = var(non_chaining_varis[i]);
+      non_chaining_varis_[N_ * n + i] = new vari(state.coeff(i), false);
+      state_return.coeffRef(i) = var(non_chaining_varis_[N_ * n + i]);
     }
   }
 
-  void store_state(vari** non_chaining_varis,
+  void store_state(std::size_t n,
                    const Eigen::VectorXd& state,
                    Eigen::Matrix<double, Eigen::Dynamic, 1>& state_return) {
+    solver_->y_[n] = state;
     state_return = state;
   }
 
+  /**
+   * Since the CVODES solver manages memory with malloc calls, these resources must be freed using a destructor call (which is not being called for the vari class).
+   */
+  struct cvodes_solver : public chainable_alloc {
+    size_t N_;
+    std::vector<Eigen::VectorXd> y_;
+    const std::string function_name_str_;
+    const char* function_name_;
+    void* cvodes_mem_;
+    N_Vector nv_state_forward_;
+    N_Vector nv_state_backward_;
+    N_Vector nv_quad_;
+    N_Vector nv_absolute_tolerance_forward_;
+    N_Vector nv_absolute_tolerance_backward_;
+
+    SUNMatrix A_forward_;
+    SUNLinearSolver LS_forward_;
+
+    SUNMatrix A_backward_;
+    SUNLinearSolver LS_backward_;
+    template <typename StateFwd, typename StateBwd, typename Quad,
+              typename AbsTolFwd, typename AbsTolBwd>
+    cvodes_solver(const char* function_name, size_t N,
+                  size_t num_args_vars, size_t ts_size, int solver_forward, StateFwd& state_forward,
+                  StateBwd& state_backward, Quad& quad,
+                  AbsTolFwd& absolute_tolerance_forward,
+                  AbsTolBwd& absolute_tolerance_backward) :
+        chainable_alloc(), N_(N), y_(ts_size),
+        function_name_str_(function_name),
+        function_name_(function_name_str_.c_str()) {
+      nv_state_forward_ = N_VMake_Serial(N, state_forward.data());
+      nv_state_backward_ = N_VMake_Serial(N, state_backward.data());
+      nv_quad_ = N_VMake_Serial(num_args_vars, quad.data());
+      nv_absolute_tolerance_forward_
+          = N_VMake_Serial(N, absolute_tolerance_forward.data());
+      nv_absolute_tolerance_backward_
+          = N_VMake_Serial(N, absolute_tolerance_backward.data());
+      
+      A_forward_ = SUNDenseMatrix(N, N);
+      A_backward_ = SUNDenseMatrix(N, N);
+      if (N > 0 ) {
+        LS_forward_ = SUNDenseLinearSolver(nv_state_forward_, A_forward_);
+        LS_backward_ = SUNDenseLinearSolver(nv_state_backward_, A_backward_);
+      }
+      cvodes_mem_ = CVodeCreate(solver_forward);
+      
+      if (cvodes_mem_ == nullptr) {
+        throw std::runtime_error("CVodeCreate failed to allocate memory");
+      }
+    }
+    
+    virtual ~cvodes_solver() {
+      SUNMatDestroy(A_forward_);
+      SUNMatDestroy(A_backward_);
+      if (N_ > 0) {
+        SUNLinSolFree(LS_forward_);
+        SUNLinSolFree(LS_backward_);
+      }
+      N_VDestroy_Serial(nv_state_forward_);
+      N_VDestroy_Serial(nv_state_backward_);
+      N_VDestroy_Serial(nv_quad_);
+      N_VDestroy_Serial(nv_absolute_tolerance_forward_);
+      N_VDestroy_Serial(nv_absolute_tolerance_backward_);
+      
+      if (cvodes_mem_) {
+        CVodeFree(&cvodes_mem_);
+      }
+    }
+  };
+
+
  public:
   /**
    * Construct cvodes_integrator object. Note: All arguments must be stored as
@@ -465,7 +482,7 @@ class cvodes_integrator_adjoint_vari : public vari {
             num_y0_vars_)),
         args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             num_args_vars_)),
-        mem(new cvodes_integrator_adjoint_mem(function_name, N_,
+        solver_(new cvodes_solver(function_name, N_,
             num_args_vars_, ts_.size(), solver_forward_, state_forward_,
             state_backward_, quad_,
             absolute_tolerance_forward_,
@@ -475,82 +492,82 @@ class cvodes_integrator_adjoint_vari : public vari {
     save_varis(y0_varis_, y0);
     save_varis(args_varis_, args...);
 
-    check_finite(mem->function_name_, "initial state", y0);
-    check_finite(mem->function_name_, "initial time", t0);
-    check_finite(mem->function_name_, "times", ts);
+    check_finite(solver_->function_name_, "initial state", y0);
+    check_finite(solver_->function_name_, "initial time", t0);
+    check_finite(solver_->function_name_, "times", ts);
 
     // Code from: https://stackoverflow.com/a/17340003 . Should probably do
     // something better
     apply(
         [&](auto&&... args) {
           std::vector<int> unused_temp{
-              0, (check_finite(mem->function_name_, "ode parameters and data", args),
+              0, (check_finite(solver_->function_name_, "ode parameters and data", args),
                   0)...};
         },
         local_args_tuple_);
 
-    check_nonzero_size(mem->function_name_, "times", ts);
-    check_nonzero_size(mem->function_name_, "initial state", y0);
-    check_sorted(mem->function_name_, "times", ts);
-    check_less(mem->function_name_, "initial time", t0, ts[0]);
-    check_positive_finite(mem->function_name_, "relative_tolerance_forward",
+    check_nonzero_size(solver_->function_name_, "times", ts);
+    check_nonzero_size(solver_->function_name_, "initial state", y0);
+    check_sorted(solver_->function_name_, "times", ts);
+    check_less(solver_->function_name_, "initial time", t0, ts[0]);
+    check_positive_finite(solver_->function_name_, "relative_tolerance_forward",
                           relative_tolerance_forward_);
-    check_positive_finite(mem->function_name_, "absolute_tolerance_forward",
+    check_positive_finite(solver_->function_name_, "absolute_tolerance_forward",
                           absolute_tolerance_forward_);
-    check_size_match(mem->function_name_, "absolute_tolerance_forward",
+    check_size_match(solver_->function_name_, "absolute_tolerance_forward",
                      absolute_tolerance_forward_.size(), "states", N_);
-    check_positive_finite(mem->function_name_, "relative_tolerance_backward",
+    check_positive_finite(solver_->function_name_, "relative_tolerance_backward",
                           relative_tolerance_backward_);
-    check_positive_finite(mem->function_name_, "absolute_tolerance_backward",
+    check_positive_finite(solver_->function_name_, "absolute_tolerance_backward",
                           absolute_tolerance_backward_);
-    check_size_match(mem->function_name_, "absolute_tolerance_backward",
+    check_size_match(solver_->function_name_, "absolute_tolerance_backward",
                      absolute_tolerance_backward_.size(), "states", N_);
-    check_positive_finite(mem->function_name_, "relative_tolerance_quadrature",
+    check_positive_finite(solver_->function_name_, "relative_tolerance_quadrature",
                           relative_tolerance_quadrature_);
-    check_positive_finite(mem->function_name_, "absolute_tolerance_quadrature",
+    check_positive_finite(solver_->function_name_, "absolute_tolerance_quadrature",
                           absolute_tolerance_quadrature_);
-    check_positive(mem->function_name_, "max_num_steps", max_num_steps_);
-    check_positive(mem->function_name_, "num_steps_between_checkpoints",
+    check_positive(solver_->function_name_, "max_num_steps", max_num_steps_);
+    check_positive(solver_->function_name_, "num_steps_between_checkpoints",
                    num_steps_between_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
-    check_range(mem->function_name_, "interpolation_polynomial", 2,
+    check_range(solver_->function_name_, "interpolation_polynomial", 2,
                 interpolation_polynomial_);
     // 1=Adams=CV_ADAMS, 2=BDF=CV_BDF
-    check_range(mem->function_name_, "solver_forward", 2, solver_forward_);
-    check_range(mem->function_name_, "solver_backward", 2, solver_backward_);
+    check_range(solver_->function_name_, "solver_forward", 2, solver_forward_);
+    check_range(solver_->function_name_, "solver_backward", 2, solver_backward_);
 
 
     check_flag_sundials(
-        CVodeInit(mem->cvodes_mem_, &cvodes_integrator_adjoint_vari::cv_rhs,
-                  value_of(t0_), mem->nv_state_forward_),
+        CVodeInit(solver_->cvodes_mem_, &cvodes_integrator_adjoint_vari::cv_rhs,
+                  value_of(t0_), solver_->nv_state_forward_),
         "CVodeInit");
 
     // Assign pointer to this as user data
     check_flag_sundials(
-        CVodeSetUserData(mem->cvodes_mem_, reinterpret_cast<void*>(this)),
+        CVodeSetUserData(solver_->cvodes_mem_, reinterpret_cast<void*>(this)),
         "CVodeSetUserData");
 
-    cvodes_set_options(mem->cvodes_mem_, relative_tolerance_forward_,
+    cvodes_set_options(solver_->cvodes_mem_, relative_tolerance_forward_,
                        absolute_tolerance_forward_(0), max_num_steps_);
 
     check_flag_sundials(
-        CVodeSVtolerances(mem->cvodes_mem_, relative_tolerance_forward_,
-                          mem->nv_absolute_tolerance_forward_),
+        CVodeSVtolerances(solver_->cvodes_mem_, relative_tolerance_forward_,
+                          solver_->nv_absolute_tolerance_forward_),
         "CVodeSVtolerances");
 
     check_flag_sundials(
-        CVodeSetLinearSolver(mem->cvodes_mem_, mem->LS_forward_, mem->A_forward_),
+        CVodeSetLinearSolver(solver_->cvodes_mem_, solver_->LS_forward_, solver_->A_forward_),
         "CVodeSetLinearSolver");
 
     check_flag_sundials(
-        CVodeSetJacFn(mem->cvodes_mem_,
+        CVodeSetJacFn(solver_->cvodes_mem_,
                       &cvodes_integrator_adjoint_vari::cv_jacobian_states),
         "CVodeSetJacFn");
 
     // initialize backward sensitivity system of CVODES as needed
     if (num_vars_ != 0) {
       check_flag_sundials(
-          CVodeAdjInit(mem->cvodes_mem_, num_steps_between_checkpoints_,
+          CVodeAdjInit(solver_->cvodes_mem_, num_steps_between_checkpoints_,
                        interpolation_polynomial_),
           "CVodeAdjInit");
     }
@@ -579,11 +596,11 @@ class cvodes_integrator_adjoint_vari : public vari {
         if (num_vars_ != 0) {
           int ncheck;
 
-          int error_code = CVodeF(mem->cvodes_mem_, t_final, mem->nv_state_forward_,
+          int error_code = CVodeF(solver_->cvodes_mem_, t_final, solver_->nv_state_forward_,
                                   &t_init, CV_NORMAL, &ncheck);
 
           if (error_code == CV_TOO_MUCH_WORK) {
-            throw_domain_error(mem->function_name_, "", t_final,
+            throw_domain_error(solver_->function_name_, "", t_final,
                                "Failed to integrate to next output time (",
                                ") in less than max_num_steps steps");
           } else {
@@ -591,11 +608,11 @@ class cvodes_integrator_adjoint_vari : public vari {
           }
 
         } else {
-          int error_code = CVode(mem->cvodes_mem_, t_final, mem->nv_state_forward_,
+          int error_code = CVode(solver_->cvodes_mem_, t_final, solver_->nv_state_forward_,
                                  &t_init, CV_NORMAL);
 
           if (error_code == CV_TOO_MUCH_WORK) {
-            throw_domain_error(mem->function_name_, "", t_final,
+            throw_domain_error(solver_->function_name_, "", t_final,
                                "Failed to integrate to next output time (",
                                ") in less than max_num_steps steps");
           } else {
@@ -603,12 +620,7 @@ class cvodes_integrator_adjoint_vari : public vari {
           }
         }
       }
-      if (non_chaining_varis_) {
-        store_state(non_chaining_varis_ + N_ * n, state_forward_, y_return[n]);
-      } else {
-        store_state(non_chaining_varis_, state_forward_, y_return[n]);
-      }
-      mem->y_[n] = state_forward_;
+      store_state(n, state_forward_, y_return[n]);
 
       t_init = t_final;
     }
@@ -625,7 +637,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     // std::cout << "chain" << std::endl; //<-- Good way to verify it's only
     //  being called once
 
-    if (mem->cvodes_mem_ == nullptr) {
+    if (solver_->cvodes_mem_ == nullptr) {
       return;
     }
 
@@ -677,7 +689,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
       if (num_ts_vars_ > 0 && i >= 0) {
         ts_varis_[i]->adj_
-            += step_sens.dot(rhs(t_init, mem->y_[i], value_of_args_tuple_));
+            += step_sens.dot(rhs(t_init, solver_->y_[i], value_of_args_tuple_));
         /*
           apply(
           [&](auto&&... args) {
@@ -696,10 +708,10 @@ class cvodes_integrator_adjoint_vari : public vari {
       if (t_final != t_init) {
         if (!backward_is_initialized_) {
           check_flag_sundials(
-              CVodeCreateB(mem->cvodes_mem_, solver_backward_, &index_backward_),
+              CVodeCreateB(solver_->cvodes_mem_, solver_backward_, &index_backward_),
               "CVodeCreateB");
 
-          check_flag_sundials(CVodeSetUserDataB(mem->cvodes_mem_, index_backward_,
+          check_flag_sundials(CVodeSetUserDataB(solver_->cvodes_mem_, index_backward_,
                                                 reinterpret_cast<void*>(this)),
                               "CVodeSetUserDataB");
 
@@ -707,28 +719,28 @@ class cvodes_integrator_adjoint_vari : public vari {
           // the states of the backward problem *are* the adjoints
           // of the ode states
           check_flag_sundials(
-              CVodeInitB(mem->cvodes_mem_, index_backward_,
+              CVodeInitB(solver_->cvodes_mem_, index_backward_,
                          &cvodes_integrator_adjoint_vari::cv_rhs_adj_sens,
-                         t_init, mem->nv_state_backward_),
+                         t_init, solver_->nv_state_backward_),
               "CVodeInitB");
 
           check_flag_sundials(
-              CVodeSVtolerancesB(mem->cvodes_mem_, index_backward_,
+              CVodeSVtolerancesB(solver_->cvodes_mem_, index_backward_,
                                  relative_tolerance_backward_,
-                                 mem->nv_absolute_tolerance_backward_),
+                                 solver_->nv_absolute_tolerance_backward_),
               "CVodeSVtolerancesB");
 
-          check_flag_sundials(CVodeSetMaxNumStepsB(mem->cvodes_mem_, index_backward_,
+          check_flag_sundials(CVodeSetMaxNumStepsB(solver_->cvodes_mem_, index_backward_,
                                                    max_num_steps_),
                               "CVodeSetMaxNumStepsB");
 
           check_flag_sundials(
-              CVodeSetLinearSolverB(mem->cvodes_mem_, index_backward_, mem->LS_backward_,
-                                    mem->A_backward_),
+              CVodeSetLinearSolverB(solver_->cvodes_mem_, index_backward_, solver_->LS_backward_,
+                                    solver_->A_backward_),
               "CVodeSetLinearSolverB");
 
           check_flag_sundials(
-              CVodeSetJacFnB(mem->cvodes_mem_, index_backward_,
+              CVodeSetJacFnB(solver_->cvodes_mem_, index_backward_,
                              &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
               "CVodeSetJacFnB");
 
@@ -736,40 +748,40 @@ class cvodes_integrator_adjoint_vari : public vari {
           // parameters vary.
           if (num_args_vars_ > 0) {
             check_flag_sundials(
-                CVodeQuadInitB(mem->cvodes_mem_, index_backward_,
+                CVodeQuadInitB(solver_->cvodes_mem_, index_backward_,
                                &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj,
-                               mem->nv_quad_),
+                               solver_->nv_quad_),
                 "CVodeQuadInitB");
 
             check_flag_sundials(
-                CVodeQuadSStolerancesB(mem->cvodes_mem_, index_backward_,
+                CVodeQuadSStolerancesB(solver_->cvodes_mem_, index_backward_,
                                        relative_tolerance_quadrature_,
                                        absolute_tolerance_quadrature_),
                 "CVodeQuadSStolerancesB");
 
             check_flag_sundials(
-                CVodeSetQuadErrConB(mem->cvodes_mem_, index_backward_, SUNTRUE),
+                CVodeSetQuadErrConB(solver_->cvodes_mem_, index_backward_, SUNTRUE),
                 "CVodeSetQuadErrConB");
           }
 
           backward_is_initialized_ = true;
         } else {
           // just re-initialize the solver
-          check_flag_sundials(CVodeReInitB(mem->cvodes_mem_, index_backward_, t_init,
-                                           mem->nv_state_backward_),
+          check_flag_sundials(CVodeReInitB(solver_->cvodes_mem_, index_backward_, t_init,
+                                           solver_->nv_state_backward_),
                               "CVodeReInitB");
 
           if (num_args_vars_ > 0) {
             check_flag_sundials(
-                CVodeQuadReInitB(mem->cvodes_mem_, index_backward_, mem->nv_quad_),
+                CVodeQuadReInitB(solver_->cvodes_mem_, index_backward_, solver_->nv_quad_),
                 "CVodeQuadReInitB");
           }
         }
 
-        int error_code = CVodeB(mem->cvodes_mem_, t_final, CV_NORMAL);
+        int error_code = CVodeB(solver_->cvodes_mem_, t_final, CV_NORMAL);
 
         if (error_code == CV_TOO_MUCH_WORK) {
-          throw_domain_error(mem->function_name_, "", t_final,
+          throw_domain_error(solver_->function_name_, "", t_final,
                              "Failed to integrate backward to output time (",
                              ") in less than max_num_steps steps");
         } else {
@@ -782,13 +794,13 @@ class cvodes_integrator_adjoint_vari : public vari {
 
         // obtain adjoint states and update t_init to time point
         // reached of t_final
-        check_flag_sundials(CVodeGetB(mem->cvodes_mem_, index_backward_, &t_init,
-                                      mem->nv_state_backward_),
+        check_flag_sundials(CVodeGetB(solver_->cvodes_mem_, index_backward_, &t_init,
+                                      solver_->nv_state_backward_),
                             "CVodeGetB");
 
         if (num_args_vars_ > 0) {
           check_flag_sundials(
-              CVodeGetQuadB(mem->cvodes_mem_, index_backward_, &t_init, mem->nv_quad_),
+              CVodeGetQuadB(solver_->cvodes_mem_, index_backward_, &t_init, solver_->nv_quad_),
               "CVodeGetQuadB");
         }
       }
diff --git a/test/unit/math/prim/functor/harmonic_oscillator.hpp b/test/unit/math/prim/functor/harmonic_oscillator.hpp
index 458c5c2ca4b..851c0d08601 100644
--- a/test/unit/math/prim/functor/harmonic_oscillator.hpp
+++ b/test/unit/math/prim/functor/harmonic_oscillator.hpp
@@ -6,16 +6,16 @@
 #include <vector>
 
 struct harm_osc_ode_fun {
-  template <typename T0, typename T1, typename T2>
-  inline std::vector<stan::return_type_t<T1, T2>>
   // initial time
   // initial positions
   // parameters
   // double data
   // integer data
-  operator()(const T0& t_in, const std::vector<T1>& y_in,
-             const std::vector<T2>& theta, const std::vector<double>& x,
-             const std::vector<int>& x_int, std::ostream* msgs) const {
+  template <typename T0, typename T1, typename T2, typename T3, typename T4>
+  inline std::vector<stan::return_type_t<T1, T2>>
+  operator()(const T0& t_in, const T1& y_in,
+             const T2& theta, const T3& x,
+             const T4& x_int, std::ostream* msgs) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
@@ -29,12 +29,12 @@ struct harm_osc_ode_fun {
 };
 
 struct harm_osc_ode_fun_eigen {
-  template <typename T0, typename T1, typename T2>
+  template <typename T0, typename T1, typename T2, typename T3, typename T4>
   inline auto operator()(const T0& t_in,
-                         const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y_in,
-                         std::ostream* msgs, const std::vector<T2>& theta,
-                         const std::vector<double>& x,
-                         const std::vector<int>& x_int) const {
+                         const T1& y_in,
+                         std::ostream* msgs, const T2& theta,
+                         const T3& x,
+                         const T4& x_int) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
@@ -48,16 +48,16 @@ struct harm_osc_ode_fun_eigen {
 };
 
 struct harm_osc_ode_data_fun {
-  template <typename T0, typename T1, typename T2>
-  inline std::vector<stan::return_type_t<T1, T2>>
   // initial time
   // initial positions
   // parameters
   // double data
   // integer data
-  operator()(const T0& t_in, const std::vector<T1>& y_in,
-             const std::vector<T2>& theta, const std::vector<double>& x,
-             const std::vector<int>& x_int, std::ostream* msgs) const {
+  template <typename T0, typename T1, typename T2, typename T3, typename T4>
+  inline std::vector<stan::return_type_t<T1, T2>>
+  operator()(const T0& t_in, const T1& y_in,
+             const T2& theta, const T3& x,
+             const T4& x_int, std::ostream* msgs) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
@@ -72,17 +72,17 @@ struct harm_osc_ode_data_fun {
 };
 
 struct harm_osc_ode_data_fun_eigen {
-  template <typename T0, typename T1, typename T2>
+  template <typename T0, typename T1, typename T2, typename T3, typename T4>
   inline auto operator()(const T0& t_in,
-                         const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y_in,
-                         std::ostream* msgs, const std::vector<T2>& theta,
-                         const std::vector<double>& x,
-                         const std::vector<int>& x_int) const {
+                         const T1& y_in,
+                         std::ostream* msgs, const T2& theta,
+                         const T3& x,
+                         const T4& x_int) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
 
-    const T2& p = theta.at(0);
+    const auto& p = theta.at(0);
 
     Eigen::Matrix<stan::return_type_t<T1, T2>, -1, 1> res(2);
     res << (x.at(0) * y_in(1) + x_int.at(0)),
@@ -93,16 +93,16 @@ struct harm_osc_ode_data_fun_eigen {
 };
 
 struct harm_osc_ode_wrong_size_1_fun {
-  template <typename T0, typename T1, typename T2>
-  inline std::vector<stan::return_type_t<T1, T2>>
   // initial time
   // initial positions
   // parameters
   // double data
   // integer data
-  operator()(const T0& t_in, const std::vector<T1>& y_in,
-             const std::vector<T2>& theta, const std::vector<double>& x,
-             const std::vector<int>& x_int, std::ostream* msgs) const {
+  template <typename T0, typename T1, typename T2, typename T3, typename T4>
+  inline std::vector<stan::return_type_t<T1, T2>>
+  operator()(const T0& t_in, const T1& y_in,
+             const T2& theta, const T3& x,
+             const T4& x_int, std::ostream* msgs) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
@@ -117,16 +117,16 @@ struct harm_osc_ode_wrong_size_1_fun {
 };
 
 struct harm_osc_ode_wrong_size_2_fun {
-  template <typename T0, typename T1, typename T2>
-  inline std::vector<stan::return_type_t<T1, T2>>
   // initial time
   // initial positions
   // parameters
   // double data
   // integer data
-  operator()(const T0& t_in, const std::vector<T1>& y_in,
-             const std::vector<T2>& theta, const std::vector<double>& x,
-             const std::vector<int>& x_int, std::ostream* msgs) const {
+  template <typename T0, typename T1, typename T2, typename T3, typename T4>
+  inline std::vector<stan::return_type_t<T1, T2>>
+  operator()(const T0& t_in, const T1& y_in,
+             const T2& theta, const T3& x,
+             const T4& x_int, std::ostream* msgs) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");

From ace3931435f60e30d6e5d79be27a43105c540392 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 26 Apr 2021 23:28:39 +0200
Subject: [PATCH 097/157] optimize ts only var case

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 65 ++++++++++++-------
 1 file changed, 40 insertions(+), 25 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 93a11e7eca4..9e4d73056ae 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -94,6 +94,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   //static constexpr std::array<bool, sizeof...(T_Args)> is_var_args{is_var<scalar_type_t<T_Args>>::value...};
   static constexpr bool is_any_var_args{disjunction<is_var<scalar_type_t<T_Args>>...>::value};
   static constexpr bool is_var_return{is_var<T_Return>::value};
+  static constexpr bool is_var_only_ts{is_var_ts && !(is_var_t0 || is_var_y0 || is_any_var_args)};
   
   /**
    * Call the ODE RHS with given tuple.
@@ -471,9 +472,9 @@ class cvodes_integrator_adjoint_vari : public vari {
         state_backward_(N_),
         quad_(num_args_vars_),
 
-        non_chaining_varis_(num_vars_ == 0 ?
-                            nullptr :
-                            ChainableStack::instance_->memalloc_.alloc_array<vari*>(ts_.size() * N_)),
+        non_chaining_varis_(is_var_return ?
+                            ChainableStack::instance_->memalloc_.alloc_array<vari*>(ts_.size() * N_) :
+                            nullptr),
         t0_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             num_t0_vars_)),
         ts_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
@@ -565,7 +566,7 @@ class cvodes_integrator_adjoint_vari : public vari {
         "CVodeSetJacFn");
 
     // initialize backward sensitivity system of CVODES as needed
-    if (num_vars_ != 0) {
+    if (is_var_return) {
       check_flag_sundials(
           CVodeAdjInit(solver_->cvodes_mem_, num_steps_between_checkpoints_,
                        interpolation_polynomial_),
@@ -593,7 +594,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       // std::cout << "n = " << n << ": t_init = " << t_init << ", t_final = "
       // << t_final << std::endl;
       if (t_final != t_init) {
-        if (num_vars_ != 0) {
+        if (is_var_return) {
           int ncheck;
 
           int error_code = CVodeF(solver_->cvodes_mem_, t_final, solver_->nv_state_forward_,
@@ -645,10 +646,39 @@ class cvodes_integrator_adjoint_vari : public vari {
       return;
     }
 
-    if (num_vars_ == 0) {
+    if (!is_var_return) {
       return;
     }
 
+    // for sensitivities wrt to ts we do not need to run the backward
+    // integration
+    if (is_var_ts) {
+      for (int i = 0; i < ts_.size(); ++i) {
+        Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
+        for (int j = 0; j < N_; j++) {
+          // std::cout << "i: " << i << ", j: " << j << std::endl;
+          step_sens.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
+        }
+
+        ts_varis_[i]->adj_
+            += step_sens.dot(rhs(value_of(ts_[i]), solver_->y_[i], value_of_args_tuple_));
+        /*
+          apply(
+          [&](auto&&... args) {
+          double adj = step_sens.dot(
+          memory->f_(t_init, memory->y_[i], msgs_, args...));
+          // std::cout << "adj: " << adj << ", i: " << i << std::endl;
+          return adj;
+            },
+            memory->value_of_args_tuple_);
+        */
+      }
+
+      if (is_var_only_ts) {
+        return;
+      }
+    }
+
     state_backward_.setZero();
     quad_.setZero();
     // N_VConst(0.0, memory->nv_state_sens_);
@@ -680,29 +710,14 @@ class cvodes_integrator_adjoint_vari : public vari {
     for (int i = ts_.size() - 1; i >= 0; --i) {
       // Take in the adjoints from all the output variables at this point
       // in time
-      Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
+      //Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
       for (int j = 0; j < N_; j++) {
         // std::cout << "i: " << i << ", j: " << j << std::endl;
         state_backward_.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
-        step_sens.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
-      }
-
-      if (num_ts_vars_ > 0 && i >= 0) {
-        ts_varis_[i]->adj_
-            += step_sens.dot(rhs(t_init, solver_->y_[i], value_of_args_tuple_));
-        /*
-          apply(
-          [&](auto&&... args) {
-          double adj = step_sens.dot(
-          memory->f_(t_init, memory->y_[i], msgs_, args...));
-          // std::cout << "adj: " << adj << ", i: " << i << std::endl;
-          return adj;
-          },
-          memory->value_of_args_tuple_);
-        */
+        //step_sens.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
       }
 
-      double t_final = value_of((i > 0) ? ts_[i - 1] : t0_);
+     double t_final = value_of((i > 0) ? ts_[i - 1] : t0_);
       // std::cout << "backward: time-point " << i << "; t_init = " << t_init <<
       // "; t_final = " << t_final << std::endl;
       if (t_final != t_init) {
@@ -806,7 +821,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       }
     }
 
-    if (num_t0_vars_ > 0) {
+    if (is_var_t0) {
       Eigen::VectorXd y0d = value_of(y0_);
       t0_varis_[0]->adj_
           += -state_backward_.dot(rhs(t_init, y0d, value_of_args_tuple_));

From 6e10a7db9d148c9ed6f88032be7ace6f0770103b Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Mon, 26 Apr 2021 21:32:07 +0000
Subject: [PATCH 098/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/core/zero_adjoints.hpp          |   9 +-
 stan/math/rev/functor/coupled_ode_system.hpp  |   3 +-
 .../rev/functor/cvodes_integrator_adjoint.hpp | 203 ++++++++++--------
 .../math/prim/functor/harmonic_oscillator.hpp |  42 ++--
 .../math/prim/functor/ode_test_functors.hpp   |  12 +-
 .../math/rev/functor/test_fixture_ode_fho.hpp |   4 +-
 .../rev/functor/test_fixture_ode_lorenz.hpp   |   4 +-
 .../math/rev/functor/test_fixture_ode_sho.hpp |   3 +-
 8 files changed, 151 insertions(+), 129 deletions(-)

diff --git a/stan/math/rev/core/zero_adjoints.hpp b/stan/math/rev/core/zero_adjoints.hpp
index c49012702e3..a05362fc79d 100644
--- a/stan/math/rev/core/zero_adjoints.hpp
+++ b/stan/math/rev/core/zero_adjoints.hpp
@@ -36,9 +36,7 @@ inline void zero_adjoints(T& x) noexcept {}
  * @param x current argument
  * @param args rest of arguments to zero
  */
-inline void zero_adjoints(var& x) {
-  x.adj() = 0;
-}
+inline void zero_adjoints(var& x) { x.adj() = 0; }
 
 /**
  * Zero the adjoints of the varis of every var in an Eigen::Matrix
@@ -66,10 +64,11 @@ inline void zero_adjoints(EigMat& x) {
  * @param x current argument
  * @param args rest of arguments to zero
  */
-template <typename StdVec, require_std_vector_st<is_autodiff, StdVec>* = nullptr>
+template <typename StdVec,
+          require_std_vector_st<is_autodiff, StdVec>* = nullptr>
 inline void zero_adjoints(StdVec& x) {
   for (size_t i = 0; i < x.size(); ++i) {
-    zero_adjoints(x[i]);    
+    zero_adjoints(x[i]);
   }
 }
 
diff --git a/stan/math/rev/functor/coupled_ode_system.hpp b/stan/math/rev/functor/coupled_ode_system.hpp
index 08b7dd557fe..c7f608c0622 100644
--- a/stan/math/rev/functor/coupled_ode_system.hpp
+++ b/stan/math/rev/functor/coupled_ode_system.hpp
@@ -148,7 +148,8 @@ struct coupled_ode_system_impl<false, F, T_y0, Args...> {
 
       // The vars here do not live on the nested stack so must be zero'd
       // separately
-      stan::math::for_each([](auto&& arg) { zero_adjoints(arg); }, local_args_tuple_);
+      stan::math::for_each([](auto&& arg) { zero_adjoints(arg); },
+                           local_args_tuple_);
 
       // No need to zero adjoints after last sweep
       if (i + 1 < N_) {
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 9e4d73056ae..026e00b884d 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -20,7 +20,6 @@
 namespace stan {
 namespace math {
 
-
 /**
  * Integrator interface for CVODES' ODE solvers (Adams & BDF
  * methods).
@@ -56,10 +55,11 @@ class cvodes_integrator_adjoint_vari : public vari {
 
   std::ostream* msgs_;
 
-  std::tuple<
-      arena_t<plain_type_t<decltype(deep_copy_vars(std::declval<const T_Args&>()))>>...>
+  std::tuple<arena_t<
+      plain_type_t<decltype(deep_copy_vars(std::declval<const T_Args&>()))>>...>
       local_args_tuple_;
-  std::tuple<arena_t<plain_type_t<decltype(value_of(std::declval<const T_Args&>()))>>...>
+  std::tuple<arena_t<
+      plain_type_t<decltype(value_of(std::declval<const T_Args&>()))>>...>
       value_of_args_tuple_;
 
   size_t N_;
@@ -84,23 +84,27 @@ class cvodes_integrator_adjoint_vari : public vari {
   vari** args_varis_;
 
   int index_backward_;
-  
+
   struct cvodes_solver;
   cvodes_solver* solver_;
 
   static constexpr bool is_var_ts{is_var<T_ts>::value};
   static constexpr bool is_var_t0{is_var<T_t0>::value};
   static constexpr bool is_var_y0{is_var<T_y0_t0>::value};
-  //static constexpr std::array<bool, sizeof...(T_Args)> is_var_args{is_var<scalar_type_t<T_Args>>::value...};
-  static constexpr bool is_any_var_args{disjunction<is_var<scalar_type_t<T_Args>>...>::value};
+  // static constexpr std::array<bool, sizeof...(T_Args)>
+  // is_var_args{is_var<scalar_type_t<T_Args>>::value...};
+  static constexpr bool is_any_var_args{
+      disjunction<is_var<scalar_type_t<T_Args>>...>::value};
   static constexpr bool is_var_return{is_var<T_Return>::value};
-  static constexpr bool is_var_only_ts{is_var_ts && !(is_var_t0 || is_var_y0 || is_any_var_args)};
-  
+  static constexpr bool is_var_only_ts{
+      is_var_ts && !(is_var_t0 || is_var_y0 || is_any_var_args)};
+
   /**
    * Call the ODE RHS with given tuple.
    */
   template <typename yT, typename... ArgsT>
-  constexpr auto rhs(double t, const yT& y, const std::tuple<ArgsT...>& args_tuple) const {
+  constexpr auto rhs(double t, const yT& y,
+                     const std::tuple<ArgsT...>& args_tuple) const {
     return apply([&](auto&&... args) { return f_(t, y, msgs_, args...); },
                  args_tuple);
   }
@@ -120,7 +124,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Implements the function of type CVRhsFnB which is the
    * RHS of the backward ODE system.
    */
-  static int cv_rhs_adj_sens(realtype t, N_Vector y, N_Vector yB, N_Vector yBdot, void* user_data) {
+  static int cv_rhs_adj_sens(realtype t, N_Vector y, N_Vector yB,
+                             N_Vector yBdot, void* user_data) {
     const cvodes_integrator_adjoint_vari* integrator
         = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->rhs_adj_sens(t, y, yB, yBdot);
@@ -131,7 +136,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Implements the function of type CVQuadRhsFnB which is the
    * RHS of the backward ODE system's quadrature.
    */
-  static int cv_quad_rhs_adj(realtype t, N_Vector y, N_Vector yB, N_Vector qBdot, void* user_data) {
+  static int cv_quad_rhs_adj(realtype t, N_Vector y, N_Vector yB,
+                             N_Vector qBdot, void* user_data) {
     cvodes_integrator_adjoint_vari* integrator
         = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
     integrator->quad_rhs_adj(t, y, yB, qBdot);
@@ -144,7 +150,9 @@ class cvodes_integrator_adjoint_vari : public vari {
    * ode_rhs wrt to the states y. The jacobian is stored in column
    * major format.
    */
-  static int cv_jacobian_states(realtype t, N_Vector y, N_Vector fy, SUNMatrix J, void* user_data, N_Vector tmp1, N_Vector tmp2, N_Vector tmp3) {
+  static int cv_jacobian_states(realtype t, N_Vector y, N_Vector fy,
+                                SUNMatrix J, void* user_data, N_Vector tmp1,
+                                N_Vector tmp2, N_Vector tmp3) {
     const cvodes_integrator_adjoint_vari* integrator
         = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_states(t, y, J);
@@ -155,7 +163,9 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Implements the CVLsJacFnB function for evaluating the jacobian of
    * the adjoint problem.
    */
-  static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB, N_Vector fyB, SUNMatrix J, void* user_data, N_Vector tmp1, N_Vector tmp2, N_Vector tmp3) {
+  static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
+                             SUNMatrix J, void* user_data, N_Vector tmp1,
+                             N_Vector tmp2, N_Vector tmp3) {
     const cvodes_integrator_adjoint_vari* integrator
         = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_adj(t, y, J);
@@ -171,7 +181,8 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     const Eigen::VectorXd dy_dt_vec = rhs(t, y_vec, value_of_args_tuple_);
 
-    check_size_match(solver_->function_name_, "dy_dt", dy_dt_vec.size(), "states", N_);
+    check_size_match(solver_->function_name_, "dy_dt", dy_dt_vec.size(),
+                     "states", N_);
 
     Eigen::Map<Eigen::VectorXd>(dy_dt, N_) = dy_dt_vec;
   }
@@ -202,7 +213,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vars, value_of_args_tuple_);
 
-    check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(), "states", N_);
+    check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(),
+                     "states", N_);
 
     f_y_t_vars.adj() = -mu;
 
@@ -234,12 +246,14 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     // The vars here do not live on the nested stack so must be zero'd
     // separately
-    stan::math::for_each([](auto&& arg) { zero_adjoints(arg); }, local_args_tuple_);
+    stan::math::for_each([](auto&& arg) { zero_adjoints(arg); },
+                         local_args_tuple_);
 
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vec, local_args_tuple_);
 
-    check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(), "states", N_);
+    check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(),
+                     "states", N_);
 
     f_y_t_vars.adj() = -mu;
 
@@ -263,7 +277,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var
         = rhs(t, y_var, value_of_args_tuple_);
 
-    check_size_match(solver_->function_name_, "dy_dt", fy_var.size(), "states", N_);
+    check_size_match(solver_->function_name_, "dy_dt", fy_var.size(), "states",
+                     N_);
 
     grad(fy_var.coeffRef(0).vi_);
     Jfy.col(0) = y_var.adj();
@@ -294,10 +309,11 @@ class cvodes_integrator_adjoint_vari : public vari {
   }
 
   /**
-   * Overloads which setup the states returned from the forward solve. In case the return type is a double only, then no autodiff is needed. In case of autodiff non-chaining varis are setup accordingly.
+   * Overloads which setup the states returned from the forward solve. In case
+   * the return type is a double only, then no autodiff is needed. In case of
+   * autodiff non-chaining varis are setup accordingly.
    */
-  void store_state(std::size_t n,
-                   const Eigen::VectorXd& state,
+  void store_state(std::size_t n, const Eigen::VectorXd& state,
                    Eigen::Matrix<var, Eigen::Dynamic, 1>& state_return) {
     solver_->y_[n] = state;
     state_return.resize(N_);
@@ -307,15 +323,16 @@ class cvodes_integrator_adjoint_vari : public vari {
     }
   }
 
-  void store_state(std::size_t n,
-                   const Eigen::VectorXd& state,
+  void store_state(std::size_t n, const Eigen::VectorXd& state,
                    Eigen::Matrix<double, Eigen::Dynamic, 1>& state_return) {
     solver_->y_[n] = state;
     state_return = state;
   }
 
   /**
-   * Since the CVODES solver manages memory with malloc calls, these resources must be freed using a destructor call (which is not being called for the vari class).
+   * Since the CVODES solver manages memory with malloc calls, these resources
+   * must be freed using a destructor call (which is not being called for the
+   * vari class).
    */
   struct cvodes_solver : public chainable_alloc {
     size_t N_;
@@ -336,14 +353,16 @@ class cvodes_integrator_adjoint_vari : public vari {
     SUNLinearSolver LS_backward_;
     template <typename StateFwd, typename StateBwd, typename Quad,
               typename AbsTolFwd, typename AbsTolBwd>
-    cvodes_solver(const char* function_name, size_t N,
-                  size_t num_args_vars, size_t ts_size, int solver_forward, StateFwd& state_forward,
+    cvodes_solver(const char* function_name, size_t N, size_t num_args_vars,
+                  size_t ts_size, int solver_forward, StateFwd& state_forward,
                   StateBwd& state_backward, Quad& quad,
                   AbsTolFwd& absolute_tolerance_forward,
-                  AbsTolBwd& absolute_tolerance_backward) :
-        chainable_alloc(), N_(N), y_(ts_size),
-        function_name_str_(function_name),
-        function_name_(function_name_str_.c_str()) {
+                  AbsTolBwd& absolute_tolerance_backward)
+        : chainable_alloc(),
+          N_(N),
+          y_(ts_size),
+          function_name_str_(function_name),
+          function_name_(function_name_str_.c_str()) {
       nv_state_forward_ = N_VMake_Serial(N, state_forward.data());
       nv_state_backward_ = N_VMake_Serial(N, state_backward.data());
       nv_quad_ = N_VMake_Serial(num_args_vars, quad.data());
@@ -351,20 +370,20 @@ class cvodes_integrator_adjoint_vari : public vari {
           = N_VMake_Serial(N, absolute_tolerance_forward.data());
       nv_absolute_tolerance_backward_
           = N_VMake_Serial(N, absolute_tolerance_backward.data());
-      
+
       A_forward_ = SUNDenseMatrix(N, N);
       A_backward_ = SUNDenseMatrix(N, N);
-      if (N > 0 ) {
+      if (N > 0) {
         LS_forward_ = SUNDenseLinearSolver(nv_state_forward_, A_forward_);
         LS_backward_ = SUNDenseLinearSolver(nv_state_backward_, A_backward_);
       }
       cvodes_mem_ = CVodeCreate(solver_forward);
-      
+
       if (cvodes_mem_ == nullptr) {
         throw std::runtime_error("CVodeCreate failed to allocate memory");
       }
     }
-    
+
     virtual ~cvodes_solver() {
       SUNMatDestroy(A_forward_);
       SUNMatDestroy(A_backward_);
@@ -377,14 +396,13 @@ class cvodes_integrator_adjoint_vari : public vari {
       N_VDestroy_Serial(nv_quad_);
       N_VDestroy_Serial(nv_absolute_tolerance_forward_);
       N_VDestroy_Serial(nv_absolute_tolerance_backward_);
-      
+
       if (cvodes_mem_) {
         CVodeFree(&cvodes_mem_);
       }
     }
   };
 
-
  public:
   /**
    * Construct cvodes_integrator object. Note: All arguments must be stored as
@@ -472,9 +490,11 @@ class cvodes_integrator_adjoint_vari : public vari {
         state_backward_(N_),
         quad_(num_args_vars_),
 
-        non_chaining_varis_(is_var_return ?
-                            ChainableStack::instance_->memalloc_.alloc_array<vari*>(ts_.size() * N_) :
-                            nullptr),
+        non_chaining_varis_(
+            is_var_return
+                ? ChainableStack::instance_->memalloc_.alloc_array<vari*>(
+                      ts_.size() * N_)
+                : nullptr),
         t0_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             num_t0_vars_)),
         ts_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
@@ -483,10 +503,9 @@ class cvodes_integrator_adjoint_vari : public vari {
             num_y0_vars_)),
         args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             num_args_vars_)),
-        solver_(new cvodes_solver(function_name, N_,
-            num_args_vars_, ts_.size(), solver_forward_, state_forward_,
-            state_backward_, quad_,
-            absolute_tolerance_forward_,
+        solver_(new cvodes_solver(
+            function_name, N_, num_args_vars_, ts_.size(), solver_forward_,
+            state_forward_, state_backward_, quad_, absolute_tolerance_forward_,
             absolute_tolerance_backward_)) {
     save_varis(t0_varis_, t0);
     save_varis(ts_varis_, ts);
@@ -502,7 +521,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     apply(
         [&](auto&&... args) {
           std::vector<int> unused_temp{
-              0, (check_finite(solver_->function_name_, "ode parameters and data", args),
+              0, (check_finite(solver_->function_name_,
+                               "ode parameters and data", args),
                   0)...};
         },
         local_args_tuple_);
@@ -517,15 +537,19 @@ class cvodes_integrator_adjoint_vari : public vari {
                           absolute_tolerance_forward_);
     check_size_match(solver_->function_name_, "absolute_tolerance_forward",
                      absolute_tolerance_forward_.size(), "states", N_);
-    check_positive_finite(solver_->function_name_, "relative_tolerance_backward",
+    check_positive_finite(solver_->function_name_,
+                          "relative_tolerance_backward",
                           relative_tolerance_backward_);
-    check_positive_finite(solver_->function_name_, "absolute_tolerance_backward",
+    check_positive_finite(solver_->function_name_,
+                          "absolute_tolerance_backward",
                           absolute_tolerance_backward_);
     check_size_match(solver_->function_name_, "absolute_tolerance_backward",
                      absolute_tolerance_backward_.size(), "states", N_);
-    check_positive_finite(solver_->function_name_, "relative_tolerance_quadrature",
+    check_positive_finite(solver_->function_name_,
+                          "relative_tolerance_quadrature",
                           relative_tolerance_quadrature_);
-    check_positive_finite(solver_->function_name_, "absolute_tolerance_quadrature",
+    check_positive_finite(solver_->function_name_,
+                          "absolute_tolerance_quadrature",
                           absolute_tolerance_quadrature_);
     check_positive(solver_->function_name_, "max_num_steps", max_num_steps_);
     check_positive(solver_->function_name_, "num_steps_between_checkpoints",
@@ -535,8 +559,8 @@ class cvodes_integrator_adjoint_vari : public vari {
                 interpolation_polynomial_);
     // 1=Adams=CV_ADAMS, 2=BDF=CV_BDF
     check_range(solver_->function_name_, "solver_forward", 2, solver_forward_);
-    check_range(solver_->function_name_, "solver_backward", 2, solver_backward_);
-
+    check_range(solver_->function_name_, "solver_backward", 2,
+                solver_backward_);
 
     check_flag_sundials(
         CVodeInit(solver_->cvodes_mem_, &cvodes_integrator_adjoint_vari::cv_rhs,
@@ -557,7 +581,8 @@ class cvodes_integrator_adjoint_vari : public vari {
         "CVodeSVtolerances");
 
     check_flag_sundials(
-        CVodeSetLinearSolver(solver_->cvodes_mem_, solver_->LS_forward_, solver_->A_forward_),
+        CVodeSetLinearSolver(solver_->cvodes_mem_, solver_->LS_forward_,
+                             solver_->A_forward_),
         "CVodeSetLinearSolver");
 
     check_flag_sundials(
@@ -586,7 +611,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     // std::cout << "forward integrate..." << std::endl;
     const double t0_dbl = value_of(t0_);
     const auto ts_dbl = value_of(ts_);
-    std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return(ts_.size());
+    std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return(
+        ts_.size());
 
     double t_init = t0_dbl;
     for (size_t n = 0; n < ts_dbl.size(); ++n) {
@@ -597,8 +623,9 @@ class cvodes_integrator_adjoint_vari : public vari {
         if (is_var_return) {
           int ncheck;
 
-          int error_code = CVodeF(solver_->cvodes_mem_, t_final, solver_->nv_state_forward_,
-                                  &t_init, CV_NORMAL, &ncheck);
+          int error_code
+              = CVodeF(solver_->cvodes_mem_, t_final,
+                       solver_->nv_state_forward_, &t_init, CV_NORMAL, &ncheck);
 
           if (error_code == CV_TOO_MUCH_WORK) {
             throw_domain_error(solver_->function_name_, "", t_final,
@@ -609,8 +636,9 @@ class cvodes_integrator_adjoint_vari : public vari {
           }
 
         } else {
-          int error_code = CVode(solver_->cvodes_mem_, t_final, solver_->nv_state_forward_,
-                                 &t_init, CV_NORMAL);
+          int error_code
+              = CVode(solver_->cvodes_mem_, t_final, solver_->nv_state_forward_,
+                      &t_init, CV_NORMAL);
 
           if (error_code == CV_TOO_MUCH_WORK) {
             throw_domain_error(solver_->function_name_, "", t_final,
@@ -660,8 +688,8 @@ class cvodes_integrator_adjoint_vari : public vari {
           step_sens.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
         }
 
-        ts_varis_[i]->adj_
-            += step_sens.dot(rhs(value_of(ts_[i]), solver_->y_[i], value_of_args_tuple_));
+        ts_varis_[i]->adj_ += step_sens.dot(
+            rhs(value_of(ts_[i]), solver_->y_[i], value_of_args_tuple_));
         /*
           apply(
           [&](auto&&... args) {
@@ -710,25 +738,26 @@ class cvodes_integrator_adjoint_vari : public vari {
     for (int i = ts_.size() - 1; i >= 0; --i) {
       // Take in the adjoints from all the output variables at this point
       // in time
-      //Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
+      // Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
       for (int j = 0; j < N_; j++) {
         // std::cout << "i: " << i << ", j: " << j << std::endl;
         state_backward_.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
-        //step_sens.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
+        // step_sens.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
       }
 
-     double t_final = value_of((i > 0) ? ts_[i - 1] : t0_);
+      double t_final = value_of((i > 0) ? ts_[i - 1] : t0_);
       // std::cout << "backward: time-point " << i << "; t_init = " << t_init <<
       // "; t_final = " << t_final << std::endl;
       if (t_final != t_init) {
         if (!backward_is_initialized_) {
-          check_flag_sundials(
-              CVodeCreateB(solver_->cvodes_mem_, solver_backward_, &index_backward_),
-              "CVodeCreateB");
+          check_flag_sundials(CVodeCreateB(solver_->cvodes_mem_,
+                                           solver_backward_, &index_backward_),
+                              "CVodeCreateB");
 
-          check_flag_sundials(CVodeSetUserDataB(solver_->cvodes_mem_, index_backward_,
-                                                reinterpret_cast<void*>(this)),
-                              "CVodeSetUserDataB");
+          check_flag_sundials(
+              CVodeSetUserDataB(solver_->cvodes_mem_, index_backward_,
+                                reinterpret_cast<void*>(this)),
+              "CVodeSetUserDataB");
 
           // initialize CVODES backward machinery.
           // the states of the backward problem *are* the adjoints
@@ -745,14 +774,15 @@ class cvodes_integrator_adjoint_vari : public vari {
                                  solver_->nv_absolute_tolerance_backward_),
               "CVodeSVtolerancesB");
 
-          check_flag_sundials(CVodeSetMaxNumStepsB(solver_->cvodes_mem_, index_backward_,
-                                                   max_num_steps_),
-                              "CVodeSetMaxNumStepsB");
-
           check_flag_sundials(
-              CVodeSetLinearSolverB(solver_->cvodes_mem_, index_backward_, solver_->LS_backward_,
-                                    solver_->A_backward_),
-              "CVodeSetLinearSolverB");
+              CVodeSetMaxNumStepsB(solver_->cvodes_mem_, index_backward_,
+                                   max_num_steps_),
+              "CVodeSetMaxNumStepsB");
+
+          check_flag_sundials(CVodeSetLinearSolverB(
+                                  solver_->cvodes_mem_, index_backward_,
+                                  solver_->LS_backward_, solver_->A_backward_),
+                              "CVodeSetLinearSolverB");
 
           check_flag_sundials(
               CVodeSetJacFnB(solver_->cvodes_mem_, index_backward_,
@@ -774,21 +804,23 @@ class cvodes_integrator_adjoint_vari : public vari {
                                        absolute_tolerance_quadrature_),
                 "CVodeQuadSStolerancesB");
 
-            check_flag_sundials(
-                CVodeSetQuadErrConB(solver_->cvodes_mem_, index_backward_, SUNTRUE),
-                "CVodeSetQuadErrConB");
+            check_flag_sundials(CVodeSetQuadErrConB(solver_->cvodes_mem_,
+                                                    index_backward_, SUNTRUE),
+                                "CVodeSetQuadErrConB");
           }
 
           backward_is_initialized_ = true;
         } else {
           // just re-initialize the solver
-          check_flag_sundials(CVodeReInitB(solver_->cvodes_mem_, index_backward_, t_init,
-                                           solver_->nv_state_backward_),
-                              "CVodeReInitB");
+          check_flag_sundials(
+              CVodeReInitB(solver_->cvodes_mem_, index_backward_, t_init,
+                           solver_->nv_state_backward_),
+              "CVodeReInitB");
 
           if (num_args_vars_ > 0) {
             check_flag_sundials(
-                CVodeQuadReInitB(solver_->cvodes_mem_, index_backward_, solver_->nv_quad_),
+                CVodeQuadReInitB(solver_->cvodes_mem_, index_backward_,
+                                 solver_->nv_quad_),
                 "CVodeQuadReInitB");
           }
         }
@@ -809,13 +841,14 @@ class cvodes_integrator_adjoint_vari : public vari {
 
         // obtain adjoint states and update t_init to time point
         // reached of t_final
-        check_flag_sundials(CVodeGetB(solver_->cvodes_mem_, index_backward_, &t_init,
-                                      solver_->nv_state_backward_),
+        check_flag_sundials(CVodeGetB(solver_->cvodes_mem_, index_backward_,
+                                      &t_init, solver_->nv_state_backward_),
                             "CVodeGetB");
 
         if (num_args_vars_ > 0) {
           check_flag_sundials(
-              CVodeGetQuadB(solver_->cvodes_mem_, index_backward_, &t_init, solver_->nv_quad_),
+              CVodeGetQuadB(solver_->cvodes_mem_, index_backward_, &t_init,
+                            solver_->nv_quad_),
               "CVodeGetQuadB");
         }
       }
diff --git a/test/unit/math/prim/functor/harmonic_oscillator.hpp b/test/unit/math/prim/functor/harmonic_oscillator.hpp
index 851c0d08601..9a60e29603b 100644
--- a/test/unit/math/prim/functor/harmonic_oscillator.hpp
+++ b/test/unit/math/prim/functor/harmonic_oscillator.hpp
@@ -12,10 +12,9 @@ struct harm_osc_ode_fun {
   // double data
   // integer data
   template <typename T0, typename T1, typename T2, typename T3, typename T4>
-  inline std::vector<stan::return_type_t<T1, T2>>
-  operator()(const T0& t_in, const T1& y_in,
-             const T2& theta, const T3& x,
-             const T4& x_int, std::ostream* msgs) const {
+  inline std::vector<stan::return_type_t<T1, T2>> operator()(
+      const T0& t_in, const T1& y_in, const T2& theta, const T3& x,
+      const T4& x_int, std::ostream* msgs) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
@@ -30,11 +29,8 @@ struct harm_osc_ode_fun {
 
 struct harm_osc_ode_fun_eigen {
   template <typename T0, typename T1, typename T2, typename T3, typename T4>
-  inline auto operator()(const T0& t_in,
-                         const T1& y_in,
-                         std::ostream* msgs, const T2& theta,
-                         const T3& x,
-                         const T4& x_int) const {
+  inline auto operator()(const T0& t_in, const T1& y_in, std::ostream* msgs,
+                         const T2& theta, const T3& x, const T4& x_int) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
@@ -54,10 +50,9 @@ struct harm_osc_ode_data_fun {
   // double data
   // integer data
   template <typename T0, typename T1, typename T2, typename T3, typename T4>
-  inline std::vector<stan::return_type_t<T1, T2>>
-  operator()(const T0& t_in, const T1& y_in,
-             const T2& theta, const T3& x,
-             const T4& x_int, std::ostream* msgs) const {
+  inline std::vector<stan::return_type_t<T1, T2>> operator()(
+      const T0& t_in, const T1& y_in, const T2& theta, const T3& x,
+      const T4& x_int, std::ostream* msgs) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
@@ -73,11 +68,8 @@ struct harm_osc_ode_data_fun {
 
 struct harm_osc_ode_data_fun_eigen {
   template <typename T0, typename T1, typename T2, typename T3, typename T4>
-  inline auto operator()(const T0& t_in,
-                         const T1& y_in,
-                         std::ostream* msgs, const T2& theta,
-                         const T3& x,
-                         const T4& x_int) const {
+  inline auto operator()(const T0& t_in, const T1& y_in, std::ostream* msgs,
+                         const T2& theta, const T3& x, const T4& x_int) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
@@ -99,10 +91,9 @@ struct harm_osc_ode_wrong_size_1_fun {
   // double data
   // integer data
   template <typename T0, typename T1, typename T2, typename T3, typename T4>
-  inline std::vector<stan::return_type_t<T1, T2>>
-  operator()(const T0& t_in, const T1& y_in,
-             const T2& theta, const T3& x,
-             const T4& x_int, std::ostream* msgs) const {
+  inline std::vector<stan::return_type_t<T1, T2>> operator()(
+      const T0& t_in, const T1& y_in, const T2& theta, const T3& x,
+      const T4& x_int, std::ostream* msgs) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
@@ -123,10 +114,9 @@ struct harm_osc_ode_wrong_size_2_fun {
   // double data
   // integer data
   template <typename T0, typename T1, typename T2, typename T3, typename T4>
-  inline std::vector<stan::return_type_t<T1, T2>>
-  operator()(const T0& t_in, const T1& y_in,
-             const T2& theta, const T3& x,
-             const T4& x_int, std::ostream* msgs) const {
+  inline std::vector<stan::return_type_t<T1, T2>> operator()(
+      const T0& t_in, const T1& y_in, const T2& theta, const T3& x,
+      const T4& x_int, std::ostream* msgs) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
diff --git a/test/unit/math/prim/functor/ode_test_functors.hpp b/test/unit/math/prim/functor/ode_test_functors.hpp
index fd17d96fee0..3de5ef75f0f 100644
--- a/test/unit/math/prim/functor/ode_test_functors.hpp
+++ b/test/unit/math/prim/functor/ode_test_functors.hpp
@@ -28,8 +28,8 @@ auto sum_(Vec&& arg) {
 struct CosArg1 {
   template <typename T0, typename T1, typename... T_Args>
   inline Eigen::Matrix<stan::return_type_t<T1, T_Args...>, Eigen::Dynamic, 1>
-  operator()(const T0& t, const T1& y,
-             std::ostream* msgs, const T_Args&... a) const {
+  operator()(const T0& t, const T1& y, std::ostream* msgs,
+             const T_Args&... a) const {
     std::vector<typename stan::return_type<T0, T_Args...>::type> vec
         = {sum_(a)...};
     Eigen::Matrix<stan::return_type_t<T1, T_Args...>, Eigen::Dynamic, 1> out(1);
@@ -41,8 +41,8 @@ struct CosArg1 {
 struct Cos2Arg {
   template <typename T0, typename T1, typename T2, typename T3>
   inline Eigen::Matrix<stan::return_type_t<T1, T2, T3>, Eigen::Dynamic, 1>
-  operator()(const T0& t, const T1& y,
-             std::ostream* msgs, const T2& a, const T3& b) const {
+  operator()(const T0& t, const T1& y, std::ostream* msgs, const T2& a,
+             const T3& b) const {
     Eigen::Matrix<stan::return_type_t<T1, T2, T3>, Eigen::Dynamic, 1> out(1);
     out << stan::math::cos((sum_(a) + sum_(b)) * t);
     return out;
@@ -52,8 +52,8 @@ struct Cos2Arg {
 struct CosArgWrongSize {
   template <typename T0, typename T1, typename... T_Args>
   inline Eigen::Matrix<stan::return_type_t<T1, T_Args...>, Eigen::Dynamic, 1>
-  operator()(const T0& t, const T1& y,
-             std::ostream* msgs, const T_Args&... a) const {
+  operator()(const T0& t, const T1& y, std::ostream* msgs,
+             const T_Args&... a) const {
     std::vector<typename stan::return_type<T0, T_Args...>::type> vec
         = {sum_(a)...};
     Eigen::Matrix<stan::return_type_t<T1, T_Args...>, Eigen::Dynamic, 1> out(2);
diff --git a/test/unit/math/rev/functor/test_fixture_ode_fho.hpp b/test/unit/math/rev/functor/test_fixture_ode_fho.hpp
index 384221e04be..ac38801a388 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_fho.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_fho.hpp
@@ -17,8 +17,8 @@ struct forced_harm_osc_base {
   struct forced_harm_osc {
     template <typename T0, typename T1, typename T2>
     inline Eigen::Matrix<stan::return_type_t<T0, T1, T2>, -1, 1> operator()(
-        const T0& t_in, const T1& y_in,
-        std::ostream* msgs, const T2& theta) const {
+        const T0& t_in, const T1& y_in, std::ostream* msgs,
+        const T2& theta) const {
       if (y_in.size() != 2)
         throw std::domain_error(
             "this function was called with inconsistent state");
diff --git a/test/unit/math/rev/functor/test_fixture_ode_lorenz.hpp b/test/unit/math/rev/functor/test_fixture_ode_lorenz.hpp
index ed593f83be2..a1a50aa0f55 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_lorenz.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_lorenz.hpp
@@ -17,8 +17,8 @@ struct lorenz_ode_base {
   struct lorenz_rhs {
     template <typename T0, typename T1, typename T2>
     inline Eigen::Matrix<stan::return_type_t<T1, T2>, -1, 1> operator()(
-        const T0& t_in, const T1& y_in,
-        std::ostream* msgs, const T2& theta) const {
+        const T0& t_in, const T1& y_in, std::ostream* msgs,
+        const T2& theta) const {
       Eigen::Matrix<stan::return_type_t<T1, T2>, -1, 1> res(3);
       res << theta.at(0) * (y_in(1) - y_in(0)),
           theta.at(1) * y_in(0) - y_in(1) - y_in(0) * y_in(2),
diff --git a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
index c537bcabb54..1ea6b139f8f 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
@@ -19,8 +19,7 @@ struct harmonic_oscillator_ode_base {
   struct sho_square_fun {
     template <typename T0, typename T1, typename T2, typename T3, typename T4>
     inline Eigen::Matrix<stan::return_type_t<T1, T2>, -1, 1> operator()(
-        const T0& t_in, const T1& y_in,
-        std::ostream* msgs, const T2& theta,
+        const T0& t_in, const T1& y_in, std::ostream* msgs, const T2& theta,
         const T3& x, const T4& x_int) const {
       if (y_in.size() != 2)
         throw std::domain_error("Functor called with inconsistent state");

From a836ff643da7257883abaf84839e342f4c0760d3 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 27 Apr 2021 09:23:12 +0200
Subject: [PATCH 099/157] headers compliance

---
 stan/math/rev/functor/coupled_ode_system.hpp        | 1 +
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 3 +--
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/stan/math/rev/functor/coupled_ode_system.hpp b/stan/math/rev/functor/coupled_ode_system.hpp
index 08b7dd557fe..4fc1a2b5070 100644
--- a/stan/math/rev/functor/coupled_ode_system.hpp
+++ b/stan/math/rev/functor/coupled_ode_system.hpp
@@ -6,6 +6,7 @@
 #include <stan/math/rev/meta.hpp>
 #include <stan/math/rev/core.hpp>
 #include <stan/math/rev/fun/value_of.hpp>
+#include <stan/math/prim/functor/for_each.hpp>
 #include <stan/math/prim/err.hpp>
 #include <stdexcept>
 #include <ostream>
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 9e4d73056ae..2d4a5cf75a8 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -4,10 +4,10 @@
 #include <stan/math/rev/meta.hpp>
 #include <stan/math/rev/core/save_varis.hpp>
 #include <stan/math/rev/functor/cvodes_utils.hpp>
-#include <stan/math/rev/functor/coupled_ode_system.hpp>
 #include <stan/math/rev/functor/ode_store_sensitivities.hpp>
 #include <stan/math/prim/err.hpp>
 #include <stan/math/prim/fun/value_of.hpp>
+#include <stan/math/prim/functor/for_each.hpp>
 #include <cvodes/cvodes.h>
 #include <nvector/nvector_serial.h>
 #include <sunmatrix/sunmatrix_dense.h>
@@ -91,7 +91,6 @@ class cvodes_integrator_adjoint_vari : public vari {
   static constexpr bool is_var_ts{is_var<T_ts>::value};
   static constexpr bool is_var_t0{is_var<T_t0>::value};
   static constexpr bool is_var_y0{is_var<T_y0_t0>::value};
-  //static constexpr std::array<bool, sizeof...(T_Args)> is_var_args{is_var<scalar_type_t<T_Args>>::value...};
   static constexpr bool is_any_var_args{disjunction<is_var<scalar_type_t<T_Args>>...>::value};
   static constexpr bool is_var_return{is_var<T_Return>::value};
   static constexpr bool is_var_only_ts{is_var_ts && !(is_var_t0 || is_var_y0 || is_any_var_args)};

From 1654918eb007022f9ceb740cd5c186f4e5cb00b8 Mon Sep 17 00:00:00 2001
From: Steve Bronder <stevo15025@gmail.com>
Date: Tue, 27 Apr 2021 11:01:16 -0400
Subject: [PATCH 100/157] update zero_adjoints test

---
 test/unit/math/rev/core/zero_adjoints_test.cpp | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/test/unit/math/rev/core/zero_adjoints_test.cpp b/test/unit/math/rev/core/zero_adjoints_test.cpp
index 07ab008e47b..e58cb3ecc13 100644
--- a/test/unit/math/rev/core/zero_adjoints_test.cpp
+++ b/test/unit/math/rev/core/zero_adjoints_test.cpp
@@ -1,8 +1,9 @@
-#include <gtest/gtest.h>
-#include <test/unit/math/rev/fun/util.hpp>
 #include <stan/math/rev/core.hpp>
+#include <test/unit/math/rev/fun/util.hpp>
 #include <stan/math/rev/fun/sin.hpp>
 #include <vector>
+#include <tuple>
+#include <gtest/gtest.h>
 
 TEST(AgradRev, zero_arithmetic) {
   int a = 1.0;
@@ -30,8 +31,9 @@ TEST(AgradRev, zero_arithmetic) {
   stan::math::zero_adjoints(vc);
   stan::math::zero_adjoints(vd);
   stan::math::zero_adjoints(ve);
-
-  stan::math::zero_adjoints(a, b, va, vb, c, d, e, vva, vvb, vc, vd, ve);
+  stan::math::for_each([](auto&& x) {
+    stan::math::zero_adjoints(x);
+  }, std::forward_as_tuple(a, b, va, vb, c, d, e, vva, vvb, vc, vd, ve));
 }
 
 TEST(AgradRev, zero_var) {
@@ -214,7 +216,9 @@ TEST(AgradRev, zero_multi) {
   std::vector<int> e(5, 1);
   std::vector<double> f(5, 1.0);
 
-  stan::math::zero_adjoints(a, b, c, d, e, f);
+  stan::math::for_each([](auto&& x) {
+    stan::math::zero_adjoints(x);
+  }, std::forward_as_tuple(a, b, c, d, e, f));
   EXPECT_FLOAT_EQ(c.vi_->adj_, 0.0);
   for (size_t i = 0; i < d.size(); ++i)
     EXPECT_FLOAT_EQ(d[i].vi_->adj_, 0.0);

From 6eff162bee4771b2144a2ded59ad29b14816e78c Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Tue, 27 Apr 2021 16:12:49 +0000
Subject: [PATCH 101/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 test/unit/math/rev/core/zero_adjoints_test.cpp | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/test/unit/math/rev/core/zero_adjoints_test.cpp b/test/unit/math/rev/core/zero_adjoints_test.cpp
index e58cb3ecc13..9f0fbf0c719 100644
--- a/test/unit/math/rev/core/zero_adjoints_test.cpp
+++ b/test/unit/math/rev/core/zero_adjoints_test.cpp
@@ -31,9 +31,9 @@ TEST(AgradRev, zero_arithmetic) {
   stan::math::zero_adjoints(vc);
   stan::math::zero_adjoints(vd);
   stan::math::zero_adjoints(ve);
-  stan::math::for_each([](auto&& x) {
-    stan::math::zero_adjoints(x);
-  }, std::forward_as_tuple(a, b, va, vb, c, d, e, vva, vvb, vc, vd, ve));
+  stan::math::for_each(
+      [](auto&& x) { stan::math::zero_adjoints(x); },
+      std::forward_as_tuple(a, b, va, vb, c, d, e, vva, vvb, vc, vd, ve));
 }
 
 TEST(AgradRev, zero_var) {
@@ -216,9 +216,8 @@ TEST(AgradRev, zero_multi) {
   std::vector<int> e(5, 1);
   std::vector<double> f(5, 1.0);
 
-  stan::math::for_each([](auto&& x) {
-    stan::math::zero_adjoints(x);
-  }, std::forward_as_tuple(a, b, c, d, e, f));
+  stan::math::for_each([](auto&& x) { stan::math::zero_adjoints(x); },
+                       std::forward_as_tuple(a, b, c, d, e, f));
   EXPECT_FLOAT_EQ(c.vi_->adj_, 0.0);
   for (size_t i = 0; i < d.size(); ++i)
     EXPECT_FLOAT_EQ(d[i].vi_->adj_, 0.0);

From d99f69ec31da1912b3f776c04359c25d1530fcd2 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 27 Apr 2021 22:26:10 +0200
Subject: [PATCH 102/157] remove obsolete benchmarks

---
 .../functor/ode_bdf_adjoint_bench_test.cpp    | 110 ---------------
 .../ode_bdf_adjoint_scale_bench_test.cpp      | 130 ------------------
 2 files changed, 240 deletions(-)
 delete mode 100644 test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
 delete mode 100644 test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp

diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
deleted file mode 100644
index c6233d2b018..00000000000
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_bench_test.cpp
+++ /dev/null
@@ -1,110 +0,0 @@
-
-#include <stan/math/rev.hpp>
-#include <gtest/gtest.h>
-#include <stan/math/rev/functor/gradient.hpp>
-#include <iostream>
-#include <sstream>
-#include <vector>
-#include <stdexcept>
-
-using stan::math::lognormal_rng;
-using stan::math::var;
-
-struct pkpd_rhs {
-  template <typename T0, typename T1, typename T2, typename T3, typename T4,
-            typename T5, typename T6, typename T7, typename T8>
-  inline Eigen::Matrix<stan::return_type_t<T1, T2, T3, T4, T5, T6, T7, T8>,
-                       Eigen::Dynamic, 1>
-  operator()(const T0& t, const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y,
-             std::ostream* msgs, const T2& ka, const T3& ke, const T4& k12,
-             const T5& k21, const T6& kin, const T7& kout,
-             const T8& ea50) const {
-    Eigen::Matrix<stan::return_type_t<T1, T2, T3, T4, T5, T6, T7, T8>,
-                  Eigen::Dynamic, 1>
-        dydt(4);
-
-    dydt << -ka * y(0), +ka * y(0) - ke * y(1) - k12 * y(1) + k21 * y(2),
-        +k12 * y(1) - k21 * y(2),
-        +kin - kout * (1.0 - y(1) / (y(1) + ea50)) * y(3);
-
-    return dydt;
-  }
-};
-
-void run_benchmark(int adjoint_integrator) {
-  double true_CL = 8.0;
-  double true_Q = 18.0;
-  double true_V1 = 10.0;
-  double true_V2 = 14.0;
-  double true_ka = log(2.0) / 2.0;
-  double true_pd0 = 1.0;
-  double true_kin = 4.0;
-  double true_kout = true_kin / true_pd0;
-  double true_ec50 = 0.01;
-  std::vector<double> ts{1., 2.0, 4.0, 8.0, 16.0, 32.0, 64.0};
-  std::size_t ts_size = ts.size();
-
-  pkpd_rhs ode;
-
-  boost::ecuyer1988 base_rng(563646);
-
-  double log_sigma = 10.0;
-  double abs_tol = 1e-8;
-  double abs_tol_B = abs_tol * 100.0;
-  double abs_tol_QB = abs_tol_B * 10.0;
-  double rel_tol = 1e-6;
-  int steps_checkpoint = 100;
-  int max_num_steps = 1000000;
-
-  for (std::size_t i = 0; i != 2; i++) {
-    stan::math::nested_rev_autodiff nested;
-
-    var CL = lognormal_rng(true_CL, log_sigma, base_rng);
-    var Q = lognormal_rng(true_Q, log_sigma, base_rng);
-    var V1 = lognormal_rng(true_V1, log_sigma, base_rng);
-    var V2 = lognormal_rng(true_V2, log_sigma, base_rng);
-    var ka = lognormal_rng(true_ka, log_sigma, base_rng);
-    var pd0 = lognormal_rng(true_pd0, log_sigma, base_rng);
-    var kin = lognormal_rng(true_kin, log_sigma, base_rng);
-    var kout = kin / pd0;
-    var ec50 = lognormal_rng(true_ec50, log_sigma, base_rng);
-    var ea50 = ec50 * V1;
-
-    var ke = CL / V1;
-    var k12 = Q / V2;
-    var k21 = k12 * V1 / V2;
-
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(4);
-    y0 << 4.0, 0.0, 0.0, pd0;
-
-    double t0 = 0.0;
-
-    try {
-      if (adjoint_integrator) {
-        const int N = y0.size();
-        std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-            = ode_adjoint_tol_ctl(
-                ode, y0, t0, ts, rel_tol, Eigen::VectorXd::Constant(N, abs_tol),
-                rel_tol, Eigen::VectorXd::Constant(N, abs_tol_B), rel_tol,
-                abs_tol_QB, max_num_steps, steps_checkpoint, 1, 2, 2, nullptr,
-                ka, ke, k12, k21, kin, kout, ea50);
-
-        stan::math::grad();
-      } else {
-        std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y
-            = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol_QB, max_num_steps,
-                          nullptr, ka, ke, k12, k21, kin, kout, ea50);
-
-        stan::math::grad();
-      }
-    } catch (std::exception& exc) {
-      std::cout << "oops, keep going please!" << std::endl;
-      std::cerr << exc.what() << std::endl;
-    }
-  }
-  stan::math::recover_memory();
-}
-
-TEST(StanMathOdeBench, bdf) { run_benchmark(0); }
-
-TEST(StanMathOdeBench, bdf_adjoint) { run_benchmark(1); }
diff --git a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp b/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
deleted file mode 100644
index 35fe7f79882..00000000000
--- a/test/unit/math/rev/functor/ode_bdf_adjoint_scale_bench_test.cpp
+++ /dev/null
@@ -1,130 +0,0 @@
-
-
-#include <stan/math/rev.hpp>
-#include <gtest/gtest.h>
-#include <stan/math/rev/functor/gradient.hpp>
-#include <iostream>
-#include <sstream>
-#include <vector>
-#include <stdexcept>
-
-using stan::math::lognormal_rng;
-using stan::math::var;
-
-struct scaling_rhs {
-  template <typename T0, typename T1, typename T2, typename T3, typename T4,
-            typename T5>
-  inline Eigen::Matrix<stan::return_type_t<T1, T2, T3, T4, T5>, Eigen::Dynamic,
-                       1>
-  operator()(const T0& t, const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y,
-             std::ostream* msgs, const std::vector<T2>& kt,
-             const std::vector<T3>& e50, const std::vector<T4>& k12,
-             const std::vector<T5>& k21) const {
-    std::size_t num_main_states = kt.size();
-    std::size_t num_states = 2 * num_main_states;
-
-    using return_t = stan::return_type_t<T1, T2, T3, T4, T5>;
-
-    Eigen::Matrix<return_t, Eigen::Dynamic, 1> dydt(num_states);
-    std::vector<return_t> ksat(num_main_states);
-
-    for (std::size_t i = 0; i != num_main_states; ++i) {
-      std::size_t m = 2 * i;  // main state
-      std::size_t a = m + 1;  // auxilary state
-      ksat[i] = kt[i] * y(m) / (y(m) + e50[i]);
-
-      dydt(m) = -ksat[i] * y(m) - k12[i] * y(m) + k21[i] * y(a);
-      dydt(a) = +k12[i] * y(m) - k21[i] * y(a);
-
-      if (i != 0) {
-        dydt(m) += ksat[i - 1] * y(2 * (i - 1));
-      }
-    }
-    return dydt;
-  }
-};
-
-void run_benchmark(std::size_t system_size, int adjoint_integrator) {
-  scaling_rhs ode;
-
-  std::vector<double> ts{1., 2.0, 4.0, 8.0, 16.0, 32.0, 64.0};
-  std::size_t ts_size = ts.size();
-
-  boost::ecuyer1988 base_rng(563646);
-
-  double log_sigma = 10.0;
-  double abs_tol = 1e-8;
-  double abs_tol_B = abs_tol * 10.0;
-  double abs_tol_QB = abs_tol_B * 10.0;
-  double rel_tol = 1e-6;
-  int steps_checkpoint = 100;
-  int max_num_steps = 1000000;
-
-  for (std::size_t i = 0; i != 2; i++) {
-    stan::math::nested_rev_autodiff nested;
-
-    std::vector<var> kt(system_size);
-    std::vector<var> e50(system_size);
-    std::vector<var> k12(system_size);
-    std::vector<var> k21(system_size);
-
-    for (std::size_t j = 0; j != system_size; ++j) {
-      kt[j] = lognormal_rng(0.0, log_sigma, base_rng);
-      e50[j] = lognormal_rng(0.0, log_sigma, base_rng);
-      k12[j] = lognormal_rng(0.0, log_sigma, base_rng);
-      k21[j] = lognormal_rng(0.0, log_sigma, base_rng);
-    }
-
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y0(2 * system_size);
-    for (std::size_t j = 0; j != 2 * system_size; ++j)
-      y0(j) = lognormal_rng(0.0, log_sigma, base_rng);
-
-    double t0 = 0.0;
-
-    try {
-      std::vector<Eigen::Matrix<var, Eigen::Dynamic, 1>> y;
-      if (adjoint_integrator) {
-        const int N = y0.size();
-        y = ode_adjoint_tol_ctl(
-            ode, y0, t0, ts, rel_tol, Eigen::VectorXd::Constant(N, abs_tol),
-            rel_tol, Eigen::VectorXd::Constant(N, abs_tol_B), rel_tol,
-            abs_tol_QB, max_num_steps, steps_checkpoint, 1, 2, 2, nullptr, kt,
-            e50, k12, k21);
-      } else {
-        y = ode_bdf_tol(ode, y0, t0, ts, rel_tol, abs_tol_QB, max_num_steps,
-                        nullptr, kt, e50, k12, k21);
-      }
-
-      // Essentially sets the adjoint for all states to 1.
-      var target = stan::math::sum(y[0]);
-      for (int k = 1; k < ts_size; k++)
-        target += stan::math::sum(y[k]);
-
-      target.grad();
-    } catch (std::exception& exc) {
-      std::cout << "oops, keep going please!" << std::endl;
-      std::cerr << exc.what() << std::endl;
-    }
-  }
-  stan::math::recover_memory();
-}
-
-TEST(StanMathOdeBench, bdf_2) { run_benchmark(2, 0); }
-
-TEST(StanMathOdeBench, bdf_adjoint_2) { run_benchmark(2, 1); }
-
-TEST(StanMathOdeBench, bdf_4) { run_benchmark(4, 0); }
-
-TEST(StanMathOdeBench, bdf_adjoint_4) { run_benchmark(4, 1); }
-
-TEST(StanMathOdeBench, bdf_8) { run_benchmark(8, 0); }
-
-TEST(StanMathOdeBench, bdf_adjoint_8) { run_benchmark(8, 1); }
-
-TEST(StanMathOdeBench, bdf_16) { run_benchmark(16, 0); }
-
-TEST(StanMathOdeBench, bdf_adjoint_16) { run_benchmark(16, 1); }
-
-TEST(StanMathOdeBench, bdf_32) { run_benchmark(32, 0); }
-
-TEST(StanMathOdeBench, bdf_adjoint_32) { run_benchmark(32, 1); }

From e8138b8216742490f3431236b543818a40951f1a Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 27 Apr 2021 22:27:35 +0200
Subject: [PATCH 103/157] delete benchmark R code and move ode functor into
 chainable_alloc class as it may not be trivially destructible

---
 ode-adjoint/README.md                         |  17 --
 ode-adjoint/linked-mass-flow-eigen.stan       | 160 ------------
 ode-adjoint/linked-mass-flow-fixed.stan       | 206 ---------------
 ode-adjoint/linked-mass-flow.R                | 104 --------
 ode-adjoint/linked-mass-flow.stan             | 238 ------------------
 ode-adjoint/sbc.R                             | 102 --------
 ode-adjoint/sbc_report.R                      | 116 ---------
 ode-adjoint/sbc_tools.R                       | 152 -----------
 ode-adjoint/scale-benchmark.R                 | 186 --------------
 .../rev/functor/cvodes_integrator_adjoint.hpp |  13 +-
 10 files changed, 6 insertions(+), 1288 deletions(-)
 delete mode 100644 ode-adjoint/README.md
 delete mode 100644 ode-adjoint/linked-mass-flow-eigen.stan
 delete mode 100644 ode-adjoint/linked-mass-flow-fixed.stan
 delete mode 100644 ode-adjoint/linked-mass-flow.R
 delete mode 100644 ode-adjoint/linked-mass-flow.stan
 delete mode 100644 ode-adjoint/sbc.R
 delete mode 100644 ode-adjoint/sbc_report.R
 delete mode 100644 ode-adjoint/sbc_tools.R
 delete mode 100644 ode-adjoint/scale-benchmark.R

diff --git a/ode-adjoint/README.md b/ode-adjoint/README.md
deleted file mode 100644
index f7b9e740fed..00000000000
--- a/ode-adjoint/README.md
+++ /dev/null
@@ -1,17 +0,0 @@
-# Codes to run SBC & Speed Benchmarks for adjoint ODE
-
-The work is dispatched using batchtools (see
-https://mllg.github.io/batchtools/articles/batchtools.html
-). Batchtools allows to use a cluster as backend or just your laptop.
-
-The R codes rely on cmdstanr. So please ensure that the environment
-variable CMDSTAN points to a properly setup cmdstan checkout which
-itself is configured to use a math checkout of the branch
-feature/adjoint-odes .
-
-Files:
-
-sbc.R runs SBC
-sbc_report.R plots SBC results
-scale-benchmark.R Speed benchmark which varries the system size
-sbc_tools.R shared R functions
diff --git a/ode-adjoint/linked-mass-flow-eigen.stan b/ode-adjoint/linked-mass-flow-eigen.stan
deleted file mode 100644
index d309f236573..00000000000
--- a/ode-adjoint/linked-mass-flow-eigen.stan
+++ /dev/null
@@ -1,160 +0,0 @@
-functions {
-
-  vector linked_mass_flow(real t, vector y,
-                          vector kt, vector e50, vector k12, vector k21) {
-
-    int num_main_states = num_elements(kt);
-    int num_states = 2 * num_main_states;
-
-    vector[num_states] dydt;
-    vector[num_main_states] ksat;
-
-    for (i in 1:num_main_states) {
-      int m = 2 * (i-1) + 1;  // main state
-      int a = m + 1;  // auxilary state
-      ksat[i] = kt[i] * y[m] / (y[m] + e50[i]);
-
-      dydt[m] = -ksat[i] * y[m] - k12[i] * y[m] + k21[i] * y[a];
-      dydt[a] = +k12[i] * y[m] - k21[i] * y[a];
-
-      if (i != 1) {
-        dydt[m] += ksat[i-1] * y[2 * (i - 1) + 1];
-      }
-    }
-    return dydt;
-
-  }
-
-  vector sample_vector_rng(real m, real s, int N) {
-    vector[N] sample;
-    for(i in 1:N)
-      sample[i] = normal_rng(m, s);
-    return sample;
-  }
-
-  vector[] simulate_mean(real t0, vector log_a0, real[] ts, int adjoint_integrator,
-                         real rel_tol, real abs_tol, int max_num_steps,
-                         vector log_kt, vector log_e50, vector log_k12, vector log_k21) {
-    int num_sim = num_elements(ts);
-    int num_states = num_elements(log_a0);
-    vector[num_states] y[num_sim];
-
-    if(adjoint_integrator) {
-      y = ode_adjoint(linked_mass_flow, exp(log_a0), t0, ts,
-                      rel_tol, rep_vector(abs_tol, num_states), // forward
-                      rel_tol, 10*abs_tol, // backward
-                      rel_tol, 100*abs_tol, // quadrature
-                      max_num_steps, 50, 1,
-                      2, // bdf forward
-                      2, // bdf backward
-                      exp(log_kt), exp(log_e50), exp(log_k12), exp(log_k21));
-    } else {
-      y = ode_bdf_tol(linked_mass_flow, exp(log_a0), t0, ts, rel_tol, abs_tol, max_num_steps,
-                      exp(log_kt), exp(log_e50), exp(log_k12), exp(log_k21));
-    }
-
-    return y;
-  }
-  
-}
-data {
-  real<lower=0> rel_tol;
-  real<lower=0> abs_tol;
-  int<lower=0,upper=1> adjoint_integrator;
-  int<lower=1> max_num_steps;
-  int<lower=1> system_size;
-  int<lower=1> num_obs;
-  real<lower=0> sigma_sim;
-  real<lower=0> sigma_y;
-}
-transformed data {
-  int num_states = 2 * system_size;
-
-  vector[system_size] log_kt_ = sample_vector_rng(0.0, sigma_sim, system_size);
-  vector[system_size] log_e50_ = sample_vector_rng(0.0, sigma_sim, system_size);
-  vector[system_size] log_k12_ = sample_vector_rng(0.0, sigma_sim, system_size);
-  vector[system_size] log_k21_ = sample_vector_rng(0.0, sigma_sim, system_size);
-  vector[system_size] log_sigma_y_ = sample_vector_rng(0.0, sigma_y, system_size);
-  vector[num_states] log_a0_ = sample_vector_rng(0.0, sigma_sim, num_states);
-  vector[num_states] y_[num_obs];
-  real ts[num_obs];
-  real t0 = 0.0;
-
-  ts[1] = 1.0;
-  for(i in 2:num_obs) {
-    ts[i] = 1.5 * ts[i-1];
-  }
-
-  y_ = simulate_mean(t0, log_a0_, ts, 0, rel_tol, abs_tol, max_num_steps,
-                     log_kt_, log_e50_, log_k12_, log_k21_);
-  
-  for(i in 1:num_obs) {
-    for(j in 1:system_size) {
-      int m = 2*(j-1) + 1;
-      y_[i,m] = lognormal_rng(log(y_[i,m]+1E-3), exp(log_sigma_y_[j]));
-    }
-  }
-
-  if(adjoint_integrator) {
-    print("Using bdf_adjoint integrator.");
-  } else {
-    print("Using bdf integrator.");
-  }
-  print("relative tolerance: ", rel_tol);
-  print("absolute tolerance: ", abs_tol);
-  print("maximum number of steps: ", max_num_steps);
-  print("number of time points: ", num_obs);
-  print("system size: ", system_size);
-  print("time points: ", ts);
-  print("y_: ", y_);
-}
-parameters {
-  vector[system_size] log_kt;
-  vector[system_size] log_e50;
-  vector[system_size] log_k12;
-  vector[system_size] log_k21;
-  vector[system_size] log_sigma_y;
-  vector[num_states] log_a0;
-}
-transformed parameters {
-}
-model {
-  vector[num_states] mu[num_obs] = simulate_mean(t0, log_a0, ts, adjoint_integrator, rel_tol, abs_tol, max_num_steps,
-                                                 log_kt, log_e50, log_k12, log_k21);
-  
-  target += normal_lpdf(log_kt| 0.0, sigma_sim);
-  target += normal_lpdf(log_e50| 0.0, sigma_sim);
-  target += normal_lpdf(log_k12| 0.0, sigma_sim);
-  target += normal_lpdf(log_k21| 0.0, sigma_sim);
-  target += normal_lpdf(log_a0| 0.0, sigma_sim);
-  target += normal_lpdf(log_sigma_y| 0.0, sigma_y);
-
-  for(j in 1:system_size) {
-    int m = 2*(j-1) + 1;
-    target += lognormal_lpdf(to_vector(y_[:,m])| log(to_vector(mu[:,m])+1E-3), exp(log_sigma_y[j]));
-  }
-}
-generated quantities {
-  int rank_log_kt[system_size];
-  int rank_log_e50[system_size];
-  int rank_log_k12[system_size];
-  int rank_log_k21[system_size];
-  int rank_log_sigma_y[system_size];
-  int rank_log_a0[system_size];
-  vector[system_size] bias_log_kt = log_kt - log_kt_;
-  vector[system_size] bias_log_e50 = log_e50 - log_e50_;
-  vector[system_size] bias_log_k12 = log_k12 - log_k12_;
-  vector[system_size] bias_log_k21 = log_k21 - log_k21_;
-  vector[system_size] bias_log_sigma_y = log_sigma_y - log_sigma_y_;
-  vector[system_size] bias_log_a0 = log_a0 - log_a0_;
-
-  for(i in 1:system_size) {
-    rank_log_kt[i] = (log_kt[i] > log_kt_[i] ? 1 : 0);
-    rank_log_e50[i] = (log_e50[i] > log_e50_[i] ? 1 : 0);
-    rank_log_k12[i] = (log_k12[i] > log_k12_[i] ? 1 : 0);
-    rank_log_k21[i] = (log_k21[i] > log_k21_[i] ? 1 : 0);
-    rank_log_sigma_y[i] = (log_sigma_y[i] > log_sigma_y_[i] ? 1 : 0);
-    rank_log_a0[i] = (log_a0[i] > log_a0_[i] ? 1 : 0);
-  }
-
-}
diff --git a/ode-adjoint/linked-mass-flow-fixed.stan b/ode-adjoint/linked-mass-flow-fixed.stan
deleted file mode 100644
index d59a508750f..00000000000
--- a/ode-adjoint/linked-mass-flow-fixed.stan
+++ /dev/null
@@ -1,206 +0,0 @@
-functions {
-
-  vector linked_mass_flow(real t, vector y,
-                          real kp, real ks, real e50, real k12, real k21) {
-
-    int num_main_states = num_elements(y)/2;
-    int num_states = 2 * num_main_states;
-
-    vector[num_states] dydt;
-    vector[num_main_states] mass_flow;
-
-    for (i in 1:num_main_states) {
-      int m = 2 * (i-1) + 1;  // main state
-      int a = m + 1;  // auxilary state
-      mass_flow[i] = (kp + ks  / (y[m] + e50)) * y[m];
-
-      dydt[m] = -mass_flow[i] - k12 * y[m] + k21 * y[a];
-      dydt[a] = +k12 * y[m] - k21 * y[a];
-
-      if (i != 1) {
-        dydt[m] += mass_flow[i-1];
-      }
-    }
-    return dydt;
-
-  }
-
-  vector sample_vector_rng(real m, real s, int N) {
-    vector[N] sample;
-    for(i in 1:N)
-      sample[i] = normal_rng(m, s);
-    return sample;
-  }
-
-  vector[] simulate_mean(real t0, vector log_a0, real[] ts, int adjoint_integrator,
-                         real rel_tol, real abs_tol, int max_num_steps,
-                         int num_checkpoints,
-                         int interpolation_polynomial,
-                         int solver_f, int solver_b,
-                         real log_kp, real log_ks, real log_e50, real log_k12, real log_k21) {
-    int num_sim = num_elements(ts);
-    int num_states = num_elements(log_a0);
-    int system_size = num_states/2;
-    vector[num_states] y[num_sim];
-    real kp = exp(log_kp);
-    real ks = exp(log_ks);
-    real e50 = exp(log_e50);
-    real k12 = exp(log_k12);
-    real k21 = exp(log_k21);
-    vector[num_states] a0 = exp(log_a0);
-
-    if(adjoint_integrator) {
-      y = ode_adjoint_tol_ctl(linked_mass_flow, a0, t0, ts, 
-                              rel_tol, rep_vector(abs_tol/100.0, num_states), // forward
-                              rel_tol, rep_vector(abs_tol/10.0, num_states), // backward
-                              rel_tol, abs_tol, // quadrature
-                              max_num_steps,
-                              num_checkpoints, // number of steps between checkpoints
-                              interpolation_polynomial,  // polynomials
-                              solver_f,  // bdf forward
-                              solver_b,  // bdf backward
-                              kp, ks, e50, k12, k21);
-      /* simplified interface
-      y = ode_adjoint_tol(linked_mass_flow, a0, t0, ts, 
-                              rel_tol, abs_tol,
-                              max_num_steps,
-                              kp, ks, e50, k12, k21);
-      */
-    } else {
-      y = ode_bdf_tol(linked_mass_flow, a0, t0, ts, rel_tol, abs_tol, max_num_steps,
-                      kp, ks, e50, k12, k21);
-    }
-
-    return y;
-  }
-  
-}
-data {
-  real<lower=0> rel_tol_f;
-  real<lower=0> abs_tol_f;
-  int<lower=0,upper=1> adjoint_integrator;
-  int<lower=1> max_num_steps;
-  int<lower=1> num_checkpoints;
-  int<lower=1,upper=2> interpolation_polynomial;
-  int<lower=1,upper=2> solver_f;
-  int<lower=1,upper=2> solver_b;
-  int<lower=1> system_size;
-  int<lower=1> num_obs;
-  real<lower=0> sigma_sim;
-  real<lower=0> sigma_y;
-}
-transformed data {
-  int num_states = 2 * system_size;
-  real param_scale = sigma_sim / sqrt(num_obs/4);
-
-  real log_kp_ = normal_rng(0.0, sigma_sim);
-  real log_ks_ = normal_rng(0.0, sigma_sim);
-  real log_e50_ = normal_rng(0.0, sigma_sim);
-  real log_k12_ = normal_rng(0.0, sigma_sim);
-  real log_k21_ = normal_rng(0.0, sigma_sim);
-  vector[system_size] log_sigma_y_ = sample_vector_rng(0.0, sigma_y, system_size);
-  vector[num_states] log_a0_ = sample_vector_rng(0.0, sigma_sim, num_states);
-  vector[num_states] y_[num_obs];
-  real ts[num_obs];
-  real t0 = 0.0;
-
-  ts[1] = 1.0;
-  for(i in 2:num_obs) {
-    ts[i] = 1.5 * ts[i-1];
-  }
-
-  y_ = simulate_mean(t0, log_a0_, ts, 0, rel_tol_f, abs_tol_f, max_num_steps,
-                     num_checkpoints,
-                     interpolation_polynomial,
-                     solver_f, solver_b,
-                     log_kp_, log_ks_, log_e50_, log_k12_, log_k21_);
-  
-  for(i in 1:num_obs) {
-    for(j in 1:system_size) {
-      int m = 2*(j-1) + 1;
-      y_[i,m] = lognormal_rng(log(y_[i,m]+1E-3), exp(log_sigma_y_[j]));
-    }
-  }
-
-  if(adjoint_integrator) {
-    print("Using adjoint integrator.");
-  } else {
-    print("Using bdf integrator.");
-  }
-  print("relative tolerance: ", rel_tol_f);
-  print("absolute tolerance: ", abs_tol_f);
-  print("maximum number of steps: ", max_num_steps);
-  print("number of checkpoints: ", num_checkpoints);
-  print("interpolation polynomial: ", interpolation_polynomial);
-  print("solver forward: ", solver_f);
-  print("solver backward: ", solver_b);
-  print("number of time points: ", num_obs);
-  print("system size: ", system_size);
-  print("time points: ", ts);
-  print("y_: ", y_);
-  print("ODE states N: ", num_states);
-  print("ODE parameters varying M: ", 5 + num_states);
-}
-parameters {
-  real<multiplier=param_scale,offset=log_kp_> log_kp;
-  real<multiplier=param_scale,offset=log_ks_> log_ks;
-  real<multiplier=param_scale,offset=log_e50_> log_e50;
-  real<multiplier=param_scale,offset=log_k12_> log_k12;
-  real<multiplier=param_scale,offset=log_k21_> log_k21;
-  vector<offset=log_sigma_y_>[system_size] log_sigma_y;
-  vector<multiplier=param_scale,offset=log_a0_>[num_states] log_a0;
-}
-transformed parameters {
-}
-model {
-  vector[num_states] mu[num_obs];
-
-  profile("ode") {
-    mu = simulate_mean(t0, log_a0, ts, adjoint_integrator, rel_tol_f, abs_tol_f, max_num_steps,
-                       num_checkpoints,
-                       interpolation_polynomial,
-                       solver_f, solver_b,
-                       log_kp, log_ks, log_e50, log_k12, log_k21);
-  }
-  
-  target += normal_lpdf(log_kp| 0.0, sigma_sim);
-  target += normal_lpdf(log_ks| 0.0, sigma_sim);
-  target += normal_lpdf(log_e50| 0.0, sigma_sim);
-  target += normal_lpdf(log_k12| 0.0, sigma_sim);
-  target += normal_lpdf(log_k21| 0.0, sigma_sim);
-  target += normal_lpdf(log_a0| 0.0, sigma_sim);
-  target += normal_lpdf(log_sigma_y| 0.0, sigma_y);
-
-  for(j in 1:system_size) {
-    int m = 2*(j-1) + 1;
-    target += lognormal_lpdf(to_vector(y_[:,m])| log(to_vector(mu[:,m])+1E-3), exp(log_sigma_y[j]));
-  }
-}
-generated quantities {
-  int rank_log_kp;
-  int rank_log_ks;
-  int rank_log_e50;
-  int rank_log_k12;
-  int rank_log_k21;
-  int rank_log_sigma_y[system_size];
-  int rank_log_a0[system_size];
-  real bias_log_kp = log_kp - log_kp_;
-  real bias_log_ks = log_ks - log_ks_;
-  real bias_log_e50 = log_e50 - log_e50_;
-  real bias_log_k12 = log_k12 - log_k12_;
-  real bias_log_k21 = log_k21 - log_k21_;
-  vector[system_size] bias_log_sigma_y = log_sigma_y - log_sigma_y_;
-  vector[system_size] bias_log_a0 = log_a0 - log_a0_;
-
-  rank_log_kp = (log_kp > log_kp_ ? 1 : 0);
-  rank_log_ks = (log_ks > log_ks_ ? 1 : 0);
-  rank_log_e50 = (log_e50 > log_e50_ ? 1 : 0);
-  rank_log_k12 = (log_k12 > log_k12_ ? 1 : 0);
-  rank_log_k21 = (log_k21 > log_k21_ ? 1 : 0);
-  
-  for(i in 1:system_size) {
-    rank_log_sigma_y[i] = (log_sigma_y[i] > log_sigma_y_[i] ? 1 : 0);
-    rank_log_a0[i] = (log_a0[i] > log_a0_[i] ? 1 : 0);
-  }
-  
-}
diff --git a/ode-adjoint/linked-mass-flow.R b/ode-adjoint/linked-mass-flow.R
deleted file mode 100644
index e24cd5a61f5..00000000000
--- a/ode-adjoint/linked-mass-flow.R
+++ /dev/null
@@ -1,104 +0,0 @@
-
-if(FALSE) {
-    ## run as needed to get latest cmdstanr
-    install.packages("cmdstanr", repos = c("https://mc-stan.org/r-packages/", getOption("repos")))
-    ## or even install the latest dev version with profiling support
-    Sys.setenv(R_REMOTES_NO_ERRORS_FROM_WARNINGS="true")
-    remotes::install_github("stan-dev/cmdstanr")
-}
-
-library(cmdstanr)
-
-set_cmdstan_path("~/work/cmdstan")
-
-## model which scales number of states and parameters
-stan_model_file <- "linked-mass-flow.stan"
-## model which scales only number of states, but recycles parameters
-stan_model_fixed_file <- "linked-mass-flow-fixed.stan"
-
-mod <- cmdstan_model(stan_model_file)
-mod_fixed <- cmdstan_model(stan_model_fixed_file)
-
-rel_tol_f <- 1E-8
-abs_tol_f <- 1E-8
-base_data <- list(rel_tol_f=rel_tol_f,
-                  abs_tol_f=abs_tol_f,
-                  rel_tol_b=rel_tol_f * 100,
-                  abs_tol_b=abs_tol_f * 100,
-                  rel_tol_q=rel_tol_f * 1000,
-                  abs_tol_q=abs_tol_f * 1000,
-                  adjoint_integrator=0,
-                  max_num_steps=10000,
-                  num_checkpoints=150,
-                  interpolation_polynomial=1,
-                  solver_f=2,
-                  solver_b=2,
-                  system_size=2,
-                  num_obs=8,
-                  sigma_sim=2.0,
-                  sigma_y=0.2
-                  )
-
-run_benchmark <- function(..., model, num_iter=250, adapt_delta=0.8) {
-    fit_args <- list(...)
-    fit_data <- modifyList(base_data, fit_args)
-    print(str(fit_data))
-    fit <- model$sample(
-                   data = fit_data,
-                   iter_warmup = num_iter,
-                   iter_sampling = num_iter,
-                   seed = 123,
-                   chains = 1,
-                   parallel_chains = 1,
-                   refresh = round(num_iter/5),
-                   save_warmup=TRUE,
-                   adapt_delta=adapt_delta,
-                   init=0.1
-                   ##verbose=TRUE
-               )
-    fit
-}
-
-
-summarize_benchmark <- function(fit) {
-    sdiag_all  <- fit$sampler_diagnostics(inc_warmup=TRUE)
-    sdiag_main  <- fit$sampler_diagnostics(inc_warmup=FALSE)
-
-    num_divergent <- sum(sdiag_main[,,"divergent__"])
-    accept_stat <- mean(sdiag_main[,,"accept_stat__"])
-    treedepth <- mean(sdiag_main[,,"treedepth__"])
-    num_leaps <- sum(sdiag_all[,,"n_leapfrog__"])
-    fit_time <- fit$time()$total
-
-    ess_speed <- as.vector(fit$summary("lp__")[1,c("ess_bulk", "ess_tail")]) / fit_time
-    lp_est <- as.vector(fit$summary("lp__"))
-
-    list(num_leaps=num_leaps, time=fit_time, time_per_leap=1E3 * fit_time / num_leaps,
-         step_size=fit$metadata()$step_size_adaptation,
-         num_divergent=num_divergent, accept_stat=accept_stat, treedepth=treedepth,
-         ess_lp_speed=ess_speed, lp_est=lp_est)
-}
-
-
-bdf_fit <- run_benchmark(adjoint_integrator=0, model=mod, num_iter=50)
-adjoint_fit <- run_benchmark(adjoint_integrator=1, model=mod, num_iter=50)
-## vary solvers, e.g.
-##adjoint_fit <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=1, num_iter=50)
-
-str(summarize_benchmark(bdf_fit))
-str(summarize_benchmark(adjoint_fit))
-
-Lbdf_fit <- run_benchmark(adjoint_integrator=0, system_size=6, model=mod, num_iter=50)
-Ladjoint_fit <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=2, system_size=6, model=mod, num_iter=50)
-
-str(summarize_benchmark(Lbdf_fit))
-str(summarize_benchmark(Ladjoint_fit))
-
-## there is no gain by adjoint method if there are not more parameters
-## than states
-Lbdf_fit_fixed <- run_benchmark(adjoint_integrator=0, system_size=5, model=mod_fixed, num_iter=50)
-Ladjoint_fit_fixed <- run_benchmark(adjoint_integrator=1, solver_f=2, solver_b=2, system_size=5, model=mod_fixed, interpolation_polynomial=1, num_iter=50)
-
-str(summarize_benchmark(Lbdf_fit_fixed))
-str(summarize_benchmark(Ladjoint_fit_fixed))
-
diff --git a/ode-adjoint/linked-mass-flow.stan b/ode-adjoint/linked-mass-flow.stan
deleted file mode 100644
index f2795451593..00000000000
--- a/ode-adjoint/linked-mass-flow.stan
+++ /dev/null
@@ -1,238 +0,0 @@
-functions {
-
-  vector linked_mass_flow(real t, vector y,
-                          real[] kp, real[] ks, real[] e50, real[] k12, real[] k21) {
-
-    int num_main_states = num_elements(kp);
-    int num_states = 2 * num_main_states;
-
-    vector[num_states] dydt;
-    vector[num_main_states] mass_flow;
-
-    for (i in 1:num_main_states) {
-      int m = 2 * (i-1) + 1;  // main state
-      int a = m + 1;  // auxilary state
-      mass_flow[i] = (kp[i] + ks[i]  / (y[m] + e50[i])) * y[m];
-
-      dydt[m] = -mass_flow[i] - k12[i] * y[m] + k21[i] * y[a];
-      dydt[a] = +k12[i] * y[m] - k21[i] * y[a];
-
-      if (i != 1) {
-        dydt[m] += mass_flow[i-1];
-      }
-    }
-    return dydt;
-
-  }
-
-  vector sample_vector_rng(real m, real s, int N) {
-    vector[N] sample;
-    for(i in 1:N)
-      sample[i] = normal_rng(m, s);
-    return sample;
-  }
-
-  vector[] simulate_mean(real t0, vector log_a0, real[] ts, int adjoint_integrator,
-                         real rel_tol_f, real abs_tol_f,
-                         real rel_tol_b, real abs_tol_b,
-                         real rel_tol_q, real abs_tol_q,
-                         int max_num_steps,
-                         int num_checkpoints,
-                         int interpolation_polynomial,
-                         int solver_f, int solver_b,
-                         vector log_kp, vector log_ks, vector log_e50, vector log_k12, vector log_k21) {
-    int num_sim = num_elements(ts);
-    int num_states = num_elements(log_a0);
-    int system_size = num_elements(log_kp);
-    vector[num_states] y[num_sim];
-    real kp[system_size];
-    real ks[system_size];
-    real e50[system_size];
-    real k12[system_size];
-    real k21[system_size];
-    vector[num_states] a0;
-
-    for(i in 1:system_size) {
-      kp[i] = exp(log_kp[i]);
-      ks[i] = exp(log_ks[i]);
-      e50[i] = exp(log_e50[i]);
-      k12[i] = exp(log_k12[i]);
-      k21[i] = exp(log_k21[i]);
-    }
-
-    for(i in 1:num_states) {
-        a0[i] = exp(log_a0[i]);
-    }
-
-    if(adjoint_integrator) {
-      y = ode_adjoint_tol_ctl(linked_mass_flow, a0, t0, ts,
-                              rel_tol_f, rep_vector(abs_tol_f, num_states), // forward
-                              rel_tol_b, rep_vector(abs_tol_b, num_states), // backward
-                              rel_tol_q, abs_tol_q, // quadrature
-                              max_num_steps,
-                              num_checkpoints, // number of steps between checkpoints
-                              interpolation_polynomial,  // polynomials
-                              solver_f,  // bdf forward
-                              solver_b,  // bdf backward
-                              kp, ks, e50, k12, k21);
-      /* simplified interface 
-      y = ode_adjoint_tol(linked_mass_flow, a0, t0, ts, 
-                              rel_tol, abs_tol,
-                              max_num_steps,
-                              kp, ks, e50, k12, k21);
-      */
-    } else {
-      y = ode_bdf_tol(linked_mass_flow, a0, t0, ts,
-                      rel_tol_f, abs_tol_f,
-                      max_num_steps,
-                      kp, ks, e50, k12, k21);
-    }
-
-    return y;
-  }
-  
-}
-data {
-  real<lower=0> rel_tol_f;
-  real<lower=0> abs_tol_f;
-  real<lower=0> rel_tol_b;
-  real<lower=0> abs_tol_b;
-  real<lower=0> rel_tol_q;
-  real<lower=0> abs_tol_q;
-  int<lower=0,upper=1> adjoint_integrator;
-  int<lower=1> max_num_steps;
-  int<lower=1> num_checkpoints;
-  int<lower=1,upper=2> interpolation_polynomial;
-  int<lower=1,upper=2> solver_f;
-  int<lower=1,upper=2> solver_b;
-  int<lower=1> system_size;
-  int<lower=1> num_obs;
-  real<lower=0> sigma_sim;
-  real<lower=0> sigma_y;
-}
-transformed data {
-  int num_states = 2 * system_size;
-  real param_scale = sigma_sim / sqrt(num_obs/4);
-
-  vector[system_size] log_kp_ = sample_vector_rng(0.0, sigma_sim, system_size);
-  vector[system_size] log_ks_ = sample_vector_rng(0.0, sigma_sim, system_size);
-  vector[system_size] log_e50_ = sample_vector_rng(0.0, sigma_sim, system_size);
-  vector[system_size] log_k12_ = sample_vector_rng(0.0, sigma_sim, system_size);
-  vector[system_size] log_k21_ = sample_vector_rng(0.0, sigma_sim, system_size);
-  vector[system_size] log_sigma_y_ = sample_vector_rng(0.0, sigma_y, system_size);
-  vector[num_states] log_a0_ = sample_vector_rng(0.0, sigma_sim, num_states);
-  vector[num_states] y_[num_obs];
-  real ts[num_obs];
-  real t0 = 0.0;
-
-  ts[1] = 1.0;
-  for(i in 2:num_obs) {
-    ts[i] = 1.5 * ts[i-1];
-  }
-
-  y_ = simulate_mean(t0, log_a0_, ts, 0,
-                     rel_tol_f, abs_tol_f,
-                     rel_tol_b, abs_tol_b,
-                     rel_tol_q, abs_tol_q,
-                     max_num_steps,
-                     num_checkpoints,
-                     interpolation_polynomial,
-                     solver_f, solver_b,
-                     log_kp_, log_ks_, log_e50_, log_k12_, log_k21_);
-  
-  for(i in 1:num_obs) {
-    for(j in 1:system_size) {
-      int m = 2*(j-1) + 1;
-      y_[i,m] = lognormal_rng(log(y_[i,m]+1E-3), exp(log_sigma_y_[j]));
-    }
-  }
-
-  if(adjoint_integrator) {
-    print("Using adjoint integrator.");
-  } else {
-    print("Using bdf integrator.");
-  }
-  print("relative tolerance forward: ", rel_tol_f);
-  print("absolute tolerance forward: ", abs_tol_f);
-  print("relative tolerance backward: ", rel_tol_b);
-  print("absolute tolerance backward: ", abs_tol_b);
-  print("relative tolerance quadrature: ", rel_tol_q);
-  print("absolute tolerance quadrature: ", abs_tol_q);
-  print("maximum number of steps: ", max_num_steps);
-  print("number of checkpoints: ", num_checkpoints);
-  print("interpolation polynomial: ", interpolation_polynomial);
-  print("solver forward: ", solver_f);
-  print("solver backward: ", solver_b);
-  print("number of time points: ", num_obs);
-  print("system size: ", system_size);
-  print("time points: ", ts);
-  print("y_: ", y_);
-  print("ODE states N: ", num_states);
-  print("ODE parameters varying M: ", 5*system_size + num_states);
-}
-parameters {
-  vector<multiplier=param_scale,offset=log_kp_>[system_size] log_kp;
-  vector<multiplier=param_scale,offset=log_ks_>[system_size] log_ks;
-  vector<multiplier=param_scale,offset=log_e50_>[system_size] log_e50;
-  vector<multiplier=param_scale,offset=log_k12_>[system_size] log_k12;
-  vector<multiplier=param_scale,offset=log_k21_>[system_size] log_k21;
-  vector<offset=log_sigma_y_>[system_size] log_sigma_y;
-  vector<multiplier=param_scale,offset=log_a0_>[num_states] log_a0;
-}
-transformed parameters {
-}
-model {
-  vector[num_states] mu[num_obs];
-
-  profile("ode") {
-    mu = simulate_mean(t0, log_a0, ts, adjoint_integrator,
-                       rel_tol_f, abs_tol_f,
-                       rel_tol_b, abs_tol_b,
-                       rel_tol_q, abs_tol_q,
-                       max_num_steps,
-                       num_checkpoints,
-                       interpolation_polynomial,
-                       solver_f, solver_b,
-                       log_kp, log_ks, log_e50, log_k12, log_k21);
-  }
-  
-  target += normal_lpdf(log_kp| 0.0, sigma_sim);
-  target += normal_lpdf(log_ks| 0.0, sigma_sim);
-  target += normal_lpdf(log_e50| 0.0, sigma_sim);
-  target += normal_lpdf(log_k12| 0.0, sigma_sim);
-  target += normal_lpdf(log_k21| 0.0, sigma_sim);
-  target += normal_lpdf(log_a0| 0.0, sigma_sim);
-  target += normal_lpdf(log_sigma_y| 0.0, sigma_y);
-
-  for(j in 1:system_size) {
-    int m = 2*(j-1) + 1;
-    target += lognormal_lpdf(to_vector(y_[:,m])| log(to_vector(mu[:,m])+1E-3), exp(log_sigma_y[j]));
-  }
-}
-generated quantities {
-  int rank_log_kp[system_size];
-  int rank_log_ks[system_size];
-  int rank_log_e50[system_size];
-  int rank_log_k12[system_size];
-  int rank_log_k21[system_size];
-  int rank_log_sigma_y[system_size];
-  int rank_log_a0[system_size];
-  vector[system_size] bias_log_kp = log_kp - log_kp_;
-  vector[system_size] bias_log_ks = log_ks - log_ks_;
-  vector[system_size] bias_log_e50 = log_e50 - log_e50_;
-  vector[system_size] bias_log_k12 = log_k12 - log_k12_;
-  vector[system_size] bias_log_k21 = log_k21 - log_k21_;
-  vector[system_size] bias_log_sigma_y = log_sigma_y - log_sigma_y_;
-  vector[system_size] bias_log_a0 = log_a0 - log_a0_;
-
-  for(i in 1:system_size) {
-    rank_log_kp[i] = (log_kp[i] > log_kp_[i] ? 1 : 0);
-    rank_log_ks[i] = (log_ks[i] > log_ks_[i] ? 1 : 0);
-    rank_log_e50[i] = (log_e50[i] > log_e50_[i] ? 1 : 0);
-    rank_log_k12[i] = (log_k12[i] > log_k12_[i] ? 1 : 0);
-    rank_log_k21[i] = (log_k21[i] > log_k21_[i] ? 1 : 0);
-    rank_log_sigma_y[i] = (log_sigma_y[i] > log_sigma_y_[i] ? 1 : 0);
-    rank_log_a0[i] = (log_a0[i] > log_a0_[i] ? 1 : 0);
-  }
-  
-}
diff --git a/ode-adjoint/sbc.R b/ode-adjoint/sbc.R
deleted file mode 100644
index 550edaba257..00000000000
--- a/ode-adjoint/sbc.R
+++ /dev/null
@@ -1,102 +0,0 @@
-library(cmdstanr)
-library(posterior)
-library(batchtools)
-set.seed(453453)
-## load utilities and current dev version of OncoBayes2
-source("sbc_tools.R")
-
-#' according to the docs this speeds up the reduce step
-options(batchtools.progress = FALSE)
-
-registry_tmp <- Sys.getenv("TMP_NFS", tempdir())
-
-reg <- makeExperimentRegistry(
-    file.dir = tempfile("sbc-", registry_tmp),
-    ## use the default configured batchtools configuration batchtools.conf
-    ## - found in the environemnt variable R_BATCHTOOLS_SEARCH_PATH
-    ## - current working directory
-    ## - $(HOME)/.batchtools.conf
-    ## conf.file = NA,
-    seed = 47845854,
-    ## our worker functions and package loading
-    source="sbc_tools.R")
-
-## resources of each job: Less than 55min, 2000MB RAM and 1 core
-job_resources <- list(walltime=55, memory=2000, ncpus=1, max.concurrent.jobs=300)
-
-## choose number of sample as being dividable by 2 minus one. This gives us exactly by 2 divisable number of bins.
-base_data  <- list(num_warmup=2^8 - 1, num_sampling=2^8 - 1, adapt_delta=0.9, stan_model=stan_model)
-
-addProblem("linked_mass_flow",
-           data = base_data,
-           fun = setup_system,
-           seed = 2345,
-           ## caching speeds up the reduce step
-           cache = TRUE
-)
-
-addAlgorithm("stan_cvodes", run_benchmark)
-
-pdes <- list(linked_mass_flow = data.frame(system_size=c(1,4)))
-ades <- list(stan_cvodes= data.frame(adjoint_integrator=c(0,1)))
-
-##addExperiments(pdes, ades, repls = 1000)
-addExperiments(pdes, ades, repls = 2)
-
-summarizeExperiments()
-
-## approximate per job runtime in s
-target_runtime <- 100
-## for 30 min runtime we then need
-chunk_size <- round(30 * 60 / target_runtime)
-chunk_size
-
-##chunk_size <- 10
-
-ids <- unwrap(getJobPars())
-
-ids[,chunk:=chunk(job.id, chunk.size=chunk_size)]
-
-auto_submit(ids, reg, job_resources)
-
-
-if(FALSE) {
-job1 <- testJob(4000)
-
-names(job1)
-
-
-job1$summary("lp__")
-job1$draws("lp__")
-
-job1$metadata()$iter_sampling
-
-as.list(job1$rank)
-
-job1_def <- makeJob(4000)
-
-job1_def$instance
-
-job1_def$repl
-
-names(job1_def$problem$data)
-}
-
-calibration <- ijoin(
-  ## grab job parameters
-  unwrap(getJobPars()),
-    unwrap(reduceResultsDataTable(fun=function(x) c(as.list(x$rank),
-                                      ess_speed=as.list(x$ess_lp_speed),
-                                      x[c("num_leaps", "time", "time_per_leap", "num_divergent")]
-                                                    )) )
-    )
-
-
-
-head(calibration)
-
-saveRDS(calibration, file = "calibration.rds")
-
-removeRegistry(0)
-
-sessionInfo()
diff --git a/ode-adjoint/sbc_report.R b/ode-adjoint/sbc_report.R
deleted file mode 100644
index caa3a99a7cc..00000000000
--- a/ode-adjoint/sbc_report.R
+++ /dev/null
@@ -1,116 +0,0 @@
-#' ---
-#' title: Simulation based calibration for CVODES solvers
-#' author: ""
-#' date: "`r date()`"
-#' output: html_vignette
-#' params:
-#'   include_plots: FALSE
-#' ---
-#'
-#+ include=FALSE
-source("sbc_tools.R")
-library(knitr)
-library(tools)
-library(assertthat)
-library(dplyr)
-library(tidyr)
-library(broom)
-library(ggplot2)
-theme_set(theme_bw())
-
-knitr::opts_chunk$set(
-  fig.width = 1.62*4,
-  fig.height = 4,
-  cache=FALSE,
-  echo=FALSE
-    )
-
-calibration <- readRDS("calibration.rds")
-
-plot_binned <- function(cal_df) {
-
-    pl <- NULL
-
-    if(!include_plots)
-        return(pl)
-    
-    if(!all(cal_df$count == 0)) {
-        
-        S <- calibration$S
-        B <- calibration$B
-        
-        c95 <- qbinom(c(0.025, 0.5, 0.975), S, 1 / B)
-        
-        dd <- cal_df %>%
-            arrange(param, bin) %>%
-                group_by(param) %>%
-                    mutate(ecdf = cumsum(count) / S, ecdf_ref = (bin + 1) / B) %>%
-                        filter(!all(ecdf == 0))
-        
-        nparam <- length(unique(dd$param))
-        if(unique(dd$partype) %in% c("mu_eta", "tau_eta")){
-            nc <- nparam
-        } else{
-            nc <- 2
-        }
-        
-        nr <- max(1, ceiling(nparam / nc))
-
-        pl <- list()
-        pl[["hist"]] <- ggplot(dd, aes(bin, count)) +
-            facet_wrap(~ param, nrow = nr, ncol = nc) +
-                geom_col() +
-                    geom_hline(yintercept=c95[c(1,3)], linetype=I(2)) +
-                        geom_hline(yintercept=c95[c(2)], linetype=I(3))
-        pl[["ecdf_diff"]] <- ggplot(dd, aes(bin, ecdf-ecdf_ref)) +
-            facet_wrap(~ param, nrow = nr, ncol = nc) +
-                geom_step() +
-                    geom_hline(yintercept=0, linetype=I(3))
-        pl
-    }
-
-    return(pl)
-}
-
-
-dim(calibration)
-rank_params <- names(calibration)[grepl(names(calibration), pattern = "rank")]
-
-# calibration_data_binned <- calibration_data[, scale64(.SD), by=c("problem", "model", params)]
-
-sampled_ranks <- 2^8
-B <- sampled_ranks/ 2^3
-B
-
-calibration_binned <- calibration %>%
-    mutate_at(.vars = rank_params, .funs = function(x) ceiling((x + 1) / (sampled_ranks/ B) - 1)) %>%
-        mutate(adjoint_integrator=factor(adjoint_integrator))
-
-
-long <- calibration_binned[c("system_size", "adjoint_integrator", rank_params)] %>%
-    tidyr::gather(key = "param", value = "rank", -system_size, -adjoint_integrator) %>%
-        group_by(system_size, adjoint_integrator, param)  %>%
-            mutate(all_NA=all(is.na(rank))) %>%
-                filter(!all_NA)
-
-metrics_long <- calibration_binned[!(names(calibration_binned) %in% rank_params)] %>%
-  tidyr::gather(key = "metric", value = "value", -system_size, -adjoint_integrator, -job.id, -problem, -algorithm) 
-
-long_split <- split(long, long$system_size)
-
-pl_rank <- lapply(long_split, function(d) ggplot(d, aes(rank)) + facet_grid(param~adjoint_integrator*system_size, labeller=label_both) + geom_bar() + theme(strip.text.y = element_text(angle = 0)))
-
-pl_rank[[1]] + theme(strip.text.y = element_text(angle = 0))
-pl_rank[[2]]
-
-ggsave("rank_size_1.png", pl_rank[[1]], width=6, height=6)
-ggsave("rank_size_4.png", pl_rank[[2]], width=6, height=14)
-
-metrics_split <- split(metrics_long, metrics_long$system_size)
-
-pl_metrics <- lapply(metrics_split, function(d) ggplot(d, aes(adjoint_integrator, value)) +
-    facet_wrap(~ metric*system_size, labeller=label_both, scales="free_y") + geom_boxplot()+
-        theme(strip.text.y = element_text(angle = 0)))
-
-ggsave("metrics_1.png", pl_metrics[[1]], width=10, height=10)
-ggsave("metrics_4.png", pl_metrics[[2]], width=10, height=10)
diff --git a/ode-adjoint/sbc_tools.R b/ode-adjoint/sbc_tools.R
deleted file mode 100644
index 17d0b78bd5d..00000000000
--- a/ode-adjoint/sbc_tools.R
+++ /dev/null
@@ -1,152 +0,0 @@
-library(cmdstanr)
-library(posterior)
-library(bayesplot)
-color_scheme_set("brightblue")
-
-if(!("sbc_dir" %in%  ls())) {
-    sbc_dir <- getwd()
-}
-
-stan_model_file <- normalizePath(file.path(sbc_dir, "linked-mass-flow.stan"))
-##stan_model_file <- normalizePath(file.path(sbc_dir, "linked-mass-flow-eigen.stan"))
-
-stan_model <- cmdstan_model(stan_model_file)
-
-setup_system <- function(data, job, ...) {
-    args <- list(...)
-    base_stan_data <- list(rel_tol=1e-6,
-                           abs_tol=1e-6,
-                           max_num_steps=10000,
-                           num_checkpoints=50,
-                           solver_f=2,
-                           solver_b=2,
-                           adjoint_integrator=0,
-                           system_size=1,
-                           num_obs=8,
-                           sigma_sim=1.0,
-                           sigma_y=0.2)
-    modifyList(base_stan_data, args)
-}
-
-run_benchmark <- function(data, job, instance, adjoint_integrator, num_checkpoints, solver_f, solver_b, ...) {
-    fit_data <- modifyList(instance, list(adjoint_integrator=adjoint_integrator,
-                                          num_checkpoints=num_checkpoints,
-                                          solver_f=solver_f,
-                                          solver_b=solver_b))
-    r <- job$repl
-    fit <- data$stan_model$sample(
-        data = fit_data,
-        iter_warmup = data$num_warmup,
-        iter_sampling = data$num_sampling,
-        seed = 123 + r,
-        init=0,
-        chains = 1,
-        parallel_chains = 1,
-        refresh = 100,
-        save_warmup=TRUE,
-        adapt_delta=data$adapt_delta
-        ##verbose=TRUE
-        )
-    c(summarize_benchmark(fit), list(repl=r))
-}
-
-
-summarize_benchmark <- function(fit) {
-    sdiag_all  <- fit$sampler_diagnostics(inc_warmup=TRUE)
-    sdiag_main  <- fit$sampler_diagnostics(inc_warmup=FALSE)
-
-    num_divergent <- sum(sdiag_main[,,"divergent__"])
-    accept_stat <- mean(sdiag_main[,,"accept_stat__"])
-    treedepth <- mean(sdiag_main[,,"treedepth__"])
-    num_leaps <- sum(sdiag_all[,,"n_leapfrog__"])
-    fit_time <- fit$time()$total
-
-    ess_speed <- as.vector(unlist(fit$summary("lp__")[1,c("ess_bulk", "ess_tail")])) / fit_time
-    names(ess_speed) <- c("bulk", "tail")
-    lp_est <- as.vector(fit$summary("lp__"))
-
-    rank_vars <- grep("^rank", fit$metadata()$stan_variables, value=TRUE)
-    ranks <- fit$summary(rank_vars)
-    num_iter <- fit$metadata()$iter_sampling
-
-    count_ranks <- ranks$mean * num_iter
-    names(count_ranks) <- ranks$variable
-
-    list(num_leaps=num_leaps, time=fit_time, time_per_leap=1E3 * fit_time / num_leaps,
-         step_size=fit$metadata()$step_size_adaptation,
-         num_divergent=num_divergent, accept_stat=accept_stat, treedepth=treedepth,
-         ess_lp_speed=ess_speed, lp_est=lp_est, ranks=count_ranks)
-}
-
-
-## Submits to batchtools cluster with fault tolerance, i.e.
-## resubmitting failed jobs max_num_tries times
-auto_submit <- function(jobs, registry, resources=list(), max_num_tries = 10) {
-  all_unfinished_jobs <- jobs
-
-  num_unfinished_jobs <- nrow(all_unfinished_jobs)
-  num_all_jobs <- num_unfinished_jobs
-  remaining_tries <- max_num_tries
-  all_jobs_finished <- FALSE
-  while (remaining_tries > 0 && !all_jobs_finished) {
-    remaining_tries <- remaining_tries - 1
-
-    message("Submitting jobs at ", Sys.time())
-    # Once things run fine let's submit this work to the cluster.
-    submitJobs(all_unfinished_jobs, resources=resources)
-    # Wait for results.
-    waitForJobs()
-    message("Finished waiting for jobs at ", Sys.time())
-
-    # Check status:
-    print(getStatus())
-
-    # Ensure that all jobs are done
-    if (nrow(findNotDone()) != 0) {
-      not_done_jobs <- findNotDone()
-      print(getErrorMessages(not_done_jobs))
-      ##browser()
-      ##invisible(readline(prompt="Press [enter] to continue"))
-
-      message("Some jobs did not complete. Please check the batchtools registry ", registry$file.dir)
-      all_unfinished_jobs <- inner_join(not_done_jobs, all_unfinished_jobs)
-
-      if (num_unfinished_jobs == nrow(all_unfinished_jobs) &&  nrow(all_unfinished_jobs) > 0.25 * num_all_jobs)
-      {
-        # Unfinished job count did not change -> retrying will probably not help. Abort!
-        warning("Error: unfinished job count is not decreasing. Aborting job retries.")
-        remaining_tries <- 0
-      }
-
-      if (num_unfinished_jobs == nrow(jobs))
-      {
-        # All jobs errored -> retrying will probably not help. Abort!
-        warning("Error: all jobs errored. Aborting job retries.")
-        remaining_tries <- 0
-      }
-
-      num_unfinished_jobs <- nrow(all_unfinished_jobs)
-      message("Trying to resubmit jobs. Remaining tries: ", remaining_tries, " / ", max_num_tries)
-    } else {
-      all_jobs_finished <- TRUE
-    }
-  }
-
-  invisible(all_jobs_finished)
-}
-
-
-reduce_run <- function(x) {
-    c(as.list(x$rank),
-      ess_speed=as.list(x$ess_lp_speed),
-      x[c("num_leaps", "time", "time_per_leap", "num_divergent", "repl")]
-      )
-}
-
-reduce_run_bench <- function(x) {
-    c(
-      ess_speed=as.list(x$ess_lp_speed),
-      x[c("num_leaps", "time", "time_per_leap", "num_divergent", "repl")]
-      )
-}
-
diff --git a/ode-adjoint/scale-benchmark.R b/ode-adjoint/scale-benchmark.R
deleted file mode 100644
index bab13dc5ea3..00000000000
--- a/ode-adjoint/scale-benchmark.R
+++ /dev/null
@@ -1,186 +0,0 @@
-library(cmdstanr)
-library(posterior)
-library(batchtools)
-set.seed(453453)
-## load utilities and current dev version of OncoBayes2
-source("sbc_tools.R")
-
-#' according to the docs this speeds up the reduce step
-options(batchtools.progress = FALSE)
-
-registry_tmp <- Sys.getenv("TMP_NFS", tempdir())
-
-reg <- makeExperimentRegistry(
-    file.dir = tempfile("scale-bench-", registry_tmp),
-    ## use the default configured batchtools configuration batchtools.conf
-    ## - found in the environemnt variable R_BATCHTOOLS_SEARCH_PATH
-    ## - current working directory
-    ## - $(HOME)/.batchtools.conf
-    ## conf.file = NA,
-    seed = 47845854,
-    ## our worker functions and package loading
-    source="sbc_tools.R")
-
-## resources of each job: Less than 4h, 2000MB RAM and 1 core
-job_resources <- list(walltime=4*60, memory=2000, ncpus=1, max.concurrent.jobs=300)
-
-## choose number of sample as being dividable by 2 minus one. This gives us exactly by 2 divisable number of bins.
-base_data  <- list(num_warmup=20, num_sampling=20, adapt_delta=0.9, stan_model=stan_model)
-base_data  <- list(num_warmup=10, num_sampling=10, adapt_delta=0.9, stan_model=stan_model)
-
-addProblem("linked_mass_flow",
-           data = base_data,
-           fun = setup_system,
-           seed = 2345,
-           ## caching speeds up the reduce step
-           cache = TRUE
-)
-
-addAlgorithm("stan_cvodes", run_benchmark)
-
-##pdes <- list(linked_mass_flow = data.frame(system_size=c(1,2,4,8,16)))
-##pdes <- list(linked_mass_flow = data.frame(system_size=c(1,4)))
-
-pdes <- list(linked_mass_flow = data.frame(system_size=c(1,2,4,8)))
-ades <- list(stan_cvodes= expand.grid(adjoint_integrator=c(1), num_checkpoints=c(10, 100, 250), solver_f=1:2, solver_b=1:2))
-
-
-pdes <- list(linked_mass_flow = data.frame(system_size=c(128)))
-
-ades <- list(stan_cvodes= rbind(expand.grid(adjoint_integrator=c(1), num_checkpoints=c(250), solver_f=2, solver_b=2),
-                                expand.grid(adjoint_integrator=c(1), num_checkpoints=c(250), solver_f=4, solver_b=4))
-             )
-
-
-##ades <- list(stan_cvodes= rbind(expand.grid(adjoint_integrator=c(1), num_checkpoints=c(250), solver_f=1:2, solver_b=1:2),
-##                                expand.grid(adjoint_integrator=c(1), num_checkpoints=c(250)##, solver_f=3:4, solver_b=3:4))
-##             )
-
-##addExperiments(pdes, ades, repls = 1000)
-addExperiments(pdes, ades, repls = 1)
-
-summarizeExperiments()
-
-## approximate per job runtime in s
-target_runtime <- 100
-## for 30 min runtime we then need
-chunk_size <- round(30 * 60 / target_runtime)
-chunk_size
-
-##chunk_size <- 10
-
-ids <- unwrap(getJobPars())
-
-##ids[,chunk:=chunk(job.id, chunk.size=chunk_size)]
-
-##auto_submit(ids, reg, job_resources)
-
-submitJobs(ids)
-
-getStatus()
-
-findErrors()
-
-getLog(47)
-
-if(FALSE) {
-
-    job1 <- testJob(47)
-
-    names(job1)
-
-    job1
-
-
-job1$summary("lp__")
-job1$draws("lp__")
-
-job1$metadata()$iter_sampling
-
-as.list(job1$rank)
-
-job1_def <- makeJob(4)
-
-job1_def$instance
-
-job1_def$repl
-
-names(job1_def$problem$data)
-}
-
-
-library(ggplot2)
-library(dplyr)
-
-scale_benchmark <- ijoin(
-  ## grab job parameters
-  unwrap(getJobPars()),
-  unwrap(reduceResultsDataTable(fun=reduce_run_bench))
-) %>% filter(!(system_size==8 & num_checkpoints==100 & solver_f==2 & solver_b==2))
-
-head(scale_benchmark)
-
-scale_benchmark <- mutate(scale_benchmark,
-                          adjoint_integrator=factor(adjoint_integrator),
-                          sparse_f=solver_f > 2,
-                          sparse_b=solver_b > 2,
-                          lmm_f=case_when(solver_f %% 2 == 0 ~ "BDF", TRUE ~ "Adams"),
-                          lmm_b=case_when(solver_b %% 2 == 0 ~ "BDF", TRUE ~ "Adams")
-                          ##num_checkpoints=factor(num_checkpoints),
-                          ##solver_f=factor(solver_f),
-                          ##solver_b=factor(solver_b)
-                          )
-
-saveRDS(scale_benchmark, file = "scale_benchmark.rds")
-
-removeRegistry(0)
-
-sessionInfo()
-
-scale_benchmark <- as.data.frame(scale_benchmark)
-scale_benchmark <- mutate(scale_benchmark, repl=factor(repl), set=paste(repl,adjoint_integrator,sep="-"), solver=factor(paste(solver_f, solver_b, sep="-")))
-
-
-scale_benchmark_ref <- left_join(scale_benchmark, filter(scale_benchmark, adjoint_integrator==0)[c("repl", "system_size", "time_per_leap")], suffix=c("", "_ref"), by=c("repl", "system_size"))
-
-scale_benchmark_ref
-
-lf <- lm(log(time_per_leap) ~ 1 + solver_f + solver_b + factor(num_checkpoints), data=subset(scale_benchmark_ref, system_size==8))
-summary(lf) ## bdf for backward makes things by 13% slower... change polyonmial?
-
-theme_set(theme_bw())
-ggplot(scale_benchmark, aes(num_checkpoints, time_per_leap, colour=solver, shape=repl)) +
-    facet_wrap(~system_size, scales="free_y") +
-   geom_point()  + geom_line() +
-    ##scale_x_log10(breaks=c(1,2,4,8,16)) +
-    scale_y_log10() +
-    ggtitle("Time per leapfrog of adjoint and forward per replication")
-
-ggsave("tuning_scale_time_per_leap.png", width=1.6 *4, height=4, dpi=120)
-
-
-ggplot(scale_benchmark, aes(sparse_f, time_per_leap, colour=sparse_b, linetype=lmm_f, shape=repl)) +
-    facet_grid(system_size~lmm_b, scales="free_y") +
-   geom_point()  + geom_line() +
-    ##scale_x_log10(breaks=c(1,2,4,8,16)) +
-    scale_y_log10() +
-    ggtitle("Time per leapfrog of adjoint and forward per replication")
-
-
-theme_set(theme_bw())
-ggplot(scale_benchmark, aes(system_size, time_per_leap, group=set, colour=adjoint_integrator, shape=repl)) +
-   geom_point()  + geom_line() +
-    scale_x_log10(breaks=c(1,2,4,8,16)) +
-    scale_y_log10() +
-    ggtitle("Time per leapfrog of adjoint and forward per replication")
-
-ggsave("scale_time_per_leap.png", width=1.6 *4, height=4, dpi=120)
-
-ggplot(filter(scale_benchmark_ref, adjoint_integrator==1), aes(system_size, time_per_leap_ref/time_per_leap, group=set, shape=repl)) +
-   geom_point()  + geom_line() +
-    scale_x_log10(breaks=c(1,2,4,8,16)) +
-    scale_y_log10() +
-    geom_hline(yintercept=1.0, linetype=2) +
-    ggtitle("Speedup of adjoint vs forward per replication")
-
-ggsave("scale_speedup.png", width=1.6 *4, height=4, dpi=120)
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index accac958241..51af06f0163 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -36,7 +36,6 @@ class cvodes_integrator_adjoint_vari : public vari {
   using T_Return = return_type_t<T_y0, T_t0, T_ts, T_Args...>;
   using T_y0_t0 = return_type_t<T_y0, T_t0>;
 
-  std::decay_t<F> f_;
   arena_t<Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1>> y0_;
   arena_t<T_t0> t0_;
   std::vector<arena_t<T_ts>, arena_allocator<arena_t<T_ts>>> ts_;
@@ -91,7 +90,6 @@ class cvodes_integrator_adjoint_vari : public vari {
   static constexpr bool is_var_ts{is_var<T_ts>::value};
   static constexpr bool is_var_t0{is_var<T_t0>::value};
   static constexpr bool is_var_y0{is_var<T_y0_t0>::value};
-  // is_var_args{is_var<scalar_type_t<T_Args>>::value...};
   static constexpr bool is_any_var_args{
       disjunction<is_var<scalar_type_t<T_Args>>...>::value};
   static constexpr bool is_var_return{is_var<T_Return>::value};
@@ -104,7 +102,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   template <typename yT, typename... ArgsT>
   constexpr auto rhs(double t, const yT& y,
                      const std::tuple<ArgsT...>& args_tuple) const {
-    return apply([&](auto&&... args) { return f_(t, y, msgs_, args...); },
+    return apply([&](auto&&... args) { return solver_->f_(t, y, msgs_, args...); },
                  args_tuple);
   }
 
@@ -334,6 +332,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * vari class).
    */
   struct cvodes_solver : public chainable_alloc {
+    const std::decay_t<F> f_;
     size_t N_;
     std::vector<Eigen::VectorXd> y_;
     const std::string function_name_str_;
@@ -350,14 +349,15 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     SUNMatrix A_backward_;
     SUNLinearSolver LS_backward_;
-    template <typename StateFwd, typename StateBwd, typename Quad,
+    template <typename FF, typename StateFwd, typename StateBwd, typename Quad,
               typename AbsTolFwd, typename AbsTolBwd>
-    cvodes_solver(const char* function_name, size_t N, size_t num_args_vars,
+    cvodes_solver(const char* function_name, FF&& f, size_t N, size_t num_args_vars,
                   size_t ts_size, int solver_forward, StateFwd& state_forward,
                   StateBwd& state_backward, Quad& quad,
                   AbsTolFwd& absolute_tolerance_forward,
                   AbsTolBwd& absolute_tolerance_backward)
         : chainable_alloc(),
+          f_(f),
           N_(N),
           y_(ts_size),
           function_name_str_(function_name),
@@ -456,7 +456,6 @@ class cvodes_integrator_adjoint_vari : public vari {
       int interpolation_polynomial, int solver_forward, int solver_backward,
       std::ostream* msgs, const T_Args&... args)
       : vari(NOT_A_NUMBER),
-        f_(f),
         y0_(y0),
         t0_(t0),
         ts_(ts.begin(), ts.end()),
@@ -503,7 +502,7 @@ class cvodes_integrator_adjoint_vari : public vari {
         args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             num_args_vars_)),
         solver_(new cvodes_solver(
-            function_name, N_, num_args_vars_, ts_.size(), solver_forward_,
+            function_name, f, N_, num_args_vars_, ts_.size(), solver_forward_,
             state_forward_, state_backward_, quad_, absolute_tolerance_forward_,
             absolute_tolerance_backward_)) {
     save_varis(t0_varis_, t0);

From d14f9f0ef51db23a6c76eb7abb543dcc1ac64cc4 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 27 Apr 2021 22:45:37 +0200
Subject: [PATCH 104/157] cleanups & added some more inline & constexpr as
 appropiate

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 111 ++++--------------
 1 file changed, 21 insertions(+), 90 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 51af06f0163..93a65923c34 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -110,7 +110,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Implements the function of type CVRhsFn which is the user-defined
    * ODE RHS passed to CVODES.
    */
-  static int cv_rhs(realtype t, N_Vector y, N_Vector ydot, void* user_data) {
+  constexpr static int cv_rhs(realtype t, N_Vector y, N_Vector ydot, void* user_data) {
     const cvodes_integrator_adjoint_vari* integrator
         = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->rhs(t, NV_DATA_S(y), NV_DATA_S(ydot));
@@ -121,8 +121,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Implements the function of type CVRhsFnB which is the
    * RHS of the backward ODE system.
    */
-  static int cv_rhs_adj_sens(realtype t, N_Vector y, N_Vector yB,
-                             N_Vector yBdot, void* user_data) {
+  constexpr static int cv_rhs_adj_sens(realtype t, N_Vector y, N_Vector yB,
+                                       N_Vector yBdot, void* user_data) {
     const cvodes_integrator_adjoint_vari* integrator
         = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->rhs_adj_sens(t, y, yB, yBdot);
@@ -133,8 +133,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Implements the function of type CVQuadRhsFnB which is the
    * RHS of the backward ODE system's quadrature.
    */
-  static int cv_quad_rhs_adj(realtype t, N_Vector y, N_Vector yB,
-                             N_Vector qBdot, void* user_data) {
+  constexpr static int cv_quad_rhs_adj(realtype t, N_Vector y, N_Vector yB,
+                                       N_Vector qBdot, void* user_data) {
     cvodes_integrator_adjoint_vari* integrator
         = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
     integrator->quad_rhs_adj(t, y, yB, qBdot);
@@ -147,9 +147,9 @@ class cvodes_integrator_adjoint_vari : public vari {
    * ode_rhs wrt to the states y. The jacobian is stored in column
    * major format.
    */
-  static int cv_jacobian_states(realtype t, N_Vector y, N_Vector fy,
-                                SUNMatrix J, void* user_data, N_Vector tmp1,
-                                N_Vector tmp2, N_Vector tmp3) {
+  constexpr static int cv_jacobian_states(realtype t, N_Vector y, N_Vector fy,
+                                          SUNMatrix J, void* user_data, N_Vector tmp1,
+                                          N_Vector tmp2, N_Vector tmp3) {
     const cvodes_integrator_adjoint_vari* integrator
         = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_states(t, y, J);
@@ -160,9 +160,9 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Implements the CVLsJacFnB function for evaluating the jacobian of
    * the adjoint problem.
    */
-  static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
-                             SUNMatrix J, void* user_data, N_Vector tmp1,
-                             N_Vector tmp2, N_Vector tmp3) {
+  constexpr static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
+                                       SUNMatrix J, void* user_data, N_Vector tmp1,
+                                       N_Vector tmp2, N_Vector tmp3) {
     const cvodes_integrator_adjoint_vari* integrator
         = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_adj(t, y, J);
@@ -197,7 +197,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] yB state of the adjoint ODE system
    * @param[out] yBdot evaluation of adjoint ODE RHS
    */
-  void rhs_adj_sens(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
+  inline void rhs_adj_sens(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
     Eigen::Map<Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
     Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(yBdot), N_);
@@ -233,7 +233,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] yB state of the adjoint ODE system
    * @param[out] qBdot evaluation of adjoint ODE quadrature RHS
    */
-  void quad_rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector qBdot) {
+  inline void quad_rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector qBdot) {
     const Eigen::VectorXd y_vec = Eigen::Map<Eigen::VectorXd>(NV_DATA_S(y), N_);
     Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(qBdot), num_args_vars_);
@@ -308,7 +308,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   /**
    * Overloads which setup the states returned from the forward solve. In case
    * the return type is a double only, then no autodiff is needed. In case of
-   * autodiff non-chaining varis are setup accordingly.
+   * autodiff then non-chaining varis are setup accordingly.
    */
   void store_state(std::size_t n, const Eigen::VectorXd& state,
                    Eigen::Matrix<var, Eigen::Dynamic, 1>& state_return) {
@@ -333,7 +333,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    */
   struct cvodes_solver : public chainable_alloc {
     const std::decay_t<F> f_;
-    size_t N_;
+    const size_t N_;
     std::vector<Eigen::VectorXd> y_;
     const std::string function_name_str_;
     const char* function_name_;
@@ -349,6 +349,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     SUNMatrix A_backward_;
     SUNLinearSolver LS_backward_;
+
     template <typename FF, typename StateFwd, typename StateBwd, typename Quad,
               typename AbsTolFwd, typename AbsTolBwd>
     cvodes_solver(const char* function_name, FF&& f, size_t N, size_t num_args_vars,
@@ -606,7 +607,6 @@ class cvodes_integrator_adjoint_vari : public vari {
    *   solution time (excluding the initial state)
    */
   std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> operator()() {
-    // std::cout << "forward integrate..." << std::endl;
     const double t0_dbl = value_of(t0_);
     const auto ts_dbl = value_of(ts_);
     std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return(
@@ -615,8 +615,6 @@ class cvodes_integrator_adjoint_vari : public vari {
     double t_init = t0_dbl;
     for (size_t n = 0; n < ts_dbl.size(); ++n) {
       double t_final = ts_dbl[n];
-      // std::cout << "n = " << n << ": t_init = " << t_init << ", t_final = "
-      // << t_final << std::endl;
       if (t_final != t_init) {
         if (is_var_return) {
           int ncheck;
@@ -653,17 +651,10 @@ class cvodes_integrator_adjoint_vari : public vari {
     }
 
     forward_returned_ = true;
-    // std::cout << "forward integrate...done" << std::endl;
     return y_return;
   }
 
   virtual void chain() {
-    // std::cout << "backward v2 integrate..." << std::endl;
-    // std::cout << "backward v2 chain_count_ = " << chain_count_++ <<
-    // std::endl;
-    // std::cout << "chain" << std::endl; //<-- Good way to verify it's only
-    //  being called once
-
     if (solver_->cvodes_mem_ == nullptr) {
       return;
     }
@@ -682,22 +673,11 @@ class cvodes_integrator_adjoint_vari : public vari {
       for (int i = 0; i < ts_.size(); ++i) {
         Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
         for (int j = 0; j < N_; j++) {
-          // std::cout << "i: " << i << ", j: " << j << std::endl;
           step_sens.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
         }
 
         ts_varis_[i]->adj_ += step_sens.dot(
             rhs(value_of(ts_[i]), solver_->y_[i], value_of_args_tuple_));
-        /*
-          apply(
-          [&](auto&&... args) {
-          double adj = step_sens.dot(
-          memory->f_(t_init, memory->y_[i], msgs_, args...));
-          // std::cout << "adj: " << adj << ", i: " << i << std::endl;
-          return adj;
-            },
-            memory->value_of_args_tuple_);
-        */
       }
 
       if (is_var_only_ts) {
@@ -707,28 +687,6 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     state_backward_.setZero();
     quad_.setZero();
-    // N_VConst(0.0, memory->nv_state_sens_);
-    // N_VConst(0.0, memory->nv_quad_);
-
-    /* check these if needed
-      flag = CVodeSetUserDataB(cvode_mem, which, user_dataB);
-      flag = CVodeSetMaxOrdB(cvode_mem, which, maxordB);
-      flag = CVodeSetMaxNumStepsB(cvode_mem, which, mxstepsB);  ** WE
-      NEED THIS **
-      flag = CVodeSetInitStepB(cvode_mem, which, hinB);
-      flag = CVodeSetMinStepB(cvode_mem, which, hminB);
-      flag = CVodeSetMaxStepB(cvode_mem, which, hmaxB);
-      flag = CVodeSetStabLimDetB(cvode_mem, which, stldetB); ** MAYBE? **
-      flag = CVodeSetConstraintsB(cvode_mem, which, constraintsB);
-     */
-
-    /*
-    if (memory->backward_is_initialized_) {
-      std::cout << "backward integrate: index_b_ = " << memory->index_b_ <<
-    std::endl; } else { std::cout << "backward not yet initialized" <<
-    std::endl;
-    }
-    */
 
     // At every time step, collect the adjoints from the output
     // variables and re-initialize the solver
@@ -736,16 +694,11 @@ class cvodes_integrator_adjoint_vari : public vari {
     for (int i = ts_.size() - 1; i >= 0; --i) {
       // Take in the adjoints from all the output variables at this point
       // in time
-      // Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
       for (int j = 0; j < N_; j++) {
-        // std::cout << "i: " << i << ", j: " << j << std::endl;
         state_backward_.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
-        // step_sens.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
       }
 
       double t_final = value_of((i > 0) ? ts_[i - 1] : t0_);
-      // std::cout << "backward: time-point " << i << "; t_init = " << t_init <<
-      // "; t_final = " << t_final << std::endl;
       if (t_final != t_init) {
         if (!backward_is_initialized_) {
           check_flag_sundials(CVodeCreateB(solver_->cvodes_mem_,
@@ -833,10 +786,6 @@ class cvodes_integrator_adjoint_vari : public vari {
           check_flag_sundials(error_code, "CVodeB");
         }
 
-        // check_flag_sundials(CVodeB(memory->cvodes_mem_, t_final,
-        // CV_NORMAL),
-        //                    "CVodeB");
-
         // obtain adjoint states and update t_init to time point
         // reached of t_final
         check_flag_sundials(CVodeGetB(solver_->cvodes_mem_, index_backward_,
@@ -853,42 +802,24 @@ class cvodes_integrator_adjoint_vari : public vari {
     }
 
     if (is_var_t0) {
-      Eigen::VectorXd y0d = value_of(y0_);
+      Eigen::VectorXd value_of_y0 = value_of(y0_);
       t0_varis_[0]->adj_
-          += -state_backward_.dot(rhs(t_init, y0d, value_of_args_tuple_));
-      /*
-        apply(
-        [&](auto&&... args) {
-        return -state_sens.dot(memory->f_(t_init, y0d, msgs_, args...));
-        },
-        memory->value_of_args_tuple_);
-      */
+          += -state_backward_.dot(rhs(t_init, value_of_y0, value_of_args_tuple_));
     }
 
-    // do we need this a 2nd time? Don't think so.
-    /*
-      if (args_vars_ > 0) {
-      check_flag_sundials(
-      CVodeGetQuadB(memory->cvodes_mem_, indexB, &t_init, nv_quad),
-      "CVodeGetQuadB");
-      }
-    */
-
     // After integrating all the way back to t0, we finally have the
     // the adjoints we wanted
     // These are the dlog_density / d(initial_conditions[s]) adjoints
-    for (size_t s = 0; s < num_y0_vars_; s++) {
+    for (size_t s = 0; s < num_y0_vars_; ++s) {
       y0_varis_[s]->adj_ += state_backward_.coeff(s);
     }
 
     // These are the dlog_density / d(parameters[s]) adjoints
-    for (size_t s = 0; s < num_args_vars_; s++) {
+    for (size_t s = 0; s < num_args_vars_; ++s) {
       args_varis_[s]->adj_ += quad_.coeff(s);
     }
-
-    // std::cout << "backward v2 integrate...done" << std::endl;
   }
-};  // cvodes integrator
+};  // cvodes integrator adjoint vari
 
 }  // namespace math
 }  // namespace stan

From 875082b92a8e60d4bd3fce0875ffac4ec6677053 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Tue, 27 Apr 2021 20:46:45 +0000
Subject: [PATCH 105/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 31 +++++++++++--------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 93a65923c34..c6dc5867599 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -102,15 +102,17 @@ class cvodes_integrator_adjoint_vari : public vari {
   template <typename yT, typename... ArgsT>
   constexpr auto rhs(double t, const yT& y,
                      const std::tuple<ArgsT...>& args_tuple) const {
-    return apply([&](auto&&... args) { return solver_->f_(t, y, msgs_, args...); },
-                 args_tuple);
+    return apply(
+        [&](auto&&... args) { return solver_->f_(t, y, msgs_, args...); },
+        args_tuple);
   }
 
   /**
    * Implements the function of type CVRhsFn which is the user-defined
    * ODE RHS passed to CVODES.
    */
-  constexpr static int cv_rhs(realtype t, N_Vector y, N_Vector ydot, void* user_data) {
+  constexpr static int cv_rhs(realtype t, N_Vector y, N_Vector ydot,
+                              void* user_data) {
     const cvodes_integrator_adjoint_vari* integrator
         = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->rhs(t, NV_DATA_S(y), NV_DATA_S(ydot));
@@ -148,8 +150,9 @@ class cvodes_integrator_adjoint_vari : public vari {
    * major format.
    */
   constexpr static int cv_jacobian_states(realtype t, N_Vector y, N_Vector fy,
-                                          SUNMatrix J, void* user_data, N_Vector tmp1,
-                                          N_Vector tmp2, N_Vector tmp3) {
+                                          SUNMatrix J, void* user_data,
+                                          N_Vector tmp1, N_Vector tmp2,
+                                          N_Vector tmp3) {
     const cvodes_integrator_adjoint_vari* integrator
         = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
     integrator->jacobian_states(t, y, J);
@@ -160,8 +163,9 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Implements the CVLsJacFnB function for evaluating the jacobian of
    * the adjoint problem.
    */
-  constexpr static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB, N_Vector fyB,
-                                       SUNMatrix J, void* user_data, N_Vector tmp1,
+  constexpr static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB,
+                                       N_Vector fyB, SUNMatrix J,
+                                       void* user_data, N_Vector tmp1,
                                        N_Vector tmp2, N_Vector tmp3) {
     const cvodes_integrator_adjoint_vari* integrator
         = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
@@ -197,7 +201,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] yB state of the adjoint ODE system
    * @param[out] yBdot evaluation of adjoint ODE RHS
    */
-  inline void rhs_adj_sens(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
+  inline void rhs_adj_sens(double t, N_Vector y, N_Vector yB,
+                           N_Vector yBdot) const {
     Eigen::Map<Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
     Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(yBdot), N_);
@@ -352,9 +357,9 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     template <typename FF, typename StateFwd, typename StateBwd, typename Quad,
               typename AbsTolFwd, typename AbsTolBwd>
-    cvodes_solver(const char* function_name, FF&& f, size_t N, size_t num_args_vars,
-                  size_t ts_size, int solver_forward, StateFwd& state_forward,
-                  StateBwd& state_backward, Quad& quad,
+    cvodes_solver(const char* function_name, FF&& f, size_t N,
+                  size_t num_args_vars, size_t ts_size, int solver_forward,
+                  StateFwd& state_forward, StateBwd& state_backward, Quad& quad,
                   AbsTolFwd& absolute_tolerance_forward,
                   AbsTolBwd& absolute_tolerance_backward)
         : chainable_alloc(),
@@ -803,8 +808,8 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     if (is_var_t0) {
       Eigen::VectorXd value_of_y0 = value_of(y0_);
-      t0_varis_[0]->adj_
-          += -state_backward_.dot(rhs(t_init, value_of_y0, value_of_args_tuple_));
+      t0_varis_[0]->adj_ += -state_backward_.dot(
+          rhs(t_init, value_of_y0, value_of_args_tuple_));
     }
 
     // After integrating all the way back to t0, we finally have the

From c61958789aa4fffec6e57615ba70bf2bec57cd64 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Wed, 28 Apr 2021 13:25:19 +0200
Subject: [PATCH 106/157] fix memory issues

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index c6dc5867599..c28df340cbe 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -36,6 +36,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   using T_Return = return_type_t<T_y0, T_t0, T_ts, T_Args...>;
   using T_y0_t0 = return_type_t<T_y0, T_t0>;
 
+  const std::decay_t<F> f_;
   arena_t<Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1>> y0_;
   arena_t<T_t0> t0_;
   std::vector<arena_t<T_ts>, arena_allocator<arena_t<T_ts>>> ts_;
@@ -103,7 +104,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   constexpr auto rhs(double t, const yT& y,
                      const std::tuple<ArgsT...>& args_tuple) const {
     return apply(
-        [&](auto&&... args) { return solver_->f_(t, y, msgs_, args...); },
+        [&](auto&&... args) { return f_(t, y, msgs_, args...); },
         args_tuple);
   }
 
@@ -337,7 +338,6 @@ class cvodes_integrator_adjoint_vari : public vari {
    * vari class).
    */
   struct cvodes_solver : public chainable_alloc {
-    const std::decay_t<F> f_;
     const size_t N_;
     std::vector<Eigen::VectorXd> y_;
     const std::string function_name_str_;
@@ -355,15 +355,14 @@ class cvodes_integrator_adjoint_vari : public vari {
     SUNMatrix A_backward_;
     SUNLinearSolver LS_backward_;
 
-    template <typename FF, typename StateFwd, typename StateBwd, typename Quad,
+    template <typename StateFwd, typename StateBwd, typename Quad,
               typename AbsTolFwd, typename AbsTolBwd>
-    cvodes_solver(const char* function_name, FF&& f, size_t N,
+    cvodes_solver(const char* function_name, size_t N,
                   size_t num_args_vars, size_t ts_size, int solver_forward,
                   StateFwd& state_forward, StateBwd& state_backward, Quad& quad,
                   AbsTolFwd& absolute_tolerance_forward,
                   AbsTolBwd& absolute_tolerance_backward)
         : chainable_alloc(),
-          f_(f),
           N_(N),
           y_(ts_size),
           function_name_str_(function_name),
@@ -462,6 +461,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       int interpolation_polynomial, int solver_forward, int solver_backward,
       std::ostream* msgs, const T_Args&... args)
       : vari(NOT_A_NUMBER),
+        f_(f),
         y0_(y0),
         t0_(t0),
         ts_(ts.begin(), ts.end()),
@@ -508,7 +508,7 @@ class cvodes_integrator_adjoint_vari : public vari {
         args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             num_args_vars_)),
         solver_(new cvodes_solver(
-            function_name, f, N_, num_args_vars_, ts_.size(), solver_forward_,
+            function_name, N_, num_args_vars_, ts_.size(), solver_forward_,
             state_forward_, state_backward_, quad_, absolute_tolerance_forward_,
             absolute_tolerance_backward_)) {
     save_varis(t0_varis_, t0);

From 0150ed6873329e6539c7f63be7e45a31dbc76bb1 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Wed, 28 Apr 2021 11:26:21 +0000
Subject: [PATCH 107/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index c28df340cbe..8fb578c1c24 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -103,9 +103,8 @@ class cvodes_integrator_adjoint_vari : public vari {
   template <typename yT, typename... ArgsT>
   constexpr auto rhs(double t, const yT& y,
                      const std::tuple<ArgsT...>& args_tuple) const {
-    return apply(
-        [&](auto&&... args) { return f_(t, y, msgs_, args...); },
-        args_tuple);
+    return apply([&](auto&&... args) { return f_(t, y, msgs_, args...); },
+                 args_tuple);
   }
 
   /**
@@ -357,9 +356,9 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     template <typename StateFwd, typename StateBwd, typename Quad,
               typename AbsTolFwd, typename AbsTolBwd>
-    cvodes_solver(const char* function_name, size_t N,
-                  size_t num_args_vars, size_t ts_size, int solver_forward,
-                  StateFwd& state_forward, StateBwd& state_backward, Quad& quad,
+    cvodes_solver(const char* function_name, size_t N, size_t num_args_vars,
+                  size_t ts_size, int solver_forward, StateFwd& state_forward,
+                  StateBwd& state_backward, Quad& quad,
                   AbsTolFwd& absolute_tolerance_forward,
                   AbsTolBwd& absolute_tolerance_backward)
         : chainable_alloc(),

From 273a82b4bb039d142238a92a909161e78377ec6f Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Wed, 28 Apr 2021 20:49:26 +0200
Subject: [PATCH 108/157] some more const declarations

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 8fb578c1c24..67e48d76d63 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -203,10 +203,10 @@ class cvodes_integrator_adjoint_vari : public vari {
    */
   inline void rhs_adj_sens(double t, N_Vector y, N_Vector yB,
                            N_Vector yBdot) const {
-    Eigen::Map<Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
+    Eigen::Map<const Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
     Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(yBdot), N_);
-    mu_dot = Eigen::VectorXd::Zero(N_);
+    mu_dot.setZero();
 
     const nested_rev_autodiff nested;
 
@@ -239,10 +239,11 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[out] qBdot evaluation of adjoint ODE quadrature RHS
    */
   inline void quad_rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector qBdot) {
-    const Eigen::VectorXd y_vec = Eigen::Map<Eigen::VectorXd>(NV_DATA_S(y), N_);
+    //const Eigen::VectorXd y_vec = Eigen::Map<Eigen::VectorXd>(NV_DATA_S(y), N_);
+    Eigen::Map<const Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
     Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(qBdot), num_args_vars_);
-    mu_dot = Eigen::VectorXd::Zero(num_args_vars_);
+    mu_dot.setZero();
 
     const nested_rev_autodiff nested;
 

From c6510f86b33924eb72ef8b8b562b6bd3a8013acf Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Wed, 28 Apr 2021 21:00:39 +0200
Subject: [PATCH 109/157] cleanup

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 67e48d76d63..0a6185c525e 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -239,7 +239,6 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[out] qBdot evaluation of adjoint ODE quadrature RHS
    */
   inline void quad_rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector qBdot) {
-    //const Eigen::VectorXd y_vec = Eigen::Map<Eigen::VectorXd>(NV_DATA_S(y), N_);
     Eigen::Map<const Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
     Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(qBdot), num_args_vars_);
@@ -677,7 +676,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     if (is_var_ts) {
       for (int i = 0; i < ts_.size(); ++i) {
         Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
-        for (int j = 0; j < N_; j++) {
+        for (int j = 0; j < N_; ++j) {
           step_sens.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
         }
 
@@ -699,7 +698,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     for (int i = ts_.size() - 1; i >= 0; --i) {
       // Take in the adjoints from all the output variables at this point
       // in time
-      for (int j = 0; j < N_; j++) {
+      for (int j = 0; j < N_; ++j) {
         state_backward_.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
       }
 
@@ -807,9 +806,8 @@ class cvodes_integrator_adjoint_vari : public vari {
     }
 
     if (is_var_t0) {
-      Eigen::VectorXd value_of_y0 = value_of(y0_);
       t0_varis_[0]->adj_ += -state_backward_.dot(
-          rhs(t_init, value_of_y0, value_of_args_tuple_));
+          rhs(t_init, value_of(y0_), value_of_args_tuple_));
     }
 
     // After integrating all the way back to t0, we finally have the

From 18ecc0d2231dc4601ace4dabfe6d568be9d7b943 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Wed, 28 Apr 2021 21:19:36 +0200
Subject: [PATCH 110/157] fix recover memory issues of ode test framework

---
 test/unit/math/rev/functor/test_fixture_ode.hpp     | 5 +++++
 test/unit/math/rev/functor/test_fixture_ode_sho.hpp | 8 ++++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/test/unit/math/rev/functor/test_fixture_ode.hpp b/test/unit/math/rev/functor/test_fixture_ode.hpp
index 5707b467810..074cfcd398d 100644
--- a/test/unit/math/rev/functor/test_fixture_ode.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode.hpp
@@ -109,6 +109,11 @@
  */
 template <class ode_problem_type>
 struct ODETestFixture : public ::testing::Test {
+
+  virtual void TearDown() {
+    stan::math::recover_memory();
+  }
+  
   /**
    * test ODE solver pass
    *
diff --git a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
index 1ea6b139f8f..6a8b6017804 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
@@ -268,8 +268,7 @@ template <typename T>
 struct harmonic_oscillator_t0_ad_test
     : public harmonic_oscillator_ode_base<T>,
       public ODETestFixture<harmonic_oscillator_t0_ad_test<T>> {
-  stan::math::nested_rev_autodiff nested;
-  stan::math::var t0v;
+  stan::math::var t0v_;
 
   harmonic_oscillator_t0_ad_test()
       : harmonic_oscillator_ode_base<T>(), t0v(stan::math::to_var(this->t0)) {
@@ -283,19 +282,24 @@ struct harmonic_oscillator_t0_ad_test
   }
 
   void test_t0_ad(double tol) {
+    stan::math::nested_rev_autodiff nested;
     auto res = apply_solver();
     res[0][0].grad();
     EXPECT_NEAR(t0v.adj(), -0.66360742442816977871, tol);
     nested.set_zero_all_adjoints();
+    t0v.adj() = 0.0;
     res[0][1].grad();
     EXPECT_NEAR(t0v.adj(), 0.23542843380353062344, tol);
     nested.set_zero_all_adjoints();
+    t0v.adj() = 0.0;
     res[1][0].grad();
     EXPECT_NEAR(t0v.adj(), -0.2464078910913158893, tol);
     nested.set_zero_all_adjoints();
+    t0v.adj() = 0.0;
     res[1][1].grad();
     EXPECT_NEAR(t0v.adj(), -0.38494826636037426937, tol);
     nested.set_zero_all_adjoints();
+    t0v.adj() = 0.0;
   }
 };
 

From a2af67dab8b7321b26e5a50ad38ee3773f593f81 Mon Sep 17 00:00:00 2001
From: Steve Bronder <stevo15025@gmail.com>
Date: Wed, 28 Apr 2021 15:58:18 -0400
Subject: [PATCH 111/157] fix leak in coupled ode system and move the nested to
 be local in the harmonic oscillator test

---
 stan/math/rev/functor/coupled_ode_system.hpp        |  5 +++--
 test/unit/math/rev/functor/test_fixture_ode.hpp     |  5 +++++
 test/unit/math/rev/functor/test_fixture_ode_sho.hpp | 10 +++++-----
 3 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/stan/math/rev/functor/coupled_ode_system.hpp b/stan/math/rev/functor/coupled_ode_system.hpp
index 0236e525365..cbd4314c493 100644
--- a/stan/math/rev/functor/coupled_ode_system.hpp
+++ b/stan/math/rev/functor/coupled_ode_system.hpp
@@ -138,8 +138,9 @@ struct coupled_ode_system_impl<false, F, T_y0, Args...> {
 
       y_adjoints_ = y_vars.adj();
 
-      // memset was faster than Eigen setZero
-      memset(args_adjoints_.data(), 0, sizeof(double) * num_args_vars);
+      if (args_adjoints_.size() > 0) {
+        memset(args_adjoints_.data(), 0, sizeof(double) * num_args_vars);
+      }
 
       apply(
           [&](auto&&... args) {
diff --git a/test/unit/math/rev/functor/test_fixture_ode.hpp b/test/unit/math/rev/functor/test_fixture_ode.hpp
index 5707b467810..ddbe4a37692 100644
--- a/test/unit/math/rev/functor/test_fixture_ode.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode.hpp
@@ -109,6 +109,11 @@
  */
 template <class ode_problem_type>
 struct ODETestFixture : public ::testing::Test {
+
+
+  virtual void TearDown() {
+    stan::math::recover_memory();
+  }
   /**
    * test ODE solver pass
    *
diff --git a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
index 1ea6b139f8f..e35df6878d8 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
@@ -268,7 +268,6 @@ template <typename T>
 struct harmonic_oscillator_t0_ad_test
     : public harmonic_oscillator_ode_base<T>,
       public ODETestFixture<harmonic_oscillator_t0_ad_test<T>> {
-  stan::math::nested_rev_autodiff nested;
   stan::math::var t0v;
 
   harmonic_oscillator_t0_ad_test()
@@ -283,19 +282,20 @@ struct harmonic_oscillator_t0_ad_test
   }
 
   void test_t0_ad(double tol) {
+    stan::math::nested_rev_autodiff nested;
     auto res = apply_solver();
     res[0][0].grad();
     EXPECT_NEAR(t0v.adj(), -0.66360742442816977871, tol);
-    nested.set_zero_all_adjoints();
+    stan::math::set_zero_all_adjoints();
     res[0][1].grad();
     EXPECT_NEAR(t0v.adj(), 0.23542843380353062344, tol);
-    nested.set_zero_all_adjoints();
+    stan::math::set_zero_all_adjoints();
     res[1][0].grad();
     EXPECT_NEAR(t0v.adj(), -0.2464078910913158893, tol);
-    nested.set_zero_all_adjoints();
+    stan::math::set_zero_all_adjoints();
     res[1][1].grad();
     EXPECT_NEAR(t0v.adj(), -0.38494826636037426937, tol);
-    nested.set_zero_all_adjoints();
+    stan::math::set_zero_all_adjoints();
   }
 };
 

From 62281d213da848621d11c4d20829112b0d9374a1 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Wed, 28 Apr 2021 19:59:52 +0000
Subject: [PATCH 112/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 test/unit/math/rev/functor/test_fixture_ode.hpp | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/test/unit/math/rev/functor/test_fixture_ode.hpp b/test/unit/math/rev/functor/test_fixture_ode.hpp
index ddbe4a37692..b3457f4f68e 100644
--- a/test/unit/math/rev/functor/test_fixture_ode.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode.hpp
@@ -109,11 +109,7 @@
  */
 template <class ode_problem_type>
 struct ODETestFixture : public ::testing::Test {
-
-
-  virtual void TearDown() {
-    stan::math::recover_memory();
-  }
+  virtual void TearDown() { stan::math::recover_memory(); }
   /**
    * test ODE solver pass
    *

From e7214146dac4f261c60b93fd40f1eac80a1164f1 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Wed, 28 Apr 2021 22:27:14 +0200
Subject: [PATCH 113/157] adding extra bad tests for ode_adjoint_tol_ctl
 interface

---
 .../math/rev/functor/ode_test_functors.hpp    |  27 ++
 .../rev/functor/sho_ode_typed_error_test.cpp  |  26 ++
 .../math/rev/functor/test_fixture_ode_sho.hpp | 232 +++++++++++++++++-
 3 files changed, 284 insertions(+), 1 deletion(-)

diff --git a/test/unit/math/rev/functor/ode_test_functors.hpp b/test/unit/math/rev/functor/ode_test_functors.hpp
index d876556656d..94cafcd0f68 100644
--- a/test/unit/math/rev/functor/ode_test_functors.hpp
+++ b/test/unit/math/rev/functor/ode_test_functors.hpp
@@ -105,6 +105,33 @@ struct ode_adjoint_functor {
         num_steps_between_checkpoints, interpolation_polynomial, solver_forward,
         solver_backward, msgs, args...);
   }
+
+  template <typename F, typename T_y0, typename T_t0, typename T_ts,
+            typename... T_Args, stan::require_eigen_col_vector_t<T_y0>* = nullptr>
+  std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
+                            Eigen::Dynamic, 1>>
+  operator()(
+      const F& f, const T_y0& y0, const T_t0& t0, const std::vector<T_ts>& ts,
+      double relative_tolerance_forward,
+      const Eigen::VectorXd& absolute_tolerance_forward,
+      double relative_tolerance_backward,
+      const Eigen::VectorXd& absolute_tolerance_backward,
+      double relative_tolerance_quadrature, double absolute_tolerance_quadrature,
+      long int max_num_steps,                  // NOLINT(runtime/int)
+      long int num_steps_between_checkpoints,  // NOLINT(runtime/int)
+      int interpolation_polynomial, int solver_forward, int solver_backward,
+      std::ostream* msgs, const T_Args&... args) {
+    return stan::math::ode_adjoint_tol_ctl(f, y0, t0, ts,
+                        relative_tolerance_forward, absolute_tolerance_forward,
+                        relative_tolerance_backward, absolute_tolerance_backward,
+                        relative_tolerance_quadrature, absolute_tolerance_quadrature,
+                        max_num_steps, num_steps_between_checkpoints,
+                        interpolation_polynomial,
+                        solver_forward, solver_backward,
+                        msgs, args...);
+  }
+
+  
 };
 
 #endif
diff --git a/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp b/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
index aebf9b1d216..7470c976faa 100644
--- a/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
@@ -67,3 +67,29 @@ TYPED_TEST_P(harmonic_oscillator_bad_ode_test, bad_ode_error) {
 REGISTER_TYPED_TEST_SUITE_P(harmonic_oscillator_bad_ode_test, bad_ode_error);
 INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_bad_ode_test,
                                harmonic_oscillator_integrate_ode_test_types);
+
+
+/**
+ * Outer product of test types
+ */
+using harmonic_oscillator_ctl_test_types = boost::mp11::mp_product<
+    ode_test_tuple,
+    ::testing::Types<ode_adjoint_functor>,
+    ::testing::Types<double, stan::math::var_value<double>>,  // t
+    ::testing::Types<double, stan::math::var_value<double>>,  // y0
+    ::testing::Types<double, stan::math::var_value<double>>   // theta
+    >;
+
+TYPED_TEST_SUITE_P(harmonic_oscillator_ctl_test);
+TYPED_TEST_P(harmonic_oscillator_ctl_test, no_error) { this->test_good(); }
+TYPED_TEST_P(harmonic_oscillator_ctl_test, error_conditions) { this->test_bad(); }
+TYPED_TEST_P(harmonic_oscillator_ctl_test, value) {
+  this->test_value(0.0);
+  this->test_value(1.0);
+  this->test_value(-1.0);
+}
+REGISTER_TYPED_TEST_SUITE_P(harmonic_oscillator_ctl_test, no_error,
+                            error_conditions, value);
+INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_ctl_test,
+                               harmonic_oscillator_ctl_test_types);
+
diff --git a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
index 6a8b6017804..203c0736720 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
@@ -268,7 +268,7 @@ template <typename T>
 struct harmonic_oscillator_t0_ad_test
     : public harmonic_oscillator_ode_base<T>,
       public ODETestFixture<harmonic_oscillator_t0_ad_test<T>> {
-  stan::math::var t0v_;
+  stan::math::var t0v;
 
   harmonic_oscillator_t0_ad_test()
       : harmonic_oscillator_ode_base<T>(), t0v(stan::math::to_var(this->t0)) {
@@ -500,4 +500,234 @@ struct harmonic_oscillator_analytical_test
   }
 };
 
+/**
+ * Inheriting base type, various fixtures differs by the type of ODE
+ * functor used in <code>apply_solver</code> calls, intended for
+ * different kind of tests.
+ *
+ */
+template <typename T>
+struct harmonic_oscillator_ctl_test
+    : public harmonic_oscillator_ode_base<T>,
+      public ODETestFixture<harmonic_oscillator_test<T>> {
+
+  double rtol_f;
+  double rtol_b;
+  double rtol_q;
+  Eigen::VectorXd atol_f;
+  Eigen::VectorXd atol_b;
+  double atol_q;
+  long int num_steps_check;
+  int inter_poly;
+  int solv_f;
+  int solv_b;
+  
+  
+  harmonic_oscillator_ctl_test() :
+      harmonic_oscillator_ode_base<T>(),
+      rtol_f(this->rtol / 8.0),
+      rtol_b(this->rtol / 4.0),
+      rtol_q(this->rtol),
+      atol_f(Eigen::VectorXd::Constant(this->y0.size(), this->atol / 6.0)),
+      atol_b(Eigen::VectorXd::Constant(this->y0.size(), this->atol / 3.0)),
+      atol_q(this->atol),
+      num_steps_check(100),
+      inter_poly(CV_HERMITE),
+      solv_f(CV_BDF),
+      solv_b(CV_ADAMS)
+  {}
+
+  auto apply_solver() {
+    std::tuple_element_t<0, T> sol;
+    return sol(this->f_eigen, this->y0, this->t0, this->ts, nullptr,
+               this->theta, this->x_r, this->x_i);
+  }
+
+  template <typename T1, typename T2>
+  auto apply_solver(T1&& init, T2&& theta_in) {
+    std::tuple_element_t<0, T> sol;
+    return sol(this->f_eigen, init, this->t0, this->ts, nullptr, theta_in,
+               this->x_r, this->x_i);
+  }
+
+  auto apply_solver_tol() {
+    std::tuple_element_t<1, T> sol;
+    return sol(this->f_eigen, this->y0, this->t0, this->ts, this->rtol,
+               this->atol, this->max_num_step, nullptr, this->theta, this->x_r,
+               this->x_i);
+  }
+
+  auto apply_solver_tol_ctl() {
+    std::tuple_element_t<0, T> sol;
+    return sol(this->f_eigen, this->y0, this->t0, this->ts,
+               this->rtol_f, this->atol_f,
+               this->rtol_b, this->atol_b,
+               this->rtol_q, this->atol_q,
+               this->max_num_step, this->num_steps_check,
+               this->inter_poly, this->solv_f, this->solv_b,
+               nullptr, this->theta, this->x_r,
+               this->x_i);
+  }
+
+  void test_bad() {
+    const auto y0_(this->y0);
+
+    this->y0.resize(0);
+    EXPECT_THROW_MSG(apply_solver(), std::invalid_argument,
+                     "initial state has size 0");
+    this->y0 = y0_;
+
+    const auto t0_ = this->t0;
+    this->t0 = 2.0;
+    EXPECT_THROW_MSG(apply_solver(), std::domain_error,
+                     "initial time is 2, but must be less than 0.1");
+    this->t0 = t0_;
+
+    const auto ts_ = this->ts;
+    this->ts.resize(0);
+    EXPECT_THROW_MSG(apply_solver(), std::invalid_argument, "times has size 0");
+    this->ts = ts_;
+
+    this->ts.resize(2);
+    this->ts[0] = 3.0;
+    this->ts[1] = 1.0;
+    EXPECT_THROW_MSG(apply_solver(), std::domain_error,
+                     "times is not a valid sorted vector");
+    this->ts = ts_;
+
+    const double rtol_f_ = this->rtol_f;
+    this->rtol_f = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "relative_tolerance_forward");
+    this->rtol_f = rtol_f_;
+
+    const double atol_f_ = this->atol_f(0);
+    this->atol_f(0) = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "absolute_tolerance_forward");
+    this->atol_f(0) = atol_f_;
+
+    const auto theta_ = this->theta;
+    const auto x_r_ = this->x_r;
+    const auto x_i_ = this->x_i;
+
+    // NaN errors
+    double nan = std::numeric_limits<double>::quiet_NaN();
+    std::stringstream expected_is_nan;
+    expected_is_nan << "is " << nan;
+
+    this->y0[0] = nan;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial state");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_nan.str());
+    this->y0 = y0_;
+
+    this->t0 = nan;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial time");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_nan.str());
+    this->t0 = t0_;
+
+    this->ts[0] = nan;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "times");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_nan.str());
+    this->ts = ts_;
+
+    this->theta[0] = nan;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     "ode parameters and data");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_nan.str());
+    this->theta = theta_;
+
+    this->x_r.push_back(nan);
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     "ode parameters and data");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_nan.str());
+    this->x_r = x_r_;
+
+    // inf test
+    std::stringstream expected_is_inf;
+    expected_is_inf << "is " << std::numeric_limits<double>::infinity();
+    std::stringstream expected_is_neg_inf;
+    expected_is_neg_inf << "is " << -std::numeric_limits<double>::infinity();
+    double inf = std::numeric_limits<double>::infinity();
+
+    this->y0[0] = inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial state");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_inf.str());
+    this->y0[0] = -inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial state");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_neg_inf.str());
+    this->y0 = y0_;
+
+    this->t0 = inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial time");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_inf.str());
+    this->t0 = -inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial time");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_neg_inf.str());
+    this->t0 = t0_;
+
+    this->ts.back() = inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "times");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_inf.str());
+    this->ts.back() = -inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "times");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_neg_inf.str());
+    this->ts = ts_;
+
+    this->theta[0] = inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     "ode parameters and data");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_inf.str());
+    this->theta[0] = -inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     "ode parameters and data");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_neg_inf.str());
+    this->theta = theta_;
+
+    this->x_r = std::vector<double>{inf};
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     "ode parameters and data");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_inf.str());
+    this->x_r[0] = -inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     "ode parameters and data");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_neg_inf.str());
+    this->x_r = x_r_;
+  }
+
+  void test_value(double t0_in) {
+    this->t0 = t0_in;
+    for (size_t i = 0; i < this->ts.size(); ++i) {
+      this->ts[i] = this->t0 + 0.1 * (i + 1);
+    }
+
+    this->rtol = 1e-8;
+    this->atol = 1e-10;
+    this->max_num_step = 1e6;
+    auto res = apply_solver_tol();
+
+    EXPECT_NEAR(0.995029, stan::math::value_of(res[0][0]), 1e-5);
+    EXPECT_NEAR(-0.0990884, stan::math::value_of(res[0][1]), 1e-5);
+
+    EXPECT_NEAR(-0.421907, stan::math::value_of(res[99][0]), 1e-5);
+    EXPECT_NEAR(0.246407, stan::math::value_of(res[99][1]), 1e-5);
+  }
+};
+
+
 #endif

From c513e9425e7a8872c8258c7494af46fda753cde8 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Wed, 28 Apr 2021 20:36:34 +0000
Subject: [PATCH 114/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../math/rev/functor/ode_test_functors.hpp    | 41 ++++++++----------
 .../rev/functor/sho_ode_typed_error_test.cpp  |  9 ++--
 .../math/rev/functor/test_fixture_ode.hpp     |  5 +--
 .../math/rev/functor/test_fixture_ode_sho.hpp | 43 ++++++++-----------
 4 files changed, 42 insertions(+), 56 deletions(-)

diff --git a/test/unit/math/rev/functor/ode_test_functors.hpp b/test/unit/math/rev/functor/ode_test_functors.hpp
index 94cafcd0f68..997fc4ab075 100644
--- a/test/unit/math/rev/functor/ode_test_functors.hpp
+++ b/test/unit/math/rev/functor/ode_test_functors.hpp
@@ -107,31 +107,28 @@ struct ode_adjoint_functor {
   }
 
   template <typename F, typename T_y0, typename T_t0, typename T_ts,
-            typename... T_Args, stan::require_eigen_col_vector_t<T_y0>* = nullptr>
+            typename... T_Args,
+            stan::require_eigen_col_vector_t<T_y0>* = nullptr>
   std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
                             Eigen::Dynamic, 1>>
-  operator()(
-      const F& f, const T_y0& y0, const T_t0& t0, const std::vector<T_ts>& ts,
-      double relative_tolerance_forward,
-      const Eigen::VectorXd& absolute_tolerance_forward,
-      double relative_tolerance_backward,
-      const Eigen::VectorXd& absolute_tolerance_backward,
-      double relative_tolerance_quadrature, double absolute_tolerance_quadrature,
-      long int max_num_steps,                  // NOLINT(runtime/int)
-      long int num_steps_between_checkpoints,  // NOLINT(runtime/int)
-      int interpolation_polynomial, int solver_forward, int solver_backward,
-      std::ostream* msgs, const T_Args&... args) {
-    return stan::math::ode_adjoint_tol_ctl(f, y0, t0, ts,
-                        relative_tolerance_forward, absolute_tolerance_forward,
-                        relative_tolerance_backward, absolute_tolerance_backward,
-                        relative_tolerance_quadrature, absolute_tolerance_quadrature,
-                        max_num_steps, num_steps_between_checkpoints,
-                        interpolation_polynomial,
-                        solver_forward, solver_backward,
-                        msgs, args...);
+  operator()(const F& f, const T_y0& y0, const T_t0& t0,
+             const std::vector<T_ts>& ts, double relative_tolerance_forward,
+             const Eigen::VectorXd& absolute_tolerance_forward,
+             double relative_tolerance_backward,
+             const Eigen::VectorXd& absolute_tolerance_backward,
+             double relative_tolerance_quadrature,
+             double absolute_tolerance_quadrature,
+             long int max_num_steps,                  // NOLINT(runtime/int)
+             long int num_steps_between_checkpoints,  // NOLINT(runtime/int)
+             int interpolation_polynomial, int solver_forward,
+             int solver_backward, std::ostream* msgs, const T_Args&... args) {
+    return stan::math::ode_adjoint_tol_ctl(
+        f, y0, t0, ts, relative_tolerance_forward, absolute_tolerance_forward,
+        relative_tolerance_backward, absolute_tolerance_backward,
+        relative_tolerance_quadrature, absolute_tolerance_quadrature,
+        max_num_steps, num_steps_between_checkpoints, interpolation_polynomial,
+        solver_forward, solver_backward, msgs, args...);
   }
-
-  
 };
 
 #endif
diff --git a/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp b/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
index 7470c976faa..2c284571ac0 100644
--- a/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
@@ -68,13 +68,11 @@ REGISTER_TYPED_TEST_SUITE_P(harmonic_oscillator_bad_ode_test, bad_ode_error);
 INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_bad_ode_test,
                                harmonic_oscillator_integrate_ode_test_types);
 
-
 /**
  * Outer product of test types
  */
 using harmonic_oscillator_ctl_test_types = boost::mp11::mp_product<
-    ode_test_tuple,
-    ::testing::Types<ode_adjoint_functor>,
+    ode_test_tuple, ::testing::Types<ode_adjoint_functor>,
     ::testing::Types<double, stan::math::var_value<double>>,  // t
     ::testing::Types<double, stan::math::var_value<double>>,  // y0
     ::testing::Types<double, stan::math::var_value<double>>   // theta
@@ -82,7 +80,9 @@ using harmonic_oscillator_ctl_test_types = boost::mp11::mp_product<
 
 TYPED_TEST_SUITE_P(harmonic_oscillator_ctl_test);
 TYPED_TEST_P(harmonic_oscillator_ctl_test, no_error) { this->test_good(); }
-TYPED_TEST_P(harmonic_oscillator_ctl_test, error_conditions) { this->test_bad(); }
+TYPED_TEST_P(harmonic_oscillator_ctl_test, error_conditions) {
+  this->test_bad();
+}
 TYPED_TEST_P(harmonic_oscillator_ctl_test, value) {
   this->test_value(0.0);
   this->test_value(1.0);
@@ -92,4 +92,3 @@ REGISTER_TYPED_TEST_SUITE_P(harmonic_oscillator_ctl_test, no_error,
                             error_conditions, value);
 INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_ctl_test,
                                harmonic_oscillator_ctl_test_types);
-
diff --git a/test/unit/math/rev/functor/test_fixture_ode.hpp b/test/unit/math/rev/functor/test_fixture_ode.hpp
index bb8a3c87ebc..b11fb16b4a2 100644
--- a/test/unit/math/rev/functor/test_fixture_ode.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode.hpp
@@ -110,11 +110,8 @@
  */
 template <class ode_problem_type>
 struct ODETestFixture : public ::testing::Test {
+  virtual void TearDown() { stan::math::recover_memory(); }
 
-  virtual void TearDown() {
-    stan::math::recover_memory();
-  }
-  
   /**
    * test ODE solver pass
    *
diff --git a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
index 203c0736720..68c5b41630e 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
@@ -510,7 +510,6 @@ template <typename T>
 struct harmonic_oscillator_ctl_test
     : public harmonic_oscillator_ode_base<T>,
       public ODETestFixture<harmonic_oscillator_test<T>> {
-
   double rtol_f;
   double rtol_b;
   double rtol_q;
@@ -521,21 +520,19 @@ struct harmonic_oscillator_ctl_test
   int inter_poly;
   int solv_f;
   int solv_b;
-  
-  
-  harmonic_oscillator_ctl_test() :
-      harmonic_oscillator_ode_base<T>(),
-      rtol_f(this->rtol / 8.0),
-      rtol_b(this->rtol / 4.0),
-      rtol_q(this->rtol),
-      atol_f(Eigen::VectorXd::Constant(this->y0.size(), this->atol / 6.0)),
-      atol_b(Eigen::VectorXd::Constant(this->y0.size(), this->atol / 3.0)),
-      atol_q(this->atol),
-      num_steps_check(100),
-      inter_poly(CV_HERMITE),
-      solv_f(CV_BDF),
-      solv_b(CV_ADAMS)
-  {}
+
+  harmonic_oscillator_ctl_test()
+      : harmonic_oscillator_ode_base<T>(),
+        rtol_f(this->rtol / 8.0),
+        rtol_b(this->rtol / 4.0),
+        rtol_q(this->rtol),
+        atol_f(Eigen::VectorXd::Constant(this->y0.size(), this->atol / 6.0)),
+        atol_b(Eigen::VectorXd::Constant(this->y0.size(), this->atol / 3.0)),
+        atol_q(this->atol),
+        num_steps_check(100),
+        inter_poly(CV_HERMITE),
+        solv_f(CV_BDF),
+        solv_b(CV_ADAMS) {}
 
   auto apply_solver() {
     std::tuple_element_t<0, T> sol;
@@ -559,14 +556,11 @@ struct harmonic_oscillator_ctl_test
 
   auto apply_solver_tol_ctl() {
     std::tuple_element_t<0, T> sol;
-    return sol(this->f_eigen, this->y0, this->t0, this->ts,
-               this->rtol_f, this->atol_f,
-               this->rtol_b, this->atol_b,
-               this->rtol_q, this->atol_q,
-               this->max_num_step, this->num_steps_check,
-               this->inter_poly, this->solv_f, this->solv_b,
-               nullptr, this->theta, this->x_r,
-               this->x_i);
+    return sol(this->f_eigen, this->y0, this->t0, this->ts, this->rtol_f,
+               this->atol_f, this->rtol_b, this->atol_b, this->rtol_q,
+               this->atol_q, this->max_num_step, this->num_steps_check,
+               this->inter_poly, this->solv_f, this->solv_b, nullptr,
+               this->theta, this->x_r, this->x_i);
   }
 
   void test_bad() {
@@ -729,5 +723,4 @@ struct harmonic_oscillator_ctl_test
   }
 };
 
-
 #endif

From c6455877477af9cf00f5dd904c4f893d9b6b6c9e Mon Sep 17 00:00:00 2001
From: Steve Bronder <stevo15025@gmail.com>
Date: Wed, 28 Apr 2021 19:23:20 -0400
Subject: [PATCH 115/157] fix cpplint error

---
 test/unit/math/rev/functor/test_fixture_ode_sho.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
index 68c5b41630e..c358e3338b1 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
@@ -516,7 +516,7 @@ struct harmonic_oscillator_ctl_test
   Eigen::VectorXd atol_f;
   Eigen::VectorXd atol_b;
   double atol_q;
-  long int num_steps_check;
+  int num_steps_check;
   int inter_poly;
   int solv_f;
   int solv_b;

From ab4acbeef6f8d6ad99d293ecec06cacf3e354c6b Mon Sep 17 00:00:00 2001
From: wds15 <sdw.post@waebers.de>
Date: Sat, 1 May 2021 21:46:41 +0200
Subject: [PATCH 116/157] Update
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp

Co-authored-by: Steve Bronder <Stevo15025@gmail.com>
---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 0a6185c525e..2b0219e544f 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -521,14 +521,9 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     // Code from: https://stackoverflow.com/a/17340003 . Should probably do
     // something better
-    apply(
-        [&](auto&&... args) {
-          std::vector<int> unused_temp{
-              0, (check_finite(solver_->function_name_,
-                               "ode parameters and data", args),
-                  0)...};
-        },
-        local_args_tuple_);
+    stan::math::for_each([func_name = this->function_name_](auto&& arg) {
+      check_finite(func_name, "ode parameters and data", arg);
+    }, local_args_tuple);
 
     check_nonzero_size(solver_->function_name_, "times", ts);
     check_nonzero_size(solver_->function_name_, "initial state", y0);

From ea7e8082c2c794134fb3d92926e8741aff9f74df Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Sat, 1 May 2021 19:47:18 +0000
Subject: [PATCH 117/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 2b0219e544f..6087c21b13f 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -521,9 +521,11 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     // Code from: https://stackoverflow.com/a/17340003 . Should probably do
     // something better
-    stan::math::for_each([func_name = this->function_name_](auto&& arg) {
-      check_finite(func_name, "ode parameters and data", arg);
-    }, local_args_tuple);
+    stan::math::for_each(
+        [func_name = this->function_name_](auto&& arg) {
+          check_finite(func_name, "ode parameters and data", arg);
+        },
+        local_args_tuple);
 
     check_nonzero_size(solver_->function_name_, "times", ts);
     check_nonzero_size(solver_->function_name_, "initial state", y0);

From 9ec431aa46ec92a5e363d2e8b3a20d986d9efe87 Mon Sep 17 00:00:00 2001
From: wds15 <sdw.post@waebers.de>
Date: Sat, 1 May 2021 21:55:16 +0200
Subject: [PATCH 118/157] Update
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp

Co-authored-by: Steve Bronder <Stevo15025@gmail.com>
---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 6087c21b13f..566577889ad 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -275,7 +275,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     nested_rev_autodiff nested;
 
-    const Eigen::Matrix<var, Eigen::Dynamic, 1> y_var(x);
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y_var(x);
     Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var
         = rhs(t, y_var, value_of_args_tuple_);
 

From 7d914d126baaa1afe0b5a3bfe25f6467aad1f264 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sat, 1 May 2021 22:53:06 +0200
Subject: [PATCH 119/157] review comments

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 184 ++++++++----------
 stan/math/rev/functor/ode_adjoint.hpp         |   2 +-
 2 files changed, 83 insertions(+), 103 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 566577889ad..dd17df03cf9 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -1,5 +1,5 @@
-#ifndef STAN_MATH_REV_FUNCTOR_INTEGRATE_ODE_CVODES_ADJOINT_HPP
-#define STAN_MATH_REV_FUNCTOR_INTEGRATE_ODE_CVODES_ADJOINT_HPP
+#ifndef STAN_MATH_REV_FUNCTOR_CVODES_INTEGRATOR_ADJOINT_HPP
+#define STAN_MATH_REV_FUNCTOR_CVODES_INTEGRATOR_ADJOINT_HPP
 
 #include <stan/math/rev/meta.hpp>
 #include <stan/math/rev/core/save_varis.hpp>
@@ -63,7 +63,6 @@ class cvodes_integrator_adjoint_vari : public vari {
       value_of_args_tuple_;
 
   size_t N_;
-  bool forward_returned_;
   bool backward_is_initialized_;
 
   size_t num_t0_vars_;
@@ -88,14 +87,14 @@ class cvodes_integrator_adjoint_vari : public vari {
   struct cvodes_solver;
   cvodes_solver* solver_;
 
-  static constexpr bool is_var_ts{is_var<T_ts>::value};
-  static constexpr bool is_var_t0{is_var<T_t0>::value};
-  static constexpr bool is_var_y0{is_var<T_y0_t0>::value};
-  static constexpr bool is_any_var_args{
+  static constexpr bool is_var_ts_{is_var<T_ts>::value};
+  static constexpr bool is_var_t0_{is_var<T_t0>::value};
+  static constexpr bool is_var_y0_{is_var<T_y0_t0>::value};
+  static constexpr bool is_any_var_args_{
       disjunction<is_var<scalar_type_t<T_Args>>...>::value};
-  static constexpr bool is_var_return{is_var<T_Return>::value};
-  static constexpr bool is_var_only_ts{
-      is_var_ts && !(is_var_t0 || is_var_y0 || is_any_var_args)};
+  static constexpr bool is_var_return_{is_var<T_Return>::value};
+  static constexpr bool is_var_only_ts_{
+      is_var_ts_ && !(is_var_t0_ || is_var_y0_ || is_any_var_args_)};
 
   /**
    * Call the ODE RHS with given tuple.
@@ -107,15 +106,20 @@ class cvodes_integrator_adjoint_vari : public vari {
                  args_tuple);
   }
 
+  /**
+   * Utility to cast user memory pointer passed in from CVODES to actual typed object pointer.
+   */
+  constexpr static cvodes_integrator_adjoint_vari* cast_to_self(void* mem) {
+    return static_cast<cvodes_integrator_adjoint_vari*>(mem);
+  }
+
   /**
    * Implements the function of type CVRhsFn which is the user-defined
    * ODE RHS passed to CVODES.
    */
   constexpr static int cv_rhs(realtype t, N_Vector y, N_Vector ydot,
                               void* user_data) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
-    integrator->rhs(t, NV_DATA_S(y), NV_DATA_S(ydot));
+    cast_to_self(user_data)->rhs(t, NV_DATA_S(y), NV_DATA_S(ydot));
     return 0;
   }
 
@@ -123,11 +127,9 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Implements the function of type CVRhsFnB which is the
    * RHS of the backward ODE system.
    */
-  constexpr static int cv_rhs_adj_sens(realtype t, N_Vector y, N_Vector yB,
+  constexpr static int cv_rhs_adj(realtype t, N_Vector y, N_Vector yB,
                                        N_Vector yBdot, void* user_data) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
-    integrator->rhs_adj_sens(t, y, yB, yBdot);
+    cast_to_self(user_data)->rhs_adj(t, y, yB, yBdot);
     return 0;
   }
 
@@ -137,9 +139,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    */
   constexpr static int cv_quad_rhs_adj(realtype t, N_Vector y, N_Vector yB,
                                        N_Vector qBdot, void* user_data) {
-    cvodes_integrator_adjoint_vari* integrator
-        = static_cast<cvodes_integrator_adjoint_vari*>(user_data);
-    integrator->quad_rhs_adj(t, y, yB, qBdot);
+    cast_to_self(user_data)->quad_rhs_adj(t, y, yB, qBdot);
     return 0;
   }
 
@@ -149,27 +149,23 @@ class cvodes_integrator_adjoint_vari : public vari {
    * ode_rhs wrt to the states y. The jacobian is stored in column
    * major format.
    */
-  constexpr static int cv_jacobian_states(realtype t, N_Vector y, N_Vector fy,
-                                          SUNMatrix J, void* user_data,
-                                          N_Vector tmp1, N_Vector tmp2,
-                                          N_Vector tmp3) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
-    integrator->jacobian_states(t, y, J);
+  constexpr static int cv_jacobian_rhs_states(realtype t, N_Vector y, N_Vector fy,
+                                              SUNMatrix J, void* user_data,
+                                              N_Vector tmp1, N_Vector tmp2,
+                                              N_Vector tmp3) {
+    cast_to_self(user_data)->jacobian_rhs_states(t, y, J);
     return 0;
   }
 
   /**
    * Implements the CVLsJacFnB function for evaluating the jacobian of
-   * the adjoint problem.
+   * the adjoint problem wrt to the backward states.
    */
-  constexpr static int cv_jacobian_adj(realtype t, N_Vector y, N_Vector yB,
-                                       N_Vector fyB, SUNMatrix J,
-                                       void* user_data, N_Vector tmp1,
-                                       N_Vector tmp2, N_Vector tmp3) {
-    const cvodes_integrator_adjoint_vari* integrator
-        = static_cast<const cvodes_integrator_adjoint_vari*>(user_data);
-    integrator->jacobian_adj(t, y, J);
+  constexpr static int cv_jacobian_rhs_adj_states(realtype t, N_Vector y, N_Vector yB,
+                                                  N_Vector fyB, SUNMatrix J,
+                                                  void* user_data, N_Vector tmp1,
+                                                  N_Vector tmp2, N_Vector tmp3) {
+    cast_to_self(user_data)->jacobian_rhs_adj_states(t, y, J);
     return 0;
   }
 
@@ -201,8 +197,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] yB state of the adjoint ODE system
    * @param[out] yBdot evaluation of adjoint ODE RHS
    */
-  inline void rhs_adj_sens(double t, N_Vector y, N_Vector yB,
-                           N_Vector yBdot) const {
+  inline void rhs_adj(double t, N_Vector y, N_Vector yB,
+                      N_Vector yBdot) const {
     Eigen::Map<const Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
     Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(yBdot), N_);
@@ -269,7 +265,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Calculates the jacobian of the ODE RHS wrt to its states y at the
    * given time-point t and state y.
    */
-  inline void jacobian_states(double t, N_Vector y, SUNMatrix J) const {
+  inline void jacobian_rhs_states(double t, N_Vector y, SUNMatrix J) const {
     Eigen::Map<Eigen::MatrixXd> Jfy(SM_DATA_D(J), N_, N_);
     Eigen::Map<const Eigen::VectorXd> x(NV_DATA_S(y), N_);
 
@@ -293,18 +289,18 @@ class cvodes_integrator_adjoint_vari : public vari {
   }
 
   /*
-   * Calculate the Jacobian of the RHS of the adjoint ODE (see rhs_adj_sens
+   * Calculate the Jacobian of the RHS of the adjoint ODE (see rhs_adj
    * below for citation for how this is done)
    *
    * @param[in] y State of system
    * @param[in] t Time
    * @param[out] J CVode structure where output is to be stored
    */
-  inline void jacobian_adj(double t, N_Vector y, SUNMatrix J) const {
+  inline void jacobian_rhs_adj_states(double t, N_Vector y, SUNMatrix J) const {
     Eigen::Map<Eigen::MatrixXd> J_adj_y(SM_DATA_D(J), N_, N_);
 
     // J_adj_y = -1 * transpose(J_y)
-    jacobian_states(t, y, J);
+    jacobian_rhs_states(t, y, J);
 
     J_adj_y.transposeInPlace();
     J_adj_y.array() *= -1.0;
@@ -339,6 +335,7 @@ class cvodes_integrator_adjoint_vari : public vari {
   struct cvodes_solver : public chainable_alloc {
     const size_t N_;
     std::vector<Eigen::VectorXd> y_;
+    std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return_;
     const std::string function_name_str_;
     const char* function_name_;
     void* cvodes_mem_;
@@ -364,24 +361,21 @@ class cvodes_integrator_adjoint_vari : public vari {
         : chainable_alloc(),
           N_(N),
           y_(ts_size),
+          y_return_(ts_size),
           function_name_str_(function_name),
-          function_name_(function_name_str_.c_str()) {
-      nv_state_forward_ = N_VMake_Serial(N, state_forward.data());
-      nv_state_backward_ = N_VMake_Serial(N, state_backward.data());
-      nv_quad_ = N_VMake_Serial(num_args_vars, quad.data());
-      nv_absolute_tolerance_forward_
-          = N_VMake_Serial(N, absolute_tolerance_forward.data());
-      nv_absolute_tolerance_backward_
-          = N_VMake_Serial(N, absolute_tolerance_backward.data());
-
-      A_forward_ = SUNDenseMatrix(N, N);
-      A_backward_ = SUNDenseMatrix(N, N);
-      if (N > 0) {
-        LS_forward_ = SUNDenseLinearSolver(nv_state_forward_, A_forward_);
-        LS_backward_ = SUNDenseLinearSolver(nv_state_backward_, A_backward_);
-      }
-      cvodes_mem_ = CVodeCreate(solver_forward);
-
+          function_name_(function_name_str_.c_str()),
+          nv_state_forward_(N_VMake_Serial(N, state_forward.data())),
+          nv_state_backward_(N_VMake_Serial(N, state_backward.data())),
+          nv_quad_(N_VMake_Serial(num_args_vars, quad.data())),
+          nv_absolute_tolerance_forward_(N_VMake_Serial(N, absolute_tolerance_forward.data())),
+          nv_absolute_tolerance_backward_(N_VMake_Serial(N, absolute_tolerance_backward.data())),
+      
+          A_forward_(SUNDenseMatrix(N, N)),
+          A_backward_(SUNDenseMatrix(N, N)),
+          LS_forward_(N == 0 ? nullptr : SUNDenseLinearSolver(nv_state_forward_, A_forward_)),
+          LS_backward_(N == 0 ? nullptr : SUNDenseLinearSolver(nv_state_backward_, A_backward_)),
+          cvodes_mem_(CVodeCreate(solver_forward))
+    {
       if (cvodes_mem_ == nullptr) {
         throw std::runtime_error("CVodeCreate failed to allocate memory");
       }
@@ -400,9 +394,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       N_VDestroy_Serial(nv_absolute_tolerance_forward_);
       N_VDestroy_Serial(nv_absolute_tolerance_backward_);
 
-      if (cvodes_mem_) {
-        CVodeFree(&cvodes_mem_);
-      }
+      CVodeFree(&cvodes_mem_);
     }
   };
 
@@ -480,7 +472,6 @@ class cvodes_integrator_adjoint_vari : public vari {
         value_of_args_tuple_(to_arena(value_of(args))...),
 
         N_(y0.size()),
-        forward_returned_(false),
         backward_is_initialized_(false),
 
         num_t0_vars_(count_vars(t0)),
@@ -494,7 +485,7 @@ class cvodes_integrator_adjoint_vari : public vari {
         quad_(num_args_vars_),
 
         non_chaining_varis_(
-            is_var_return
+            is_var_return_
                 ? ChainableStack::instance_->memalloc_.alloc_array<vari*>(
                       ts_.size() * N_)
                 : nullptr),
@@ -519,13 +510,11 @@ class cvodes_integrator_adjoint_vari : public vari {
     check_finite(solver_->function_name_, "initial time", t0);
     check_finite(solver_->function_name_, "times", ts);
 
-    // Code from: https://stackoverflow.com/a/17340003 . Should probably do
-    // something better
     stan::math::for_each(
-        [func_name = this->function_name_](auto&& arg) {
+        [func_name = solver_->function_name_](auto&& arg) {
           check_finite(func_name, "ode parameters and data", arg);
         },
-        local_args_tuple);
+        local_args_tuple_);
 
     check_nonzero_size(solver_->function_name_, "times", ts);
     check_nonzero_size(solver_->function_name_, "initial state", y0);
@@ -587,37 +576,28 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     check_flag_sundials(
         CVodeSetJacFn(solver_->cvodes_mem_,
-                      &cvodes_integrator_adjoint_vari::cv_jacobian_states),
+                      &cvodes_integrator_adjoint_vari::cv_jacobian_rhs_states),
         "CVodeSetJacFn");
 
     // initialize backward sensitivity system of CVODES as needed
-    if (is_var_return) {
+    if (is_var_return_) {
       check_flag_sundials(
           CVodeAdjInit(solver_->cvodes_mem_, num_steps_between_checkpoints_,
                        interpolation_polynomial_),
           "CVodeAdjInit");
     }
-  }
 
-  /**
-   * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
-   * times, { t1, t2, t3, ... } using the stiff backward differentiation formula
-   * (BDF) solver in CVODES.
-   *
-   * @return std::vector of Eigen::Matrix of the states of the ODE, one for each
-   *   solution time (excluding the initial state)
-   */
-  std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> operator()() {
-    const double t0_dbl = value_of(t0_);
+    /**
+     * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
+     * times, { t1, t2, t3, ... } using the requested forward solver of CVODES.
+     */
     const auto ts_dbl = value_of(ts_);
-    std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return(
-        ts_.size());
-
-    double t_init = t0_dbl;
+    
+    double t_init = value_of(t0_);
     for (size_t n = 0; n < ts_dbl.size(); ++n) {
       double t_final = ts_dbl[n];
       if (t_final != t_init) {
-        if (is_var_return) {
+        if (is_var_return_) {
           int ncheck;
 
           int error_code
@@ -646,31 +626,31 @@ class cvodes_integrator_adjoint_vari : public vari {
           }
         }
       }
-      store_state(n, state_forward_, y_return[n]);
+      store_state(n, state_forward_, solver_->y_return_[n]);
 
       t_init = t_final;
     }
+    
+  }
 
-    forward_returned_ = true;
-    return y_return;
+  /**
+   * Obtain solution of ODE.
+   *
+   * @return std::vector of Eigen::Matrix of the states of the ODE, one for each
+   *   solution time (excluding the initial state)
+   */
+  std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> solution() {
+    return solver_->y_return_;
   }
 
   virtual void chain() {
-    if (solver_->cvodes_mem_ == nullptr) {
-      return;
-    }
-
-    if (!forward_returned_) {
-      return;
-    }
-
-    if (!is_var_return) {
+    if (!is_var_return_) {
       return;
     }
 
     // for sensitivities wrt to ts we do not need to run the backward
     // integration
-    if (is_var_ts) {
+    if (is_var_ts_) {
       for (int i = 0; i < ts_.size(); ++i) {
         Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
         for (int j = 0; j < N_; ++j) {
@@ -681,7 +661,7 @@ class cvodes_integrator_adjoint_vari : public vari {
             rhs(value_of(ts_[i]), solver_->y_[i], value_of_args_tuple_));
       }
 
-      if (is_var_only_ts) {
+      if (is_var_only_ts_) {
         return;
       }
     }
@@ -716,7 +696,7 @@ class cvodes_integrator_adjoint_vari : public vari {
           // of the ode states
           check_flag_sundials(
               CVodeInitB(solver_->cvodes_mem_, index_backward_,
-                         &cvodes_integrator_adjoint_vari::cv_rhs_adj_sens,
+                         &cvodes_integrator_adjoint_vari::cv_rhs_adj,
                          t_init, solver_->nv_state_backward_),
               "CVodeInitB");
 
@@ -738,7 +718,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
           check_flag_sundials(
               CVodeSetJacFnB(solver_->cvodes_mem_, index_backward_,
-                             &cvodes_integrator_adjoint_vari::cv_jacobian_adj),
+                             &cvodes_integrator_adjoint_vari::cv_jacobian_rhs_adj_states),
               "CVodeSetJacFnB");
 
           // Allocate space for backwards quadrature needed when
@@ -802,7 +782,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       }
     }
 
-    if (is_var_t0) {
+    if (is_var_t0_) {
       t0_varis_[0]->adj_ += -state_backward_.dot(
           rhs(t_init, value_of(y0_), value_of_args_tuple_));
     }
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 82178eccce9..a32f64c741a 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -84,7 +84,7 @@ ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
       absolute_tolerance_quadrature, max_num_steps,
       num_steps_between_checkpoints, interpolation_polynomial, solver_forward,
       solver_backward, msgs, to_ref(args)...);
-  return (*integrator)();
+  return integrator->solution();
 }
 
 /**

From f0104fc0e6fde5bcd36327b8c29a1e51322a7c37 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Sat, 1 May 2021 20:54:02 +0000
Subject: [PATCH 120/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 56 ++++++++++---------
 1 file changed, 31 insertions(+), 25 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index dd17df03cf9..c355708e7bd 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -107,7 +107,8 @@ class cvodes_integrator_adjoint_vari : public vari {
   }
 
   /**
-   * Utility to cast user memory pointer passed in from CVODES to actual typed object pointer.
+   * Utility to cast user memory pointer passed in from CVODES to actual typed
+   * object pointer.
    */
   constexpr static cvodes_integrator_adjoint_vari* cast_to_self(void* mem) {
     return static_cast<cvodes_integrator_adjoint_vari*>(mem);
@@ -128,7 +129,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * RHS of the backward ODE system.
    */
   constexpr static int cv_rhs_adj(realtype t, N_Vector y, N_Vector yB,
-                                       N_Vector yBdot, void* user_data) {
+                                  N_Vector yBdot, void* user_data) {
     cast_to_self(user_data)->rhs_adj(t, y, yB, yBdot);
     return 0;
   }
@@ -149,10 +150,10 @@ class cvodes_integrator_adjoint_vari : public vari {
    * ode_rhs wrt to the states y. The jacobian is stored in column
    * major format.
    */
-  constexpr static int cv_jacobian_rhs_states(realtype t, N_Vector y, N_Vector fy,
-                                              SUNMatrix J, void* user_data,
-                                              N_Vector tmp1, N_Vector tmp2,
-                                              N_Vector tmp3) {
+  constexpr static int cv_jacobian_rhs_states(realtype t, N_Vector y,
+                                              N_Vector fy, SUNMatrix J,
+                                              void* user_data, N_Vector tmp1,
+                                              N_Vector tmp2, N_Vector tmp3) {
     cast_to_self(user_data)->jacobian_rhs_states(t, y, J);
     return 0;
   }
@@ -161,10 +162,11 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Implements the CVLsJacFnB function for evaluating the jacobian of
    * the adjoint problem wrt to the backward states.
    */
-  constexpr static int cv_jacobian_rhs_adj_states(realtype t, N_Vector y, N_Vector yB,
-                                                  N_Vector fyB, SUNMatrix J,
-                                                  void* user_data, N_Vector tmp1,
-                                                  N_Vector tmp2, N_Vector tmp3) {
+  constexpr static int cv_jacobian_rhs_adj_states(realtype t, N_Vector y,
+                                                  N_Vector yB, N_Vector fyB,
+                                                  SUNMatrix J, void* user_data,
+                                                  N_Vector tmp1, N_Vector tmp2,
+                                                  N_Vector tmp3) {
     cast_to_self(user_data)->jacobian_rhs_adj_states(t, y, J);
     return 0;
   }
@@ -197,8 +199,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] yB state of the adjoint ODE system
    * @param[out] yBdot evaluation of adjoint ODE RHS
    */
-  inline void rhs_adj(double t, N_Vector y, N_Vector yB,
-                      N_Vector yBdot) const {
+  inline void rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
     Eigen::Map<const Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
     Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
     Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(yBdot), N_);
@@ -367,15 +368,20 @@ class cvodes_integrator_adjoint_vari : public vari {
           nv_state_forward_(N_VMake_Serial(N, state_forward.data())),
           nv_state_backward_(N_VMake_Serial(N, state_backward.data())),
           nv_quad_(N_VMake_Serial(num_args_vars, quad.data())),
-          nv_absolute_tolerance_forward_(N_VMake_Serial(N, absolute_tolerance_forward.data())),
-          nv_absolute_tolerance_backward_(N_VMake_Serial(N, absolute_tolerance_backward.data())),
-      
+          nv_absolute_tolerance_forward_(
+              N_VMake_Serial(N, absolute_tolerance_forward.data())),
+          nv_absolute_tolerance_backward_(
+              N_VMake_Serial(N, absolute_tolerance_backward.data())),
+
           A_forward_(SUNDenseMatrix(N, N)),
           A_backward_(SUNDenseMatrix(N, N)),
-          LS_forward_(N == 0 ? nullptr : SUNDenseLinearSolver(nv_state_forward_, A_forward_)),
-          LS_backward_(N == 0 ? nullptr : SUNDenseLinearSolver(nv_state_backward_, A_backward_)),
-          cvodes_mem_(CVodeCreate(solver_forward))
-    {
+          LS_forward_(
+              N == 0 ? nullptr
+                     : SUNDenseLinearSolver(nv_state_forward_, A_forward_)),
+          LS_backward_(
+              N == 0 ? nullptr
+                     : SUNDenseLinearSolver(nv_state_backward_, A_backward_)),
+          cvodes_mem_(CVodeCreate(solver_forward)) {
       if (cvodes_mem_ == nullptr) {
         throw std::runtime_error("CVodeCreate failed to allocate memory");
       }
@@ -592,7 +598,7 @@ class cvodes_integrator_adjoint_vari : public vari {
      * times, { t1, t2, t3, ... } using the requested forward solver of CVODES.
      */
     const auto ts_dbl = value_of(ts_);
-    
+
     double t_init = value_of(t0_);
     for (size_t n = 0; n < ts_dbl.size(); ++n) {
       double t_final = ts_dbl[n];
@@ -630,7 +636,6 @@ class cvodes_integrator_adjoint_vari : public vari {
 
       t_init = t_final;
     }
-    
   }
 
   /**
@@ -696,8 +701,8 @@ class cvodes_integrator_adjoint_vari : public vari {
           // of the ode states
           check_flag_sundials(
               CVodeInitB(solver_->cvodes_mem_, index_backward_,
-                         &cvodes_integrator_adjoint_vari::cv_rhs_adj,
-                         t_init, solver_->nv_state_backward_),
+                         &cvodes_integrator_adjoint_vari::cv_rhs_adj, t_init,
+                         solver_->nv_state_backward_),
               "CVodeInitB");
 
           check_flag_sundials(
@@ -717,8 +722,9 @@ class cvodes_integrator_adjoint_vari : public vari {
                               "CVodeSetLinearSolverB");
 
           check_flag_sundials(
-              CVodeSetJacFnB(solver_->cvodes_mem_, index_backward_,
-                             &cvodes_integrator_adjoint_vari::cv_jacobian_rhs_adj_states),
+              CVodeSetJacFnB(
+                  solver_->cvodes_mem_, index_backward_,
+                  &cvodes_integrator_adjoint_vari::cv_jacobian_rhs_adj_states),
               "CVodeSetJacFnB");
 
           // Allocate space for backwards quadrature needed when

From 3ab73f1c9dca504ce3a9e5557d2558ee8068604c Mon Sep 17 00:00:00 2001
From: stevebronder <stevo15025@gmail.com>
Date: Mon, 3 May 2021 17:54:12 -0400
Subject: [PATCH 121/157] update adjoint ode to not have to make seperate vari

---
 stan/math/prim/fun/get.hpp                    | 18 ++++--
 .../rev/functor/cvodes_integrator_adjoint.hpp | 62 +++++++------------
 2 files changed, 36 insertions(+), 44 deletions(-)

diff --git a/stan/math/prim/fun/get.hpp b/stan/math/prim/fun/get.hpp
index 8a466561278..7ab3d4afd53 100644
--- a/stan/math/prim/fun/get.hpp
+++ b/stan/math/prim/fun/get.hpp
@@ -20,7 +20,7 @@ namespace stan {
  * @return input scalar
  */
 template <typename T, typename = require_stan_scalar_t<T>>
-inline T get(const T& x, size_t n) {
+inline T& get(T& x, size_t n) {
   return x;
 }
 
@@ -31,8 +31,13 @@ inline T get(const T& x, size_t n) {
  * @param n index of the element to return
  * @return n-th element of the input vector
  */
-template <typename T>
-inline T get(const std::vector<T>& x, size_t n) {
+template <typename T, typename Alloc>
+inline T& get(std::vector<T, Alloc>& x, size_t n) {
+  return x[n];
+}
+
+template <typename T, typename Alloc>
+inline const T& get(const std::vector<T, Alloc>& x, size_t n) {
   return x[n];
 }
 
@@ -44,7 +49,12 @@ inline T get(const std::vector<T>& x, size_t n) {
  * @return n-th element of the \c Eigen \c Matrix or expression
  */
 template <typename T, typename = require_eigen_t<T>>
-inline scalar_type_t<T> get(const T& m, size_t n) {
+inline scalar_type_t<T>& get(T& m, size_t n) {
+  return m(static_cast<int>(n));
+}
+
+template <typename T, typename = require_eigen_t<T>>
+inline const scalar_type_t<T>& get(const T& m, size_t n) {
   return m(static_cast<int>(n));
 }
 
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index c355708e7bd..46cf7551304 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -6,6 +6,7 @@
 #include <stan/math/rev/functor/cvodes_utils.hpp>
 #include <stan/math/rev/functor/ode_store_sensitivities.hpp>
 #include <stan/math/prim/err.hpp>
+#include <stan/math/prim/fun/get.hpp>
 #include <stan/math/prim/fun/value_of.hpp>
 #include <stan/math/prim/functor/for_each.hpp>
 #include <cvodes/cvodes.h>
@@ -65,9 +66,6 @@ class cvodes_integrator_adjoint_vari : public vari {
   size_t N_;
   bool backward_is_initialized_;
 
-  size_t num_t0_vars_;
-  size_t num_ts_vars_;
-  size_t num_y0_vars_;
   size_t num_args_vars_;
   size_t num_vars_;
 
@@ -77,9 +75,6 @@ class cvodes_integrator_adjoint_vari : public vari {
 
   vari** non_chaining_varis_;
 
-  vari** t0_varis_;
-  vari** ts_varis_;
-  vari** y0_varis_;
   vari** args_varis_;
 
   int index_backward_;
@@ -100,8 +95,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Call the ODE RHS with given tuple.
    */
   template <typename yT, typename... ArgsT>
-  constexpr auto rhs(double t, const yT& y,
-                     const std::tuple<ArgsT...>& args_tuple) const {
+  constexpr auto rhs(double t, const yT& y, const std::tuple<ArgsT...>& args_tuple) const {
     return apply([&](auto&&... args) { return f_(t, y, msgs_, args...); },
                  args_tuple);
   }
@@ -317,8 +311,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     solver_->y_[n] = state;
     state_return.resize(N_);
     for (size_t i = 0; i < N_; i++) {
-      non_chaining_varis_[N_ * n + i] = new vari(state.coeff(i), false);
-      state_return.coeffRef(i) = var(non_chaining_varis_[N_ * n + i]);
+      state_return.coeffRef(i) = var(new vari(state.coeff(i), false));
     }
   }
 
@@ -476,40 +469,24 @@ class cvodes_integrator_adjoint_vari : public vari {
         msgs_(msgs),
         local_args_tuple_(to_arena(deep_copy_vars(args))...),
         value_of_args_tuple_(to_arena(value_of(args))...),
-
         N_(y0.size()),
         backward_is_initialized_(false),
-
-        num_t0_vars_(count_vars(t0)),
-        num_ts_vars_(count_vars(ts)),
-        num_y0_vars_(count_vars(y0)),
         num_args_vars_(count_vars(args...)),
-        num_vars_(num_t0_vars_ + num_ts_vars_ + num_y0_vars_ + num_args_vars_),
-
+        num_vars_(count_vars(t0) + count_vars(ts) + count_vars(y0) + num_args_vars_),
         state_forward_(value_of(y0)),
         state_backward_(N_),
         quad_(num_args_vars_),
-
         non_chaining_varis_(
             is_var_return_
                 ? ChainableStack::instance_->memalloc_.alloc_array<vari*>(
                       ts_.size() * N_)
                 : nullptr),
-        t0_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
-            num_t0_vars_)),
-        ts_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
-            num_ts_vars_)),
-        y0_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
-            num_y0_vars_)),
         args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             num_args_vars_)),
         solver_(new cvodes_solver(
             function_name, N_, num_args_vars_, ts_.size(), solver_forward_,
             state_forward_, state_backward_, quad_, absolute_tolerance_forward_,
             absolute_tolerance_backward_)) {
-    save_varis(t0_varis_, t0);
-    save_varis(ts_varis_, ts);
-    save_varis(y0_varis_, y0);
     save_varis(args_varis_, args...);
 
     check_finite(solver_->function_name_, "initial state", y0);
@@ -644,7 +621,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @return std::vector of Eigen::Matrix of the states of the ODE, one for each
    *   solution time (excluding the initial state)
    */
-  std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> solution() {
+  std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>>& solution() noexcept {
     return solver_->y_return_;
   }
 
@@ -656,14 +633,15 @@ class cvodes_integrator_adjoint_vari : public vari {
     // for sensitivities wrt to ts we do not need to run the backward
     // integration
     if (is_var_ts_) {
+      Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
       for (int i = 0; i < ts_.size(); ++i) {
-        Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
         for (int j = 0; j < N_; ++j) {
-          step_sens.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
+          step_sens.coeffRef(j) += forward_as<var>(solver_->y_return_[i][j]).adj();
         }
 
-        ts_varis_[i]->adj_ += step_sens.dot(
+        forward_as<var>(stan::get(ts_, i))->adj_ += step_sens.dot(
             rhs(value_of(ts_[i]), solver_->y_[i], value_of_args_tuple_));
+        step_sens.setZero();
       }
 
       if (is_var_only_ts_) {
@@ -681,7 +659,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       // Take in the adjoints from all the output variables at this point
       // in time
       for (int j = 0; j < N_; ++j) {
-        state_backward_.coeffRef(j) += non_chaining_varis_[i * N_ + j]->adj_;
+        state_backward_.coeffRef(j) += forward_as<var>(solver_->y_return_[i][j]).adj();
       }
 
       double t_final = value_of((i > 0) ? ts_[i - 1] : t0_);
@@ -729,7 +707,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
           // Allocate space for backwards quadrature needed when
           // parameters vary.
-          if (num_args_vars_ > 0) {
+          if (is_any_var_args_) {
             check_flag_sundials(
                 CVodeQuadInitB(solver_->cvodes_mem_, index_backward_,
                                &cvodes_integrator_adjoint_vari::cv_quad_rhs_adj,
@@ -755,7 +733,7 @@ class cvodes_integrator_adjoint_vari : public vari {
                            solver_->nv_state_backward_),
               "CVodeReInitB");
 
-          if (num_args_vars_ > 0) {
+          if (is_any_var_args_) {
             check_flag_sundials(
                 CVodeQuadReInitB(solver_->cvodes_mem_, index_backward_,
                                  solver_->nv_quad_),
@@ -779,7 +757,7 @@ class cvodes_integrator_adjoint_vari : public vari {
                                       &t_init, solver_->nv_state_backward_),
                             "CVodeGetB");
 
-        if (num_args_vars_ > 0) {
+        if (is_any_var_args_) {
           check_flag_sundials(
               CVodeGetQuadB(solver_->cvodes_mem_, index_backward_, &t_init,
                             solver_->nv_quad_),
@@ -789,20 +767,24 @@ class cvodes_integrator_adjoint_vari : public vari {
     }
 
     if (is_var_t0_) {
-      t0_varis_[0]->adj_ += -state_backward_.dot(
+      forward_as<var>(stan::get(t0_, 0))->adj_ += -state_backward_.dot(
           rhs(t_init, value_of(y0_), value_of_args_tuple_));
     }
 
     // After integrating all the way back to t0, we finally have the
     // the adjoints we wanted
     // These are the dlog_density / d(initial_conditions[s]) adjoints
-    for (size_t s = 0; s < num_y0_vars_; ++s) {
-      y0_varis_[s]->adj_ += state_backward_.coeff(s);
+    if (is_var_y0_) {
+      for (size_t i = 0; i < N_; ++i) {
+        forward_as<var>(stan::get(y0_, i))->adj_ += state_backward_.coeff(i);
+      }
     }
 
     // These are the dlog_density / d(parameters[s]) adjoints
-    for (size_t s = 0; s < num_args_vars_; ++s) {
-      args_varis_[s]->adj_ += quad_.coeff(s);
+    if (is_any_var_args_) {
+      for (size_t s = 0; s < num_args_vars_; ++s) {
+        args_varis_[s]->adj_ += quad_.coeff(s);
+      }
     }
   }
 };  // cvodes integrator adjoint vari

From ba82ee4894fcd3ce5f1b07bbe654f888cb44a2ee Mon Sep 17 00:00:00 2001
From: stevebronder <stevo15025@gmail.com>
Date: Mon, 3 May 2021 17:56:55 -0400
Subject: [PATCH 122/157] remove num_vars

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 46cf7551304..e92fdabff47 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -67,7 +67,6 @@ class cvodes_integrator_adjoint_vari : public vari {
   bool backward_is_initialized_;
 
   size_t num_args_vars_;
-  size_t num_vars_;
 
   arena_t<Eigen::VectorXd> state_forward_;
   arena_t<Eigen::VectorXd> state_backward_;
@@ -472,7 +471,6 @@ class cvodes_integrator_adjoint_vari : public vari {
         N_(y0.size()),
         backward_is_initialized_(false),
         num_args_vars_(count_vars(args...)),
-        num_vars_(count_vars(t0) + count_vars(ts) + count_vars(y0) + num_args_vars_),
         state_forward_(value_of(y0)),
         state_backward_(N_),
         quad_(num_args_vars_),

From 784be28f828170c46c1f1a2b6e272fd07c0e6a6a Mon Sep 17 00:00:00 2001
From: stevebronder <stevo15025@gmail.com>
Date: Mon, 3 May 2021 18:04:08 -0400
Subject: [PATCH 123/157] remove num_vars

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index e92fdabff47..441f73977f7 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -474,11 +474,6 @@ class cvodes_integrator_adjoint_vari : public vari {
         state_forward_(value_of(y0)),
         state_backward_(N_),
         quad_(num_args_vars_),
-        non_chaining_varis_(
-            is_var_return_
-                ? ChainableStack::instance_->memalloc_.alloc_array<vari*>(
-                      ts_.size() * N_)
-                : nullptr),
         args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             num_args_vars_)),
         solver_(new cvodes_solver(

From 3f91c24d7d8d6aa4db8d5a3c9552f32d1a7740b1 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 4 May 2021 21:59:48 +0200
Subject: [PATCH 124/157] more testing

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 56 +++++++++--------
 .../math/rev/functor/test_fixture_ode_sho.hpp | 62 +++++++++++++++++++
 2 files changed, 91 insertions(+), 27 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index c355708e7bd..cf84ed5bcae 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -503,59 +503,61 @@ class cvodes_integrator_adjoint_vari : public vari {
             num_y0_vars_)),
         args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
             num_args_vars_)),
-        solver_(new cvodes_solver(
-            function_name, N_, num_args_vars_, ts_.size(), solver_forward_,
-            state_forward_, state_backward_, quad_, absolute_tolerance_forward_,
-            absolute_tolerance_backward_)) {
+        solver_(nullptr) {
     save_varis(t0_varis_, t0);
     save_varis(ts_varis_, ts);
     save_varis(y0_varis_, y0);
     save_varis(args_varis_, args...);
 
-    check_finite(solver_->function_name_, "initial state", y0);
-    check_finite(solver_->function_name_, "initial time", t0);
-    check_finite(solver_->function_name_, "times", ts);
+    check_finite(function_name, "initial state", y0);
+    check_finite(function_name, "initial time", t0);
+    check_finite(function_name, "times", ts);
 
     stan::math::for_each(
-        [func_name = solver_->function_name_](auto&& arg) {
+        [func_name = function_name](auto&& arg) {
           check_finite(func_name, "ode parameters and data", arg);
         },
         local_args_tuple_);
 
-    check_nonzero_size(solver_->function_name_, "times", ts);
-    check_nonzero_size(solver_->function_name_, "initial state", y0);
-    check_sorted(solver_->function_name_, "times", ts);
-    check_less(solver_->function_name_, "initial time", t0, ts[0]);
-    check_positive_finite(solver_->function_name_, "relative_tolerance_forward",
+    check_nonzero_size(function_name, "times", ts);
+    check_nonzero_size(function_name, "initial state", y0);
+    check_sorted(function_name, "times", ts);
+    check_less(function_name, "initial time", t0, ts[0]);
+    check_positive_finite(function_name, "relative_tolerance_forward",
                           relative_tolerance_forward_);
-    check_positive_finite(solver_->function_name_, "absolute_tolerance_forward",
+    check_positive_finite(function_name, "absolute_tolerance_forward",
                           absolute_tolerance_forward_);
-    check_size_match(solver_->function_name_, "absolute_tolerance_forward",
+    check_size_match(function_name, "absolute_tolerance_forward",
                      absolute_tolerance_forward_.size(), "states", N_);
-    check_positive_finite(solver_->function_name_,
+    check_positive_finite(function_name,
                           "relative_tolerance_backward",
                           relative_tolerance_backward_);
-    check_positive_finite(solver_->function_name_,
+    check_positive_finite(function_name,
                           "absolute_tolerance_backward",
                           absolute_tolerance_backward_);
-    check_size_match(solver_->function_name_, "absolute_tolerance_backward",
+    check_size_match(function_name, "absolute_tolerance_backward",
                      absolute_tolerance_backward_.size(), "states", N_);
-    check_positive_finite(solver_->function_name_,
+    check_positive_finite(function_name,
                           "relative_tolerance_quadrature",
                           relative_tolerance_quadrature_);
-    check_positive_finite(solver_->function_name_,
+    check_positive_finite(function_name,
                           "absolute_tolerance_quadrature",
                           absolute_tolerance_quadrature_);
-    check_positive(solver_->function_name_, "max_num_steps", max_num_steps_);
-    check_positive(solver_->function_name_, "num_steps_between_checkpoints",
+    check_positive(function_name, "max_num_steps", max_num_steps_);
+    check_positive(function_name, "num_steps_between_checkpoints",
                    num_steps_between_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
-    check_range(solver_->function_name_, "interpolation_polynomial", 2,
-                interpolation_polynomial_);
+    check_bounded(function_name, "interpolation_polynomial", 
+                  interpolation_polynomial_, 1, 2);
     // 1=Adams=CV_ADAMS, 2=BDF=CV_BDF
-    check_range(solver_->function_name_, "solver_forward", 2, solver_forward_);
-    check_range(solver_->function_name_, "solver_backward", 2,
-                solver_backward_);
+    check_bounded(function_name, "solver_forward", solver_forward_, 1, 2);
+    check_bounded(function_name, "solver_backward",
+                  solver_backward_, 1, 2);
+
+    solver_ = new cvodes_solver(
+        function_name, N_, num_args_vars_, ts_.size(), solver_forward_,
+        state_forward_, state_backward_, quad_, absolute_tolerance_forward_,
+        absolute_tolerance_backward_);
 
     check_flag_sundials(
         CVodeInit(solver_->cvodes_mem_, &cvodes_integrator_adjoint_vari::cv_rhs,
diff --git a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
index c358e3338b1..50a5130b3bf 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
@@ -601,6 +601,68 @@ struct harmonic_oscillator_ctl_test
                      "absolute_tolerance_forward");
     this->atol_f(0) = atol_f_;
 
+    this->atol_f.resize(1);
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::invalid_argument,
+                     "absolute_tolerance_forward");
+    this->atol_f.resize(2);
+    this->atol_f(1) = atol_f_;
+    
+    const double rtol_b_ = this->rtol_b;
+    this->rtol_b = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "relative_tolerance_backward");
+    this->rtol_b = rtol_b_;
+
+    const double atol_b_ = this->atol_b(0);
+    this->atol_b(0) = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "absolute_tolerance_backward");
+    this->atol_b(0) = atol_b_;
+
+    this->atol_b.resize(1);
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::invalid_argument,
+                     "absolute_tolerance_backward");
+    this->atol_b.resize(2);
+    this->atol_b(0) = atol_b_;
+    this->atol_b(1) = atol_b_;
+    
+    const double rtol_q_ = this->rtol_q;
+    this->rtol_q = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "relative_tolerance_quadrature");
+    this->rtol_q = rtol_q_;
+
+    const double atol_q_ = this->atol_q;
+    this->atol_q = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "absolute_tolerance_quadrature");
+    this->atol_q = atol_q_;
+
+    const int max_num_step_ = this->max_num_step;
+    this->max_num_step = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error, "max_num_steps");
+    this->max_num_step = max_num_step_;
+ 
+    const int num_steps_check_ = this->num_steps_check;
+    this->num_steps_check = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error, "num_steps_between_checkpoints");
+    this->num_steps_check = num_steps_check_;
+ 
+    const int inter_poly_ = this->inter_poly;
+    this->inter_poly = 0;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error, "interpolation_polynomial");
+    this->inter_poly = inter_poly_;
+ 
+    const int solv_f_ = this->solv_f;
+    this->solv_f = 0;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error, "solver_forward");
+    this->solv_f = solv_f_;
+ 
+    const int solv_b_ = this->solv_b;
+    this->solv_b = 0;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error, "solver_backward");
+    this->solv_b = solv_b_;
+
     const auto theta_ = this->theta;
     const auto x_r_ = this->x_r;
     const auto x_i_ = this->x_i;

From 19774a204aa6ae57de640bdf34e42dc354beafbc Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Tue, 4 May 2021 20:06:04 +0000
Subject: [PATCH 125/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 441f73977f7..23c5cd51d01 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -94,7 +94,8 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Call the ODE RHS with given tuple.
    */
   template <typename yT, typename... ArgsT>
-  constexpr auto rhs(double t, const yT& y, const std::tuple<ArgsT...>& args_tuple) const {
+  constexpr auto rhs(double t, const yT& y,
+                     const std::tuple<ArgsT...>& args_tuple) const {
     return apply([&](auto&&... args) { return f_(t, y, msgs_, args...); },
                  args_tuple);
   }
@@ -629,7 +630,8 @@ class cvodes_integrator_adjoint_vari : public vari {
       Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
       for (int i = 0; i < ts_.size(); ++i) {
         for (int j = 0; j < N_; ++j) {
-          step_sens.coeffRef(j) += forward_as<var>(solver_->y_return_[i][j]).adj();
+          step_sens.coeffRef(j)
+              += forward_as<var>(solver_->y_return_[i][j]).adj();
         }
 
         forward_as<var>(stan::get(ts_, i))->adj_ += step_sens.dot(
@@ -652,7 +654,8 @@ class cvodes_integrator_adjoint_vari : public vari {
       // Take in the adjoints from all the output variables at this point
       // in time
       for (int j = 0; j < N_; ++j) {
-        state_backward_.coeffRef(j) += forward_as<var>(solver_->y_return_[i][j]).adj();
+        state_backward_.coeffRef(j)
+            += forward_as<var>(solver_->y_return_[i][j]).adj();
       }
 
       double t_final = value_of((i > 0) ? ts_[i - 1] : t0_);

From 8a4525af06b80ad047f7aec67d573c592efdfd3f Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 4 May 2021 22:23:01 +0200
Subject: [PATCH 126/157] doc and review comments

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 496 +++++++++---------
 stan/math/rev/functor/ode_adjoint.hpp         |  35 +-
 2 files changed, 275 insertions(+), 256 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index b0ebe2a7087..fa1f2ffc2a7 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -27,9 +27,9 @@ namespace math {
  *
  * @tparam F Type of ODE right hand side
  * @tparam T_y0 Type of scalars for initial state
- * @tparam T_param Type of scalars for parameters
- * @tparam T_t0 Type of scalar of initial time point
+ * @tparam T_t0 Type of initial time
  * @tparam T_ts Type of time-points where ODE solution is returned
+ * @tparam T_Args Types of pass-through parameters
  */
 template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args>
@@ -37,6 +37,15 @@ class cvodes_integrator_adjoint_vari : public vari {
   using T_Return = return_type_t<T_y0, T_t0, T_ts, T_Args...>;
   using T_y0_t0 = return_type_t<T_y0, T_t0>;
 
+  static constexpr bool is_var_ts_{is_var<T_ts>::value};
+  static constexpr bool is_var_t0_{is_var<T_t0>::value};
+  static constexpr bool is_var_y0_{is_var<T_y0_t0>::value};
+  static constexpr bool is_any_var_args_{
+      disjunction<is_var<scalar_type_t<T_Args>>...>::value};
+  static constexpr bool is_var_return_{is_var<T_Return>::value};
+  static constexpr bool is_var_only_ts_{
+      is_var_ts_ && !(is_var_t0_ || is_var_y0_ || is_any_var_args_)};
+
   const std::decay_t<F> f_;
   arena_t<Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1>> y0_;
   arena_t<T_t0> t0_;
@@ -78,248 +87,6 @@ class cvodes_integrator_adjoint_vari : public vari {
 
   int index_backward_;
 
-  struct cvodes_solver;
-  cvodes_solver* solver_;
-
-  static constexpr bool is_var_ts_{is_var<T_ts>::value};
-  static constexpr bool is_var_t0_{is_var<T_t0>::value};
-  static constexpr bool is_var_y0_{is_var<T_y0_t0>::value};
-  static constexpr bool is_any_var_args_{
-      disjunction<is_var<scalar_type_t<T_Args>>...>::value};
-  static constexpr bool is_var_return_{is_var<T_Return>::value};
-  static constexpr bool is_var_only_ts_{
-      is_var_ts_ && !(is_var_t0_ || is_var_y0_ || is_any_var_args_)};
-
-  /**
-   * Call the ODE RHS with given tuple.
-   */
-  template <typename yT, typename... ArgsT>
-  constexpr auto rhs(double t, const yT& y, const std::tuple<ArgsT...>& args_tuple) const {
-    return apply([&](auto&&... args) { return f_(t, y, msgs_, args...); },
-                 args_tuple);
-  }
-
-  /**
-   * Utility to cast user memory pointer passed in from CVODES to actual typed
-   * object pointer.
-   */
-  constexpr static cvodes_integrator_adjoint_vari* cast_to_self(void* mem) {
-    return static_cast<cvodes_integrator_adjoint_vari*>(mem);
-  }
-
-  /**
-   * Implements the function of type CVRhsFn which is the user-defined
-   * ODE RHS passed to CVODES.
-   */
-  constexpr static int cv_rhs(realtype t, N_Vector y, N_Vector ydot,
-                              void* user_data) {
-    cast_to_self(user_data)->rhs(t, NV_DATA_S(y), NV_DATA_S(ydot));
-    return 0;
-  }
-
-  /**
-   * Implements the function of type CVRhsFnB which is the
-   * RHS of the backward ODE system.
-   */
-  constexpr static int cv_rhs_adj(realtype t, N_Vector y, N_Vector yB,
-                                  N_Vector yBdot, void* user_data) {
-    cast_to_self(user_data)->rhs_adj(t, y, yB, yBdot);
-    return 0;
-  }
-
-  /**
-   * Implements the function of type CVQuadRhsFnB which is the
-   * RHS of the backward ODE system's quadrature.
-   */
-  constexpr static int cv_quad_rhs_adj(realtype t, N_Vector y, N_Vector yB,
-                                       N_Vector qBdot, void* user_data) {
-    cast_to_self(user_data)->quad_rhs_adj(t, y, yB, qBdot);
-    return 0;
-  }
-
-  /**
-   * Implements the function of type CVDlsJacFn which is the
-   * user-defined callback for CVODES to calculate the jacobian of the
-   * ode_rhs wrt to the states y. The jacobian is stored in column
-   * major format.
-   */
-  constexpr static int cv_jacobian_rhs_states(realtype t, N_Vector y,
-                                              N_Vector fy, SUNMatrix J,
-                                              void* user_data, N_Vector tmp1,
-                                              N_Vector tmp2, N_Vector tmp3) {
-    cast_to_self(user_data)->jacobian_rhs_states(t, y, J);
-    return 0;
-  }
-
-  /**
-   * Implements the CVLsJacFnB function for evaluating the jacobian of
-   * the adjoint problem wrt to the backward states.
-   */
-  constexpr static int cv_jacobian_rhs_adj_states(realtype t, N_Vector y,
-                                                  N_Vector yB, N_Vector fyB,
-                                                  SUNMatrix J, void* user_data,
-                                                  N_Vector tmp1, N_Vector tmp2,
-                                                  N_Vector tmp3) {
-    cast_to_self(user_data)->jacobian_rhs_adj_states(t, y, J);
-    return 0;
-  }
-
-  /**
-   * Calculates the ODE RHS, dy_dt, using the user-supplied functor at
-   * the given time t and state y.
-   */
-  inline void rhs(double t, const double y[], double dy_dt[]) const {
-    const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
-
-    const Eigen::VectorXd dy_dt_vec = rhs(t, y_vec, value_of_args_tuple_);
-
-    check_size_match(solver_->function_name_, "dy_dt", dy_dt_vec.size(),
-                     "states", N_);
-
-    Eigen::Map<Eigen::VectorXd>(dy_dt, N_) = dy_dt_vec;
-  }
-
-  /*
-   * Calculate the adjoint sensitivity RHS for varying initial conditions
-   * and parameters
-   *
-   * Equation 2.23 in the cvs_guide.
-   *
-   * @param[in] initial var vector
-   * @param[in] param var vector
-   * @param[in] t time
-   * @param[in] y state of the base ODE system
-   * @param[in] yB state of the adjoint ODE system
-   * @param[out] yBdot evaluation of adjoint ODE RHS
-   */
-  inline void rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
-    Eigen::Map<const Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
-    Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
-    Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(yBdot), N_);
-    mu_dot.setZero();
-
-    const nested_rev_autodiff nested;
-
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y_vars(y_vec);
-
-    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
-        = rhs(t, y_vars, value_of_args_tuple_);
-
-    check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(),
-                     "states", N_);
-
-    f_y_t_vars.adj() = -mu;
-
-    grad();
-
-    mu_dot = y_vars.adj();
-  }
-
-  /*
-   * Calculate the RHS for the quadrature part of the adjoint ODE
-   * problem.
-   *
-   * This is the integrand of equation 2.22 in the cvs_guide.
-   *
-   * @param[in] initial var vector
-   * @param[in] param var vector
-   * @param[in] t time
-   * @param[in] y state of the base ODE system
-   * @param[in] yB state of the adjoint ODE system
-   * @param[out] qBdot evaluation of adjoint ODE quadrature RHS
-   */
-  inline void quad_rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector qBdot) {
-    Eigen::Map<const Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
-    Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
-    Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(qBdot), num_args_vars_);
-    mu_dot.setZero();
-
-    const nested_rev_autodiff nested;
-
-    // The vars here do not live on the nested stack so must be zero'd
-    // separately
-    stan::math::for_each([](auto&& arg) { zero_adjoints(arg); },
-                         local_args_tuple_);
-
-    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
-        = rhs(t, y_vec, local_args_tuple_);
-
-    check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(),
-                     "states", N_);
-
-    f_y_t_vars.adj() = -mu;
-
-    grad();
-
-    apply([&](auto&&... args) { accumulate_adjoints(mu_dot.data(), args...); },
-          local_args_tuple_);
-  }
-
-  /**
-   * Calculates the jacobian of the ODE RHS wrt to its states y at the
-   * given time-point t and state y.
-   */
-  inline void jacobian_rhs_states(double t, N_Vector y, SUNMatrix J) const {
-    Eigen::Map<Eigen::MatrixXd> Jfy(SM_DATA_D(J), N_, N_);
-    Eigen::Map<const Eigen::VectorXd> x(NV_DATA_S(y), N_);
-
-    nested_rev_autodiff nested;
-
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y_var(x);
-    Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var
-        = rhs(t, y_var, value_of_args_tuple_);
-
-    check_size_match(solver_->function_name_, "dy_dt", fy_var.size(), "states",
-                     N_);
-
-    grad(fy_var.coeffRef(0).vi_);
-    Jfy.col(0) = y_var.adj();
-    for (int i = 1; i < fy_var.size(); ++i) {
-      nested.set_zero_all_adjoints();
-      grad(fy_var.coeffRef(i).vi_);
-      Jfy.col(i) = y_var.adj();
-    }
-    Jfy.transposeInPlace();
-  }
-
-  /*
-   * Calculate the Jacobian of the RHS of the adjoint ODE (see rhs_adj
-   * below for citation for how this is done)
-   *
-   * @param[in] y State of system
-   * @param[in] t Time
-   * @param[out] J CVode structure where output is to be stored
-   */
-  inline void jacobian_rhs_adj_states(double t, N_Vector y, SUNMatrix J) const {
-    Eigen::Map<Eigen::MatrixXd> J_adj_y(SM_DATA_D(J), N_, N_);
-
-    // J_adj_y = -1 * transpose(J_y)
-    jacobian_rhs_states(t, y, J);
-
-    J_adj_y.transposeInPlace();
-    J_adj_y.array() *= -1.0;
-  }
-
-  /**
-   * Overloads which setup the states returned from the forward solve. In case
-   * the return type is a double only, then no autodiff is needed. In case of
-   * autodiff then non-chaining varis are setup accordingly.
-   */
-  void store_state(std::size_t n, const Eigen::VectorXd& state,
-                   Eigen::Matrix<var, Eigen::Dynamic, 1>& state_return) {
-    solver_->y_[n] = state;
-    state_return.resize(N_);
-    for (size_t i = 0; i < N_; i++) {
-      state_return.coeffRef(i) = var(new vari(state.coeff(i), false));
-    }
-  }
-
-  void store_state(std::size_t n, const Eigen::VectorXd& state,
-                   Eigen::Matrix<double, Eigen::Dynamic, 1>& state_return) {
-    solver_->y_[n] = state;
-    state_return = state;
-  }
-
   /**
    * Since the CVODES solver manages memory with malloc calls, these resources
    * must be freed using a destructor call (which is not being called for the
@@ -395,6 +162,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       CVodeFree(&cvodes_mem_);
     }
   };
+  cvodes_solver* solver_;
 
  public:
   /**
@@ -411,11 +179,11 @@ class cvodes_integrator_adjoint_vari : public vari {
    *   not less than t0.
    * @param relative_tolerance_forward Relative tolerance for forward problem
    * passed to CVODES
-   * @param absolute_tolerance_forward Absolute tolerance for forward problem
+   * @param absolute_tolerance_forward Absolute tolerance per ODE state for forward problem
    * passed to CVODES
    * @param relative_tolerance_backward Relative tolerance for backward problem
    * passed to CVODES
-   * @param absolute_tolerance_backward Absolute tolerance for backward problem
+   * @param absolute_tolerance_backward Absolute tolerance per ODE state for backward problem
    * passed to CVODES
    * @param relative_tolerance_quadrature Relative tolerance for quadrature
    * problem passed to CVODES
@@ -631,7 +399,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
       for (int i = 0; i < ts_.size(); ++i) {
         for (int j = 0; j < N_; ++j) {
-          step_sens.coeffRef(j) += forward_as<var>(solver_->y_return_[i][j]).adj();
+          step_sens.coeffRef(j) += forward_as<var>(solver_->y_return_[i].coeff(j)).adj();
         }
 
         forward_as<var>(stan::get(ts_, i))->adj_ += step_sens.dot(
@@ -654,7 +422,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       // Take in the adjoints from all the output variables at this point
       // in time
       for (int j = 0; j < N_; ++j) {
-        state_backward_.coeffRef(j) += forward_as<var>(solver_->y_return_[i][j]).adj();
+        state_backward_.coeffRef(j) += forward_as<var>(solver_->y_return_[i].coeff(j)).adj();
       }
 
       double t_final = value_of((i > 0) ? ts_[i - 1] : t0_);
@@ -782,6 +550,238 @@ class cvodes_integrator_adjoint_vari : public vari {
       }
     }
   }
+
+ private:
+  
+  /**
+   * Call the ODE RHS with given tuple.
+   */
+  template <typename yT, typename... ArgsT>
+  constexpr auto rhs(double t, const yT& y, const std::tuple<ArgsT...>& args_tuple) const {
+    return apply([&](auto&&... args) { return f_(t, y, msgs_, args...); },
+                 args_tuple);
+  }
+
+  /**
+   * Utility to cast user memory pointer passed in from CVODES to actual typed
+   * object pointer.
+   */
+  constexpr static cvodes_integrator_adjoint_vari* cast_to_self(void* mem) {
+    return static_cast<cvodes_integrator_adjoint_vari*>(mem);
+  }
+
+  /**
+   * Implements the function of type CVRhsFn which is the user-defined
+   * ODE RHS passed to CVODES.
+   */
+  constexpr static int cv_rhs(realtype t, N_Vector y, N_Vector ydot,
+                              void* user_data) {
+    cast_to_self(user_data)->rhs(t, NV_DATA_S(y), NV_DATA_S(ydot));
+    return 0;
+  }
+
+  /**
+   * Implements the function of type CVRhsFnB which is the
+   * RHS of the backward ODE system.
+   */
+  constexpr static int cv_rhs_adj(realtype t, N_Vector y, N_Vector yB,
+                                  N_Vector yBdot, void* user_data) {
+    cast_to_self(user_data)->rhs_adj(t, y, yB, yBdot);
+    return 0;
+  }
+
+  /**
+   * Implements the function of type CVQuadRhsFnB which is the
+   * RHS of the backward ODE system's quadrature.
+   */
+  constexpr static int cv_quad_rhs_adj(realtype t, N_Vector y, N_Vector yB,
+                                       N_Vector qBdot, void* user_data) {
+    cast_to_self(user_data)->quad_rhs_adj(t, y, yB, qBdot);
+    return 0;
+  }
+
+  /**
+   * Implements the function of type CVDlsJacFn which is the
+   * user-defined callback for CVODES to calculate the jacobian of the
+   * ode_rhs wrt to the states y. The jacobian is stored in column
+   * major format.
+   */
+  constexpr static int cv_jacobian_rhs_states(realtype t, N_Vector y,
+                                              N_Vector fy, SUNMatrix J,
+                                              void* user_data, N_Vector tmp1,
+                                              N_Vector tmp2, N_Vector tmp3) {
+    cast_to_self(user_data)->jacobian_rhs_states(t, y, J);
+    return 0;
+  }
+
+  /**
+   * Implements the CVLsJacFnB function for evaluating the jacobian of
+   * the adjoint problem wrt to the backward states.
+   */
+  constexpr static int cv_jacobian_rhs_adj_states(realtype t, N_Vector y,
+                                                  N_Vector yB, N_Vector fyB,
+                                                  SUNMatrix J, void* user_data,
+                                                  N_Vector tmp1, N_Vector tmp2,
+                                                  N_Vector tmp3) {
+    cast_to_self(user_data)->jacobian_rhs_adj_states(t, y, J);
+    return 0;
+  }
+
+  /**
+   * Calculates the ODE RHS, dy_dt, using the user-supplied functor at
+   * the given time t and state y.
+   */
+  inline void rhs(double t, const double y[], double dy_dt[]) const {
+    const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
+
+    const Eigen::VectorXd dy_dt_vec = rhs(t, y_vec, value_of_args_tuple_);
+
+    check_size_match(solver_->function_name_, "dy_dt", dy_dt_vec.size(),
+                     "states", N_);
+
+    Eigen::Map<Eigen::VectorXd>(dy_dt, N_) = dy_dt_vec;
+  }
+
+  /*
+   * Calculate the adjoint sensitivity RHS for varying initial conditions
+   * and parameters
+   *
+   * Equation 2.23 in the cvs_guide.
+   *
+   * @param[in] initial var vector
+   * @param[in] param var vector
+   * @param[in] t time
+   * @param[in] y state of the base ODE system
+   * @param[in] yB state of the adjoint ODE system
+   * @param[out] yBdot evaluation of adjoint ODE RHS
+   */
+  inline void rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
+    Eigen::Map<const Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
+    Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
+    Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(yBdot), N_);
+    mu_dot.setZero();
+
+    const nested_rev_autodiff nested;
+
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y_vars(y_vec);
+
+    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
+        = rhs(t, y_vars, value_of_args_tuple_);
+
+    check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(),
+                     "states", N_);
+
+    f_y_t_vars.adj() = -mu;
+
+    grad();
+
+    mu_dot = y_vars.adj();
+  }
+
+  /*
+   * Calculate the RHS for the quadrature part of the adjoint ODE
+   * problem.
+   *
+   * This is the integrand of equation 2.22 in the cvs_guide.
+   *
+   * @param[in] initial var vector
+   * @param[in] param var vector
+   * @param[in] t time
+   * @param[in] y state of the base ODE system
+   * @param[in] yB state of the adjoint ODE system
+   * @param[out] qBdot evaluation of adjoint ODE quadrature RHS
+   */
+  inline void quad_rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector qBdot) {
+    Eigen::Map<const Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
+    Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
+    Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(qBdot), num_args_vars_);
+    mu_dot.setZero();
+
+    const nested_rev_autodiff nested;
+
+    // The vars here do not live on the nested stack so must be zero'd
+    // separately
+    stan::math::for_each([](auto&& arg) { zero_adjoints(arg); },
+                         local_args_tuple_);
+
+    Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
+        = rhs(t, y_vec, local_args_tuple_);
+
+    check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(),
+                     "states", N_);
+
+    f_y_t_vars.adj() = -mu;
+
+    grad();
+
+    apply([&](auto&&... args) { accumulate_adjoints(mu_dot.data(), args...); },
+          local_args_tuple_);
+  }
+
+  /**
+   * Calculates the jacobian of the ODE RHS wrt to its states y at the
+   * given time-point t and state y.
+   */
+  inline void jacobian_rhs_states(double t, N_Vector y, SUNMatrix J) const {
+    Eigen::Map<Eigen::MatrixXd> Jfy(SM_DATA_D(J), N_, N_);
+    Eigen::Map<const Eigen::VectorXd> x(NV_DATA_S(y), N_);
+
+    nested_rev_autodiff nested;
+
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y_var(x);
+    Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var
+        = rhs(t, y_var, value_of_args_tuple_);
+
+    check_size_match(solver_->function_name_, "dy_dt", fy_var.size(), "states",
+                     N_);
+
+    grad(fy_var.coeffRef(0).vi_);
+    Jfy.col(0) = y_var.adj();
+    for (int i = 1; i < fy_var.size(); ++i) {
+      nested.set_zero_all_adjoints();
+      grad(fy_var.coeffRef(i).vi_);
+      Jfy.col(i) = y_var.adj();
+    }
+    Jfy.transposeInPlace();
+  }
+
+  /*
+   * Calculate the Jacobian of the RHS of the adjoint ODE (see rhs_adj
+   * below for citation for how this is done)
+   *
+   * @param[in] y State of system
+   * @param[in] t Time
+   * @param[out] J CVode structure where output is to be stored
+   */
+  inline void jacobian_rhs_adj_states(double t, N_Vector y, SUNMatrix J) const {
+    Eigen::Map<Eigen::MatrixXd> J_adj_y(SM_DATA_D(J), N_, N_);
+
+    // J_adj_y = -1 * transpose(J_y)
+    jacobian_rhs_states(t, y, J);
+
+    J_adj_y.transposeInPlace();
+    J_adj_y.array() *= -1.0;
+  }
+
+  /**
+   * Overloads which setup the states returned from the forward solve. In case
+   * the return type is a double only, then no autodiff is needed. In case of
+   * autodiff then non-chaining varis are setup accordingly.
+   */
+  void store_state(std::size_t n, const Eigen::VectorXd& state,
+                   Eigen::Matrix<var, Eigen::Dynamic, 1>& state_return) {
+    solver_->y_[n] = state;
+    state_return.resize(N_);
+    for (size_t i = 0; i < N_; i++) {
+      state_return.coeffRef(i) = var(new vari(state.coeff(i), false));
+    }
+  }
+
+  void store_state(std::size_t n, const Eigen::VectorXd& state,
+                   Eigen::Matrix<double, Eigen::Dynamic, 1>& state_return) {
+    solver_->y_[n] = state;
+    state_return = state;
+  }
 };  // cvodes integrator adjoint vari
 
 }  // namespace math
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index a32f64c741a..bd9e65c977b 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -13,7 +13,8 @@ namespace math {
 /**
  * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
  * times, { t1, t2, t3, ... } using the stiff backward differentiation formula
- * BDF solver from CVODES.
+ * BDF solver or the non-stiff Adams solver from CVODES. The ODE system is
+ * integrated using the adjoint sensitivity approach of CVODES.
  *
  * \p f must define an operator() with the signature as:
  *   template<typename T_t, typename T_y, typename... T_Args>
@@ -26,7 +27,8 @@ namespace math {
  * through to \p f without modification).
  *
  * @tparam F Type of ODE right hand side
- * @tparam T_0 Type of initial time
+ * @tparam T_y0 Type of initial state
+ * @tparam T_t0 Type of initial time
  * @tparam T_ts Type of output times
  * @tparam T_Args Types of pass-through parameters
  *
@@ -38,11 +40,11 @@ namespace math {
  *   not less than t0.
  * @param relative_tolerance_forward Relative tolerance for forward problem
  * passed to CVODES
- * @param absolute_tolerance_forward Absolute tolerance for forward problem
+ * @param absolute_tolerance_forward Absolute tolerance per ODE state for forward problem
  * passed to CVODES
  * @param relative_tolerance_backward Relative tolerance for backward problem
  * passed to CVODES
- * @param absolute_tolerance_backward Absolute tolerance for backward problem
+ * @param absolute_tolerance_backward Absolute tolerance per ODE state for backward problem
  * passed to CVODES
  * @param relative_tolerance_quadrature Relative tolerance for quadrature
  * problem passed to CVODES
@@ -90,7 +92,8 @@ ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
 /**
  * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
  * times, { t1, t2, t3, ... } using the stiff backward differentiation formula
- * BDF solver from CVODES.
+ * BDF solver or the non-stiff Adams solver from CVODES. The ODE system is
+ * integrated using the adjoint sensitivity approach of CVODES.
  *
  * \p f must define an operator() with the signature as:
  *   template<typename T_t, typename T_y, typename... T_Args>
@@ -103,7 +106,8 @@ ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
  * through to \p f without modification).
  *
  * @tparam F Type of ODE right hand side
- * @tparam T_0 Type of initial time
+ * @tparam T_y0 Type of initial state
+ * @tparam T_t0 Type of initial time
  * @tparam T_ts Type of output times
  * @tparam T_Args Types of pass-through parameters
  *
@@ -112,10 +116,25 @@ ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
  * @param t0 Initial time
  * @param ts Times at which to solve the ODE at. All values must be sorted and
  *   not less than t0.
- * @param relative_tolerance Relative tolerance passed to CVODES
- * @param absolute_tolerance Absolute tolerance passed to CVODES
+ * @param relative_tolerance_forward Relative tolerance for forward problem
+ * passed to CVODES
+ * @param absolute_tolerance_forward Absolute tolerance per ODE state for forward problem
+ * passed to CVODES
+ * @param relative_tolerance_backward Relative tolerance for backward problem
+ * passed to CVODES
+ * @param absolute_tolerance_backward Absolute tolerance per ODE state for backward problem
+ * passed to CVODES
+ * @param relative_tolerance_quadrature Relative tolerance for quadrature
+ * problem passed to CVODES
+ * @param absolute_tolerance_quadrature Absolute tolerance for quadrature
+ * problem passed to CVODES
  * @param max_num_steps Upper limit on the number of integration steps to
  *   take between each output (error if exceeded)
+ * @param num_steps_between_checkpoints Number of integrator steps after which a
+ * checkpoint is stored for the backward pass
+ * @param interpolation_polynomial type of polynomial used for interpolation
+ * @param solver_forward solver used for forward pass
+ * @param solver_backward solver used for backward pass
  * @param[in, out] msgs the print stream for warning messages
  * @param args Extra arguments passed unmodified through to ODE right hand side
  * @return Solution to ODE at times \p ts

From eb62f8caeca60992e319f7fe09f7fa135a66b579 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Tue, 4 May 2021 20:28:00 +0000
Subject: [PATCH 127/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 35 +++++++++----------
 stan/math/rev/functor/ode_adjoint.hpp         | 16 ++++-----
 .../math/rev/functor/test_fixture_ode_sho.hpp | 27 ++++++++------
 3 files changed, 40 insertions(+), 38 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index fa1f2ffc2a7..6e0597eecfc 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -179,12 +179,12 @@ class cvodes_integrator_adjoint_vari : public vari {
    *   not less than t0.
    * @param relative_tolerance_forward Relative tolerance for forward problem
    * passed to CVODES
-   * @param absolute_tolerance_forward Absolute tolerance per ODE state for forward problem
-   * passed to CVODES
+   * @param absolute_tolerance_forward Absolute tolerance per ODE state for
+   * forward problem passed to CVODES
    * @param relative_tolerance_backward Relative tolerance for backward problem
    * passed to CVODES
-   * @param absolute_tolerance_backward Absolute tolerance per ODE state for backward problem
-   * passed to CVODES
+   * @param absolute_tolerance_backward Absolute tolerance per ODE state for
+   * backward problem passed to CVODES
    * @param relative_tolerance_quadrature Relative tolerance for quadrature
    * problem passed to CVODES
    * @param absolute_tolerance_quadrature Absolute tolerance for quadrature
@@ -267,30 +267,25 @@ class cvodes_integrator_adjoint_vari : public vari {
                           absolute_tolerance_forward_);
     check_size_match(function_name, "absolute_tolerance_forward",
                      absolute_tolerance_forward_.size(), "states", N_);
-    check_positive_finite(function_name,
-                          "relative_tolerance_backward",
+    check_positive_finite(function_name, "relative_tolerance_backward",
                           relative_tolerance_backward_);
-    check_positive_finite(function_name,
-                          "absolute_tolerance_backward",
+    check_positive_finite(function_name, "absolute_tolerance_backward",
                           absolute_tolerance_backward_);
     check_size_match(function_name, "absolute_tolerance_backward",
                      absolute_tolerance_backward_.size(), "states", N_);
-    check_positive_finite(function_name,
-                          "relative_tolerance_quadrature",
+    check_positive_finite(function_name, "relative_tolerance_quadrature",
                           relative_tolerance_quadrature_);
-    check_positive_finite(function_name,
-                          "absolute_tolerance_quadrature",
+    check_positive_finite(function_name, "absolute_tolerance_quadrature",
                           absolute_tolerance_quadrature_);
     check_positive(function_name, "max_num_steps", max_num_steps_);
     check_positive(function_name, "num_steps_between_checkpoints",
                    num_steps_between_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
-    check_bounded(function_name, "interpolation_polynomial", 
+    check_bounded(function_name, "interpolation_polynomial",
                   interpolation_polynomial_, 1, 2);
     // 1=Adams=CV_ADAMS, 2=BDF=CV_BDF
     check_bounded(function_name, "solver_forward", solver_forward_, 1, 2);
-    check_bounded(function_name, "solver_backward",
-                  solver_backward_, 1, 2);
+    check_bounded(function_name, "solver_backward", solver_backward_, 1, 2);
 
     solver_ = new cvodes_solver(
         function_name, N_, num_args_vars_, ts_.size(), solver_forward_,
@@ -399,7 +394,8 @@ class cvodes_integrator_adjoint_vari : public vari {
       Eigen::VectorXd step_sens = Eigen::VectorXd::Zero(N_);
       for (int i = 0; i < ts_.size(); ++i) {
         for (int j = 0; j < N_; ++j) {
-          step_sens.coeffRef(j) += forward_as<var>(solver_->y_return_[i].coeff(j)).adj();
+          step_sens.coeffRef(j)
+              += forward_as<var>(solver_->y_return_[i].coeff(j)).adj();
         }
 
         forward_as<var>(stan::get(ts_, i))->adj_ += step_sens.dot(
@@ -422,7 +418,8 @@ class cvodes_integrator_adjoint_vari : public vari {
       // Take in the adjoints from all the output variables at this point
       // in time
       for (int j = 0; j < N_; ++j) {
-        state_backward_.coeffRef(j) += forward_as<var>(solver_->y_return_[i].coeff(j)).adj();
+        state_backward_.coeffRef(j)
+            += forward_as<var>(solver_->y_return_[i].coeff(j)).adj();
       }
 
       double t_final = value_of((i > 0) ? ts_[i - 1] : t0_);
@@ -552,12 +549,12 @@ class cvodes_integrator_adjoint_vari : public vari {
   }
 
  private:
-  
   /**
    * Call the ODE RHS with given tuple.
    */
   template <typename yT, typename... ArgsT>
-  constexpr auto rhs(double t, const yT& y, const std::tuple<ArgsT...>& args_tuple) const {
+  constexpr auto rhs(double t, const yT& y,
+                     const std::tuple<ArgsT...>& args_tuple) const {
     return apply([&](auto&&... args) { return f_(t, y, msgs_, args...); },
                  args_tuple);
   }
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index bd9e65c977b..0aba04f1217 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -40,12 +40,12 @@ namespace math {
  *   not less than t0.
  * @param relative_tolerance_forward Relative tolerance for forward problem
  * passed to CVODES
- * @param absolute_tolerance_forward Absolute tolerance per ODE state for forward problem
- * passed to CVODES
+ * @param absolute_tolerance_forward Absolute tolerance per ODE state for
+ * forward problem passed to CVODES
  * @param relative_tolerance_backward Relative tolerance for backward problem
  * passed to CVODES
- * @param absolute_tolerance_backward Absolute tolerance per ODE state for backward problem
- * passed to CVODES
+ * @param absolute_tolerance_backward Absolute tolerance per ODE state for
+ * backward problem passed to CVODES
  * @param relative_tolerance_quadrature Relative tolerance for quadrature
  * problem passed to CVODES
  * @param absolute_tolerance_quadrature Absolute tolerance for quadrature
@@ -118,12 +118,12 @@ ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
  *   not less than t0.
  * @param relative_tolerance_forward Relative tolerance for forward problem
  * passed to CVODES
- * @param absolute_tolerance_forward Absolute tolerance per ODE state for forward problem
- * passed to CVODES
+ * @param absolute_tolerance_forward Absolute tolerance per ODE state for
+ * forward problem passed to CVODES
  * @param relative_tolerance_backward Relative tolerance for backward problem
  * passed to CVODES
- * @param absolute_tolerance_backward Absolute tolerance per ODE state for backward problem
- * passed to CVODES
+ * @param absolute_tolerance_backward Absolute tolerance per ODE state for
+ * backward problem passed to CVODES
  * @param relative_tolerance_quadrature Relative tolerance for quadrature
  * problem passed to CVODES
  * @param absolute_tolerance_quadrature Absolute tolerance for quadrature
diff --git a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
index 50a5130b3bf..410f2a43d61 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
@@ -606,7 +606,7 @@ struct harmonic_oscillator_ctl_test
                      "absolute_tolerance_forward");
     this->atol_f.resize(2);
     this->atol_f(1) = atol_f_;
-    
+
     const double rtol_b_ = this->rtol_b;
     this->rtol_b = -1;
     EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
@@ -625,7 +625,7 @@ struct harmonic_oscillator_ctl_test
     this->atol_b.resize(2);
     this->atol_b(0) = atol_b_;
     this->atol_b(1) = atol_b_;
-    
+
     const double rtol_q_ = this->rtol_q;
     this->rtol_q = -1;
     EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
@@ -640,27 +640,32 @@ struct harmonic_oscillator_ctl_test
 
     const int max_num_step_ = this->max_num_step;
     this->max_num_step = -1;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error, "max_num_steps");
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "max_num_steps");
     this->max_num_step = max_num_step_;
- 
+
     const int num_steps_check_ = this->num_steps_check;
     this->num_steps_check = -1;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error, "num_steps_between_checkpoints");
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "num_steps_between_checkpoints");
     this->num_steps_check = num_steps_check_;
- 
+
     const int inter_poly_ = this->inter_poly;
     this->inter_poly = 0;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error, "interpolation_polynomial");
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "interpolation_polynomial");
     this->inter_poly = inter_poly_;
- 
+
     const int solv_f_ = this->solv_f;
     this->solv_f = 0;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error, "solver_forward");
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "solver_forward");
     this->solv_f = solv_f_;
- 
+
     const int solv_b_ = this->solv_b;
     this->solv_b = 0;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error, "solver_backward");
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "solver_backward");
     this->solv_b = solv_b_;
 
     const auto theta_ = this->theta;

From 12b4a92dc927aaa650e63bbc6736d7ead06d78e0 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 4 May 2021 22:34:25 +0200
Subject: [PATCH 128/157] move functor into deallocated solve object

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index fa1f2ffc2a7..c9e83689856 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -46,7 +46,6 @@ class cvodes_integrator_adjoint_vari : public vari {
   static constexpr bool is_var_only_ts_{
       is_var_ts_ && !(is_var_t0_ || is_var_y0_ || is_any_var_args_)};
 
-  const std::decay_t<F> f_;
   arena_t<Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1>> y0_;
   arena_t<T_t0> t0_;
   std::vector<arena_t<T_ts>, arena_allocator<arena_t<T_ts>>> ts_;
@@ -93,6 +92,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    * vari class).
    */
   struct cvodes_solver : public chainable_alloc {
+    const std::decay_t<F> f_;
     const size_t N_;
     std::vector<Eigen::VectorXd> y_;
     std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return_;
@@ -111,14 +111,15 @@ class cvodes_integrator_adjoint_vari : public vari {
     SUNMatrix A_backward_;
     SUNLinearSolver LS_backward_;
 
-    template <typename StateFwd, typename StateBwd, typename Quad,
+    template <typename FF, typename StateFwd, typename StateBwd, typename Quad,
               typename AbsTolFwd, typename AbsTolBwd>
-    cvodes_solver(const char* function_name, size_t N, size_t num_args_vars,
+    cvodes_solver(const char* function_name, FF&& f, size_t N, size_t num_args_vars,
                   size_t ts_size, int solver_forward, StateFwd& state_forward,
                   StateBwd& state_backward, Quad& quad,
                   AbsTolFwd& absolute_tolerance_forward,
                   AbsTolBwd& absolute_tolerance_backward)
         : chainable_alloc(),
+          f_(f),
           N_(N),
           y_(ts_size),
           y_return_(ts_size),
@@ -218,7 +219,6 @@ class cvodes_integrator_adjoint_vari : public vari {
       int interpolation_polynomial, int solver_forward, int solver_backward,
       std::ostream* msgs, const T_Args&... args)
       : vari(NOT_A_NUMBER),
-        f_(f),
         y0_(y0),
         t0_(t0),
         ts_(ts.begin(), ts.end()),
@@ -293,7 +293,7 @@ class cvodes_integrator_adjoint_vari : public vari {
                   solver_backward_, 1, 2);
 
     solver_ = new cvodes_solver(
-        function_name, N_, num_args_vars_, ts_.size(), solver_forward_,
+        function_name, f, N_, num_args_vars_, ts_.size(), solver_forward_,
         state_forward_, state_backward_, quad_, absolute_tolerance_forward_,
         absolute_tolerance_backward_);
 
@@ -558,7 +558,7 @@ class cvodes_integrator_adjoint_vari : public vari {
    */
   template <typename yT, typename... ArgsT>
   constexpr auto rhs(double t, const yT& y, const std::tuple<ArgsT...>& args_tuple) const {
-    return apply([&](auto&&... args) { return f_(t, y, msgs_, args...); },
+    return apply([&](auto&&... args) { return solver_->f_(t, y, msgs_, args...); },
                  args_tuple);
   }
 

From c0fa201521f2b489112aa6de36a253d6f2bce8dd Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Tue, 4 May 2021 20:37:48 +0000
Subject: [PATCH 129/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../math/rev/functor/cvodes_integrator_adjoint.hpp | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index a162b60070c..704e68d4f82 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -113,9 +113,9 @@ class cvodes_integrator_adjoint_vari : public vari {
 
     template <typename FF, typename StateFwd, typename StateBwd, typename Quad,
               typename AbsTolFwd, typename AbsTolBwd>
-    cvodes_solver(const char* function_name, FF&& f, size_t N, size_t num_args_vars,
-                  size_t ts_size, int solver_forward, StateFwd& state_forward,
-                  StateBwd& state_backward, Quad& quad,
+    cvodes_solver(const char* function_name, FF&& f, size_t N,
+                  size_t num_args_vars, size_t ts_size, int solver_forward,
+                  StateFwd& state_forward, StateBwd& state_backward, Quad& quad,
                   AbsTolFwd& absolute_tolerance_forward,
                   AbsTolBwd& absolute_tolerance_backward)
         : chainable_alloc(),
@@ -553,9 +553,11 @@ class cvodes_integrator_adjoint_vari : public vari {
    * Call the ODE RHS with given tuple.
    */
   template <typename yT, typename... ArgsT>
-  constexpr auto rhs(double t, const yT& y, const std::tuple<ArgsT...>& args_tuple) const {
-    return apply([&](auto&&... args) { return solver_->f_(t, y, msgs_, args...); },
-                 args_tuple);
+  constexpr auto rhs(double t, const yT& y,
+                     const std::tuple<ArgsT...>& args_tuple) const {
+    return apply(
+        [&](auto&&... args) { return solver_->f_(t, y, msgs_, args...); },
+        args_tuple);
   }
 
   /**

From 2bf6d94b0e9f4efae53de5a729cd5cd1f017e12c Mon Sep 17 00:00:00 2001
From: stevebronder <stevo15025@gmail.com>
Date: Tue, 4 May 2021 17:12:46 -0400
Subject: [PATCH 130/157] fix get

---
 stan/math/prim/fun/get.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/stan/math/prim/fun/get.hpp b/stan/math/prim/fun/get.hpp
index 7ab3d4afd53..50486511a1e 100644
--- a/stan/math/prim/fun/get.hpp
+++ b/stan/math/prim/fun/get.hpp
@@ -50,12 +50,12 @@ inline const T& get(const std::vector<T, Alloc>& x, size_t n) {
  */
 template <typename T, typename = require_eigen_t<T>>
 inline scalar_type_t<T>& get(T& m, size_t n) {
-  return m(static_cast<int>(n));
+  return m.coeffRef(static_cast<int>(n));
 }
 
 template <typename T, typename = require_eigen_t<T>>
 inline const scalar_type_t<T>& get(const T& m, size_t n) {
-  return m(static_cast<int>(n));
+  return m.coeffRef(static_cast<int>(n));
 }
 
 }  // namespace stan

From 79f097078f2d18d944a021f7eaae0e3a0eeed5ac Mon Sep 17 00:00:00 2001
From: stevebronder <stevo15025@gmail.com>
Date: Tue, 4 May 2021 17:52:59 -0400
Subject: [PATCH 131/157] return value for const ref input for get()

---
 stan/math/prim/fun/get.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/stan/math/prim/fun/get.hpp b/stan/math/prim/fun/get.hpp
index 50486511a1e..a60e149c514 100644
--- a/stan/math/prim/fun/get.hpp
+++ b/stan/math/prim/fun/get.hpp
@@ -54,8 +54,8 @@ inline scalar_type_t<T>& get(T& m, size_t n) {
 }
 
 template <typename T, typename = require_eigen_t<T>>
-inline const scalar_type_t<T>& get(const T& m, size_t n) {
-  return m.coeffRef(static_cast<int>(n));
+inline scalar_type_t<T> get(const T& m, size_t n) {
+  return m.coeff(static_cast<int>(n));
 }
 
 }  // namespace stan

From 46412b1f90d37652991d0ecada0b23532dbe5605 Mon Sep 17 00:00:00 2001
From: stevebronder <stevo15025@gmail.com>
Date: Tue, 4 May 2021 17:54:30 -0400
Subject: [PATCH 132/157] return value for scalar const ref in get()

---
 stan/math/prim/fun/get.hpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/stan/math/prim/fun/get.hpp b/stan/math/prim/fun/get.hpp
index a60e149c514..32a30dad5f8 100644
--- a/stan/math/prim/fun/get.hpp
+++ b/stan/math/prim/fun/get.hpp
@@ -24,6 +24,11 @@ inline T& get(T& x, size_t n) {
   return x;
 }
 
+template <typename T, typename = require_stan_scalar_t<T>>
+inline T get(const T& x, size_t n) {
+  return x;
+}
+
 /** \ingroup type_trait
  * Returns the n-th element of the provided std::vector.
  *

From ff16c049723bc7b48b128adfdfbfbefbd6c6f55a Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Wed, 5 May 2021 08:31:47 +0200
Subject: [PATCH 133/157] fix tests (hopefully)

---
 test/unit/math/rev/functor/test_fixture_ode_sho.hpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
index 410f2a43d61..ea61a994fd6 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
@@ -602,9 +602,11 @@ struct harmonic_oscillator_ctl_test
     this->atol_f(0) = atol_f_;
 
     this->atol_f.resize(1);
+    this->atol_f(0) = atol_f_;
     EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::invalid_argument,
                      "absolute_tolerance_forward");
     this->atol_f.resize(2);
+    this->atol_f(0) = atol_f_;
     this->atol_f(1) = atol_f_;
 
     const double rtol_b_ = this->rtol_b;
@@ -620,6 +622,7 @@ struct harmonic_oscillator_ctl_test
     this->atol_b(0) = atol_b_;
 
     this->atol_b.resize(1);
+    this->atol_b(0) = atol_b_;
     EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::invalid_argument,
                      "absolute_tolerance_backward");
     this->atol_b.resize(2);

From 33205044590acedc4ab84b63b4c421e4c442884a Mon Sep 17 00:00:00 2001
From: stevebronder <stevo15025@gmail.com>
Date: Sun, 9 May 2021 16:04:58 -0400
Subject: [PATCH 134/157] 1. Cleaning up the actual functions these are called
 from in order to accept eigen expressions. 2. The `cv_*` functions now just
 return back the int from the `*` function they call, which I think looks a
 bit cleaner. I also moved them around so that the `cv_*` function is next to
 the `*` function which I think is a bit easier to read. 3. Cleaned up the
 private member functions to be a bit more efficient. 4. moved `store_state`
 up to near the constructor mostly again so that's it's defined near where
 it's called. 5. Removed the `get()` changes (@bbbales2 great find that makes
 things a lot simpler) 6. `cvodes_integrator_adjoint_vari` now inherits from
 `vari_base` instead of `vari`.

---
 stan/math/prim/fun/get.hpp                    |  21 +-
 stan/math/rev/functor/coupled_ode_system.hpp  |   2 +-
 .../rev/functor/cvodes_integrator_adjoint.hpp | 325 ++++++++----------
 stan/math/rev/functor/ode_adjoint.hpp         |  42 +--
 .../math/rev/functor/sho_ode_typed_test.cpp   |   9 +-
 5 files changed, 177 insertions(+), 222 deletions(-)

diff --git a/stan/math/prim/fun/get.hpp b/stan/math/prim/fun/get.hpp
index 32a30dad5f8..8a466561278 100644
--- a/stan/math/prim/fun/get.hpp
+++ b/stan/math/prim/fun/get.hpp
@@ -19,11 +19,6 @@ namespace stan {
  * @param n index of the element to return
  * @return input scalar
  */
-template <typename T, typename = require_stan_scalar_t<T>>
-inline T& get(T& x, size_t n) {
-  return x;
-}
-
 template <typename T, typename = require_stan_scalar_t<T>>
 inline T get(const T& x, size_t n) {
   return x;
@@ -36,13 +31,8 @@ inline T get(const T& x, size_t n) {
  * @param n index of the element to return
  * @return n-th element of the input vector
  */
-template <typename T, typename Alloc>
-inline T& get(std::vector<T, Alloc>& x, size_t n) {
-  return x[n];
-}
-
-template <typename T, typename Alloc>
-inline const T& get(const std::vector<T, Alloc>& x, size_t n) {
+template <typename T>
+inline T get(const std::vector<T>& x, size_t n) {
   return x[n];
 }
 
@@ -53,14 +43,9 @@ inline const T& get(const std::vector<T, Alloc>& x, size_t n) {
  * @param n index of the element to return
  * @return n-th element of the \c Eigen \c Matrix or expression
  */
-template <typename T, typename = require_eigen_t<T>>
-inline scalar_type_t<T>& get(T& m, size_t n) {
-  return m.coeffRef(static_cast<int>(n));
-}
-
 template <typename T, typename = require_eigen_t<T>>
 inline scalar_type_t<T> get(const T& m, size_t n) {
-  return m.coeff(static_cast<int>(n));
+  return m(static_cast<int>(n));
 }
 
 }  // namespace stan
diff --git a/stan/math/rev/functor/coupled_ode_system.hpp b/stan/math/rev/functor/coupled_ode_system.hpp
index cbd4314c493..264845eaf4f 100644
--- a/stan/math/rev/functor/coupled_ode_system.hpp
+++ b/stan/math/rev/functor/coupled_ode_system.hpp
@@ -139,7 +139,7 @@ struct coupled_ode_system_impl<false, F, T_y0, Args...> {
       y_adjoints_ = y_vars.adj();
 
       if (args_adjoints_.size() > 0) {
-        memset(args_adjoints_.data(), 0, sizeof(double) * num_args_vars);
+        memset(args_adjoints_.data(), 0, sizeof(double) * args_adjoints_.size());
       }
 
       apply(
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 704e68d4f82..a16ec4c2c1e 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -6,7 +6,6 @@
 #include <stan/math/rev/functor/cvodes_utils.hpp>
 #include <stan/math/rev/functor/ode_store_sensitivities.hpp>
 #include <stan/math/prim/err.hpp>
-#include <stan/math/prim/fun/get.hpp>
 #include <stan/math/prim/fun/value_of.hpp>
 #include <stan/math/prim/functor/for_each.hpp>
 #include <cvodes/cvodes.h>
@@ -33,7 +32,7 @@ namespace math {
  */
 template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename... T_Args>
-class cvodes_integrator_adjoint_vari : public vari {
+class cvodes_integrator_adjoint_vari : public vari_base {
   using T_Return = return_type_t<T_y0, T_t0, T_ts, T_Args...>;
   using T_y0_t0 = return_type_t<T_y0, T_t0>;
 
@@ -46,45 +45,35 @@ class cvodes_integrator_adjoint_vari : public vari {
   static constexpr bool is_var_only_ts_{
       is_var_ts_ && !(is_var_t0_ || is_var_y0_ || is_any_var_args_)};
 
+  std::tuple<arena_t<T_Args>...> local_args_tuple_;
+  std::tuple<arena_t<promote_scalar_t<partials_type_t<scalar_type_t<T_Args>>, T_Args>>...>
+      value_of_args_tuple_;
+
+  arena_t<std::vector<Eigen::VectorXd>> y_;
+  arena_t<std::vector<T_ts>> ts_;
   arena_t<Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1>> y0_;
+  arena_t<Eigen::VectorXd> absolute_tolerance_forward_;
+  arena_t<Eigen::VectorXd> absolute_tolerance_backward_;
+  arena_t<Eigen::VectorXd> state_forward_;
+  arena_t<Eigen::VectorXd> state_backward_;
+  size_t num_args_vars_;
+  arena_t<Eigen::VectorXd> quad_;
   arena_t<T_t0> t0_;
-  std::vector<arena_t<T_ts>, arena_allocator<arena_t<T_ts>>> ts_;
 
   double relative_tolerance_forward_;
-  arena_t<Eigen::VectorXd> absolute_tolerance_forward_;
   double relative_tolerance_backward_;
-  arena_t<Eigen::VectorXd> absolute_tolerance_backward_;
   double relative_tolerance_quadrature_;
   double absolute_tolerance_quadrature_;
   long int max_num_steps_;                  // NOLINT(runtime/int)
   long int num_steps_between_checkpoints_;  // NOLINT(runtime/int)
+  size_t N_;
+  std::ostream* msgs_;
+  vari** args_varis_;
   int interpolation_polynomial_;
   int solver_forward_;
   int solver_backward_;
-
-  std::ostream* msgs_;
-
-  std::tuple<arena_t<
-      plain_type_t<decltype(deep_copy_vars(std::declval<const T_Args&>()))>>...>
-      local_args_tuple_;
-  std::tuple<arena_t<
-      plain_type_t<decltype(value_of(std::declval<const T_Args&>()))>>...>
-      value_of_args_tuple_;
-
-  size_t N_;
-  bool backward_is_initialized_;
-
-  size_t num_args_vars_;
-
-  arena_t<Eigen::VectorXd> state_forward_;
-  arena_t<Eigen::VectorXd> state_backward_;
-  arena_t<Eigen::VectorXd> quad_;
-
-  vari** non_chaining_varis_;
-
-  vari** args_varis_;
-
   int index_backward_;
+  bool backward_is_initialized_{false};
 
   /**
    * Since the CVODES solver manages memory with malloc calls, these resources
@@ -92,24 +81,20 @@ class cvodes_integrator_adjoint_vari : public vari {
    * vari class).
    */
   struct cvodes_solver : public chainable_alloc {
-    const std::decay_t<F> f_;
-    const size_t N_;
-    std::vector<Eigen::VectorXd> y_;
     std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return_;
-    const std::string function_name_str_;
-    const char* function_name_;
-    void* cvodes_mem_;
     N_Vector nv_state_forward_;
     N_Vector nv_state_backward_;
     N_Vector nv_quad_;
     N_Vector nv_absolute_tolerance_forward_;
     N_Vector nv_absolute_tolerance_backward_;
-
     SUNMatrix A_forward_;
     SUNLinearSolver LS_forward_;
-
     SUNMatrix A_backward_;
     SUNLinearSolver LS_backward_;
+    const size_t N_;
+    const char* function_name_;
+    void* cvodes_mem_;
+    const std::decay_t<F> f_;
 
     template <typename FF, typename StateFwd, typename StateBwd, typename Quad,
               typename AbsTolFwd, typename AbsTolBwd>
@@ -119,12 +104,7 @@ class cvodes_integrator_adjoint_vari : public vari {
                   AbsTolFwd& absolute_tolerance_forward,
                   AbsTolBwd& absolute_tolerance_backward)
         : chainable_alloc(),
-          f_(f),
-          N_(N),
-          y_(ts_size),
           y_return_(ts_size),
-          function_name_str_(function_name),
-          function_name_(function_name_str_.c_str()),
           nv_state_forward_(N_VMake_Serial(N, state_forward.data())),
           nv_state_backward_(N_VMake_Serial(N, state_backward.data())),
           nv_quad_(N_VMake_Serial(num_args_vars, quad.data())),
@@ -132,7 +112,6 @@ class cvodes_integrator_adjoint_vari : public vari {
               N_VMake_Serial(N, absolute_tolerance_forward.data())),
           nv_absolute_tolerance_backward_(
               N_VMake_Serial(N, absolute_tolerance_backward.data())),
-
           A_forward_(SUNDenseMatrix(N, N)),
           A_backward_(SUNDenseMatrix(N, N)),
           LS_forward_(
@@ -141,7 +120,10 @@ class cvodes_integrator_adjoint_vari : public vari {
           LS_backward_(
               N == 0 ? nullptr
                      : SUNDenseLinearSolver(nv_state_backward_, A_backward_)),
-          cvodes_mem_(CVodeCreate(solver_forward)) {
+          N_(N),
+          function_name_(function_name),
+          cvodes_mem_(CVodeCreate(solver_forward)),
+          f_(std::forward<FF>(f)) {
       if (cvodes_mem_ == nullptr) {
         throw std::runtime_error("CVodeCreate failed to allocate memory");
       }
@@ -163,7 +145,7 @@ class cvodes_integrator_adjoint_vari : public vari {
       CVodeFree(&cvodes_mem_);
     }
   };
-  cvodes_solver* solver_;
+  cvodes_solver* solver_{nullptr};
 
  public:
   /**
@@ -218,34 +200,37 @@ class cvodes_integrator_adjoint_vari : public vari {
       long int num_steps_between_checkpoints,  // NOLINT(runtime/int)
       int interpolation_polynomial, int solver_forward, int solver_backward,
       std::ostream* msgs, const T_Args&... args)
-      : vari(NOT_A_NUMBER),
+      : vari_base(),
+        local_args_tuple_(to_arena(deep_copy_vars(args))...),
+        value_of_args_tuple_(to_arena(value_of(args))...),
+        y_(ts.size()),
+        ts_(ts.begin(), ts.end()),
         y0_(y0),
+        absolute_tolerance_forward_(absolute_tolerance_forward),
+        absolute_tolerance_backward_(absolute_tolerance_backward),
+        state_forward_(value_of(y0)),
+        state_backward_(y0.size()),
+        num_args_vars_(count_vars(args...)),
+        quad_(num_args_vars_),
         t0_(t0),
-        ts_(ts.begin(), ts.end()),
         relative_tolerance_forward_(relative_tolerance_forward),
-        absolute_tolerance_forward_(absolute_tolerance_forward),
         relative_tolerance_backward_(relative_tolerance_backward),
-        absolute_tolerance_backward_(absolute_tolerance_backward),
         relative_tolerance_quadrature_(relative_tolerance_quadrature),
         absolute_tolerance_quadrature_(absolute_tolerance_quadrature),
         max_num_steps_(max_num_steps),
         num_steps_between_checkpoints_(num_steps_between_checkpoints),
+        N_(y0.size()),
+        msgs_(msgs),
+        args_varis_([&args..., num_vars = this->num_args_vars_]() {
+          vari** vari_mem = ChainableStack::instance_->memalloc_.alloc_array<vari*>(num_vars);
+          save_varis(vari_mem, args...);
+          return vari_mem;
+        }()),
         interpolation_polynomial_(interpolation_polynomial),
         solver_forward_(solver_forward),
         solver_backward_(solver_backward),
-        msgs_(msgs),
-        local_args_tuple_(to_arena(deep_copy_vars(args))...),
-        value_of_args_tuple_(to_arena(value_of(args))...),
-        N_(y0.size()),
         backward_is_initialized_(false),
-        num_args_vars_(count_vars(args...)),
-        state_forward_(value_of(y0)),
-        state_backward_(N_),
-        quad_(num_args_vars_),
-        args_varis_(ChainableStack::instance_->memalloc_.alloc_array<vari*>(
-            num_args_vars_)),
         solver_(nullptr) {
-    save_varis(args_varis_, args...);
 
     check_finite(function_name, "initial state", y0);
     check_finite(function_name, "initial time", t0);
@@ -345,7 +330,7 @@ class cvodes_integrator_adjoint_vari : public vari {
               = CVodeF(solver_->cvodes_mem_, t_final,
                        solver_->nv_state_forward_, &t_init, CV_NORMAL, &ncheck);
 
-          if (error_code == CV_TOO_MUCH_WORK) {
+          if (unlikely(error_code == CV_TOO_MUCH_WORK)) {
             throw_domain_error(solver_->function_name_, "", t_final,
                                "Failed to integrate to next output time (",
                                ") in less than max_num_steps steps");
@@ -371,19 +356,48 @@ class cvodes_integrator_adjoint_vari : public vari {
 
       t_init = t_final;
     }
+    ChainableStack::instance_->var_stack_.push_back(this);
+  }
+
+private:
+  /**
+   * Overloads which setup the states returned from the forward solve. In case
+   * the return type is a double only, then no autodiff is needed. In case of
+   * autodiff then non-chaining varis are setup accordingly.
+   */
+  void store_state(std::size_t n, const Eigen::VectorXd& state,
+                   Eigen::Matrix<var, Eigen::Dynamic, 1>& state_return) {
+    y_[n] = state;
+    state_return.resize(N_);
+    for (size_t i = 0; i < N_; i++) {
+      state_return.coeffRef(i) = var(new vari(state.coeff(i), false));
+    }
+  }
+
+  void store_state(std::size_t n, const Eigen::VectorXd& state,
+                   Eigen::Matrix<double, Eigen::Dynamic, 1>& state_return) {
+    y_[n] = state;
+    state_return = state;
   }
 
+public:
+
   /**
    * Obtain solution of ODE.
    *
    * @return std::vector of Eigen::Matrix of the states of the ODE, one for each
    *   solution time (excluding the initial state)
    */
-  std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>>& solution() noexcept {
+  std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> solution() noexcept {
     return solver_->y_return_;
   }
 
-  virtual void chain() {
+  /**
+   * No-op for setting adjoints since this class does not own any adjoints.
+   */
+  void set_zero_adjoint() final {};
+
+  void chain() final {
     if (!is_var_return_) {
       return;
     }
@@ -398,8 +412,8 @@ class cvodes_integrator_adjoint_vari : public vari {
               += forward_as<var>(solver_->y_return_[i].coeff(j)).adj();
         }
 
-        forward_as<var>(stan::get(ts_, i))->adj_ += step_sens.dot(
-            rhs(value_of(ts_[i]), solver_->y_[i], value_of_args_tuple_));
+        forward_as<var>(ts_[i])->adj_ += step_sens.dot(
+            rhs(value_of(ts_[i]), y_[i], value_of_args_tuple_));
         step_sens.setZero();
       }
 
@@ -424,7 +438,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
       double t_final = value_of((i > 0) ? ts_[i - 1] : t0_);
       if (t_final != t_init) {
-        if (!backward_is_initialized_) {
+        if (unlikely(!backward_is_initialized_)) {
           check_flag_sundials(CVodeCreateB(solver_->cvodes_mem_,
                                            solver_backward_, &index_backward_),
                               "CVodeCreateB");
@@ -503,7 +517,7 @@ class cvodes_integrator_adjoint_vari : public vari {
 
         int error_code = CVodeB(solver_->cvodes_mem_, t_final, CV_NORMAL);
 
-        if (error_code == CV_TOO_MUCH_WORK) {
+        if (unlikely(error_code == CV_TOO_MUCH_WORK)) {
           throw_domain_error(solver_->function_name_, "", t_final,
                              "Failed to integrate backward to output time (",
                              ") in less than max_num_steps steps");
@@ -527,7 +541,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     }
 
     if (is_var_t0_) {
-      forward_as<var>(stan::get(t0_, 0))->adj_ += -state_backward_.dot(
+      forward_as<var>(t0_)->adj_ += -state_backward_.dot(
           rhs(t_init, value_of(y0_), value_of_args_tuple_));
     }
 
@@ -535,9 +549,7 @@ class cvodes_integrator_adjoint_vari : public vari {
     // the adjoints we wanted
     // These are the dlog_density / d(initial_conditions[s]) adjoints
     if (is_var_y0_) {
-      for (size_t i = 0; i < N_; ++i) {
-        forward_as<var>(stan::get(y0_, i))->adj_ += state_backward_.coeff(i);
-      }
+      forward_as<arena_t<Eigen::Matrix<var, Eigen::Dynamic, 1>>>(y0_).adj() += state_backward_;
     }
 
     // These are the dlog_density / d(parameters[s]) adjoints
@@ -568,76 +580,26 @@ class cvodes_integrator_adjoint_vari : public vari {
     return static_cast<cvodes_integrator_adjoint_vari*>(mem);
   }
 
-  /**
-   * Implements the function of type CVRhsFn which is the user-defined
-   * ODE RHS passed to CVODES.
-   */
-  constexpr static int cv_rhs(realtype t, N_Vector y, N_Vector ydot,
-                              void* user_data) {
-    cast_to_self(user_data)->rhs(t, NV_DATA_S(y), NV_DATA_S(ydot));
-    return 0;
-  }
-
-  /**
-   * Implements the function of type CVRhsFnB which is the
-   * RHS of the backward ODE system.
-   */
-  constexpr static int cv_rhs_adj(realtype t, N_Vector y, N_Vector yB,
-                                  N_Vector yBdot, void* user_data) {
-    cast_to_self(user_data)->rhs_adj(t, y, yB, yBdot);
-    return 0;
-  }
-
-  /**
-   * Implements the function of type CVQuadRhsFnB which is the
-   * RHS of the backward ODE system's quadrature.
-   */
-  constexpr static int cv_quad_rhs_adj(realtype t, N_Vector y, N_Vector yB,
-                                       N_Vector qBdot, void* user_data) {
-    cast_to_self(user_data)->quad_rhs_adj(t, y, yB, qBdot);
-    return 0;
-  }
-
-  /**
-   * Implements the function of type CVDlsJacFn which is the
-   * user-defined callback for CVODES to calculate the jacobian of the
-   * ode_rhs wrt to the states y. The jacobian is stored in column
-   * major format.
-   */
-  constexpr static int cv_jacobian_rhs_states(realtype t, N_Vector y,
-                                              N_Vector fy, SUNMatrix J,
-                                              void* user_data, N_Vector tmp1,
-                                              N_Vector tmp2, N_Vector tmp3) {
-    cast_to_self(user_data)->jacobian_rhs_states(t, y, J);
-    return 0;
-  }
-
-  /**
-   * Implements the CVLsJacFnB function for evaluating the jacobian of
-   * the adjoint problem wrt to the backward states.
-   */
-  constexpr static int cv_jacobian_rhs_adj_states(realtype t, N_Vector y,
-                                                  N_Vector yB, N_Vector fyB,
-                                                  SUNMatrix J, void* user_data,
-                                                  N_Vector tmp1, N_Vector tmp2,
-                                                  N_Vector tmp3) {
-    cast_to_self(user_data)->jacobian_rhs_adj_states(t, y, J);
-    return 0;
-  }
-
   /**
    * Calculates the ODE RHS, dy_dt, using the user-supplied functor at
    * the given time t and state y.
    */
-  inline void rhs(double t, const double y[], double dy_dt[]) const {
+  inline int rhs(double t, const double* y, double*& dy_dt) const {
     const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
-
     const Eigen::VectorXd dy_dt_vec = rhs(t, y_vec, value_of_args_tuple_);
-
     check_size_match(solver_->function_name_, "dy_dt", dy_dt_vec.size(),
                      "states", N_);
-
     Eigen::Map<Eigen::VectorXd>(dy_dt, N_) = dy_dt_vec;
+    return 0;
+  }
+
+  /**
+   * Implements the function of type CVRhsFn which is the user-defined
+   * ODE RHS passed to CVODES.
+   */
+  constexpr static int cv_rhs(realtype t, N_Vector y, N_Vector ydot,
+                              void* user_data) {
+    return cast_to_self(user_data)->rhs(t, NV_DATA_S(y), NV_DATA_S(ydot));
   }
 
   /*
@@ -653,27 +615,27 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] yB state of the adjoint ODE system
    * @param[out] yBdot evaluation of adjoint ODE RHS
    */
-  inline void rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
-    Eigen::Map<const Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
-    Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
-    Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(yBdot), N_);
-    mu_dot.setZero();
-
+  inline int rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
     const nested_rev_autodiff nested;
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y_vars(y_vec);
-
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y_vars(Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_));
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vars, value_of_args_tuple_);
-
     check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(),
                      "states", N_);
-
-    f_y_t_vars.adj() = -mu;
-
+    f_y_t_vars.adj() = -Eigen::Map<Eigen::VectorXd>(NV_DATA_S(yB), N_);
     grad();
+    Eigen::Map<Eigen::VectorXd>(NV_DATA_S(yBdot), N_) = y_vars.adj();
+    return 0;
+  }
 
-    mu_dot = y_vars.adj();
+  /**
+   * Implements the function of type CVRhsFnB which is the
+   * RHS of the backward ODE system.
+   */
+  constexpr static int cv_rhs_adj(realtype t, N_Vector y, N_Vector yB,
+                                  N_Vector yBdot, void* user_data) {
+    return cast_to_self(user_data)->rhs_adj(t, y, yB, yBdot);
   }
 
   /*
@@ -689,44 +651,44 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] yB state of the adjoint ODE system
    * @param[out] qBdot evaluation of adjoint ODE quadrature RHS
    */
-  inline void quad_rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector qBdot) {
+  inline int quad_rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector qBdot) {
     Eigen::Map<const Eigen::VectorXd> y_vec(NV_DATA_S(y), N_);
-    Eigen::Map<Eigen::VectorXd> mu(NV_DATA_S(yB), N_);
-    Eigen::Map<Eigen::VectorXd> mu_dot(NV_DATA_S(qBdot), num_args_vars_);
-    mu_dot.setZero();
-
     const nested_rev_autodiff nested;
 
     // The vars here do not live on the nested stack so must be zero'd
     // separately
     stan::math::for_each([](auto&& arg) { zero_adjoints(arg); },
                          local_args_tuple_);
-
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vec, local_args_tuple_);
-
     check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(),
                      "states", N_);
-
-    f_y_t_vars.adj() = -mu;
-
+    f_y_t_vars.adj() = -Eigen::Map<Eigen::VectorXd>(NV_DATA_S(yB), N_);
     grad();
-
-    apply([&](auto&&... args) { accumulate_adjoints(mu_dot.data(), args...); },
+    apply([&qBdot](auto&&... args) { accumulate_adjoints(NV_DATA_S(qBdot), args...); },
           local_args_tuple_);
+    return 0;
+  }
+
+  /**
+   * Implements the function of type CVQuadRhsFnB which is the
+   * RHS of the backward ODE system's quadrature.
+   */
+  constexpr static int cv_quad_rhs_adj(realtype t, N_Vector y, N_Vector yB,
+                                       N_Vector qBdot, void* user_data) {
+    return cast_to_self(user_data)->quad_rhs_adj(t, y, yB, qBdot);
   }
 
   /**
    * Calculates the jacobian of the ODE RHS wrt to its states y at the
    * given time-point t and state y.
    */
-  inline void jacobian_rhs_states(double t, N_Vector y, SUNMatrix J) const {
+  inline int jacobian_rhs_states(double t, N_Vector y, SUNMatrix J) const {
     Eigen::Map<Eigen::MatrixXd> Jfy(SM_DATA_D(J), N_, N_);
-    Eigen::Map<const Eigen::VectorXd> x(NV_DATA_S(y), N_);
 
     nested_rev_autodiff nested;
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y_var(x);
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y_var(Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_));
     Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var
         = rhs(t, y_var, value_of_args_tuple_);
 
@@ -741,6 +703,20 @@ class cvodes_integrator_adjoint_vari : public vari {
       Jfy.col(i) = y_var.adj();
     }
     Jfy.transposeInPlace();
+    return 0;
+  }
+
+  /**
+   * Implements the function of type CVDlsJacFn which is the
+   * user-defined callback for CVODES to calculate the jacobian of the
+   * ode_rhs wrt to the states y. The jacobian is stored in column
+   * major format.
+   */
+  constexpr static int cv_jacobian_rhs_states(realtype t, N_Vector y,
+                                              N_Vector fy, SUNMatrix J,
+                                              void* user_data, N_Vector tmp1,
+                                              N_Vector tmp2, N_Vector tmp3) {
+    return cast_to_self(user_data)->jacobian_rhs_states(t, y, J);
   }
 
   /*
@@ -751,35 +727,30 @@ class cvodes_integrator_adjoint_vari : public vari {
    * @param[in] t Time
    * @param[out] J CVode structure where output is to be stored
    */
-  inline void jacobian_rhs_adj_states(double t, N_Vector y, SUNMatrix J) const {
-    Eigen::Map<Eigen::MatrixXd> J_adj_y(SM_DATA_D(J), N_, N_);
+  inline int jacobian_rhs_adj_states(double t, N_Vector y, SUNMatrix J) const {
 
     // J_adj_y = -1 * transpose(J_y)
-    jacobian_rhs_states(t, y, J);
+    int error_code = jacobian_rhs_states(t, y, J);
 
+    Eigen::Map<Eigen::MatrixXd> J_adj_y(SM_DATA_D(J), N_, N_);
     J_adj_y.transposeInPlace();
     J_adj_y.array() *= -1.0;
+    return error_code;
   }
 
   /**
-   * Overloads which setup the states returned from the forward solve. In case
-   * the return type is a double only, then no autodiff is needed. In case of
-   * autodiff then non-chaining varis are setup accordingly.
+   * Implements the CVLsJacFnB function for evaluating the jacobian of
+   * the adjoint problem wrt to the backward states.
    */
-  void store_state(std::size_t n, const Eigen::VectorXd& state,
-                   Eigen::Matrix<var, Eigen::Dynamic, 1>& state_return) {
-    solver_->y_[n] = state;
-    state_return.resize(N_);
-    for (size_t i = 0; i < N_; i++) {
-      state_return.coeffRef(i) = var(new vari(state.coeff(i), false));
-    }
+  constexpr static int cv_jacobian_rhs_adj_states(realtype t, N_Vector y,
+                                                  N_Vector yB, N_Vector fyB,
+                                                  SUNMatrix J, void* user_data,
+                                                  N_Vector tmp1, N_Vector tmp2,
+                                                  N_Vector tmp3) {
+    return cast_to_self(user_data)->jacobian_rhs_adj_states(t, y, J);
   }
 
-  void store_state(std::size_t n, const Eigen::VectorXd& state,
-                   Eigen::Matrix<double, Eigen::Dynamic, 1>& state_return) {
-    solver_->y_[n] = state;
-    state_return = state;
-  }
+
 };  // cvodes integrator adjoint vari
 
 }  // namespace math
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 0aba04f1217..e83003794ec 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -59,18 +59,19 @@ namespace math {
  * @param solver_backward solver used for backward pass
  * @param[in, out] msgs the print stream for warning messages
  * @param args Extra arguments passed unmodified through to ODE right hand side
- * @return Solution to ODE at times \p ts
+ * @return An `std::vector` of Eigen column vectors with scalars equal to
+ *  the least upper bound of `T_y0`, `T_t0`, `T_ts`, and the lambda's arguments.
+ *  This represents the solution to ODE at times \p ts
  */
 template <typename F, typename T_y0, typename T_t0, typename T_ts,
-          typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
-std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
-                          Eigen::Dynamic, 1>>
-ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
+typename T_abs_tol_fwd, typename T_abs_tol_bwd, typename... T_Args,
+require_all_eigen_col_vector_t<T_y0, T_abs_tol_fwd, T_abs_tol_bwd>* = nullptr>
+auto ode_adjoint_impl(const char* function_name, F&& f, const T_y0& y0,
                  const T_t0& t0, const std::vector<T_ts>& ts,
                  double relative_tolerance_forward,
-                 const Eigen::VectorXd& absolute_tolerance_forward,
+                 const T_abs_tol_fwd& absolute_tolerance_forward,
                  double relative_tolerance_backward,
-                 const Eigen::VectorXd& absolute_tolerance_backward,
+                 const T_abs_tol_bwd& absolute_tolerance_backward,
                  double relative_tolerance_quadrature,
                  double absolute_tolerance_quadrature,
                  long int max_num_steps,                  // NOLINT(runtime/int)
@@ -78,14 +79,15 @@ ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
                  int interpolation_polynomial, int solver_forward,
                  int solver_backward, std::ostream* msgs,
                  const T_Args&... args) {
-  auto integrator = new stan::math::cvodes_integrator_adjoint_vari<
-      F, plain_type_t<T_y0>, T_t0, T_ts, ref_type_t<T_Args>...>(
-      function_name, f, eval(y0), t0, ts, relative_tolerance_forward,
+  using integrator_vari = cvodes_integrator_adjoint_vari<
+      F, plain_type_t<T_y0>, T_t0, T_ts, plain_type_t<T_Args>...>;
+  auto integrator = new integrator_vari(
+      function_name, std::forward<F>(f), y0, t0, ts, relative_tolerance_forward,
       absolute_tolerance_forward, relative_tolerance_backward,
       absolute_tolerance_backward, relative_tolerance_quadrature,
       absolute_tolerance_quadrature, max_num_steps,
       num_steps_between_checkpoints, interpolation_polynomial, solver_forward,
-      solver_backward, msgs, to_ref(args)...);
+      solver_backward, msgs, args...);
   return integrator->solution();
 }
 
@@ -137,25 +139,25 @@ ode_adjoint_impl(const char* function_name, const F& f, const T_y0& y0,
  * @param solver_backward solver used for backward pass
  * @param[in, out] msgs the print stream for warning messages
  * @param args Extra arguments passed unmodified through to ODE right hand side
- * @return Solution to ODE at times \p ts
+ * @return An `std::vector` of Eigen column vectors with scalars equal to
+ *  the least upper bound of `T_y0`, `T_t0`, `T_ts`, and the lambda's arguments.
+ *  This represents the solution to ODE at times \p ts
  */
 template <typename F, typename T_y0, typename T_t0, typename T_ts,
-          typename... T_Args, require_eigen_col_vector_t<T_y0>* = nullptr>
-std::vector<Eigen::Matrix<stan::return_type_t<T_y0, T_t0, T_ts, T_Args...>,
-                          Eigen::Dynamic, 1>>
-ode_adjoint_tol_ctl(
-    const F& f, const T_y0& y0, const T_t0& t0, const std::vector<T_ts>& ts,
+ typename T_abs_tol_fwd, typename T_abs_tol_bwd, typename... T_Args,
+ require_all_eigen_col_vector_t<T_y0, T_abs_tol_fwd, T_abs_tol_bwd>* = nullptr>
+auto ode_adjoint_tol_ctl(F&& f, const T_y0& y0, const T_t0& t0, const std::vector<T_ts>& ts,
     double relative_tolerance_forward,
-    const Eigen::VectorXd& absolute_tolerance_forward,
+    const T_abs_tol_fwd& absolute_tolerance_forward,
     double relative_tolerance_backward,
-    const Eigen::VectorXd& absolute_tolerance_backward,
+    const T_abs_tol_bwd& absolute_tolerance_backward,
     double relative_tolerance_quadrature, double absolute_tolerance_quadrature,
     long int max_num_steps,                  // NOLINT(runtime/int)
     long int num_steps_between_checkpoints,  // NOLINT(runtime/int)
     int interpolation_polynomial, int solver_forward, int solver_backward,
     std::ostream* msgs, const T_Args&... args) {
   return ode_adjoint_impl(
-      "ode_adjoint_tol_ctl", f, y0, t0, ts, relative_tolerance_forward,
+      "ode_adjoint_tol_ctl", std::forward<F>(f), y0, t0, ts, relative_tolerance_forward,
       absolute_tolerance_forward, relative_tolerance_backward,
       absolute_tolerance_backward, relative_tolerance_quadrature,
       absolute_tolerance_quadrature, max_num_steps,
diff --git a/test/unit/math/rev/functor/sho_ode_typed_test.cpp b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
index 6056bd3579a..207e070c9da 100644
--- a/test/unit/math/rev/functor/sho_ode_typed_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
@@ -18,9 +18,7 @@ using ode_test_tuple = std::tuple<solve_type, solve_type, Ts...>;
  */
 using harmonic_oscillator_fd_test_types = boost::mp11::mp_product<
     ode_test_tuple,
-    ::testing::Types<  // ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
-                       // ode_rk45_functor,
-        ode_adjoint_functor>,
+    ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor, ode_rk45_functor, ode_adjoint_functor>,
     ::testing::Types<double>,  // t
     ::testing::Types<double>,  // y0
     ::testing::Types<double>   // theta
@@ -90,9 +88,8 @@ INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_data_test,
 
 using harmonic_oscillator_test_types = boost::mp11::mp_product<
     ode_test_tuple,
-    ::testing::Types<  // ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
-                       // ode_rk45_functor,
-        ode_adjoint_functor>,
+    ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
+                     ode_rk45_functor, ode_adjoint_functor>,
     ::testing::Types<double>,                                  // t
     ::testing::Types<double, stan::math::var_value<double> >,  // y0
     ::testing::Types<double, stan::math::var_value<double> >   // theta

From 0b7324794dbe2fa71f6b96c1a03aaa97d49b8061 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Sun, 9 May 2021 20:06:03 +0000
Subject: [PATCH 135/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/coupled_ode_system.hpp  |  3 +-
 .../rev/functor/cvodes_integrator_adjoint.hpp | 33 +++++++-----
 stan/math/rev/functor/ode_adjoint.hpp         | 54 ++++++++++---------
 .../math/rev/functor/sho_ode_typed_test.cpp   |  3 +-
 4 files changed, 51 insertions(+), 42 deletions(-)

diff --git a/stan/math/rev/functor/coupled_ode_system.hpp b/stan/math/rev/functor/coupled_ode_system.hpp
index 264845eaf4f..1f043fe0e9a 100644
--- a/stan/math/rev/functor/coupled_ode_system.hpp
+++ b/stan/math/rev/functor/coupled_ode_system.hpp
@@ -139,7 +139,8 @@ struct coupled_ode_system_impl<false, F, T_y0, Args...> {
       y_adjoints_ = y_vars.adj();
 
       if (args_adjoints_.size() > 0) {
-        memset(args_adjoints_.data(), 0, sizeof(double) * args_adjoints_.size());
+        memset(args_adjoints_.data(), 0,
+               sizeof(double) * args_adjoints_.size());
       }
 
       apply(
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index a16ec4c2c1e..a45f72940c2 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -46,7 +46,8 @@ class cvodes_integrator_adjoint_vari : public vari_base {
       is_var_ts_ && !(is_var_t0_ || is_var_y0_ || is_any_var_args_)};
 
   std::tuple<arena_t<T_Args>...> local_args_tuple_;
-  std::tuple<arena_t<promote_scalar_t<partials_type_t<scalar_type_t<T_Args>>, T_Args>>...>
+  std::tuple<arena_t<
+      promote_scalar_t<partials_type_t<scalar_type_t<T_Args>>, T_Args>>...>
       value_of_args_tuple_;
 
   arena_t<std::vector<Eigen::VectorXd>> y_;
@@ -222,7 +223,9 @@ class cvodes_integrator_adjoint_vari : public vari_base {
         N_(y0.size()),
         msgs_(msgs),
         args_varis_([&args..., num_vars = this->num_args_vars_]() {
-          vari** vari_mem = ChainableStack::instance_->memalloc_.alloc_array<vari*>(num_vars);
+          vari** vari_mem
+              = ChainableStack::instance_->memalloc_.alloc_array<vari*>(
+                  num_vars);
           save_varis(vari_mem, args...);
           return vari_mem;
         }()),
@@ -231,7 +234,6 @@ class cvodes_integrator_adjoint_vari : public vari_base {
         solver_backward_(solver_backward),
         backward_is_initialized_(false),
         solver_(nullptr) {
-
     check_finite(function_name, "initial state", y0);
     check_finite(function_name, "initial time", t0);
     check_finite(function_name, "times", ts);
@@ -359,7 +361,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     ChainableStack::instance_->var_stack_.push_back(this);
   }
 
-private:
+ private:
   /**
    * Overloads which setup the states returned from the forward solve. In case
    * the return type is a double only, then no autodiff is needed. In case of
@@ -380,8 +382,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     state_return = state;
   }
 
-public:
-
+ public:
   /**
    * Obtain solution of ODE.
    *
@@ -395,7 +396,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
   /**
    * No-op for setting adjoints since this class does not own any adjoints.
    */
-  void set_zero_adjoint() final {};
+  void set_zero_adjoint() final{};
 
   void chain() final {
     if (!is_var_return_) {
@@ -549,7 +550,8 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     // the adjoints we wanted
     // These are the dlog_density / d(initial_conditions[s]) adjoints
     if (is_var_y0_) {
-      forward_as<arena_t<Eigen::Matrix<var, Eigen::Dynamic, 1>>>(y0_).adj() += state_backward_;
+      forward_as<arena_t<Eigen::Matrix<var, Eigen::Dynamic, 1>>>(y0_).adj()
+          += state_backward_;
     }
 
     // These are the dlog_density / d(parameters[s]) adjoints
@@ -618,7 +620,8 @@ class cvodes_integrator_adjoint_vari : public vari_base {
   inline int rhs_adj(double t, N_Vector y, N_Vector yB, N_Vector yBdot) const {
     const nested_rev_autodiff nested;
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y_vars(Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_));
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y_vars(
+        Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_));
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vars, value_of_args_tuple_);
     check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(),
@@ -665,8 +668,11 @@ class cvodes_integrator_adjoint_vari : public vari_base {
                      "states", N_);
     f_y_t_vars.adj() = -Eigen::Map<Eigen::VectorXd>(NV_DATA_S(yB), N_);
     grad();
-    apply([&qBdot](auto&&... args) { accumulate_adjoints(NV_DATA_S(qBdot), args...); },
-          local_args_tuple_);
+    apply(
+        [&qBdot](auto&&... args) {
+          accumulate_adjoints(NV_DATA_S(qBdot), args...);
+        },
+        local_args_tuple_);
     return 0;
   }
 
@@ -688,7 +694,8 @@ class cvodes_integrator_adjoint_vari : public vari_base {
 
     nested_rev_autodiff nested;
 
-    Eigen::Matrix<var, Eigen::Dynamic, 1> y_var(Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_));
+    Eigen::Matrix<var, Eigen::Dynamic, 1> y_var(
+        Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_));
     Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var
         = rhs(t, y_var, value_of_args_tuple_);
 
@@ -728,7 +735,6 @@ class cvodes_integrator_adjoint_vari : public vari_base {
    * @param[out] J CVode structure where output is to be stored
    */
   inline int jacobian_rhs_adj_states(double t, N_Vector y, SUNMatrix J) const {
-
     // J_adj_y = -1 * transpose(J_y)
     int error_code = jacobian_rhs_states(t, y, J);
 
@@ -750,7 +756,6 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     return cast_to_self(user_data)->jacobian_rhs_adj_states(t, y, J);
   }
 
-
 };  // cvodes integrator adjoint vari
 
 }  // namespace math
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index e83003794ec..e2b700c1c2f 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -64,23 +64,23 @@ namespace math {
  *  This represents the solution to ODE at times \p ts
  */
 template <typename F, typename T_y0, typename T_t0, typename T_ts,
-typename T_abs_tol_fwd, typename T_abs_tol_bwd, typename... T_Args,
-require_all_eigen_col_vector_t<T_y0, T_abs_tol_fwd, T_abs_tol_bwd>* = nullptr>
-auto ode_adjoint_impl(const char* function_name, F&& f, const T_y0& y0,
-                 const T_t0& t0, const std::vector<T_ts>& ts,
-                 double relative_tolerance_forward,
-                 const T_abs_tol_fwd& absolute_tolerance_forward,
-                 double relative_tolerance_backward,
-                 const T_abs_tol_bwd& absolute_tolerance_backward,
-                 double relative_tolerance_quadrature,
-                 double absolute_tolerance_quadrature,
-                 long int max_num_steps,                  // NOLINT(runtime/int)
-                 long int num_steps_between_checkpoints,  // NOLINT(runtime/int)
-                 int interpolation_polynomial, int solver_forward,
-                 int solver_backward, std::ostream* msgs,
-                 const T_Args&... args) {
-  using integrator_vari = cvodes_integrator_adjoint_vari<
-      F, plain_type_t<T_y0>, T_t0, T_ts, plain_type_t<T_Args>...>;
+          typename T_abs_tol_fwd, typename T_abs_tol_bwd, typename... T_Args,
+          require_all_eigen_col_vector_t<T_y0, T_abs_tol_fwd,
+                                         T_abs_tol_bwd>* = nullptr>
+auto ode_adjoint_impl(
+    const char* function_name, F&& f, const T_y0& y0, const T_t0& t0,
+    const std::vector<T_ts>& ts, double relative_tolerance_forward,
+    const T_abs_tol_fwd& absolute_tolerance_forward,
+    double relative_tolerance_backward,
+    const T_abs_tol_bwd& absolute_tolerance_backward,
+    double relative_tolerance_quadrature, double absolute_tolerance_quadrature,
+    long int max_num_steps,                  // NOLINT(runtime/int)
+    long int num_steps_between_checkpoints,  // NOLINT(runtime/int)
+    int interpolation_polynomial, int solver_forward, int solver_backward,
+    std::ostream* msgs, const T_Args&... args) {
+  using integrator_vari
+      = cvodes_integrator_adjoint_vari<F, plain_type_t<T_y0>, T_t0, T_ts,
+                                       plain_type_t<T_Args>...>;
   auto integrator = new integrator_vari(
       function_name, std::forward<F>(f), y0, t0, ts, relative_tolerance_forward,
       absolute_tolerance_forward, relative_tolerance_backward,
@@ -144,9 +144,11 @@ auto ode_adjoint_impl(const char* function_name, F&& f, const T_y0& y0,
  *  This represents the solution to ODE at times \p ts
  */
 template <typename F, typename T_y0, typename T_t0, typename T_ts,
- typename T_abs_tol_fwd, typename T_abs_tol_bwd, typename... T_Args,
- require_all_eigen_col_vector_t<T_y0, T_abs_tol_fwd, T_abs_tol_bwd>* = nullptr>
-auto ode_adjoint_tol_ctl(F&& f, const T_y0& y0, const T_t0& t0, const std::vector<T_ts>& ts,
+          typename T_abs_tol_fwd, typename T_abs_tol_bwd, typename... T_Args,
+          require_all_eigen_col_vector_t<T_y0, T_abs_tol_fwd,
+                                         T_abs_tol_bwd>* = nullptr>
+auto ode_adjoint_tol_ctl(
+    F&& f, const T_y0& y0, const T_t0& t0, const std::vector<T_ts>& ts,
     double relative_tolerance_forward,
     const T_abs_tol_fwd& absolute_tolerance_forward,
     double relative_tolerance_backward,
@@ -157,12 +159,12 @@ auto ode_adjoint_tol_ctl(F&& f, const T_y0& y0, const T_t0& t0, const std::vecto
     int interpolation_polynomial, int solver_forward, int solver_backward,
     std::ostream* msgs, const T_Args&... args) {
   return ode_adjoint_impl(
-      "ode_adjoint_tol_ctl", std::forward<F>(f), y0, t0, ts, relative_tolerance_forward,
-      absolute_tolerance_forward, relative_tolerance_backward,
-      absolute_tolerance_backward, relative_tolerance_quadrature,
-      absolute_tolerance_quadrature, max_num_steps,
-      num_steps_between_checkpoints, interpolation_polynomial, solver_forward,
-      solver_backward, msgs, args...);
+      "ode_adjoint_tol_ctl", std::forward<F>(f), y0, t0, ts,
+      relative_tolerance_forward, absolute_tolerance_forward,
+      relative_tolerance_backward, absolute_tolerance_backward,
+      relative_tolerance_quadrature, absolute_tolerance_quadrature,
+      max_num_steps, num_steps_between_checkpoints, interpolation_polynomial,
+      solver_forward, solver_backward, msgs, args...);
 }
 
 }  // namespace math
diff --git a/test/unit/math/rev/functor/sho_ode_typed_test.cpp b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
index 207e070c9da..9d8b56f4855 100644
--- a/test/unit/math/rev/functor/sho_ode_typed_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_typed_test.cpp
@@ -18,7 +18,8 @@ using ode_test_tuple = std::tuple<solve_type, solve_type, Ts...>;
  */
 using harmonic_oscillator_fd_test_types = boost::mp11::mp_product<
     ode_test_tuple,
-    ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor, ode_rk45_functor, ode_adjoint_functor>,
+    ::testing::Types<ode_adams_functor, ode_bdf_functor, ode_ckrk_functor,
+                     ode_rk45_functor, ode_adjoint_functor>,
     ::testing::Types<double>,  // t
     ::testing::Types<double>,  // y0
     ::testing::Types<double>   // theta

From 8612ac606318ae4461b5a2baa82fa15c6d599132 Mon Sep 17 00:00:00 2001
From: stevebronder <stevo15025@gmail.com>
Date: Sun, 9 May 2021 16:28:33 -0400
Subject: [PATCH 136/157] remove extra whitespace for cpplint

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index a45f72940c2..9f7c1f49dd2 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -755,7 +755,6 @@ class cvodes_integrator_adjoint_vari : public vari_base {
                                                   N_Vector tmp3) {
     return cast_to_self(user_data)->jacobian_rhs_adj_states(t, y, J);
   }
-
 };  // cvodes integrator adjoint vari
 
 }  // namespace math

From 68c3f88cd4cda66f22c55be7e6f7c3c219780c3d Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 10 May 2021 10:52:57 +0200
Subject: [PATCH 137/157] address review comments (split tests, test for no
 vari on stack when double only case)

---
 make/tests                                    |   2 +-
 stan/math/rev/functor/cvodes_integrator.hpp   |   6 +-
 .../rev/functor/cvodes_integrator_adjoint.hpp |   3 +-
 stan/math/rev/functor/cvodes_utils.hpp        |   5 +-
 .../sho_ode_adjoint_typed_error_test.cpp      |  51 +++
 .../rev/functor/sho_ode_typed_error_test.cpp  |  24 --
 .../math/rev/functor/test_fixture_ode_sho.hpp | 293 -----------------
 .../rev/functor/test_fixture_ode_sho_ctl.hpp  | 311 ++++++++++++++++++
 8 files changed, 369 insertions(+), 326 deletions(-)
 create mode 100644 test/unit/math/rev/functor/sho_ode_adjoint_typed_error_test.cpp
 create mode 100644 test/unit/math/rev/functor/test_fixture_ode_sho_ctl.hpp

diff --git a/make/tests b/make/tests
index 30b195a3870..6a90618d62f 100644
--- a/make/tests
+++ b/make/tests
@@ -64,7 +64,7 @@ $(EXPRESSION_TESTS) : $(LIBSUNDIALS)
 # CVODES tests
 ##
 
-CVODES_TESTS := $(subst .cpp,$(EXE),$(call findfiles,test,*cvodes*_test.cpp) $(call findfiles,test,*_bdf_*_test.cpp) $(call findfiles,test,*_adams_*_test.cpp) $(call findfiles,test,*_ode_typed_*test.cpp))
+CVODES_TESTS := $(subst .cpp,$(EXE),$(call findfiles,test,*cvodes*_test.cpp) $(call findfiles,test,*_bdf_*_test.cpp) $(call findfiles,test,*_adams_*_test.cpp) $(call findfiles,test,*_ode_typed_*test.cpp) $(call findfiles,test,*_ode_adjoint_typed_*test.cpp))
 $(CVODES_TESTS) : $(LIBSUNDIALS)
 
 
diff --git a/stan/math/rev/functor/cvodes_integrator.hpp b/stan/math/rev/functor/cvodes_integrator.hpp
index 4e40354540b..ce6d8f960a4 100644
--- a/stan/math/rev/functor/cvodes_integrator.hpp
+++ b/stan/math/rev/functor/cvodes_integrator.hpp
@@ -279,8 +279,10 @@ class cvodes_integrator {
           CVodeSetUserData(cvodes_mem, reinterpret_cast<void*>(this)),
           "CVodeSetUserData");
 
-      cvodes_set_options(cvodes_mem, relative_tolerance_, absolute_tolerance_,
-                         max_num_steps_);
+      cvodes_set_options(cvodes_mem, max_num_steps_);
+
+      check_flag_sundials(CVodeSStolerances(cvodes_mem, relative_tolerance_, absolute_tolerance_),
+                          "CVodeSStolerances");
 
       check_flag_sundials(CVodeSetLinearSolver(cvodes_mem, LS_, A_),
                           "CVodeSetLinearSolver");
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 9f7c1f49dd2..d91aa336880 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -289,8 +289,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
         CVodeSetUserData(solver_->cvodes_mem_, reinterpret_cast<void*>(this)),
         "CVodeSetUserData");
 
-    cvodes_set_options(solver_->cvodes_mem_, relative_tolerance_forward_,
-                       absolute_tolerance_forward_(0), max_num_steps_);
+    cvodes_set_options(solver_->cvodes_mem_, max_num_steps_);
 
     check_flag_sundials(
         CVodeSVtolerances(solver_->cvodes_mem_, relative_tolerance_forward_,
diff --git a/stan/math/rev/functor/cvodes_utils.hpp b/stan/math/rev/functor/cvodes_utils.hpp
index d747f223959..8ab960b6519 100644
--- a/stan/math/rev/functor/cvodes_utils.hpp
+++ b/stan/math/rev/functor/cvodes_utils.hpp
@@ -21,16 +21,13 @@ extern "C" inline void cvodes_err_handler(int error_code, const char* module,
   }
 }
 
-inline void cvodes_set_options(void* cvodes_mem, double rel_tol, double abs_tol,
+inline void cvodes_set_options(void* cvodes_mem,
                                // NOLINTNEXTLINE(runtime/int)
                                long int max_num_steps) {
   // forward CVode errors to noop error handler
   CVodeSetErrHandlerFn(cvodes_mem, cvodes_err_handler, nullptr);
 
   // Initialize solver parameters
-  check_flag_sundials(CVodeSStolerances(cvodes_mem, rel_tol, abs_tol),
-                      "CVodeSStolerances");
-
   check_flag_sundials(CVodeSetMaxNumSteps(cvodes_mem, max_num_steps),
                       "CVodeSetMaxNumSteps");
 
diff --git a/test/unit/math/rev/functor/sho_ode_adjoint_typed_error_test.cpp b/test/unit/math/rev/functor/sho_ode_adjoint_typed_error_test.cpp
new file mode 100644
index 00000000000..f1e4da5892d
--- /dev/null
+++ b/test/unit/math/rev/functor/sho_ode_adjoint_typed_error_test.cpp
@@ -0,0 +1,51 @@
+#include <stan/math/rev.hpp>
+#include <boost/mp11.hpp>
+#include <gtest/gtest.h>
+#include <test/unit/math/rev/functor/test_fixture_ode.hpp>
+#include <test/unit/math/rev/functor/test_fixture_ode_sho_ctl.hpp>
+#include <test/unit/math/rev/functor/ode_test_functors.hpp>
+
+/**
+ *
+ * Use same solver functor type for both w & w/o tolerance control
+ */
+template <typename solve_type, typename... Ts>
+using ode_test_tuple = std::tuple<solve_type, solve_type, Ts...>;
+
+/**
+ * Outer product of test types
+ */
+using harmonic_oscillator_ctl_test_types = boost::mp11::mp_product<
+    ode_test_tuple, ::testing::Types<ode_adjoint_functor>,
+    ::testing::Types<double, stan::math::var_value<double>>,  // t
+    ::testing::Types<double, stan::math::var_value<double>>,  // y0
+    ::testing::Types<double, stan::math::var_value<double>>   // theta
+    >;
+
+TYPED_TEST_SUITE_P(harmonic_oscillator_ctl_test);
+TYPED_TEST_P(harmonic_oscillator_ctl_test, no_error) { this->test_good(); }
+TYPED_TEST_P(harmonic_oscillator_ctl_test, error_conditions) {
+  this->test_bad(); 
+}
+TYPED_TEST_P(harmonic_oscillator_ctl_test, value) {
+  stan::math::nested_rev_autodiff nested;
+  
+  this->test_value(0.0);
+  this->test_value(1.0);
+  this->test_value(-1.0);
+
+  if(std::is_same<std::tuple_element_t<2, TypeParam>, double>::value &&
+     std::is_same<std::tuple_element_t<3, TypeParam>, double>::value &&
+     std::is_same<std::tuple_element_t<4, TypeParam>, double>::value
+     ) {
+    std::cout << "testing all doubles!!!" << std::endl;
+    EXPECT_EQ(stan::math::nested_size(), 0);
+  } else {
+    EXPECT_GT(stan::math::nested_size(), 0);
+  }
+}
+
+REGISTER_TYPED_TEST_SUITE_P(harmonic_oscillator_ctl_test, no_error,
+                            error_conditions, value);
+INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_ctl_test,
+                               harmonic_oscillator_ctl_test_types);
diff --git a/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp b/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
index 2c284571ac0..0ca125c70c0 100644
--- a/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
@@ -68,27 +68,3 @@ REGISTER_TYPED_TEST_SUITE_P(harmonic_oscillator_bad_ode_test, bad_ode_error);
 INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_bad_ode_test,
                                harmonic_oscillator_integrate_ode_test_types);
 
-/**
- * Outer product of test types
- */
-using harmonic_oscillator_ctl_test_types = boost::mp11::mp_product<
-    ode_test_tuple, ::testing::Types<ode_adjoint_functor>,
-    ::testing::Types<double, stan::math::var_value<double>>,  // t
-    ::testing::Types<double, stan::math::var_value<double>>,  // y0
-    ::testing::Types<double, stan::math::var_value<double>>   // theta
-    >;
-
-TYPED_TEST_SUITE_P(harmonic_oscillator_ctl_test);
-TYPED_TEST_P(harmonic_oscillator_ctl_test, no_error) { this->test_good(); }
-TYPED_TEST_P(harmonic_oscillator_ctl_test, error_conditions) {
-  this->test_bad();
-}
-TYPED_TEST_P(harmonic_oscillator_ctl_test, value) {
-  this->test_value(0.0);
-  this->test_value(1.0);
-  this->test_value(-1.0);
-}
-REGISTER_TYPED_TEST_SUITE_P(harmonic_oscillator_ctl_test, no_error,
-                            error_conditions, value);
-INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_ctl_test,
-                               harmonic_oscillator_ctl_test_types);
diff --git a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
index ea61a994fd6..560d3d375c6 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_sho.hpp
@@ -500,297 +500,4 @@ struct harmonic_oscillator_analytical_test
   }
 };
 
-/**
- * Inheriting base type, various fixtures differs by the type of ODE
- * functor used in <code>apply_solver</code> calls, intended for
- * different kind of tests.
- *
- */
-template <typename T>
-struct harmonic_oscillator_ctl_test
-    : public harmonic_oscillator_ode_base<T>,
-      public ODETestFixture<harmonic_oscillator_test<T>> {
-  double rtol_f;
-  double rtol_b;
-  double rtol_q;
-  Eigen::VectorXd atol_f;
-  Eigen::VectorXd atol_b;
-  double atol_q;
-  int num_steps_check;
-  int inter_poly;
-  int solv_f;
-  int solv_b;
-
-  harmonic_oscillator_ctl_test()
-      : harmonic_oscillator_ode_base<T>(),
-        rtol_f(this->rtol / 8.0),
-        rtol_b(this->rtol / 4.0),
-        rtol_q(this->rtol),
-        atol_f(Eigen::VectorXd::Constant(this->y0.size(), this->atol / 6.0)),
-        atol_b(Eigen::VectorXd::Constant(this->y0.size(), this->atol / 3.0)),
-        atol_q(this->atol),
-        num_steps_check(100),
-        inter_poly(CV_HERMITE),
-        solv_f(CV_BDF),
-        solv_b(CV_ADAMS) {}
-
-  auto apply_solver() {
-    std::tuple_element_t<0, T> sol;
-    return sol(this->f_eigen, this->y0, this->t0, this->ts, nullptr,
-               this->theta, this->x_r, this->x_i);
-  }
-
-  template <typename T1, typename T2>
-  auto apply_solver(T1&& init, T2&& theta_in) {
-    std::tuple_element_t<0, T> sol;
-    return sol(this->f_eigen, init, this->t0, this->ts, nullptr, theta_in,
-               this->x_r, this->x_i);
-  }
-
-  auto apply_solver_tol() {
-    std::tuple_element_t<1, T> sol;
-    return sol(this->f_eigen, this->y0, this->t0, this->ts, this->rtol,
-               this->atol, this->max_num_step, nullptr, this->theta, this->x_r,
-               this->x_i);
-  }
-
-  auto apply_solver_tol_ctl() {
-    std::tuple_element_t<0, T> sol;
-    return sol(this->f_eigen, this->y0, this->t0, this->ts, this->rtol_f,
-               this->atol_f, this->rtol_b, this->atol_b, this->rtol_q,
-               this->atol_q, this->max_num_step, this->num_steps_check,
-               this->inter_poly, this->solv_f, this->solv_b, nullptr,
-               this->theta, this->x_r, this->x_i);
-  }
-
-  void test_bad() {
-    const auto y0_(this->y0);
-
-    this->y0.resize(0);
-    EXPECT_THROW_MSG(apply_solver(), std::invalid_argument,
-                     "initial state has size 0");
-    this->y0 = y0_;
-
-    const auto t0_ = this->t0;
-    this->t0 = 2.0;
-    EXPECT_THROW_MSG(apply_solver(), std::domain_error,
-                     "initial time is 2, but must be less than 0.1");
-    this->t0 = t0_;
-
-    const auto ts_ = this->ts;
-    this->ts.resize(0);
-    EXPECT_THROW_MSG(apply_solver(), std::invalid_argument, "times has size 0");
-    this->ts = ts_;
-
-    this->ts.resize(2);
-    this->ts[0] = 3.0;
-    this->ts[1] = 1.0;
-    EXPECT_THROW_MSG(apply_solver(), std::domain_error,
-                     "times is not a valid sorted vector");
-    this->ts = ts_;
-
-    const double rtol_f_ = this->rtol_f;
-    this->rtol_f = -1;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
-                     "relative_tolerance_forward");
-    this->rtol_f = rtol_f_;
-
-    const double atol_f_ = this->atol_f(0);
-    this->atol_f(0) = -1;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
-                     "absolute_tolerance_forward");
-    this->atol_f(0) = atol_f_;
-
-    this->atol_f.resize(1);
-    this->atol_f(0) = atol_f_;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::invalid_argument,
-                     "absolute_tolerance_forward");
-    this->atol_f.resize(2);
-    this->atol_f(0) = atol_f_;
-    this->atol_f(1) = atol_f_;
-
-    const double rtol_b_ = this->rtol_b;
-    this->rtol_b = -1;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
-                     "relative_tolerance_backward");
-    this->rtol_b = rtol_b_;
-
-    const double atol_b_ = this->atol_b(0);
-    this->atol_b(0) = -1;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
-                     "absolute_tolerance_backward");
-    this->atol_b(0) = atol_b_;
-
-    this->atol_b.resize(1);
-    this->atol_b(0) = atol_b_;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::invalid_argument,
-                     "absolute_tolerance_backward");
-    this->atol_b.resize(2);
-    this->atol_b(0) = atol_b_;
-    this->atol_b(1) = atol_b_;
-
-    const double rtol_q_ = this->rtol_q;
-    this->rtol_q = -1;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
-                     "relative_tolerance_quadrature");
-    this->rtol_q = rtol_q_;
-
-    const double atol_q_ = this->atol_q;
-    this->atol_q = -1;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
-                     "absolute_tolerance_quadrature");
-    this->atol_q = atol_q_;
-
-    const int max_num_step_ = this->max_num_step;
-    this->max_num_step = -1;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
-                     "max_num_steps");
-    this->max_num_step = max_num_step_;
-
-    const int num_steps_check_ = this->num_steps_check;
-    this->num_steps_check = -1;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
-                     "num_steps_between_checkpoints");
-    this->num_steps_check = num_steps_check_;
-
-    const int inter_poly_ = this->inter_poly;
-    this->inter_poly = 0;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
-                     "interpolation_polynomial");
-    this->inter_poly = inter_poly_;
-
-    const int solv_f_ = this->solv_f;
-    this->solv_f = 0;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
-                     "solver_forward");
-    this->solv_f = solv_f_;
-
-    const int solv_b_ = this->solv_b;
-    this->solv_b = 0;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
-                     "solver_backward");
-    this->solv_b = solv_b_;
-
-    const auto theta_ = this->theta;
-    const auto x_r_ = this->x_r;
-    const auto x_i_ = this->x_i;
-
-    // NaN errors
-    double nan = std::numeric_limits<double>::quiet_NaN();
-    std::stringstream expected_is_nan;
-    expected_is_nan << "is " << nan;
-
-    this->y0[0] = nan;
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial state");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_nan.str());
-    this->y0 = y0_;
-
-    this->t0 = nan;
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial time");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_nan.str());
-    this->t0 = t0_;
-
-    this->ts[0] = nan;
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "times");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_nan.str());
-    this->ts = ts_;
-
-    this->theta[0] = nan;
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     "ode parameters and data");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_nan.str());
-    this->theta = theta_;
-
-    this->x_r.push_back(nan);
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     "ode parameters and data");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_nan.str());
-    this->x_r = x_r_;
-
-    // inf test
-    std::stringstream expected_is_inf;
-    expected_is_inf << "is " << std::numeric_limits<double>::infinity();
-    std::stringstream expected_is_neg_inf;
-    expected_is_neg_inf << "is " << -std::numeric_limits<double>::infinity();
-    double inf = std::numeric_limits<double>::infinity();
-
-    this->y0[0] = inf;
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial state");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_inf.str());
-    this->y0[0] = -inf;
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial state");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_neg_inf.str());
-    this->y0 = y0_;
-
-    this->t0 = inf;
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial time");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_inf.str());
-    this->t0 = -inf;
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial time");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_neg_inf.str());
-    this->t0 = t0_;
-
-    this->ts.back() = inf;
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "times");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_inf.str());
-    this->ts.back() = -inf;
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "times");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_neg_inf.str());
-    this->ts = ts_;
-
-    this->theta[0] = inf;
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     "ode parameters and data");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_inf.str());
-    this->theta[0] = -inf;
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     "ode parameters and data");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_neg_inf.str());
-    this->theta = theta_;
-
-    this->x_r = std::vector<double>{inf};
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     "ode parameters and data");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_inf.str());
-    this->x_r[0] = -inf;
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     "ode parameters and data");
-    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
-                     expected_is_neg_inf.str());
-    this->x_r = x_r_;
-  }
-
-  void test_value(double t0_in) {
-    this->t0 = t0_in;
-    for (size_t i = 0; i < this->ts.size(); ++i) {
-      this->ts[i] = this->t0 + 0.1 * (i + 1);
-    }
-
-    this->rtol = 1e-8;
-    this->atol = 1e-10;
-    this->max_num_step = 1e6;
-    auto res = apply_solver_tol();
-
-    EXPECT_NEAR(0.995029, stan::math::value_of(res[0][0]), 1e-5);
-    EXPECT_NEAR(-0.0990884, stan::math::value_of(res[0][1]), 1e-5);
-
-    EXPECT_NEAR(-0.421907, stan::math::value_of(res[99][0]), 1e-5);
-    EXPECT_NEAR(0.246407, stan::math::value_of(res[99][1]), 1e-5);
-  }
-};
-
 #endif
diff --git a/test/unit/math/rev/functor/test_fixture_ode_sho_ctl.hpp b/test/unit/math/rev/functor/test_fixture_ode_sho_ctl.hpp
new file mode 100644
index 00000000000..061c29393e7
--- /dev/null
+++ b/test/unit/math/rev/functor/test_fixture_ode_sho_ctl.hpp
@@ -0,0 +1,311 @@
+#ifndef STAN_MATH_TEST_FIXTURE_ODE_SHO_CTL_HPP
+#define STAN_MATH_TEST_FIXTURE_ODE_SHO_CTL_HPP
+
+#include <stan/math/rev.hpp>
+#include <boost/numeric/odeint.hpp>
+#include <gtest/gtest.h>
+#include <test/unit/math/prim/functor/harmonic_oscillator.hpp>
+#include <test/unit/math/rev/functor/coupled_mm.hpp>
+#include <test/unit/math/rev/functor/test_fixture_ode.hpp>
+#include <test/unit/math/rev/functor/test_fixture_ode_sho.hpp>
+#include <test/unit/util.hpp>
+#include <iostream>
+#include <sstream>
+#include <vector>
+#include <limits>
+#include <string>
+
+
+/**
+ * Inheriting base type, various fixtures differs by the type of ODE functor used in <code>apply_solver</code> calls, intended for different kind of tests. This harmonic oscillator test class is intended for use with the adjoint ODE solver which has additional control parameters.
+ *
+ */
+template <typename T>
+struct harmonic_oscillator_ctl_test
+    : public harmonic_oscillator_ode_base<T>,
+      public ODETestFixture<harmonic_oscillator_test<T>> {
+  double rtol_f;
+  double rtol_b;
+  double rtol_q;
+  Eigen::VectorXd atol_f;
+  Eigen::VectorXd atol_b;
+  double atol_q;
+  int num_steps_check;
+  int inter_poly;
+  int solv_f;
+  int solv_b;
+
+  harmonic_oscillator_ctl_test()
+      : harmonic_oscillator_ode_base<T>(),
+        rtol_f(this->rtol / 8.0),
+        rtol_b(this->rtol / 4.0),
+        rtol_q(this->rtol),
+        atol_f(Eigen::VectorXd::Constant(this->y0.size(), this->atol / 6.0)),
+        atol_b(Eigen::VectorXd::Constant(this->y0.size(), this->atol / 3.0)),
+        atol_q(this->atol),
+        num_steps_check(100),
+        inter_poly(CV_HERMITE),
+        solv_f(CV_BDF),
+        solv_b(CV_ADAMS) {}
+
+  auto apply_solver() {
+    std::tuple_element_t<0, T> sol;
+    return sol(this->f_eigen, this->y0, this->t0, this->ts, nullptr,
+               this->theta, this->x_r, this->x_i);
+  }
+
+  template <typename T1, typename T2>
+  auto apply_solver(T1&& init, T2&& theta_in) {
+    std::tuple_element_t<0, T> sol;
+    return sol(this->f_eigen, init, this->t0, this->ts, nullptr, theta_in,
+               this->x_r, this->x_i);
+  }
+
+  auto apply_solver_tol() {
+    std::tuple_element_t<1, T> sol;
+    return sol(this->f_eigen, this->y0, this->t0, this->ts, this->rtol,
+               this->atol, this->max_num_step, nullptr, this->theta, this->x_r,
+               this->x_i);
+  }
+
+  auto apply_solver_tol_ctl() {
+    std::tuple_element_t<0, T> sol;
+    return sol(this->f_eigen, this->y0, this->t0, this->ts, this->rtol_f,
+               this->atol_f, this->rtol_b, this->atol_b, this->rtol_q,
+               this->atol_q, this->max_num_step, this->num_steps_check,
+               this->inter_poly, this->solv_f, this->solv_b, nullptr,
+               this->theta, this->x_r, this->x_i);
+  }
+
+  void test_bad() {
+    const auto y0_(this->y0);
+
+    this->y0.resize(0);
+    EXPECT_THROW_MSG(apply_solver(), std::invalid_argument,
+                     "initial state has size 0");
+    this->y0 = y0_;
+
+    const auto t0_ = this->t0;
+    this->t0 = 2.0;
+    EXPECT_THROW_MSG(apply_solver(), std::domain_error,
+                     "initial time is 2, but must be less than 0.1");
+    this->t0 = t0_;
+
+    const auto ts_ = this->ts;
+    this->ts.resize(0);
+    EXPECT_THROW_MSG(apply_solver(), std::invalid_argument, "times has size 0");
+    this->ts = ts_;
+
+    this->ts.resize(2);
+    this->ts[0] = 3.0;
+    this->ts[1] = 1.0;
+    EXPECT_THROW_MSG(apply_solver(), std::domain_error,
+                     "times is not a valid sorted vector");
+    this->ts = ts_;
+
+    const double rtol_f_ = this->rtol_f;
+    this->rtol_f = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "relative_tolerance_forward");
+    this->rtol_f = rtol_f_;
+
+    const double atol_f_ = this->atol_f(0);
+    this->atol_f(0) = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "absolute_tolerance_forward");
+    this->atol_f(0) = atol_f_;
+
+    this->atol_f.resize(1);
+    this->atol_f(0) = atol_f_;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::invalid_argument,
+                     "absolute_tolerance_forward");
+    this->atol_f.resize(2);
+    this->atol_f(0) = atol_f_;
+    this->atol_f(1) = atol_f_;
+
+    const double rtol_b_ = this->rtol_b;
+    this->rtol_b = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "relative_tolerance_backward");
+    this->rtol_b = rtol_b_;
+
+    const double atol_b_ = this->atol_b(0);
+    this->atol_b(0) = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "absolute_tolerance_backward");
+    this->atol_b(0) = atol_b_;
+
+    this->atol_b.resize(1);
+    this->atol_b(0) = atol_b_;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::invalid_argument,
+                     "absolute_tolerance_backward");
+    this->atol_b.resize(2);
+    this->atol_b(0) = atol_b_;
+    this->atol_b(1) = atol_b_;
+
+    const double rtol_q_ = this->rtol_q;
+    this->rtol_q = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "relative_tolerance_quadrature");
+    this->rtol_q = rtol_q_;
+
+    const double atol_q_ = this->atol_q;
+    this->atol_q = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "absolute_tolerance_quadrature");
+    this->atol_q = atol_q_;
+
+    const int max_num_step_ = this->max_num_step;
+    this->max_num_step = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "max_num_steps");
+    this->max_num_step = max_num_step_;
+
+    const int num_steps_check_ = this->num_steps_check;
+    this->num_steps_check = -1;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "num_steps_between_checkpoints");
+    this->num_steps_check = num_steps_check_;
+
+    const int inter_poly_ = this->inter_poly;
+    this->inter_poly = 0;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "interpolation_polynomial");
+    this->inter_poly = inter_poly_;
+
+    const int solv_f_ = this->solv_f;
+    this->solv_f = 0;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "solver_forward");
+    this->solv_f = solv_f_;
+
+    const int solv_b_ = this->solv_b;
+    this->solv_b = 0;
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+                     "solver_backward");
+    this->solv_b = solv_b_;
+
+    const auto theta_ = this->theta;
+    const auto x_r_ = this->x_r;
+    const auto x_i_ = this->x_i;
+
+    // NaN errors
+    double nan = std::numeric_limits<double>::quiet_NaN();
+    std::stringstream expected_is_nan;
+    expected_is_nan << "is " << nan;
+
+    this->y0[0] = nan;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial state");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_nan.str());
+    this->y0 = y0_;
+
+    this->t0 = nan;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial time");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_nan.str());
+    this->t0 = t0_;
+
+    this->ts[0] = nan;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "times");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_nan.str());
+    this->ts = ts_;
+
+    this->theta[0] = nan;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     "ode parameters and data");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_nan.str());
+    this->theta = theta_;
+
+    this->x_r.push_back(nan);
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     "ode parameters and data");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_nan.str());
+    this->x_r = x_r_;
+
+    // inf test
+    std::stringstream expected_is_inf;
+    expected_is_inf << "is " << std::numeric_limits<double>::infinity();
+    std::stringstream expected_is_neg_inf;
+    expected_is_neg_inf << "is " << -std::numeric_limits<double>::infinity();
+    double inf = std::numeric_limits<double>::infinity();
+
+    this->y0[0] = inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial state");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_inf.str());
+    this->y0[0] = -inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial state");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_neg_inf.str());
+    this->y0 = y0_;
+
+    this->t0 = inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial time");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_inf.str());
+    this->t0 = -inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "initial time");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_neg_inf.str());
+    this->t0 = t0_;
+
+    this->ts.back() = inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "times");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_inf.str());
+    this->ts.back() = -inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error, "times");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_neg_inf.str());
+    this->ts = ts_;
+
+    this->theta[0] = inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     "ode parameters and data");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_inf.str());
+    this->theta[0] = -inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     "ode parameters and data");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_neg_inf.str());
+    this->theta = theta_;
+
+    this->x_r = std::vector<double>{inf};
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     "ode parameters and data");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_inf.str());
+    this->x_r[0] = -inf;
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     "ode parameters and data");
+    EXPECT_THROW_MSG(apply_solver_tol(), std::domain_error,
+                     expected_is_neg_inf.str());
+    this->x_r = x_r_;
+  }
+
+  void test_value(double t0_in) {
+    this->t0 = t0_in;
+    for (size_t i = 0; i < this->ts.size(); ++i) {
+      this->ts[i] = this->t0 + 0.1 * (i + 1);
+    }
+
+    this->rtol = 1e-8;
+    this->atol = 1e-10;
+    this->max_num_step = 1e6;
+    auto res = apply_solver_tol();
+
+    EXPECT_NEAR(0.995029, stan::math::value_of(res[0][0]), 1e-5);
+    EXPECT_NEAR(-0.0990884, stan::math::value_of(res[0][1]), 1e-5);
+
+    EXPECT_NEAR(-0.421907, stan::math::value_of(res[99][0]), 1e-5);
+    EXPECT_NEAR(0.246407, stan::math::value_of(res[99][1]), 1e-5);
+  }
+};
+
+
+#endif

From 828f40915c2b552860f2b991a1883f5f889d9de3 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 10 May 2021 11:22:01 +0200
Subject: [PATCH 138/157] save function name in a string

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index d91aa336880..c8337ba7d6a 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -38,12 +38,12 @@ class cvodes_integrator_adjoint_vari : public vari_base {
 
   static constexpr bool is_var_ts_{is_var<T_ts>::value};
   static constexpr bool is_var_t0_{is_var<T_t0>::value};
-  static constexpr bool is_var_y0_{is_var<T_y0_t0>::value};
+  static constexpr bool is_var_y0_t0_{is_var<T_y0_t0>::value};
   static constexpr bool is_any_var_args_{
       disjunction<is_var<scalar_type_t<T_Args>>...>::value};
   static constexpr bool is_var_return_{is_var<T_Return>::value};
   static constexpr bool is_var_only_ts_{
-      is_var_ts_ && !(is_var_t0_ || is_var_y0_ || is_any_var_args_)};
+      is_var_ts_ && !(is_var_t0_ || is_var_y0_t0_ || is_any_var_args_)};
 
   std::tuple<arena_t<T_Args>...> local_args_tuple_;
   std::tuple<arena_t<
@@ -93,7 +93,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     SUNMatrix A_backward_;
     SUNLinearSolver LS_backward_;
     const size_t N_;
-    const char* function_name_;
+    const std::string function_name_str_;
     void* cvodes_mem_;
     const std::decay_t<F> f_;
 
@@ -122,7 +122,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
               N == 0 ? nullptr
                      : SUNDenseLinearSolver(nv_state_backward_, A_backward_)),
           N_(N),
-          function_name_(function_name),
+          function_name_str_(function_name),
           cvodes_mem_(CVodeCreate(solver_forward)),
           f_(std::forward<FF>(f)) {
       if (cvodes_mem_ == nullptr) {
@@ -332,7 +332,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
                        solver_->nv_state_forward_, &t_init, CV_NORMAL, &ncheck);
 
           if (unlikely(error_code == CV_TOO_MUCH_WORK)) {
-            throw_domain_error(solver_->function_name_, "", t_final,
+            throw_domain_error(solver_->function_name_str_.c_str(), "", t_final,
                                "Failed to integrate to next output time (",
                                ") in less than max_num_steps steps");
           } else {
@@ -345,7 +345,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
                       &t_init, CV_NORMAL);
 
           if (error_code == CV_TOO_MUCH_WORK) {
-            throw_domain_error(solver_->function_name_, "", t_final,
+            throw_domain_error(solver_->function_name_str_.c_str(), "", t_final,
                                "Failed to integrate to next output time (",
                                ") in less than max_num_steps steps");
           } else {
@@ -518,7 +518,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
         int error_code = CVodeB(solver_->cvodes_mem_, t_final, CV_NORMAL);
 
         if (unlikely(error_code == CV_TOO_MUCH_WORK)) {
-          throw_domain_error(solver_->function_name_, "", t_final,
+          throw_domain_error(solver_->function_name_str_.c_str(), "", t_final,
                              "Failed to integrate backward to output time (",
                              ") in less than max_num_steps steps");
         } else {
@@ -548,7 +548,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     // After integrating all the way back to t0, we finally have the
     // the adjoints we wanted
     // These are the dlog_density / d(initial_conditions[s]) adjoints
-    if (is_var_y0_) {
+    if (is_var_y0_t0_) {
       forward_as<arena_t<Eigen::Matrix<var, Eigen::Dynamic, 1>>>(y0_).adj()
           += state_backward_;
     }
@@ -588,7 +588,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
   inline int rhs(double t, const double* y, double*& dy_dt) const {
     const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
     const Eigen::VectorXd dy_dt_vec = rhs(t, y_vec, value_of_args_tuple_);
-    check_size_match(solver_->function_name_, "dy_dt", dy_dt_vec.size(),
+    check_size_match(solver_->function_name_str_.c_str(), "dy_dt", dy_dt_vec.size(),
                      "states", N_);
     Eigen::Map<Eigen::VectorXd>(dy_dt, N_) = dy_dt_vec;
     return 0;
@@ -623,7 +623,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
         Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_));
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vars, value_of_args_tuple_);
-    check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(),
+    check_size_match(solver_->function_name_str_.c_str(), "dy_dt", f_y_t_vars.size(),
                      "states", N_);
     f_y_t_vars.adj() = -Eigen::Map<Eigen::VectorXd>(NV_DATA_S(yB), N_);
     grad();
@@ -663,7 +663,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
                          local_args_tuple_);
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vec, local_args_tuple_);
-    check_size_match(solver_->function_name_, "dy_dt", f_y_t_vars.size(),
+    check_size_match(solver_->function_name_str_.c_str(), "dy_dt", f_y_t_vars.size(),
                      "states", N_);
     f_y_t_vars.adj() = -Eigen::Map<Eigen::VectorXd>(NV_DATA_S(yB), N_);
     grad();
@@ -698,7 +698,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var
         = rhs(t, y_var, value_of_args_tuple_);
 
-    check_size_match(solver_->function_name_, "dy_dt", fy_var.size(), "states",
+    check_size_match(solver_->function_name_str_.c_str(), "dy_dt", fy_var.size(), "states",
                      N_);
 
     grad(fy_var.coeffRef(0).vi_);

From cc488c036bc8712dbfaed91b94de16c8aafb96ac Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 10 May 2021 11:26:38 +0200
Subject: [PATCH 139/157] register solver adjoint only in case of AD

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index c8337ba7d6a..2e36df7ea1a 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -357,7 +357,9 @@ class cvodes_integrator_adjoint_vari : public vari_base {
 
       t_init = t_final;
     }
-    ChainableStack::instance_->var_stack_.push_back(this);
+    if (is_var_return_) {
+      ChainableStack::instance_->var_stack_.push_back(this);
+    }
   }
 
  private:

From e5ad9c10aa9145df69d15724c64f4c3de579e691 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Mon, 10 May 2021 11:33:34 +0200
Subject: [PATCH 140/157] optimize case when only ts is a var

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp           | 4 ++--
 .../math/rev/functor/sho_ode_adjoint_typed_error_test.cpp     | 1 -
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 2e36df7ea1a..37ee4a1ea96 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -307,7 +307,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
         "CVodeSetJacFn");
 
     // initialize backward sensitivity system of CVODES as needed
-    if (is_var_return_) {
+    if (is_var_return_ && !is_var_only_ts_) {
       check_flag_sundials(
           CVodeAdjInit(solver_->cvodes_mem_, num_steps_between_checkpoints_,
                        interpolation_polynomial_),
@@ -324,7 +324,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     for (size_t n = 0; n < ts_dbl.size(); ++n) {
       double t_final = ts_dbl[n];
       if (t_final != t_init) {
-        if (is_var_return_) {
+        if (is_var_return_ && !is_var_only_ts_) {
           int ncheck;
 
           int error_code
diff --git a/test/unit/math/rev/functor/sho_ode_adjoint_typed_error_test.cpp b/test/unit/math/rev/functor/sho_ode_adjoint_typed_error_test.cpp
index f1e4da5892d..1c33d31b55c 100644
--- a/test/unit/math/rev/functor/sho_ode_adjoint_typed_error_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_adjoint_typed_error_test.cpp
@@ -38,7 +38,6 @@ TYPED_TEST_P(harmonic_oscillator_ctl_test, value) {
      std::is_same<std::tuple_element_t<3, TypeParam>, double>::value &&
      std::is_same<std::tuple_element_t<4, TypeParam>, double>::value
      ) {
-    std::cout << "testing all doubles!!!" << std::endl;
     EXPECT_EQ(stan::math::nested_size(), 0);
   } else {
     EXPECT_GT(stan::math::nested_size(), 0);

From 459d64bd04d6fd7e6ca588fcc5a46b4832188110 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Mon, 10 May 2021 09:42:28 +0000
Subject: [PATCH 141/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/cvodes_integrator.hpp      |  3 ++-
 .../rev/functor/cvodes_integrator_adjoint.hpp    | 16 ++++++++--------
 .../functor/sho_ode_adjoint_typed_error_test.cpp | 11 +++++------
 .../rev/functor/sho_ode_typed_error_test.cpp     |  1 -
 .../rev/functor/test_fixture_ode_sho_ctl.hpp     |  7 ++++---
 5 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator.hpp b/stan/math/rev/functor/cvodes_integrator.hpp
index ce6d8f960a4..554b77e1f54 100644
--- a/stan/math/rev/functor/cvodes_integrator.hpp
+++ b/stan/math/rev/functor/cvodes_integrator.hpp
@@ -281,7 +281,8 @@ class cvodes_integrator {
 
       cvodes_set_options(cvodes_mem, max_num_steps_);
 
-      check_flag_sundials(CVodeSStolerances(cvodes_mem, relative_tolerance_, absolute_tolerance_),
+      check_flag_sundials(CVodeSStolerances(cvodes_mem, relative_tolerance_,
+                                            absolute_tolerance_),
                           "CVodeSStolerances");
 
       check_flag_sundials(CVodeSetLinearSolver(cvodes_mem, LS_, A_),
diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 37ee4a1ea96..a1fecc2a8b0 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -590,8 +590,8 @@ class cvodes_integrator_adjoint_vari : public vari_base {
   inline int rhs(double t, const double* y, double*& dy_dt) const {
     const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
     const Eigen::VectorXd dy_dt_vec = rhs(t, y_vec, value_of_args_tuple_);
-    check_size_match(solver_->function_name_str_.c_str(), "dy_dt", dy_dt_vec.size(),
-                     "states", N_);
+    check_size_match(solver_->function_name_str_.c_str(), "dy_dt",
+                     dy_dt_vec.size(), "states", N_);
     Eigen::Map<Eigen::VectorXd>(dy_dt, N_) = dy_dt_vec;
     return 0;
   }
@@ -625,8 +625,8 @@ class cvodes_integrator_adjoint_vari : public vari_base {
         Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_));
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vars, value_of_args_tuple_);
-    check_size_match(solver_->function_name_str_.c_str(), "dy_dt", f_y_t_vars.size(),
-                     "states", N_);
+    check_size_match(solver_->function_name_str_.c_str(), "dy_dt",
+                     f_y_t_vars.size(), "states", N_);
     f_y_t_vars.adj() = -Eigen::Map<Eigen::VectorXd>(NV_DATA_S(yB), N_);
     grad();
     Eigen::Map<Eigen::VectorXd>(NV_DATA_S(yBdot), N_) = y_vars.adj();
@@ -665,8 +665,8 @@ class cvodes_integrator_adjoint_vari : public vari_base {
                          local_args_tuple_);
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
         = rhs(t, y_vec, local_args_tuple_);
-    check_size_match(solver_->function_name_str_.c_str(), "dy_dt", f_y_t_vars.size(),
-                     "states", N_);
+    check_size_match(solver_->function_name_str_.c_str(), "dy_dt",
+                     f_y_t_vars.size(), "states", N_);
     f_y_t_vars.adj() = -Eigen::Map<Eigen::VectorXd>(NV_DATA_S(yB), N_);
     grad();
     apply(
@@ -700,8 +700,8 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var
         = rhs(t, y_var, value_of_args_tuple_);
 
-    check_size_match(solver_->function_name_str_.c_str(), "dy_dt", fy_var.size(), "states",
-                     N_);
+    check_size_match(solver_->function_name_str_.c_str(), "dy_dt",
+                     fy_var.size(), "states", N_);
 
     grad(fy_var.coeffRef(0).vi_);
     Jfy.col(0) = y_var.adj();
diff --git a/test/unit/math/rev/functor/sho_ode_adjoint_typed_error_test.cpp b/test/unit/math/rev/functor/sho_ode_adjoint_typed_error_test.cpp
index 1c33d31b55c..89d21c789ef 100644
--- a/test/unit/math/rev/functor/sho_ode_adjoint_typed_error_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_adjoint_typed_error_test.cpp
@@ -25,19 +25,18 @@ using harmonic_oscillator_ctl_test_types = boost::mp11::mp_product<
 TYPED_TEST_SUITE_P(harmonic_oscillator_ctl_test);
 TYPED_TEST_P(harmonic_oscillator_ctl_test, no_error) { this->test_good(); }
 TYPED_TEST_P(harmonic_oscillator_ctl_test, error_conditions) {
-  this->test_bad(); 
+  this->test_bad();
 }
 TYPED_TEST_P(harmonic_oscillator_ctl_test, value) {
   stan::math::nested_rev_autodiff nested;
-  
+
   this->test_value(0.0);
   this->test_value(1.0);
   this->test_value(-1.0);
 
-  if(std::is_same<std::tuple_element_t<2, TypeParam>, double>::value &&
-     std::is_same<std::tuple_element_t<3, TypeParam>, double>::value &&
-     std::is_same<std::tuple_element_t<4, TypeParam>, double>::value
-     ) {
+  if (std::is_same<std::tuple_element_t<2, TypeParam>, double>::value
+      && std::is_same<std::tuple_element_t<3, TypeParam>, double>::value
+      && std::is_same<std::tuple_element_t<4, TypeParam>, double>::value) {
     EXPECT_EQ(stan::math::nested_size(), 0);
   } else {
     EXPECT_GT(stan::math::nested_size(), 0);
diff --git a/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp b/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
index 0ca125c70c0..aebf9b1d216 100644
--- a/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
+++ b/test/unit/math/rev/functor/sho_ode_typed_error_test.cpp
@@ -67,4 +67,3 @@ TYPED_TEST_P(harmonic_oscillator_bad_ode_test, bad_ode_error) {
 REGISTER_TYPED_TEST_SUITE_P(harmonic_oscillator_bad_ode_test, bad_ode_error);
 INSTANTIATE_TYPED_TEST_SUITE_P(StanOde, harmonic_oscillator_bad_ode_test,
                                harmonic_oscillator_integrate_ode_test_types);
-
diff --git a/test/unit/math/rev/functor/test_fixture_ode_sho_ctl.hpp b/test/unit/math/rev/functor/test_fixture_ode_sho_ctl.hpp
index 061c29393e7..f079e8952fe 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_sho_ctl.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_sho_ctl.hpp
@@ -15,9 +15,11 @@
 #include <limits>
 #include <string>
 
-
 /**
- * Inheriting base type, various fixtures differs by the type of ODE functor used in <code>apply_solver</code> calls, intended for different kind of tests. This harmonic oscillator test class is intended for use with the adjoint ODE solver which has additional control parameters.
+ * Inheriting base type, various fixtures differs by the type of ODE functor
+ * used in <code>apply_solver</code> calls, intended for different kind of
+ * tests. This harmonic oscillator test class is intended for use with the
+ * adjoint ODE solver which has additional control parameters.
  *
  */
 template <typename T>
@@ -307,5 +309,4 @@ struct harmonic_oscillator_ctl_test
   }
 };
 
-
 #endif

From 07495812a6e31184cb98022942a5d7c95d2053ba Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 11 May 2021 09:06:26 +0200
Subject: [PATCH 142/157] nest adjoint ODE integrator vari call for double only
 case

---
 .../rev/functor/cvodes_integrator_adjoint.hpp |  4 +-
 stan/math/rev/functor/ode_adjoint.hpp         | 89 ++++++++++++++++++-
 2 files changed, 89 insertions(+), 4 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index a1fecc2a8b0..445ae380dff 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -357,9 +357,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
 
       t_init = t_final;
     }
-    if (is_var_return_) {
-      ChainableStack::instance_->var_stack_.push_back(this);
-    }
+    ChainableStack::instance_->var_stack_.push_back(this);
   }
 
  private:
diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index e2b700c1c2f..798459e9012 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -66,7 +66,8 @@ namespace math {
 template <typename F, typename T_y0, typename T_t0, typename T_ts,
           typename T_abs_tol_fwd, typename T_abs_tol_bwd, typename... T_Args,
           require_all_eigen_col_vector_t<T_y0, T_abs_tol_fwd,
-                                         T_abs_tol_bwd>* = nullptr>
+                                         T_abs_tol_bwd>* = nullptr,
+          require_any_not_st_arithmetic<T_y0, T_t0, T_ts, T_Args...>* = nullptr>
 auto ode_adjoint_impl(
     const char* function_name, F&& f, const T_y0& y0, const T_t0& t0,
     const std::vector<T_ts>& ts, double relative_tolerance_forward,
@@ -91,6 +92,92 @@ auto ode_adjoint_impl(
   return integrator->solution();
 }
 
+/**
+ * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of times, { t1, t2, t3, ... } using the stiff backward differentiation formula BDF solver or the non-stiff Adams solver from CVODES. The ODE system is integrated using the adjoint sensitivity approach of CVODES. This implementation handles the case of a double return type which ensures that no resources are left on the AD stack.
+ *
+ * \p f must define an operator() with the signature as:
+ *   template<typename T_t, typename T_y, typename... T_Args>
+ *   Eigen::Matrix<stan::return_type_t<T_t, T_y, T_Args...>, Eigen::Dynamic, 1>
+ *     operator()(const T_t& t, const Eigen::Matrix<T_y, Eigen::Dynamic, 1>& y,
+ *     std::ostream* msgs, const T_Args&... args);
+ *
+ * t is the time, y is the state, msgs is a stream for error messages, and args
+ * are optional arguments passed to the ODE solve function (which are passed
+ * through to \p f without modification).
+ *
+ * @tparam F Type of ODE right hand side
+ * @tparam T_y0 Type of initial state
+ * @tparam T_t0 Type of initial time
+ * @tparam T_ts Type of output times
+ * @tparam T_Args Types of pass-through parameters
+ *
+ * @param function_name Calling function name (for printing debugging messages)
+ * @param f Right hand side of the ODE
+ * @param y0 Initial state
+ * @param t0 Initial time
+ * @param ts Times at which to solve the ODE at. All values must be sorted and
+ *   not less than t0.
+ * @param relative_tolerance_forward Relative tolerance for forward problem
+ * passed to CVODES
+ * @param absolute_tolerance_forward Absolute tolerance per ODE state for
+ * forward problem passed to CVODES
+ * @param relative_tolerance_backward Relative tolerance for backward problem
+ * passed to CVODES
+ * @param absolute_tolerance_backward Absolute tolerance per ODE state for
+ * backward problem passed to CVODES
+ * @param relative_tolerance_quadrature Relative tolerance for quadrature
+ * problem passed to CVODES
+ * @param absolute_tolerance_quadrature Absolute tolerance for quadrature
+ * problem passed to CVODES
+ * @param max_num_steps Upper limit on the number of integration steps to
+ *   take between each output (error if exceeded)
+ * @param num_steps_between_checkpoints Number of integrator steps after which a
+ * checkpoint is stored for the backward pass
+ * @param interpolation_polynomial type of polynomial used for interpolation
+ * @param solver_forward solver used for forward pass
+ * @param solver_backward solver used for backward pass
+ * @param[in, out] msgs the print stream for warning messages
+ * @param args Extra arguments passed unmodified through to ODE right hand side
+ * @return An `std::vector` of Eigen column vectors with scalars equal to
+ *  the least upper bound of `T_y0`, `T_t0`, `T_ts`, and the lambda's arguments.
+ *  This represents the solution to ODE at times \p ts
+ */
+template <typename F, typename T_y0, typename T_t0, typename T_ts,
+  typename T_abs_tol_fwd, typename T_abs_tol_bwd, typename... T_Args,
+  require_all_eigen_col_vector_t<T_y0, T_abs_tol_fwd, T_abs_tol_bwd>* = nullptr,
+  require_all_st_arithmetic<T_y0, T_t0, T_ts, T_Args...>* = nullptr>
+std::vector<Eigen::VectorXd> ode_adjoint_impl(
+    const char* function_name, F&& f, const T_y0& y0, const T_t0& t0,
+    const std::vector<T_ts>& ts, double relative_tolerance_forward,
+    const T_abs_tol_fwd& absolute_tolerance_forward,
+    double relative_tolerance_backward,
+    const T_abs_tol_bwd& absolute_tolerance_backward,
+    double relative_tolerance_quadrature, double absolute_tolerance_quadrature,
+    long int max_num_steps,                  // NOLINT(runtime/int)
+    long int num_steps_between_checkpoints,  // NOLINT(runtime/int)
+    int interpolation_polynomial, int solver_forward, int solver_backward,
+    std::ostream* msgs, const T_Args&... args) {
+  std::vector<Eigen::VectorXd> ode_solution;
+  {
+    nested_rev_autodiff nested;
+
+    using integrator_vari
+        = cvodes_integrator_adjoint_vari<F, plain_type_t<T_y0>, T_t0, T_ts,
+                                       plain_type_t<T_Args>...>;
+
+    auto integrator = new integrator_vari(
+        function_name, std::forward<F>(f), y0, t0, ts, relative_tolerance_forward,
+        absolute_tolerance_forward, relative_tolerance_backward,
+        absolute_tolerance_backward, relative_tolerance_quadrature,
+        absolute_tolerance_quadrature, max_num_steps,
+        num_steps_between_checkpoints, interpolation_polynomial, solver_forward,
+        solver_backward, msgs, args...);
+    
+    ode_solution = integrator->solution();
+  }
+  return ode_solution;
+}
+
 /**
  * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
  * times, { t1, t2, t3, ... } using the stiff backward differentiation formula

From a83fbd6ec92a6bdf2c1312be52007ae42c95aba3 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Tue, 11 May 2021 07:07:36 +0000
Subject: [PATCH 143/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/ode_adjoint.hpp | 30 ++++++++++++++++-----------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/stan/math/rev/functor/ode_adjoint.hpp b/stan/math/rev/functor/ode_adjoint.hpp
index 798459e9012..9c866032981 100644
--- a/stan/math/rev/functor/ode_adjoint.hpp
+++ b/stan/math/rev/functor/ode_adjoint.hpp
@@ -93,7 +93,12 @@ auto ode_adjoint_impl(
 }
 
 /**
- * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of times, { t1, t2, t3, ... } using the stiff backward differentiation formula BDF solver or the non-stiff Adams solver from CVODES. The ODE system is integrated using the adjoint sensitivity approach of CVODES. This implementation handles the case of a double return type which ensures that no resources are left on the AD stack.
+ * Solve the ODE initial value problem y' = f(t, y), y(t0) = y0 at a set of
+ * times, { t1, t2, t3, ... } using the stiff backward differentiation formula
+ * BDF solver or the non-stiff Adams solver from CVODES. The ODE system is
+ * integrated using the adjoint sensitivity approach of CVODES. This
+ * implementation handles the case of a double return type which ensures that no
+ * resources are left on the AD stack.
  *
  * \p f must define an operator() with the signature as:
  *   template<typename T_t, typename T_y, typename... T_Args>
@@ -143,9 +148,10 @@ auto ode_adjoint_impl(
  *  This represents the solution to ODE at times \p ts
  */
 template <typename F, typename T_y0, typename T_t0, typename T_ts,
-  typename T_abs_tol_fwd, typename T_abs_tol_bwd, typename... T_Args,
-  require_all_eigen_col_vector_t<T_y0, T_abs_tol_fwd, T_abs_tol_bwd>* = nullptr,
-  require_all_st_arithmetic<T_y0, T_t0, T_ts, T_Args...>* = nullptr>
+          typename T_abs_tol_fwd, typename T_abs_tol_bwd, typename... T_Args,
+          require_all_eigen_col_vector_t<T_y0, T_abs_tol_fwd,
+                                         T_abs_tol_bwd>* = nullptr,
+          require_all_st_arithmetic<T_y0, T_t0, T_ts, T_Args...>* = nullptr>
 std::vector<Eigen::VectorXd> ode_adjoint_impl(
     const char* function_name, F&& f, const T_y0& y0, const T_t0& t0,
     const std::vector<T_ts>& ts, double relative_tolerance_forward,
@@ -163,16 +169,16 @@ std::vector<Eigen::VectorXd> ode_adjoint_impl(
 
     using integrator_vari
         = cvodes_integrator_adjoint_vari<F, plain_type_t<T_y0>, T_t0, T_ts,
-                                       plain_type_t<T_Args>...>;
+                                         plain_type_t<T_Args>...>;
 
     auto integrator = new integrator_vari(
-        function_name, std::forward<F>(f), y0, t0, ts, relative_tolerance_forward,
-        absolute_tolerance_forward, relative_tolerance_backward,
-        absolute_tolerance_backward, relative_tolerance_quadrature,
-        absolute_tolerance_quadrature, max_num_steps,
-        num_steps_between_checkpoints, interpolation_polynomial, solver_forward,
-        solver_backward, msgs, args...);
-    
+        function_name, std::forward<F>(f), y0, t0, ts,
+        relative_tolerance_forward, absolute_tolerance_forward,
+        relative_tolerance_backward, absolute_tolerance_backward,
+        relative_tolerance_quadrature, absolute_tolerance_quadrature,
+        max_num_steps, num_steps_between_checkpoints, interpolation_polynomial,
+        solver_forward, solver_backward, msgs, args...);
+
     ode_solution = integrator->solution();
   }
   return ode_solution;

From 8b01a90f1c32d1264c24cd8e676cc2e825b205dd Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 11 May 2021 09:28:34 +0200
Subject: [PATCH 144/157] make use of adjoint_of where it works

---
 .../math/rev/functor/cvodes_integrator_adjoint.hpp | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 445ae380dff..9e5593a892a 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -338,13 +338,12 @@ class cvodes_integrator_adjoint_vari : public vari_base {
           } else {
             check_flag_sundials(error_code, "CVodeF");
           }
-
         } else {
           int error_code
               = CVode(solver_->cvodes_mem_, t_final, solver_->nv_state_forward_,
                       &t_init, CV_NORMAL);
 
-          if (error_code == CV_TOO_MUCH_WORK) {
+          if (unlikely(error_code == CV_TOO_MUCH_WORK)) {
             throw_domain_error(solver_->function_name_str_.c_str(), "", t_final,
                                "Failed to integrate to next output time (",
                                ") in less than max_num_steps steps");
@@ -411,8 +410,8 @@ class cvodes_integrator_adjoint_vari : public vari_base {
           step_sens.coeffRef(j)
               += forward_as<var>(solver_->y_return_[i].coeff(j)).adj();
         }
-
-        forward_as<var>(ts_[i])->adj_ += step_sens.dot(
+        
+        adjoint_of(ts_[i]) += step_sens.dot(
             rhs(value_of(ts_[i]), y_[i], value_of_args_tuple_));
         step_sens.setZero();
       }
@@ -435,7 +434,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
         state_backward_.coeffRef(j)
             += forward_as<var>(solver_->y_return_[i].coeff(j)).adj();
       }
-
+      
       double t_final = value_of((i > 0) ? ts_[i - 1] : t0_);
       if (t_final != t_init) {
         if (unlikely(!backward_is_initialized_)) {
@@ -541,7 +540,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     }
 
     if (is_var_t0_) {
-      forward_as<var>(t0_)->adj_ += -state_backward_.dot(
+      adjoint_of(t0_) += -state_backward_.dot(
           rhs(t_init, value_of(y0_), value_of_args_tuple_));
     }
 
@@ -549,8 +548,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     // the adjoints we wanted
     // These are the dlog_density / d(initial_conditions[s]) adjoints
     if (is_var_y0_t0_) {
-      forward_as<arena_t<Eigen::Matrix<var, Eigen::Dynamic, 1>>>(y0_).adj()
-          += state_backward_;
+      adjoint_of(y0_) += state_backward_;
     }
 
     // These are the dlog_density / d(parameters[s]) adjoints

From 437791c01ac276100fa1647aec9054ed264ee426 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Tue, 11 May 2021 07:29:37 +0000
Subject: [PATCH 145/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 9e5593a892a..c78918aa6fa 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -410,7 +410,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
           step_sens.coeffRef(j)
               += forward_as<var>(solver_->y_return_[i].coeff(j)).adj();
         }
-        
+
         adjoint_of(ts_[i]) += step_sens.dot(
             rhs(value_of(ts_[i]), y_[i], value_of_args_tuple_));
         step_sens.setZero();
@@ -434,7 +434,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
         state_backward_.coeffRef(j)
             += forward_as<var>(solver_->y_return_[i].coeff(j)).adj();
       }
-      
+
       double t_final = value_of((i > 0) ? ts_[i - 1] : t0_);
       if (t_final != t_init) {
         if (unlikely(!backward_is_initialized_)) {

From c26a82fdf49a55cfe8f562fb5bbf85ed35cf5fec Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 11 May 2021 10:34:06 +0200
Subject: [PATCH 146/157] fix

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index c78918aa6fa..10d74cba6b3 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -38,6 +38,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
 
   static constexpr bool is_var_ts_{is_var<T_ts>::value};
   static constexpr bool is_var_t0_{is_var<T_t0>::value};
+  static constexpr bool is_var_y0_{is_var<T_y0>::value};
   static constexpr bool is_var_y0_t0_{is_var<T_y0_t0>::value};
   static constexpr bool is_any_var_args_{
       disjunction<is_var<scalar_type_t<T_Args>>...>::value};
@@ -547,7 +548,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     // After integrating all the way back to t0, we finally have the
     // the adjoints we wanted
     // These are the dlog_density / d(initial_conditions[s]) adjoints
-    if (is_var_y0_t0_) {
+    if (is_var_y0_) {
       adjoint_of(y0_) += state_backward_;
     }
 

From d88e021ccb61d447fab564b4ebe64b9dcb8e8a9b Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Tue, 11 May 2021 13:43:02 +0200
Subject: [PATCH 147/157] fix tests

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 10d74cba6b3..bacb015466b 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -548,8 +548,9 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     // After integrating all the way back to t0, we finally have the
     // the adjoints we wanted
     // These are the dlog_density / d(initial_conditions[s]) adjoints
-    if (is_var_y0_) {
-      adjoint_of(y0_) += state_backward_;
+    if (is_var_y0_t0_) {
+      forward_as<arena_t<Eigen::Matrix<var, Eigen::Dynamic, 1>>>(y0_).adj()
+          += state_backward_;
     }
 
     // These are the dlog_density / d(parameters[s]) adjoints

From ba01fad95400c2bc4df6c5f9f63ffa07d855951f Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sat, 15 May 2021 22:44:48 +0200
Subject: [PATCH 148/157] improve error messages

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp    | 10 ++++++----
 .../unit/math/rev/functor/test_fixture_ode_sho_ctl.hpp |  6 +++---
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index bacb015466b..9b79ae5732c 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -269,11 +269,13 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     check_positive(function_name, "num_steps_between_checkpoints",
                    num_steps_between_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
-    check_bounded(function_name, "interpolation_polynomial",
-                  interpolation_polynomial_, 1, 2);
+    if(interpolation_polynomial_ != 1 && interpolation_polynomial_ != 2)
+      invalid_argument(function_name, "interpolation_polynomial", interpolation_polynomial_, "", ", must be 1 for Hermite or 2 for polynomial interpolation of ODE solution");
     // 1=Adams=CV_ADAMS, 2=BDF=CV_BDF
-    check_bounded(function_name, "solver_forward", solver_forward_, 1, 2);
-    check_bounded(function_name, "solver_backward", solver_backward_, 1, 2);
+    if(solver_forward_ != 1 && solver_forward_ != 2)
+      invalid_argument(function_name, "solver_forward", solver_forward_, "", ", must be 1 for Adams or 2 for BDF forward solver");
+    if(solver_backward_ != 1 && solver_backward_ != 2)
+      invalid_argument(function_name, "solver_backward", solver_backward_, "", ", must be 1 for Adams or 2 for BDF backward solver");
 
     solver_ = new cvodes_solver(
         function_name, f, N_, num_args_vars_, ts_.size(), solver_forward_,
diff --git a/test/unit/math/rev/functor/test_fixture_ode_sho_ctl.hpp b/test/unit/math/rev/functor/test_fixture_ode_sho_ctl.hpp
index f079e8952fe..1c95d1d25dd 100644
--- a/test/unit/math/rev/functor/test_fixture_ode_sho_ctl.hpp
+++ b/test/unit/math/rev/functor/test_fixture_ode_sho_ctl.hpp
@@ -171,19 +171,19 @@ struct harmonic_oscillator_ctl_test
 
     const int inter_poly_ = this->inter_poly;
     this->inter_poly = 0;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::invalid_argument,
                      "interpolation_polynomial");
     this->inter_poly = inter_poly_;
 
     const int solv_f_ = this->solv_f;
     this->solv_f = 0;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::invalid_argument,
                      "solver_forward");
     this->solv_f = solv_f_;
 
     const int solv_b_ = this->solv_b;
     this->solv_b = 0;
-    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::domain_error,
+    EXPECT_THROW_MSG(apply_solver_tol_ctl(), std::invalid_argument,
                      "solver_backward");
     this->solv_b = solv_b_;
 

From 826e0c94c81c006eec5fdf696dcf8b02aac81227 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Sat, 15 May 2021 20:45:45 +0000
Subject: [PATCH 149/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp   | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 9b79ae5732c..5a1437640f5 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -269,13 +269,18 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     check_positive(function_name, "num_steps_between_checkpoints",
                    num_steps_between_checkpoints_);
     // for polynomial: 1=CV_HERMITE / 2=CV_POLYNOMIAL
-    if(interpolation_polynomial_ != 1 && interpolation_polynomial_ != 2)
-      invalid_argument(function_name, "interpolation_polynomial", interpolation_polynomial_, "", ", must be 1 for Hermite or 2 for polynomial interpolation of ODE solution");
+    if (interpolation_polynomial_ != 1 && interpolation_polynomial_ != 2)
+      invalid_argument(function_name, "interpolation_polynomial",
+                       interpolation_polynomial_, "",
+                       ", must be 1 for Hermite or 2 for polynomial "
+                       "interpolation of ODE solution");
     // 1=Adams=CV_ADAMS, 2=BDF=CV_BDF
-    if(solver_forward_ != 1 && solver_forward_ != 2)
-      invalid_argument(function_name, "solver_forward", solver_forward_, "", ", must be 1 for Adams or 2 for BDF forward solver");
-    if(solver_backward_ != 1 && solver_backward_ != 2)
-      invalid_argument(function_name, "solver_backward", solver_backward_, "", ", must be 1 for Adams or 2 for BDF backward solver");
+    if (solver_forward_ != 1 && solver_forward_ != 2)
+      invalid_argument(function_name, "solver_forward", solver_forward_, "",
+                       ", must be 1 for Adams or 2 for BDF forward solver");
+    if (solver_backward_ != 1 && solver_backward_ != 2)
+      invalid_argument(function_name, "solver_backward", solver_backward_, "",
+                       ", must be 1 for Adams or 2 for BDF backward solver");
 
     solver_ = new cvodes_solver(
         function_name, f, N_, num_args_vars_, ts_.size(), solver_forward_,

From 784226c95035d6eb9a204bbd4545dc7ff0051d9a Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sun, 16 May 2021 09:25:18 +0200
Subject: [PATCH 150/157] revert harmonic oscillator

---
 .../math/prim/functor/harmonic_oscillator.hpp | 56 +++++++++++--------
 1 file changed, 33 insertions(+), 23 deletions(-)

diff --git a/test/unit/math/prim/functor/harmonic_oscillator.hpp b/test/unit/math/prim/functor/harmonic_oscillator.hpp
index 9a60e29603b..458c5c2ca4b 100644
--- a/test/unit/math/prim/functor/harmonic_oscillator.hpp
+++ b/test/unit/math/prim/functor/harmonic_oscillator.hpp
@@ -6,15 +6,16 @@
 #include <vector>
 
 struct harm_osc_ode_fun {
+  template <typename T0, typename T1, typename T2>
+  inline std::vector<stan::return_type_t<T1, T2>>
   // initial time
   // initial positions
   // parameters
   // double data
   // integer data
-  template <typename T0, typename T1, typename T2, typename T3, typename T4>
-  inline std::vector<stan::return_type_t<T1, T2>> operator()(
-      const T0& t_in, const T1& y_in, const T2& theta, const T3& x,
-      const T4& x_int, std::ostream* msgs) const {
+  operator()(const T0& t_in, const std::vector<T1>& y_in,
+             const std::vector<T2>& theta, const std::vector<double>& x,
+             const std::vector<int>& x_int, std::ostream* msgs) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
@@ -28,9 +29,12 @@ struct harm_osc_ode_fun {
 };
 
 struct harm_osc_ode_fun_eigen {
-  template <typename T0, typename T1, typename T2, typename T3, typename T4>
-  inline auto operator()(const T0& t_in, const T1& y_in, std::ostream* msgs,
-                         const T2& theta, const T3& x, const T4& x_int) const {
+  template <typename T0, typename T1, typename T2>
+  inline auto operator()(const T0& t_in,
+                         const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y_in,
+                         std::ostream* msgs, const std::vector<T2>& theta,
+                         const std::vector<double>& x,
+                         const std::vector<int>& x_int) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
@@ -44,15 +48,16 @@ struct harm_osc_ode_fun_eigen {
 };
 
 struct harm_osc_ode_data_fun {
+  template <typename T0, typename T1, typename T2>
+  inline std::vector<stan::return_type_t<T1, T2>>
   // initial time
   // initial positions
   // parameters
   // double data
   // integer data
-  template <typename T0, typename T1, typename T2, typename T3, typename T4>
-  inline std::vector<stan::return_type_t<T1, T2>> operator()(
-      const T0& t_in, const T1& y_in, const T2& theta, const T3& x,
-      const T4& x_int, std::ostream* msgs) const {
+  operator()(const T0& t_in, const std::vector<T1>& y_in,
+             const std::vector<T2>& theta, const std::vector<double>& x,
+             const std::vector<int>& x_int, std::ostream* msgs) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
@@ -67,14 +72,17 @@ struct harm_osc_ode_data_fun {
 };
 
 struct harm_osc_ode_data_fun_eigen {
-  template <typename T0, typename T1, typename T2, typename T3, typename T4>
-  inline auto operator()(const T0& t_in, const T1& y_in, std::ostream* msgs,
-                         const T2& theta, const T3& x, const T4& x_int) const {
+  template <typename T0, typename T1, typename T2>
+  inline auto operator()(const T0& t_in,
+                         const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y_in,
+                         std::ostream* msgs, const std::vector<T2>& theta,
+                         const std::vector<double>& x,
+                         const std::vector<int>& x_int) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
 
-    const auto& p = theta.at(0);
+    const T2& p = theta.at(0);
 
     Eigen::Matrix<stan::return_type_t<T1, T2>, -1, 1> res(2);
     res << (x.at(0) * y_in(1) + x_int.at(0)),
@@ -85,15 +93,16 @@ struct harm_osc_ode_data_fun_eigen {
 };
 
 struct harm_osc_ode_wrong_size_1_fun {
+  template <typename T0, typename T1, typename T2>
+  inline std::vector<stan::return_type_t<T1, T2>>
   // initial time
   // initial positions
   // parameters
   // double data
   // integer data
-  template <typename T0, typename T1, typename T2, typename T3, typename T4>
-  inline std::vector<stan::return_type_t<T1, T2>> operator()(
-      const T0& t_in, const T1& y_in, const T2& theta, const T3& x,
-      const T4& x_int, std::ostream* msgs) const {
+  operator()(const T0& t_in, const std::vector<T1>& y_in,
+             const std::vector<T2>& theta, const std::vector<double>& x,
+             const std::vector<int>& x_int, std::ostream* msgs) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");
@@ -108,15 +117,16 @@ struct harm_osc_ode_wrong_size_1_fun {
 };
 
 struct harm_osc_ode_wrong_size_2_fun {
+  template <typename T0, typename T1, typename T2>
+  inline std::vector<stan::return_type_t<T1, T2>>
   // initial time
   // initial positions
   // parameters
   // double data
   // integer data
-  template <typename T0, typename T1, typename T2, typename T3, typename T4>
-  inline std::vector<stan::return_type_t<T1, T2>> operator()(
-      const T0& t_in, const T1& y_in, const T2& theta, const T3& x,
-      const T4& x_int, std::ostream* msgs) const {
+  operator()(const T0& t_in, const std::vector<T1>& y_in,
+             const std::vector<T2>& theta, const std::vector<double>& x,
+             const std::vector<int>& x_int, std::ostream* msgs) const {
     if (y_in.size() != 2)
       throw std::domain_error(
           "this function was called with inconsistent state");

From 69158dde26450a888ec529171fa50f525988f080 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sun, 16 May 2021 09:36:05 +0200
Subject: [PATCH 151/157] make Eigen inputs expression compatible

---
 test/unit/math/prim/functor/harmonic_oscillator.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/unit/math/prim/functor/harmonic_oscillator.hpp b/test/unit/math/prim/functor/harmonic_oscillator.hpp
index 458c5c2ca4b..b7a1383f9b2 100644
--- a/test/unit/math/prim/functor/harmonic_oscillator.hpp
+++ b/test/unit/math/prim/functor/harmonic_oscillator.hpp
@@ -31,7 +31,7 @@ struct harm_osc_ode_fun {
 struct harm_osc_ode_fun_eigen {
   template <typename T0, typename T1, typename T2>
   inline auto operator()(const T0& t_in,
-                         const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y_in,
+                         const T1& y_in,
                          std::ostream* msgs, const std::vector<T2>& theta,
                          const std::vector<double>& x,
                          const std::vector<int>& x_int) const {
@@ -74,7 +74,7 @@ struct harm_osc_ode_data_fun {
 struct harm_osc_ode_data_fun_eigen {
   template <typename T0, typename T1, typename T2>
   inline auto operator()(const T0& t_in,
-                         const Eigen::Matrix<T1, Eigen::Dynamic, 1>& y_in,
+                         const T1& y_in,
                          std::ostream* msgs, const std::vector<T2>& theta,
                          const std::vector<double>& x,
                          const std::vector<int>& x_int) const {

From 8d7106781ab8627d872f8bce3d72f9a7b5a8f035 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Sun, 16 May 2021 07:38:38 +0000
Subject: [PATCH 152/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 test/unit/math/prim/functor/harmonic_oscillator.hpp | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/test/unit/math/prim/functor/harmonic_oscillator.hpp b/test/unit/math/prim/functor/harmonic_oscillator.hpp
index b7a1383f9b2..4a0903906f0 100644
--- a/test/unit/math/prim/functor/harmonic_oscillator.hpp
+++ b/test/unit/math/prim/functor/harmonic_oscillator.hpp
@@ -30,9 +30,8 @@ struct harm_osc_ode_fun {
 
 struct harm_osc_ode_fun_eigen {
   template <typename T0, typename T1, typename T2>
-  inline auto operator()(const T0& t_in,
-                         const T1& y_in,
-                         std::ostream* msgs, const std::vector<T2>& theta,
+  inline auto operator()(const T0& t_in, const T1& y_in, std::ostream* msgs,
+                         const std::vector<T2>& theta,
                          const std::vector<double>& x,
                          const std::vector<int>& x_int) const {
     if (y_in.size() != 2)
@@ -73,9 +72,8 @@ struct harm_osc_ode_data_fun {
 
 struct harm_osc_ode_data_fun_eigen {
   template <typename T0, typename T1, typename T2>
-  inline auto operator()(const T0& t_in,
-                         const T1& y_in,
-                         std::ostream* msgs, const std::vector<T2>& theta,
+  inline auto operator()(const T0& t_in, const T1& y_in, std::ostream* msgs,
+                         const std::vector<T2>& theta,
                          const std::vector<double>& x,
                          const std::vector<int>& x_int) const {
     if (y_in.size() != 2)

From 1b000081367b5bfd9f5a23ed79c118b44e004ef2 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sun, 16 May 2021 09:54:50 +0200
Subject: [PATCH 153/157] move ode argument tuple into the cvodes_solvers
 object

---
 .../rev/functor/cvodes_integrator_adjoint.hpp | 63 ++++++++++---------
 1 file changed, 33 insertions(+), 30 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 5a1437640f5..b399da595d2 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -46,11 +46,6 @@ class cvodes_integrator_adjoint_vari : public vari_base {
   static constexpr bool is_var_only_ts_{
       is_var_ts_ && !(is_var_t0_ || is_var_y0_t0_ || is_any_var_args_)};
 
-  std::tuple<arena_t<T_Args>...> local_args_tuple_;
-  std::tuple<arena_t<
-      promote_scalar_t<partials_type_t<scalar_type_t<T_Args>>, T_Args>>...>
-      value_of_args_tuple_;
-
   arena_t<std::vector<Eigen::VectorXd>> y_;
   arena_t<std::vector<T_ts>> ts_;
   arena_t<Eigen::Matrix<T_y0_t0, Eigen::Dynamic, 1>> y0_;
@@ -83,7 +78,9 @@ class cvodes_integrator_adjoint_vari : public vari_base {
    * vari class).
    */
   struct cvodes_solver : public chainable_alloc {
-    std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return_;
+    const std::string function_name_str_;
+    const F f_;
+    const size_t N_;
     N_Vector nv_state_forward_;
     N_Vector nv_state_backward_;
     N_Vector nv_quad_;
@@ -93,10 +90,13 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     SUNLinearSolver LS_forward_;
     SUNMatrix A_backward_;
     SUNLinearSolver LS_backward_;
-    const size_t N_;
-    const std::string function_name_str_;
     void* cvodes_mem_;
-    const std::decay_t<F> f_;
+    std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return_;
+    std::tuple<T_Args...> local_args_tuple_;
+    const std::tuple<
+      promote_scalar_t<partials_type_t<scalar_type_t<T_Args>>, T_Args>...>
+    value_of_args_tuple_;
+
 
     template <typename FF, typename StateFwd, typename StateBwd, typename Quad,
               typename AbsTolFwd, typename AbsTolBwd>
@@ -104,8 +104,11 @@ class cvodes_integrator_adjoint_vari : public vari_base {
                   size_t num_args_vars, size_t ts_size, int solver_forward,
                   StateFwd& state_forward, StateBwd& state_backward, Quad& quad,
                   AbsTolFwd& absolute_tolerance_forward,
-                  AbsTolBwd& absolute_tolerance_backward)
+                  AbsTolBwd& absolute_tolerance_backward,
+                  const T_Args&... args)
         : chainable_alloc(),
+          f_(std::forward<FF>(f)),
+          function_name_str_(function_name),
           y_return_(ts_size),
           nv_state_forward_(N_VMake_Serial(N, state_forward.data())),
           nv_state_backward_(N_VMake_Serial(N, state_backward.data())),
@@ -123,9 +126,9 @@ class cvodes_integrator_adjoint_vari : public vari_base {
               N == 0 ? nullptr
                      : SUNDenseLinearSolver(nv_state_backward_, A_backward_)),
           N_(N),
-          function_name_str_(function_name),
           cvodes_mem_(CVodeCreate(solver_forward)),
-          f_(std::forward<FF>(f)) {
+          local_args_tuple_(deep_copy_vars(args)...),
+          value_of_args_tuple_(value_of(args)...) {
       if (cvodes_mem_ == nullptr) {
         throw std::runtime_error("CVodeCreate failed to allocate memory");
       }
@@ -189,7 +192,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
    * @return a vector of states, each state being a vector of the
    * same size as the state variable, corresponding to a time in ts.
    */
-  template <typename FF, require_eigen_col_vector_t<T_y0>* = nullptr>
+template <typename FF, require_eigen_col_vector_t<T_y0>* = nullptr>
   cvodes_integrator_adjoint_vari(
       const char* function_name, FF&& f, const T_y0& y0, const T_t0& t0,
       const std::vector<T_ts>& ts, double relative_tolerance_forward,
@@ -203,8 +206,8 @@ class cvodes_integrator_adjoint_vari : public vari_base {
       int interpolation_polynomial, int solver_forward, int solver_backward,
       std::ostream* msgs, const T_Args&... args)
       : vari_base(),
-        local_args_tuple_(to_arena(deep_copy_vars(args))...),
-        value_of_args_tuple_(to_arena(value_of(args))...),
+        //local_args_tuple_(deep_copy_vars(args)...),
+        //value_of_args_tuple_(value_of(args)...),
         y_(ts.size()),
         ts_(ts.begin(), ts.end()),
         y0_(y0),
@@ -239,12 +242,6 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     check_finite(function_name, "initial time", t0);
     check_finite(function_name, "times", ts);
 
-    stan::math::for_each(
-        [func_name = function_name](auto&& arg) {
-          check_finite(func_name, "ode parameters and data", arg);
-        },
-        local_args_tuple_);
-
     check_nonzero_size(function_name, "times", ts);
     check_nonzero_size(function_name, "initial state", y0);
     check_sorted(function_name, "times", ts);
@@ -285,7 +282,13 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     solver_ = new cvodes_solver(
         function_name, f, N_, num_args_vars_, ts_.size(), solver_forward_,
         state_forward_, state_backward_, quad_, absolute_tolerance_forward_,
-        absolute_tolerance_backward_);
+        absolute_tolerance_backward_, args...);
+
+    stan::math::for_each(
+        [func_name = function_name](auto&& arg) {
+          check_finite(func_name, "ode parameters and data", arg);
+        },
+        solver_->local_args_tuple_);
 
     check_flag_sundials(
         CVodeInit(solver_->cvodes_mem_, &cvodes_integrator_adjoint_vari::cv_rhs,
@@ -420,7 +423,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
         }
 
         adjoint_of(ts_[i]) += step_sens.dot(
-            rhs(value_of(ts_[i]), y_[i], value_of_args_tuple_));
+            rhs(value_of(ts_[i]), y_[i], solver_->value_of_args_tuple_));
         step_sens.setZero();
       }
 
@@ -549,7 +552,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
 
     if (is_var_t0_) {
       adjoint_of(t0_) += -state_backward_.dot(
-          rhs(t_init, value_of(y0_), value_of_args_tuple_));
+          rhs(t_init, value_of(y0_), solver_->value_of_args_tuple_));
     }
 
     // After integrating all the way back to t0, we finally have the
@@ -594,7 +597,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
    */
   inline int rhs(double t, const double* y, double*& dy_dt) const {
     const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
-    const Eigen::VectorXd dy_dt_vec = rhs(t, y_vec, value_of_args_tuple_);
+    const Eigen::VectorXd dy_dt_vec = rhs(t, y_vec, solver_->value_of_args_tuple_);
     check_size_match(solver_->function_name_str_.c_str(), "dy_dt",
                      dy_dt_vec.size(), "states", N_);
     Eigen::Map<Eigen::VectorXd>(dy_dt, N_) = dy_dt_vec;
@@ -629,7 +632,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     Eigen::Matrix<var, Eigen::Dynamic, 1> y_vars(
         Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_));
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
-        = rhs(t, y_vars, value_of_args_tuple_);
+        = rhs(t, y_vars, solver_->value_of_args_tuple_);
     check_size_match(solver_->function_name_str_.c_str(), "dy_dt",
                      f_y_t_vars.size(), "states", N_);
     f_y_t_vars.adj() = -Eigen::Map<Eigen::VectorXd>(NV_DATA_S(yB), N_);
@@ -667,9 +670,9 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     // The vars here do not live on the nested stack so must be zero'd
     // separately
     stan::math::for_each([](auto&& arg) { zero_adjoints(arg); },
-                         local_args_tuple_);
+                         solver_->local_args_tuple_);
     Eigen::Matrix<var, Eigen::Dynamic, 1> f_y_t_vars
-        = rhs(t, y_vec, local_args_tuple_);
+        = rhs(t, y_vec, solver_->local_args_tuple_);
     check_size_match(solver_->function_name_str_.c_str(), "dy_dt",
                      f_y_t_vars.size(), "states", N_);
     f_y_t_vars.adj() = -Eigen::Map<Eigen::VectorXd>(NV_DATA_S(yB), N_);
@@ -678,7 +681,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
         [&qBdot](auto&&... args) {
           accumulate_adjoints(NV_DATA_S(qBdot), args...);
         },
-        local_args_tuple_);
+        solver_->local_args_tuple_);
     return 0;
   }
 
@@ -703,7 +706,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     Eigen::Matrix<var, Eigen::Dynamic, 1> y_var(
         Eigen::Map<const Eigen::VectorXd>(NV_DATA_S(y), N_));
     Eigen::Matrix<var, Eigen::Dynamic, 1> fy_var
-        = rhs(t, y_var, value_of_args_tuple_);
+        = rhs(t, y_var, solver_->value_of_args_tuple_);
 
     check_size_match(solver_->function_name_str_.c_str(), "dy_dt",
                      fy_var.size(), "states", N_);

From 67c7c8fc5316e318bc81c140df20e711e2ed5c89 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Sun, 16 May 2021 07:56:13 +0000
Subject: [PATCH 154/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 .../rev/functor/cvodes_integrator_adjoint.hpp   | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index b399da595d2..6474e7502ec 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -94,9 +94,8 @@ class cvodes_integrator_adjoint_vari : public vari_base {
     std::vector<Eigen::Matrix<T_Return, Eigen::Dynamic, 1>> y_return_;
     std::tuple<T_Args...> local_args_tuple_;
     const std::tuple<
-      promote_scalar_t<partials_type_t<scalar_type_t<T_Args>>, T_Args>...>
-    value_of_args_tuple_;
-
+        promote_scalar_t<partials_type_t<scalar_type_t<T_Args>>, T_Args>...>
+        value_of_args_tuple_;
 
     template <typename FF, typename StateFwd, typename StateBwd, typename Quad,
               typename AbsTolFwd, typename AbsTolBwd>
@@ -104,8 +103,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
                   size_t num_args_vars, size_t ts_size, int solver_forward,
                   StateFwd& state_forward, StateBwd& state_backward, Quad& quad,
                   AbsTolFwd& absolute_tolerance_forward,
-                  AbsTolBwd& absolute_tolerance_backward,
-                  const T_Args&... args)
+                  AbsTolBwd& absolute_tolerance_backward, const T_Args&... args)
         : chainable_alloc(),
           f_(std::forward<FF>(f)),
           function_name_str_(function_name),
@@ -192,7 +190,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
    * @return a vector of states, each state being a vector of the
    * same size as the state variable, corresponding to a time in ts.
    */
-template <typename FF, require_eigen_col_vector_t<T_y0>* = nullptr>
+  template <typename FF, require_eigen_col_vector_t<T_y0>* = nullptr>
   cvodes_integrator_adjoint_vari(
       const char* function_name, FF&& f, const T_y0& y0, const T_t0& t0,
       const std::vector<T_ts>& ts, double relative_tolerance_forward,
@@ -206,8 +204,8 @@ template <typename FF, require_eigen_col_vector_t<T_y0>* = nullptr>
       int interpolation_polynomial, int solver_forward, int solver_backward,
       std::ostream* msgs, const T_Args&... args)
       : vari_base(),
-        //local_args_tuple_(deep_copy_vars(args)...),
-        //value_of_args_tuple_(value_of(args)...),
+        // local_args_tuple_(deep_copy_vars(args)...),
+        // value_of_args_tuple_(value_of(args)...),
         y_(ts.size()),
         ts_(ts.begin(), ts.end()),
         y0_(y0),
@@ -597,7 +595,8 @@ template <typename FF, require_eigen_col_vector_t<T_y0>* = nullptr>
    */
   inline int rhs(double t, const double* y, double*& dy_dt) const {
     const Eigen::VectorXd y_vec = Eigen::Map<const Eigen::VectorXd>(y, N_);
-    const Eigen::VectorXd dy_dt_vec = rhs(t, y_vec, solver_->value_of_args_tuple_);
+    const Eigen::VectorXd dy_dt_vec
+        = rhs(t, y_vec, solver_->value_of_args_tuple_);
     check_size_match(solver_->function_name_str_.c_str(), "dy_dt",
                      dy_dt_vec.size(), "states", N_);
     Eigen::Map<Eigen::VectorXd>(dy_dt, N_) = dy_dt_vec;

From 03d50c68b7dc0fe12b130ab0281fdea9c36b6ea0 Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sun, 16 May 2021 10:03:35 +0200
Subject: [PATCH 155/157] add in decay_t again as before

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index b399da595d2..703436779b5 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -79,7 +79,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
    */
   struct cvodes_solver : public chainable_alloc {
     const std::string function_name_str_;
-    const F f_;
+    const std::decay_t<F> f_;
     const size_t N_;
     N_Vector nv_state_forward_;
     N_Vector nv_state_backward_;

From 630e1ed14b578c8ca53aae85b3a0139c495e888e Mon Sep 17 00:00:00 2001
From: Sebastian Weber <sdw.post@waebers.de>
Date: Sun, 16 May 2021 20:43:55 +0200
Subject: [PATCH 156/157] review comments

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index 21da24f6b02..b28657eb122 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -21,7 +21,7 @@ namespace stan {
 namespace math {
 
 /**
- * Integrator interface for CVODES' ODE solvers (Adams & BDF
+ * Integrator interface for CVODES' adjoint ODE solvers (Adams & BDF
  * methods).
  *
  * @tparam F Type of ODE right hand side
@@ -204,8 +204,6 @@ class cvodes_integrator_adjoint_vari : public vari_base {
       int interpolation_polynomial, int solver_forward, int solver_backward,
       std::ostream* msgs, const T_Args&... args)
       : vari_base(),
-        // local_args_tuple_(deep_copy_vars(args)...),
-        // value_of_args_tuple_(value_of(args)...),
         y_(ts.size()),
         ts_(ts.begin(), ts.end()),
         y0_(y0),
@@ -403,7 +401,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
   /**
    * No-op for setting adjoints since this class does not own any adjoints.
    */
-  void set_zero_adjoint() final{};
+  void set_zero_adjoint() final {};
 
   void chain() final {
     if (!is_var_return_) {
@@ -618,8 +616,6 @@ class cvodes_integrator_adjoint_vari : public vari_base {
    *
    * Equation 2.23 in the cvs_guide.
    *
-   * @param[in] initial var vector
-   * @param[in] param var vector
    * @param[in] t time
    * @param[in] y state of the base ODE system
    * @param[in] yB state of the adjoint ODE system
@@ -655,8 +651,6 @@ class cvodes_integrator_adjoint_vari : public vari_base {
    *
    * This is the integrand of equation 2.22 in the cvs_guide.
    *
-   * @param[in] initial var vector
-   * @param[in] param var vector
    * @param[in] t time
    * @param[in] y state of the base ODE system
    * @param[in] yB state of the adjoint ODE system
@@ -738,8 +732,8 @@ class cvodes_integrator_adjoint_vari : public vari_base {
    * Calculate the Jacobian of the RHS of the adjoint ODE (see rhs_adj
    * below for citation for how this is done)
    *
-   * @param[in] y State of system
    * @param[in] t Time
+   * @param[in] y State of system
    * @param[out] J CVode structure where output is to be stored
    */
   inline int jacobian_rhs_adj_states(double t, N_Vector y, SUNMatrix J) const {

From 01c9fa1ead64d7f3774d27c0938ac7a01a6bdc25 Mon Sep 17 00:00:00 2001
From: Stan Jenkins <mc.stanislaw@gmail.com>
Date: Sun, 16 May 2021 18:45:04 +0000
Subject: [PATCH 157/157] [Jenkins] auto-formatting by clang-format version
 6.0.0-1ubuntu2~16.04.1 (tags/RELEASE_600/final)

---
 stan/math/rev/functor/cvodes_integrator_adjoint.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
index b28657eb122..d470c39f495 100644
--- a/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
+++ b/stan/math/rev/functor/cvodes_integrator_adjoint.hpp
@@ -401,7 +401,7 @@ class cvodes_integrator_adjoint_vari : public vari_base {
   /**
    * No-op for setting adjoints since this class does not own any adjoints.
    */
-  void set_zero_adjoint() final {};
+  void set_zero_adjoint() final{};
 
   void chain() final {
     if (!is_var_return_) {