use optimizer as default

Author: greole

include/OGL/DevicePersistent/ExecutorHandler.hpp

@@ -311,6 +311,8 @@ class ExecutorHandler
 
     label get_ranks_per_gpu() const { return device_id_handler_.ranks_per_gpu; }
 
+    void set_ranks_per_gpu(label ranks_per_gpu) { device_id_handler_.ranks_per_gpu= ranks_per_gpu; }
+
     label get_owner_rank() const { return device_id_handler_.global_owner(); }
 
     const std::shared_ptr<gko::Executor> get_device_exec() const

include/OGL/StoppingCriterion.hpp

@@ -201,34 +201,30 @@ class StoppingCriterion {
                                   bool export_res, label prev_solve_iters,
                                   scalar prev_rel_cost) const
     {
+        word frequencyMode = "optimizer";
         label minIter = minIter_;
         label frequency = frequency_;
         // in case of export_res all residuals need to be computed
-        std::cout << __FILE__ << "adapt minIter and frequency0 \n";
         if (!export_res) {
-            std::cout << __FILE__ << "adapt minIter and frequency1 \n";
             if (prev_solve_iters > 0 && adapt_minIter_ && prev_rel_cost > 0) {
-                std::cout << __FILE__ << "adapt minIter and frequency2 \n";
                 minIter = prev_solve_iters * relaxationFactor_;
-                if (frequencyMode_ == "optimizer") {
+                if (frequencyMode == "optimizer") {
                     auto alpha = sqrt(
                         1.0 / (prev_solve_iters * (1.0 - relaxationFactor_)) *
                         prev_rel_cost);
                     frequency = min(norm_eval_limit_, max(1, label(1 / alpha)));
                 }
-                // if (frequencyMode_ == "relative") {
-                frequency = label(prev_solve_iters * 0.075) + 1;
-                // }
+                if (frequencyMode == "relative") {
+                    frequency = label(prev_solve_iters * 0.075) + 1;
+                }
             }
         }
 
         word msg = "Creating stopping criterion with minIter " +
                    std::to_string(minIter) + " frequency " +
                    std::to_string(frequency) + " prev_solve_iters " +
-                   std::to_string(prev_solve_iters) + +" adapt_minIter_  " +
-                   std::to_string(adapt_minIter_) + +" prev_rel_cost  " +
-                   std::to_string(prev_rel_cost) + " prev_solve_iters*0.075  " +
-                   std::to_string(prev_solve_iters * 0.075);
+                   std::to_string(prev_solve_iters) + " adapt_minIter_  " +
+                   std::to_string(adapt_minIter_) + " prev_rel_cost  ";
 
         MLOG_0(verbose, msg)
 

src/MatrixWrapper/Distributed.cpp

@@ -527,7 +527,6 @@ std::shared_ptr<RepartDistMatrix> create_impl(
     label rank = exec_handler.get_host_rank();
     auto exec = exec_handler.get_ref_exec();
     auto host_comm = *exec_handler.get_host_comm().get();
-    exec_handler.init_device_comm();
     auto device_comm = *exec_handler.get_device_comm().get();
     bool owner = repartitioner->is_owner(exec_handler);
 

test/unit/MatrixWrapper/Distributed.cpp

@@ -56,6 +56,7 @@ class Environment : public testing::Environment {
                            Foam::IOobject::MUST_READ),
             false);
 
+        // FIXME this needs the device_id_handler
         exec = std::make_shared<ExecutorHandler>(runTime_->thisDb(), dict,
                                                  "dummy", true);
 
@@ -172,6 +173,7 @@ TEST_P(DistMatL2D, canCreateDistributedMatrix)
 {
     /* The test mesh is 6x6 grid decomposed into 4 3x3 subdomains */
     auto [ranks_per_gpu, matrix_format, fused] = GetParam();
+    exec.set_ranks_per_gpu(ranks_per_gpu);
 
     auto mesh = ((Environment *)global_env)->mesh;
     auto hostMatrix = ((Environment *)global_env)->hostMatrix;
@@ -182,6 +184,7 @@ TEST_P(DistMatL2D, canCreateDistributedMatrix)
     gko::dim<2> global_vec_dim{repartitioner->get_orig_partition()->get_size(),
                                1};
     gko::dim<2> local_vec_dim{repartitioner->get_repart_dim()[0], 1};
+    exec.init_device_comm();
 
     auto distributed = create_distributed(exec, repartitioner, hostMatrix,
                                           matrix_format, fused, 0);
@@ -200,6 +203,8 @@ TEST_P(DistMatL2D, hasCorrectLocalMatrix)
 {
     /* The test mesh is 6x6 grid decomposed into 4 3x3 subdomains */
     auto [ranks_per_gpu, matrix_format, fused] = GetParam();
+    exec.set_ranks_per_gpu(ranks_per_gpu);
+    exec.init_device_comm();
     auto mesh = ((Environment *)global_env)->mesh;
     auto hostMatrix = ((Environment *)global_env)->hostMatrix;
     auto repartitioner = std::make_shared<Repartitioner>(
@@ -254,6 +259,8 @@ TEST_P(DistMatL2D, hasCorrectNonLocalMatrix)
 {
     /* The test mesh is 6x6 grid decomposed into 4 3x3 subdomains */
     auto [ranks_per_gpu, matrix_format, fused] = GetParam();
+    exec.set_ranks_per_gpu(ranks_per_gpu);
+    exec.init_device_comm();
     auto mesh = ((Environment *)global_env)->mesh;
     auto hostMatrix = ((Environment *)global_env)->hostMatrix;
     auto name = ((Environment *)global_env)->name_;
@@ -292,6 +299,8 @@ TEST_P(DistMatL2D, hasCorrectNonLocalMatrix)
 TEST_P(DistMatL2D, canApplyCorrectly)
 {
     auto [ranks_per_gpu, format, fused] = GetParam();
+    exec.set_ranks_per_gpu(ranks_per_gpu);
+    exec.init_device_comm();
     auto mesh = ((Environment *)global_env)->mesh;
     auto hostMatrix = ((Environment *)global_env)->hostMatrix;
     auto name = ((Environment *)global_env)->name_;
@@ -314,12 +323,15 @@ TEST_P(DistMatL2D, canApplyCorrectly)
     x->fill(0);
 
     // Act
+    bool active = repartitioner->get_repart_size() != 0;
+    if (active){
     distributed->apply(b, x);
     auto res_x = std::vector<scalar>(
         x->get_local_vector()->get_const_values(),
         x->get_local_vector()->get_const_values() + local_vec_dim[0]);
 
     ASSERT_EQ(res_x, exp_x[name][fused][ranks_per_gpu][rank]);
+    }
 }
 
 int main(int argc, char *argv[])