Compare extreme value branch with current dev branch

.gitignore

@@ -3,6 +3,7 @@
 *core*
 *.asv
 *.swn
+*.swm
 *.swo
 *.eps
 *.png

examples/workflow_example_extr.jl

@@ -2,26 +2,43 @@
 
 include(normpath(joinpath(dirname(@__FILE__),"..","src","ClustForOpt_priv_development.jl")))
 #using ClustForOpt_priv
-#using Gurobi
+using Gurobi
+env = Gurobi.Env() # reusing the same gurobi environment for multiple solves
+# select solver
+solver=GurobiSolver(env,OutputFlag=0)
 
 # load data
-ts_input_data, = load_timeseries_data("CEP", "GER_18";K=365, T=24) #CEP
+state="TX_1" # or "GER_18" or "GER_1" or "CA_1" or "TX_1"
+ts_input_data, = load_timeseries_data("CEP", state;K=365, T=24) #CEP
 
-cep_input_data_GER=load_cep_data("GER_18")
+cep_input_data_GER=load_cep_data(state)
 
  # define simple extreme days of interest
- ev1 = SimpleExtremeValueDescr("wind-dena42","max","absolute")
- ev2 = SimpleExtremeValueDescr("solar-dena42","min","integral")
- ev3 = SimpleExtremeValueDescr("el_demand-dena21","max","absolute")
- ev = [ev1, ev2, ev3]
- # simple extreme day selection
- ts_input_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_input_data,ev;rep_mod_method="feasibility")
+#ev1 = SimpleExtremeValueDescr("wind-node61","max","absolute") # TODO: min?
+#ev2 = SimpleExtremeValueDescr("solar-node61","min","integral") 
+#ev3 = SimpleExtremeValueDescr("el_demand-node61","max","absolute")
+# ev1 = SimpleExtremeValueDescr("wind-dena42","max","absolute")
+# ev2 = SimpleExtremeValueDescr("solar-dena42","min","integral")
+# ev3 = SimpleExtremeValueDescr("el_demand-dena21","max","absolute")
+# ev = [ev1, ev2, ev3]
 
- # run clustering
-ts_clust_res = run_clust(ts_input_data_mod;method="kmeans",representation="centroid",n_init=10,n_clust=5) # default k-means
+rep_mod_method="append"  #feasibility, append
+extreme_event_selection_method= "slack" #slack, feasibility
+slack_cost=1e5
+co2_limit=1000.0
+
+#without simple extreme days
+ ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER;rep_mod_method=rep_mod_method,method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",co2_limit=co2_limit,extreme_event_selection_method=extreme_event_selection_method,slack_cost=slack_cost,print_flag=false) 
+ #ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",extreme_event_selection_method="feasibility",print_flag=false) 
+
+#with simple extreme days
+# ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER,ev;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",print_flag=false) 
+
+# optimization
+opt_res = run_opt(ts_clust_res.best_results,cep_input_data_GER;solver=GurobiSolver(),storage="intra",co2_limit=co2_limit)
+
+#TODO: write functions to get generation, capacity etc.
+
+# TODO: write plotting functions in other package
 
-# representation modification
-ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results)
 
- # optimization
-opt_res = run_opt(ts_clust_extr,cep_input_data_GER;solver=GurobiSolver(),co2_limit=1000.0)

examples/workflow_example_extr_simple.jl

@@ -0,0 +1,32 @@
+# This file exemplifies the workflow from data input to optimization result generation
+#QUESTION using ClustForOpt_priv.col in module Main conflicts with an existing identifier., using ClustForOpt_priv.cols in module Main conflicts with an existing identifier.
+
+include(normpath(joinpath(dirname(@__FILE__),"..","src","ClustForOpt_priv_development.jl")))
+#using ClustForOpt_priv
+#using Gurobi
+
+# load data
+ts_input_data, = load_timeseries_data("CEP", "GER_18";K=365, T=24) #CEP
+
+cep_input_data_GER=load_cep_data("GER_18")
+
+ # define simple extreme days of interest
+ ev1 = SimpleExtremeValueDescr("wind-dena42","max","absolute")
+ ev2 = SimpleExtremeValueDescr("solar-dena42","min","integral")
+ ev3 = SimpleExtremeValueDescr("el_demand-dena21","max","absolute")
+ ev = [ev1, ev2, ev3]
+
+ rep_mod_method="feasibility"
+ # simple extreme day selection
+ ts_input_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_input_data,ev;rep_mod_method=rep_mod_method)
+
+ # run clustering
+ts_clust_res = run_clust(ts_input_data_mod;method="kmeans",representation="centroid",n_init=10,n_clust=5) # default k-means
+
+# representation modification
+ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results)
+
+ts_clust_res_extr = ClustResult(ts_clust_res,ts_clust_extr,extr_idcs,rep_mod_method,ev,"none")
+
+ # optimization
+opt_res = run_opt(ts_clust_res_extr.best_results,cep_input_data_GER;solver=GurobiSolver(),co2_limit=1000.0)

src/clustering/extreme_vals.jl

@@ -1,3 +1,145 @@
+"""
+
+ts_data: The full input data 365. Used for individual opt run
+ts_data_mod: The input data used for clustering (365, or 365-extreme values in the case of append)
+clust_data: The clustered data: n_clust + extreme values 
+"""
+function run_clust_extr(
+      ts_data::ClustData,
+      opt_data::OptDataCEP,
+      simple_extr_value_descr_ar::Array{SimpleExtremeValueDescr,1};
+      norm_op::String="zscore",
+      norm_scope::String="full",
+      method::String="kmeans",
+      representation::String="centroid",
+      n_clust::Int=5,
+      n_init::Int=100,
+      iterations::Int=300,
+      attribute_weights::Dict{String,Float64}=Dict{String,Float64}(),
+      extreme_event_selection_method::String="feasibility",
+      rep_mod_method::String="feasibility",
+      save::String="",
+      get_all_clust_results::Bool=false,
+      solver::Any=CbcSolver(),
+      descriptor::String="",
+      co2_limit::Number=Inf,
+      slack_cost::Number=Inf,
+      existing_infrastructure::Bool=false,
+      limit_infrastructure::Bool=false,
+      storage::String="non",
+      transmission::Bool=false,
+      k_ids::Array{Int64,1}=Array{Int64,1}(),
+      print_flag::Bool=true,
+      kwargs...
+                        # simple_extreme_days=true
+                        # extreme_day_selection_method="feasibility", "slack", "none"
+                        # + extreme_value_descr_ar
+                        # needs input data for optimization problem
+                        )
+    # QUESTION: should keyword arguments be specified or rather be kwargs? kwargs may not work because the subsequent functions would through an error that some of the keyword arguments are not supported
+    # TODO: Specify keywords arguemnt as three seperate arrays, one for clustering, and one for extreme values, one for optimization problem
+    # simple extreme value selection
+    use_simple_extr = !isempty(simple_extr_value_descr_ar)
+    extr_vals=ClustData
+    extr_idcs=Int[]
+    ts_data_mod=ts_data
+    if use_simple_extr
+       ts_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_data,simple_extr_value_descr_ar;rep_mod_method=rep_mod_method)
+    end
+
+    # run initial clustering 
+    clust_res = run_clust(ts_data_mod;norm_op=norm_op,norm_scope=norm_scope,method=method,representation=representation,n_clust=n_clust,n_init=n_init,iterations=iterations,attribute_weights=attribute_weights,save=save,get_all_clust_results=get_all_clust_results,kwargs...)
+
+   # if simple: representation modification
+    clust_data=clust_res.best_results
+    if use_simple_extr
+      clust_data = representation_modification(extr_vals,clust_data)
+    end
+
+    if extreme_event_selection_method=="none"
+     return ClustResult(clust_res,clust_data,extr_idcs,rep_mod_method,simple_extr_value_descr_ar,extreme_event_selection_method) 
+    elseif (extreme_event_selection_method !="feasibility") && (extreme_event_selection_method != "slack")
+      @warn "extreme_event_selection_method - "*extreme_event_selection_method*" - does not match any of the three predefined keywords: feasibility, append, none. The function assumes -none-."
+
+     return ClustResult(clust_res,clust_data,extr_idcs,rep_mod_method,simple_extr_value_descr_ar,extreme_event_selection_method) 
+    end
+
+    # convert ts_data into N individual ClustData structs
+    ts_data_indiv_ar = clustData_individual(ts_data) 
+    is_feasible = false # indicates if optimization result from clustered input data is feasible on operatoins optimization with full input data 
+
+    i=0 
+    while !is_feasible 
+      i+=1
+      # initial design and operations optimization
+      d_o_opt = run_opt(clust_data,opt_data;solver=solver,descriptor=descriptor,co2_limit=co2_limit,existing_infrastructure=existing_infrastructure,limit_infrastructure=limit_infrastructure,storage=storage,transmission=transmission,slack_cost=Inf,print_flag=print_flag)
+      dvs = get_cep_design_variables(d_o_opt)
+      # run individual optimization with fixed design
+      o_opt_individual = OptResult[]
+      if extreme_event_selection_method=="feasibility"
+        eval_res = Symbol[] 
+      elseif extreme_event_selection_method=="slack"
+        eval_res = OptVariable[] 
+      end 
+      for k=1:ts_data.K
+        # TODO: include in run_opt an option to turn off warnings. This optimization is often infeasible, and it currently gives a warning every time. There should be an option for this case to turn it off. 
+        if extreme_event_selection_method=="feasibility"
+           push!(o_opt_individual,run_opt(ts_data_indiv_ar[k],opt_data,d_o_opt.opt_config,dvs;solver=solver,slack_cost=Inf))
+           push!(eval_res,o_opt_individual[k].status)
+        elseif extreme_event_selection_method=="slack"
+           slack_cost==Inf && (@warn "extreme_event_selection_method is -slack-,but slack cost are Inf")
+          # ### TODO: THIS IS JUST A TEST
+          # set_clust_config!(d_o_opt.opt_config;co2_limit=Inf)
+           push!(o_opt_individual,run_opt(ts_data_indiv_ar[k],opt_data,d_o_opt.opt_config,dvs;solver=solver,slack_cost=slack_cost))
+           push!(eval_res,get_cep_slack_variables(o_opt_individual[k]))
+         end 
+      end
+      is_feasible = check_indiv_opt_feasibility(eval_res)
+      println("feasibility: ",is_feasible, " i=",i)  # TODO - delete this line
+      is_feasible && return ClustResult(clust_res,clust_data,extr_idcs,rep_mod_method,simple_extr_value_descr_ar,extreme_event_selection_method) 
+
+      # get infeasible value
+      idx_infeas = get_index_inf(eval_res)
+      push!(extr_idcs,idx_infeas)
+      println(idx_infeas)
+      extr_val_inf = extreme_val_output(ts_data,idx_infeas,rep_mod_method=rep_mod_method)
+      # add extr_val_inf to extr_vals (using representation modification method)
+      if typeof(extr_vals)==DataType
+        extr_vals=extr_val_inf
+      else
+        extr_vals = representation_modification(extr_val_inf,extr_vals)
+      end
+      if rep_mod_method=="append"
+        ts_data_mod = input_data_modification(ts_data,extr_idcs)
+        clust_res = run_clust(ts_data_mod;norm_op=norm_op,norm_scope=norm_scope,method=method,representation=representation,n_clust=n_clust,n_init=n_init,iterations=iterations,attribute_weights=attribute_weights,save=save,get_all_clust_results=get_all_clust_results,kwargs...)
+        clust_data=clust_res.best_results
+        clust_data = representation_modification(extr_vals,clust_data) 
+      elseif rep_mod_method == "feasibility"
+        clust_data = representation_modification(extr_val_inf,clust_data) 
+      else 
+        @error "rep_mod_method does not exist" # TODO: Write automatic check functions for the different options
+      end
+    end
+    return ClustResult(clust_res,clust_data,extr_idcs,rep_mod_method,simple_extr_value_descr_ar,extreme_event_selection_method) # TODO: adjust clust_config in these functions
+end
+
+"""
+    function run_clust_extr(
+          ts_data::ClustData,
+          opt_data::OptDataCEP;
+          kwargs...
+          )
+
+Clustering and extreme value selection WITHOUT simple extreme values.
+"""
+function run_clust_extr(
+      ts_data::ClustData,
+      opt_data::OptDataCEP;
+      kwargs...
+      )
+      return run_clust_extr(ts_data,opt_data,SimpleExtremeValueDescr[];kwargs...)
+end
+
 """
 function simple_extr_val_sel(data::ClustData,
                              extreme_value_descr_ar::Array{SimpleExtremeValueDescr,1};
@@ -197,6 +339,7 @@ Merges the clustered data and extreme vals into one ClustData struct. Weights ar
 function representation_modification(extr_vals::ClustData,
                                      clust_data::ClustData,
                                      )
+                                     #TODO: The input order of extr_vals and clust_data should probably be reversed. Usually, we return the modified version of the first input argument.
   K_mod = clust_data.K + extr_vals.K
   data_mod=Dict{String,Array}()
   for dt in keys(clust_data.data)

src/optim_problems/opt_cep.jl

@@ -397,11 +397,12 @@ function setup_opt_cep_co2_limit!(cep::OptModelCEP,
   set=cep.set
   #ts          Dict( tech-node ): t x k
   ts=ts_data.data
-
+  w = ts_data.weights
+
   ## EMISSIONS ##
   # Limit the Emissions with co2_limit if it exists
   push!(cep.info,"ΣCOST_{account,tech}[account,'$(set["impact"][1])',tech] ≤ co2_limit*Σ_{node,t,k}ts[el_demand-node,t,k]")
-  @constraint(cep.model, sum(cep.model[:COST][account,"CO2",tech] for account=set["account"], tech=set["tech"])<= co2_limit*sum(sum(ts["el_demand-"*node]) for node=set["nodes"]))
+  @constraint(cep.model, sum(cep.model[:COST][account,"CO2",tech] for account=set["account"], tech=set["tech"])<= co2_limit*sum(sum(ts["el_demand-"*node]*w) for node=set["nodes"]))
   return cep
 end
 

src/optim_problems/run_opt.jl

@@ -59,6 +59,7 @@ function run_opt(ts_data::ClustData,
                     slack_cost::Number=Inf,
                     k_ids::Array{Int64,1}=Array{Int64,1}())
   # Add the fixed_design_variables and new setting for slack costs to the existing config
+  # TODO: Think about changing opt config as a deepcopy function.Possible issue: the opt_config of the original  problem is modified as well.
   set_opt_config_cep!(opt_config;fixed_design_variables=fixed_design_variables, slack_cost=slack_cost)
   return run_opt(ts_data,opt_data,opt_config;solver=solver,k_ids=k_ids)
 end

src/utils/datastructs.jl

@@ -306,6 +306,25 @@ function ClustData(data::FullInputData,
   return ClustData(data.region,K,T,data_reshape,ones(K))
 end
 
+"""
+function ClustData_individual(data::ClustData)
+
+Takes a ClustData struct and returns an array of ClustData structs that contains each period individually.
+"""
+function clustData_individual(data::ClustData)
+  clust_data_indiv = ClustData[]
+  for kk=1:data.K
+    # initialize new dict
+    data_dict_indiv = Dict{String,Array}()
+    # fill dict with data
+    for (k,v) in data.data
+      data_dict_indiv[k] = v[:,kk:kk]  # kk:kk instead of k ensures that it returns a two-dimensional array instead of a vector during array slicing with singleton dimension
+    end
+    push!(clust_data_indiv,ClustData(data.region,1,data.T,data_dict_indiv,[data.weights[kk]];mean=data.mean,sdv=data.sdv))    
+  end
+  return clust_data_indiv
+end
+
 """
 constructor 1: construct ClustDataMerged
 function ClustDataMerged(region::String,
@@ -354,3 +373,63 @@ function ClustDataMerged(data::ClustData)
  end
  ClustDataMerged(data.region,data.K,data.T,data_merged,data_type,data.weights,data.mean,data.sdv)
 end
+
+"""
+function ClustResult(clust_res::ClustResultBest,clust_data_mod::ClustData;kwargs...)
+
+adjusts ClustResult best_results. To be used to modify clustered data with extreme values.
+Adds kwargs to clust_config (can e.g. be used to add extreme value information)
+"""
+function ClustResult(clust_res::ClustResultBest,
+                     clust_data_mod::ClustData;
+                     kwargs...)
+  clust_config=clust_res.clust_config
+  set_clust_config!(clust_config;kwargs...)
+  return ClustResultBest(clust_data_mod,clust_res.best_ids,clust_res.best_cost,clust_res.data_type,clust_config)
+end
+
+
+"""
+function ClustResult(clust_res::ClustResultBest,clust_data_mod::ClustData;kwargs...)
+
+adjusts ClustResult best_results. To be used to modify clustered data with extreme values.
+Wrapper function that makes explicit the arguments that it adds (extreme value arguments) to clust config
+"""
+function ClustResult(clust_res::ClustResultBest,
+                     clust_data_mod::ClustData,
+                     extr_idcs::Array{Int,1},
+                     rep_mod_method::String,
+                     simple_extr_value_descr_ar::Array{SimpleExtremeValueDescr,1},
+                     extreme_event_selection_method::String
+                     )
+  return ClustResult(clust_res,clust_data_mod;extr_idcs=extr_idcs,rep_mod_method=rep_mod_method,simple_extr_value_descr_ar=simple_extr_value_descr_ar,extreme_event_selection_method=extreme_event_selection_method)
+end
+
+"""
+function ClustResult(clust_res::ClustResultAll,clust_data_mod::ClustData;kwargs...)
+
+adjusts ClustResult best_results. To be used to modify clustered data with extreme values.
+"""
+function ClustResult(clust_res::ClustResultAll,
+                     clust_data_mod::ClustData;
+                     kwargs...)
+  clust_config=clust_res.clust_config
+  set_clust_config!(clust_config;kwargs...)
+  return ClustResultAll(clust_data_mod,clust_res.best_ids,clust_res.best_cost,clust_res.data_type,clust_config,clust_res.centers,clust_res.weights,clust_res.clustids,clust_res.cost,clust_res.iter)
+end
+
+"""
+function ClustResult(clust_res::ClustResultAll,clust_data_mod::ClustData;kwargs...)
+
+adjusts ClustResult all. To be used to modify clustered data with extreme values.
+Wrapper function that makes explicit the arguments that it adds (extreme value arguments) to clust config
+"""
+function ClustResult(clust_res::ClustResultAll,
+                     clust_data_mod::ClustData,
+                     extr_idcs::Array{Int,1},
+                     rep_mod_method::String,
+                     simple_extr_value_descr_ar::Array{SimpleExtremeValueDescr,1},
+                     extreme_event_selection_method::String
+                     )
+  return ClustResult(clust_res,clust_data_mod;extr_idcs=extr_idcs,rep_mod_method=rep_mod_method,simple_extr_value_descr_ar=simple_extr_value_descr_ar,extreme_event_selection_method=extreme_event_selection_method)
+end