From 454be31b4511eea7cfee6e6eded12062811b8d9e Mon Sep 17 00:00:00 2001 From: holgerteichgraeber Date: Wed, 12 Dec 2018 15:53:03 -0800 Subject: [PATCH 01/11] pseudocode extreme value selection procedure --- src/clustering/extreme_vals.jl | 64 ++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/clustering/extreme_vals.jl b/src/clustering/extreme_vals.jl index 98cb776..af32d59 100644 --- a/src/clustering/extreme_vals.jl +++ b/src/clustering/extreme_vals.jl @@ -1,3 +1,67 @@ +""" +""" +function run_clust_extr( + data::ClustData; + norm_op::String="zscore", + norm_scope::String="full", + method::String="kmeans", + representation::String="centroid", + n_clust::Int=5, + n_init::Int=100, + iterations::Int=300, + attribute_weights::Dict{String,Float64}=Dict{String,Float64}(), + simple_extreme_event::Bool=true, + extreme_event_selection_method="feasibility" + save::String="", + get_all_clust_results::Bool=false, + kwargs... + # same inputs as in run_clust() + # simple_extreme_days=true + # extreme_day_selection_method="feasibility", "append", "none" + # + extreme_value_descr_ar + # needs input data for optimization problem + ) + # if simple: simple_extr_val_sel + # clust_res = run_clust() + # if simple: representation modification + # DVs = run_opt().variables["CAP"] + # is_feasible = false + # while !is_feasible + # for i=1:365 + # method that puts one day out of ClustData into its own ClustData struct + # run_opt() single day, given DVs(operations only) + # end + # is_feasible = [depends on if any day was infeasible] + # if is_feasible: break out of while loop here + # idx_extr_val = feasibility: first infeasible index; append: idx with highest slack variable + # extr_val_output(idx_extr_val) + # representation_modification() + # if append: run_clust() # really? should representation modification not be afterwards? / figure 2.5 Constantin Thesis + # + # DVs = run_opt().variables["CAP"] + # end + # + # + # + # + # + # + # + # + # + # + # + # + # + # + # + # + # + return 0 + +end + + """ function simple_extr_val_sel(data::ClustData, extreme_value_descr_ar::Array{SimpleExtremeValueDescr,1}; From 5bc4cff53e249b2336ebc11593e6a4463c91be7d Mon Sep 17 00:00:00 2001 From: holgerteichgraeber Date: Mon, 17 Dec 2018 15:47:35 -0800 Subject: [PATCH 02/11] simple extreme values in one function --- examples/workflow_example_extr.jl | 16 ++++++--- examples/workflow_example_extr_simple.jl | 28 ++++++++++++++++ src/clustering/extreme_vals.jl | 41 ++++++++++++++++++++---- src/utils/datastructs.jl | 22 +++++++++++++ 4 files changed, 96 insertions(+), 11 deletions(-) create mode 100644 examples/workflow_example_extr_simple.jl diff --git a/examples/workflow_example_extr.jl b/examples/workflow_example_extr.jl index 7cfa6a2..98feb00 100644 --- a/examples/workflow_example_extr.jl +++ b/examples/workflow_example_extr.jl @@ -16,13 +16,21 @@ cep_input_data_GER=load_cep_data("GER_18") ev3 = SimpleExtremeValueDescr("el_demand-dena21","max","absolute") ev = [ev1, ev2, ev3] # simple extreme day selection - ts_input_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_input_data,ev;rep_mod_method="feasibility") + #ts_input_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_input_data,ev;rep_mod_method="feasibility") # run clustering -ts_clust_res = run_clust(ts_input_data_mod;method="kmeans",representation="centroid",n_init=10,n_clust=5) # default k-means +#ts_clust_res = run_clust(ts_input_data_mod;method="kmeans",representation="centroid",n_init=10,n_clust=5) # default k-means # representation modification -ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results) +#ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results) + + ts_clust_res = run_clust_extr(ts_input_data,ev;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5) # optimization -opt_res = run_opt(ts_clust_extr,cep_input_data_GER;solver=GurobiSolver(),co2limit=1000.0) +opt_res = run_opt(ts_clust_res.best_results,cep_input_data_GER;solver=GurobiSolver(),co2_limit=1000.0) + +#TODO: write functions to get generation, capacity etc. + +# TODO: write plotting functions in other package + + diff --git a/examples/workflow_example_extr_simple.jl b/examples/workflow_example_extr_simple.jl new file mode 100644 index 0000000..81a0216 --- /dev/null +++ b/examples/workflow_example_extr_simple.jl @@ -0,0 +1,28 @@ +# This file exemplifies the workflow from data input to optimization result generation +#QUESTION using ClustForOpt_priv.col in module Main conflicts with an existing identifier., using ClustForOpt_priv.cols in module Main conflicts with an existing identifier. + +include(normpath(joinpath(dirname(@__FILE__),"..","src","ClustForOpt_priv_development.jl"))) +#using ClustForOpt_priv +#using Gurobi + +# load data +ts_input_data, = load_timeseries_data("CEP", "GER_18";K=365, T=24) #CEP + +cep_input_data_GER=load_cep_data("GER_18") + + # define simple extreme days of interest + ev1 = SimpleExtremeValueDescr("wind-dena42","max","absolute") + ev2 = SimpleExtremeValueDescr("solar-dena42","min","integral") + ev3 = SimpleExtremeValueDescr("el_demand-dena21","max","absolute") + ev = [ev1, ev2, ev3] + # simple extreme day selection + ts_input_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_input_data,ev;rep_mod_method="feasibility") + + # run clustering +ts_clust_res = run_clust(ts_input_data_mod;method="kmeans",representation="centroid",n_init=10,n_clust=5) # default k-means + +# representation modification +ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results) + + # optimization +opt_res = run_opt(ts_clust_extr,cep_input_data_GER;solver=GurobiSolver(),co2_limit=1000.0) diff --git a/src/clustering/extreme_vals.jl b/src/clustering/extreme_vals.jl index af32d59..cc3aaba 100644 --- a/src/clustering/extreme_vals.jl +++ b/src/clustering/extreme_vals.jl @@ -1,7 +1,9 @@ """ """ function run_clust_extr( - data::ClustData; + data::ClustData, + extr_value_descr_ar::Array{SimpleExtremeValueDescr,1}; + rep_mod_method::String="feasibility", norm_op::String="zscore", norm_scope::String="full", method::String="kmeans", @@ -11,7 +13,7 @@ function run_clust_extr( iterations::Int=300, attribute_weights::Dict{String,Float64}=Dict{String,Float64}(), simple_extreme_event::Bool=true, - extreme_event_selection_method="feasibility" + extreme_event_selection_method="feasibility", save::String="", get_all_clust_results::Bool=false, kwargs... @@ -21,10 +23,22 @@ function run_clust_extr( # + extreme_value_descr_ar # needs input data for optimization problem ) - # if simple: simple_extr_val_sel - # clust_res = run_clust() + # QUESTION: should keyword arguments be specified or rather be kwargs? kwargs may not work because the subsequent functions would through an error that some of the keyword arguments are not supported + # simple extreme value selection + use_simple_extr = !isempty(extr_value_descr_ar) + if use_simple_extr + data_mod,extr_vals,extr_idcs = simple_extr_val_sel(data,extr_value_descr_ar;rep_mod_method=rep_mod_method) + else + data_mod=data + end + # run initial clustering + clust_res = run_clust(data_mod;norm_op=norm_op,norm_scope=norm_scope,method=method,representation=representation,n_clust=n_clust,n_init=n_init,iterations=iterations,attribute_weights=attribute_weights,save=save,get_all_clust_results=get_all_clust_results,kwargs...) # if simple: representation modification - # DVs = run_opt().variables["CAP"] + clust_data=clust_res.best_results + if use_simple_extr + clust_data = representation_modification(extr_vals,clust_data) + end + # DVs = run_opt().variables["CAP"] # how to make this generic, so that it alwasy takes dvs - write get_opt_dvs() # is_feasible = false # while !is_feasible # for i=1:365 @@ -56,11 +70,24 @@ function run_clust_extr( # # # - # - return 0 + + + return ClustResult(clust_res,clust_data) # TODO: adjust clust_config in these functions end +""" +wrapper function without simple extreme values +""" +function run_clust_extr( + data::ClustData; + kwargs... + ) + return run_clust_extr(data,[];kwargs...) + +end + + """ function simple_extr_val_sel(data::ClustData, diff --git a/src/utils/datastructs.jl b/src/utils/datastructs.jl index 6a01e84..684d678 100644 --- a/src/utils/datastructs.jl +++ b/src/utils/datastructs.jl @@ -353,3 +353,25 @@ function ClustDataMerged(data::ClustData) end ClustDataMerged(data.region,data.K,data.T,data_merged,data_type,data.weights,data.mean,data.sdv) end + +""" +constructor for ClustResultBest + +function ClustResult(clust_res::ClustResultBest,clust_data_mod::ClustData) + +adjusts ClustResult best_results. To be used to modify clustered data with extreme values. +""" +function ClustResult(clust_res::ClustResultBest,clust_data_mod::ClustData) + return ClustResultBest(clust_data_mod,clust_res.best_ids,clust_res.best_cost,clust_res.data_type,clust_res.clust_config) +end + +""" +constructor for ClustResultAll + +function ClustResult(clust_res::ClustResultAll,clust_data_mod::ClustData) + +adjusts ClustResult best_results. To be used to modify clustered data with extreme values. +""" +function ClustResult(clust_res::ClustResultAll,clust_data_mod::ClustData) + return ClustResultAll(clust_data_mod,clust_res.best_ids,clust_res.best_cost,clust_res.data_type,clust_res.clust_config,clust_res.centers,clust_res.weights,clust_res.clustids,clust_res.cost,clust_res.iter) +end From d2dd1e74e32aaa02f05ad44f8ecef5bf07f4aa38 Mon Sep 17 00:00:00 2001 From: holgerteichgraeber Date: Mon, 17 Dec 2018 18:12:54 -0800 Subject: [PATCH 03/11] Put ClustData in individual ClustData structs for each k in K. --- examples/workflow_example_extr.jl | 2 +- src/clustering/extreme_vals.jl | 48 +++++++++++++++++++++++++------ src/utils/datastructs.jl | 23 ++++++++++++--- 3 files changed, 59 insertions(+), 14 deletions(-) diff --git a/examples/workflow_example_extr.jl b/examples/workflow_example_extr.jl index 5dde785..4a0939d 100644 --- a/examples/workflow_example_extr.jl +++ b/examples/workflow_example_extr.jl @@ -23,7 +23,7 @@ cep_input_data_GER=load_cep_data("GER_18") # representation modification #ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results) - ts_clust_res = run_clust_extr(ts_input_data,ev;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5) + ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER,ev;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=GurobiSolver()) # optimization opt_res = run_opt(ts_clust_res.best_results,cep_input_data_GER;solver=GurobiSolver(),co2_limit=1000.0) diff --git a/src/clustering/extreme_vals.jl b/src/clustering/extreme_vals.jl index cc3aaba..e3a1eb1 100644 --- a/src/clustering/extreme_vals.jl +++ b/src/clustering/extreme_vals.jl @@ -1,9 +1,9 @@ """ """ function run_clust_extr( - data::ClustData, + ts_data::ClustData, + opt_data::OptDataCEP, extr_value_descr_ar::Array{SimpleExtremeValueDescr,1}; - rep_mod_method::String="feasibility", norm_op::String="zscore", norm_scope::String="full", method::String="kmeans", @@ -12,12 +12,20 @@ function run_clust_extr( n_init::Int=100, iterations::Int=300, attribute_weights::Dict{String,Float64}=Dict{String,Float64}(), - simple_extreme_event::Bool=true, extreme_event_selection_method="feasibility", + rep_mod_method::String="feasibility", save::String="", get_all_clust_results::Bool=false, + solver::Any=CbcSolver(), + descriptor::String="", + co2_limit::Number=Inf, + slack_cost::Number=Inf, + existing_infrastructure::Bool=false, + limit_infrastructure::Bool=false, + storage::String="non", + transmission::Bool=false, + k_ids::Array{Int64,1}=Array{Int64,1}(), kwargs... - # same inputs as in run_clust() # simple_extreme_days=true # extreme_day_selection_method="feasibility", "append", "none" # + extreme_value_descr_ar @@ -27,19 +35,41 @@ function run_clust_extr( # simple extreme value selection use_simple_extr = !isempty(extr_value_descr_ar) if use_simple_extr - data_mod,extr_vals,extr_idcs = simple_extr_val_sel(data,extr_value_descr_ar;rep_mod_method=rep_mod_method) + ts_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_data,extr_value_descr_ar;rep_mod_method=rep_mod_method) else - data_mod=data + ts_data_mod=ts_data end # run initial clustering - clust_res = run_clust(data_mod;norm_op=norm_op,norm_scope=norm_scope,method=method,representation=representation,n_clust=n_clust,n_init=n_init,iterations=iterations,attribute_weights=attribute_weights,save=save,get_all_clust_results=get_all_clust_results,kwargs...) + clust_res = run_clust(ts_data_mod;norm_op=norm_op,norm_scope=norm_scope,method=method,representation=representation,n_clust=n_clust,n_init=n_init,iterations=iterations,attribute_weights=attribute_weights,save=save,get_all_clust_results=get_all_clust_results,kwargs...) # if simple: representation modification clust_data=clust_res.best_results if use_simple_extr clust_data = representation_modification(extr_vals,clust_data) end - # DVs = run_opt().variables["CAP"] # how to make this generic, so that it alwasy takes dvs - write get_opt_dvs() - # is_feasible = false + # initial design and operations optimization + d_o_opt = run_opt(clust_data,opt_data;solver=solver,descriptor=descriptor,co2_limit=co2_limit,slack_cost=slack_cost,existing_infrastructure=existing_infrastructure,limit_infrastructure=limit_infrastructure,storage=storage,transmission=transmission,k_ids=k_ids) + dvs = get_cep_design_variables(d_o_opt) + + # convert ts_data into K individual ClustData structs + ts_data_mod_indiv_ar = clustData_individual(ts_data_mod) + is_feasible = false # indicates if optimization result from clustered input data is feasible on operatoins optimization with full input data + i=0 + while !is_feasible + i+=1 + ### TODO: Pick up here on TUESDAY + o_opt_individual = OptResult[] + status = Symbol[] + slack_vars = [] + for k=1:ts_data_mod.K + # if feasibility: + # run without slack,store status + # if append + # run with slack, store slack + push!(o_opt_individual,run_opt()) + + end + + end # while !is_feasible # for i=1:365 # method that puts one day out of ClustData into its own ClustData struct diff --git a/src/utils/datastructs.jl b/src/utils/datastructs.jl index aa7af59..c47dac4 100644 --- a/src/utils/datastructs.jl +++ b/src/utils/datastructs.jl @@ -306,6 +306,25 @@ function ClustData(data::FullInputData, return ClustData(data.region,K,T,data_reshape,ones(K)) end +""" +function ClustData_individual(data::ClustData) + +Takes a ClustData struct and returns an array of ClustData structs that contains each period individually. +""" +function clustData_individual(data::ClustData) + clust_data_indiv = ClustData[] + for kk=1:data.K + # initialize new dict + data_dict_indiv = Dict{String,Array}() + # fill dict with data + for (k,v) in data.data + data_dict_indiv[k] = v[:,kk:kk] # kk:kk instead of k ensures that it returns a two-dimensional array instead of a vector during array slicing with singleton dimension + end + push!(clust_data_indiv,ClustData(data.region,1,data.T,data_dict_indiv,[data.weights[kk]];mean=data.mean,sdv=data.sdv)) + end + return clust_data_indiv +end + """ constructor 1: construct ClustDataMerged function ClustDataMerged(region::String, @@ -356,8 +375,6 @@ function ClustDataMerged(data::ClustData) end """ -constructor for ClustResultBest - function ClustResult(clust_res::ClustResultBest,clust_data_mod::ClustData) adjusts ClustResult best_results. To be used to modify clustered data with extreme values. @@ -367,8 +384,6 @@ function ClustResult(clust_res::ClustResultBest,clust_data_mod::ClustData) end """ -constructor for ClustResultAll - function ClustResult(clust_res::ClustResultAll,clust_data_mod::ClustData) adjusts ClustResult best_results. To be used to modify clustered data with extreme values. From 91fec3d9ca4eac5416c68ee5de0a0a42b2385a8b Mon Sep 17 00:00:00 2001 From: holgerteichgraeber Date: Tue, 18 Dec 2018 14:15:52 -0800 Subject: [PATCH 04/11] individual optimization and feasibility check --- examples/workflow_example_extr.jl | 22 +++++++----- src/clustering/extreme_vals.jl | 57 +++++++++++++++++++++---------- src/utils/utils.jl | 22 ++++++++++-- 3 files changed, 73 insertions(+), 28 deletions(-) diff --git a/examples/workflow_example_extr.jl b/examples/workflow_example_extr.jl index 4a0939d..2c88241 100644 --- a/examples/workflow_example_extr.jl +++ b/examples/workflow_example_extr.jl @@ -2,17 +2,23 @@ include(normpath(joinpath(dirname(@__FILE__),"..","src","ClustForOpt_priv_development.jl"))) #using ClustForOpt_priv -#using Gurobi +using Gurobi +env = Gurobi.Env() # reusing the same gurobi environment for multiple solves +# select solver +solver=GurobiSolver(env,OutputFlag=0) # load data -ts_input_data, = load_timeseries_data("CEP", "GER_18";K=365, T=24) #CEP +ts_input_data, = load_timeseries_data("CEP", "TX_1";K=365, T=24) #CEP -cep_input_data_GER=load_cep_data("GER_18") +cep_input_data_GER=load_cep_data("TX_1") # define simple extreme days of interest - ev1 = SimpleExtremeValueDescr("wind-dena42","max","absolute") - ev2 = SimpleExtremeValueDescr("solar-dena42","min","integral") - ev3 = SimpleExtremeValueDescr("el_demand-dena21","max","absolute") +ev1 = SimpleExtremeValueDescr("wind-node61","max","absolute") +ev2 = SimpleExtremeValueDescr("solar-node61","min","integral") +ev3 = SimpleExtremeValueDescr("el_demand-node61","max","absolute") +# ev1 = SimpleExtremeValueDescr("wind-dena42","max","absolute") +# ev2 = SimpleExtremeValueDescr("solar-dena42","min","integral") +# ev3 = SimpleExtremeValueDescr("el_demand-dena21","max","absolute") ev = [ev1, ev2, ev3] # simple extreme day selection #ts_input_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_input_data,ev;rep_mod_method="feasibility") @@ -23,10 +29,10 @@ cep_input_data_GER=load_cep_data("GER_18") # representation modification #ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results) - ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER,ev;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=GurobiSolver()) + ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER,ev;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra") # optimization -opt_res = run_opt(ts_clust_res.best_results,cep_input_data_GER;solver=GurobiSolver(),co2_limit=1000.0) +#opt_res = run_opt(ts_clust_res.best_results,cep_input_data_GER;solver=GurobiSolver(),co2_limit=1000.0) #TODO: write functions to get generation, capacity etc. diff --git a/src/clustering/extreme_vals.jl b/src/clustering/extreme_vals.jl index e3a1eb1..b9e4d04 100644 --- a/src/clustering/extreme_vals.jl +++ b/src/clustering/extreme_vals.jl @@ -27,7 +27,7 @@ function run_clust_extr( k_ids::Array{Int64,1}=Array{Int64,1}(), kwargs... # simple_extreme_days=true - # extreme_day_selection_method="feasibility", "append", "none" + # extreme_day_selection_method="feasibility", "slack", "none" # + extreme_value_descr_ar # needs input data for optimization problem ) @@ -46,29 +46,46 @@ function run_clust_extr( if use_simple_extr clust_data = representation_modification(extr_vals,clust_data) end + + if extreme_event_selection_method=="none" + return ClustResult(clust_res,clust_data) # TODO: adjust clust_config in these functions + elseif (extreme_event_selection_method !="feasibility") && (extreme_event_selection_method != "slack") + @warn "extreme_event_selection_method - "*extreme_event_selection_method*" - does not match any of the three predefined keywords: feasibility, append, none. The function assumes -none-." + return ClustResult(clust_res,clust_data) # TODO: adjust clust_config in these functions + end + # initial design and operations optimization - d_o_opt = run_opt(clust_data,opt_data;solver=solver,descriptor=descriptor,co2_limit=co2_limit,slack_cost=slack_cost,existing_infrastructure=existing_infrastructure,limit_infrastructure=limit_infrastructure,storage=storage,transmission=transmission,k_ids=k_ids) + d_o_opt = run_opt(clust_data,opt_data;solver=solver,descriptor=descriptor,co2_limit=co2_limit,existing_infrastructure=existing_infrastructure,limit_infrastructure=limit_infrastructure,storage=storage,transmission=transmission,slack_cost=Inf) dvs = get_cep_design_variables(d_o_opt) # convert ts_data into K individual ClustData structs ts_data_mod_indiv_ar = clustData_individual(ts_data_mod) is_feasible = false # indicates if optimization result from clustered input data is feasible on operatoins optimization with full input data + if extreme_event_selection_method=="feasibility" + eval_res = Symbol[] + elseif extreme_event_selection_method=="slack" + eval_res = OptVariable[] + end i=0 while !is_feasible i+=1 - ### TODO: Pick up here on TUESDAY o_opt_individual = OptResult[] - status = Symbol[] - slack_vars = [] + # run individual optimization with fixed design for k=1:ts_data_mod.K - # if feasibility: - # run without slack,store status - # if append - # run with slack, store slack - push!(o_opt_individual,run_opt()) - + if extreme_event_selection_method=="feasibility" + push!(o_opt_individual,run_opt(ts_data_mod_indiv_ar[k],opt_data,d_o_opt.opt_config,dvs;solver=solver,slack_cost=Inf)) + push!(eval_res,o_opt_individual[k].status) + elseif extreme_event_selection_method=="slack" + slack_cost==Inf && (@warn "extreme_event_selection_method is -slack-,but slack cost are Inf") + push!(o_opt_individual,run_opt(ts_data_mod_indiv_ar[k],opt_data,d_o_opt.opt_config,dvs;solver=solver,slack_cost=slack_cost)) + push!(eval_res,get_cep_slack_variables(o_opt_individual[k])) + end end - + is_feasible = check_indiv_opt_feasibility(eval_res) + println(eval_res, "feasibility: ",is_feasible, " i=",i) + is_feasible && return ClustResult(clust_res,clust_data) # TODO: adjust clust_config in these functions + + end # while !is_feasible # for i=1:365 @@ -107,18 +124,22 @@ function run_clust_extr( end """ -wrapper function without simple extreme values + function run_clust_extr( + ts_data::ClustData, + opt_data::OptDataCEP; + kwargs... + ) + +Clustering and extreme value selection WITHOUT simple extreme values. """ function run_clust_extr( - data::ClustData; + ts_data::ClustData, + opt_data::OptDataCEP; kwargs... ) - return run_clust_extr(data,[];kwargs...) - + return run_clust_extr(ts_data,opt_data,SimpleExtremeValueDescr[];kwargs...) end - - """ function simple_extr_val_sel(data::ClustData, extreme_value_descr_ar::Array{SimpleExtremeValueDescr,1}; diff --git a/src/utils/utils.jl b/src/utils/utils.jl index 5673c48..6d7fd1e 100644 --- a/src/utils/utils.jl +++ b/src/utils/utils.jl @@ -474,9 +474,9 @@ end """ function get_cep_slack_variables(opt_result::OptResult) - Returns the SLACK variable of this opt_result matching "sv" + Returns the SLACK variables of this opt_result matching "sv" """ -function get_cep_slack_variable(opt_result::OptResult) +function get_cep_slack_variables(opt_result::OptResult) if "SLACK" in keys(opt_result.variables) return opt_result.variables["SLACK"] else @@ -484,6 +484,24 @@ function get_cep_slack_variable(opt_result::OptResult) end end +""" + function check_indiv_opt_feasibility(status::Array{Symbol,1}) + +checks feasibility of optimization problem based on status +""" +function check_indiv_opt_feasibility(status::Array{Symbol,1}) + return (sum(status.!=:Optimal) == 0) +end + +""" + function check_indiv_opt_feasibility(slack_vars::Array{OptVariable,1}) + +checks feasibility of optimization problem based on slack variables +""" +function check_indiv_opt_feasibility(slack_vars::Array{OptVariable,1}) + return all(sum.([slack_vars[i].data for i=1:length(slack_vars)]) .== 0) +end + """ function set_opt_config_cep(opt_data::OptDataCEP; kwargs...) kwargs can be whatever you need to run the run_opt From eeee7a10a274adce1fd199ba5067fded23720fa4 Mon Sep 17 00:00:00 2001 From: holgerteichgraeber Date: Fri, 21 Dec 2018 11:49:53 -0800 Subject: [PATCH 05/11] run_clust_extr() first version. Bug exists. --- examples/workflow_example_extr.jl | 17 ++++++---- src/clustering/extreme_vals.jl | 54 ++++++++++++++++++++++--------- src/utils/utils.jl | 22 ++++++++++++- 3 files changed, 70 insertions(+), 23 deletions(-) diff --git a/examples/workflow_example_extr.jl b/examples/workflow_example_extr.jl index 2c88241..1c1636b 100644 --- a/examples/workflow_example_extr.jl +++ b/examples/workflow_example_extr.jl @@ -8,13 +8,14 @@ env = Gurobi.Env() # reusing the same gurobi environment for multiple solves solver=GurobiSolver(env,OutputFlag=0) # load data -ts_input_data, = load_timeseries_data("CEP", "TX_1";K=365, T=24) #CEP +state="TX_1" # or "GER_18" or "GER_1" or "CA_1" or "TX_1" +ts_input_data, = load_timeseries_data("CEP", state;K=365, T=24) #CEP -cep_input_data_GER=load_cep_data("TX_1") +cep_input_data_GER=load_cep_data(state) # define simple extreme days of interest -ev1 = SimpleExtremeValueDescr("wind-node61","max","absolute") -ev2 = SimpleExtremeValueDescr("solar-node61","min","integral") +ev1 = SimpleExtremeValueDescr("wind-node61","max","absolute") # TODO: min? +ev2 = SimpleExtremeValueDescr("solar-node61","min","integral") ev3 = SimpleExtremeValueDescr("el_demand-node61","max","absolute") # ev1 = SimpleExtremeValueDescr("wind-dena42","max","absolute") # ev2 = SimpleExtremeValueDescr("solar-dena42","min","integral") @@ -29,9 +30,13 @@ ev3 = SimpleExtremeValueDescr("el_demand-node61","max","absolute") # representation modification #ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results) - ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER,ev;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra") +#without simple extreme days + ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",print_flag=false) - # optimization +#with simple extreme days +# ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER,ev;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",print_flag=false) + +# optimization #opt_res = run_opt(ts_clust_res.best_results,cep_input_data_GER;solver=GurobiSolver(),co2_limit=1000.0) #TODO: write functions to get generation, capacity etc. diff --git a/src/clustering/extreme_vals.jl b/src/clustering/extreme_vals.jl index b9e4d04..fb6be87 100644 --- a/src/clustering/extreme_vals.jl +++ b/src/clustering/extreme_vals.jl @@ -1,4 +1,8 @@ """ + +ts_data: The full input data 365. Used for individual opt run +ts_data_mod: The input data used for clustering (365, or 365-extreme values in the case of append) +clust_data: The clustered data: n_clust + extreme values """ function run_clust_extr( ts_data::ClustData, @@ -25,6 +29,7 @@ function run_clust_extr( storage::String="non", transmission::Bool=false, k_ids::Array{Int64,1}=Array{Int64,1}(), + print_flag::Bool=true, kwargs... # simple_extreme_days=true # extreme_day_selection_method="feasibility", "slack", "none" @@ -34,14 +39,17 @@ function run_clust_extr( # QUESTION: should keyword arguments be specified or rather be kwargs? kwargs may not work because the subsequent functions would through an error that some of the keyword arguments are not supported # simple extreme value selection use_simple_extr = !isempty(extr_value_descr_ar) + extr_vals=ClustData + extr_idcs=Int[] + ts_data_mod=ts_data if use_simple_extr ts_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_data,extr_value_descr_ar;rep_mod_method=rep_mod_method) - else - ts_data_mod=ts_data end + # run initial clustering clust_res = run_clust(ts_data_mod;norm_op=norm_op,norm_scope=norm_scope,method=method,representation=representation,n_clust=n_clust,n_init=n_init,iterations=iterations,attribute_weights=attribute_weights,save=save,get_all_clust_results=get_all_clust_results,kwargs...) - # if simple: representation modification + + # if simple: representation modification clust_data=clust_res.best_results if use_simple_extr clust_data = representation_modification(extr_vals,clust_data) @@ -53,39 +61,52 @@ function run_clust_extr( @warn "extreme_event_selection_method - "*extreme_event_selection_method*" - does not match any of the three predefined keywords: feasibility, append, none. The function assumes -none-." return ClustResult(clust_res,clust_data) # TODO: adjust clust_config in these functions end - - # initial design and operations optimization - d_o_opt = run_opt(clust_data,opt_data;solver=solver,descriptor=descriptor,co2_limit=co2_limit,existing_infrastructure=existing_infrastructure,limit_infrastructure=limit_infrastructure,storage=storage,transmission=transmission,slack_cost=Inf) - dvs = get_cep_design_variables(d_o_opt) - # convert ts_data into K individual ClustData structs - ts_data_mod_indiv_ar = clustData_individual(ts_data_mod) + # convert ts_data into N individual ClustData structs + ts_data_indiv_ar = clustData_individual(ts_data) is_feasible = false # indicates if optimization result from clustered input data is feasible on operatoins optimization with full input data if extreme_event_selection_method=="feasibility" eval_res = Symbol[] elseif extreme_event_selection_method=="slack" eval_res = OptVariable[] end + i=0 while !is_feasible i+=1 - o_opt_individual = OptResult[] + # initial design and operations optimization + d_o_opt = run_opt(clust_data,opt_data;solver=solver,descriptor=descriptor,co2_limit=co2_limit,existing_infrastructure=existing_infrastructure,limit_infrastructure=limit_infrastructure,storage=storage,transmission=transmission,slack_cost=Inf,print_flag=print_flag) + dvs = get_cep_design_variables(d_o_opt) + # run individual optimization with fixed design - for k=1:ts_data_mod.K + o_opt_individual = OptResult[] + for k=1:ts_data.K if extreme_event_selection_method=="feasibility" - push!(o_opt_individual,run_opt(ts_data_mod_indiv_ar[k],opt_data,d_o_opt.opt_config,dvs;solver=solver,slack_cost=Inf)) + push!(o_opt_individual,run_opt(ts_data_indiv_ar[k],opt_data,d_o_opt.opt_config,dvs;solver=solver,slack_cost=Inf)) push!(eval_res,o_opt_individual[k].status) elseif extreme_event_selection_method=="slack" slack_cost==Inf && (@warn "extreme_event_selection_method is -slack-,but slack cost are Inf") - push!(o_opt_individual,run_opt(ts_data_mod_indiv_ar[k],opt_data,d_o_opt.opt_config,dvs;solver=solver,slack_cost=slack_cost)) + push!(o_opt_individual,run_opt(ts_data_indiv_ar[k],opt_data,d_o_opt.opt_config,dvs;solver=solver,slack_cost=slack_cost)) push!(eval_res,get_cep_slack_variables(o_opt_individual[k])) end end is_feasible = check_indiv_opt_feasibility(eval_res) - println(eval_res, "feasibility: ",is_feasible, " i=",i) + println("feasibility: ",is_feasible, " i=",i) # TODO - delete this line is_feasible && return ClustResult(clust_res,clust_data) # TODO: adjust clust_config in these functions - - + + # get infeasible value + idx_infeas = get_index_inf(eval_res) + push!(extr_idcs,idx_infeas) + extr_val_inf = extreme_val_output(ts_data,idx_infeas,rep_mod_method=rep_mod_method) + # add extr_val_inf to extr_vals (using representation modification method) + extr_vals = representation_modification(extr_val_inf,extr_vals) + if rep_mod_method=="append" + ts_data_mod = input_data_modification(ts_data,extr_idcs) + clust_res = run_clust(ts_data_mod;norm_op=norm_op,norm_scope=norm_scope,method=method,representation=representation,n_clust=n_clust,n_init=n_init,iterations=iterations,attribute_weights=attribute_weights,save=save,get_all_clust_results=get_all_clust_results,kwargs...) + clust_data=clust_res.best_results + end + clust_data = representation_modification(extr_vals,clust_data) + end # while !is_feasible # for i=1:365 @@ -339,6 +360,7 @@ Merges the clustered data and extreme vals into one ClustData struct. Weights ar function representation_modification(extr_vals::ClustData, clust_data::ClustData, ) + #TODO: The input order of extr_vals and clust_data should probably be reversed. Usually, we return the modified version of the first input argument. K_mod = clust_data.K + extr_vals.K data_mod=Dict{String,Array}() for dt in keys(clust_data.data) diff --git a/src/utils/utils.jl b/src/utils/utils.jl index 6d7fd1e..1e0fc3b 100644 --- a/src/utils/utils.jl +++ b/src/utils/utils.jl @@ -480,7 +480,7 @@ function get_cep_slack_variables(opt_result::OptResult) if "SLACK" in keys(opt_result.variables) return opt_result.variables["SLACK"] else - throw(@error("SLACK-Variable not provided in $(opt_result.descriptor)")) + @error("SLACK-Variable not provided in $(opt_result.descriptor)") end end @@ -502,6 +502,26 @@ function check_indiv_opt_feasibility(slack_vars::Array{OptVariable,1}) return all(sum.([slack_vars[i].data for i=1:length(slack_vars)]) .== 0) end +""" + function get_index_inf(status::Array{Symbol,1}) + +Finds first day in array that is infeasible +""" +function get_index_inf(status::Array{Symbol,1}) + return findfirst(x->x.!=:Optimal,status) +end + +""" + function get_index_inf(slack_vars::Array{OptVariable,1}) + +Finds day that contains the maximum slack variable, returns index of that day +""" +function get_index_inf(slack_vars::Array{OptVariable,1}) + # TODO: make optional to choose integrally maximum slack day + return findmax([findmax(slack_vars[i].data)[1] for i=1:length(slack_vars)])[2] +end + + """ function set_opt_config_cep(opt_data::OptDataCEP; kwargs...) kwargs can be whatever you need to run the run_opt From 5b7ff7a55ee8706ad9d6e1e0a76786f6f62f2e7e Mon Sep 17 00:00:00 2001 From: holgerteichgraeber Date: Thu, 27 Dec 2018 23:17:31 -0800 Subject: [PATCH 06/11] extreme value selection method -feasibility- first version working. -slack- still bug. --- examples/workflow_example_extr.jl | 3 ++- src/clustering/extreme_vals.jl | 31 ++++++++++++++++++++++--------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/examples/workflow_example_extr.jl b/examples/workflow_example_extr.jl index 1c1636b..4ee8029 100644 --- a/examples/workflow_example_extr.jl +++ b/examples/workflow_example_extr.jl @@ -31,7 +31,8 @@ ev3 = SimpleExtremeValueDescr("el_demand-node61","max","absolute") #ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results) #without simple extreme days - ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",print_flag=false) + ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",extreme_event_selection_method="slack",slack_cost=1e4,print_flag=false) + #ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",extreme_event_selection_method="feasibility",print_flag=false) #with simple extreme days # ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER,ev;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",print_flag=false) diff --git a/src/clustering/extreme_vals.jl b/src/clustering/extreme_vals.jl index fb6be87..f55e53d 100644 --- a/src/clustering/extreme_vals.jl +++ b/src/clustering/extreme_vals.jl @@ -65,22 +65,24 @@ function run_clust_extr( # convert ts_data into N individual ClustData structs ts_data_indiv_ar = clustData_individual(ts_data) is_feasible = false # indicates if optimization result from clustered input data is feasible on operatoins optimization with full input data - if extreme_event_selection_method=="feasibility" - eval_res = Symbol[] - elseif extreme_event_selection_method=="slack" - eval_res = OptVariable[] - end i=0 - while !is_feasible + while !is_feasible i+=1 + println("beg: ",clust_data) # initial design and operations optimization d_o_opt = run_opt(clust_data,opt_data;solver=solver,descriptor=descriptor,co2_limit=co2_limit,existing_infrastructure=existing_infrastructure,limit_infrastructure=limit_infrastructure,storage=storage,transmission=transmission,slack_cost=Inf,print_flag=print_flag) dvs = get_cep_design_variables(d_o_opt) - + #println("opt_res: ", d_o_opt.variables["CAP"]) # run individual optimization with fixed design o_opt_individual = OptResult[] + if extreme_event_selection_method=="feasibility" + eval_res = Symbol[] + elseif extreme_event_selection_method=="slack" + eval_res = OptVariable[] + end for k=1:ts_data.K + # TODO: include in run_opt an option to turn off warnings. This optimization is often infeasible, and it currently gives a warning every time. There should be an option for this case to turn it off. if extreme_event_selection_method=="feasibility" push!(o_opt_individual,run_opt(ts_data_indiv_ar[k],opt_data,d_o_opt.opt_config,dvs;solver=solver,slack_cost=Inf)) push!(eval_res,o_opt_individual[k].status) @@ -92,20 +94,31 @@ function run_clust_extr( end is_feasible = check_indiv_opt_feasibility(eval_res) println("feasibility: ",is_feasible, " i=",i) # TODO - delete this line + #println(eval_res) is_feasible && return ClustResult(clust_res,clust_data) # TODO: adjust clust_config in these functions # get infeasible value idx_infeas = get_index_inf(eval_res) push!(extr_idcs,idx_infeas) + println(idx_infeas) extr_val_inf = extreme_val_output(ts_data,idx_infeas,rep_mod_method=rep_mod_method) # add extr_val_inf to extr_vals (using representation modification method) - extr_vals = representation_modification(extr_val_inf,extr_vals) + if typeof(extr_vals)==DataType + extr_vals=extr_val_inf + else + extr_vals = representation_modification(extr_val_inf,extr_vals) + end if rep_mod_method=="append" ts_data_mod = input_data_modification(ts_data,extr_idcs) clust_res = run_clust(ts_data_mod;norm_op=norm_op,norm_scope=norm_scope,method=method,representation=representation,n_clust=n_clust,n_init=n_init,iterations=iterations,attribute_weights=attribute_weights,save=save,get_all_clust_results=get_all_clust_results,kwargs...) clust_data=clust_res.best_results + clust_data = representation_modification(extr_vals,clust_data) + elseif rep_mod_method == "feasibility" + clust_data = representation_modification(extr_val_inf,clust_data) + else + @error "rep_mod_method does not exist" # TODO: Write automatic check functions for the different options end - clust_data = representation_modification(extr_vals,clust_data) + println("end: ",extr_vals) end # while !is_feasible From 60bd12ae8d562e2eb97edaed2d74c1337ea35f1e Mon Sep 17 00:00:00 2001 From: holgerteichgraeber Date: Fri, 28 Dec 2018 18:31:24 -0800 Subject: [PATCH 07/11] slack based selection working --- examples/workflow_example_extr.jl | 2 +- src/clustering/extreme_vals.jl | 6 +----- src/optim_problems/run_opt.jl | 1 + src/utils/utils.jl | 17 +++++++++++++++-- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/examples/workflow_example_extr.jl b/examples/workflow_example_extr.jl index 4ee8029..1556f61 100644 --- a/examples/workflow_example_extr.jl +++ b/examples/workflow_example_extr.jl @@ -31,7 +31,7 @@ ev3 = SimpleExtremeValueDescr("el_demand-node61","max","absolute") #ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results) #without simple extreme days - ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",extreme_event_selection_method="slack",slack_cost=1e4,print_flag=false) + ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",extreme_event_selection_method="slack",slack_cost=1e5,print_flag=false) #ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",extreme_event_selection_method="feasibility",print_flag=false) #with simple extreme days diff --git a/src/clustering/extreme_vals.jl b/src/clustering/extreme_vals.jl index f55e53d..4355ebf 100644 --- a/src/clustering/extreme_vals.jl +++ b/src/clustering/extreme_vals.jl @@ -69,11 +69,9 @@ function run_clust_extr( i=0 while !is_feasible i+=1 - println("beg: ",clust_data) # initial design and operations optimization d_o_opt = run_opt(clust_data,opt_data;solver=solver,descriptor=descriptor,co2_limit=co2_limit,existing_infrastructure=existing_infrastructure,limit_infrastructure=limit_infrastructure,storage=storage,transmission=transmission,slack_cost=Inf,print_flag=print_flag) dvs = get_cep_design_variables(d_o_opt) - #println("opt_res: ", d_o_opt.variables["CAP"]) # run individual optimization with fixed design o_opt_individual = OptResult[] if extreme_event_selection_method=="feasibility" @@ -90,11 +88,10 @@ function run_clust_extr( slack_cost==Inf && (@warn "extreme_event_selection_method is -slack-,but slack cost are Inf") push!(o_opt_individual,run_opt(ts_data_indiv_ar[k],opt_data,d_o_opt.opt_config,dvs;solver=solver,slack_cost=slack_cost)) push!(eval_res,get_cep_slack_variables(o_opt_individual[k])) - end + end end is_feasible = check_indiv_opt_feasibility(eval_res) println("feasibility: ",is_feasible, " i=",i) # TODO - delete this line - #println(eval_res) is_feasible && return ClustResult(clust_res,clust_data) # TODO: adjust clust_config in these functions # get infeasible value @@ -118,7 +115,6 @@ function run_clust_extr( else @error "rep_mod_method does not exist" # TODO: Write automatic check functions for the different options end - println("end: ",extr_vals) end # while !is_feasible diff --git a/src/optim_problems/run_opt.jl b/src/optim_problems/run_opt.jl index 1e542f7..f064151 100644 --- a/src/optim_problems/run_opt.jl +++ b/src/optim_problems/run_opt.jl @@ -59,6 +59,7 @@ function run_opt(ts_data::ClustData, slack_cost::Number=Inf, k_ids::Array{Int64,1}=Array{Int64,1}()) # Add the fixed_design_variables and new setting for slack costs to the existing config + # TODO: Think about changing opt config as a deepcopy function.Possible issue: the opt_config of the original problem is modified as well. set_opt_config_cep!(opt_config;fixed_design_variables=fixed_design_variables, slack_cost=slack_cost) return run_opt(ts_data,opt_data,opt_config;solver=solver,k_ids=k_ids) end diff --git a/src/utils/utils.jl b/src/utils/utils.jl index 1e0fc3b..43ddac9 100644 --- a/src/utils/utils.jl +++ b/src/utils/utils.jl @@ -499,7 +499,7 @@ end checks feasibility of optimization problem based on slack variables """ function check_indiv_opt_feasibility(slack_vars::Array{OptVariable,1}) - return all(sum.([slack_vars[i].data for i=1:length(slack_vars)]) .== 0) + return all(sum.([slack_vars[i].data for i=1:length(slack_vars)]) .<= 1e-9) # use 1e-9 as a tolerance level for nonzero values end """ @@ -514,7 +514,7 @@ end """ function get_index_inf(slack_vars::Array{OptVariable,1}) -Finds day that contains the maximum slack variable, returns index of that day +Finds day that contains the maximum absolute slack variable, returns index of that day """ function get_index_inf(slack_vars::Array{OptVariable,1}) # TODO: make optional to choose integrally maximum slack day @@ -621,3 +621,16 @@ function set_clust_config(;kwargs...) # Return Directory with the information of kwargs return config end + +""" +function set_clust_config!(config::Dict{String,Any};kwargs...) +""" +function set_clust_config!(config::Dict{String,Any};kwargs...) + # Loop through the kwargs and add them to Dictionary + for kwarg in kwargs + config[String(kwarg[1])]=kwarg[2] + end + # Return Directory with the information + return config +end + From dd48eba3170402205a0d1e693d8f28123930814d Mon Sep 17 00:00:00 2001 From: holgerteichgraeber Date: Fri, 28 Dec 2018 22:04:45 -0800 Subject: [PATCH 08/11] include extreme values in clust_config within ClustResult --- .gitignore | 1 + examples/workflow_example_extr_simple.jl | 8 +++- src/clustering/extreme_vals.jl | 51 ++++------------------ src/utils/datastructs.jl | 54 +++++++++++++++++++++--- src/utils/utils.jl | 5 ++- 5 files changed, 68 insertions(+), 51 deletions(-) diff --git a/.gitignore b/.gitignore index d1cdb13..0397009 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *core* *.asv *.swn +*.swm *.swo *.eps *.png diff --git a/examples/workflow_example_extr_simple.jl b/examples/workflow_example_extr_simple.jl index 81a0216..a121c71 100644 --- a/examples/workflow_example_extr_simple.jl +++ b/examples/workflow_example_extr_simple.jl @@ -15,8 +15,10 @@ cep_input_data_GER=load_cep_data("GER_18") ev2 = SimpleExtremeValueDescr("solar-dena42","min","integral") ev3 = SimpleExtremeValueDescr("el_demand-dena21","max","absolute") ev = [ev1, ev2, ev3] + + rep_mod_method="feasibility" # simple extreme day selection - ts_input_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_input_data,ev;rep_mod_method="feasibility") + ts_input_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_input_data,ev;rep_mod_method=rep_mod_method) # run clustering ts_clust_res = run_clust(ts_input_data_mod;method="kmeans",representation="centroid",n_init=10,n_clust=5) # default k-means @@ -24,5 +26,7 @@ ts_clust_res = run_clust(ts_input_data_mod;method="kmeans",representation="centr # representation modification ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results) +ts_clust_res_extr = ClustResult(ts_clust_res,ts_clust_extr,extr_idcs,rep_mod_method,ev,"none") + # optimization -opt_res = run_opt(ts_clust_extr,cep_input_data_GER;solver=GurobiSolver(),co2_limit=1000.0) +opt_res = run_opt(ts_clust_res_extr.best_results,cep_input_data_GER;solver=GurobiSolver(),co2_limit=1000.0) diff --git a/src/clustering/extreme_vals.jl b/src/clustering/extreme_vals.jl index 4355ebf..be44cec 100644 --- a/src/clustering/extreme_vals.jl +++ b/src/clustering/extreme_vals.jl @@ -7,7 +7,7 @@ clust_data: The clustered data: n_clust + extreme values function run_clust_extr( ts_data::ClustData, opt_data::OptDataCEP, - extr_value_descr_ar::Array{SimpleExtremeValueDescr,1}; + simple_extr_value_descr_ar::Array{SimpleExtremeValueDescr,1}; norm_op::String="zscore", norm_scope::String="full", method::String="kmeans", @@ -16,7 +16,7 @@ function run_clust_extr( n_init::Int=100, iterations::Int=300, attribute_weights::Dict{String,Float64}=Dict{String,Float64}(), - extreme_event_selection_method="feasibility", + extreme_event_selection_method::String="feasibility", rep_mod_method::String="feasibility", save::String="", get_all_clust_results::Bool=false, @@ -38,12 +38,12 @@ function run_clust_extr( ) # QUESTION: should keyword arguments be specified or rather be kwargs? kwargs may not work because the subsequent functions would through an error that some of the keyword arguments are not supported # simple extreme value selection - use_simple_extr = !isempty(extr_value_descr_ar) + use_simple_extr = !isempty(simple_extr_value_descr_ar) extr_vals=ClustData extr_idcs=Int[] ts_data_mod=ts_data if use_simple_extr - ts_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_data,extr_value_descr_ar;rep_mod_method=rep_mod_method) + ts_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_data,simple_extr_value_descr_ar;rep_mod_method=rep_mod_method) end # run initial clustering @@ -56,10 +56,11 @@ function run_clust_extr( end if extreme_event_selection_method=="none" - return ClustResult(clust_res,clust_data) # TODO: adjust clust_config in these functions + return ClustResult(clust_res,clust_data,extr_idcs,rep_mod_method,simple_extr_value_descr_ar,extreme_event_selection_method) elseif (extreme_event_selection_method !="feasibility") && (extreme_event_selection_method != "slack") @warn "extreme_event_selection_method - "*extreme_event_selection_method*" - does not match any of the three predefined keywords: feasibility, append, none. The function assumes -none-." - return ClustResult(clust_res,clust_data) # TODO: adjust clust_config in these functions + + return ClustResult(clust_res,clust_data,extr_idcs,rep_mod_method,simple_extr_value_descr_ar,extreme_event_selection_method) end # convert ts_data into N individual ClustData structs @@ -92,7 +93,7 @@ function run_clust_extr( end is_feasible = check_indiv_opt_feasibility(eval_res) println("feasibility: ",is_feasible, " i=",i) # TODO - delete this line - is_feasible && return ClustResult(clust_res,clust_data) # TODO: adjust clust_config in these functions + is_feasible && return ClustResult(clust_res,clust_data,extr_idcs,rep_mod_method,simple_extr_value_descr_ar,extreme_event_selection_method) # get infeasible value idx_infeas = get_index_inf(eval_res) @@ -115,42 +116,8 @@ function run_clust_extr( else @error "rep_mod_method does not exist" # TODO: Write automatic check functions for the different options end - end - # while !is_feasible - # for i=1:365 - # method that puts one day out of ClustData into its own ClustData struct - # run_opt() single day, given DVs(operations only) - # end - # is_feasible = [depends on if any day was infeasible] - # if is_feasible: break out of while loop here - # idx_extr_val = feasibility: first infeasible index; append: idx with highest slack variable - # extr_val_output(idx_extr_val) - # representation_modification() - # if append: run_clust() # really? should representation modification not be afterwards? / figure 2.5 Constantin Thesis - # - # DVs = run_opt().variables["CAP"] - # end - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - # - - - return ClustResult(clust_res,clust_data) # TODO: adjust clust_config in these functions - + return ClustResult(clust_res,clust_data,extr_idcs,rep_mod_method,simple_extr_value_descr_ar,extreme_event_selection_method) # TODO: adjust clust_config in these functions end """ diff --git a/src/utils/datastructs.jl b/src/utils/datastructs.jl index c47dac4..eed4220 100644 --- a/src/utils/datastructs.jl +++ b/src/utils/datastructs.jl @@ -375,19 +375,61 @@ function ClustDataMerged(data::ClustData) end """ -function ClustResult(clust_res::ClustResultBest,clust_data_mod::ClustData) +function ClustResult(clust_res::ClustResultBest,clust_data_mod::ClustData;kwargs...) adjusts ClustResult best_results. To be used to modify clustered data with extreme values. +Adds kwargs to clust_config (can e.g. be used to add extreme value information) """ -function ClustResult(clust_res::ClustResultBest,clust_data_mod::ClustData) - return ClustResultBest(clust_data_mod,clust_res.best_ids,clust_res.best_cost,clust_res.data_type,clust_res.clust_config) +function ClustResult(clust_res::ClustResultBest, + clust_data_mod::ClustData; + kwargs...) + clust_config=clust_res.clust_config + set_clust_config!(clust_config;kwargs...) + return ClustResultBest(clust_data_mod,clust_res.best_ids,clust_res.best_cost,clust_res.data_type,clust_config) +end + + +""" +function ClustResult(clust_res::ClustResultBest,clust_data_mod::ClustData;kwargs...) + +adjusts ClustResult best_results. To be used to modify clustered data with extreme values. +Wrapper function that makes explicit the arguments that it adds (extreme value arguments) to clust config +""" +function ClustResult(clust_res::ClustResultBest, + clust_data_mod::ClustData, + extr_idcs::Array{Int,1}, + rep_mod_method::String, + simple_extr_value_descr_ar::Array{SimpleExtremeValueDescr,1}, + extreme_event_selection_method::String + ) + return ClustResult(clust_res,clust_data_mod;extr_idcs=extr_idcs,rep_mod_method=rep_mod_method,simple_extr_value_descr_ar=simple_extr_value_descr_ar,extreme_event_selection_method=extreme_event_selection_method) end """ -function ClustResult(clust_res::ClustResultAll,clust_data_mod::ClustData) +function ClustResult(clust_res::ClustResultAll,clust_data_mod::ClustData;kwargs...) adjusts ClustResult best_results. To be used to modify clustered data with extreme values. """ -function ClustResult(clust_res::ClustResultAll,clust_data_mod::ClustData) - return ClustResultAll(clust_data_mod,clust_res.best_ids,clust_res.best_cost,clust_res.data_type,clust_res.clust_config,clust_res.centers,clust_res.weights,clust_res.clustids,clust_res.cost,clust_res.iter) +function ClustResult(clust_res::ClustResultAll, + clust_data_mod::ClustData; + kwargs...) + clust_config=clust_res.clust_config + set_clust_config!(clust_config;kwargs...) + return ClustResultAll(clust_data_mod,clust_res.best_ids,clust_res.best_cost,clust_res.data_type,clust_config,clust_res.centers,clust_res.weights,clust_res.clustids,clust_res.cost,clust_res.iter) +end + +""" +function ClustResult(clust_res::ClustResultAll,clust_data_mod::ClustData;kwargs...) + +adjusts ClustResult all. To be used to modify clustered data with extreme values. +Wrapper function that makes explicit the arguments that it adds (extreme value arguments) to clust config +""" +function ClustResult(clust_res::ClustResultAll, + clust_data_mod::ClustData, + extr_idcs::Array{Int,1}, + rep_mod_method::String, + simple_extr_value_descr_ar::Array{SimpleExtremeValueDescr,1}, + extreme_event_selection_method::String + ) + return ClustResult(clust_res,clust_data_mod;extr_idcs=extr_idcs,rep_mod_method=rep_mod_method,simple_extr_value_descr_ar=simple_extr_value_descr_ar,extreme_event_selection_method=extreme_event_selection_method) end diff --git a/src/utils/utils.jl b/src/utils/utils.jl index 43ddac9..0f5c952 100644 --- a/src/utils/utils.jl +++ b/src/utils/utils.jl @@ -623,7 +623,10 @@ function set_clust_config(;kwargs...) end """ -function set_clust_config!(config::Dict{String,Any};kwargs...) + function set_clust_config!(config::Dict{String,Any};kwargs...) + +add or replace items to an existing config +e.g. extreme value selection information """ function set_clust_config!(config::Dict{String,Any};kwargs...) # Loop through the kwargs and add them to Dictionary From 66429c46edd6d88335b0587491465a01b3322581 Mon Sep 17 00:00:00 2001 From: holgerteichgraeber Date: Mon, 31 Dec 2018 11:59:00 -0800 Subject: [PATCH 09/11] updated example --- examples/workflow_example_extr.jl | 23 ++++++++++------------- src/clustering/extreme_vals.jl | 1 + 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/examples/workflow_example_extr.jl b/examples/workflow_example_extr.jl index 1556f61..1b10e26 100644 --- a/examples/workflow_example_extr.jl +++ b/examples/workflow_example_extr.jl @@ -14,31 +14,28 @@ ts_input_data, = load_timeseries_data("CEP", state;K=365, T=24) #CEP cep_input_data_GER=load_cep_data(state) # define simple extreme days of interest -ev1 = SimpleExtremeValueDescr("wind-node61","max","absolute") # TODO: min? -ev2 = SimpleExtremeValueDescr("solar-node61","min","integral") -ev3 = SimpleExtremeValueDescr("el_demand-node61","max","absolute") +#ev1 = SimpleExtremeValueDescr("wind-node61","max","absolute") # TODO: min? +#ev2 = SimpleExtremeValueDescr("solar-node61","min","integral") +#ev3 = SimpleExtremeValueDescr("el_demand-node61","max","absolute") # ev1 = SimpleExtremeValueDescr("wind-dena42","max","absolute") # ev2 = SimpleExtremeValueDescr("solar-dena42","min","integral") # ev3 = SimpleExtremeValueDescr("el_demand-dena21","max","absolute") - ev = [ev1, ev2, ev3] - # simple extreme day selection - #ts_input_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_input_data,ev;rep_mod_method="feasibility") +# ev = [ev1, ev2, ev3] - # run clustering -#ts_clust_res = run_clust(ts_input_data_mod;method="kmeans",representation="centroid",n_init=10,n_clust=5) # default k-means - -# representation modification -#ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results) +rep_mod_method="append" #feasibility, append +extreme_event_selection_method= "slack" #slack, feasibility +slack_cost=1e5 +co2_limit=1000.0 #without simple extreme days - ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",extreme_event_selection_method="slack",slack_cost=1e5,print_flag=false) + ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER;rep_mod_method=rep_mod_method,method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",co2_limit=co2_limit,extreme_event_selection_method=extreme_event_selection_method,slack_cost=slack_cost,print_flag=false) #ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",extreme_event_selection_method="feasibility",print_flag=false) #with simple extreme days # ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER,ev;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",print_flag=false) # optimization -#opt_res = run_opt(ts_clust_res.best_results,cep_input_data_GER;solver=GurobiSolver(),co2_limit=1000.0) +opt_res = run_opt(ts_clust_res.best_results,cep_input_data_GER;solver=GurobiSolver(),storage="intra",co2_limit=co2_limit) #TODO: write functions to get generation, capacity etc. diff --git a/src/clustering/extreme_vals.jl b/src/clustering/extreme_vals.jl index be44cec..549d9fe 100644 --- a/src/clustering/extreme_vals.jl +++ b/src/clustering/extreme_vals.jl @@ -37,6 +37,7 @@ function run_clust_extr( # needs input data for optimization problem ) # QUESTION: should keyword arguments be specified or rather be kwargs? kwargs may not work because the subsequent functions would through an error that some of the keyword arguments are not supported + # TODO: Specify keywords arguemnt as three seperate arrays, one for clustering, and one for extreme values, one for optimization problem # simple extreme value selection use_simple_extr = !isempty(simple_extr_value_descr_ar) extr_vals=ClustData From b87293f477cf0cef9a966bc61822c9bb2f8f48ce Mon Sep 17 00:00:00 2001 From: holgerteichgraeber Date: Wed, 2 Jan 2019 20:21:51 -0800 Subject: [PATCH 10/11] bugfix: WEIGHTED Co2 emissions constraint --- src/optim_problems/opt_cep.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/optim_problems/opt_cep.jl b/src/optim_problems/opt_cep.jl index 220c76a..3c8c456 100644 --- a/src/optim_problems/opt_cep.jl +++ b/src/optim_problems/opt_cep.jl @@ -397,11 +397,12 @@ function setup_opt_cep_co2_limit!(cep::OptModelCEP, set=cep.set #ts Dict( tech-node ): t x k ts=ts_data.data - + w = ts_data.weights + ## EMISSIONS ## # Limit the Emissions with co2_limit if it exists push!(cep.info,"ΣCOST_{account,tech}[account,'$(set["impact"][1])',tech] ≤ co2_limit*Σ_{node,t,k}ts[el_demand-node,t,k]") - @constraint(cep.model, sum(cep.model[:COST][account,"CO2",tech] for account=set["account"], tech=set["tech"])<= co2_limit*sum(sum(ts["el_demand-"*node]) for node=set["nodes"])) + @constraint(cep.model, sum(cep.model[:COST][account,"CO2",tech] for account=set["account"], tech=set["tech"])<= co2_limit*sum(sum(ts["el_demand-"*node]*w) for node=set["nodes"])) return cep end From 969df83ed0cbe72edae4ef953c940a954301058f Mon Sep 17 00:00:00 2001 From: holgerteichgraeber Date: Mon, 11 Feb 2019 19:13:06 -0800 Subject: [PATCH 11/11] test slack_cost loop --- src/clustering/extreme_vals.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/clustering/extreme_vals.jl b/src/clustering/extreme_vals.jl index 549d9fe..1daa616 100644 --- a/src/clustering/extreme_vals.jl +++ b/src/clustering/extreme_vals.jl @@ -88,6 +88,8 @@ function run_clust_extr( push!(eval_res,o_opt_individual[k].status) elseif extreme_event_selection_method=="slack" slack_cost==Inf && (@warn "extreme_event_selection_method is -slack-,but slack cost are Inf") + # ### TODO: THIS IS JUST A TEST + # set_clust_config!(d_o_opt.opt_config;co2_limit=Inf) push!(o_opt_individual,run_opt(ts_data_indiv_ar[k],opt_data,d_o_opt.opt_config,dvs;solver=solver,slack_cost=slack_cost)) push!(eval_res,get_cep_slack_variables(o_opt_individual[k])) end