diff --git a/.gitignore b/.gitignore index d1cdb13..0397009 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *core* *.asv *.swn +*.swm *.swo *.eps *.png diff --git a/examples/workflow_example_extr.jl b/examples/workflow_example_extr.jl index 6332ce7..1b10e26 100644 --- a/examples/workflow_example_extr.jl +++ b/examples/workflow_example_extr.jl @@ -2,26 +2,43 @@ include(normpath(joinpath(dirname(@__FILE__),"..","src","ClustForOpt_priv_development.jl"))) #using ClustForOpt_priv -#using Gurobi +using Gurobi +env = Gurobi.Env() # reusing the same gurobi environment for multiple solves +# select solver +solver=GurobiSolver(env,OutputFlag=0) # load data -ts_input_data, = load_timeseries_data("CEP", "GER_18";K=365, T=24) #CEP +state="TX_1" # or "GER_18" or "GER_1" or "CA_1" or "TX_1" +ts_input_data, = load_timeseries_data("CEP", state;K=365, T=24) #CEP -cep_input_data_GER=load_cep_data("GER_18") +cep_input_data_GER=load_cep_data(state) # define simple extreme days of interest - ev1 = SimpleExtremeValueDescr("wind-dena42","max","absolute") - ev2 = SimpleExtremeValueDescr("solar-dena42","min","integral") - ev3 = SimpleExtremeValueDescr("el_demand-dena21","max","absolute") - ev = [ev1, ev2, ev3] - # simple extreme day selection - ts_input_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_input_data,ev;rep_mod_method="feasibility") +#ev1 = SimpleExtremeValueDescr("wind-node61","max","absolute") # TODO: min? +#ev2 = SimpleExtremeValueDescr("solar-node61","min","integral") +#ev3 = SimpleExtremeValueDescr("el_demand-node61","max","absolute") +# ev1 = SimpleExtremeValueDescr("wind-dena42","max","absolute") +# ev2 = SimpleExtremeValueDescr("solar-dena42","min","integral") +# ev3 = SimpleExtremeValueDescr("el_demand-dena21","max","absolute") +# ev = [ev1, ev2, ev3] - # run clustering -ts_clust_res = run_clust(ts_input_data_mod;method="kmeans",representation="centroid",n_init=10,n_clust=5) # default k-means +rep_mod_method="append" #feasibility, append +extreme_event_selection_method= "slack" #slack, feasibility +slack_cost=1e5 +co2_limit=1000.0 + +#without simple extreme days + ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER;rep_mod_method=rep_mod_method,method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",co2_limit=co2_limit,extreme_event_selection_method=extreme_event_selection_method,slack_cost=slack_cost,print_flag=false) + #ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",extreme_event_selection_method="feasibility",print_flag=false) + +#with simple extreme days +# ts_clust_res = run_clust_extr(ts_input_data,cep_input_data_GER,ev;rep_mod_method="feasibility",method="kmeans",representation="centroid",n_init=10,n_clust=5,solver=solver,storage="intra",print_flag=false) + +# optimization +opt_res = run_opt(ts_clust_res.best_results,cep_input_data_GER;solver=GurobiSolver(),storage="intra",co2_limit=co2_limit) + +#TODO: write functions to get generation, capacity etc. + +# TODO: write plotting functions in other package -# representation modification -ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results) - # optimization -opt_res = run_opt(ts_clust_extr,cep_input_data_GER;solver=GurobiSolver(),co2_limit=1000.0) diff --git a/examples/workflow_example_extr_simple.jl b/examples/workflow_example_extr_simple.jl new file mode 100644 index 0000000..a121c71 --- /dev/null +++ b/examples/workflow_example_extr_simple.jl @@ -0,0 +1,32 @@ +# This file exemplifies the workflow from data input to optimization result generation +#QUESTION using ClustForOpt_priv.col in module Main conflicts with an existing identifier., using ClustForOpt_priv.cols in module Main conflicts with an existing identifier. + +include(normpath(joinpath(dirname(@__FILE__),"..","src","ClustForOpt_priv_development.jl"))) +#using ClustForOpt_priv +#using Gurobi + +# load data +ts_input_data, = load_timeseries_data("CEP", "GER_18";K=365, T=24) #CEP + +cep_input_data_GER=load_cep_data("GER_18") + + # define simple extreme days of interest + ev1 = SimpleExtremeValueDescr("wind-dena42","max","absolute") + ev2 = SimpleExtremeValueDescr("solar-dena42","min","integral") + ev3 = SimpleExtremeValueDescr("el_demand-dena21","max","absolute") + ev = [ev1, ev2, ev3] + + rep_mod_method="feasibility" + # simple extreme day selection + ts_input_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_input_data,ev;rep_mod_method=rep_mod_method) + + # run clustering +ts_clust_res = run_clust(ts_input_data_mod;method="kmeans",representation="centroid",n_init=10,n_clust=5) # default k-means + +# representation modification +ts_clust_extr = representation_modification(extr_vals,ts_clust_res.best_results) + +ts_clust_res_extr = ClustResult(ts_clust_res,ts_clust_extr,extr_idcs,rep_mod_method,ev,"none") + + # optimization +opt_res = run_opt(ts_clust_res_extr.best_results,cep_input_data_GER;solver=GurobiSolver(),co2_limit=1000.0) diff --git a/src/clustering/extreme_vals.jl b/src/clustering/extreme_vals.jl index 98cb776..1daa616 100644 --- a/src/clustering/extreme_vals.jl +++ b/src/clustering/extreme_vals.jl @@ -1,3 +1,145 @@ +""" + +ts_data: The full input data 365. Used for individual opt run +ts_data_mod: The input data used for clustering (365, or 365-extreme values in the case of append) +clust_data: The clustered data: n_clust + extreme values +""" +function run_clust_extr( + ts_data::ClustData, + opt_data::OptDataCEP, + simple_extr_value_descr_ar::Array{SimpleExtremeValueDescr,1}; + norm_op::String="zscore", + norm_scope::String="full", + method::String="kmeans", + representation::String="centroid", + n_clust::Int=5, + n_init::Int=100, + iterations::Int=300, + attribute_weights::Dict{String,Float64}=Dict{String,Float64}(), + extreme_event_selection_method::String="feasibility", + rep_mod_method::String="feasibility", + save::String="", + get_all_clust_results::Bool=false, + solver::Any=CbcSolver(), + descriptor::String="", + co2_limit::Number=Inf, + slack_cost::Number=Inf, + existing_infrastructure::Bool=false, + limit_infrastructure::Bool=false, + storage::String="non", + transmission::Bool=false, + k_ids::Array{Int64,1}=Array{Int64,1}(), + print_flag::Bool=true, + kwargs... + # simple_extreme_days=true + # extreme_day_selection_method="feasibility", "slack", "none" + # + extreme_value_descr_ar + # needs input data for optimization problem + ) + # QUESTION: should keyword arguments be specified or rather be kwargs? kwargs may not work because the subsequent functions would through an error that some of the keyword arguments are not supported + # TODO: Specify keywords arguemnt as three seperate arrays, one for clustering, and one for extreme values, one for optimization problem + # simple extreme value selection + use_simple_extr = !isempty(simple_extr_value_descr_ar) + extr_vals=ClustData + extr_idcs=Int[] + ts_data_mod=ts_data + if use_simple_extr + ts_data_mod,extr_vals,extr_idcs = simple_extr_val_sel(ts_data,simple_extr_value_descr_ar;rep_mod_method=rep_mod_method) + end + + # run initial clustering + clust_res = run_clust(ts_data_mod;norm_op=norm_op,norm_scope=norm_scope,method=method,representation=representation,n_clust=n_clust,n_init=n_init,iterations=iterations,attribute_weights=attribute_weights,save=save,get_all_clust_results=get_all_clust_results,kwargs...) + + # if simple: representation modification + clust_data=clust_res.best_results + if use_simple_extr + clust_data = representation_modification(extr_vals,clust_data) + end + + if extreme_event_selection_method=="none" + return ClustResult(clust_res,clust_data,extr_idcs,rep_mod_method,simple_extr_value_descr_ar,extreme_event_selection_method) + elseif (extreme_event_selection_method !="feasibility") && (extreme_event_selection_method != "slack") + @warn "extreme_event_selection_method - "*extreme_event_selection_method*" - does not match any of the three predefined keywords: feasibility, append, none. The function assumes -none-." + + return ClustResult(clust_res,clust_data,extr_idcs,rep_mod_method,simple_extr_value_descr_ar,extreme_event_selection_method) + end + + # convert ts_data into N individual ClustData structs + ts_data_indiv_ar = clustData_individual(ts_data) + is_feasible = false # indicates if optimization result from clustered input data is feasible on operatoins optimization with full input data + + i=0 + while !is_feasible + i+=1 + # initial design and operations optimization + d_o_opt = run_opt(clust_data,opt_data;solver=solver,descriptor=descriptor,co2_limit=co2_limit,existing_infrastructure=existing_infrastructure,limit_infrastructure=limit_infrastructure,storage=storage,transmission=transmission,slack_cost=Inf,print_flag=print_flag) + dvs = get_cep_design_variables(d_o_opt) + # run individual optimization with fixed design + o_opt_individual = OptResult[] + if extreme_event_selection_method=="feasibility" + eval_res = Symbol[] + elseif extreme_event_selection_method=="slack" + eval_res = OptVariable[] + end + for k=1:ts_data.K + # TODO: include in run_opt an option to turn off warnings. This optimization is often infeasible, and it currently gives a warning every time. There should be an option for this case to turn it off. + if extreme_event_selection_method=="feasibility" + push!(o_opt_individual,run_opt(ts_data_indiv_ar[k],opt_data,d_o_opt.opt_config,dvs;solver=solver,slack_cost=Inf)) + push!(eval_res,o_opt_individual[k].status) + elseif extreme_event_selection_method=="slack" + slack_cost==Inf && (@warn "extreme_event_selection_method is -slack-,but slack cost are Inf") + # ### TODO: THIS IS JUST A TEST + # set_clust_config!(d_o_opt.opt_config;co2_limit=Inf) + push!(o_opt_individual,run_opt(ts_data_indiv_ar[k],opt_data,d_o_opt.opt_config,dvs;solver=solver,slack_cost=slack_cost)) + push!(eval_res,get_cep_slack_variables(o_opt_individual[k])) + end + end + is_feasible = check_indiv_opt_feasibility(eval_res) + println("feasibility: ",is_feasible, " i=",i) # TODO - delete this line + is_feasible && return ClustResult(clust_res,clust_data,extr_idcs,rep_mod_method,simple_extr_value_descr_ar,extreme_event_selection_method) + + # get infeasible value + idx_infeas = get_index_inf(eval_res) + push!(extr_idcs,idx_infeas) + println(idx_infeas) + extr_val_inf = extreme_val_output(ts_data,idx_infeas,rep_mod_method=rep_mod_method) + # add extr_val_inf to extr_vals (using representation modification method) + if typeof(extr_vals)==DataType + extr_vals=extr_val_inf + else + extr_vals = representation_modification(extr_val_inf,extr_vals) + end + if rep_mod_method=="append" + ts_data_mod = input_data_modification(ts_data,extr_idcs) + clust_res = run_clust(ts_data_mod;norm_op=norm_op,norm_scope=norm_scope,method=method,representation=representation,n_clust=n_clust,n_init=n_init,iterations=iterations,attribute_weights=attribute_weights,save=save,get_all_clust_results=get_all_clust_results,kwargs...) + clust_data=clust_res.best_results + clust_data = representation_modification(extr_vals,clust_data) + elseif rep_mod_method == "feasibility" + clust_data = representation_modification(extr_val_inf,clust_data) + else + @error "rep_mod_method does not exist" # TODO: Write automatic check functions for the different options + end + end + return ClustResult(clust_res,clust_data,extr_idcs,rep_mod_method,simple_extr_value_descr_ar,extreme_event_selection_method) # TODO: adjust clust_config in these functions +end + +""" + function run_clust_extr( + ts_data::ClustData, + opt_data::OptDataCEP; + kwargs... + ) + +Clustering and extreme value selection WITHOUT simple extreme values. +""" +function run_clust_extr( + ts_data::ClustData, + opt_data::OptDataCEP; + kwargs... + ) + return run_clust_extr(ts_data,opt_data,SimpleExtremeValueDescr[];kwargs...) +end + """ function simple_extr_val_sel(data::ClustData, extreme_value_descr_ar::Array{SimpleExtremeValueDescr,1}; @@ -197,6 +339,7 @@ Merges the clustered data and extreme vals into one ClustData struct. Weights ar function representation_modification(extr_vals::ClustData, clust_data::ClustData, ) + #TODO: The input order of extr_vals and clust_data should probably be reversed. Usually, we return the modified version of the first input argument. K_mod = clust_data.K + extr_vals.K data_mod=Dict{String,Array}() for dt in keys(clust_data.data) diff --git a/src/optim_problems/opt_cep.jl b/src/optim_problems/opt_cep.jl index 220c76a..3c8c456 100644 --- a/src/optim_problems/opt_cep.jl +++ b/src/optim_problems/opt_cep.jl @@ -397,11 +397,12 @@ function setup_opt_cep_co2_limit!(cep::OptModelCEP, set=cep.set #ts Dict( tech-node ): t x k ts=ts_data.data - + w = ts_data.weights + ## EMISSIONS ## # Limit the Emissions with co2_limit if it exists push!(cep.info,"ΣCOST_{account,tech}[account,'$(set["impact"][1])',tech] ≤ co2_limit*Σ_{node,t,k}ts[el_demand-node,t,k]") - @constraint(cep.model, sum(cep.model[:COST][account,"CO2",tech] for account=set["account"], tech=set["tech"])<= co2_limit*sum(sum(ts["el_demand-"*node]) for node=set["nodes"])) + @constraint(cep.model, sum(cep.model[:COST][account,"CO2",tech] for account=set["account"], tech=set["tech"])<= co2_limit*sum(sum(ts["el_demand-"*node]*w) for node=set["nodes"])) return cep end diff --git a/src/optim_problems/run_opt.jl b/src/optim_problems/run_opt.jl index 1e542f7..f064151 100644 --- a/src/optim_problems/run_opt.jl +++ b/src/optim_problems/run_opt.jl @@ -59,6 +59,7 @@ function run_opt(ts_data::ClustData, slack_cost::Number=Inf, k_ids::Array{Int64,1}=Array{Int64,1}()) # Add the fixed_design_variables and new setting for slack costs to the existing config + # TODO: Think about changing opt config as a deepcopy function.Possible issue: the opt_config of the original problem is modified as well. set_opt_config_cep!(opt_config;fixed_design_variables=fixed_design_variables, slack_cost=slack_cost) return run_opt(ts_data,opt_data,opt_config;solver=solver,k_ids=k_ids) end diff --git a/src/utils/datastructs.jl b/src/utils/datastructs.jl index c9eecb4..eed4220 100644 --- a/src/utils/datastructs.jl +++ b/src/utils/datastructs.jl @@ -306,6 +306,25 @@ function ClustData(data::FullInputData, return ClustData(data.region,K,T,data_reshape,ones(K)) end +""" +function ClustData_individual(data::ClustData) + +Takes a ClustData struct and returns an array of ClustData structs that contains each period individually. +""" +function clustData_individual(data::ClustData) + clust_data_indiv = ClustData[] + for kk=1:data.K + # initialize new dict + data_dict_indiv = Dict{String,Array}() + # fill dict with data + for (k,v) in data.data + data_dict_indiv[k] = v[:,kk:kk] # kk:kk instead of k ensures that it returns a two-dimensional array instead of a vector during array slicing with singleton dimension + end + push!(clust_data_indiv,ClustData(data.region,1,data.T,data_dict_indiv,[data.weights[kk]];mean=data.mean,sdv=data.sdv)) + end + return clust_data_indiv +end + """ constructor 1: construct ClustDataMerged function ClustDataMerged(region::String, @@ -354,3 +373,63 @@ function ClustDataMerged(data::ClustData) end ClustDataMerged(data.region,data.K,data.T,data_merged,data_type,data.weights,data.mean,data.sdv) end + +""" +function ClustResult(clust_res::ClustResultBest,clust_data_mod::ClustData;kwargs...) + +adjusts ClustResult best_results. To be used to modify clustered data with extreme values. +Adds kwargs to clust_config (can e.g. be used to add extreme value information) +""" +function ClustResult(clust_res::ClustResultBest, + clust_data_mod::ClustData; + kwargs...) + clust_config=clust_res.clust_config + set_clust_config!(clust_config;kwargs...) + return ClustResultBest(clust_data_mod,clust_res.best_ids,clust_res.best_cost,clust_res.data_type,clust_config) +end + + +""" +function ClustResult(clust_res::ClustResultBest,clust_data_mod::ClustData;kwargs...) + +adjusts ClustResult best_results. To be used to modify clustered data with extreme values. +Wrapper function that makes explicit the arguments that it adds (extreme value arguments) to clust config +""" +function ClustResult(clust_res::ClustResultBest, + clust_data_mod::ClustData, + extr_idcs::Array{Int,1}, + rep_mod_method::String, + simple_extr_value_descr_ar::Array{SimpleExtremeValueDescr,1}, + extreme_event_selection_method::String + ) + return ClustResult(clust_res,clust_data_mod;extr_idcs=extr_idcs,rep_mod_method=rep_mod_method,simple_extr_value_descr_ar=simple_extr_value_descr_ar,extreme_event_selection_method=extreme_event_selection_method) +end + +""" +function ClustResult(clust_res::ClustResultAll,clust_data_mod::ClustData;kwargs...) + +adjusts ClustResult best_results. To be used to modify clustered data with extreme values. +""" +function ClustResult(clust_res::ClustResultAll, + clust_data_mod::ClustData; + kwargs...) + clust_config=clust_res.clust_config + set_clust_config!(clust_config;kwargs...) + return ClustResultAll(clust_data_mod,clust_res.best_ids,clust_res.best_cost,clust_res.data_type,clust_config,clust_res.centers,clust_res.weights,clust_res.clustids,clust_res.cost,clust_res.iter) +end + +""" +function ClustResult(clust_res::ClustResultAll,clust_data_mod::ClustData;kwargs...) + +adjusts ClustResult all. To be used to modify clustered data with extreme values. +Wrapper function that makes explicit the arguments that it adds (extreme value arguments) to clust config +""" +function ClustResult(clust_res::ClustResultAll, + clust_data_mod::ClustData, + extr_idcs::Array{Int,1}, + rep_mod_method::String, + simple_extr_value_descr_ar::Array{SimpleExtremeValueDescr,1}, + extreme_event_selection_method::String + ) + return ClustResult(clust_res,clust_data_mod;extr_idcs=extr_idcs,rep_mod_method=rep_mod_method,simple_extr_value_descr_ar=simple_extr_value_descr_ar,extreme_event_selection_method=extreme_event_selection_method) +end diff --git a/src/utils/utils.jl b/src/utils/utils.jl index 5673c48..0f5c952 100644 --- a/src/utils/utils.jl +++ b/src/utils/utils.jl @@ -474,16 +474,54 @@ end """ function get_cep_slack_variables(opt_result::OptResult) - Returns the SLACK variable of this opt_result matching "sv" + Returns the SLACK variables of this opt_result matching "sv" """ -function get_cep_slack_variable(opt_result::OptResult) +function get_cep_slack_variables(opt_result::OptResult) if "SLACK" in keys(opt_result.variables) return opt_result.variables["SLACK"] else - throw(@error("SLACK-Variable not provided in $(opt_result.descriptor)")) + @error("SLACK-Variable not provided in $(opt_result.descriptor)") end end +""" + function check_indiv_opt_feasibility(status::Array{Symbol,1}) + +checks feasibility of optimization problem based on status +""" +function check_indiv_opt_feasibility(status::Array{Symbol,1}) + return (sum(status.!=:Optimal) == 0) +end + +""" + function check_indiv_opt_feasibility(slack_vars::Array{OptVariable,1}) + +checks feasibility of optimization problem based on slack variables +""" +function check_indiv_opt_feasibility(slack_vars::Array{OptVariable,1}) + return all(sum.([slack_vars[i].data for i=1:length(slack_vars)]) .<= 1e-9) # use 1e-9 as a tolerance level for nonzero values +end + +""" + function get_index_inf(status::Array{Symbol,1}) + +Finds first day in array that is infeasible +""" +function get_index_inf(status::Array{Symbol,1}) + return findfirst(x->x.!=:Optimal,status) +end + +""" + function get_index_inf(slack_vars::Array{OptVariable,1}) + +Finds day that contains the maximum absolute slack variable, returns index of that day +""" +function get_index_inf(slack_vars::Array{OptVariable,1}) + # TODO: make optional to choose integrally maximum slack day + return findmax([findmax(slack_vars[i].data)[1] for i=1:length(slack_vars)])[2] +end + + """ function set_opt_config_cep(opt_data::OptDataCEP; kwargs...) kwargs can be whatever you need to run the run_opt @@ -583,3 +621,19 @@ function set_clust_config(;kwargs...) # Return Directory with the information of kwargs return config end + +""" + function set_clust_config!(config::Dict{String,Any};kwargs...) + +add or replace items to an existing config +e.g. extreme value selection information +""" +function set_clust_config!(config::Dict{String,Any};kwargs...) + # Loop through the kwargs and add them to Dictionary + for kwarg in kwargs + config[String(kwarg[1])]=kwarg[2] + end + # Return Directory with the information + return config +end +