From a81e305584632ecc529ee879c587d4b1f3ce1c4b Mon Sep 17 00:00:00 2001 From: Rose Crocker Date: Mon, 4 Dec 2023 14:37:07 +1000 Subject: [PATCH 1/7] Changes to `summarize_env_data` so mean and std for scenarios and sites is calculated --- src/io/result_io.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/io/result_io.jl b/src/io/result_io.jl index 6d9bbb5fa..ef3fcc21c 100644 --- a/src/io/result_io.jl +++ b/src/io/result_io.jl @@ -41,10 +41,10 @@ Matrix{Float64, 2}, of mean and standard deviation for each environmental scenar """ function summarize_env_data(data::AbstractArray)::Array{Float64} # TODO: Update once - dc_mean = dropdims(mean(data, dims=(1, 2)), dims=(1, 2))' - dc_std = dropdims(std(data, dims=(1, 2)), dims=(1, 2))' - - return Array{Float64}(vcat(dc_mean, dc_std)) + stats_store = Array{Float64}(zeros(2, size(data, 3), size(data, 2))) + stats_store[1, :, :] .= Array(dropdims(mean(data; dims=1); dims=1)') + stats_store[2, :, :] = Array(dropdims(std(data; dims=1); dims=1)') + return stats_store end """ From 48d0b1403886c8822823fff676d95123d7c043be Mon Sep 17 00:00:00 2001 From: Rose Crocker Date: Mon, 4 Dec 2023 14:37:59 +1000 Subject: [PATCH 2/7] Change size of `stats_store` to allow storage of mean and std for scenarios and sites in `ResultSet` --- src/io/result_io.jl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/io/result_io.jl b/src/io/result_io.jl index ef3fcc21c..924d9e7c2 100644 --- a/src/io/result_io.jl +++ b/src/io/result_io.jl @@ -68,17 +68,20 @@ N is the number of dhw/wave scenarios. function store_env_summary(data_cube::NamedDimsArray, type::String, file_loc::String, rcp::String, compressor::Zarr.Compressor)::ZArray stats = summarize_env_data(data_cube) - stats_store = zcreate(Float32, (2, size(stats, 2))...; + stats_store = zcreate( + Float32, + (2, size(stats, 2), size(stats, 3))...; fill_value=nothing, fill_as_missing=false, path=joinpath(file_loc, rcp), attrs=Dict( - :structure => ("stat", type), - :rows => ["mean", "std"], - :cols => string.(1:size(stats, 2)), + :structure => ("stat", type, "locations"), + :stats => ["mean", "std"], + :scenarios => string.(1:size(stats, 2)), + :locations => string.(1:size(stats, 3)), :rcp => rcp), compressor=compressor) - stats_store[:, :] .= stats + stats_store[:, :, :] .= stats return stats_store end From 7826392c47f94dab3ba73a8fb88795827bb5d77a Mon Sep 17 00:00:00 2001 From: Rose Crocker Date: Mon, 4 Dec 2023 14:38:43 +1000 Subject: [PATCH 3/7] Add locations dimension when recreating stats store --- src/io/result_io.jl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/io/result_io.jl b/src/io/result_io.jl index 924d9e7c2..a288bc701 100644 --- a/src/io/result_io.jl +++ b/src/io/result_io.jl @@ -362,9 +362,12 @@ function _recreate_stats_from_store(zarr_store_path::String)::Dict{String,Abstra store = zopen(sd, fill_as_missing=false) dims = store.attrs["structure"] - row_names = string.(store.attrs["rows"]) - col_names = string.(store.attrs["cols"]) - stat_set = NamedDimsArray(store[:, :]; zip(Symbol.(dims), [row_names, col_names])...) + stats = string.(store.attrs["stats"]) + scenario_ids = string.(store.attrs["scenarios"]) + loc_ids = string.(store.attrs["locations"]) + stat_set = NamedDimsArray( + store[:, :, :]; zip(Symbol.(dims), [stats, scenario_ids, loc_ids])... + ) stat_d[rcp_dirs[i]] = stat_set end From 2de4b292b1ec957a8757ed20d21e05edf461882b Mon Sep 17 00:00:00 2001 From: Rose Crocker Date: Tue, 5 Dec 2023 11:50:24 +1000 Subject: [PATCH 4/7] Fix missing element-wise assignment --- src/io/result_io.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/result_io.jl b/src/io/result_io.jl index a288bc701..e3fd08365 100644 --- a/src/io/result_io.jl +++ b/src/io/result_io.jl @@ -43,7 +43,7 @@ function summarize_env_data(data::AbstractArray)::Array{Float64} # TODO: Update once stats_store = Array{Float64}(zeros(2, size(data, 3), size(data, 2))) stats_store[1, :, :] .= Array(dropdims(mean(data; dims=1); dims=1)') - stats_store[2, :, :] = Array(dropdims(std(data; dims=1); dims=1)') + stats_store[2, :, :] .= Array(dropdims(std(data; dims=1); dims=1)') return stats_store end From 6d79d1240d1649b087cb8f6f24017d327b6d6e2f Mon Sep 17 00:00:00 2001 From: Rose Crocker Date: Tue, 5 Dec 2023 11:55:36 +1000 Subject: [PATCH 5/7] Convert `dims` to symbol on assignment --- src/io/result_io.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/io/result_io.jl b/src/io/result_io.jl index e3fd08365..1e3f3e774 100644 --- a/src/io/result_io.jl +++ b/src/io/result_io.jl @@ -361,12 +361,13 @@ function _recreate_stats_from_store(zarr_store_path::String)::Dict{String,Abstra for (i, sd) in enumerate(rcp_stat_dirs) store = zopen(sd, fill_as_missing=false) - dims = store.attrs["structure"] + dim_names = Symbol.(store.attrs["structure"]) stats = string.(store.attrs["stats"]) scenario_ids = string.(store.attrs["scenarios"]) loc_ids = string.(store.attrs["locations"]) stat_set = NamedDimsArray( - store[:, :, :]; zip(Symbol.(dims), [stats, scenario_ids, loc_ids])... + store[:, :, :]; + zip(dim_names, [stats, scenario_ids, loc_ids])..., ) stat_d[rcp_dirs[i]] = stat_set From 6eed4ee87322a53f297df85808815e98a325821e Mon Sep 17 00:00:00 2001 From: Rose Crocker Date: Tue, 5 Dec 2023 12:28:18 +1000 Subject: [PATCH 6/7] Change to assigning type for stats storage array --- src/io/result_io.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/io/result_io.jl b/src/io/result_io.jl index 1e3f3e774..a7012855c 100644 --- a/src/io/result_io.jl +++ b/src/io/result_io.jl @@ -41,7 +41,7 @@ Matrix{Float64, 2}, of mean and standard deviation for each environmental scenar """ function summarize_env_data(data::AbstractArray)::Array{Float64} # TODO: Update once - stats_store = Array{Float64}(zeros(2, size(data, 3), size(data, 2))) + stats_store::Array{Float64} = zeros(2, size(data, 3), size(data, 2)) stats_store[1, :, :] .= Array(dropdims(mean(data; dims=1); dims=1)') stats_store[2, :, :] .= Array(dropdims(std(data; dims=1); dims=1)') return stats_store From eb9645ea90fb1ad3179723073e15508d1f755d9a Mon Sep 17 00:00:00 2001 From: Rose Crocker Date: Wed, 6 Dec 2023 08:36:42 +1000 Subject: [PATCH 7/7] Remove unnecessary Array specification --- src/io/result_io.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/io/result_io.jl b/src/io/result_io.jl index a7012855c..168be255c 100644 --- a/src/io/result_io.jl +++ b/src/io/result_io.jl @@ -42,8 +42,8 @@ Matrix{Float64, 2}, of mean and standard deviation for each environmental scenar function summarize_env_data(data::AbstractArray)::Array{Float64} # TODO: Update once stats_store::Array{Float64} = zeros(2, size(data, 3), size(data, 2)) - stats_store[1, :, :] .= Array(dropdims(mean(data; dims=1); dims=1)') - stats_store[2, :, :] .= Array(dropdims(std(data; dims=1); dims=1)') + stats_store[1, :, :] .= dropdims(mean(data; dims=1); dims=1)' + stats_store[2, :, :] .= dropdims(std(data; dims=1); dims=1)' return stats_store end