From 8c684abd94281af6badf17148d127b142e5280ab Mon Sep 17 00:00:00 2001 From: tiemvanderdeure Date: Fri, 27 Jun 2025 12:33:14 +0200 Subject: [PATCH 1/2] implement attributemod --- src/modifieddiskarray.jl | 42 ++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/src/modifieddiskarray.jl b/src/modifieddiskarray.jl index 4628e37bb..c54125b47 100644 --- a/src/modifieddiskarray.jl +++ b/src/modifieddiskarray.jl @@ -40,10 +40,28 @@ function _resolve_mod_eltype(::Type{T}, missingval, scale, offset) where T return T3 end -missingval(m::Mod) = m.missingval -missingval(m::NoMod) = m.missingval +# Modifies by dictionary lookup - similar to categorical array +struct AttributeMod{T, Mi, Di} <: AbstractModifications{T} + table::NamedTuple + column::Symbol + dict::Di + missingval::Mi -_inner_missingval(m::Mod) = _inner_missingval(m.missingval) + function AttributeMod(table::NamedTuple, column::Symbol, missingval::Mi) where {Mi} + keycol = identity.(first(table)) + valcol = identity.(table[column]) + dict = DD.OrderedCollections.LittleDict(keycol, valcol) |> DD.OrderedCollections.freeze + new{eltype(table[column]), Mi, typeof(dict)}(table, column, dict, missingval) + end +end + +## Todo in a DBFTables extension +_tablemod(args...) = error() + +## missingval handling +missingval(m::AbstractModifications) = m.missingval + +_inner_missingval(m::AbstractModifications) = _inner_missingval(m.missingval) _inner_missingval(mv) = mv _inner_missingval(mv::Pair) = mv[1] @@ -78,7 +96,7 @@ DiskArrays.haschunks(A::ModifiedDiskArray) = DiskArrays.haschunks(parent(A)) DiskArrays.eachchunk(A::ModifiedDiskArray) = DiskArrays.eachchunk(parent(A)) function DiskArrays.readblock!( - A::ModifiedDiskArray{<:Any,<:Any,<:Any,<:Mod}, outer_block, I::AbstractVector... + A::ModifiedDiskArray{<:Any,<:Any,<:Any,<:AbstractModifications}, outer_block, I::AbstractVector... ) if isdisk(parent(A)) inner_block = similar(outer_block, eltype(parent(A))) @@ -121,6 +139,11 @@ function DiskArrays.writeblock!( parent(A)[I...] = block end end +function DiskArrays.writeblock!( + A::ModifiedDiskArray{<:Any,<:Any,<:Any,<:AttributeMod}, block, I::AbstractVector... +) + error("Writing using an Attribute Table is not supported") +end Base.@assume_effects :foldable function _applymod(x, m::Mod) if _ismissing(x, _inner_missingval(m)) @@ -130,6 +153,13 @@ Base.@assume_effects :foldable function _applymod(x, m::Mod) end end Base.@assume_effects :foldable _applymod(x, ::NoMod) = x +Base.@assume_effects :foldable function _applymod(x, m::AttributeMod) + if _ismissing(x, _inner_missingval(m)) + _outer_missingval(m) + else + m.dict[x] + end +end _ismissing(x, mv) = ismissing(x) || x === mv _ismissing(x, ::Nothing) = ismissing(x) @@ -198,7 +228,7 @@ function get_scale(metadata, scaled::Bool) end _mod_eltype(::AbstractArray{T}, ::NoMod) where T = T -_mod_eltype(::AbstractArray, m::Mod{T}) where T = T +_mod_eltype(::AbstractArray, m::AbstractModifications{T}) where T = T _mod_inverse_eltype(::AbstractArray{T}, ::NoMod) where T = T _mod_inverse_eltype(::AbstractArray{T}, m::Mod) where T = @@ -248,4 +278,4 @@ function _read_missingval_pair(var, metadata, missingval) # Otherwise: detect missing value and convert it to `missingval` Rasters.missingval(var, metadata) => missingval end -end \ No newline at end of file +end From e1ccc56b3d0d913d2a74a0796131abd9df77d801 Mon Sep 17 00:00:00 2001 From: tiemvanderdeure Date: Fri, 27 Jun 2025 12:33:28 +0200 Subject: [PATCH 2/2] check if a file has a .vat.dbf --- src/array.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/array.jl b/src/array.jl index 431f7b7ad..9ef0e864c 100644 --- a/src/array.jl +++ b/src/array.jl @@ -365,8 +365,17 @@ function _raster(ds; data_out, dims_out, metadata_out, missingval_out = _open(source, ds; name=name1, group, mod=nothing) do var metadata_out = isnokw(metadata) ? _metadata(var) : metadata missingval_out = _read_missingval_pair(var, metadata_out, missingval) + # Generate mod for scaling - mod = isnokw(mod) ? _mod(eltype(var), metadata_out, missingval_out; scaled, coerce) : mod + mod = if isnokw(mod) + if !raw && isfile(filename .* ".vat.dbf") # todo: how to detect a RAT/VAT? + _tablemod(filename .* ".vat.dbf", missingval_out, name) # name or name1? + else + _mod(eltype(var), metadata_out, missingval_out; scaled, coerce) + end + else + mod + end # Define or load the data array data_out = if lazy # Define a lay FileArray