From c46abb091d4b34fd244f5ca3aacc32c9d80a4499 Mon Sep 17 00:00:00 2001 From: Michael Uleysky Date: Fri, 14 Jun 2024 15:56:45 +1000 Subject: [PATCH] Removed some features from the NCZarr interface. Removed direct access to variables and dimensions in NCZarr, dimensions are now bound to variables. Separate CoordsNames and DimNames structures in the NCFuncs class. --- include/layereddata.h | 2 +- include/layereddataz.h | 8 ++-- include/ncfuncs.h | 8 +++- include/nczarr.h | 15 ++----- include/nczarrcommon.h | 34 +++++++-------- sources/AVISOLOCAL.cpp | 3 +- src/layereddataz.cpp | 6 +-- src/ncfuncs.cpp | 93 +++++++++++++++++++----------------------- 8 files changed, 79 insertions(+), 90 deletions(-) diff --git a/include/layereddata.h b/include/layereddata.h index 0fdbd0f..0a4f893 100644 --- a/include/layereddata.h +++ b/include/layereddata.h @@ -80,7 +80,7 @@ class LayeredData: public NCFuncs std::vector nc; std::vector depths; std::vector times; - struct CoordNames dname; + struct DimNames dname; real lonb, latb, lone, late; real lonstep, latstep; MString title; diff --git a/include/layereddataz.h b/include/layereddataz.h index f70b6ca..5048ef8 100644 --- a/include/layereddataz.h +++ b/include/layereddataz.h @@ -77,7 +77,7 @@ class LayeredDataZ: public NCFuncs std::vector depths; bool depthinv; std::vector times; - struct CoordNames dname; + struct DimNames dname; real lonb, latb, lone, late; real lonstep, latstep; MString title; @@ -195,11 +195,11 @@ class LayeredDataZ: public NCFuncs { auto tind = nc[i].Index(tm); if(tind == 0) continue; - for(const auto& v: nc[i].Vars()) + for(const auto& v: nc[i].VarNames()) { - auto stname = nc[i].AttString(v.Name(), "standard_name"); + auto stname = nc[i].AttString(v, "standard_name"); if(!stname.Exist()) continue; - if(StName2Name(stname) == vname) return {v.Name(), i, tind - 1}; + if(StName2Name(stname) == vname) return {v, i, tind - 1}; } } return {"", 0, 0}; diff --git a/include/ncfuncs.h b/include/ncfuncs.h index fc508fb..7d49494 100644 --- a/include/ncfuncs.h +++ b/include/ncfuncs.h @@ -15,6 +15,10 @@ class NCFuncs { public: struct CoordNames + { + MString lonname, latname, depthname, timename; + }; + struct DimNames { MString lonname, latname, depthname, timename; size_t nx, ny, nz, nt; @@ -26,12 +30,12 @@ class NCFuncs static std::tuple Refdate(const MString& refdate); static void GetVars(const NCFileA& nc, std::set& vars); static CoordNames GetCNames(const NCFileA& nc); - static CoordNames GetDNames(const NCFileA& nc); + static DimNames GetDNames(const NCFileA& nc); static bool HaveVar(const NCFileA& nc, const MString& vname); static void GetVars(const NCZarr& nc, std::set& vars); static CoordNames GetCNames(const NCZarr& nc); - static CoordNames GetDNames(const NCZarr& nc); + static DimNames GetDNames(const NCZarr& nc); static bool HaveVar(const NCZarr& nc, const MString& vname); template static VarPresence CheckVar(const MString& vname, HV hv) diff --git a/include/nczarr.h b/include/nczarr.h index 17902e2..50886c5 100644 --- a/include/nczarr.h +++ b/include/nczarr.h @@ -31,7 +31,6 @@ class NCZarr: private NCZarrBase, private DimReqDef return V([](const auto& arg) { return static_cast(arg); }); } - DEFFUNC(NDim) DEFFUNC1(NDim) DEFFUNC(NAtt) @@ -44,10 +43,9 @@ class NCZarr: private NCZarrBase, private DimReqDef DEFFUNC1(VarT) DEFFUNC1(VarFill) - DEFFUNC(DimNames) DEFFUNC1(DimNames) - DEFFUNC1(DimSize) + DEFFUNC2(DimSize) DEFFUNC2(AttT) DEFFUNC2(AttInt) @@ -63,14 +61,11 @@ class NCZarr: private NCZarrBase, private DimReqDef DEFFUNC1(AttString) DEFFUNC1(AttBool) - DEFFUNC1(HasDim) + DEFFUNC2(HasDim) DEFFUNC1(HasVar) DEFFUNC1(HasAtt) DEFFUNC2(HasAtt) - DEFFUNC(Vars) - DEFFUNC(Dims) - template Error Read(const MString& vname, Data& data, Transform transform, const char* request) const { return V([&vname = std::as_const(vname), &data = data, &transform = transform, request = request](const auto& arg) { return arg.Read(vname, data, transform, request); }); @@ -87,15 +82,13 @@ class NCZarr: private NCZarrBase, private DimReqDef { return arg.Read(vname, data, transform, std::move(req1), std::move(req2)); }); } - template - Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2, DimReq&& req3) const + template Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2, DimReq&& req3) const { return V([&vname = std::as_const(vname), &data = data, &transform = transform, &req1 = req1, &req2 = req2, &req3 = req3](const auto& arg) { return arg.Read(vname, data, transform, std::move(req1), std::move(req2), std::move(req3)); }); } - template - Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2, DimReq&& req3, DimReq&& req4) const + template Error Read(const MString& vname, Data& data, Transform transform, DimReq&& req1, DimReq&& req2, DimReq&& req3, DimReq&& req4) const { return V([&vname = std::as_const(vname), &data = data, &transform = transform, &req1 = req1, &req2 = req2, &req3 = req3, &req4 = req4](const auto& arg) { return arg.Read(vname, data, transform, std::move(req1), std::move(req2), std::move(req3), std::move(req4)); }); diff --git a/include/nczarrcommon.h b/include/nczarrcommon.h index 783e467..f1312c2 100644 --- a/include/nczarrcommon.h +++ b/include/nczarrcommon.h @@ -381,8 +381,6 @@ class NcZarrTypes public: explicit operator bool() const { return !vars.empty(); } - size_t NDim() const { return dims.size(); } - size_t NDim(const MString& var) const { size_t ind = FindInd(var, vars); @@ -431,13 +429,6 @@ class NcZarrTypes return ind < vars.size() ? vars[ind].Fill() : Variable::FillType(); } - auto DimNames() const - { - std::vector out; - std::transform(dims.cbegin(), dims.cend(), std::back_inserter(out), [](const Dimension& d) { return d.Name(); }); - return out; - } - auto DimNames(const MString& var) const { size_t ind = FindInd(var, vars); @@ -450,8 +441,10 @@ class NcZarrTypes return out; } - size_t DimSize(const MString& dim) const + size_t DimSize(const MString& var, const MString& dim) const { + if(!HasDim(var, dim)) return 0; + size_t ind = FindInd(dim, dims); return ind < dims.size() ? dims[ind].Size() : 0; } @@ -535,13 +528,19 @@ class NcZarrTypes auto AttString(const MString& name) const { return AttString("", name); } auto AttBool(const MString& name) const { return AttBool("", name); } - bool HasDim(const MString& name) const { return FindInd(name, dims) < dims.size(); } + bool HasDim(const MString& var, const MString& name) const + { + size_t vind = FindInd(var, vars); + if(vind >= vars.size()) return false; + + for(const auto dind: vars[vind].Dims()) + if(dims[dind].Name() == name) return true; + return false; + } + bool HasVar(const MString& name) const { return FindInd(name, vars) < vars.size(); } bool HasAtt(const MString& vname, const MString& aname) const { return AttT(vname, aname) != AttType::UNDEF; } bool HasAtt(const MString& aname) const { return AttT(aname) != AttType::UNDEF; } - - const auto& Vars() const { return vars; } - const auto& Dims() const { return dims; } }; class DimReqDef @@ -713,7 +712,8 @@ template class NcZarrRead: public C, public DimReqDef std::vector pdims; const auto vdims = C::DimNames(vname); - std::transform(vdims.cbegin(), vdims.cend(), std::back_inserter(pdims), [this](const MString& n) -> struct DimReq { return {n, 0, C::DimSize(n)}; }); + std::transform(vdims.cbegin(), vdims.cend(), std::back_inserter(pdims), + [this, &vname = std::as_const(vname)](const MString& n) -> struct DimReq { return {n, 0, C::DimSize(vname, n)}; }); return Read(vname, data, transform, pdims); } @@ -744,7 +744,7 @@ template class NcZarrRead: public C, public DimReqDef if(transindex[i] == ind) return {pref, "Parameters for dimension " + req.name + " already defined"}; transindex.push_back(ind); - size_t dlen = C::DimSize(pdims[ind].name); + size_t dlen = C::DimSize(vname, pdims[ind].name); if(req.beg == req.fill && req.count == req.fill) // Only name, so, we request full length { pdims[ind].beg = 0; @@ -818,7 +818,7 @@ template class NcZarrRead: public C, public DimReqDef template Error Read(const MString& vname, std::vector& out) const { const auto& dnames = C::DimNames(vname); - if(dnames.size() > 0) out.resize(C::DimSize(dnames[0])); + if(dnames.size() > 0) out.resize(C::DimSize(vname, dnames[0])); auto data = [&vec = out](size_t i) -> Type& { return vec[i]; }; return Read(vname, data, std::identity()); diff --git a/sources/AVISOLOCAL.cpp b/sources/AVISOLOCAL.cpp index a61fd1c..325430f 100644 --- a/sources/AVISOLOCAL.cpp +++ b/sources/AVISOLOCAL.cpp @@ -6,7 +6,8 @@ MString AVISOLOCALData::Info() const if(!isOk()) return ""; NCFileA nc; - struct CoordNames cn, dn; + struct CoordNames cn; + struct DimNames dn; std::set vars; nc.Reset(datapath + "/uv-" + times[0].ToString() + ".nc"); diff --git a/src/layereddataz.cpp b/src/layereddataz.cpp index b8a21be..4ea0494 100644 --- a/src/layereddataz.cpp +++ b/src/layereddataz.cpp @@ -229,10 +229,10 @@ bool LayeredDataZ::Read(const MString& vname, std::mapHeader(); - for(const auto& v: nc[id].Vars()) - if(v.Name() == name) + for(const auto& v: nc[id].VarNames()) + if(v == name) { - if(v.NDim() == 3) nodepth = true; + if(nc[id].NDim(v) == 3) nodepth = true; data = ReadVarRaw(nc[id], name, tid, nodepth, p); if(data) { diff --git a/src/ncfuncs.cpp b/src/ncfuncs.cpp index c799fb1..7bebafe 100644 --- a/src/ncfuncs.cpp +++ b/src/ncfuncs.cpp @@ -1,10 +1,10 @@ #define MICHLIB_NOSOURCE #include "ncfuncs.h" -NCFuncs::CoordNames NCFuncs::GetDNames(const NCFileA& nc) +NCFuncs::DimNames NCFuncs::GetDNames(const NCFileA& nc) { - CoordNames out; - auto head = nc.Header(); + DimNames out; + auto head = nc.Header(); for(const auto& dim: head.Dimensions()) { if(dim.Name() == "lon" || dim.Name() == "longitude") @@ -31,32 +31,33 @@ NCFuncs::CoordNames NCFuncs::GetDNames(const NCFileA& nc) return out; } -NCFuncs::CoordNames NCFuncs::GetDNames(const NCZarr& nc) +NCFuncs::DimNames NCFuncs::GetDNames(const NCZarr& nc) { - CoordNames out; - for(const auto& dim: nc.Dims()) - { - if(dim.Name() == "lon" || dim.Name() == "longitude") - { - out.lonname = dim.Name(); - out.nx = dim.Size(); - } - if(dim.Name() == "lat" || dim.Name() == "latitude") - { - out.latname = dim.Name(); - out.ny = dim.Size(); - } - if(dim.Name() == "depth" || dim.Name() == "elevation") + DimNames out; + for(const auto& v: nc.VarNames()) + for(const auto& dim: nc.DimNames(v)) { - out.depthname = dim.Name(); - out.nz = dim.Size(); - } - if(dim.Name() == "time") - { - out.timename = dim.Name(); - out.nt = dim.Size(); + if(dim == "lon" || dim == "longitude") + { + out.lonname = dim; + out.nx = nc.DimSize(v, dim); + } + if(dim == "lat" || dim == "latitude") + { + out.latname = dim; + out.ny = nc.DimSize(v, dim); + } + if(dim == "depth" || dim == "elevation") + { + out.depthname = dim; + out.nz = nc.DimSize(v, dim); + } + if(dim == "time") + { + out.timename = dim; + out.nt = nc.DimSize(v, dim); + } } - } return out; } @@ -84,11 +85,6 @@ NCFuncs::CoordNames NCFuncs::GetCNames(const NCFileA& nc) if(islat) out.latname = v.Name(); if(isdepth) out.depthname = v.Name(); if(istime) out.timename = v.Name(); - - if(islon) out.nx = v.Dimensions().size(); - if(islat) out.ny = v.Dimensions().size(); - if(isdepth) out.nz = v.Dimensions().size(); - if(istime) out.nt = v.Dimensions().size(); } return out; } @@ -96,14 +92,14 @@ NCFuncs::CoordNames NCFuncs::GetCNames(const NCFileA& nc) NCFuncs::CoordNames NCFuncs::GetCNames(const NCZarr& nc) { CoordNames out; - for(const auto& v: nc.Vars()) // Try to define coordinates by attribute standard_name or attribute axis + for(const auto& v: nc.VarNames()) // Try to define coordinates by attribute standard_name or attribute axis { - auto havestname = nc.HasAtt(v.Name(), "standard_name"); - auto haveaxis = nc.HasAtt(v.Name(), "axis"); + auto havestname = nc.HasAtt(v, "standard_name"); + auto haveaxis = nc.HasAtt(v, "axis"); if(!(havestname || haveaxis)) continue; - auto stname = nc.AttString(v.Name(), "standard_name"); - auto axis = nc.AttString(v.Name(), "axis"); + auto stname = nc.AttString(v, "standard_name"); + auto axis = nc.AttString(v, "axis"); bool islon = false, islat = false, isdepth = false, istime = false; if(stname == "longitude") islon = true; if(stname == "latitude") islat = true; @@ -115,15 +111,10 @@ NCFuncs::CoordNames NCFuncs::GetCNames(const NCZarr& nc) if(!out.depthname.Exist() && axis == "Z") isdepth = true; if(!out.timename.Exist() && axis == "T") istime = true; - if(islon) out.lonname = v.Name(); - if(islat) out.latname = v.Name(); - if(isdepth) out.depthname = v.Name(); - if(istime) out.timename = v.Name(); - - if(islon) out.nx = v.Dims().size(); - if(islat) out.ny = v.Dims().size(); - if(isdepth) out.nz = v.Dims().size(); - if(istime) out.nt = v.Dims().size(); + if(islon) out.lonname = v; + if(islat) out.latname = v; + if(isdepth) out.depthname = v; + if(istime) out.timename = v; } // If time not found just check variable "time" @@ -152,10 +143,10 @@ void NCFuncs::GetVars(const NCFileA& nc, std::set& vars) void NCFuncs::GetVars(const NCZarr& nc, std::set& vars) { - for(const auto& v: nc.Vars()) + for(const auto& v: nc.VarNames()) { - if(!nc.HasAtt(v.Name(), "standard_name")) continue; - auto ret = nc.AttString(v.Name(), "standard_name"); + if(!nc.HasAtt(v, "standard_name")) continue; + auto ret = nc.AttString(v, "standard_name"); if(StName2Name(ret).Exist()) vars.emplace(StName2Name(ret)); } if((vars.contains("ptemp") || vars.contains("temp")) && vars.contains("sal")) vars.emplace("pdens"); @@ -274,10 +265,10 @@ bool NCFuncs::HaveVar(const NCFileA& nc, const MString& vname) bool NCFuncs::HaveVar(const NCZarr& nc, const MString& vname) { - for(const auto& v: nc.Vars()) + for(const auto& v: nc.VarNames()) { - if(!nc.HasAtt(v.Name(), "standard_name")) continue; - auto stname = nc.AttString(v.Name(), "standard_name"); + if(!nc.HasAtt(v, "standard_name")) continue; + auto stname = nc.AttString(v, "standard_name"); if(StName2Name(stname) == vname) return true; } return false;