From 2a99a98192c2d89cbf775b497c216b49eb089f5f Mon Sep 17 00:00:00 2001 From: Michael Uleysky Date: Thu, 11 Apr 2024 16:52:36 +1000 Subject: [PATCH] Fix some errors in ZARR reader --- include/nczarrcommon.h | 24 +++++++++++++++++++----- include/zarr.h | 9 +++++---- src/zarr.cpp | 14 ++++++++++++-- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/include/nczarrcommon.h b/include/nczarrcommon.h index a1a6807..414b1c8 100644 --- a/include/nczarrcommon.h +++ b/include/nczarrcommon.h @@ -539,6 +539,9 @@ class NcZarrTypes bool HasVar(const MString& name) const { return FindInd(name, vars) < vars.size(); } bool HasAtt(const MString& vname, const MString& aname) const { return AttT(vname, aname) != AttType::UNDEF; } bool HasAtt(const MString& aname) const { return AttT(aname) != AttType::UNDEF; } + + const auto& Vars() const { return vars; } + const auto& Dims() const { return dims; } }; class DimReqDef @@ -634,7 +637,6 @@ template class NcZarrRead: public C, public DimReqDef d.Fillval() } -> std::convertible_to; }) - fillout = data.Fillval(); else // Data does'nt have own fillvalue, using variable fillvalue fillout = static_cast(fillin); @@ -675,8 +677,6 @@ template class NcZarrRead: public C, public DimReqDef inind++; } - michlib::message("Variable " + vname + ", request size " + nval); - for(const auto& r: reqs) michlib::message(r.name + " from " + r.beg + ", count " + r.count); return Error(); } @@ -775,9 +775,13 @@ template class NcZarrRead: public C, public DimReqDef if(pdims[ind].beg >= dlen) return {pref, MString("Error parsing request: start index ") + pdims[ind].beg + " must be lesser then " + pdims[ind].name + " size " + dlen}; if(pdims[ind].beg + pdims[ind].count > dlen) return {pref, MString("Error parsing request: start index ") + pdims[ind].beg + " with count " + pdims[ind].count + " exceeds " + pdims[ind].name + " size " + dlen}; + + // Ignore hyperplanes in requests for calculation of data dimensionality + if(pdims[transindex.back()].count == 1) transindex.pop_back(); } - if(transindex.size() != Dimensionity()) return {pref, "Output data dimensions not correspondind request dimensions"}; + if(transindex.size() != Dimensionity()) + return {pref, MString("Output data dimensions (") + Dimensionity() + ") not corresponding request dimensions (" + transindex.size() + ")"}; switch(C::VarT(vname)) { case(C::VarType::UNDEF): return {pref, "No variable with name " + vname + " (impossible)"}; @@ -787,7 +791,7 @@ template class NcZarrRead: public C, public DimReqDef case(C::VarType::INT2): return Read>(vname, transindex, data, transform, pdims); case(C::VarType::INT4): return Read>(vname, transindex, data, transform, pdims); case(C::VarType::INT8): return Read>(vname, transindex, data, transform, pdims); - case(C::VarType::UINT1): return Read>(vname, transindex, data, transform, pdims); + case(C::VarType::UINT1): return Read>(vname, transindex, data, transform, pdims); } return {pref, "Internal error (impossible)"}; @@ -819,4 +823,14 @@ template class NcZarrRead: public C, public DimReqDef return Read(vname, data, transform, pdims); } + + // Request full one-dimensional variable + template Error Read(const MString& vname, std::vector& out) const + { + const auto& dnames = C::DimNames(vname); + if(dnames.size() > 0) out.resize(C::DimSize(dnames[0])); + auto data = [&vec = out](size_t i) -> Type& { return vec[i]; }; + + return Read(vname, data, std::identity()); + } }; diff --git a/include/zarr.h b/include/zarr.h index 0f692ec..5794add 100644 --- a/include/zarr.h +++ b/include/zarr.h @@ -20,7 +20,7 @@ class ZarrTypes: public NcZarrTypes std::vector> data; public: - ReadedData():mainind(Vec()),chunkind(Vec()),inchunkind(Vec()){} + ReadedData(): mainind(Vec()), chunkind(Vec()), inchunkind(Vec()) {} ReadedData(size_t N, const size_t* start, const size_t* count, const size_t* csize, std::vector>&& d): start(start, start + N), @@ -122,7 +122,8 @@ class ZarrFunctions: public ZarrTypes size_t ind = FindInd(var, vars); const size_t N = vars[ind].NDim(); const auto& csize = chunks[ind]; - Vec chunkstart( + + Vec chunkstart( [](size_t N, const size_t* st, const size_t* cs) { Vec out(N); @@ -133,7 +134,7 @@ class ZarrFunctions: public ZarrTypes [](size_t N, const size_t* st, const size_t* cn, const size_t* cs) { Vec out(N); - for(size_t i = 0; i < N; i++) out[i] = (st[i] + cn[i]) / cs[i] - st[i] / cs[i] + 1; + for(size_t i = 0; i < N; i++) out[i] = (st[i] + cn[i] - 1) / cs[i] - st[i] / cs[i] + 1; return out; }(N, start, count, csize.data())); @@ -158,7 +159,7 @@ class ZarrFunctions: public ZarrTypes for(; chunkind; ++chunkind) { cdata[chunkind.Index()].reset(new VType[chunksize]); - auto res = GetChunk(var, chunkind.VIndex(chunkstart), chunksize, sizeof(VType), cdata[chunkind.Index()].get(),havefill?&fill:nullptr); + auto res = GetChunk(var, chunkind.VIndex(chunkstart), chunksize, sizeof(VType), cdata[chunkind.Index()].get(), havefill ? &fill : nullptr); if(!res) return res; } diff --git a/src/zarr.cpp b/src/zarr.cpp index 0ae4d06..5b36587 100644 --- a/src/zarr.cpp +++ b/src/zarr.cpp @@ -208,9 +208,19 @@ Error ZarrFunctions::Open(const MString& product, const MString& dataset, bool t CopernicusCatalog cat; Json::Value json; + MString realdataset; + if(!dataset.Exist()) { - auto urlret = time ? cat.DatasetTimeURL(product, dataset) : cat.DatasetGeoURL(product, dataset); - if(!urlret) return urlret.Add(pref, "Can't get url for the dataset " + dataset + " of product " + product); + auto dsets = cat.DatasetList(product); + if(!dsets) return dsets.Add(pref, "Can't get default dataset of product " + product); + realdataset = dsets.Value()[0]; + } + else + realdataset = dataset; + + { + auto urlret = time ? cat.DatasetTimeURL(product, realdataset) : cat.DatasetGeoURL(product, realdataset); + if(!urlret) return urlret.Add(pref, "Can't get url for the dataset " + realdataset + " of product " + product); url = urlret.Value(); auto ret = cat.GetJSON(url + "/.zmetadata");