Browse Source

Fix some errors in ZARR reader

lintest
Michael Uleysky 8 months ago
parent
commit
2a99a98192
  1. 24
      include/nczarrcommon.h
  2. 3
      include/zarr.h
  3. 14
      src/zarr.cpp

24
include/nczarrcommon.h

@ -539,6 +539,9 @@ class NcZarrTypes
bool HasVar(const MString& name) const { return FindInd(name, vars) < vars.size(); } bool HasVar(const MString& name) const { return FindInd(name, vars) < vars.size(); }
bool HasAtt(const MString& vname, const MString& aname) const { return AttT(vname, aname) != AttType::UNDEF; } bool HasAtt(const MString& vname, const MString& aname) const { return AttT(vname, aname) != AttType::UNDEF; }
bool HasAtt(const MString& aname) const { return AttT(aname) != AttType::UNDEF; } bool HasAtt(const MString& aname) const { return AttT(aname) != AttType::UNDEF; }
const auto& Vars() const { return vars; }
const auto& Dims() const { return dims; }
}; };
class DimReqDef class DimReqDef
@ -634,7 +637,6 @@ template<class C> class NcZarrRead: public C, public DimReqDef
d.Fillval() d.Fillval()
} -> std::convertible_to<DataType>; } -> std::convertible_to<DataType>;
}) })
fillout = data.Fillval(); fillout = data.Fillval();
else // Data does'nt have own fillvalue, using variable fillvalue else // Data does'nt have own fillvalue, using variable fillvalue
fillout = static_cast<DataType>(fillin); fillout = static_cast<DataType>(fillin);
@ -675,8 +677,6 @@ template<class C> class NcZarrRead: public C, public DimReqDef
inind++; inind++;
} }
michlib::message("Variable " + vname + ", request size " + nval);
for(const auto& r: reqs) michlib::message(r.name + " from " + r.beg + ", count " + r.count);
return Error(); return Error();
} }
@ -775,9 +775,13 @@ template<class C> class NcZarrRead: public C, public DimReqDef
if(pdims[ind].beg >= dlen) return {pref, MString("Error parsing request: start index ") + pdims[ind].beg + " must be lesser then " + pdims[ind].name + " size " + dlen}; if(pdims[ind].beg >= dlen) return {pref, MString("Error parsing request: start index ") + pdims[ind].beg + " must be lesser then " + pdims[ind].name + " size " + dlen};
if(pdims[ind].beg + pdims[ind].count > dlen) if(pdims[ind].beg + pdims[ind].count > dlen)
return {pref, MString("Error parsing request: start index ") + pdims[ind].beg + " with count " + pdims[ind].count + " exceeds " + pdims[ind].name + " size " + dlen}; return {pref, MString("Error parsing request: start index ") + pdims[ind].beg + " with count " + pdims[ind].count + " exceeds " + pdims[ind].name + " size " + dlen};
// Ignore hyperplanes in requests for calculation of data dimensionality
if(pdims[transindex.back()].count == 1) transindex.pop_back();
} }
if(transindex.size() != Dimensionity<Data>()) return {pref, "Output data dimensions not correspondind request dimensions"}; if(transindex.size() != Dimensionity<Data>())
return {pref, MString("Output data dimensions (") + Dimensionity<Data>() + ") not corresponding request dimensions (" + transindex.size() + ")"};
switch(C::VarT(vname)) switch(C::VarT(vname))
{ {
case(C::VarType::UNDEF): return {pref, "No variable with name " + vname + " (impossible)"}; case(C::VarType::UNDEF): return {pref, "No variable with name " + vname + " (impossible)"};
@ -787,7 +791,7 @@ template<class C> class NcZarrRead: public C, public DimReqDef
case(C::VarType::INT2): return Read<typename C::template Type<C::VarType::INT2>>(vname, transindex, data, transform, pdims); case(C::VarType::INT2): return Read<typename C::template Type<C::VarType::INT2>>(vname, transindex, data, transform, pdims);
case(C::VarType::INT4): return Read<typename C::template Type<C::VarType::INT4>>(vname, transindex, data, transform, pdims); case(C::VarType::INT4): return Read<typename C::template Type<C::VarType::INT4>>(vname, transindex, data, transform, pdims);
case(C::VarType::INT8): return Read<typename C::template Type<C::VarType::INT8>>(vname, transindex, data, transform, pdims); case(C::VarType::INT8): return Read<typename C::template Type<C::VarType::INT8>>(vname, transindex, data, transform, pdims);
case(C::VarType::UINT1): return Read<typename C::template Type<C::VarType::UINT1>>(vname, transindex, data, transform, pdims); case(C::VarType::UINT1): return Read<typename C::template Type<C::VarType::INT1>>(vname, transindex, data, transform, pdims);
} }
return {pref, "Internal error (impossible)"}; return {pref, "Internal error (impossible)"};
@ -819,4 +823,14 @@ template<class C> class NcZarrRead: public C, public DimReqDef
return Read(vname, data, transform, pdims); return Read(vname, data, transform, pdims);
} }
// Request full one-dimensional variable
template<class Type> Error Read(const MString& vname, std::vector<Type>& out) const
{
const auto& dnames = C::DimNames(vname);
if(dnames.size() > 0) out.resize(C::DimSize(dnames[0]));
auto data = [&vec = out](size_t i) -> Type& { return vec[i]; };
return Read(vname, data, std::identity());
}
}; };

3
include/zarr.h

@ -122,6 +122,7 @@ class ZarrFunctions: public ZarrTypes
size_t ind = FindInd(var, vars); size_t ind = FindInd(var, vars);
const size_t N = vars[ind].NDim(); const size_t N = vars[ind].NDim();
const auto& csize = chunks[ind]; const auto& csize = chunks[ind];
Vec chunkstart( Vec chunkstart(
[](size_t N, const size_t* st, const size_t* cs) [](size_t N, const size_t* st, const size_t* cs)
{ {
@ -133,7 +134,7 @@ class ZarrFunctions: public ZarrTypes
[](size_t N, const size_t* st, const size_t* cn, const size_t* cs) [](size_t N, const size_t* st, const size_t* cn, const size_t* cs)
{ {
Vec out(N); Vec out(N);
for(size_t i = 0; i < N; i++) out[i] = (st[i] + cn[i]) / cs[i] - st[i] / cs[i] + 1; for(size_t i = 0; i < N; i++) out[i] = (st[i] + cn[i] - 1) / cs[i] - st[i] / cs[i] + 1;
return out; return out;
}(N, start, count, csize.data())); }(N, start, count, csize.data()));

14
src/zarr.cpp

@ -208,9 +208,19 @@ Error ZarrFunctions::Open(const MString& product, const MString& dataset, bool t
CopernicusCatalog cat; CopernicusCatalog cat;
Json::Value json; Json::Value json;
MString realdataset;
if(!dataset.Exist())
{ {
auto urlret = time ? cat.DatasetTimeURL(product, dataset) : cat.DatasetGeoURL(product, dataset); auto dsets = cat.DatasetList(product);
if(!urlret) return urlret.Add(pref, "Can't get url for the dataset " + dataset + " of product " + product); if(!dsets) return dsets.Add(pref, "Can't get default dataset of product " + product);
realdataset = dsets.Value()[0];
}
else
realdataset = dataset;
{
auto urlret = time ? cat.DatasetTimeURL(product, realdataset) : cat.DatasetGeoURL(product, realdataset);
if(!urlret) return urlret.Add(pref, "Can't get url for the dataset " + realdataset + " of product " + product);
url = urlret.Value(); url = urlret.Value();
auto ret = cat.GetJSON(url + "/.zmetadata"); auto ret = cat.GetJSON(url + "/.zmetadata");

Loading…
Cancel
Save