Browse Source

Fix some errors in ZARR reader

lintest
Michael Uleysky 8 months ago
parent
commit
2a99a98192
  1. 24
      include/nczarrcommon.h
  2. 9
      include/zarr.h
  3. 14
      src/zarr.cpp

24
include/nczarrcommon.h

@ -539,6 +539,9 @@ class NcZarrTypes
bool HasVar(const MString& name) const { return FindInd(name, vars) < vars.size(); }
bool HasAtt(const MString& vname, const MString& aname) const { return AttT(vname, aname) != AttType::UNDEF; }
bool HasAtt(const MString& aname) const { return AttT(aname) != AttType::UNDEF; }
const auto& Vars() const { return vars; }
const auto& Dims() const { return dims; }
};
class DimReqDef
@ -634,7 +637,6 @@ template<class C> class NcZarrRead: public C, public DimReqDef
d.Fillval()
} -> std::convertible_to<DataType>;
})
fillout = data.Fillval();
else // Data does'nt have own fillvalue, using variable fillvalue
fillout = static_cast<DataType>(fillin);
@ -675,8 +677,6 @@ template<class C> class NcZarrRead: public C, public DimReqDef
inind++;
}
michlib::message("Variable " + vname + ", request size " + nval);
for(const auto& r: reqs) michlib::message(r.name + " from " + r.beg + ", count " + r.count);
return Error();
}
@ -775,9 +775,13 @@ template<class C> class NcZarrRead: public C, public DimReqDef
if(pdims[ind].beg >= dlen) return {pref, MString("Error parsing request: start index ") + pdims[ind].beg + " must be lesser then " + pdims[ind].name + " size " + dlen};
if(pdims[ind].beg + pdims[ind].count > dlen)
return {pref, MString("Error parsing request: start index ") + pdims[ind].beg + " with count " + pdims[ind].count + " exceeds " + pdims[ind].name + " size " + dlen};
// Ignore hyperplanes in requests for calculation of data dimensionality
if(pdims[transindex.back()].count == 1) transindex.pop_back();
}
if(transindex.size() != Dimensionity<Data>()) return {pref, "Output data dimensions not correspondind request dimensions"};
if(transindex.size() != Dimensionity<Data>())
return {pref, MString("Output data dimensions (") + Dimensionity<Data>() + ") not corresponding request dimensions (" + transindex.size() + ")"};
switch(C::VarT(vname))
{
case(C::VarType::UNDEF): return {pref, "No variable with name " + vname + " (impossible)"};
@ -787,7 +791,7 @@ template<class C> class NcZarrRead: public C, public DimReqDef
case(C::VarType::INT2): return Read<typename C::template Type<C::VarType::INT2>>(vname, transindex, data, transform, pdims);
case(C::VarType::INT4): return Read<typename C::template Type<C::VarType::INT4>>(vname, transindex, data, transform, pdims);
case(C::VarType::INT8): return Read<typename C::template Type<C::VarType::INT8>>(vname, transindex, data, transform, pdims);
case(C::VarType::UINT1): return Read<typename C::template Type<C::VarType::UINT1>>(vname, transindex, data, transform, pdims);
case(C::VarType::UINT1): return Read<typename C::template Type<C::VarType::INT1>>(vname, transindex, data, transform, pdims);
}
return {pref, "Internal error (impossible)"};
@ -819,4 +823,14 @@ template<class C> class NcZarrRead: public C, public DimReqDef
return Read(vname, data, transform, pdims);
}
// Request full one-dimensional variable
template<class Type> Error Read(const MString& vname, std::vector<Type>& out) const
{
const auto& dnames = C::DimNames(vname);
if(dnames.size() > 0) out.resize(C::DimSize(dnames[0]));
auto data = [&vec = out](size_t i) -> Type& { return vec[i]; };
return Read(vname, data, std::identity());
}
};

9
include/zarr.h

@ -20,7 +20,7 @@ class ZarrTypes: public NcZarrTypes
std::vector<std::unique_ptr<VType[]>> data;
public:
ReadedData():mainind(Vec()),chunkind(Vec()),inchunkind(Vec()){}
ReadedData(): mainind(Vec()), chunkind(Vec()), inchunkind(Vec()) {}
ReadedData(size_t N, const size_t* start, const size_t* count, const size_t* csize, std::vector<std::unique_ptr<VType[]>>&& d):
start(start, start + N),
@ -122,7 +122,8 @@ class ZarrFunctions: public ZarrTypes
size_t ind = FindInd(var, vars);
const size_t N = vars[ind].NDim();
const auto& csize = chunks[ind];
Vec chunkstart(
Vec chunkstart(
[](size_t N, const size_t* st, const size_t* cs)
{
Vec out(N);
@ -133,7 +134,7 @@ class ZarrFunctions: public ZarrTypes
[](size_t N, const size_t* st, const size_t* cn, const size_t* cs)
{
Vec out(N);
for(size_t i = 0; i < N; i++) out[i] = (st[i] + cn[i]) / cs[i] - st[i] / cs[i] + 1;
for(size_t i = 0; i < N; i++) out[i] = (st[i] + cn[i] - 1) / cs[i] - st[i] / cs[i] + 1;
return out;
}(N, start, count, csize.data()));
@ -158,7 +159,7 @@ class ZarrFunctions: public ZarrTypes
for(; chunkind; ++chunkind)
{
cdata[chunkind.Index()].reset(new VType[chunksize]);
auto res = GetChunk(var, chunkind.VIndex(chunkstart), chunksize, sizeof(VType), cdata[chunkind.Index()].get(),havefill?&fill:nullptr);
auto res = GetChunk(var, chunkind.VIndex(chunkstart), chunksize, sizeof(VType), cdata[chunkind.Index()].get(), havefill ? &fill : nullptr);
if(!res) return res;
}

14
src/zarr.cpp

@ -208,9 +208,19 @@ Error ZarrFunctions::Open(const MString& product, const MString& dataset, bool t
CopernicusCatalog cat;
Json::Value json;
MString realdataset;
if(!dataset.Exist())
{
auto urlret = time ? cat.DatasetTimeURL(product, dataset) : cat.DatasetGeoURL(product, dataset);
if(!urlret) return urlret.Add(pref, "Can't get url for the dataset " + dataset + " of product " + product);
auto dsets = cat.DatasetList(product);
if(!dsets) return dsets.Add(pref, "Can't get default dataset of product " + product);
realdataset = dsets.Value()[0];
}
else
realdataset = dataset;
{
auto urlret = time ? cat.DatasetTimeURL(product, realdataset) : cat.DatasetGeoURL(product, realdataset);
if(!urlret) return urlret.Add(pref, "Can't get url for the dataset " + realdataset + " of product " + product);
url = urlret.Value();
auto ret = cat.GetJSON(url + "/.zmetadata");

Loading…
Cancel
Save