michael_ul
/
odm
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

New interface connecting input and output data

master
Michael Uleysky 4 weeks ago
parent
55b70742a0
commit
83cae2f06a
2 changed files with 496 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 152
      include/Adapter.h
  2. 344
      src/Adapter.cpp

152
include/Adapter.h
Unescape View File

@ -0,0 +1,152 @@
#pragma once
#include "Data3D.h"
#include "ParseArgs.h"
#include "mdatetime.h"
#include "merrors.h"
#include "nczarr.h"
#include <functional>
using michlib::MDateTime;
using michlib::RetVal;
class Adapter
{
public:
// Информация о блоке чтения, имена и индексы размерностей
struct ReadInfo
{
MString xdname, ydname, zdname, tdname;
size_t xb, xe;
size_t yb, ye;
size_t zb, ze;
auto operator<=>(const ReadInfo&) const = default;
};
// Информация о переменной
struct VInfo
{
MString name; // Внутреннее имя в файле
MString unit, stname, lname, comment; // Единица измерения в файле, стандартное имя, человеческое имя, комментарий
MString targetunit; // Единица измерения прочитанных данных
std::shared_ptr<struct ReadInfo> rinfo; // Информация о блоке чтения
real scale; // Масштаб и
real offset; // смещение величины в файле
real fillval; // _FillVal
};
// Полная информация о переменной
struct VarInfo
{
struct VInfo info; // Информация о переменной в файле
std::shared_ptr<Projection> proj; // Горизонтальная структура прочитанных данных
std::shared_ptr<Vertical> vert; // Вертикальная структура прочитанных данных
// Функции чтения двумерного и трёхмерного блоков
std::function<RetVal<std::shared_ptr<Data2D>>(const Adapter&, const void*, std::shared_ptr<Projection>, size_t)> Read2D; // = Def2DReader;
std::function<RetVal<std::shared_ptr<Data3D>>(const Adapter&, const void*, std::shared_ptr<Projection>, std::shared_ptr<Vertical>, size_t)> Read3D; // = Def3DReader;
};
// Запись в таблице файлов
struct TimeRow
{
size_t beg, end; // Начальный и конечный временные индексы
MString file; // Имя файла, в котором хранятся данные для этого интервала времени
};
enum class ZarrMethod
{
NC,
ZARR,
MNC,
MZARR
};
Adapter(): Adapter(ZarrMethod::NC) {};
Adapter(ZarrMethod m): method(m) {}
Adapter(Adapter&&) = default;
Adapter& operator=(Adapter&&) = default;
template<class T>
requires std::same_as<std::vector<MDateTime>, std::decay_t<T>>
Error SetTimes(T&& t, const CLArgs& args, michlib_internal::ParameterListEx& pars)
{
times = std::forward<T>(t);
curtime = times.size();
return FilterTimes(args, pars);
}
template<class T>
requires std::same_as<std::vector<TimeRow>, std::decay_t<T>>
void SetTimeTable(T&& t)
{
timetable = std::forward<T>(t);
}
void SetTitle(const MString& t) { title = t; }
void Add2DVariable(const MString& name, struct VInfo&& vinfo, std::shared_ptr<Projection> proj, decltype(VarInfo().Read2D) func = Def2DReader)
{
vars[name] = {.info = std::move(vinfo), .proj = proj, .Read2D = func, .Read3D = {}};
}
void Add3DVariable(const MString& name, struct VInfo&& vinfo, std::shared_ptr<Projection> proj, std::shared_ptr<Vertical> vert, decltype(VarInfo().Read3D) func = Def3DReader)
{
vars[name] = {.info = std::move(vinfo), .proj = proj, .vert = vert, .Read2D = {}, .Read3D = func};
}
void CleanVariables(const std::set<MString>& keep)
{
std::erase_if(vars, [&keep](const auto& k) { return !keep.contains(k.first); });
}
const auto& Times() const { return times; }
const auto& TimeIndexes() const { return tindexes; }
const auto& Vars() const { return vars; }
const auto& Title() const { return title; }
const auto& Var(const MString& name) const { return vars.at(name); }
std::vector<MDateTime> IndexedTimes() const
{
std::vector<MDateTime> out(tindexes.size());
for(size_t i = 0; i < out.size(); i++) out[i] = times[tindexes[i]];
return out;
}
bool HasVar(const MString& name) const { return vars.contains(name); }
bool Is2D(const MString& name) const { return HasVar(name) && vars.at(name).Read2D; }
bool Is3D(const MString& name) const { return HasVar(name) && vars.at(name).Read3D; }
private:
// Функция чтения двумерных данных по умолчанию
static RetVal<std::shared_ptr<Data2D>> Def2DReader(const Adapter& ad, const void* vinfo, std::shared_ptr<Projection> proj, size_t it);
// Функция чтения трёхмерных данных по умолчанию
static RetVal<std::shared_ptr<Data3D>> Def3DReader(const Adapter& ad, const void* vinfo, std::shared_ptr<Projection> proj, std::shared_ptr<Vertical> vert, size_t it);
// Создание индексов tindexes сообразно параметрам командной строки
Error FilterTimes(const CLArgs& args, michlib_internal::ParameterListEx& pars);
MString title; // Описание данных
std::unique_ptr<NCZarr> nc; // Текущий файл, с которого происходит чтение
MString curfile; // Его имя
std::map<MString, VarInfo> vars; // Переменные, которые можно читать
ZarrMethod method; // Метод доступа к данным файла
std::vector<TimeRow> timetable; // Таблица файлов, каждый файл содержит даннные для соответствующего временного интервала
size_t curtime; // Номер текущей записи в таблице
size_t curit; // Смещение текущего времени в текущем файле
std::vector<MDateTime> times; // Моменты времени, для которых доступны данные
std::vector<size_t> tindexes; // Индексы в times, для которых запрашиваются данные
// Кэши прочитаннх данных
std::unique_ptr<std::map<MString, std::shared_ptr<Data2D>>> cache2D;
std::unique_ptr<std::map<MString, std::shared_ptr<Data3D>>> cache3D;
// Проверка корректности запроса переменной и времени, подготовка адаптера к чтению данных
Error ReadCheck(const MString& var, size_t it);
public:
RetVal<std::shared_ptr<Data2D>> Read2D(const MString& var, size_t it);
RetVal<std::shared_ptr<Data3D>> Read3D(const MString& var, size_t it);
};

344
src/Adapter.cpp
Unescape View File

@ -0,0 +1,344 @@
#define MICHLIB_NOSOURCE
#include "Adapter.h"
Error Adapter::FilterTimes(const CLArgs& args, michlib_internal::ParameterListEx& pars)
{
static const MString pref = "Adapter::FilterTimes";
if(args.contains("time") && (args.contains("timeb") || args.contains("timee"))) return {pref, "Time must be set via time parameter or timeb and timee parameter but not via both"};
if(args.contains("time") && args.contains("timefilt")) return {pref, "Time filter must be set via time parameter or timefilter parameter but not via both"};
MDateTime b, e;
MString tb = args.contains("timeb") ? args.at("timeb") : "BEG";
MString te = args.contains("timee") ? args.at("timee") : "END";
MString tf = args.contains("timefilt") ? args.at("timefilt") : "";
if(args.contains("time"))
{
MDateTime time;
MString t = args.at("time");
if(time.FromString(t) || t == "BEGIN" || t == "BEG" || t == "FIRST" || t == "END" || t == "LAST") // Time, not regex
tb = te = t;
else // Regular expression
tf = t;
}
if(tb == "BEGIN" || tb == "BEG" || tb == "FIRST")
b = times.front();
else if(tb == "END" || tb == "LAST")
b = times.back();
else
b.FromString(tb);
if(te == "BEGIN" || te == "BEG" || te == "FIRST")
e = times.front();
else if(te == "END" || te == "LAST")
e = times.back();
else
e.FromString(te);
if(e < b) return {pref, "Timeb must not be greater then timee"};
michlib::RegExpSimple reg(tf.Buf());
bool filt = tf.Exist();
if(filt && reg.Compile() != 0) return {pref, "Bad regular expression: " + tf};
tindexes.clear();
for(size_t it = 0; it < times.size(); it++)
if(times[it] >= b && times[it] <= e && (!filt || reg.Match(times[it].ToTString().Buf()))) tindexes.push_back(it);
if(tindexes.size() == 0) return {pref, "No times with given criteria are found"};
if(tindexes.size() == 1)
pars.AddParameter("time", times[tindexes[0]].ToTString());
else
{
pars.AddParameter("timebegin", times[tindexes.front()].ToTString());
pars.AddParameter("timeend", times[tindexes.back()].ToTString());
if(filt) pars.AddParameter("timefilter", tf);
}
return {};
}
RetVal<std::shared_ptr<Data2D>> Adapter::Def2DReader(const Adapter& ad, const void* vinfo, std::shared_ptr<Projection> proj, size_t it)
{
static const MString pref = "Adapter::Def2DReader";
auto v = michlib::pointer_cast<const struct VInfo*>(vinfo);
const struct ReadInfo* ri = v->rinfo.get();
const bool layered = ri->zdname.Exist() && ad.nc->HasDim(v->name, ri->zdname);
std::shared_ptr<Data2D> out(new Data2D(proj, {.unit = v->targetunit, .stname = v->stname, .lname = v->lname, .comment = v->comment, .fillval = v->fillval}));
auto trans = [scale = v->scale, offset = v->offset](auto raw) -> real { return raw * scale + offset; };
auto data = [pdata = out.get()](size_t ix, size_t iy) -> real& { return (*pdata)(ix, iy); };
const UtUnit fr(v->unit), to(v->targetunit);
const auto cnv = Converter(fr, to);
auto cnvtrans = [scale = v->scale, offset = v->offset, &cnv = std::as_const(cnv)](auto raw) -> real { return cnv(raw * scale + offset); };
const bool needsconvert = (v->unit.Exist() && v->targetunit.Exist() && fr && to && fr != to);
if(needsconvert && !cnv) return {pref, "Can't convert units from \"" + v->unit + "\" to \"" + v->targetunit + "\""};
MString xreq, yreq, zreq, treq;
yreq = ri->ydname + ":" + ri->yb + ":" + (ri->ye - ri->yb + 1);
zreq = layered ? (ri->zdname + ":" + ri->zb + ":1") : "";
treq = ri->tdname + ":" + it + ":1";
auto req = [&x = xreq, &y = yreq, &z = zreq, &t = treq]() -> MString { return x + ";" + y + ";" + (z.Exist() ? (z + ";") : "") + t; };
if(ri->xb < ri->xe)
{
xreq = ri->xdname + ":" + ri->xb + ":" + (ri->xe - ri->xb + 1);
auto ret = needsconvert ? ad.nc->Read(v->name, data, cnvtrans, req()) : ad.nc->Read(v->name, data, trans, req());
if(!ret) return ret.Add(pref, "Can't read variable " + v->name);
}
else
{
size_t nx = ad.nc->DimSize(v->name, ri->xdname);
{
xreq = ri->xdname + ":" + ri->xb + ":" + (nx - ri->xb + 1);
auto ret = needsconvert ? ad.nc->Read(v->name, data, cnvtrans, req()) : ad.nc->Read(v->name, data, trans, req());
if(!ret) return ret.Add(pref, "Can't read variable " + v->name);
}
{
size_t shift = nx - ri->xb + 1;
auto shifteddata = [pdata = out.get(), shift](size_t ix, size_t iy) -> real& { return (*pdata)(ix + shift, iy); };
xreq = ri->xdname + ":0:" + (ri->xe - 1);
auto ret = needsconvert ? ad.nc->Read(v->name, shifteddata, cnvtrans, req()) : ad.nc->Read(v->name, shifteddata, trans, req());
if(!ret) return ret.Add(pref, "Can't read variable " + v->name);
}
}
return out;
}
RetVal<std::shared_ptr<Data3D>> Adapter::Def3DReader(const Adapter& ad, const void* vinfo, std::shared_ptr<Projection> proj, std::shared_ptr<Vertical> vert, size_t it)
{
static const MString pref = "Adapter::Def3DReader";
auto v = michlib::pointer_cast<const struct VInfo*>(vinfo);
const struct ReadInfo* ri = v->rinfo.get();
const bool layered = ri->zdname.Exist() && ad.nc->HasDim(v->name, ri->zdname);
const bool invertz = ri->zb > ri->ze;
const bool onez = ri->zb == ri->ze;
std::shared_ptr<Data3D> out(new Data3D(proj, vert, {.unit = v->targetunit, .stname = v->stname, .lname = v->lname, .comment = v->comment, .fillval = v->fillval}));
auto trans = [scale = v->scale, offset = v->offset](auto raw) -> real { return raw * scale + offset; };
auto data = [pdata = out.get(), invertz](size_t ix, size_t iy, size_t iz) -> real& { return (*pdata)(ix, iy, invertz ? (pdata->Nz() - iz - 1) : iz); };
auto data2D = [pdata = out.get()](size_t ix, size_t iy) -> real& { return (*pdata)(ix, iy, 0); };
const UtUnit fr(v->unit), to(v->targetunit);
const auto cnv = Converter(fr, to);
auto cnvtrans = [scale = v->scale, offset = v->offset, &cnv = std::as_const(cnv)](auto raw) -> real { return cnv(raw * scale + offset); };
const bool needsconvert = (v->unit.Exist() && v->targetunit.Exist() && fr && to && fr != to);
if(needsconvert && !cnv) return {pref, "Can't convert units from \"" + v->unit + "\" to \"" + v->targetunit + "\""};
MString xreq, yreq, zreq, treq;
yreq = ri->ydname + ":" + ri->yb + ":" + (ri->ye - ri->yb + 1);
zreq = layered ? (ri->zdname + ":" + std::min(ri->zb, ri->ze) + ":" + (std::max(ri->zb, ri->ze) - std::min(ri->zb, ri->ze) + 1)) : "";
treq = ri->tdname + ":" + it + ":1";
auto req = [&x = xreq, &y = yreq, &z = zreq, &t = treq]() -> MString { return x + ";" + y + ";" + (z.Exist() ? (z + ";") : "") + t; };
if(ri->xb < ri->xe)
{
xreq = ri->xdname + ":" + ri->xb + ":" + (ri->xe - ri->xb + 1);
auto ret = needsconvert ? (onez ? ad.nc->Read(v->name, data2D, cnvtrans, req()) : ad.nc->Read(v->name, data, cnvtrans, req()))
: (onez ? ad.nc->Read(v->name, data2D, trans, req()) : ad.nc->Read(v->name, data, trans, req()));
if(!ret) return ret.Add(pref, "Can't read variable " + v->name);
}
else
{
size_t nx = ad.nc->DimSize(v->name, ri->xdname);
{
xreq = ri->xdname + ":" + ri->xb + ":" + (nx - ri->xb + 1);
auto ret = needsconvert ? ad.nc->Read(v->name, data, cnvtrans, req()) : ad.nc->Read(v->name, data, trans, req());
if(!ret) return ret.Add(pref, "Can't read variable " + v->name);
}
{
size_t shift = nx - ri->xb + 1;
auto shifteddata = [pdata = out.get(), shift, invertz](size_t ix, size_t iy, size_t iz) -> real& { return (*pdata)(ix + shift, iy, invertz ? (pdata->Nz() - iz - 1) : iz); };
auto shifteddata2D = [pdata = out.get(), shift](size_t ix, size_t iy) -> real& { return (*pdata)(ix + shift, iy, 0); };
xreq = ri->xdname + ":0:" + (ri->xe - 1);
auto ret = needsconvert ? (onez ? ad.nc->Read(v->name, shifteddata2D, cnvtrans, req()) : ad.nc->Read(v->name, shifteddata, cnvtrans, req()))
: (onez ? ad.nc->Read(v->name, shifteddata2D, trans, req()) : ad.nc->Read(v->name, shifteddata, trans, req()));
if(!ret) return ret.Add(pref, "Can't read variable " + v->name);
}
}
return out;
}
Error Adapter::ReadCheck(const MString& var, size_t it)
{
static const MString pref = "Adapter::ReadCheck";
// Check if time is correct
if(it > times.size()) return {pref, MString("Time index ") + it + " is greater then number of times " + times.size()};
// Check presence of variable
if(!vars.contains(var)) return {pref, "Variable " + var + " not found"};
// Find and check entry in timetable
size_t timerow = timetable.size();
for(size_t i = 0; i < timetable.size(); i++)
if(it >= timetable[i].beg && it <= timetable[i].end)
{
timerow = i;
break;
}
if(timerow == timetable.size()) return {pref, "Time table does'nt contains row for time " + times[it].ToTString() + "/" + it};
// Time index inside file
curit = it - timetable[timerow].beg;
// Reopen data file if needed
if(timetable[timerow].file != curfile)
{
nc.reset(new NCZarr);
curfile = timetable[timerow].file;
switch(method)
{
case(ZarrMethod::NC):
{
auto ret = nc->OpenNC(curfile);
if(!ret) return ret.Add(pref, "OpenNC failed for file " + curfile);
break;
}
case(ZarrMethod::ZARR):
{
auto args = curfile.Split(":");
if(args.size() < 2 || args.size() > 3) return {pref, "Can't open Zarr file " + curfile + ": incorrect url"};
auto ret = (args.size() == 2) ? nc->OpenZarr(args[0], args[1]) : nc->OpenZarr(args[0], args[1], args[2].ToBool());
if(!ret) return ret.Add(pref, "OpenZarr failed for file " + curfile);
break;
}
case(ZarrMethod::MNC):
{
auto ret = nc->OpenMultiNC(curfile.Split(":"));
if(!ret) return ret.Add(pref, "OpenMultiNC failed for file " + curfile);
break;
}
case(ZarrMethod::MZARR):
{
auto args = curfile.Split(":");
if(args.size() < 2 || args.size() > 3) return {pref, "Can't open Zarr file " + curfile + ": incorrect url"};
auto ret = (args.size() == 2) ? nc->OpenMultiZarr(args[0], args[1].Split(",")) : nc->OpenMultiZarr(args[0], args[1].Split(","), args[2].ToBool());
if(!ret) return ret.Add(pref, "OpenMultiZarr failed for file " + curfile);
break;
}
}
}
{
const auto& v = vars[var];
const auto* ri = v.info.rinfo.get();
const auto& name = v.info.name;
const bool layered = ri->zdname.Exist() && nc->HasDim(name, ri->zdname);
if(layered)
{
//if(ri->ze < ri->zb) return {pref, "Internal error: ze < zb for variable " + var + "(" + name + ")" + " (" + ri->ze + " < " + ri->zb + ")"};
const size_t nz = nc->DimSize(name, ri->zdname);
if(ri->zb > nz - 1) return {pref, "Internal error: zb > nz - 1 for variable " + var + "(" + name + ")" + " (" + ri->zb + " > " + nz + " - 1)"};
if(ri->ze > nz - 1) return {pref, "Internal error: ze > nz - 1 for variable " + var + "(" + name + ")" + " (" + ri->ze + " > " + nz + " - 1)"};
if(v.vert && (ri->zb < ri->ze ? ri->ze - ri->zb : ri->zb - ri->ze) + 1 != v.vert->Nz())
return {pref, "Internal error: number of requested layers does'nt correspond parameters of vertical column for variable " + var + "(" + name + ")" + " (zb = " + ri->zb +
", ze = " + ri->ze + ", Nz = " + v.vert->Nz()};
}
if(ri->ye < ri->yb) return {pref, "Internal error: ye < yb for variable " + var + "(" + name + ")" + " (" + ri->ye + " < " + ri->yb + ")"};
const size_t nx = nc->DimSize(name, ri->xdname);
const size_t ny = nc->DimSize(name, ri->ydname);
if(ri->ye > ny - 1) return {pref, "Internal error: ye > ny - 1 for variable " + var + "(" + name + ")" + " (" + ri->ye + " > " + ny + " - 1)"};
if(ri->ye - ri->yb + 1 != v.proj->Ny())
return {pref, "Internal error: number of requested y-planes does'nt correspond parameters of projection for variable " + var + "(" + name + ")" + " (yb = " + ri->yb +
", ye = " + ri->ye + ", Ny = " + v.proj->Ny()};
if(ri->xe > nx - 1) return {pref, "Internal error: xe > nx - 1 for variable " + var + "(" + name + ")" + " (" + ri->xe + " > " + nx + " - 1)"};
if(ri->xb > nx - 1) return {pref, "Internal error: xb > nx - 1 for variable " + var + "(" + name + ")" + " (" + ri->xb + " > " + nx + " - 1)"};
const size_t rnx = (ri->xb <= ri->xe) ? (ri->xe - ri->xb + 1) : (nx + ri->xe - ri->xb + 1);
if(rnx != v.proj->Nx())
return {pref, "Internal error: number of requested x-planes does'nt correspond parameters of projection for variable " + var + "(" + name + ")" + " (xb = " + ri->xb +
", xe = " + ri->xe + ", Nx = " + v.proj->Nx()};
}
return {};
}
RetVal<std::shared_ptr<Data2D>> Adapter::Read2D(const MString& var, size_t it)
{
static const MString pref = "Adapter::Read2D";
{
auto ret = ReadCheck(var, it);
if(!ret) return ret.Add(pref, "Check not passed");
}
// Clear cache
if(!cache2D) cache2D.reset(new decltype(cache2D)::element_type);
if(it != curtime) cache2D->clear();
curtime = it;
// Check cache
if(cache2D->contains(var)) return (*cache2D)[var];
const auto& v = vars[var];
if(!v.Read2D) return {pref, "No read function for the variable " + var};
auto ret = v.Read2D(*this, &v.info, v.proj, curit);
if(!ret) return ret.Add(pref, "Can't read variable " + var);
(*cache2D)[var] = ret.Value();
return (*cache2D)[var];
}
RetVal<std::shared_ptr<Data3D>> Adapter::Read3D(const MString& var, size_t it)
{
static const MString pref = "Adapter::Read3D";
{
auto ret = ReadCheck(var, it);
if(!ret) return ret.Add(pref, "Check not passed");
}
// Clear cache
if(!cache3D) cache3D.reset(new decltype(cache3D)::element_type);
if(it != curtime) cache3D->clear();
curtime = it;
// Check cache
if(cache3D->contains(var)) return (*cache3D)[var];
const auto& v = vars[var];
if(!v.vert) return {pref, "Variable " + var + " is 2D in the this dataset"};
if(!v.Read3D) return {pref, "No read function for the variable " + var};
auto ret = v.Read3D(*this, &v.info, v.proj, v.vert, curit);
if(!ret) return ret.Add(pref, "Can't read variable " + var);
(*cache3D)[var] = ret.Value();
return (*cache3D)[var];
}
Write Preview
Loading…
Cancel
Save