michael_ul
/
odm
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Added class FileInfoCache - PostgreSQL-based cache for the information about files

master
Michael Uleysky 2 months ago
parent
29a7a98548
commit
19f63956a5
2 changed files with 270 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 30
      include/cache.h
  2. 240
      src/cache.cpp

30
include/cache.h
Unescape View File

@ -1,12 +1,16 @@
#pragma once
#include "GPL.h"
#include "mirrorfuncs.h"
#include <functional>
#include <libpq-fe.h>
#include <optional>
#include <sqlite3.h>
#include <time.h>
#include <variant>
using michlib::GPL;
using michlib::int1;
using michlib::int4;
using michlib::int_cast;
using michlib::MString;
using michlib::pointer_cast;
@ -415,3 +419,29 @@ inline GenericCache* CreateCache(const MString& cachedesc)
return nullptr;
}
class FileInfoCache: public PostgreSQLHelpers
{
public:
using DataType = std::optional<MString>;
using CallbackType = std::function<DataType(const MString&)>;
private:
static bool regdest;
CallbackType readfunc;
MString dir;
int4 dirid;
FileInfoCache() = delete;
CallbackType::result_type GetData(const MString& fname) const { return readfunc(dir + "/" + fname); }
void GetDirId();
public:
FileInfoCache(CallbackType&& readfunc_, const MString& dir_);
Error UpdateCache(bool force = false) const;
DataType GetInfo(const MString& name) const;
};

240
src/cache.cpp
Unescape View File

@ -12,3 +12,243 @@ std::vector<PostgreSQLConnection::FuncType> PostgreSQLConnection::destructs = {}
bool SQLiteCache::regdest = false;
bool PostgreSQLCache::regdest = false;
bool FileInfoCache::regdest = false;
void FileInfoCache::GetDirId()
{
if(dirid != 0) return;
const char* params[] = {dir.Buf()};
int plens[] = {int_cast<int>(dir.Len())};
int pfor[] = {0};
PGresultRAII res = PQexecParams(conn, "SELECT id FROM dirs WHERE name=$1::text;", 1, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_TUPLES_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
return;
}
else if(PQntuples(res) == 0)
{
res = PQexecParams(conn,
"INSERT INTO dirs(name,id) VALUES ($1, (SELECT min(num.numid) FROM (SELECT generate_series(1, (SELECT COALESCE((SELECT max(id) FROM dirs), 1)) + "
"1, 1) AS numid) num LEFT JOIN dirs ON dirs.id=num.numid WHERE id IS NULL)) RETURNING id;",
1, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
}
if(PQntuples(res) == 0) return;
}
if(PQgetlength(res, 0, 0) == sizeof(dirid)) dirid = *pointer_cast<const decltype(dirid)*>(PQgetvalue(res, 0, 0));
michlib::message("Dirid: ", Invert(dirid));
}
FileInfoCache::FileInfoCache(FileInfoCache::CallbackType&& readfunc_, const MString& dir_): readfunc(std::move(readfunc_)), dir(dir_), dirid(0)
{
if(!conn) return;
if(!regdest)
{
// Create table
PGresultRAII res = PQexec(conn, "SET client_min_messages=WARNING;");
res = PQexec(conn, "BEGIN;"
"CREATE TABLE IF NOT EXISTS dirs(name TEXT PRIMARY KEY, id INTEGER UNIQUE NOT NULL CONSTRAINT id_is_positive CHECK(id>0));"
"CREATE TABLE IF NOT EXISTS files(name TEXT NOT NULL, size BIGINT NOT NULL CONSTRAINT size_is_positive CHECK(size>0), modtime TIMESTAMP(0) NOT NULL, "
"dirid INTEGER REFERENCES dirs(id) ON DELETE CASCADE, lastaccess TIMESTAMP(0) NOT NULL, data BYTEA NOT NULL, PRIMARY KEY(name,dirid));"
"COMMIT;");
if(PQresultStatus(res) != PGRES_COMMAND_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
}
res = PQexec(conn, "SET client_min_messages=NOTICE;");
conn.AddDestructor(
[](PostgreSQLConnection::DBType conn)
{
PGresultRAII res = PQexec(conn, "BEGIN;"
"DELETE FROM files WHERE lastaccess+'100 days'::interval<localtimestamp;"
"DELETE FROM dirs WHERE id NOT IN (SELECT dirid FROM files);"
"COMMIT;");
});
regdest = true;
}
GetDirId();
//UpdateCache();
}
Error FileInfoCache::UpdateCache(bool force) const
{
const static MString pref = "FileInfoCache::UpdateCache";
DIRRAII dhandle;
dhandle.reset(opendir(dir.Buf()));
if(!dhandle) return {pref, "Can't open directory " + dir};
int dfd = dirfd(dhandle);
errno = 0;
struct dirent* dent = readdir(dhandle);
if(errno != 0) return {pref, "Can't read directory " + dir};
struct stat st;
do {
if(dent->d_name[0] != '.')
{
int ret = fstatat(dfd, dent->d_name, &st, 0);
if(ret != 0) return {pref, "Can't stat " + dir + "/" + dent->d_name};
if(S_ISREG(st.st_mode)) // Regular file
{
const char* params[] = {dent->d_name, pointer_cast<const char*>(&dirid)};
int plens[] = {int_cast<int>(strlen(dent->d_name)), sizeof(dirid)};
int pfor[] = {0, 1};
bool querysucc = true;
time_t modtime;
size_t size;
PGresultRAII res = PQexecParams(conn, "SELECT size,modtime FROM files WHERE name=$1::text AND dirid=$2::integer;", 2, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
querysucc = false;
}
else if(PQntuples(res) == 0 || PQntuples(res) > 1)
querysucc = false;
if(querysucc)
{
size = *pointer_cast<const decltype(size)*>(PQgetvalue(res, 0, 0));
modtime = raw2epoch(*pointer_cast<const time_t*>(PQgetvalue(res, 0, 1)));
}
else
{
size = int_cast<size_t>(st.st_size);
modtime = st.st_mtim.tv_sec;
}
if(!querysucc || force || size != int_cast<size_t>(st.st_size) || modtime != st.st_mtim.tv_sec)
{
auto ret = GetData(dent->d_name);
// Remove entry
if(!ret && querysucc)
{
PGresultRAII dres = PQexecParams(conn, "DELETE FROM files WHERE name=$1::text AND dirid=$2::integer;", 2, nullptr, params, plens, pfor, 1);
if(PQresultStatus(dres) != PGRES_COMMAND_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(dres)));
michlib::errmessage(PQerrorMessage(conn));
}
}
else // Update or insert
{
auto sizei = Invert(size);
auto modtimei = epoch2raw(modtime);
const char* params[] = {dent->d_name, pointer_cast<const char*>(&sizei), pointer_cast<const char*>(&modtimei), pointer_cast<const char*>(&dirid), ret.value().Buf()};
int plens[] = {int_cast<int>(strlen(dent->d_name)), sizeof(sizei), sizeof(modtimei), sizeof(dirid), int_cast<int>(ret.value().Len())};
int pfor[] = {0, 1, 1, 1, 1};
PGresultRAII res = PQexecParams(conn,
"INSERT INTO files (name,size,modtime,dirid,lastaccess,data) VALUES($1::text, $2::bigint, $3::timestamp, $4::integer, localtimestamp, $5) "
"ON CONFLICT ON CONSTRAINT files_pkey DO UPDATE SET "
"size=EXCLUDED.size, modtime=EXCLUDED.modtime, lastaccess=EXCLUDED.lastaccess, data=EXCLUDED.data;",
5, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
}
} // Insert or update branch
} // Need data update
} // Regular file
} // if(dent->d_name[0] != '.')
dent = readdir(dhandle);
} while(dent != nullptr || errno != 0);
return Error();
}
FileInfoCache::DataType FileInfoCache::GetInfo(const MString& name) const
{
if(!*this) return GetData(name);
bool querysucc = true;
MString data;
time_t modtime;
size_t size;
{
const char* params[] = {name.Buf(), pointer_cast<const char*>(&dirid)};
int plens[] = {int_cast<int>(name.Len()), sizeof(dirid)};
int pfor[] = {0, 1};
PGresultRAII res =
PQexecParams(conn, "UPDATE files SET lastaccess=localtimestamp WHERE name=$1::text AND dirid=$2::integer RETURNING data,size,modtime;", 2, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
querysucc = false;
}
if(PQntuples(res) == 0 || PQntuples(res) > 1)
{
michlib::errmessage("Data for file ", dir + "/" + name, (PQntuples(res) == 0 ? " not found " : " duplicated "), "in cache");
querysucc = false;
}
if(querysucc)
{
data = MString(PQgetvalue(res, 0, 0), PQgetlength(res, 0, 0));
size = *pointer_cast<const decltype(size)*>(PQgetvalue(res, 0, 1));
modtime = raw2epoch(*pointer_cast<const time_t*>(PQgetvalue(res, 0, 2)));
}
}
{
struct stat st;
int ret = stat((dir + "/" + name).Buf(), &st);
if(ret != 0) return DataType();
if(querysucc && st.st_mtim.tv_sec == modtime && size == int_cast<size_t>(st.st_size)) return data;
modtime = st.st_mtim.tv_sec;
size = st.st_size;
}
auto ret = GetData(name);
if(ret)
{
auto sizei = Invert(size);
auto modtimei = epoch2raw(modtime);
const char* params[] = {name.Buf(), pointer_cast<const char*>(&sizei), pointer_cast<const char*>(&modtimei), pointer_cast<const char*>(&dirid), ret.value().Buf()};
int plens[] = {int_cast<int>(name.Len()), sizeof(sizei), sizeof(modtimei), sizeof(dirid), int_cast<int>(ret.value().Len())};
int pfor[] = {0, 1, 1, 1, 1};
PGresultRAII res = PQexecParams(conn,
"INSERT INTO files (name,size,modtime,dirid,lastaccess,data) VALUES($1::text, $2::bigint, $3::timestamp, $4::integer, localtimestamp, $5) "
"ON CONFLICT ON CONSTRAINT files_pkey DO UPDATE SET "
"size=EXCLUDED.size, modtime=EXCLUDED.modtime, lastaccess=EXCLUDED.lastaccess, data=EXCLUDED.data;",
5, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
}
}
return ret;
}

Write Preview
Loading…
Cancel
Save