Browse Source

Added class FileInfoCache - PostgreSQL-based cache for the information about files

master
Michael Uleysky 3 months ago
parent
commit
19f63956a5
  1. 30
      include/cache.h
  2. 240
      src/cache.cpp

30
include/cache.h

@ -1,12 +1,16 @@
#pragma once
#include "GPL.h"
#include "mirrorfuncs.h"
#include <functional>
#include <libpq-fe.h>
#include <optional>
#include <sqlite3.h>
#include <time.h>
#include <variant>
using michlib::GPL;
using michlib::int1;
using michlib::int4;
using michlib::int_cast;
using michlib::MString;
using michlib::pointer_cast;
@ -415,3 +419,29 @@ inline GenericCache* CreateCache(const MString& cachedesc)
return nullptr;
}
class FileInfoCache: public PostgreSQLHelpers
{
public:
using DataType = std::optional<MString>;
using CallbackType = std::function<DataType(const MString&)>;
private:
static bool regdest;
CallbackType readfunc;
MString dir;
int4 dirid;
FileInfoCache() = delete;
CallbackType::result_type GetData(const MString& fname) const { return readfunc(dir + "/" + fname); }
void GetDirId();
public:
FileInfoCache(CallbackType&& readfunc_, const MString& dir_);
Error UpdateCache(bool force = false) const;
DataType GetInfo(const MString& name) const;
};

240
src/cache.cpp

@ -12,3 +12,243 @@ std::vector<PostgreSQLConnection::FuncType> PostgreSQLConnection::destructs = {}
bool SQLiteCache::regdest = false;
bool PostgreSQLCache::regdest = false;
bool FileInfoCache::regdest = false;
void FileInfoCache::GetDirId()
{
if(dirid != 0) return;
const char* params[] = {dir.Buf()};
int plens[] = {int_cast<int>(dir.Len())};
int pfor[] = {0};
PGresultRAII res = PQexecParams(conn, "SELECT id FROM dirs WHERE name=$1::text;", 1, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_TUPLES_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
return;
}
else if(PQntuples(res) == 0)
{
res = PQexecParams(conn,
"INSERT INTO dirs(name,id) VALUES ($1, (SELECT min(num.numid) FROM (SELECT generate_series(1, (SELECT COALESCE((SELECT max(id) FROM dirs), 1)) + "
"1, 1) AS numid) num LEFT JOIN dirs ON dirs.id=num.numid WHERE id IS NULL)) RETURNING id;",
1, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
}
if(PQntuples(res) == 0) return;
}
if(PQgetlength(res, 0, 0) == sizeof(dirid)) dirid = *pointer_cast<const decltype(dirid)*>(PQgetvalue(res, 0, 0));
michlib::message("Dirid: ", Invert(dirid));
}
FileInfoCache::FileInfoCache(FileInfoCache::CallbackType&& readfunc_, const MString& dir_): readfunc(std::move(readfunc_)), dir(dir_), dirid(0)
{
if(!conn) return;
if(!regdest)
{
// Create table
PGresultRAII res = PQexec(conn, "SET client_min_messages=WARNING;");
res = PQexec(conn, "BEGIN;"
"CREATE TABLE IF NOT EXISTS dirs(name TEXT PRIMARY KEY, id INTEGER UNIQUE NOT NULL CONSTRAINT id_is_positive CHECK(id>0));"
"CREATE TABLE IF NOT EXISTS files(name TEXT NOT NULL, size BIGINT NOT NULL CONSTRAINT size_is_positive CHECK(size>0), modtime TIMESTAMP(0) NOT NULL, "
"dirid INTEGER REFERENCES dirs(id) ON DELETE CASCADE, lastaccess TIMESTAMP(0) NOT NULL, data BYTEA NOT NULL, PRIMARY KEY(name,dirid));"
"COMMIT;");
if(PQresultStatus(res) != PGRES_COMMAND_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
}
res = PQexec(conn, "SET client_min_messages=NOTICE;");
conn.AddDestructor(
[](PostgreSQLConnection::DBType conn)
{
PGresultRAII res = PQexec(conn, "BEGIN;"
"DELETE FROM files WHERE lastaccess+'100 days'::interval<localtimestamp;"
"DELETE FROM dirs WHERE id NOT IN (SELECT dirid FROM files);"
"COMMIT;");
});
regdest = true;
}
GetDirId();
//UpdateCache();
}
Error FileInfoCache::UpdateCache(bool force) const
{
const static MString pref = "FileInfoCache::UpdateCache";
DIRRAII dhandle;
dhandle.reset(opendir(dir.Buf()));
if(!dhandle) return {pref, "Can't open directory " + dir};
int dfd = dirfd(dhandle);
errno = 0;
struct dirent* dent = readdir(dhandle);
if(errno != 0) return {pref, "Can't read directory " + dir};
struct stat st;
do {
if(dent->d_name[0] != '.')
{
int ret = fstatat(dfd, dent->d_name, &st, 0);
if(ret != 0) return {pref, "Can't stat " + dir + "/" + dent->d_name};
if(S_ISREG(st.st_mode)) // Regular file
{
const char* params[] = {dent->d_name, pointer_cast<const char*>(&dirid)};
int plens[] = {int_cast<int>(strlen(dent->d_name)), sizeof(dirid)};
int pfor[] = {0, 1};
bool querysucc = true;
time_t modtime;
size_t size;
PGresultRAII res = PQexecParams(conn, "SELECT size,modtime FROM files WHERE name=$1::text AND dirid=$2::integer;", 2, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
querysucc = false;
}
else if(PQntuples(res) == 0 || PQntuples(res) > 1)
querysucc = false;
if(querysucc)
{
size = *pointer_cast<const decltype(size)*>(PQgetvalue(res, 0, 0));
modtime = raw2epoch(*pointer_cast<const time_t*>(PQgetvalue(res, 0, 1)));
}
else
{
size = int_cast<size_t>(st.st_size);
modtime = st.st_mtim.tv_sec;
}
if(!querysucc || force || size != int_cast<size_t>(st.st_size) || modtime != st.st_mtim.tv_sec)
{
auto ret = GetData(dent->d_name);
// Remove entry
if(!ret && querysucc)
{
PGresultRAII dres = PQexecParams(conn, "DELETE FROM files WHERE name=$1::text AND dirid=$2::integer;", 2, nullptr, params, plens, pfor, 1);
if(PQresultStatus(dres) != PGRES_COMMAND_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(dres)));
michlib::errmessage(PQerrorMessage(conn));
}
}
else // Update or insert
{
auto sizei = Invert(size);
auto modtimei = epoch2raw(modtime);
const char* params[] = {dent->d_name, pointer_cast<const char*>(&sizei), pointer_cast<const char*>(&modtimei), pointer_cast<const char*>(&dirid), ret.value().Buf()};
int plens[] = {int_cast<int>(strlen(dent->d_name)), sizeof(sizei), sizeof(modtimei), sizeof(dirid), int_cast<int>(ret.value().Len())};
int pfor[] = {0, 1, 1, 1, 1};
PGresultRAII res = PQexecParams(conn,
"INSERT INTO files (name,size,modtime,dirid,lastaccess,data) VALUES($1::text, $2::bigint, $3::timestamp, $4::integer, localtimestamp, $5) "
"ON CONFLICT ON CONSTRAINT files_pkey DO UPDATE SET "
"size=EXCLUDED.size, modtime=EXCLUDED.modtime, lastaccess=EXCLUDED.lastaccess, data=EXCLUDED.data;",
5, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
}
} // Insert or update branch
} // Need data update
} // Regular file
} // if(dent->d_name[0] != '.')
dent = readdir(dhandle);
} while(dent != nullptr || errno != 0);
return Error();
}
FileInfoCache::DataType FileInfoCache::GetInfo(const MString& name) const
{
if(!*this) return GetData(name);
bool querysucc = true;
MString data;
time_t modtime;
size_t size;
{
const char* params[] = {name.Buf(), pointer_cast<const char*>(&dirid)};
int plens[] = {int_cast<int>(name.Len()), sizeof(dirid)};
int pfor[] = {0, 1};
PGresultRAII res =
PQexecParams(conn, "UPDATE files SET lastaccess=localtimestamp WHERE name=$1::text AND dirid=$2::integer RETURNING data,size,modtime;", 2, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
querysucc = false;
}
if(PQntuples(res) == 0 || PQntuples(res) > 1)
{
michlib::errmessage("Data for file ", dir + "/" + name, (PQntuples(res) == 0 ? " not found " : " duplicated "), "in cache");
querysucc = false;
}
if(querysucc)
{
data = MString(PQgetvalue(res, 0, 0), PQgetlength(res, 0, 0));
size = *pointer_cast<const decltype(size)*>(PQgetvalue(res, 0, 1));
modtime = raw2epoch(*pointer_cast<const time_t*>(PQgetvalue(res, 0, 2)));
}
}
{
struct stat st;
int ret = stat((dir + "/" + name).Buf(), &st);
if(ret != 0) return DataType();
if(querysucc && st.st_mtim.tv_sec == modtime && size == int_cast<size_t>(st.st_size)) return data;
modtime = st.st_mtim.tv_sec;
size = st.st_size;
}
auto ret = GetData(name);
if(ret)
{
auto sizei = Invert(size);
auto modtimei = epoch2raw(modtime);
const char* params[] = {name.Buf(), pointer_cast<const char*>(&sizei), pointer_cast<const char*>(&modtimei), pointer_cast<const char*>(&dirid), ret.value().Buf()};
int plens[] = {int_cast<int>(name.Len()), sizeof(sizei), sizeof(modtimei), sizeof(dirid), int_cast<int>(ret.value().Len())};
int pfor[] = {0, 1, 1, 1, 1};
PGresultRAII res = PQexecParams(conn,
"INSERT INTO files (name,size,modtime,dirid,lastaccess,data) VALUES($1::text, $2::bigint, $3::timestamp, $4::integer, localtimestamp, $5) "
"ON CONFLICT ON CONSTRAINT files_pkey DO UPDATE SET "
"size=EXCLUDED.size, modtime=EXCLUDED.modtime, lastaccess=EXCLUDED.lastaccess, data=EXCLUDED.data;",
5, nullptr, params, plens, pfor, 1);
if(PQresultStatus(res) != PGRES_COMMAND_OK)
{
michlib::errmessage(PQresStatus(PQresultStatus(res)));
michlib::errmessage(PQerrorMessage(conn));
}
}
return ret;
}

Loading…
Cancel
Save