Compare commits
1 Commits
| Author | SHA1 | Date |
|---|---|---|
 Michael Uleysky
|
aaafdf28e7 | 5 months ago |
16 changed files with 222 additions and 627 deletions
@ -1,254 +0,0 @@
|
||||
#define MICHLIB_NOSOURCE |
||||
#include "cache.h" |
||||
|
||||
SQLiteConnection::DBType SQLiteConnection::db = nullptr; |
||||
size_t SQLiteConnection::count = 0; |
||||
std::vector<SQLiteConnection::FuncType> SQLiteConnection::destructs = {}; |
||||
|
||||
PostgreSQLConnection::DBType PostgreSQLConnection::conn = nullptr; |
||||
size_t PostgreSQLConnection::count = 0; |
||||
std::vector<PostgreSQLConnection::FuncType> PostgreSQLConnection::destructs = {}; |
||||
|
||||
bool SQLiteCache::regdest = false; |
||||
|
||||
bool PostgreSQLCache::regdest = false; |
||||
|
||||
bool FileInfoCache::regdest = false; |
||||
|
||||
void FileInfoCache::GetDirId() |
||||
{ |
||||
if(dirid != 0) return; |
||||
const char* params[] = {dir.Buf()}; |
||||
int plens[] = {int_cast<int>(dir.Len())}; |
||||
int pfor[] = {0}; |
||||
|
||||
PGresultRAII res = PQexecParams(conn, "SELECT id FROM dirs WHERE name=$1::text;", 1, nullptr, params, plens, pfor, 1); |
||||
if(PQresultStatus(res) != PGRES_TUPLES_OK) |
||||
{ |
||||
michlib::errmessage(PQresStatus(PQresultStatus(res))); |
||||
michlib::errmessage(PQerrorMessage(conn)); |
||||
return; |
||||
} |
||||
else if(PQntuples(res) == 0) |
||||
{ |
||||
res = PQexecParams(conn, |
||||
"INSERT INTO dirs(name,id) VALUES ($1, (SELECT min(num.numid) FROM (SELECT generate_series(1, (SELECT COALESCE((SELECT max(id) FROM dirs), 1)) + " |
||||
"1, 1) AS numid) num LEFT JOIN dirs ON dirs.id=num.numid WHERE id IS NULL)) RETURNING id;", |
||||
1, nullptr, params, plens, pfor, 1); |
||||
|
||||
if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK) |
||||
{ |
||||
michlib::errmessage(PQresStatus(PQresultStatus(res))); |
||||
michlib::errmessage(PQerrorMessage(conn)); |
||||
} |
||||
if(PQntuples(res) == 0) return; |
||||
} |
||||
|
||||
if(PQgetlength(res, 0, 0) == sizeof(dirid)) dirid = *pointer_cast<const decltype(dirid)*>(PQgetvalue(res, 0, 0)); |
||||
michlib::message("Dirid: ", Invert(dirid)); |
||||
} |
||||
|
||||
FileInfoCache::FileInfoCache(FileInfoCache::CallbackType&& readfunc_, const MString& dir_): readfunc(std::move(readfunc_)), dir(dir_), dirid(0) |
||||
{ |
||||
if(!conn) return; |
||||
|
||||
if(!regdest) |
||||
{ |
||||
// Create table
|
||||
PGresultRAII res = PQexec(conn, "SET client_min_messages=WARNING;"); |
||||
|
||||
res = PQexec(conn, "BEGIN;" |
||||
"CREATE TABLE IF NOT EXISTS dirs(name TEXT PRIMARY KEY, id INTEGER UNIQUE NOT NULL CONSTRAINT id_is_positive CHECK(id>0));" |
||||
"CREATE TABLE IF NOT EXISTS files(name TEXT NOT NULL, size BIGINT NOT NULL CONSTRAINT size_is_positive CHECK(size>0), modtime TIMESTAMP(0) NOT NULL, " |
||||
"dirid INTEGER REFERENCES dirs(id) ON DELETE CASCADE, lastaccess TIMESTAMP(0) NOT NULL, data BYTEA NOT NULL, PRIMARY KEY(name,dirid));" |
||||
"COMMIT;"); |
||||
if(PQresultStatus(res) != PGRES_COMMAND_OK) |
||||
{ |
||||
michlib::errmessage(PQresStatus(PQresultStatus(res))); |
||||
michlib::errmessage(PQerrorMessage(conn)); |
||||
} |
||||
|
||||
res = PQexec(conn, "SET client_min_messages=NOTICE;"); |
||||
|
||||
conn.AddDestructor( |
||||
[](PostgreSQLConnection::DBType conn) |
||||
{ |
||||
PGresultRAII res = PQexec(conn, "BEGIN;" |
||||
"DELETE FROM files WHERE lastaccess+'100 days'::interval<localtimestamp;" |
||||
"DELETE FROM dirs WHERE id NOT IN (SELECT dirid FROM files);" |
||||
"COMMIT;"); |
||||
}); |
||||
|
||||
regdest = true; |
||||
} |
||||
|
||||
GetDirId(); |
||||
//UpdateCache();
|
||||
} |
||||
|
||||
Error FileInfoCache::UpdateCache(bool force) const |
||||
{ |
||||
const static MString pref = "FileInfoCache::UpdateCache"; |
||||
|
||||
DIRRAII dhandle; |
||||
|
||||
dhandle.reset(opendir(dir.Buf())); |
||||
|
||||
if(!dhandle) return {pref, "Can't open directory " + dir}; |
||||
|
||||
int dfd = dirfd(dhandle); |
||||
errno = 0; |
||||
struct dirent* dent = readdir(dhandle); |
||||
if(errno != 0) return {pref, "Can't read directory " + dir}; |
||||
struct stat st; |
||||
|
||||
do { |
||||
if(dent->d_name[0] != '.') |
||||
{ |
||||
int ret = fstatat(dfd, dent->d_name, &st, 0); |
||||
if(ret != 0) return {pref, "Can't stat " + dir + "/" + dent->d_name}; |
||||
if(S_ISREG(st.st_mode)) // Regular file
|
||||
{ |
||||
const char* params[] = {dent->d_name, pointer_cast<const char*>(&dirid)}; |
||||
int plens[] = {int_cast<int>(strlen(dent->d_name)), sizeof(dirid)}; |
||||
int pfor[] = {0, 1}; |
||||
bool querysucc = true; |
||||
time_t modtime; |
||||
size_t size; |
||||
|
||||
PGresultRAII res = PQexecParams(conn, "SELECT size,modtime FROM files WHERE name=$1::text AND dirid=$2::integer;", 2, nullptr, params, plens, pfor, 1); |
||||
|
||||
if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK) |
||||
{ |
||||
michlib::errmessage(PQresStatus(PQresultStatus(res))); |
||||
michlib::errmessage(PQerrorMessage(conn)); |
||||
querysucc = false; |
||||
} |
||||
else if(PQntuples(res) == 0 || PQntuples(res) > 1) |
||||
querysucc = false; |
||||
|
||||
if(querysucc) |
||||
{ |
||||
size = *pointer_cast<const decltype(size)*>(PQgetvalue(res, 0, 0)); |
||||
modtime = raw2epoch(*pointer_cast<const time_t*>(PQgetvalue(res, 0, 1))); |
||||
} |
||||
else |
||||
{ |
||||
size = int_cast<size_t>(st.st_size); |
||||
modtime = st.st_mtim.tv_sec; |
||||
} |
||||
|
||||
if(!querysucc || force || size != int_cast<size_t>(st.st_size) || modtime != st.st_mtim.tv_sec) |
||||
{ |
||||
auto ret = GetData(dent->d_name); |
||||
// Remove entry
|
||||
if(!ret && querysucc) |
||||
{ |
||||
PGresultRAII dres = PQexecParams(conn, "DELETE FROM files WHERE name=$1::text AND dirid=$2::integer;", 2, nullptr, params, plens, pfor, 1); |
||||
if(PQresultStatus(dres) != PGRES_COMMAND_OK) |
||||
{ |
||||
michlib::errmessage(PQresStatus(PQresultStatus(dres))); |
||||
michlib::errmessage(PQerrorMessage(conn)); |
||||
} |
||||
} |
||||
else // Update or insert
|
||||
{ |
||||
auto sizei = Invert(size); |
||||
auto modtimei = epoch2raw(modtime); |
||||
|
||||
const char* params[] = {dent->d_name, pointer_cast<const char*>(&sizei), pointer_cast<const char*>(&modtimei), pointer_cast<const char*>(&dirid), ret.value().Buf()}; |
||||
int plens[] = {int_cast<int>(strlen(dent->d_name)), sizeof(sizei), sizeof(modtimei), sizeof(dirid), int_cast<int>(ret.value().Len())}; |
||||
int pfor[] = {0, 1, 1, 1, 1}; |
||||
|
||||
PGresultRAII res = PQexecParams(conn, |
||||
"INSERT INTO files (name,size,modtime,dirid,lastaccess,data) VALUES($1::text, $2::bigint, $3::timestamp, $4::integer, localtimestamp, $5) " |
||||
"ON CONFLICT ON CONSTRAINT files_pkey DO UPDATE SET " |
||||
"size=EXCLUDED.size, modtime=EXCLUDED.modtime, lastaccess=EXCLUDED.lastaccess, data=EXCLUDED.data;", |
||||
5, nullptr, params, plens, pfor, 1); |
||||
if(PQresultStatus(res) != PGRES_COMMAND_OK) |
||||
{ |
||||
michlib::errmessage(PQresStatus(PQresultStatus(res))); |
||||
michlib::errmessage(PQerrorMessage(conn)); |
||||
} |
||||
} // Insert or update branch
|
||||
} // Need data update
|
||||
} // Regular file
|
||||
} // if(dent->d_name[0] != '.')
|
||||
dent = readdir(dhandle); |
||||
} while(dent != nullptr || errno != 0); |
||||
|
||||
return Error(); |
||||
} |
||||
|
||||
FileInfoCache::DataType FileInfoCache::GetInfo(const MString& name) const |
||||
{ |
||||
if(!*this) return GetData(name); |
||||
|
||||
bool querysucc = true; |
||||
MString data; |
||||
time_t modtime; |
||||
size_t size; |
||||
|
||||
{ |
||||
const char* params[] = {name.Buf(), pointer_cast<const char*>(&dirid)}; |
||||
int plens[] = {int_cast<int>(name.Len()), sizeof(dirid)}; |
||||
int pfor[] = {0, 1}; |
||||
|
||||
PGresultRAII res = |
||||
PQexecParams(conn, "UPDATE files SET lastaccess=localtimestamp WHERE name=$1::text AND dirid=$2::integer RETURNING data,size,modtime;", 2, nullptr, params, plens, pfor, 1); |
||||
if(PQresultStatus(res) != PGRES_COMMAND_OK && PQresultStatus(res) != PGRES_TUPLES_OK) |
||||
{ |
||||
michlib::errmessage(PQresStatus(PQresultStatus(res))); |
||||
michlib::errmessage(PQerrorMessage(conn)); |
||||
querysucc = false; |
||||
} |
||||
|
||||
if(PQntuples(res) == 0 || PQntuples(res) > 1) |
||||
{ |
||||
michlib::errmessage("Data for file ", dir + "/" + name, (PQntuples(res) == 0 ? " not found " : " duplicated "), "in cache"); |
||||
querysucc = false; |
||||
} |
||||
|
||||
if(querysucc) |
||||
{ |
||||
data = MString(PQgetvalue(res, 0, 0), PQgetlength(res, 0, 0)); |
||||
size = *pointer_cast<const decltype(size)*>(PQgetvalue(res, 0, 1)); |
||||
modtime = raw2epoch(*pointer_cast<const time_t*>(PQgetvalue(res, 0, 2))); |
||||
} |
||||
} |
||||
|
||||
{ |
||||
struct stat st; |
||||
|
||||
int ret = stat((dir + "/" + name).Buf(), &st); |
||||
if(ret != 0) return DataType(); |
||||
|
||||
if(querysucc && st.st_mtim.tv_sec == modtime && size == int_cast<size_t>(st.st_size)) return data; |
||||
modtime = st.st_mtim.tv_sec; |
||||
size = st.st_size; |
||||
} |
||||
|
||||
auto ret = GetData(name); |
||||
if(ret) |
||||
{ |
||||
auto sizei = Invert(size); |
||||
auto modtimei = epoch2raw(modtime); |
||||
|
||||
const char* params[] = {name.Buf(), pointer_cast<const char*>(&sizei), pointer_cast<const char*>(&modtimei), pointer_cast<const char*>(&dirid), ret.value().Buf()}; |
||||
int plens[] = {int_cast<int>(name.Len()), sizeof(sizei), sizeof(modtimei), sizeof(dirid), int_cast<int>(ret.value().Len())}; |
||||
int pfor[] = {0, 1, 1, 1, 1}; |
||||
|
||||
PGresultRAII res = PQexecParams(conn, |
||||
"INSERT INTO files (name,size,modtime,dirid,lastaccess,data) VALUES($1::text, $2::bigint, $3::timestamp, $4::integer, localtimestamp, $5) " |
||||
"ON CONFLICT ON CONSTRAINT files_pkey DO UPDATE SET " |
||||
"size=EXCLUDED.size, modtime=EXCLUDED.modtime, lastaccess=EXCLUDED.lastaccess, data=EXCLUDED.data;", |
||||
5, nullptr, params, plens, pfor, 1); |
||||
if(PQresultStatus(res) != PGRES_COMMAND_OK) |
||||
{ |
||||
michlib::errmessage(PQresStatus(PQresultStatus(res))); |
||||
michlib::errmessage(PQerrorMessage(conn)); |
||||
} |
||||
} |
||||
|
||||
return ret; |
||||
} |
||||
Loading…
Reference in new issue