ad_user
/
libsimdb
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

* global rename : imdb -> simdb

master
Alex 'AdUser' Z 8 years ago
parent
0fdd03cab6
commit
934e9d546e
10 changed files with 221 additions and 223 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 26
      src/CMakeLists.txt
  2. 2
      src/common.h
  3. 156
      src/database.c
  4. 74
      src/database.h
  5. 4
      src/sample.c
  6. 3
      src/sample.h
  7. 54
      src/simdb-1to2.c
  8. 50
      src/simdb-query.c
  9. 24
      src/simdb-write.c
  10. 39
      tests/database.c

26
src/CMakeLists.txt
Unescape View File

@ -1,20 +1,20 @@
set(LIB_SOURCES "database.c" "bitmap.c") set(LIB_SOURCES "database.c" "bitmap.c")
add_library("imgdup" SHARED ${LIB_SOURCES}) add_library("simdb" SHARED ${LIB_SOURCES})
set_property(TARGET "imgdup" PROPERTY SONAME ${VERSION}) set_property(TARGET "simdb" PROPERTY SONAME ${VERSION})
install(TARGETS "imgdup" LIBRARY DESTINATION "lib") install(TARGETS "simdb" LIBRARY DESTINATION "lib")
add_executable("imdb-1to2" "imdb-1to2.c") add_executable("simdb-1to2" "simdb-1to2.c")
install(TARGETS "imdb-1to2" RUNTIME DESTINATION "bin") install(TARGETS "simdb-1to2" RUNTIME DESTINATION "bin")
add_executable("imdb-query" "imdb-query.c") add_executable("simdb-query" "simdb-query.c")
target_link_libraries("imdb-query" "imgdup") target_link_libraries("simdb-query" "simdb")
install(TARGETS "imdb-query" RUNTIME DESTINATION "bin") install(TARGETS "simdb-query" RUNTIME DESTINATION "bin")
find_package(ImageMagick COMPONENTS MagickCore MagickWand) find_package(ImageMagick COMPONENTS MagickCore MagickWand)
add_executable("imdb-write" "imdb-write.c" "sample.c") add_executable("simdb-write" "simdb-write.c" "sample.c")
target_link_libraries("imdb-write" "imgdup") target_link_libraries("simdb-write" "simdb")
target_link_libraries("imdb-write" ${ImageMagick_MagickCore_LIBRARY}) target_link_libraries("simdb-write" ${ImageMagick_MagickCore_LIBRARY})
target_link_libraries("imdb-write" ${ImageMagick_MagickWand_LIBRARY}) target_link_libraries("simdb-write" ${ImageMagick_MagickWand_LIBRARY})
install(TARGETS "imdb-write" RUNTIME DESTINATION "bin") install(TARGETS "simdb-write" RUNTIME DESTINATION "bin")

2
src/common.h
Unescape View File

@ -27,5 +27,5 @@
extern int image_add(uint64_t, char *path); extern int image_add(uint64_t, char *path);
extern int image_del(uint64_t); extern int image_del(uint64_t);
extern int image_exists(uint64_t); extern int image_exists(uint64_t);
extern int image_search(uint64_t, imdb_rec_t **data, size_t limit); extern int image_search(uint64_t, simdb_rec_t **data, size_t limit);
*/ */

156
src/database.c
Unescape View File

@ -22,32 +22,32 @@
errno = 0; \ errno = 0; \
memset((buf), 0x0, (len)); \ memset((buf), 0x0, (len)); \
bytes = pread((db)->fd, (buf), (len), (off)); \ bytes = pread((db)->fd, (buf), (len), (off)); \
if (errno) { return IMDB_ERR_SYSTEM; } if (errno) { return SIMDB_ERR_SYSTEM; }
#define DB_WRITE(db, buf, len, off) \ #define DB_WRITE(db, buf, len, off) \
errno = 0; \ errno = 0; \
bytes = pwrite((db)->fd, (buf), (len), (off)); \ bytes = pwrite((db)->fd, (buf), (len), (off)); \
if (errno) { return IMDB_ERR_SYSTEM; } if (errno) { return SIMDB_ERR_SYSTEM; }
const char *imdb_hdr_fmt = "IMDB v%02u, CAPS: %s;"; const char *simdb_hdr_fmt = "IMDB v%02u, CAPS: %s;";
bool bool
imdb_create(const char *path) { simdb_create(const char *path) {
ssize_t bytes = 0; ssize_t bytes = 0;
unsigned char buf[IMDB_REC_LEN]; unsigned char buf[SIMDB_REC_LEN];
const char *caps = "M-R"; const char *caps = "M-R";
bool result = false; bool result = false;
int fd = -1; int fd = -1;
memset(buf, 0x0, sizeof(char) * IMDB_REC_LEN); memset(buf, 0x0, sizeof(char) * SIMDB_REC_LEN);
if ((fd = creat(path, 0644)) < 0) if ((fd = creat(path, 0644)) < 0)
return result; return result;
snprintf((char *) buf, IMDB_REC_LEN, imdb_hdr_fmt, IMDB_VERSION, caps); snprintf((char *) buf, SIMDB_REC_LEN, simdb_hdr_fmt, SIMDB_VERSION, caps);
bytes = pwrite(fd, buf, IMDB_REC_LEN, 0); bytes = pwrite(fd, buf, SIMDB_REC_LEN, 0);
if (bytes == IMDB_REC_LEN) if (bytes == SIMDB_REC_LEN)
result = true; /* success */ result = true; /* success */
close(fd); close(fd);
@ -55,12 +55,12 @@ imdb_create(const char *path) {
return result; return result;
} }
imdb_db_t * simdb_t *
imdb_open(const char *path, int mode, int *error) { simdb_open(const char *path, int mode, int *error) {
imdb_db_t *db = NULL; simdb_t *db = NULL;
ssize_t bytes = 0; ssize_t bytes = 0;
struct stat st; struct stat st;
char buf[IMDB_REC_LEN] = "\0"; char buf[SIMDB_REC_LEN] = "\0";
int flags = 0, fd = -1; int flags = 0, fd = -1;
char *p; char *p;
@ -68,31 +68,31 @@ imdb_open(const char *path, int mode, int *error) {
assert(error != NULL); assert(error != NULL);
if (stat(path, &st) < 0) { if (stat(path, &st) < 0) {
*error = IMDB_ERR_SYSTEM; *error = SIMDB_ERR_SYSTEM;
return NULL; return NULL;
} }
if ((fd = open(path, (mode & IMDB_FLAG_WRITE) ? O_RDWR : O_RDONLY)) < 0) { if ((fd = open(path, (mode & SIMDB_FLAG_WRITE) ? O_RDWR : O_RDONLY)) < 0) {
*error = IMDB_ERR_SYSTEM; *error = SIMDB_ERR_SYSTEM;
return NULL; return NULL;
} }
errno = 0; errno = 0;
bytes = pread(fd, buf, IMDB_REC_LEN, 0); bytes = pread(fd, buf, SIMDB_REC_LEN, 0);
if (bytes < IMDB_REC_LEN) { if (bytes < SIMDB_REC_LEN) {
*error = errno ? IMDB_ERR_SYSTEM : IMDB_ERR_CORRUPTDB; *error = errno ? SIMDB_ERR_SYSTEM : SIMDB_ERR_CORRUPTDB;
return NULL; return NULL;
} }
p = buf + 0; p = buf + 0;
if (memcmp("IMDB", p, 4) != 0) { if (memcmp("IMDB", p, 4) != 0) {
*error = IMDB_ERR_CORRUPTDB; *error = SIMDB_ERR_CORRUPTDB;
return NULL; return NULL;
} }
p = buf + 6; p = buf + 6;
if (atoi(p) != IMDB_VERSION) { if (atoi(p) != SIMDB_VERSION) {
*error = IMDB_ERR_WRONGVERS; *error = SIMDB_ERR_WRONGVERS;
return NULL; return NULL;
} }
@ -102,22 +102,22 @@ imdb_open(const char *path, int mode, int *error) {
return NULL; return NULL;
} }
if (mode & IMDB_FLAG_WRITE) if (mode & SIMDB_FLAG_WRITE)
flags |= IMDB_FLAG_WRITE; flags |= SIMDB_FLAG_WRITE;
/* all seems to be ok */ /* all seems to be ok */
if ((db = calloc(1, sizeof(imdb_db_t))) == NULL) { if ((db = calloc(1, sizeof(simdb_t))) == NULL) {
*error = IMDB_ERR_OOM; *error = SIMDB_ERR_OOM;
return NULL; return NULL;
} }
p = buf + 16; p = buf + 16;
for (size_t i = 0; i < 8 && *p != '\0'; p++) { for (size_t i = 0; i < 8 && *p != '\0'; p++) {
switch (*p) { switch (*p) {
case 'M' : flags |= IMDB_CAP_BITMAP; break; case 'M' : flags |= SIMDB_CAP_BITMAP; break;
case 'C' : flags |= IMDB_CAP_COLORS; break; case 'C' : flags |= SIMDB_CAP_COLORS; break;
case 'R' : flags |= IMDB_CAP_RATIO; break; case 'R' : flags |= SIMDB_CAP_RATIO; break;
case ';' : i = 9; /* end of flags */ break; case ';' : i = 9; /* end of flags */ break;
default: /* ignore */ break; default: /* ignore */ break;
} }
@ -132,7 +132,7 @@ imdb_open(const char *path, int mode, int *error) {
} }
void void
imdb_close(imdb_db_t *db) simdb_close(simdb_t *db)
{ {
assert(db != NULL); assert(db != NULL);
@ -141,26 +141,26 @@ imdb_close(imdb_db_t *db)
} }
const char * const char *
imdb_error(int error) { simdb_error(int error) {
if (error == IMDB_SUCCESS) { if (error == SIMDB_SUCCESS) {
return "success"; return "success";
} else if (error == IMDB_ERR_SYSTEM) { } else if (error == SIMDB_ERR_SYSTEM) {
return strerror(errno); return strerror(errno);
} else if (error == IMDB_ERR_OOM) { } else if (error == SIMDB_ERR_OOM) {
return "can't allocate memory"; return "can't allocate memory";
} else if (error == IMDB_ERR_CORRUPTDB) { } else if (error == SIMDB_ERR_CORRUPTDB) {
return "database corrupted"; return "database corrupted";
} else if (error == IMDB_ERR_WRONGVERS) { } else if (error == SIMDB_ERR_WRONGVERS) {
return "database version differs from library version"; return "database version differs from library version";
} else if (error == IMDB_ERR_READONLY) { } else if (error == SIMDB_ERR_READONLY) {
return "database opened in read-only mode"; return "database opened in read-only mode";
} else if (error == IMDB_ERR_NXRECORD) { } else if (error == SIMDB_ERR_NXRECORD) {
return "no such record in database"; return "no such record in database";
} }
return "unknown error"; return "unknown error";
} }
int imdb_read_rec(imdb_db_t *db, imdb_rec_t *rec) int simdb_read_rec(simdb_t *db, simdb_rec_t *rec)
{ {
ssize_t bytes = 0; ssize_t bytes = 0;
@ -168,10 +168,10 @@ int imdb_read_rec(imdb_db_t *db, imdb_rec_t *rec)
assert(rec != NULL); assert(rec != NULL);
assert(rec->num > 0); assert(rec->num > 0);
DB_READ(db, rec->data, IMDB_REC_LEN, rec->num * IMDB_REC_LEN); DB_READ(db, rec->data, SIMDB_REC_LEN, rec->num * SIMDB_REC_LEN);
if (bytes != IMDB_REC_LEN) if (bytes != SIMDB_REC_LEN)
return errno ? IMDB_ERR_SYSTEM : IMDB_ERR_NXRECORD; return errno ? SIMDB_ERR_SYSTEM : SIMDB_ERR_NXRECORD;
if (rec->data[0] != 0xFF) if (rec->data[0] != 0xFF)
return 0; return 0;
@ -179,7 +179,7 @@ int imdb_read_rec(imdb_db_t *db, imdb_rec_t *rec)
return 1; return 1;
} }
int imdb_write_rec(imdb_db_t *db, imdb_rec_t *rec) int simdb_write_rec(simdb_t *db, simdb_rec_t *rec)
{ {
ssize_t bytes = 0; ssize_t bytes = 0;
@ -187,18 +187,18 @@ int imdb_write_rec(imdb_db_t *db, imdb_rec_t *rec)
assert(rec != NULL); assert(rec != NULL);
assert(rec->num > 0); assert(rec->num > 0);
if (!(db->flags & IMDB_FLAG_WRITE)) if (!(db->flags & SIMDB_FLAG_WRITE))
return IMDB_ERR_READONLY; return SIMDB_ERR_READONLY;
DB_WRITE(db, rec->data, IMDB_REC_LEN, rec->num * IMDB_REC_LEN); DB_WRITE(db, rec->data, SIMDB_REC_LEN, rec->num * SIMDB_REC_LEN);
if (bytes != IMDB_REC_LEN) if (bytes != SIMDB_REC_LEN)
return IMDB_ERR_SYSTEM; return SIMDB_ERR_SYSTEM;
return 1; return 1;
} }
int imdb_read_blk(imdb_db_t *db, imdb_block_t *blk) int simdb_read_blk(simdb_t *db, simdb_block_t *blk)
{ {
ssize_t bytes = 0; ssize_t bytes = 0;
@ -208,15 +208,15 @@ int imdb_read_blk(imdb_db_t *db, imdb_block_t *blk)
assert(blk->records > 0); assert(blk->records > 0);
FREE(blk->data); FREE(blk->data);
CALLOC(blk->data, blk->records, IMDB_REC_LEN); CALLOC(blk->data, blk->records, SIMDB_REC_LEN);
DB_READ(db, blk->data, blk->records * IMDB_REC_LEN, blk->start * IMDB_REC_LEN); DB_READ(db, blk->data, blk->records * SIMDB_REC_LEN, blk->start * SIMDB_REC_LEN);
blk->records = bytes / IMDB_REC_LEN; blk->records = bytes / SIMDB_REC_LEN;
return blk->records; return blk->records;
} }
uint64_t uint64_t
imdb_records_count(imdb_db_t * const db) { simdb_records_count(simdb_t * const db) {
struct stat st; struct stat st;
off_t size = 0; off_t size = 0;
@ -225,7 +225,7 @@ imdb_records_count(imdb_db_t * const db) {
fstat(db->fd, &st); fstat(db->fd, &st);
size = st.st_size; size = st.st_size;
return size / IMDB_REC_LEN; return size / SIMDB_REC_LEN;
} }
float float
@ -239,13 +239,13 @@ ratio_from_rec_data(unsigned char * const data) {
} }
int int
imdb_search(imdb_db_t * const db, simdb_search(simdb_t * const db,
imdb_rec_t * const sample, simdb_rec_t * const sample,
imdb_search_t * const search, simdb_search_t * const search,
imdb_match_t **matches) simdb_match_t **matches)
{ {
imdb_block_t blk; simdb_block_t blk;
imdb_match_t match; simdb_match_t match;
const int blk_size = 4096; const int blk_size = 4096;
uint64_t found = 0; uint64_t found = 0;
unsigned int i = 0; unsigned int i = 0;
@ -261,12 +261,12 @@ imdb_search(imdb_db_t * const db,
assert(search->maxdiff_ratio >= 0.0 && search->maxdiff_ratio <= 1.0); assert(search->maxdiff_ratio >= 0.0 && search->maxdiff_ratio <= 1.0);
assert(search->maxdiff_bitmap >= 0.0 && search->maxdiff_bitmap <= 1.0); assert(search->maxdiff_bitmap >= 0.0 && search->maxdiff_bitmap <= 1.0);
memset(&blk, 0x0, sizeof(imdb_block_t)); memset(&blk, 0x0, sizeof(simdb_block_t));
memset(&match, 0x0, sizeof(imdb_match_t)); memset(&match, 0x0, sizeof(simdb_match_t));
blk.start = 1; blk.start = 1;
blk.records = blk_size; blk.records = blk_size;
if ((ret = imdb_read_rec(db, sample)) < 1) if ((ret = simdb_read_rec(db, sample)) < 1)
return ret; return ret;
if (search->limit == 0) if (search->limit == 0)
@ -275,11 +275,11 @@ imdb_search(imdb_db_t * const db,
if (search->maxdiff_ratio > 0.0) if (search->maxdiff_ratio > 0.0)
ratio_s = ratio_from_rec_data(sample->data); ratio_s = ratio_from_rec_data(sample->data);
CALLOC(*matches, search->limit, sizeof(imdb_match_t)); CALLOC(*matches, search->limit, sizeof(simdb_match_t));
while (imdb_read_blk(db, &blk) > 0) { while (simdb_read_blk(db, &blk) > 0) {
p = blk.data; p = blk.data;
for (i = 0; i < blk.records; i++, p += IMDB_REC_LEN) { for (i = 0; i < blk.records; i++, p += SIMDB_REC_LEN) {
if (*(p + REC_OFF_RU) == 0x0) if (*(p + REC_OFF_RU) == 0x0)
continue; /* record missing */ continue; /* record missing */
@ -304,7 +304,7 @@ imdb_search(imdb_db_t * const db,
/* create match record */ /* create match record */
match.num = blk.start + i; match.num = blk.start + i;
memcpy(&(*matches)[found], &match, sizeof(imdb_match_t)); memcpy(&(*matches)[found], &match, sizeof(simdb_match_t));
found++; found++;
if (found >= search->limit) if (found >= search->limit)
break; break;
@ -319,26 +319,26 @@ imdb_search(imdb_db_t * const db,
} }
uint64_t uint64_t
imdb_usage_map(imdb_db_t * const db, simdb_usage_map(simdb_t * const db,
char ** const map) { char ** const map) {
const int blk_size = 4096; const int blk_size = 4096;
imdb_block_t blk; simdb_block_t blk;
uint64_t records; uint64_t records;
unsigned char *r; /* mnemonics : block, record */ unsigned char *r; /* mnemonics : block, record */
char *m = NULL; /* mnemonics : map */ char *m = NULL; /* mnemonics : map */
memset(&blk, 0x0, sizeof(imdb_block_t)); memset(&blk, 0x0, sizeof(simdb_block_t));
records = imdb_records_count(db); records = simdb_records_count(db);
CALLOC(*map, records + 1, sizeof(char)); CALLOC(*map, records + 1, sizeof(char));
m = *map; m = *map;
blk.start = 1; blk.start = 1;
blk.records = blk_size; blk.records = blk_size;
while (imdb_read_blk(db, &blk) > 0) { while (simdb_read_blk(db, &blk) > 0) {
r = blk.data; r = blk.data;
for (unsigned int i = 0; i < blk.records; i++, m++, r += IMDB_REC_LEN) { for (unsigned int i = 0; i < blk.records; i++, m++, r += SIMDB_REC_LEN) {
*m = (r[REC_OFF_RU] == 0xFF) ? CHAR_USED : CHAR_NONE; *m = (r[REC_OFF_RU] == 0xFF) ? CHAR_USED : CHAR_NONE;
} }
blk.start += blk_size; blk.start += blk_size;
@ -349,24 +349,24 @@ imdb_usage_map(imdb_db_t * const db,
} }
uint16_t uint16_t
imdb_usage_slice(imdb_db_t * const db, simdb_usage_slice(simdb_t * const db,
char ** const map, char ** const map,
uint64_t offset, uint64_t offset,
uint16_t limit) { uint16_t limit) {
imdb_block_t blk; simdb_block_t blk;
unsigned char *r; /* mnemonics : block, record */ unsigned char *r; /* mnemonics : block, record */
char *m = NULL; /* mnemonics : map */ char *m = NULL; /* mnemonics : map */
memset(&blk, 0x0, sizeof(imdb_block_t)); memset(&blk, 0x0, sizeof(simdb_block_t));
CALLOC(*map, limit + 1, sizeof(char)); CALLOC(*map, limit + 1, sizeof(char));
m = *map; m = *map;
blk.start = offset; blk.start = offset;
blk.records = limit; blk.records = limit;
limit = imdb_read_blk(db, &blk); limit = simdb_read_blk(db, &blk);
r = blk.data; r = blk.data;
for (uint16_t i = 0; i < blk.records; i++, m++, r += IMDB_REC_LEN) { for (uint16_t i = 0; i < blk.records; i++, m++, r += SIMDB_REC_LEN) {
*m = (r[REC_OFF_RU] == 0xFF) ? CHAR_USED : CHAR_NONE; *m = (r[REC_OFF_RU] == 0xFF) ? CHAR_USED : CHAR_NONE;
} }

74
src/database.h
Unescape View File

@ -1,32 +1,32 @@
#ifndef HAS_DATABASE_H #ifndef HAS_DATABASE_H
#define HAS_DATABASE_H 1 #define HAS_DATABASE_H 1
#define IMDB_REC_LEN 48 #define SIMDB_REC_LEN 48
#define IMDB_VERSION 2 #define SIMDB_VERSION 2
/* runtime flags */ /* runtime flags */
#define IMDB_FLAG_WRITE 1 << (0 + 0) #define SIMDB_FLAG_WRITE 1 << (0 + 0)
/* database capabilities */ /* database capabilities */
#define IMDB_CAP_BITMAP 1 << (8 + 0) #define SIMDB_CAP_BITMAP 1 << (8 + 0)
#define IMDB_CAP_COLORS 1 << (8 + 1) #define SIMDB_CAP_COLORS 1 << (8 + 1)
#define IMDB_CAP_RATIO 1 << (8 + 2) #define SIMDB_CAP_RATIO 1 << (8 + 2)
/* 3 used, 5 reserved */ /* 3 used, 5 reserved */
#define IMDB_SUCCESS 0 #define SIMDB_SUCCESS 0
/* database errors */ /* database errors */
#define IMDB_ERR_SYSTEM -1 /* see errno for details */ #define SIMDB_ERR_SYSTEM -1 /* see errno for details */
#define IMDB_ERR_OOM -2 /* can't allocate memory */ #define SIMDB_ERR_OOM -2 /* can't allocate memory */
#define IMDB_ERR_CORRUPTDB -3 /* empty or currupted database */ #define SIMDB_ERR_CORRUPTDB -3 /* empty or currupted database */
#define IMDB_ERR_WRONGVERS -4 /* database version mismatch */ #define SIMDB_ERR_WRONGVERS -4 /* database version mismatch */
#define IMDB_ERR_NXRECORD -5 /* no such record in database */ #define SIMDB_ERR_NXRECORD -5 /* no such record in database */
#define IMDB_ERR_READONLY -6 /* database opened in read-only mode */ #define SIMDB_ERR_READONLY -6 /* database opened in read-only mode */
typedef struct { typedef struct {
int fd; int fd;
int flags; int flags;
char path[PATH_MAX]; char path[PATH_MAX];
} imdb_db_t; } simdb_t;
/** /**
Database header format - fixed length, 48 bytes Database header format - fixed length, 48 bytes
@ -70,15 +70,15 @@ typedef struct {
uint64_t start; uint64_t start;
size_t records; size_t records;
unsigned char *data; unsigned char *data;
} imdb_block_t; } simdb_block_t;
/** /**
* database record * database record
*/ */
typedef struct { typedef struct {
uint64_t num; uint64_t num;
unsigned char data[IMDB_REC_LEN]; unsigned char data[SIMDB_REC_LEN];
} imdb_rec_t; } simdb_rec_t;
/** /**
* search parameters * search parameters
@ -88,7 +88,7 @@ typedef struct {
uint8_t limit; /**< max results */ uint8_t limit; /**< max results */
float maxdiff_bitmap; /**< max difference of luma bitmaps */ float maxdiff_bitmap; /**< max difference of luma bitmaps */
float maxdiff_ratio; /**< max difference of ratios, default - 7% */ float maxdiff_ratio; /**< max difference of ratios, default - 7% */
} imdb_search_t; } simdb_search_t;
/** /**
* search matches * search matches
@ -97,16 +97,16 @@ typedef struct {
uint64_t num; /**< record id */ uint64_t num; /**< record id */
float diff_ratio; /**< difference of ratio */ float diff_ratio; /**< difference of ratio */
float diff_bitmap; /**< difference of bitmap */ float diff_bitmap; /**< difference of bitmap */
} imdb_match_t; } simdb_match_t;
/** /**
* @returns 1 on success, 0 if record missing and <0 on error * @returns 1 on success, 0 if record missing and <0 on error
*/ */
int imdb_read_rec (imdb_db_t *db, imdb_rec_t *rec); int simdb_read_rec (simdb_t *db, simdb_rec_t *rec);
int imdb_write_rec(imdb_db_t *db, imdb_rec_t *rec); int simdb_write_rec(simdb_t *db, simdb_rec_t *rec);
int imdb_read_blk (imdb_db_t *db, imdb_block_t *blk); int simdb_read_blk (simdb_t *db, simdb_block_t *blk);
int imdb_write_blk(imdb_db_t *db, imdb_block_t *blk); int simdb_write_blk(simdb_t *db, simdb_block_t *blk);
/** /**
* @brief Creates empty database at given path * @brief Creates empty database at given path
@ -115,29 +115,29 @@ int imdb_write_blk(imdb_db_t *db, imdb_block_t *blk);
* @note See errno value for details * @note See errno value for details
* @todo 2nd arg: caps * @todo 2nd arg: caps
*/ */
bool imdb_create(const char *path); bool simdb_create(const char *path);
/** /**
* @brief Open database at given path * @brief Open database at given path
* @param path Path to database * @param path Path to database
* @param mode Database open modes. See IMDB_FLAG_* defines above * @param mode Database open modes. See SIMDB_FLAG_* defines above
* @param error Pointer to error code storage * @param error Pointer to error code storage
* @returns Pointer to database handle on success, NULL on error * @returns Pointer to database handle on success, NULL on error
* @note use @a imdb_error() to get error description * @note use @a simdb_error() to get error description
*/ */
imdb_db_t * imdb_open(const char *path, int mode, int *error); simdb_t * simdb_open(const char *path, int mode, int *error);
/** /**
* @brief Close database and free associated resources * @brief Close database and free associated resources
* @param db Database handle * @param db Database handle
*/ */
void imdb_close(imdb_db_t *db); void simdb_close(simdb_t *db);
/** /**
* @brief Get error desctiption by error code * @brief Get error desctiption by error code
* @param error Error code, see IMDB_ERR_* defines above * @param error Error code, see SIMDB_ERR_* defines above
*/ */
const char * imdb_error(int code); const char * simdb_error(int code);
float float
ratio_from_rec_data(unsigned char * const data); ratio_from_rec_data(unsigned char * const data);
@ -146,23 +146,23 @@ ratio_from_rec_data(unsigned char * const data);
@returns: >0 if found some matches, 0 if nothing found, <0 on error @returns: >0 if found some matches, 0 if nothing found, <0 on error
*/ */
int int
imdb_search(imdb_db_t * const db, simdb_search(simdb_t * const db,
imdb_rec_t * const sample, simdb_rec_t * const sample,
imdb_search_t * const search, simdb_search_t * const search,
imdb_match_t ** matches); simdb_match_t ** matches);
/** /**
@returns: number of records in database @returns: number of records in database
*/ */
uint64_t uint64_t
imdb_records_count(imdb_db_t * const db); simdb_records_count(simdb_t * const db);
/** /**
@brief fills buffer 'map' according to records existense in database @brief fills buffer 'map' according to records existense in database
@returns records processed (and also buffer size) @returns records processed (and also buffer size)
*/ */
uint64_t uint64_t
imdb_usage_map(imdb_db_t * const db, simdb_usage_map(simdb_t * const db,
char ** const map); char ** const map);
/** /**
@ -172,7 +172,7 @@ imdb_usage_map(imdb_db_t * const db,
@returns records processed (and also buffer size) @returns records processed (and also buffer size)
*/ */
uint16_t uint16_t
imdb_usage_slice(imdb_db_t * const db, simdb_usage_slice(simdb_t * const db,
char ** const map, char ** const map,
uint64_t offset, uint64_t offset,
uint16_t limit); uint16_t limit);

4
src/sample.c
Unescape View File

@ -6,7 +6,7 @@
#include <wand/magick_wand.h> #include <wand/magick_wand.h>
int int
imdb_sample(imdb_rec_t * const rec, simdb_sample(simdb_rec_t * const rec,
const char * const source) const char * const source)
{ {
MagickWand *wand = NULL; MagickWand *wand = NULL;
@ -83,7 +83,7 @@ imdb_sample(imdb_rec_t * const rec,
if (status == MagickPass) { if (status == MagickPass) {
assert(buf_size == BITMAP_SIZE); assert(buf_size == BITMAP_SIZE);
memset(rec->data, 0x0, IMDB_REC_LEN); memset(rec->data, 0x0, SIMDB_REC_LEN);
rec->data[REC_OFF_RU] = 0xFF; rec->data[REC_OFF_RU] = 0xFF;
memcpy(&rec->data[REC_OFF_IW], &w, sizeof(uint16_t)); memcpy(&rec->data[REC_OFF_IW], &w, sizeof(uint16_t));
memcpy(&rec->data[REC_OFF_IH], &h, sizeof(uint16_t)); memcpy(&rec->data[REC_OFF_IH], &h, sizeof(uint16_t));

3
src/sample.h
Unescape View File

@ -2,7 +2,6 @@
#define HAS_SAMPLER_H 1 #define HAS_SAMPLER_H 1
int int
imdb_sample(imdb_rec_t * const rec, simdb_sample(simdb_rec_t * const rec, const char * const source);
const char * const source);
#endif /* HAS_SAMPLER_H */ #endif /* HAS_SAMPLER_H */

54
src/imdb-1to2.c → src/simdb-1to2.c
Unescape View File

@ -7,28 +7,28 @@
#include <sys/types.h> #include <sys/types.h>
#include <unistd.h> #include <unistd.h>
#define IMDB_REC_LEN 48 #define SIMDB_REC_LEN 48
#define BLK_SIZE 1000 #define BLK_SIZE 1000
void usage(const char *message) { void usage(const char *message) {
if (message) if (message)
printf("error: %s\n", message); printf("error: %s\n", message);
printf("Usage: imdb_convert_1to2 <infile> <outfile>\n"); printf("Usage: simdb_convert_1to2 <infile> <outfile>\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
int main(int argc, char **argv) { int main(int argc, char **argv) {
int in, out; int in, out;
long unsigned int imdb_rec_total, rec_first, rec_last, records; long unsigned int simdb_rec_total, rec_first, rec_last, records;
unsigned char in_buf[IMDB_REC_LEN * BLK_SIZE]; unsigned char in_buf[SIMDB_REC_LEN * BLK_SIZE];
unsigned char out_buf[IMDB_REC_LEN * BLK_SIZE]; unsigned char out_buf[SIMDB_REC_LEN * BLK_SIZE];
unsigned char header[IMDB_REC_LEN]; unsigned char header[SIMDB_REC_LEN];
unsigned char *src, *dst; unsigned char *src, *dst;
struct stat st; struct stat st;
ssize_t bytes; ssize_t bytes;
memset(header, 0x0, IMDB_REC_LEN); memset(header, 0x0, SIMDB_REC_LEN);
snprintf((char *) header, IMDB_REC_LEN, "IMDB v%02u, CAPS: %s;", 2, "M-R"); snprintf((char *) header, SIMDB_REC_LEN, "IMDB v%02u, CAPS: %s;", 2, "M-R");
if (argc < 2) if (argc < 2)
usage(NULL); usage(NULL);
@ -43,54 +43,54 @@ int main(int argc, char **argv) {
if (fstat(in, &st) != 0) if (fstat(in, &st) != 0)
usage(strerror(errno)); usage(strerror(errno));
if ((st.st_size % IMDB_REC_LEN) != 0) if ((st.st_size % SIMDB_REC_LEN) != 0)
usage("database size expected to be multiples to 48"); usage("database size expected to be multiples to 48");
if (lseek(in, 0, SEEK_SET) < 0) if (lseek(in, 0, SEEK_SET) < 0)
usage(strerror(errno)); usage(strerror(errno));
if (read(in, in_buf, IMDB_REC_LEN) != IMDB_REC_LEN) if (read(in, in_buf, SIMDB_REC_LEN) != SIMDB_REC_LEN)
usage("can't read header of database"); usage("can't read header of database");
if (memcmp(in_buf, "DB of image fingerprints (ver 1)", 32) != 0) if (memcmp(in_buf, "DB of image fingerprints (ver 1)", 32) != 0)
usage("wrong database header / version mismatch"); usage("wrong database header / version mismatch");
imdb_rec_total = (st.st_size / IMDB_REC_LEN); simdb_rec_total = (st.st_size / SIMDB_REC_LEN);
printf("Processing %lu records\n", imdb_rec_total - 1); printf("Processing %lu records\n", simdb_rec_total - 1);
if (lseek(in, IMDB_REC_LEN, SEEK_SET) < 0) if (lseek(in, SIMDB_REC_LEN, SEEK_SET) < 0)
usage(strerror(errno)); usage(strerror(errno));
if (lseek(out, 0, SEEK_SET) < 0) if (lseek(out, 0, SEEK_SET) < 0)
usage(strerror(errno)); usage(strerror(errno));
if (write(out, header, IMDB_REC_LEN) != IMDB_REC_LEN) if (write(out, header, SIMDB_REC_LEN) != SIMDB_REC_LEN)
usage(strerror(errno)); usage(strerror(errno));
for (unsigned int block = 0; block <= (imdb_rec_total / BLK_SIZE); block++) { for (unsigned int block = 0; block <= (simdb_rec_total / BLK_SIZE); block++) {
rec_first = rec_last = (block * BLK_SIZE) + 1; rec_first = rec_last = (block * BLK_SIZE) + 1;
records = ((rec_first + BLK_SIZE) > imdb_rec_total) ? imdb_rec_total - rec_first : BLK_SIZE; records = ((rec_first + BLK_SIZE) > simdb_rec_total) ? simdb_rec_total - rec_first : BLK_SIZE;
rec_last = rec_first + records; rec_last = rec_first + records;
printf("* block %u, %4lu records [%lu, %lu]\n", printf("* block %u, %4lu records [%lu, %lu]\n",
block + 1, records, rec_first, rec_last - 1); block + 1, records, rec_first, rec_last - 1);
lseek(in, rec_first * IMDB_REC_LEN, SEEK_SET); lseek(in, rec_first * SIMDB_REC_LEN, SEEK_SET);
lseek(out, rec_first * IMDB_REC_LEN, SEEK_SET); lseek(out, rec_first * SIMDB_REC_LEN, SEEK_SET);
bytes = read(in, in_buf, IMDB_REC_LEN * records); bytes = read(in, in_buf, SIMDB_REC_LEN * records);
if (bytes != (ssize_t) records * IMDB_REC_LEN) { if (bytes != (ssize_t) records * SIMDB_REC_LEN) {
printf("Read size mismatch, expected %lu, got: %zi\n", printf("Read size mismatch, expected %lu, got: %zi\n",
IMDB_REC_LEN * records, bytes); SIMDB_REC_LEN * records, bytes);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
memset(out_buf, 0x0, records * IMDB_REC_LEN); memset(out_buf, 0x0, records * SIMDB_REC_LEN);
for (unsigned int i = 0; i < records; i++) { for (unsigned int i = 0; i < records; i++) {
src = &in_buf[i * IMDB_REC_LEN]; src = &in_buf[i * SIMDB_REC_LEN];
dst = &out_buf[i * IMDB_REC_LEN]; dst = &out_buf[i * SIMDB_REC_LEN];
memcpy(dst + 0, src + 0, sizeof(char) * 1); // usage flag memcpy(dst + 0, src + 0, sizeof(char) * 1); // usage flag
memcpy(dst + 16, src + 2, sizeof(char) * 32); // image bitmap memcpy(dst + 16, src + 2, sizeof(char) * 32); // image bitmap
} }
bytes = write(out, out_buf, IMDB_REC_LEN * records); bytes = write(out, out_buf, SIMDB_REC_LEN * records);
if (bytes != (ssize_t) records * IMDB_REC_LEN) { if (bytes != (ssize_t) records * SIMDB_REC_LEN) {
printf("Write size mismatch, expected %lu, got: %zi\n", printf("Write size mismatch, expected %lu, got: %zi\n",
IMDB_REC_LEN * records, bytes); SIMDB_REC_LEN * records, bytes);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }

50
src/imdb-query.c → src/simdb-query.c
Unescape View File

@ -23,7 +23,7 @@
void usage(int exitcode) { void usage(int exitcode) {
fprintf(stderr, fprintf(stderr,
"Usage: imdb-query <opts>\n" "Usage: simdb-query <opts>\n"
" -b <path> Path to database\n" " -b <path> Path to database\n"
" -t <int> Maximum difference pct (0 - 50, default: 10%%)\n" " -t <int> Maximum difference pct (0 - 50, default: 10%%)\n"
); );
@ -40,21 +40,21 @@ void usage(int exitcode) {
exit(exitcode); exit(exitcode);
} }
int search_similar(imdb_db_t *db, uint64_t number, float maxdiff) int search_similar(simdb_t *db, uint64_t number, float maxdiff)
{ {
int ret = 0, i = 0; int ret = 0, i = 0;
imdb_rec_t sample; simdb_rec_t sample;
imdb_match_t *matches = NULL; simdb_match_t *matches = NULL;
imdb_search_t search; simdb_search_t search;
memset(&sample, 0x0, sizeof(imdb_rec_t)); memset(&sample, 0x0, sizeof(simdb_rec_t));
memset(&search, 0x0, sizeof(imdb_search_t)); memset(&search, 0x0, sizeof(simdb_search_t));
search.maxdiff_ratio = 0.2; /* 20% */ search.maxdiff_ratio = 0.2; /* 20% */
search.maxdiff_bitmap = maxdiff; search.maxdiff_bitmap = maxdiff;
sample.num = number; sample.num = number;
if ((ret = imdb_search(db, &sample, &search, &matches)) < 0) { if ((ret = simdb_search(db, &sample, &search, &matches)) < 0) {
fprintf(stderr, "%s\n", imdb_error(ret)); fprintf(stderr, "%s\n", simdb_error(ret));
return 1; return 1;
} }
@ -70,7 +70,7 @@ int search_similar(imdb_db_t *db, uint64_t number, float maxdiff)
return 0; return 0;
} }
int db_usage_map(imdb_db_t *db, unsigned short int cols) int db_usage_map(simdb_t *db, unsigned short int cols)
{ {
char *map = NULL; char *map = NULL;
char *m = NULL; char *m = NULL;
@ -80,7 +80,7 @@ int db_usage_map(imdb_db_t *db, unsigned short int cols)
memset(row, 0x0, sizeof(char) * 256); memset(row, 0x0, sizeof(char) * 256);
if ((records = imdb_usage_map(db, &map)) == 0) { if ((records = simdb_usage_map(db, &map)) == 0) {
fprintf(stderr, "database usage: can't get database map\n"); fprintf(stderr, "database usage: can't get database map\n");
FREE(map); FREE(map);
return 1; return 1;
@ -107,26 +107,26 @@ int db_usage_map(imdb_db_t *db, unsigned short int cols)
return 0; return 0;
} }
int db_usage_slice(imdb_db_t *db, uint64_t offset, uint16_t limit) int db_usage_slice(simdb_t *db, uint64_t offset, uint16_t limit)
{ {
char *map = NULL; char *map = NULL;
limit = imdb_usage_slice(db, &map, offset, limit); limit = simdb_usage_slice(db, &map, offset, limit);
puts(map); puts(map);
FREE(map); FREE(map);
return 0; return 0;
} }
int rec_bitmap(imdb_db_t *db, uint64_t number) int rec_bitmap(simdb_t *db, uint64_t number)
{ {
imdb_rec_t rec; simdb_rec_t rec;
assert(db != NULL); assert(db != NULL);
memset(&rec, 0x0, sizeof(imdb_rec_t)); memset(&rec, 0x0, sizeof(simdb_rec_t));
rec.num = number; rec.num = number;
if (imdb_read_rec(db, &rec) < 1) { if (simdb_read_rec(db, &rec) < 1) {
fprintf(stderr, "bitmap: %s\n", "sample not found"); fprintf(stderr, "bitmap: %s\n", "sample not found");
return 1; return 1;
} }
@ -136,26 +136,26 @@ int rec_bitmap(imdb_db_t *db, uint64_t number)
return 0; return 0;
} }
int rec_diff(imdb_db_t *db, uint64_t a, uint64_t b, unsigned short int showmap) int rec_diff(simdb_t *db, uint64_t a, uint64_t b, unsigned short int showmap)
{ {
imdb_rec_t rec; simdb_rec_t rec;
float diff = 0.0; float diff = 0.0;
unsigned char one[BITMAP_SIZE]; unsigned char one[BITMAP_SIZE];
unsigned char two[BITMAP_SIZE]; unsigned char two[BITMAP_SIZE];
unsigned char map[BITMAP_SIZE]; unsigned char map[BITMAP_SIZE];
assert(db != NULL); assert(db != NULL);
memset(&rec, 0x0, sizeof(imdb_rec_t)); memset(&rec, 0x0, sizeof(simdb_rec_t));
rec.num = a; rec.num = a;
if (imdb_read_rec(db, &rec) < 1) { if (simdb_read_rec(db, &rec) < 1) {
fprintf(stderr, "record diff: first sample not exists\n"); fprintf(stderr, "record diff: first sample not exists\n");
return 1; return 1;
} }
memcpy(one, &rec.data[REC_OFF_BM], BITMAP_SIZE); memcpy(one, &rec.data[REC_OFF_BM], BITMAP_SIZE);
rec.num = b; rec.num = b;
if (imdb_read_rec(db, &rec) < 1) { if (simdb_read_rec(db, &rec) < 1) {
fprintf(stderr, "record diff: second sample not exists\n"); fprintf(stderr, "record diff: second sample not exists\n");
return 1; return 1;
} }
@ -179,7 +179,7 @@ int main(int argc, char **argv)
const char *db_path = NULL; const char *db_path = NULL;
float maxdiff = 0.10; float maxdiff = 0.10;
unsigned short int cols = 64, map = 0, ret = 0; unsigned short int cols = 64, map = 0, ret = 0;
imdb_db_t *db = NULL; simdb_t *db = NULL;
uint64_t a = 0, b = 0; uint64_t a = 0, b = 0;
char *c = NULL; char *c = NULL;
int err; int err;
@ -242,7 +242,7 @@ int main(int argc, char **argv)
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if ((db = imdb_open(db_path, 0, &err)) == NULL) { if ((db = simdb_open(db_path, 0, &err)) == NULL) {
fprintf(stderr, "database open: %d\n", err); fprintf(stderr, "database open: %d\n", err);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
@ -280,7 +280,7 @@ int main(int argc, char **argv)
break; break;
} }
imdb_close(db); simdb_close(db);
return ret; return ret;
} }

24
src/imdb-write.c → src/simdb-write.c
Unescape View File

@ -7,7 +7,7 @@
void usage(int exitcode) { void usage(int exitcode) {
fprintf(stderr, fprintf(stderr,
"Usage: imdb-write <opts>\n" "Usage: simdb-write <opts>\n"
" -b <path> Path to database\n" " -b <path> Path to database\n"
); );
fprintf(stderr, fprintf(stderr,
@ -25,11 +25,11 @@ int main(int argc, char **argv)
const char *db_path = NULL; const char *db_path = NULL;
const char *sample = NULL; const char *sample = NULL;
const char *c = NULL; const char *c = NULL;
imdb_db_t *db = NULL; simdb_t *db = NULL;
imdb_rec_t rec; simdb_rec_t rec;
int err; int err;
memset(&rec, 0x0, sizeof(imdb_rec_t)); memset(&rec, 0x0, sizeof(simdb_rec_t));
if (argc < 3) if (argc < 3)
usage(EXIT_FAILURE); usage(EXIT_FAILURE);
@ -68,12 +68,12 @@ int main(int argc, char **argv)
} }
if (mode == init) { if (mode == init) {
if (!imdb_create(db_path)) { if (!simdb_create(db_path)) {
fprintf(stderr, "database init: %s\n", strerror(errno)); fprintf(stderr, "database init: %s\n", strerror(errno));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
} }
if ((db = imdb_open(db_path, 1, &err)) == NULL) { if ((db = simdb_open(db_path, 1, &err)) == NULL) {
fprintf(stderr, "can't open database: %d\n", err); fprintf(stderr, "can't open database: %d\n", err);
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
@ -82,18 +82,18 @@ int main(int argc, char **argv)
case add : case add :
if (rec.num == 0 || sample == NULL) if (rec.num == 0 || sample == NULL)
usage(EXIT_FAILURE); usage(EXIT_FAILURE);
if (imdb_sample(&rec, sample) != 0) { if (simdb_sample(&rec, sample) != 0) {
fprintf(stderr, "sampler failure\n"); fprintf(stderr, "sampler failure\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
if ((err = imdb_write_rec(db, &rec)) < 1) { if ((err = simdb_write_rec(db, &rec)) < 1) {
fprintf(stderr, "%s\n", imdb_error(err)); fprintf(stderr, "%s\n", simdb_error(err));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
break; break;
case del : case del :
if ((err = imdb_write_rec(db, &rec)) < 1) { if ((err = simdb_write_rec(db, &rec)) < 1) {
fprintf(stderr, "%s\n", imdb_error(err)); fprintf(stderr, "%s\n", simdb_error(err));
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
break; break;
@ -104,7 +104,7 @@ int main(int argc, char **argv)
usage(EXIT_FAILURE); usage(EXIT_FAILURE);
break; break;
} }
imdb_close(db); simdb_close(db);
exit(EXIT_SUCCESS); exit(EXIT_SUCCESS);
} }

39
tests/database.c
Unescape View File

@ -1,53 +1,52 @@
#include "../src/common.h" #include "../src/common.h"
#include "../src/database.h" #include "../src/database.h"
int main() int main() {
{ simdb_t *db;
imdb_db_t *db; simdb_rec_t rec[2];
imdb_rec_t rec[2]; simdb_block_t blk;
imdb_block_t blk;
char *path = "test.db"; char *path = "test.db";
int mode = 0, err = 0, num; int mode = 0, err = 0, num;
bool ret; bool ret;
unlink(path); unlink(path);
db = imdb_open(path, mode, &err); db = simdb_open(path, mode, &err);
assert(db == NULL); assert(db == NULL);
assert(err == -1); /* no such file */ assert(err == -1); /* no such file */
ret = imdb_create(path); ret = simdb_create(path);
assert(ret == true); assert(ret == true);
db = imdb_open(path, mode, &err); db = simdb_open(path, mode, &err);
assert(db != NULL); assert(db != NULL);
rec[0].num = 1; rec[0].num = 1;
err = imdb_read_rec(db, rec); err = simdb_read_rec(db, rec);
assert(err == IMDB_ERR_NXRECORD); assert(err == SIMDB_ERR_NXRECORD);
memset(rec[0].data, 0xAA, IMDB_REC_LEN); memset(rec[0].data, 0xAA, SIMDB_REC_LEN);
memset(rec[0].data, 0xFF, 1); /* record is used */ memset(rec[0].data, 0xFF, 1); /* record is used */
num = imdb_write_rec(db, rec); num = simdb_write_rec(db, rec);
assert(num == IMDB_ERR_READONLY); /* database open in read-only mode */ assert(num == SIMDB_ERR_READONLY); /* database open in read-only mode */
imdb_close(db); simdb_close(db);
mode |= IMDB_FLAG_WRITE; mode |= SIMDB_FLAG_WRITE;
db = imdb_open(path, mode, &err); db = simdb_open(path, mode, &err);
assert(db != NULL); assert(db != NULL);
num = imdb_write_rec(db, rec); num = simdb_write_rec(db, rec);
assert(num == 1); /* success */ assert(num == 1); /* success */
num = imdb_read_rec(db, rec); num = simdb_read_rec(db, rec);
assert(num == 1); assert(num == 1);
blk.start = 1; blk.start = 1;
blk.records = 2; blk.records = 2;
blk.data = NULL; blk.data = NULL;
num = imdb_read_blk(db, &blk); num = simdb_read_blk(db, &blk);
assert(num == 1); assert(num == 1);
assert(blk.records == 1); assert(blk.records == 1);
assert(blk.data != NULL); assert(blk.data != NULL);
@ -55,7 +54,7 @@ int main()
rec[0].num = 1; rec[0].num = 1;
rec[1].num = 3; rec[1].num = 3;
imdb_close(db); simdb_close(db);
return 0; return 0;
} }

Write Preview
Loading…
Cancel
Save