ad_user
/
libsimdb
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

* use simdb_read() instead simdb_{record,block}_read()

master
Alex 'AdUser' Z 8 years ago
parent
245222705d
commit
17f23023e8
3 changed files with 99 additions and 105 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 119
      src/database.c
  2. 65
      src/simdb-query.c
  3. 18
      src/simdb.h

119
src/database.c
Unescape View File

@ -289,7 +289,7 @@ int simdb_block_read(simdb_t *db, simdb_block_t *blk) {
return blk->records; return blk->records;
} }
uint64_t int
simdb_records_count(simdb_t * const db) { simdb_records_count(simdb_t * const db) {
struct stat st; struct stat st;
off_t size = 0; off_t size = 0;
@ -313,54 +313,55 @@ simdb_record_ratio(simdb_urec_t *r) {
} }
int int
simdb_search(simdb_t * const db, simdb_search(simdb_t * const db, int num,
simdb_rec_t * const sample,
simdb_search_t * const search, simdb_search_t * const search,
simdb_match_t **matches) simdb_match_t **matches)
{ {
simdb_block_t blk;
simdb_match_t match; simdb_match_t match;
simdb_urec_t *rec; simdb_urec_t *data = NULL;
const int blk_size = 4096; simdb_urec_t *rec, sample;
const int blksize = 4096;
uint64_t found = 0; uint64_t found = 0;
unsigned int i = 0;
float ratio_s = 0.0; /* source */ float ratio_s = 0.0; /* source */
float ratio_t = 0.0; /* tested */ float ratio_t = 0.0; /* tested */
int ret = 0; int ret = 0;
assert(db != NULL); assert(db != NULL);
assert(sample != NULL);
assert(search != NULL); assert(search != NULL);
assert(matches != NULL); assert(matches != NULL);
assert(search->maxdiff_ratio >= 0.0 && search->maxdiff_ratio <= 1.0); assert(search->maxdiff_ratio >= 0.0 && search->maxdiff_ratio <= 1.0);
assert(search->maxdiff_bitmap >= 0.0 && search->maxdiff_bitmap <= 1.0); assert(search->maxdiff_bitmap >= 0.0 && search->maxdiff_bitmap <= 1.0);
memset(&blk, 0x0, sizeof(simdb_block_t));
memset(&match, 0x0, sizeof(simdb_match_t)); memset(&match, 0x0, sizeof(simdb_match_t));
blk.start = 1;
blk.records = blk_size;
if ((ret = simdb_record_read(db, sample)) < 1) if ((ret = simdb_read(db, num, 1, &rec)) < 1)
return ret; return ret;
memcpy(&sample, rec, sizeof(sample));
FREE(rec);
if (search->limit == 0) if (search->limit == 0)
search->limit = -1; /* unsigned -> max */ search->limit = -1; /* unsigned -> max */
if (search->maxdiff_ratio > 0.0) if (search->maxdiff_ratio > 0.0)
ratio_s = simdb_record_ratio((simdb_urec_t *) sample->data); ratio_s = simdb_record_ratio(&sample);
CALLOC(*matches, search->limit, sizeof(simdb_match_t)); CALLOC(*matches, search->limit, sizeof(simdb_match_t));
while (simdb_block_read(db, &blk) > 0) { for (num = 1; ; num += blksize) {
rec = (simdb_urec_t *) blk.data; ret = simdb_read(db, num, blksize, &data);
for (i = 0; i < blk.records; i++, rec++) { if (ret < 0)
if (rec->used == 0x0) return ret;
rec = data;
for (int i = 0; i < ret; i++, rec++) {
if (!rec->used)
continue; /* record missing */ continue; /* record missing */
match.diff_ratio = 0.0; match.diff_ratio = 0.0;
match.diff_bitmap = 0.0; match.diff_bitmap = 0.0;
/* - compare ratio - cheap */ /* - compare ratio - cheap */
/* TODO: check caps */
if (ratio_s > 0.0 && (ratio_t = simdb_record_ratio(rec)) > 0.0) { if (ratio_s > 0.0 && (ratio_t = simdb_record_ratio(rec)) > 0.0) {
match.diff_ratio = ratio_s - ratio_t; match.diff_ratio = ratio_s - ratio_t;
match.diff_ratio *= (ratio_s > ratio_t) ? 1.0 : -1.0; match.diff_ratio *= (ratio_s > ratio_t) ? 1.0 : -1.0;
@ -371,77 +372,83 @@ simdb_search(simdb_t * const db,
} }
/* - compare bitmap - more expensive */ /* - compare bitmap - more expensive */
match.diff_bitmap = simdb_bitmap_compare(rec->bitmap, ((simdb_urec_t *) sample)->bitmap) / SIMDB_BITMAP_BITS; match.diff_bitmap = simdb_bitmap_compare(rec->bitmap, sample.bitmap) / SIMDB_BITMAP_BITS;
if (match.diff_bitmap > search->maxdiff_bitmap) if (match.diff_bitmap > search->maxdiff_bitmap)
continue; continue;
/* create match record */ /* create match record */
match.num = blk.start + i; match.num = num + i;
memcpy(&(*matches)[found], &match, sizeof(simdb_match_t)); memcpy(&(*matches)[found], &match, sizeof(simdb_match_t));
found++; found++;
if (found >= search->limit) if (found >= search->limit)
break; break;
} }
FREE(blk.data); FREE(data);
if (found >= search->limit) if (found >= search->limit)
break; break;
blk.start += blk_size;
} }
return found; return found;
} }
uint64_t int
simdb_usage_map(simdb_t * const db, simdb_usage_map(simdb_t * const db, char ** const map) {
char ** const map) { const int blksize = 4096;
const int blk_size = 4096; simdb_urec_t *data = NULL;
simdb_block_t blk; simdb_urec_t *r; /* mnemonics : record pointer */
uint64_t records; char *m = NULL; /* mnemonics : map pointer */
simdb_urec_t *r; /* mnemonics : block, record */ int ret = 0, records = 0;
char *m = NULL; /* mnemonics : map */
memset(&blk, 0x0, sizeof(simdb_block_t)); assert(db != NULL);
assert(map != NULL);
records = simdb_records_count(db); records = simdb_records_count(db);
CALLOC(*map, records + 1, sizeof(char)); CALLOC(m, records + 1, sizeof(char));
*map = m;
m = *map; for (int num = 1; ; num += blksize) {
blk.start = 1; ret = simdb_read(db, num, blksize, &data);
blk.records = blk_size; if (ret < 0) {
FREE(*map);
while (simdb_block_read(db, &blk) > 0) { return ret;
r = (simdb_urec_t *) blk.data; }
for (unsigned int i = 0; i < blk.records; i++, m++, r++) { if (ret == 0)
break;
r = data;
for (int i = 0; i < ret; i++, m++, r++) {
*m = (r->used == 0xFF) ? CHAR_USED : CHAR_NONE; *m = (r->used == 0xFF) ? CHAR_USED : CHAR_NONE;
} }
blk.start += blk_size; FREE(data);
} }
FREE(blk.data);
return records; return records;
} }
uint16_t int
simdb_usage_slice(simdb_t * const db, simdb_usage_slice(simdb_t * const db, char ** const map, int offset, int limit) {
char ** const map, simdb_urec_t *data = NULL;
uint64_t offset, simdb_urec_t *r; /* mnemonics : record pointer */
uint16_t limit) { char *m = NULL; /* mnemonics : map pointer */
simdb_block_t blk; int ret = 0;
simdb_urec_t *r; /* mnemonics : block, record */
char *m = NULL; /* mnemonics : map */
memset(&blk, 0x0, sizeof(simdb_block_t)); assert(db != NULL);
CALLOC(*map, limit + 1, sizeof(char)); assert(map != NULL);
if (offset < 1 || limit < 1)
return SIMDB_ERR_USAGE;
ret = simdb_read(db, offset, limit, &data);
if (ret <= 0)
return ret;
CALLOC(m, limit + 1, sizeof(char));
m = *map; m = *map;
blk.start = offset;
blk.records = limit;
limit = simdb_block_read(db, &blk); r = data;
r = (simdb_urec_t *) blk.data; for (int i = 0; i < limit; i++, m++, r++) {
for (uint16_t i = 0; i < blk.records; i++, m++, r++) {
*m = (r->used == 0xFF) ? CHAR_USED : CHAR_NONE; *m = (r->used == 0xFF) ? CHAR_USED : CHAR_NONE;
} }
FREE(data);
return limit; return ret;
} }

65
src/simdb-query.c
Unescape View File

@ -16,8 +16,9 @@
#include "common.h" #include "common.h"
#include "bitmap.h" #include "bitmap.h"
#include "simdb.h"
#include "record.h" #include "record.h"
#include "io.h"
#include "simdb.h"
#include <unistd.h> #include <unistd.h>
#include <getopt.h> #include <getopt.h>
@ -41,20 +42,16 @@ void usage(int exitcode) {
exit(exitcode); exit(exitcode);
} }
int search_similar(simdb_t *db, uint64_t number, float maxdiff) int search_similar(simdb_t *db, int num, float maxdiff) {
{
int ret = 0, i = 0; int ret = 0, i = 0;
simdb_rec_t sample;
simdb_match_t *matches = NULL; simdb_match_t *matches = NULL;
simdb_search_t search; simdb_search_t search;
memset(&sample, 0x0, sizeof(simdb_rec_t));
memset(&search, 0x0, sizeof(simdb_search_t)); memset(&search, 0x0, sizeof(simdb_search_t));
search.maxdiff_ratio = 0.2; /* 20% */ search.maxdiff_ratio = 0.2; /* 20% */
search.maxdiff_bitmap = maxdiff; search.maxdiff_bitmap = maxdiff;
sample.num = number; if ((ret = simdb_search(db, num, &search, &matches)) < 0) {
if ((ret = simdb_search(db, &sample, &search, &matches)) < 0) {
fprintf(stderr, "%s\n", simdb_error(ret)); fprintf(stderr, "%s\n", simdb_error(ret));
return 1; return 1;
} }
@ -119,64 +116,62 @@ int db_usage_slice(simdb_t *db, uint64_t offset, uint16_t limit)
return 0; return 0;
} }
int rec_bitmap(simdb_t *db, uint64_t number) int rec_bitmap(simdb_t *db, int num) {
{
simdb_rec_t rec;
simdb_urec_t *r; simdb_urec_t *r;
int ret;
assert(db != NULL); assert(db != NULL);
memset(&rec, 0x0, sizeof(simdb_rec_t));
rec.num = number; if ((ret = simdb_read(db, num, 1, &r)) < 0) {
if (simdb_record_read(db, &rec) < 1) { fprintf(stderr, "read failed: %s\n", simdb_error(ret));
return 1;
} else if (ret == 0) {
fprintf(stderr, "bitmap: %s\n", "sample not found"); fprintf(stderr, "bitmap: %s\n", "sample not found");
return 1; return 1;
} }
r = (simdb_urec_t *) rec.data;
simdb_bitmap_print(r->bitmap); simdb_bitmap_print(r->bitmap);
free(r);
return 0; return 0;
} }
int rec_diff(simdb_t *db, uint64_t a, uint64_t b, unsigned short int showmap) int rec_diff(simdb_t *db, int a, int b, unsigned short int showmap) {
{
simdb_rec_t rec;
simdb_urec_t *r;
float diff = 0.0;
unsigned char one[SIMDB_BITMAP_SIZE];
unsigned char two[SIMDB_BITMAP_SIZE];
unsigned char map[SIMDB_BITMAP_SIZE]; unsigned char map[SIMDB_BITMAP_SIZE];
simdb_urec_t *one, *two;
float diff = 0.0;
int ret;
assert(db != NULL); assert(db != NULL);
memset(&rec, 0x0, sizeof(rec));
memset(one, 0x0, sizeof(one));
memset(two, 0x0, sizeof(two));
rec.num = a; if ((ret = simdb_read(db, a, 1, &one)) < 0) {
if (simdb_record_read(db, &rec) < 1) { fprintf(stderr, "read error: %s\n", simdb_error(ret));
return 1;
} else if (ret == 0 || (ret > 0 && !one->used)) {
fprintf(stderr, "record diff: first sample not exists\n"); fprintf(stderr, "record diff: first sample not exists\n");
if (ret > 0) free(one);
return 1; return 1;
} }
r = (simdb_urec_t *) rec.data;
memcpy(one, r->bitmap, sizeof(one));
rec.num = b; if ((ret = simdb_read(db, b, 1, &two)) < 0) {
if (simdb_record_read(db, &rec) < 1) { fprintf(stderr, "read error: %s\n", simdb_error(ret));
fprintf(stderr, "record diff: second sample not exists\n"); return 1;
} else if (ret == 0 || (ret > 0 && !two->used)) {
fprintf(stderr, "record diff: first sample not exists\n");
if (ret > 0) free(two);
return 1; return 1;
} }
r = (simdb_urec_t *) rec.data;
memcpy(two, r->bitmap, sizeof(two));
if (showmap) { if (showmap) {
simdb_bitmap_diffmap(one, two, map); simdb_bitmap_diffmap(one->bitmap, two->bitmap, map);
simdb_bitmap_print(map); simdb_bitmap_print(map);
return 0; return 0;
} }
diff = (float) simdb_bitmap_compare(one, two); diff = (float) simdb_bitmap_compare(one->bitmap, two->bitmap);
printf("%.2f%%\n", (diff / SIMDB_BITMAP_BITS) * 100); printf("%.2f%%\n", (diff / SIMDB_BITMAP_BITS) * 100);
free(one);
free(two);
return 0; return 0;
} }

18
src/simdb.h
Unescape View File

@ -110,15 +110,14 @@ const char * simdb_error(int code);
/** /**
* @brief Search compare given record in database to other images * @brief Search compare given record in database to other images
* @param db Database handle * @param db Database handle
* @param sample Record sample * @param num Record sample number
* @param search Search parameters * @param search Search parameters
* @param matches Pointer to storage for found matches * @param matches Pointer to storage for found matches
* @retval >0 if found some matches * @retval >0 if found some matches
* @retval 0 if nothing found * @retval 0 if nothing found
* @retval <0 on error * @retval <0 on error
*/ */
int simdb_search(simdb_t * const db, int simdb_search(simdb_t * const db, int num,
simdb_rec_t * const sample,
simdb_search_t * const search, simdb_search_t * const search,
simdb_match_t ** matches); simdb_match_t ** matches);
@ -162,16 +161,13 @@ int simdb_block_read(simdb_t *db, simdb_block_t *blk);
/** /**
* @brief Get database capacity * @brief Get database capacity
*/ */
uint64_t int simdb_records_count(simdb_t * const db);
simdb_records_count(simdb_t * const db);
/** /**
@brief Fills buffer 'map' according to records existense in database @brief Fills buffer 'map' according to records existense in database
@returns records processed (and also buffer size) @returns records processed (and also buffer size)
*/ */
uint64_t int simdb_usage_map(simdb_t * const db, char ** const map);
simdb_usage_map(simdb_t * const db,
char ** const map);
/** /**
* @brief Fills buffer 'map' according to records existense in given range * @brief Fills buffer 'map' according to records existense in given range
@ -181,10 +177,6 @@ simdb_usage_map(simdb_t * const db,
* @param limit Slice size * @param limit Slice size
* @returns Records processed (and also buffer size) * @returns Records processed (and also buffer size)
*/ */
uint16_t int simdb_usage_slice(simdb_t * const db, char ** const map, int offset, int limit);
simdb_usage_slice(simdb_t * const db,
char ** const map,
uint64_t offset,
uint16_t limit);
#endif #endif

Write Preview
Loading…
Cancel
Save