ad_user
/
libsimdb
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity
Browse Source

* use simdb_read() instead simdb_{record,block}_read()

master
Alex 'AdUser' Z 8 years ago
parent
245222705d
commit
17f23023e8
3 changed files with 99 additions and 105 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 121
      src/database.c
  2. 65
      src/simdb-query.c
  3. 18
      src/simdb.h

121
src/database.c
Unescape View File

@ -289,7 +289,7 @@ int simdb_block_read(simdb_t *db, simdb_block_t *blk) {
return blk->records;
}
uint64_t
int
simdb_records_count(simdb_t * const db) {
struct stat st;
off_t size = 0;
@ -313,54 +313,55 @@ simdb_record_ratio(simdb_urec_t *r) {
}
int
simdb_search(simdb_t * const db,
simdb_rec_t * const sample,
simdb_search(simdb_t * const db, int num,
simdb_search_t * const search,
simdb_match_t **matches)
{
simdb_block_t blk;
simdb_match_t match;
simdb_urec_t *rec;
const int blk_size = 4096;
simdb_urec_t *data = NULL;
simdb_urec_t *rec, sample;
const int blksize = 4096;
uint64_t found = 0;
unsigned int i = 0;
float ratio_s = 0.0; /* source */
float ratio_t = 0.0; /* tested */
int ret = 0;
assert(db != NULL);
assert(sample != NULL);
assert(search != NULL);
assert(matches != NULL);
assert(search->maxdiff_ratio >= 0.0 && search->maxdiff_ratio <= 1.0);
assert(search->maxdiff_bitmap >= 0.0 && search->maxdiff_bitmap <= 1.0);
memset(&blk, 0x0, sizeof(simdb_block_t));
memset(&match, 0x0, sizeof(simdb_match_t));
blk.start = 1;
blk.records = blk_size;
if ((ret = simdb_record_read(db, sample)) < 1)
if ((ret = simdb_read(db, num, 1, &rec)) < 1)
return ret;
memcpy(&sample, rec, sizeof(sample));
FREE(rec);
if (search->limit == 0)
search->limit = -1; /* unsigned -> max */
if (search->maxdiff_ratio > 0.0)
ratio_s = simdb_record_ratio((simdb_urec_t *) sample->data);
ratio_s = simdb_record_ratio(&sample);
CALLOC(*matches, search->limit, sizeof(simdb_match_t));
while (simdb_block_read(db, &blk) > 0) {
rec = (simdb_urec_t *) blk.data;
for (i = 0; i < blk.records; i++, rec++) {
if (rec->used == 0x0)
for (num = 1; ; num += blksize) {
ret = simdb_read(db, num, blksize, &data);
if (ret < 0)
return ret;
rec = data;
for (int i = 0; i < ret; i++, rec++) {
if (!rec->used)
continue; /* record missing */
match.diff_ratio = 0.0;
match.diff_bitmap = 0.0;
/* - compare ratio - cheap */
/* TODO: check caps */
if (ratio_s > 0.0 && (ratio_t = simdb_record_ratio(rec)) > 0.0) {
match.diff_ratio = ratio_s - ratio_t;
match.diff_ratio *= (ratio_s > ratio_t) ? 1.0 : -1.0;
@ -371,77 +372,83 @@ simdb_search(simdb_t * const db,
}
/* - compare bitmap - more expensive */
match.diff_bitmap = simdb_bitmap_compare(rec->bitmap, ((simdb_urec_t *) sample)->bitmap) / SIMDB_BITMAP_BITS;
match.diff_bitmap = simdb_bitmap_compare(rec->bitmap, sample.bitmap) / SIMDB_BITMAP_BITS;
if (match.diff_bitmap > search->maxdiff_bitmap)
continue;
/* create match record */
match.num = blk.start + i;
match.num = num + i;
memcpy(&(*matches)[found], &match, sizeof(simdb_match_t));
found++;
if (found >= search->limit)
break;
}
FREE(blk.data);
FREE(data);
if (found >= search->limit)
break;
blk.start += blk_size;
}
return found;
}
uint64_t
simdb_usage_map(simdb_t * const db,
char ** const map) {
const int blk_size = 4096;
simdb_block_t blk;
uint64_t records;
simdb_urec_t *r; /* mnemonics : block, record */
char *m = NULL; /* mnemonics : map */
int
simdb_usage_map(simdb_t * const db, char ** const map) {
const int blksize = 4096;
simdb_urec_t *data = NULL;
simdb_urec_t *r; /* mnemonics : record pointer */
char *m = NULL; /* mnemonics : map pointer */
int ret = 0, records = 0;
memset(&blk, 0x0, sizeof(simdb_block_t));
assert(db != NULL);
assert(map != NULL);
records = simdb_records_count(db);
CALLOC(*map, records + 1, sizeof(char));
m = *map;
blk.start = 1;
blk.records = blk_size;
while (simdb_block_read(db, &blk) > 0) {
r = (simdb_urec_t *) blk.data;
for (unsigned int i = 0; i < blk.records; i++, m++, r++) {
CALLOC(m, records + 1, sizeof(char));
*map = m;
for (int num = 1; ; num += blksize) {
ret = simdb_read(db, num, blksize, &data);
if (ret < 0) {
FREE(*map);
return ret;
}
if (ret == 0)
break;
r = data;
for (int i = 0; i < ret; i++, m++, r++) {
*m = (r->used == 0xFF) ? CHAR_USED : CHAR_NONE;
}
blk.start += blk_size;
FREE(data);
}
FREE(blk.data);
return records;
}
uint16_t
simdb_usage_slice(simdb_t * const db,
char ** const map,
uint64_t offset,
uint16_t limit) {
simdb_block_t blk;
simdb_urec_t *r; /* mnemonics : block, record */
char *m = NULL; /* mnemonics : map */
int
simdb_usage_slice(simdb_t * const db, char ** const map, int offset, int limit) {
simdb_urec_t *data = NULL;
simdb_urec_t *r; /* mnemonics : record pointer */
char *m = NULL; /* mnemonics : map pointer */
int ret = 0;
assert(db != NULL);
assert(map != NULL);
if (offset < 1 || limit < 1)
return SIMDB_ERR_USAGE;
memset(&blk, 0x0, sizeof(simdb_block_t));
CALLOC(*map, limit + 1, sizeof(char));
ret = simdb_read(db, offset, limit, &data);
if (ret <= 0)
return ret;
CALLOC(m, limit + 1, sizeof(char));
m = *map;
blk.start = offset;
blk.records = limit;
limit = simdb_block_read(db, &blk);
r = (simdb_urec_t *) blk.data;
for (uint16_t i = 0; i < blk.records; i++, m++, r++) {
r = data;
for (int i = 0; i < limit; i++, m++, r++) {
*m = (r->used == 0xFF) ? CHAR_USED : CHAR_NONE;
}
FREE(data);
return limit;
return ret;
}

65
src/simdb-query.c
Unescape View File

@ -16,8 +16,9 @@
#include "common.h"
#include "bitmap.h"
#include "simdb.h"
#include "record.h"
#include "io.h"
#include "simdb.h"
#include <unistd.h>
#include <getopt.h>
@ -41,20 +42,16 @@ void usage(int exitcode) {
exit(exitcode);
}
int search_similar(simdb_t *db, uint64_t number, float maxdiff)
{
int search_similar(simdb_t *db, int num, float maxdiff) {
int ret = 0, i = 0;
simdb_rec_t sample;
simdb_match_t *matches = NULL;
simdb_search_t search;
memset(&sample, 0x0, sizeof(simdb_rec_t));
memset(&search, 0x0, sizeof(simdb_search_t));
search.maxdiff_ratio = 0.2; /* 20% */
search.maxdiff_bitmap = maxdiff;
sample.num = number;
if ((ret = simdb_search(db, &sample, &search, &matches)) < 0) {
if ((ret = simdb_search(db, num, &search, &matches)) < 0) {
fprintf(stderr, "%s\n", simdb_error(ret));
return 1;
}
@ -119,64 +116,62 @@ int db_usage_slice(simdb_t *db, uint64_t offset, uint16_t limit)
return 0;
}
int rec_bitmap(simdb_t *db, uint64_t number)
{
simdb_rec_t rec;
int rec_bitmap(simdb_t *db, int num) {
simdb_urec_t *r;
int ret;
assert(db != NULL);
memset(&rec, 0x0, sizeof(simdb_rec_t));
rec.num = number;
if (simdb_record_read(db, &rec) < 1) {
if ((ret = simdb_read(db, num, 1, &r)) < 0) {
fprintf(stderr, "read failed: %s\n", simdb_error(ret));
return 1;
} else if (ret == 0) {
fprintf(stderr, "bitmap: %s\n", "sample not found");
return 1;
}
r = (simdb_urec_t *) rec.data;
simdb_bitmap_print(r->bitmap);
free(r);
return 0;
}
int rec_diff(simdb_t *db, uint64_t a, uint64_t b, unsigned short int showmap)
{
simdb_rec_t rec;
simdb_urec_t *r;
float diff = 0.0;
unsigned char one[SIMDB_BITMAP_SIZE];
unsigned char two[SIMDB_BITMAP_SIZE];
int rec_diff(simdb_t *db, int a, int b, unsigned short int showmap) {
unsigned char map[SIMDB_BITMAP_SIZE];
simdb_urec_t *one, *two;
float diff = 0.0;
int ret;
assert(db != NULL);
memset(&rec, 0x0, sizeof(rec));
memset(one, 0x0, sizeof(one));
memset(two, 0x0, sizeof(two));
rec.num = a;
if (simdb_record_read(db, &rec) < 1) {
if ((ret = simdb_read(db, a, 1, &one)) < 0) {
fprintf(stderr, "read error: %s\n", simdb_error(ret));
return 1;
} else if (ret == 0 || (ret > 0 && !one->used)) {
fprintf(stderr, "record diff: first sample not exists\n");
if (ret > 0) free(one);
return 1;
}
r = (simdb_urec_t *) rec.data;
memcpy(one, r->bitmap, sizeof(one));
rec.num = b;
if (simdb_record_read(db, &rec) < 1) {
fprintf(stderr, "record diff: second sample not exists\n");
if ((ret = simdb_read(db, b, 1, &two)) < 0) {
fprintf(stderr, "read error: %s\n", simdb_error(ret));
return 1;
} else if (ret == 0 || (ret > 0 && !two->used)) {
fprintf(stderr, "record diff: first sample not exists\n");
if (ret > 0) free(two);
return 1;
}
r = (simdb_urec_t *) rec.data;
memcpy(two, r->bitmap, sizeof(two));
if (showmap) {
simdb_bitmap_diffmap(one, two, map);
simdb_bitmap_diffmap(one->bitmap, two->bitmap, map);
simdb_bitmap_print(map);
return 0;
}
diff = (float) simdb_bitmap_compare(one, two);
diff = (float) simdb_bitmap_compare(one->bitmap, two->bitmap);
printf("%.2f%%\n", (diff / SIMDB_BITMAP_BITS) * 100);
free(one);
free(two);
return 0;
}

18
src/simdb.h
Unescape View File

@ -110,15 +110,14 @@ const char * simdb_error(int code);
/**
* @brief Search compare given record in database to other images
* @param db Database handle
* @param sample Record sample
* @param num Record sample number
* @param search Search parameters
* @param matches Pointer to storage for found matches
* @retval >0 if found some matches
* @retval 0 if nothing found
* @retval <0 on error
*/
int simdb_search(simdb_t * const db,
simdb_rec_t * const sample,
int simdb_search(simdb_t * const db, int num,
simdb_search_t * const search,
simdb_match_t ** matches);
@ -162,16 +161,13 @@ int simdb_block_read(simdb_t *db, simdb_block_t *blk);
/**
* @brief Get database capacity
*/
uint64_t
simdb_records_count(simdb_t * const db);
int simdb_records_count(simdb_t * const db);
/**
@brief Fills buffer 'map' according to records existense in database
@returns records processed (and also buffer size)
*/
uint64_t
simdb_usage_map(simdb_t * const db,
char ** const map);
int simdb_usage_map(simdb_t * const db, char ** const map);
/**
* @brief Fills buffer 'map' according to records existense in given range
@ -181,10 +177,6 @@ simdb_usage_map(simdb_t * const db,
* @param limit Slice size
* @returns Records processed (and also buffer size)
*/
uint16_t
simdb_usage_slice(simdb_t * const db,
char ** const map,
uint64_t offset,
uint16_t limit);
int simdb_usage_slice(simdb_t * const db, char ** const map, int offset, int limit);
#endif

Write Preview
Loading…
Cancel
Save