diff options
Diffstat (limited to 'src/datacache')
-rw-r--r-- | src/datacache/datacache.c | 24 | ||||
-rw-r--r-- | src/datacache/perf_datacache.c | 18 | ||||
-rw-r--r-- | src/datacache/plugin_datacache_heap.c | 164 | ||||
-rw-r--r-- | src/datacache/plugin_datacache_postgres.c | 99 | ||||
-rw-r--r-- | src/datacache/plugin_datacache_sqlite.c | 112 | ||||
-rw-r--r-- | src/datacache/plugin_datacache_template.c | 20 | ||||
-rw-r--r-- | src/datacache/test_datacache.c | 101 | ||||
-rw-r--r-- | src/datacache/test_datacache_quota.c | 80 |
8 files changed, 423 insertions, 195 deletions
diff --git a/src/datacache/datacache.c b/src/datacache/datacache.c index 0646019bd7..2a83e07a72 100644 --- a/src/datacache/datacache.c +++ b/src/datacache/datacache.c @@ -2,20 +2,18 @@ This file is part of GNUnet Copyright (C) 2004, 2005, 2006, 2007, 2009, 2010, 2015 GNUnet e.V. - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3, or (at your - option) any later version. + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - Boston, MA 02110-1301, USA. + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. */ /** * @file datacache/datacache.c @@ -260,6 +258,7 @@ GNUNET_DATACACHE_destroy (struct GNUNET_DATACACHE_Handle *h) * * @param h handle to the datacache * @param key key to store data under + * @param xor_distance distance of @a key to our PID * @param data_size number of bytes in @a data * @param data data to store * @param type type of the value @@ -271,6 +270,7 @@ GNUNET_DATACACHE_destroy (struct GNUNET_DATACACHE_Handle *h) int GNUNET_DATACACHE_put (struct GNUNET_DATACACHE_Handle *h, const struct GNUNET_HashCode *key, + uint32_t xor_distance, size_t data_size, const char *data, enum GNUNET_BLOCK_Type type, @@ -282,6 +282,7 @@ GNUNET_DATACACHE_put (struct GNUNET_DATACACHE_Handle *h, used = h->api->put (h->api->cls, key, + xor_distance, data_size, data, type, @@ -313,7 +314,8 @@ GNUNET_DATACACHE_put (struct GNUNET_DATACACHE_Handle *h, 1, GNUNET_NO); while (h->utilization + used > h->env.quota) - GNUNET_assert (GNUNET_OK == h->api->del (h->api->cls)); + GNUNET_assert (GNUNET_OK == + h->api->del (h->api->cls)); h->utilization += used; return GNUNET_OK; } diff --git a/src/datacache/perf_datacache.c b/src/datacache/perf_datacache.c index b01b9f9c49..7f699f2af4 100644 --- a/src/datacache/perf_datacache.c +++ b/src/datacache/perf_datacache.c @@ -2,20 +2,18 @@ This file is part of GNUnet. Copyright (C) 2006, 2009, 2010 GNUnet e.V. - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3, or (at your - option) any later version. + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - Boston, MA 02110-1301, USA. + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. */ /* * @file datacache/perf_datacache.c diff --git a/src/datacache/plugin_datacache_heap.c b/src/datacache/plugin_datacache_heap.c index 49e60bca1d..494d1ae17c 100644 --- a/src/datacache/plugin_datacache_heap.c +++ b/src/datacache/plugin_datacache_heap.c @@ -2,20 +2,18 @@ This file is part of GNUnet Copyright (C) 2012, 2015 GNUnet e.V. - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3, or (at your - option) any later version. + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - Boston, MA 02110-1301, USA. + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. */ /** @@ -31,7 +29,7 @@ #define LOG_STRERROR_FILE(kind,op,fn) GNUNET_log_from_strerror_file (kind, "datacache-heap", op, fn) - +#define NUM_HEAPS 24 /** * Context for all functions in this plugin. @@ -49,9 +47,9 @@ struct Plugin struct GNUNET_CONTAINER_MultiHashMap *map; /** - * Heap for expirations. + * Heaps sorted by distance. */ - struct GNUNET_CONTAINER_Heap *heap; + struct GNUNET_CONTAINER_Heap *heaps[NUM_HEAPS]; }; @@ -92,6 +90,11 @@ struct Value unsigned int path_info_len; /** + * How close is the hash to us? Determines which heap we are in! + */ + uint32_t distance; + + /** * Type of the block. */ enum GNUNET_BLOCK_Type type; @@ -118,11 +121,6 @@ struct PutContext const char *data; /** - * Heap from the plugin. - */ - struct GNUNET_CONTAINER_Heap *heap; - - /** * Path information. */ const struct GNUNET_PeerIdentity *path_info; @@ -168,7 +166,9 @@ put_cb (void *cls, if ( (val->size == put_ctx->size) && (val->type == put_ctx->type) && - (0 == memcmp (&val[1], put_ctx->data, put_ctx->size)) ) + (0 == memcmp (&val[1], + put_ctx->data, + put_ctx->size)) ) { put_ctx->found = GNUNET_YES; val->discard_time = GNUNET_TIME_absolute_max (val->discard_time, @@ -178,8 +178,8 @@ put_cb (void *cls, val->path_info_len, put_ctx->path_info_len); GNUNET_memcpy (val->path_info, - put_ctx->path_info, - put_ctx->path_info_len * sizeof (struct GNUNET_PeerIdentity)); + put_ctx->path_info, + put_ctx->path_info_len * sizeof (struct GNUNET_PeerIdentity)); GNUNET_CONTAINER_heap_update_cost (val->hn, val->discard_time.abs_value_us); GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, @@ -199,6 +199,7 @@ put_cb (void *cls, * * @param cls closure (our `struct Plugin`) * @param key key to store data under + * @param xor_distance how close is @a key to our PID? * @param size number of bytes in @a data * @param data data to store * @param type type of the value @@ -210,6 +211,7 @@ put_cb (void *cls, static ssize_t heap_plugin_put (void *cls, const struct GNUNET_HashCode *key, + uint32_t xor_distance, size_t size, const char *data, enum GNUNET_BLOCK_Type type, @@ -222,7 +224,6 @@ heap_plugin_put (void *cls, struct PutContext put_ctx; put_ctx.found = GNUNET_NO; - put_ctx.heap = plugin->heap; put_ctx.data = data; put_ctx.size = size; put_ctx.path_info = path_info; @@ -236,22 +237,28 @@ heap_plugin_put (void *cls, if (GNUNET_YES == put_ctx.found) return 0; val = GNUNET_malloc (sizeof (struct Value) + size); - GNUNET_memcpy (&val[1], data, size); + GNUNET_memcpy (&val[1], + data, + size); val->key = *key; val->type = type; val->discard_time = discard_time; val->size = size; + if (xor_distance >= NUM_HEAPS) + val->distance = NUM_HEAPS - 1; + else + val->distance = xor_distance; GNUNET_array_grow (val->path_info, val->path_info_len, path_info_len); GNUNET_memcpy (val->path_info, - path_info, - path_info_len * sizeof (struct GNUNET_PeerIdentity)); + path_info, + path_info_len * sizeof (struct GNUNET_PeerIdentity)); (void) GNUNET_CONTAINER_multihashmap_put (plugin->map, &val->key, val, GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); - val->hn = GNUNET_CONTAINER_heap_insert (plugin->heap, + val->hn = GNUNET_CONTAINER_heap_insert (plugin->heaps[val->distance], val, val->discard_time.abs_value_us); return size + OVERHEAD; @@ -307,6 +314,9 @@ get_cb (void *cls, if ( (get_ctx->type != val->type) && (GNUNET_BLOCK_TYPE_ANY != get_ctx->type) ) return GNUNET_OK; + if (0 == + GNUNET_TIME_absolute_get_remaining (val->discard_time).rel_value_us) + return GNUNET_OK; if (NULL != get_ctx->iter) ret = get_ctx->iter (get_ctx->iter_cls, key, @@ -369,7 +379,12 @@ heap_plugin_del (void *cls) struct Plugin *plugin = cls; struct Value *val; - val = GNUNET_CONTAINER_heap_remove_root (plugin->heap); + for (unsigned int i=0;i<NUM_HEAPS;i++) + { + val = GNUNET_CONTAINER_heap_remove_root (plugin->heaps[i]); + if (NULL != val) + break; + } if (NULL == val) return GNUNET_SYSERR; GNUNET_assert (GNUNET_YES == @@ -413,6 +428,53 @@ heap_plugin_get_random (void *cls, /** + * Closure for #find_closest(). + */ +struct GetClosestContext +{ + struct Value **values; + + unsigned int num_results; + + const struct GNUNET_HashCode *key; +}; + + +static int +find_closest (void *cls, + const struct GNUNET_HashCode *key, + void *value) +{ + struct GetClosestContext *gcc = cls; + struct Value *val = value; + unsigned int j; + + if (1 != GNUNET_CRYPTO_hash_cmp (key, + gcc->key)) + return GNUNET_OK; /* useless */ + j = gcc->num_results; + for (unsigned int i=0;i<gcc->num_results;i++) + { + if (NULL == gcc->values[i]) + { + j = i; + break; + } + if (1 == GNUNET_CRYPTO_hash_cmp (&gcc->values[i]->key, + key)) + { + j = i; + break; + } + } + if (j == gcc->num_results) + return GNUNET_OK; + gcc->values[j] = val; + return GNUNET_OK; +} + + +/** * Iterate over the results that are "close" to a particular key in * the datacache. "close" is defined as numerically larger than @a * key (when interpreted as a circular address space), with small @@ -432,8 +494,30 @@ heap_plugin_get_closest (void *cls, GNUNET_DATACACHE_Iterator iter, void *iter_cls) { - GNUNET_break (0); // not implemented! - return 0; + struct Plugin *plugin = cls; + struct Value *values[num_results]; + struct GetClosestContext gcc = { + .values = values, + .num_results = num_results, + .key = key + }; + GNUNET_CONTAINER_multihashmap_iterate (plugin->map, + &find_closest, + &gcc); + for (unsigned int i=0;i<num_results;i++) + { + if (NULL == values[i]) + return i; + iter (iter_cls, + &values[i]->key, + values[i]->size, + (void *) &values[i][1], + values[i]->type, + values[i]->discard_time, + values[i]->path_info_len, + values[i]->path_info); + } + return num_results; } @@ -453,7 +537,8 @@ libgnunet_plugin_datacache_heap_init (void *cls) plugin = GNUNET_new (struct Plugin); plugin->map = GNUNET_CONTAINER_multihashmap_create (1024, /* FIXME: base on quota! */ GNUNET_YES); - plugin->heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); + for (unsigned int i=0;i<NUM_HEAPS;i++) + plugin->heaps[i] = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); plugin->env = env; api = GNUNET_new (struct GNUNET_DATACACHE_PluginFunctions); api->cls = plugin; @@ -481,16 +566,19 @@ libgnunet_plugin_datacache_heap_done (void *cls) struct Plugin *plugin = api->cls; struct Value *val; - while (NULL != (val = GNUNET_CONTAINER_heap_remove_root (plugin->heap))) + for (unsigned int i=0;i<NUM_HEAPS;i++) { - GNUNET_assert (GNUNET_YES == - GNUNET_CONTAINER_multihashmap_remove (plugin->map, - &val->key, - val)); - GNUNET_free_non_null (val->path_info); - GNUNET_free (val); + while (NULL != (val = GNUNET_CONTAINER_heap_remove_root (plugin->heaps[i]))) + { + GNUNET_assert (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (plugin->map, + &val->key, + val)); + GNUNET_free_non_null (val->path_info); + GNUNET_free (val); + } + GNUNET_CONTAINER_heap_destroy (plugin->heaps[i]); } - GNUNET_CONTAINER_heap_destroy (plugin->heap); GNUNET_CONTAINER_multihashmap_destroy (plugin->map); GNUNET_free (plugin); GNUNET_free (api); diff --git a/src/datacache/plugin_datacache_postgres.c b/src/datacache/plugin_datacache_postgres.c index 2c233c4c21..ea87acc1f0 100644 --- a/src/datacache/plugin_datacache_postgres.c +++ b/src/datacache/plugin_datacache_postgres.c @@ -1,21 +1,19 @@ /* This file is part of GNUnet - Copyright (C) 2006, 2009, 2010, 2012, 2015, 2017 GNUnet e.V. + Copyright (C) 2006, 2009, 2010, 2012, 2015, 2017, 2018 GNUnet e.V. - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3, or (at your - option) any later version. + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - Boston, MA 02110-1301, USA. + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. */ /** @@ -67,47 +65,54 @@ static int init_connection (struct Plugin *plugin) { struct GNUNET_PQ_ExecuteStatement es[] = { - GNUNET_PQ_make_execute ("CREATE TEMPORARY TABLE IF NOT EXISTS gn090dc (" + GNUNET_PQ_make_execute ("CREATE TEMPORARY TABLE IF NOT EXISTS gn011dc (" " type INTEGER NOT NULL," + " prox INTEGER NOT NULL," " discard_time BIGINT NOT NULL," " key BYTEA NOT NULL," " value BYTEA NOT NULL," " path BYTEA DEFAULT NULL)" "WITH OIDS"), - GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS idx_key ON gn090dc (key)"), - GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS idx_dt ON gn090dc (discard_time)"), - GNUNET_PQ_make_execute ("ALTER TABLE gn090dc ALTER value SET STORAGE EXTERNAL"), - GNUNET_PQ_make_execute ("ALTER TABLE gn090dc ALTER key SET STORAGE PLAIN"), + GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS idx_key ON gn011dc (key)"), + GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS idx_dt ON gn011dc (discard_time)"), + GNUNET_PQ_make_execute ("ALTER TABLE gn011dc ALTER value SET STORAGE EXTERNAL"), + GNUNET_PQ_make_execute ("ALTER TABLE gn011dc ALTER key SET STORAGE PLAIN"), GNUNET_PQ_EXECUTE_STATEMENT_END }; struct GNUNET_PQ_PreparedStatement ps[] = { GNUNET_PQ_make_prepare ("getkt", - "SELECT discard_time,type,value,path FROM gn090dc " - "WHERE key=$1 AND type=$2", - 2), + "SELECT discard_time,type,value,path FROM gn011dc " + "WHERE key=$1 AND type=$2 AND discard_time >= $3", + 3), GNUNET_PQ_make_prepare ("getk", - "SELECT discard_time,type,value,path FROM gn090dc " - "WHERE key=$1", + "SELECT discard_time,type,value,path FROM gn011dc " + "WHERE key=$1 AND discard_time >= $2", + 2), + GNUNET_PQ_make_prepare ("getex", + "SELECT length(value) AS len,oid,key FROM gn011dc" + " WHERE discard_time < $1" + " ORDER BY discard_time ASC LIMIT 1", 1), GNUNET_PQ_make_prepare ("getm", - "SELECT length(value) AS len,oid,key FROM gn090dc " - "ORDER BY discard_time ASC LIMIT 1", + "SELECT length(value) AS len,oid,key FROM gn011dc" + " ORDER BY prox ASC, discard_time ASC LIMIT 1", 0), GNUNET_PQ_make_prepare ("get_random", - "SELECT discard_time,type,value,path,key FROM gn090dc " - "ORDER BY key ASC LIMIT 1 OFFSET $1", - 1), + "SELECT discard_time,type,value,path,key FROM gn011dc" + " WHERE discard_time >= $1" + " ORDER BY key ASC LIMIT 1 OFFSET $2", + 2), GNUNET_PQ_make_prepare ("get_closest", - "SELECT discard_time,type,value,path,key FROM gn090dc " - "WHERE key>=$1 ORDER BY key ASC LIMIT $2", - 1), + "SELECT discard_time,type,value,path,key FROM gn011dc " + "WHERE key>=$1 AND discard_time >= $2 ORDER BY key ASC LIMIT $3", + 3), GNUNET_PQ_make_prepare ("delrow", - "DELETE FROM gn090dc WHERE oid=$1", + "DELETE FROM gn011dc WHERE oid=$1", 1), GNUNET_PQ_make_prepare ("put", - "INSERT INTO gn090dc (type, discard_time, key, value, path) " - "VALUES ($1, $2, $3, $4, $5)", - 5), + "INSERT INTO gn011dc (type, prox, discard_time, key, value, path) " + "VALUES ($1, $2, $3, $4, $5, $6)", + 6), GNUNET_PQ_PREPARED_STATEMENT_END }; @@ -141,6 +146,7 @@ init_connection (struct Plugin *plugin) * * @param cls closure (our `struct Plugin`) * @param key key to store @a data under + * @param prox proximity of @a key to my PID * @param data_size number of bytes in @a data * @param data data to store * @param type type of the value @@ -152,6 +158,7 @@ init_connection (struct Plugin *plugin) static ssize_t postgres_plugin_put (void *cls, const struct GNUNET_HashCode *key, + uint32_t prox, size_t data_size, const char *data, enum GNUNET_BLOCK_Type type, @@ -163,6 +170,7 @@ postgres_plugin_put (void *cls, uint32_t type32 = (uint32_t) type; struct GNUNET_PQ_QueryParam params[] = { GNUNET_PQ_query_param_uint32 (&type32), + GNUNET_PQ_query_param_uint32 (&prox), GNUNET_PQ_query_param_absolute_time (&discard_time), GNUNET_PQ_query_param_auto_from_type (key), GNUNET_PQ_query_param_fixed_size (data, data_size), @@ -302,18 +310,22 @@ postgres_plugin_get (void *cls, { struct Plugin *plugin = cls; uint32_t type32 = (uint32_t) type; + struct GNUNET_TIME_Absolute now; struct GNUNET_PQ_QueryParam paramk[] = { GNUNET_PQ_query_param_auto_from_type (key), + GNUNET_PQ_query_param_absolute_time (&now), GNUNET_PQ_query_param_end }; struct GNUNET_PQ_QueryParam paramkt[] = { GNUNET_PQ_query_param_auto_from_type (key), GNUNET_PQ_query_param_uint32 (&type32), + GNUNET_PQ_query_param_absolute_time (&now), GNUNET_PQ_query_param_end }; enum GNUNET_DB_QueryStatus res; struct HandleResultContext hr_ctx; + now = GNUNET_TIME_absolute_get (); hr_ctx.iter = iter; hr_ctx.iter_cls = iter_cls; hr_ctx.key = key; @@ -359,11 +371,22 @@ postgres_plugin_del (void *cls) GNUNET_PQ_query_param_uint32 (&oid), GNUNET_PQ_query_param_end }; + struct GNUNET_TIME_Absolute now; + struct GNUNET_PQ_QueryParam xparam[] = { + GNUNET_PQ_query_param_absolute_time (&now), + GNUNET_PQ_query_param_end + }; + now = GNUNET_TIME_absolute_get (); res = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh, - "getm", - pempty, + "getex", + xparam, rs); + if (0 >= res) + res = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh, + "getm", + pempty, + rs); if (0 > res) return GNUNET_SYSERR; if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == res) @@ -405,6 +428,7 @@ postgres_plugin_get_random (void *cls, { struct Plugin *plugin = cls; uint32_t off; + struct GNUNET_TIME_Absolute now; struct GNUNET_TIME_Absolute expiration_time; size_t data_size; void *data; @@ -414,6 +438,7 @@ postgres_plugin_get_random (void *cls, uint32_t type; enum GNUNET_DB_QueryStatus res; struct GNUNET_PQ_QueryParam params[] = { + GNUNET_PQ_query_param_absolute_time (&now), GNUNET_PQ_query_param_uint32 (&off), GNUNET_PQ_query_param_end }; @@ -437,6 +462,7 @@ postgres_plugin_get_random (void *cls, return 0; if (NULL == iter) return 1; + now = GNUNET_TIME_absolute_get (); off = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE, plugin->num_items); res = GNUNET_PQ_eval_prepared_singleton_select (plugin->dbh, @@ -598,8 +624,10 @@ postgres_plugin_get_closest (void *cls, { struct Plugin *plugin = cls; uint32_t num_results32 = (uint32_t) num_results; + struct GNUNET_TIME_Absolute now; struct GNUNET_PQ_QueryParam params[] = { GNUNET_PQ_query_param_auto_from_type (key), + GNUNET_PQ_query_param_absolute_time (&now), GNUNET_PQ_query_param_uint32 (&num_results32), GNUNET_PQ_query_param_end }; @@ -608,6 +636,7 @@ postgres_plugin_get_closest (void *cls, erc.iter = iter; erc.iter_cls = iter_cls; + now = GNUNET_TIME_absolute_get (); res = GNUNET_PQ_eval_prepared_multi_select (plugin->dbh, "get_closest", params, diff --git a/src/datacache/plugin_datacache_sqlite.c b/src/datacache/plugin_datacache_sqlite.c index 15438b29b8..4684e514c7 100644 --- a/src/datacache/plugin_datacache_sqlite.c +++ b/src/datacache/plugin_datacache_sqlite.c @@ -2,20 +2,18 @@ This file is part of GNUnet Copyright (C) 2006, 2009, 2015 GNUnet e.V. - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3, or (at your - option) any later version. + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - Boston, MA 02110-1301, USA. + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. */ /** @@ -38,7 +36,7 @@ * How much overhead do we assume per entry in the * datacache? */ -#define OVERHEAD (sizeof(struct GNUNET_HashCode) + 32) +#define OVERHEAD (sizeof(struct GNUNET_HashCode) + 36) /** * Context for all functions in this plugin. @@ -83,6 +81,11 @@ struct Plugin /** * Prepared statement for #sqlite_plugin_del. */ + sqlite3_stmt *del_expired_stmt; + + /** + * Prepared statement for #sqlite_plugin_del. + */ sqlite3_stmt *del_stmt; /** @@ -150,6 +153,7 @@ sq_prepare (sqlite3 *dbh, * * @param cls closure (our `struct Plugin`) * @param key key to store @a data under + * @param xor_distance how close is @a key to our PID? * @param size number of bytes in @a data * @param data data to store * @param type type of the value @@ -161,6 +165,7 @@ sq_prepare (sqlite3 *dbh, static ssize_t sqlite_plugin_put (void *cls, const struct GNUNET_HashCode *key, + uint32_t xor_distance, size_t size, const char *data, enum GNUNET_BLOCK_Type type, @@ -174,6 +179,7 @@ sqlite_plugin_put (void *cls, GNUNET_SQ_query_param_uint32 (&type32), GNUNET_SQ_query_param_absolute_time (&discard_time), GNUNET_SQ_query_param_auto_from_type (key), + GNUNET_SQ_query_param_uint32 (&xor_distance), GNUNET_SQ_query_param_fixed_size (data, size), GNUNET_SQ_query_param_fixed_size (path_info, path_info_len * sizeof (struct GNUNET_PeerIdentity)), @@ -387,6 +393,7 @@ sqlite_plugin_del (void *cls) void *data; size_t dsize; struct GNUNET_HashCode hc; + struct GNUNET_TIME_Absolute now; struct GNUNET_SQ_ResultSpec rs[] = { GNUNET_SQ_result_spec_uint64 (&rowid), GNUNET_SQ_result_spec_auto_from_type (&hc), @@ -398,27 +405,52 @@ sqlite_plugin_del (void *cls) GNUNET_SQ_query_param_uint64 (&rowid), GNUNET_SQ_query_param_end }; + struct GNUNET_SQ_QueryParam time_params[] = { + GNUNET_SQ_query_param_absolute_time (&now), + GNUNET_SQ_query_param_end + }; LOG (GNUNET_ERROR_TYPE_DEBUG, "Processing DEL\n"); - if (SQLITE_ROW != - sqlite3_step (plugin->del_select_stmt)) + now = GNUNET_TIME_absolute_get (); + if (GNUNET_OK != + GNUNET_SQ_bind (plugin->del_expired_stmt, + time_params)) { LOG_SQLITE (plugin->dbh, GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, - "sqlite3_step"); + "sqlite3_bind"); GNUNET_SQ_reset (plugin->dbh, - plugin->del_select_stmt); + plugin->del_expired_stmt); return GNUNET_SYSERR; } - if (GNUNET_OK != - GNUNET_SQ_extract_result (plugin->del_select_stmt, - rs)) + if ( (SQLITE_ROW != + sqlite3_step (plugin->del_expired_stmt)) || + (GNUNET_OK != + GNUNET_SQ_extract_result (plugin->del_expired_stmt, + rs)) ) { - GNUNET_break (0); GNUNET_SQ_reset (plugin->dbh, - plugin->del_select_stmt); - return GNUNET_SYSERR; + plugin->del_expired_stmt); + if (SQLITE_ROW != + sqlite3_step (plugin->del_select_stmt)) + { + LOG_SQLITE (plugin->dbh, + GNUNET_ERROR_TYPE_ERROR | GNUNET_ERROR_TYPE_BULK, + "sqlite3_step"); + GNUNET_SQ_reset (plugin->dbh, + plugin->del_select_stmt); + return GNUNET_SYSERR; + } + if (GNUNET_OK != + GNUNET_SQ_extract_result (plugin->del_select_stmt, + rs)) + { + GNUNET_SQ_reset (plugin->dbh, + plugin->del_select_stmt); + GNUNET_break (0); + return GNUNET_SYSERR; + } } GNUNET_SQ_cleanup_result (rs); GNUNET_SQ_reset (plugin->dbh, @@ -709,13 +741,15 @@ libgnunet_plugin_datacache_sqlite_init (void *cls) SQLITE3_EXEC (dbh, "PRAGMA sqlite_temp_store=3"); SQLITE3_EXEC (dbh, - "CREATE TABLE ds090 (" " type INTEGER NOT NULL DEFAULT 0," - " expire INTEGER NOT NULL DEFAULT 0," + "CREATE TABLE ds091 (" + " type INTEGER NOT NULL DEFAULT 0," + " expire INTEGER NOT NULL," " key BLOB NOT NULL DEFAULT ''," - " value BLOB NOT NULL DEFAULT ''," + " prox INTEGER NOT NULL," + " value BLOB NOT NULL," " path BLOB DEFAULT '')"); - SQLITE3_EXEC (dbh, "CREATE INDEX idx_hashidx ON ds090 (key,type,expire)"); - SQLITE3_EXEC (dbh, "CREATE INDEX idx_expire ON ds090 (expire)"); + SQLITE3_EXEC (dbh, "CREATE INDEX idx_hashidx ON ds091 (key,type,expire)"); + SQLITE3_EXEC (dbh, "CREATE INDEX idx_expire ON ds091 (prox,expire)"); plugin = GNUNET_new (struct Plugin); plugin->env = env; plugin->dbh = dbh; @@ -723,35 +757,42 @@ libgnunet_plugin_datacache_sqlite_init (void *cls) if ( (SQLITE_OK != sq_prepare (plugin->dbh, - "INSERT INTO ds090 (type, expire, key, value, path) " - "VALUES (?, ?, ?, ?, ?)", + "INSERT INTO ds091 (type, expire, key, prox, value, path) " + "VALUES (?, ?, ?, ?, ?, ?)", &plugin->insert_stmt)) || (SQLITE_OK != sq_prepare (plugin->dbh, - "SELECT count(*) FROM ds090 " + "SELECT count(*) FROM ds091 " "WHERE key=? AND type=? AND expire >= ?", &plugin->get_count_stmt)) || (SQLITE_OK != sq_prepare (plugin->dbh, - "SELECT value,expire,path FROM ds090 " - "WHERE key=? AND type=? AND expire >= ? LIMIT 1 OFFSET ?", + "SELECT value,expire,path FROM ds091" + " WHERE key=? AND type=? AND expire >= ? LIMIT 1 OFFSET ?", &plugin->get_stmt)) || (SQLITE_OK != sq_prepare (plugin->dbh, - "SELECT _ROWID_,key,value FROM ds090 ORDER BY expire ASC LIMIT 1", + "SELECT _ROWID_,key,value FROM ds091" + " WHERE expire < ?" + " ORDER BY expire ASC LIMIT 1", + &plugin->del_expired_stmt)) || + (SQLITE_OK != + sq_prepare (plugin->dbh, + "SELECT _ROWID_,key,value FROM ds091" + " ORDER BY prox ASC, expire ASC LIMIT 1", &plugin->del_select_stmt)) || (SQLITE_OK != sq_prepare (plugin->dbh, - "DELETE FROM ds090 WHERE _ROWID_=?", + "DELETE FROM ds091 WHERE _ROWID_=?", &plugin->del_stmt)) || (SQLITE_OK != sq_prepare (plugin->dbh, - "SELECT value,expire,path,key,type FROM ds090 " + "SELECT value,expire,path,key,type FROM ds091 " "ORDER BY key LIMIT 1 OFFSET ?", &plugin->get_random_stmt)) || (SQLITE_OK != sq_prepare (plugin->dbh, - "SELECT value,expire,path,type,key FROM ds090 " + "SELECT value,expire,path,type,key FROM ds091 " "WHERE key>=? AND expire >= ? ORDER BY KEY ASC LIMIT ?", &plugin->get_closest_stmt)) ) @@ -807,6 +848,7 @@ libgnunet_plugin_datacache_sqlite_done (void *cls) sqlite3_finalize (plugin->get_count_stmt); sqlite3_finalize (plugin->get_stmt); sqlite3_finalize (plugin->del_select_stmt); + sqlite3_finalize (plugin->del_expired_stmt); sqlite3_finalize (plugin->del_stmt); sqlite3_finalize (plugin->get_random_stmt); sqlite3_finalize (plugin->get_closest_stmt); diff --git a/src/datacache/plugin_datacache_template.c b/src/datacache/plugin_datacache_template.c index b9baa64d34..25f5957d81 100644 --- a/src/datacache/plugin_datacache_template.c +++ b/src/datacache/plugin_datacache_template.c @@ -2,20 +2,18 @@ This file is part of GNUnet Copyright (C) 2006, 2009, 2015 GNUnet e.V. - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3, or (at your - option) any later version. + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - Boston, MA 02110-1301, USA. + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. */ /** @@ -45,6 +43,7 @@ struct Plugin * * @param cls closure (our `struct Plugin`) * @param key key to store @a data under + * @param xor_distance distance of @a key to our PID * @param size number of bytes in @a data * @param data data to store * @param type type of the value @@ -56,6 +55,7 @@ struct Plugin static ssize_t template_plugin_put (void *cls, const struct GNUNET_HashCode *key, + uint32_t xor_distance, size_t size, const char *data, enum GNUNET_BLOCK_Type type, diff --git a/src/datacache/test_datacache.c b/src/datacache/test_datacache.c index 79e6b6d749..50e45012df 100644 --- a/src/datacache/test_datacache.c +++ b/src/datacache/test_datacache.c @@ -2,20 +2,18 @@ This file is part of GNUnet. Copyright (C) 2006, 2009, 2010 GNUnet e.V. - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3, or (at your - option) any later version. + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - Boston, MA 02110-1301, USA. + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. */ /* * @file datacache/test_datacache.c @@ -46,6 +44,11 @@ checkIt (void *cls, unsigned int path_len, const struct GNUNET_PeerIdentity *path) { + (void) key; + (void) type; + (void) exp; + (void) path_len; + (void) path; if (size != sizeof (struct GNUNET_HashCode)) { GNUNET_break (0); @@ -61,17 +64,22 @@ checkIt (void *cls, static void -run (void *cls, char *const *args, const char *cfgfile, +run (void *cls, + char *const *args, + const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg) { struct GNUNET_DATACACHE_Handle *h; struct GNUNET_HashCode k; struct GNUNET_HashCode n; struct GNUNET_TIME_Absolute exp; - unsigned int i; + (void) cls; + (void) args; + (void) cfgfile; ok = 0; - h = GNUNET_DATACACHE_create (cfg, "testcache"); + h = GNUNET_DATACACHE_create (cfg, + "testcache"); if (h == NULL) { FPRINTF (stderr, @@ -83,32 +91,55 @@ run (void *cls, char *const *args, const char *cfgfile, exp = GNUNET_TIME_absolute_get (); exp.abs_value_us += 5 * 60 * 1000 * 1000LL; memset (&k, 0, sizeof (struct GNUNET_HashCode)); - for (i = 0; i < 100; i++) + for (unsigned int i = 0; i < 100; i++) { GNUNET_CRYPTO_hash (&k, sizeof (struct GNUNET_HashCode), &n); ASSERT (GNUNET_OK == - GNUNET_DATACACHE_put (h, &k, sizeof (struct GNUNET_HashCode), + GNUNET_DATACACHE_put (h, + &k, + GNUNET_YES, + sizeof (struct GNUNET_HashCode), (const char *) &n, 1 + i % 16, exp, 0, NULL)); k = n; } - memset (&k, 0, sizeof (struct GNUNET_HashCode)); - for (i = 0; i < 100; i++) + memset (&k, + 0, + sizeof (struct GNUNET_HashCode)); + for (unsigned int i = 0; i < 100; i++) { - GNUNET_CRYPTO_hash (&k, sizeof (struct GNUNET_HashCode), &n); - ASSERT (1 == GNUNET_DATACACHE_get (h, &k, 1 + i % 16, &checkIt, &n)); + GNUNET_CRYPTO_hash (&k, + sizeof (struct GNUNET_HashCode), + &n); + ASSERT (1 == GNUNET_DATACACHE_get (h, + &k, + 1 + i % 16, + &checkIt, + &n)); k = n; } - memset (&k, 42, sizeof (struct GNUNET_HashCode)); - GNUNET_CRYPTO_hash (&k, sizeof (struct GNUNET_HashCode), &n); + memset (&k, + 42, + sizeof (struct GNUNET_HashCode)); + GNUNET_CRYPTO_hash (&k, + sizeof (struct GNUNET_HashCode), + &n); ASSERT (GNUNET_OK == - GNUNET_DATACACHE_put (h, &k, sizeof (struct GNUNET_HashCode), - (const char *) &n, 792, + GNUNET_DATACACHE_put (h, + &k, + GNUNET_YES, + sizeof (struct GNUNET_HashCode), + (const char *) &n, + 792, GNUNET_TIME_UNIT_FOREVER_ABS, - 0, NULL)); - ASSERT (0 != GNUNET_DATACACHE_get (h, &k, 792, &checkIt, &n)); - + 0, + NULL)); + ASSERT (0 != GNUNET_DATACACHE_get (h, + &k, + 792, + &checkIt, + &n)); GNUNET_DATACACHE_destroy (h); ASSERT (ok == 0); return; @@ -133,16 +164,26 @@ main (int argc, char *argv[]) GNUNET_GETOPT_OPTION_END }; + (void) argc; GNUNET_log_setup ("test-datacache", "WARNING", NULL); plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]); - GNUNET_snprintf (cfg_name, sizeof (cfg_name), "test_datacache_data_%s.conf", + GNUNET_snprintf (cfg_name, + sizeof (cfg_name), + "test_datacache_data_%s.conf", plugin_name); - GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, xargv, - "test-datacache", "nohelp", options, &run, NULL); + GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, + xargv, + "test-datacache", + "nohelp", + options, + &run, + NULL); if ( (0 != ok) && (77 != ok) ) - FPRINTF (stderr, "Missed some testcases: %d\n", ok); + FPRINTF (stderr, + "Missed some testcases: %d\n", + ok); return ok; } diff --git a/src/datacache/test_datacache_quota.c b/src/datacache/test_datacache_quota.c index 78b56ce425..21e3736086 100644 --- a/src/datacache/test_datacache_quota.c +++ b/src/datacache/test_datacache_quota.c @@ -2,20 +2,18 @@ This file is part of GNUnet. Copyright (C) 2006, 2009, 2010 GNUnet e.V. - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3, or (at your - option) any later version. + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - Boston, MA 02110-1301, USA. + Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. */ /* * @file datacache/test_datacache_quota.c @@ -43,44 +41,63 @@ static const char *plugin_name; * some of the data from the last iteration is still there. */ static void -run (void *cls, char *const *args, const char *cfgfile, +run (void *cls, + char *const *args, + const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg) { struct GNUNET_DATACACHE_Handle *h; struct GNUNET_HashCode k; struct GNUNET_HashCode n; - unsigned int i; - unsigned int j; char buf[3200]; struct GNUNET_TIME_Absolute exp; + (void) cls; + (void) args; + (void) cfgfile; ok = 0; - h = GNUNET_DATACACHE_create (cfg, "testcache"); + h = GNUNET_DATACACHE_create (cfg, + "testcache"); if (h == NULL) { - FPRINTF (stderr, "%s", "Failed to initialize datacache. Database likely not setup, skipping test.\n"); + FPRINTF (stderr, + "%s", + "Failed to initialize datacache. Database likely not setup, skipping test.\n"); return; } exp = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS); memset (buf, 1, sizeof (buf)); memset (&k, 0, sizeof (struct GNUNET_HashCode)); - for (i = 0; i < 10; i++) + for (unsigned int i = 0; i < 10; i++) { - FPRINTF (stderr, "%s", "."); - GNUNET_CRYPTO_hash (&k, sizeof (struct GNUNET_HashCode), &n); - for (j = i; j < sizeof (buf); j += 10) + FPRINTF (stderr, + "%s", + "."); + GNUNET_CRYPTO_hash (&k, + sizeof (struct GNUNET_HashCode), + &n); + for (unsigned int j = i; j < sizeof (buf); j += 10) { exp.abs_value_us++; buf[j] = i; - ASSERT (GNUNET_OK == GNUNET_DATACACHE_put (h, &k, j, buf, 1 + i, exp, 0, NULL)); + ASSERT (GNUNET_OK == + GNUNET_DATACACHE_put (h, + &k, + GNUNET_YES, + j, + buf, + 1 + i, + exp, + 0, + NULL)); ASSERT (0 < GNUNET_DATACACHE_get (h, &k, 1 + i, NULL, NULL)); } k = n; } FPRINTF (stderr, "%s", "\n"); memset (&k, 0, sizeof (struct GNUNET_HashCode)); - for (i = 0; i < 10; i++) + for (unsigned int i = 0; i < 10; i++) { FPRINTF (stderr, "%s", "."); GNUNET_CRYPTO_hash (&k, sizeof (struct GNUNET_HashCode), &n); @@ -101,7 +118,8 @@ FAILURE: int -main (int argc, char *argv[]) +main (int argc, + char *argv[]) { char cfg_name[128]; char *const xargv[] = { @@ -114,17 +132,27 @@ main (int argc, char *argv[]) GNUNET_GETOPT_OPTION_END }; + (void) argc; GNUNET_log_setup ("test-datacache-quota", "WARNING", NULL); plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]); - GNUNET_snprintf (cfg_name, sizeof (cfg_name), "test_datacache_data_%s.conf", + GNUNET_snprintf (cfg_name, + sizeof (cfg_name), + "test_datacache_data_%s.conf", plugin_name); - GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, xargv, - "test-datacache-quota", "nohelp", options, &run, NULL); + GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, + xargv, + "test-datacache-quota", + "nohelp", + options, + &run, + NULL); if (0 != ok) - FPRINTF (stderr, "Missed some testcases: %d\n", ok); + FPRINTF (stderr, + "Missed some testcases: %d\n", + ok); return ok; } |