eliminate use of OFFSET in namestore DB queries

10 files changed, 201 insertions, 114 deletions

diff --git a/src/include/gnunet_namestore_plugin.h b/src/include/gnunet_namestore_plugin.h
index 802d7bac51..c3ab4d8bf7 100644
--- a/src/include/gnunet_namestore_plugin.h
+++ b/src/include/gnunet_namestore_plugin.h
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet
-     Copyright (C) 2012, 2013 GNUnet e.V.
+     Copyright (C) 2012, 2013, 2018 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -47,6 +47,7 @@ extern "C"
  * Function called for each matching record.
  *
  * @param cls closure
+ * @param serial unique serial number of the record
  * @param zone_key private key of the zone
  * @param label name that is being mapped (at most 255 characters long)
  * @param rd_count number of entries in @a rd array
@@ -54,6 +55,7 @@ extern "C"
  */
 typedef void
 (*GNUNET_NAMESTORE_RecordIterator) (void *cls,
+				    uint64_t serial,
 				    const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key,
 				    const char *label,
 				    unsigned int rd_count,
@@ -113,7 +115,7 @@ struct GNUNET_NAMESTORE_PluginFunctions
    *
    * @param cls closure (internal context for the plugin)
    * @param zone private key of the zone, NULL for all zones
-   * @param offset offset in the list of all matching records
+   * @param serial serial (to exclude) in the list of matching records
    * @param limit maximum number of results to return to @a iter
    * @param iter function to call with the result
    * @param iter_cls closure for @a iter
@@ -122,7 +124,7 @@ struct GNUNET_NAMESTORE_PluginFunctions
   int
   (*iterate_records) (void *cls,
 		      const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
-		      uint64_t offset,
+		      uint64_t serial,
 		      uint64_t limit,
 		      GNUNET_NAMESTORE_RecordIterator iter,
 		      void *iter_cls);
diff --git a/src/namestore/gnunet-namestore-fcfsd.c b/src/namestore/gnunet-namestore-fcfsd.c
index 6e45f82277..903253b1ae 100644
--- a/src/namestore/gnunet-namestore-fcfsd.c
+++ b/src/namestore/gnunet-namestore-fcfsd.c
@@ -324,6 +324,7 @@ iterate_cb (void *cls,
   char* pkey;
   char* new_buf;
 
+  (void) zone_key;
   if (1 != rd_len)
   {
     GNUNET_NAMESTORE_zone_iterator_next (zr->list_it,
@@ -439,6 +440,7 @@ fill_s_reply (const char *info,
   char *reply;
   struct MHD_Response *response;
 
+  (void) request;
   GNUNET_asprintf (&reply,
 		   SUBMIT_PAGE,
 		   info,
@@ -587,6 +589,8 @@ zone_to_name_cb (void *cls,
   struct Request *request = cls;
   struct GNUNET_GNSRECORD_Data r;
 
+  (void) rd;
+  (void) zone_key;
   request->qe = NULL;
   if (0 != rd_count)
   {
@@ -644,6 +648,9 @@ lookup_block_processor (void *cls,
 {
   struct Request *request = cls;
 
+  (void) label;
+  (void) rd;
+  (void) zone;
   request->qe = NULL;
   if (0 == rd_count)
   {
@@ -715,6 +722,8 @@ create_response (void *cls,
   struct GNUNET_CRYPTO_EcdsaPublicKey pub;
   int ret;
 
+  (void) cls;
+  (void) version;
   if ( (0 == strcmp (method, MHD_HTTP_METHOD_GET)) ||
        (0 == strcmp (method, MHD_HTTP_METHOD_HEAD)) )
     {
diff --git a/src/namestore/gnunet-namestore.c b/src/namestore/gnunet-namestore.c
index 7b8312b461..d329dcb3ba 100644
--- a/src/namestore/gnunet-namestore.c
+++ b/src/namestore/gnunet-namestore.c
@@ -661,6 +661,8 @@ handle_reverse_lookup (void *cls,
 {
   (void) cls;
   (void) zone;
+  (void) rd_count;
+  (void) rd;
   reverse_qe = NULL;
   if (NULL == label)
     FPRINTF (stdout,
@@ -712,6 +714,7 @@ del_monitor (void *cls,
   char *vs;
 
   (void) cls;
+  (void) zone;
   del_qe = NULL;
   if (0 == rd_count)
   {
@@ -1092,6 +1095,9 @@ id_connect_cb (void *cls,
 {
   const struct GNUNET_CONFIGURATION_Handle *cfg = cls;
 
+  (void) cls;
+  (void) ctx;
+  (void) name;
   if (NULL == ego)
   {
     get_default = GNUNET_IDENTITY_get (idh,
diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c
index c6bab15b7b..6b8f73242c 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -75,6 +75,15 @@ struct ZoneIteration
   struct GNUNET_CRYPTO_EcdsaPrivateKey zone;
 
   /**
+   * Last sequence number in the zone iteration used to address next
+   * result of the zone iteration in the store
+   *
+   * Initialy set to 0.
+   * Updated in #zone_iterate_proc()
+   */
+  uint64_t seq;
+
+  /**
    * The operation id fot the zone iteration in the response for the client
    */
   uint32_t request_id;
@@ -152,13 +161,13 @@ struct ZoneMonitor
   struct GNUNET_SCHEDULER_Task *task;
 
   /**
-   * Offset of the zone iteration used to address next result of the zone
-   * iteration in the store
+   * Last sequence number in the zone iteration used to address next
+   * result of the zone iteration in the store
    *
    * Initialy set to 0.
-   * Incremented with by every call to #handle_iteration_next
+   * Updated in #monitor_iterate_cb()
    */
-  uint32_t offset;
+  uint64_t seq;
 
 };
 
@@ -394,6 +403,7 @@ client_connect_cb (void *cls,
  * record, which (if found) is then copied to @a cls for future use.
  *
  * @param cls a `struct GNUNET_GNSRECORD_Data **` for storing the nick (if found)
+ * @param seq sequence number of the record
  * @param private_key the private key of the zone (unused)
  * @param label should be #GNUNET_GNS_EMPTY_LABEL_AT
  * @param rd_count number of records in @a rd
@@ -401,6 +411,7 @@ client_connect_cb (void *cls,
  */
 static void
 lookup_nick_it (void *cls,
+		uint64_t seq,
                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key,
                 const char *label,
                 unsigned int rd_count,
@@ -409,6 +420,7 @@ lookup_nick_it (void *cls,
   struct GNUNET_GNSRECORD_Data **res = cls;
 
   (void) private_key;
+  (void) seq;
   if (0 != strcmp (label, GNUNET_GNS_EMPTY_LABEL_AT))
   {
     GNUNET_break (0);
@@ -813,9 +825,11 @@ struct RecordLookupContext
 
 /**
  * FIXME.
+ * @param seq sequence number of the record
  */
 static void
 lookup_it (void *cls,
+	   uint64_t seq,
            const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key,
            const char *label,
            unsigned int rd_count,
@@ -826,6 +840,7 @@ lookup_it (void *cls,
   unsigned int rdc_res;
 
   (void) private_key;
+  (void) seq;
   if (0 == strcmp (label,
                    rlc->label))
   {
@@ -1212,6 +1227,7 @@ struct ZoneToNameCtx
  * Zone to name iterator
  *
  * @param cls struct ZoneToNameCtx *
+ * @param seq sequence number of the record
  * @param zone_key the zone key
  * @param name name
  * @param rd_count number of records in @a rd
@@ -1219,6 +1235,7 @@ struct ZoneToNameCtx
  */
 static void
 handle_zone_to_name_it (void *cls,
+			uint64_t seq,
 			const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key,
 			const char *name,
 			unsigned int rd_count,
@@ -1234,6 +1251,7 @@ handle_zone_to_name_it (void *cls,
   char *name_tmp;
   char *rd_tmp;
 
+  (void) seq;
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 	      "Found result for zone-to-name lookup: `%s'\n",
 	      name);
@@ -1342,8 +1360,9 @@ struct ZoneIterationProcResult
 
 /**
  * Process results for zone iteration from database
- *
- * @param cls struct ZoneIterationProcResult *proc
+ * 
+ * @param cls struct ZoneIterationProcResult
+ * @param seq sequence number of the record
  * @param zone_key the zone key
  * @param name name
  * @param rd_count number of records for this name
@@ -1351,6 +1370,7 @@ struct ZoneIterationProcResult
  */
 static void
 zone_iterate_proc (void *cls,
+		   uint64_t seq,
 		   const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key,
 		   const char *name,
 		   unsigned int rd_count,
@@ -1380,6 +1400,7 @@ zone_iterate_proc (void *cls,
     return;
   }
   proc->limit--;
+  proc->zi->seq = seq;
   send_lookup_response (proc->zi->nc,
 			proc->zi->request_id,
 			zone_key,
@@ -1432,7 +1453,7 @@ run_zone_iteration_round (struct ZoneIteration *zi,
                                                              sizeof (zero)))
                                                ? NULL
                                                : &zi->zone,
-                                               zi->offset,
+                                               zi->seq,
                                                limit,
                                                &zone_iterate_proc,
                                                &proc));
@@ -1443,7 +1464,6 @@ run_zone_iteration_round (struct ZoneIteration *zi,
                          "NAMESTORE iteration delay (μs/record)",
                          duration.rel_value_us,
                          GNUNET_NO);
-  zi->offset += (limit - proc.limit);
   if (0 == proc.limit)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -1597,6 +1617,7 @@ monitor_next (void *cls);
  * A #GNUNET_NAMESTORE_RecordIterator for monitors.
  *
  * @param cls a 'struct ZoneMonitor *' with information about the monitor
+ * @param seq sequence number of the record
  * @param zone_key zone key of the zone
  * @param name name
  * @param rd_count number of records in @a rd
@@ -1604,6 +1625,7 @@ monitor_next (void *cls);
  */
 static void
 monitor_iterate_cb (void *cls,
+		    uint64_t seq,
 		    const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone_key,
 		    const char *name,
 		    unsigned int rd_count,
@@ -1611,6 +1633,7 @@ monitor_iterate_cb (void *cls,
 {
   struct ZoneMonitor *zm = cls;
 
+  zm->seq = seq;
   if (NULL == name)
   {
     /* finished with iteration */
@@ -1683,7 +1706,7 @@ monitor_next (void *cls)
 						     sizeof (zero)))
                                        ? NULL
                                        : &zm->zone,
-				       zm->offset++,
+				       zm->seq,
                                        1,
 				       &monitor_iterate_cb,
 				       zm);
@@ -1727,7 +1750,7 @@ run (void *cls,
 							    "DISABLE");
   GSN_cfg = cfg;
   monitor_nc = GNUNET_notification_context_create (1);
-  if (GNUNET_NO == disable_namecache)
+  if (GNUNET_YES != disable_namecache)
   {
     namecache = GNUNET_NAMECACHE_connect (cfg);
     GNUNET_assert (NULL != namecache);
diff --git a/src/namestore/namestore.h b/src/namestore/namestore.h
index b398af8a91..207b35662f 100644
--- a/src/namestore/namestore.h
+++ b/src/namestore/namestore.h
@@ -161,7 +161,7 @@ struct LabelLookupResponseMessage
    * Length of serialized record data
    */
   uint16_t rd_len GNUNET_PACKED;
-
+  
   /**
    * Number of records contained
    */
@@ -169,7 +169,7 @@ struct LabelLookupResponseMessage
 
   /**
    * Was the label found in the database??
-   * GNUNET_YES or GNUNET_NO
+   * #GNUNET_YES or #GNUNET_NO
    */
   uint16_t found GNUNET_PACKED;
 
diff --git a/src/namestore/plugin_namestore_flat.c b/src/namestore/plugin_namestore_flat.c
index f061ab7d11..f40154915b 100644
--- a/src/namestore/plugin_namestore_flat.c
+++ b/src/namestore/plugin_namestore_flat.c
@@ -523,6 +523,7 @@ namestore_flat_lookup_records (void *cls,
     return GNUNET_NO;
   if (NULL != iter)
     iter (iter_cls,
+	  0,
 	  entry->private_key,
 	  entry->label,
 	  entry->record_count,
@@ -547,6 +548,12 @@ struct IterateContext
   uint64_t limit;
 
   /**
+   * What is the position of the current entry, counting
+   * starts from 1.
+   */
+  uint64_t pos;
+
+  /**
    * Target zone.
    */
   const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone;
@@ -581,6 +588,7 @@ iterate_zones (void *cls,
   struct FlatFileEntry *entry = value;
 
   (void) key;
+  ic->pos++;
   if (0 == ic->limit)
     return GNUNET_NO;
   if ( (NULL != ic->zone) &&
@@ -594,6 +602,7 @@ iterate_zones (void *cls,
     return GNUNET_YES;
   }
   ic->iter (ic->iter_cls,
+	    ic->pos,
             entry->private_key,
             entry->label,
             entry->record_count,
@@ -611,7 +620,7 @@ iterate_zones (void *cls,
  *
  * @param cls closure (internal context for the plugin)
  * @param zone hash of public key of the zone, NULL to iterate over all zones
- * @param offset offset in the list of all matching records
+ * @param serial serial number to exclude in the list of all matching records
  * @param limit maximum number of results to return to @a iter
  * @param iter function to call with the result
  * @param iter_cls closure for @a iter
@@ -620,7 +629,7 @@ iterate_zones (void *cls,
 static int
 namestore_flat_iterate_records (void *cls,
                                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
-                                uint64_t offset,
+                                uint64_t serial,
                                 uint64_t limit,
                                 GNUNET_NAMESTORE_RecordIterator iter,
                                 void *iter_cls)
@@ -628,7 +637,8 @@ namestore_flat_iterate_records (void *cls,
   struct Plugin *plugin = cls;
   struct IterateContext ic;
 
-  ic.offset = offset;
+  ic.offset = serial;
+  ic.pos = 0;
   ic.limit = limit;
   ic.iter = iter;
   ic.iter_cls = iter_cls;
@@ -663,6 +673,7 @@ zone_to_name (void *cls,
                      sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey)))
     {
       plugin->iter (plugin->iter_cls,
+		    0,
                     entry->private_key,
                     entry->label,
                     entry->record_count,
diff --git a/src/namestore/plugin_namestore_postgres.c b/src/namestore/plugin_namestore_postgres.c
index 4a24ddf88b..42bc9e4c97 100644
--- a/src/namestore/plugin_namestore_postgres.c
+++ b/src/namestore/plugin_namestore_postgres.c
@@ -65,7 +65,8 @@ static int
 database_setup (struct Plugin *plugin)
 {
   struct GNUNET_PQ_ExecuteStatement es_temporary =
-    GNUNET_PQ_make_execute ("CREATE TEMPORARY TABLE IF NOT EXISTS ns097records ("
+    GNUNET_PQ_make_execute ("CREATE TEMPORARY TABLE IF NOT EXISTS ns098records ("
+			    " seq BIGSERIAL PRIMARY KEY,"
                             " zone_private_key BYTEA NOT NULL DEFAULT '',"
                             " pkey BYTEA DEFAULT '',"
                             " rvalue BYTEA NOT NULL DEFAULT '',"
@@ -75,7 +76,8 @@ database_setup (struct Plugin *plugin)
                             ")"
                             "WITH OIDS");
   struct GNUNET_PQ_ExecuteStatement es_default =
-    GNUNET_PQ_make_execute ("CREATE TABLE IF NOT EXISTS ns097records ("
+    GNUNET_PQ_make_execute ("CREATE TABLE IF NOT EXISTS ns098records ("
+			    " seq SERIAL PRIMARY KEY,"
                             " zone_private_key BYTEA NOT NULL DEFAULT '',"
                             " pkey BYTEA DEFAULT '',"
                             " rvalue BYTEA NOT NULL DEFAULT '',"
@@ -125,13 +127,11 @@ database_setup (struct Plugin *plugin)
     struct GNUNET_PQ_ExecuteStatement es[] = {
       *cr,
       GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS ir_pkey_reverse "
-                                  "ON ns097records (zone_private_key,pkey)"),
+                                  "ON ns098records (zone_private_key,pkey)"),
       GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS ir_pkey_iter "
-                                  "ON ns097records (zone_private_key,rvalue)"),
-      GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS it_iter "
-                                  "ON ns097records (rvalue)"),
+                                  "ON ns098records (zone_private_key,seq)"),
       GNUNET_PQ_make_try_execute ("CREATE INDEX IF NOT EXISTS ir_label "
-                                  "ON ns097records (label)"),
+                                  "ON ns098records (label)"),
       GNUNET_PQ_EXECUTE_STATEMENT_END
     };
 
@@ -148,28 +148,28 @@ database_setup (struct Plugin *plugin)
   {
     struct GNUNET_PQ_PreparedStatement ps[] = {
       GNUNET_PQ_make_prepare ("store_records",
-                              "INSERT INTO ns097records (zone_private_key, pkey, rvalue, record_count, record_data, label) VALUES "
+                              "INSERT INTO ns098records (zone_private_key, pkey, rvalue, record_count, record_data, label) VALUES "
                               "($1, $2, $3, $4, $5, $6)",
                               6),
       GNUNET_PQ_make_prepare ("delete_records",
-                              "DELETE FROM ns097records "
+                              "DELETE FROM ns098records "
                               "WHERE zone_private_key=$1 AND label=$2",
                               2),
       GNUNET_PQ_make_prepare ("zone_to_name",
-                              "SELECT record_count,record_data,label FROM ns097records"
+                              "SELECT seq,record_count,record_data,label FROM ns098records"
                               " WHERE zone_private_key=$1 AND pkey=$2",
                               2),
       GNUNET_PQ_make_prepare ("iterate_zone",
-                              "SELECT record_count,record_data,label FROM ns097records "
-                              "WHERE zone_private_key=$1 ORDER BY rvalue OFFSET $2 LIMIT $3",
+                              "SELECT seq,record_count,record_data,label FROM ns098records "
+                              "WHERE zone_private_key=$1 AND seq > $2 ORDER BY seq ASC LIMIT $3",
                               3),
       GNUNET_PQ_make_prepare ("iterate_all_zones",
-                              "SELECT record_count,record_data,label,zone_private_key"
-                              " FROM ns097records ORDER BY rvalue OFFSET $1 LIMIT $2",
+                              "SELECT seq,record_count,record_data,label,zone_private_key"
+                              " FROM ns098records WHERE seq > $1 ORDER BY seq ASC LIMIT $2",
                               2),
       GNUNET_PQ_make_prepare ("lookup_label",
-                              "SELECT record_count,record_data,label "
-                              "FROM ns097records WHERE zone_private_key=$1 AND label=$2",
+                              "SELECT seq,record_count,record_data,label "
+                              "FROM ns098records WHERE zone_private_key=$1 AND label=$2",
                               2),
       GNUNET_PQ_PREPARED_STATEMENT_END
     };
@@ -224,7 +224,8 @@ namestore_postgres_store_records (void *cls,
     }
   rvalue = GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
                                      UINT64_MAX);
-  data_size = GNUNET_GNSRECORD_records_get_size (rd_count, rd);
+  data_size = GNUNET_GNSRECORD_records_get_size (rd_count,
+						 rd);
   if (data_size > 64 * 65536)
   {
     GNUNET_break (0);
@@ -312,12 +313,14 @@ parse_result_call_iterator (void *cls,
     return; /* no need to do more work */
   for (unsigned int i=0;i<num_results;i++)
   {
+    uint64_t serial;
     void *data;
     size_t data_size;
     uint32_t record_count;
     char *label;
     struct GNUNET_CRYPTO_EcdsaPrivateKey zk;
     struct GNUNET_PQ_ResultSpec rs_with_zone[] = {
+      GNUNET_PQ_result_spec_uint64 ("seq", &serial),
       GNUNET_PQ_result_spec_uint32 ("record_count", &record_count),
       GNUNET_PQ_result_spec_variable_size ("record_data", &data, &data_size),
       GNUNET_PQ_result_spec_string ("label", &label),
@@ -325,6 +328,7 @@ parse_result_call_iterator (void *cls,
       GNUNET_PQ_result_spec_end
     };
     struct GNUNET_PQ_ResultSpec rs_without_zone[] = {
+      GNUNET_PQ_result_spec_uint64 ("seq", &serial),
       GNUNET_PQ_result_spec_uint32 ("record_count", &record_count),
       GNUNET_PQ_result_spec_variable_size ("record_data", &data, &data_size),
       GNUNET_PQ_result_spec_string ("label", &label),
@@ -365,6 +369,7 @@ parse_result_call_iterator (void *cls,
         return;
       }
       pc->iter (pc->iter_cls,
+		serial,
                 (NULL == pc->zone_key) ? &zk : pc->zone_key,
                 label,
                 record_count,
@@ -422,7 +427,7 @@ namestore_postgres_lookup_records (void *cls,
  *
  * @param cls closure (internal context for the plugin)
  * @param zone hash of public key of the zone, NULL to iterate over all zones
- * @param offset offset in the list of all matching records
+ * @param serial serial number to exclude in the list of all matching records
  * @param limit maximum number of results to fetch
  * @param iter function to call with the result
  * @param iter_cls closure for @a iter
@@ -431,7 +436,7 @@ namestore_postgres_lookup_records (void *cls,
 static int
 namestore_postgres_iterate_records (void *cls,
                                     const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
-                                    uint64_t offset,
+                                    uint64_t serial,
                                     uint64_t limit,
                                     GNUNET_NAMESTORE_RecordIterator iter,
                                     void *iter_cls)
@@ -447,7 +452,7 @@ namestore_postgres_iterate_records (void *cls,
   if (NULL == zone)
   {
     struct GNUNET_PQ_QueryParam params_without_zone[] = {
-      GNUNET_PQ_query_param_uint64 (&offset),
+      GNUNET_PQ_query_param_uint64 (&serial),
       GNUNET_PQ_query_param_uint64 (&limit),
       GNUNET_PQ_query_param_end
     };
@@ -462,7 +467,7 @@ namestore_postgres_iterate_records (void *cls,
   {
     struct GNUNET_PQ_QueryParam params_with_zone[] = {
       GNUNET_PQ_query_param_auto_from_type (zone),
-      GNUNET_PQ_query_param_uint64 (&offset),
+      GNUNET_PQ_query_param_uint64 (&serial),
       GNUNET_PQ_query_param_uint64 (&limit),
       GNUNET_PQ_query_param_end
     };
diff --git a/src/namestore/plugin_namestore_sqlite.c b/src/namestore/plugin_namestore_sqlite.c
index f905787b5e..b54b4dba24 100644
--- a/src/namestore/plugin_namestore_sqlite.c
+++ b/src/namestore/plugin_namestore_sqlite.c
@@ -147,15 +147,13 @@ create_indices (sqlite3 * dbh)
   /* create indices */
   if ( (SQLITE_OK !=
 	sqlite3_exec (dbh,
-                      "CREATE INDEX IF NOT EXISTS ir_pkey_reverse ON ns097records (zone_private_key,pkey)",
+                      "CREATE INDEX IF NOT EXISTS ir_pkey_reverse "
+		      "ON ns098records (zone_private_key,pkey)",
 		      NULL, NULL, NULL)) ||
        (SQLITE_OK !=
 	sqlite3_exec (dbh,
-                      "CREATE INDEX IF NOT EXISTS ir_pkey_iter ON ns097records (zone_private_key,rvalue)",
-		      NULL, NULL, NULL)) ||
-       (SQLITE_OK !=
-	sqlite3_exec (dbh,
-                      "CREATE INDEX IF NOT EXISTS it_iter ON ns097records (rvalue)",
+                      "CREATE INDEX IF NOT EXISTS ir_pkey_iter "
+		      "ON ns098records (zone_private_key,uid)",
 		      NULL, NULL, NULL)) )
     LOG (GNUNET_ERROR_TYPE_ERROR,
 	 "Failed to create indices: %s\n",
@@ -260,22 +258,24 @@ database_setup (struct Plugin *plugin)
   /* Create table */
   CHECK (SQLITE_OK ==
          sq_prepare (plugin->dbh,
-                     "SELECT 1 FROM sqlite_master WHERE tbl_name = 'ns097records'",
+                     "SELECT 1 FROM sqlite_master WHERE tbl_name = 'ns098records'",
                      &stmt));
-  if ((sqlite3_step (stmt) == SQLITE_DONE) &&
-      (sqlite3_exec
-       (plugin->dbh,
-        "CREATE TABLE ns097records ("
-        " zone_private_key BLOB NOT NULL,"
-        " pkey BLOB,"
-	" rvalue INT8 NOT NULL,"
-	" record_count INT NOT NULL,"
-        " record_data BLOB NOT NULL,"
-        " label TEXT NOT NULL"
-	")",
-	NULL, NULL, NULL) != SQLITE_OK))
+  if ( (sqlite3_step (stmt) == SQLITE_DONE) &&
+       (SQLITE_OK !=
+	sqlite3_exec (plugin->dbh,
+		      "CREATE TABLE ns098records ("
+		      " uid INTEGER PRIMARY KEY,"
+		      " zone_private_key BLOB NOT NULL,"
+		      " pkey BLOB,"
+		      " rvalue INT8 NOT NULL,"
+		      " record_count INT NOT NULL,"
+		      " record_data BLOB NOT NULL,"
+		      " label TEXT NOT NULL"
+		      ")",
+		      NULL, NULL, NULL)) )
   {
-    LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR,
+    LOG_SQLITE (plugin,
+		GNUNET_ERROR_TYPE_ERROR,
                 "sqlite3_exec");
     sqlite3_finalize (stmt);
     return GNUNET_SYSERR;
@@ -286,33 +286,40 @@ database_setup (struct Plugin *plugin)
 
   if ( (SQLITE_OK !=
         sq_prepare (plugin->dbh,
-                    "INSERT INTO ns097records (zone_private_key, pkey, rvalue, record_count, record_data, label)"
+                    "INSERT INTO ns098records (zone_private_key, pkey, rvalue, record_count, record_data, label)"
                     " VALUES (?, ?, ?, ?, ?, ?)",
                     &plugin->store_records)) ||
        (SQLITE_OK !=
         sq_prepare (plugin->dbh,
-                    "DELETE FROM ns097records WHERE zone_private_key=? AND label=?",
+                    "DELETE FROM ns098records WHERE zone_private_key=? AND label=?",
                     &plugin->delete_records)) ||
        (SQLITE_OK !=
         sq_prepare (plugin->dbh,
-                    "SELECT record_count,record_data,label"
-                    " FROM ns097records WHERE zone_private_key=? AND pkey=?",
+                    "SELECT uid,record_count,record_data,label"
+                    " FROM ns098records"
+		    " WHERE zone_private_key=? AND pkey=?",
                     &plugin->zone_to_name)) ||
        (SQLITE_OK !=
         sq_prepare (plugin->dbh,
-                    "SELECT record_count,record_data,label"
-                    " FROM ns097records WHERE zone_private_key=?"
-                    " ORDER BY rvalue LIMIT ? OFFSET ?",
+                    "SELECT uid,record_count,record_data,label"
+                    " FROM ns098records"
+		    " WHERE zone_private_key=? AND _rowid_ >= ?"
+                    " ORDER BY _rowid_ ASC"
+		    " LIMIT ?",
                     &plugin->iterate_zone)) ||
        (SQLITE_OK !=
         sq_prepare (plugin->dbh,
-                    "SELECT record_count,record_data,label,zone_private_key"
-                    " FROM ns097records ORDER BY rvalue LIMIT ? OFFSET ?",
+                    "SELECT uid,record_count,record_data,label,zone_private_key"
+                    " FROM ns098records"
+		    " WHERE _rowid_ >= ?" 
+		    " ORDER BY _rowid_ ASC"
+		    " LIMIT ?",
                     &plugin->iterate_all_zones))  ||
        (SQLITE_OK !=
         sq_prepare (plugin->dbh,
-                    "SELECT record_count,record_data,label,zone_private_key"
-                    " FROM ns097records WHERE zone_private_key=? AND label=?",
+                    "SELECT uid,record_count,record_data,label,zone_private_key"
+                    " FROM ns098records"
+		    " WHERE zone_private_key=? AND label=?",
                     &plugin->lookup_label))
        )
   {
@@ -405,10 +412,11 @@ namestore_sqlite_store_records (void *cls,
   struct GNUNET_CRYPTO_EcdsaPublicKey pkey;
   uint64_t rvalue;
   size_t data_size;
-  unsigned int i;
 
-  memset (&pkey, 0, sizeof (pkey));
-  for (i=0;i<rd_count;i++)
+  memset (&pkey,
+	  0,
+	  sizeof (pkey));
+  for (unsigned int i=0;i<rd_count;i++)
     if (GNUNET_GNSRECORD_TYPE_PKEY == rd[i].record_type)
     {
       GNUNET_break (sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey) ==
@@ -539,28 +547,50 @@ get_records_and_call_iterator (struct Plugin *plugin,
                                GNUNET_NAMESTORE_RecordIterator iter,
                                void *iter_cls)
 {
-  uint32_t record_count;
-  size_t data_size;
-  void *data;
-  char *label;
-  struct GNUNET_CRYPTO_EcdsaPrivateKey zk;
   int ret;
   int sret;
 
   ret = GNUNET_OK;
   for (uint64_t i = 0;i<limit ; i++)
   {
-    if (SQLITE_ROW == (sret = sqlite3_step (stmt)))
+    sret = sqlite3_step (stmt);
+
+    if (SQLITE_DONE == sret)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+		  "Iteration done (no results)\n");
+      ret = GNUNET_NO;
+      break;
+    }
+    if (SQLITE_ROW != sret)
     {
+      LOG_SQLITE (plugin,
+		  GNUNET_ERROR_TYPE_ERROR,
+		  "sqlite_step");
+      ret = GNUNET_SYSERR;
+      break;
+    }
+    
+    {
+      uint64_t seq;
+      uint32_t record_count;
+      size_t data_size;
+      void *data;
+      char *label;
+      struct GNUNET_CRYPTO_EcdsaPrivateKey zk;
       struct GNUNET_SQ_ResultSpec rs[] = {
+        GNUNET_SQ_result_spec_uint64 (&seq),
         GNUNET_SQ_result_spec_uint32 (&record_count),
-        GNUNET_SQ_result_spec_variable_size (&data, &data_size),
+        GNUNET_SQ_result_spec_variable_size (&data,
+					     &data_size),
         GNUNET_SQ_result_spec_string (&label),
         GNUNET_SQ_result_spec_end
       };
       struct GNUNET_SQ_ResultSpec rsx[] = {
+        GNUNET_SQ_result_spec_uint64 (&seq),
         GNUNET_SQ_result_spec_uint32 (&record_count),
-        GNUNET_SQ_result_spec_variable_size (&data, &data_size),
+        GNUNET_SQ_result_spec_variable_size (&data,
+					     &data_size),
         GNUNET_SQ_result_spec_string (&label),
         GNUNET_SQ_result_spec_auto_from_type (&zk),
         GNUNET_SQ_result_spec_end
@@ -604,6 +634,7 @@ get_records_and_call_iterator (struct Plugin *plugin,
         {
           if (NULL != iter)
             iter (iter_cls,
+		  seq + 1,
                   zone_key,
                   label,
                   record_count,
@@ -612,21 +643,6 @@ get_records_and_call_iterator (struct Plugin *plugin,
       }
       GNUNET_SQ_cleanup_result (rs);
     }
-    else
-    {
-      if (SQLITE_DONE != sret)
-      {
-        LOG_SQLITE (plugin,
-                    GNUNET_ERROR_TYPE_ERROR,
-                    "sqlite_step");
-        ret = GNUNET_SYSERR;
-      }
-      else
-      {
-        ret = GNUNET_NO;
-      }
-      break;
-    }
   }
   GNUNET_SQ_reset (plugin->dbh,
                    stmt);
@@ -685,7 +701,7 @@ namestore_sqlite_lookup_records (void *cls,
  *
  * @param cls closure (internal context for the plugin)
  * @param zone hash of public key of the zone, NULL to iterate over all zones
- * @param offset offset in the list of all matching records
+ * @param serial serial number to exclude in the list of all matching records
  * @param limit maximum number of results to return
  * @param iter function to call with the result
  * @param iter_cls closure for @a iter
@@ -694,7 +710,7 @@ namestore_sqlite_lookup_records (void *cls,
 static int
 namestore_sqlite_iterate_records (void *cls,
 				  const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
-				  uint64_t offset,
+				  uint64_t serial,
                                   uint64_t limit,
 				  GNUNET_NAMESTORE_RecordIterator iter,
                                   void *iter_cls)
@@ -706,8 +722,8 @@ namestore_sqlite_iterate_records (void *cls,
   if (NULL == zone)
   {
     struct GNUNET_SQ_QueryParam params[] = {
+      GNUNET_SQ_query_param_uint64 (&serial),
       GNUNET_SQ_query_param_uint64 (&limit),
-      GNUNET_SQ_query_param_uint64 (&offset),
       GNUNET_SQ_query_param_end
     };
 
@@ -719,8 +735,8 @@ namestore_sqlite_iterate_records (void *cls,
   {
     struct GNUNET_SQ_QueryParam params[] = {
       GNUNET_SQ_query_param_auto_from_type (zone),
+      GNUNET_SQ_query_param_uint64 (&serial),
       GNUNET_SQ_query_param_uint64 (&limit),
-      GNUNET_SQ_query_param_uint64 (&offset),
       GNUNET_SQ_query_param_end
     };
 
@@ -761,7 +777,8 @@ static int
 namestore_sqlite_zone_to_name (void *cls,
 			       const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
 			       const struct GNUNET_CRYPTO_EcdsaPublicKey *value_zone,
-			       GNUNET_NAMESTORE_RecordIterator iter, void *iter_cls)
+			       GNUNET_NAMESTORE_RecordIterator iter,
+			       void *iter_cls)
 {
   struct Plugin *plugin = cls;
   struct GNUNET_SQ_QueryParam params[] = {
diff --git a/src/namestore/test_namestore_api_monitoring.c b/src/namestore/test_namestore_api_monitoring.c
index efbd6badfa..68a3e4fb8d 100644
--- a/src/namestore/test_namestore_api_monitoring.c
+++ b/src/namestore/test_namestore_api_monitoring.c
@@ -209,7 +209,9 @@ zone_proc (void *cls,
 
 
 static void
-put_cont (void *cls, int32_t success, const char *emsg)
+put_cont (void *cls,
+	  int32_t success,
+	  const char *emsg)
 {
   static int c = 0;
   char *label = cls;
@@ -232,10 +234,12 @@ put_cont (void *cls, int32_t success, const char *emsg)
   else
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                "Failed to created records\n");
+                "Failed to create record `%s'\n",
+		label);
     GNUNET_break (0);
     GNUNET_SCHEDULER_cancel (endbadly_task);
-    endbadly_task = GNUNET_SCHEDULER_add_now (&endbadly, NULL);
+    endbadly_task = GNUNET_SCHEDULER_add_now (&endbadly,
+					      NULL);
   }
 }
 
@@ -341,10 +345,16 @@ run (void *cls,
   /* name in different zone */
   GNUNET_asprintf(&s_name_3, "dummy3");
   s_rd_3 = create_record(1);
-  GNUNET_assert (NULL != (ns_ops[2] = GNUNET_NAMESTORE_records_store (nsh, privkey2, s_name_3,
-  		1, s_rd_3, &put_cont, s_name_3)));
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Created record 1\n");
+  GNUNET_assert (NULL != (ns_ops[2] =
+			  GNUNET_NAMESTORE_records_store (nsh,
+							  privkey2,
+							  s_name_3,
+							  1,
+							  s_rd_3,
+							  &put_cont,
+							  s_name_3)));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+	      "Created record 1\n");
   GNUNET_asprintf(&s_name_1, "dummy1");
   s_rd_1 = create_record(1);
   GNUNET_assert (NULL != (ns_ops[0] = GNUNET_NAMESTORE_records_store(nsh, privkey, s_name_1,
diff --git a/src/namestore/test_plugin_namestore.c b/src/namestore/test_plugin_namestore.c
index d591eb3255..ac2a0bba95 100644
--- a/src/namestore/test_plugin_namestore.c
+++ b/src/namestore/test_plugin_namestore.c
@@ -90,6 +90,7 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
 
 static void
 test_record (void *cls,
+	     uint64_t seq,
              const struct GNUNET_CRYPTO_EcdsaPrivateKey *private_key,
              const char *label,
              unsigned int rd_count,
@@ -112,7 +113,9 @@ test_record (void *cls,
     GNUNET_assert (rd[i].record_type == 1 + (id % 13));
     GNUNET_assert (rd[i].flags == 0);
   }
-  memset (&tzone_private_key, (id % 241), sizeof (tzone_private_key));
+  memset (&tzone_private_key,
+	  (id % 241),
+	  sizeof (tzone_private_key));
   GNUNET_assert (0 == strcmp (label, tname));
   GNUNET_assert (0 == memcmp (&tzone_private_key,
                               private_key,
@@ -156,11 +159,12 @@ put_record (struct GNUNET_NAMESTORE_PluginFunctions *nsp,
   }
   memset (&zone_private_key, (id % 241), sizeof (zone_private_key));
   memset (&signature, (id % 243), sizeof (signature));
-  GNUNET_assert (GNUNET_OK == nsp->store_records (nsp->cls,
-						&zone_private_key,
-						label,
-						rd_count,
-						rd));
+  GNUNET_assert (GNUNET_OK ==
+		 nsp->store_records (nsp->cls,
+				     &zone_private_key,
+				     label,
+				     rd_count,
+				     rd));
 }