Merge branch 'master' of gnunet.org:gnunet

6 files changed, 189 insertions, 23 deletions

diff --git a/src/include/gnunet_namestore_service.h b/src/include/gnunet_namestore_service.h
index 4828f72ade..b8c7ea810f 100644
--- a/src/include/gnunet_namestore_service.h
+++ b/src/include/gnunet_namestore_service.h
@@ -109,6 +109,12 @@ typedef void
  * it is replaced with the new record.  Use an empty array to
  * remove all records under the given name.
  *
+ * The continuation is called after the value has been stored in the
+ * database. Monitors may be notified asynchronously (basically with
+ * a buffer). However, if any monitor is consistently too slow to
+ * keep up with the changes, calling @a cont will be delayed until the
+ * monitors do keep up.
+ *
  * @param h handle to the namestore
  * @param pkey private key of the zone
  * @param label name that is being mapped
@@ -313,7 +319,7 @@ struct GNUNET_NAMESTORE_ZoneMonitor;
  *         that monitoring is down. You need to still explicitly call
  *         #GNUNET_NAMESTORE_zone_monitor_stop().
  * @param error_cb_cls closure for @a error_cb
- * @param monitor function to call on zone changes
+ * @param monitor function to call on zone changes, with an initial limit of 1
  * @param monitor_cls closure for @a monitor
  * @param sync_cb function called when we're in sync with the namestore
  * @param sync_cb_cls closure for @a sync_cb
@@ -332,6 +338,32 @@ GNUNET_NAMESTORE_zone_monitor_start (const struct GNUNET_CONFIGURATION_Handle *c
 
 
 /**
+ * Calls the monitor processor specified in #GNUNET_NAMESTORE_zone_monitor_start
+ * for the next record(s).  This function is used to allow clients that merely
+ * monitor the NAMESTORE to still throttle namestore operations, so we can be
+ * sure that the monitors can keep up.
+ *
+ * Note that #GNUNET_NAMESTORE_records_store() only waits for this
+ * call if the previous limit set by the client was already reached.
+ * Thus, by using a @a limit greater than 1, monitors basically enable
+ * a queue of notifications to be processed asynchronously with some
+ * delay.  Note that even with a limit of 1 the
+ * #GNUNET_NAMESTORE_records_store() function will run asynchronously
+ * and the continuation may be invoked before the monitors completed
+ * (or even started) processing the notification.  Thus, monitors will
+ * only closely track the current state of the namestore, but not
+ * be involved in the transactions.
+ *
+ * @param zm the monitor
+ * @param limit number of records to return to the iterator in one shot
+ *        (before #GNUNET_NAMESTORE_zone_monitor_next is to be called again)
+ */
+void
+GNUNET_NAMESTORE_zone_monitor_next (struct GNUNET_NAMESTORE_ZoneMonitor *zm,
+                                    uint64_t limit);
+
+
+/**
  * Stop monitoring a zone for changes.
  *
  * @param zm handle to the monitor activity to stop
diff --git a/src/include/gnunet_protocols.h b/src/include/gnunet_protocols.h
index bf1b48679c..1ef00cfc8e 100644
--- a/src/include/gnunet_protocols.h
+++ b/src/include/gnunet_protocols.h
@@ -1306,13 +1306,20 @@ extern "C"
 #define GNUNET_MESSAGE_TYPE_NAMESTORE_RECORD_RESULT 443
 
 /**
+ * Client to service: I am now ready for the next (set of) monitor
+ * events. Monitoring equivlaent of
+ * #GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT.
+ */
+#define GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_NEXT 444
+
+/**
  * Client to service: please start iteration; receives
  * "GNUNET_MESSAGE_TYPE_NAMESTORE_LOOKUP_NAME_RESPONSE" messages in return.
  */
 #define GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_START 445
 
 /**
- * Client to service: next record in iteration please.
+ * Client to service: next record(s) in iteration please.
  */
 #define GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_ITERATION_NEXT 447
 
diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c
index fa189dbc39..f47c8776bb 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -169,6 +169,12 @@ struct ZoneMonitor
    */
   uint64_t seq;
 
+  /**
+   * Current limit of how many more messages we are allowed
+   * to queue to this monitor.
+   */
+  uint64_t limit;
+
 };
 
 
@@ -1667,7 +1673,7 @@ monitor_iterate_cb (void *cls,
  * Handles a #GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_START message
  *
  * @param cls the client sending the message
- * @param message message from the client
+ * @param zis_msg message from the client
  */
 static void
 handle_monitor_start (void *cls,
@@ -1685,7 +1691,7 @@ handle_monitor_start (void *cls,
 			       monitor_tail,
 			       zm);
   GNUNET_SERVICE_client_mark_monitor (nc->client);
-  GNUNET_SERVICE_client_disable_continue_warning (nc->client);
+  GNUNET_SERVICE_client_continue (nc->client);
   GNUNET_notification_context_add (monitor_nc,
 				   nc->mq);
   if (GNUNET_YES == ntohl (zis_msg->iterate_first))
@@ -1733,6 +1739,51 @@ monitor_next (void *cls)
 
 
 /**
+ * Handles a #GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_NEXT message
+ *
+ * @param cls the client sending the message
+ * @param nm message from the client
+ */
+static void
+handle_monitor_next (void *cls,
+                     const struct ZoneMonitorNextMessage *nm)
+{
+  struct NamestoreClient *nc = cls;
+  struct ZoneMonitor *zm;
+  uint64_t inc;
+
+  inc = GNUNET_ntohll (nm->limit);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+	      "Received ZONE_MONITOR_NEXT message with limit %llu\n",
+              (unsigned long long) inc);
+  for (zm = monitor_head; NULL != zm; zm = zm->next)
+    if (zm->nc == nc)
+      break;
+  if (NULL == zm)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (nc->client);
+    return;
+  }
+  GNUNET_SERVICE_client_continue (nc->client);
+  if (zm->limit + inc < zm->limit)
+  {
+    GNUNET_break (0);
+    GNUNET_SERVICE_client_drop (nc->client);
+    return;
+  }
+  zm->limit += inc;
+#if 0
+  if (GNUNET_YES == ntohl (zis_msg->iterate_first))
+    zm->task = GNUNET_SCHEDULER_add_now (&monitor_next,
+					 zm);
+  else
+    monitor_sync (zm);
+#endif
+}
+
+
+/**
  * Process namestore requests.
  *
  * @param cls closure
@@ -1831,6 +1882,10 @@ GNUNET_SERVICE_MAIN
 			  GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_START,
 			  struct ZoneMonitorStartMessage,
 			  NULL),
+ GNUNET_MQ_hd_fixed_size (monitor_next,
+			  GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_NEXT,
+			  struct ZoneMonitorNextMessage,
+			  NULL),
  GNUNET_MQ_handler_end ());
 
 
diff --git a/src/namestore/namestore.h b/src/namestore/namestore.h
index 207b35662f..679ca3d3de 100644
--- a/src/namestore/namestore.h
+++ b/src/namestore/namestore.h
@@ -161,7 +161,7 @@ struct LabelLookupResponseMessage
    * Length of serialized record data
    */
   uint16_t rd_len GNUNET_PACKED;
-  
+
   /**
    * Number of records contained
    */
@@ -319,6 +319,32 @@ struct ZoneMonitorStartMessage
 
 
 /**
+ * Ask for next result of zone iteration for the given operation
+ */
+struct ZoneMonitorNextMessage
+{
+  /**
+   * Type will be #GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_NEXT
+   */
+  struct GNUNET_MessageHeader header;
+
+  /**
+   * Always zero.
+   */
+  uint32_t reserved;
+
+  /**
+   * Number of records to return to the iterator in one shot
+   * (before #GNUNET_MESSAGE_TYPE_NAMESTORE_ZONE_MONITOR_NEXT
+   * should be send again). In NBO.
+   */
+  uint64_t limit;
+
+};
+
+
+
+/**
  * Start a zone iteration for the given zone
  */
 struct ZoneIterationStartMessage
diff --git a/src/namestore/namestore_api_monitor.c b/src/namestore/namestore_api_monitor.c
index 8e6d39ad7e..9ba90833b3 100644
--- a/src/namestore/namestore_api_monitor.c
+++ b/src/namestore/namestore_api_monitor.c
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2013, 2016 GNUnet e.V.
+     Copyright (C) 2013, 2016, 2018 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -17,7 +17,6 @@
      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
      Boston, MA 02110-1301, USA.
 */
-
 /**
  * @file namestore/namestore_api_monitor.c
  * @brief API to monitor changes in the NAMESTORE
@@ -340,6 +339,42 @@ GNUNET_NAMESTORE_zone_monitor_start (const struct GNUNET_CONFIGURATION_Handle *c
 
 
 /**
+ * Calls the monitor processor specified in #GNUNET_NAMESTORE_zone_monitor_start
+ * for the next record(s).  This function is used to allow clients that merely
+ * monitor the NAMESTORE to still throttle namestore operations, so we can be
+ * sure that the monitors can keep up.
+ *
+ * Note that #GNUNET_NAMESTORE_records_store() only waits for this
+ * call if the previous limit set by the client was already reached.
+ * Thus, by using a @a limit greater than 1, monitors basically enable
+ * a queue of notifications to be processed asynchronously with some
+ * delay.  Note that even with a limit of 1 the
+ * #GNUNET_NAMESTORE_records_store() function will run asynchronously
+ * and the continuation may be invoked before the monitors completed
+ * (or even started) processing the notification.  Thus, monitors will
+ * only closely track the current state of the namestore, but not
+ * be involved in the transactions.
+ *
+ * @param zm the monitor
+ * @param limit number of records to return to the iterator in one shot
+ *        (before #GNUNET_NAMESTORE_zone_monitor_next is to be called again)
+ */
+void
+GNUNET_NAMESTORE_zone_monitor_next (struct GNUNET_NAMESTORE_ZoneMonitor *zm,
+                                    uint64_t limit)
+{
+  struct GNUNET_MQ_Envelope *env;
+  struct ZoneMonitorNextMessage *nm;
+
+  env = GNUNET_MQ_msg (nm,
+                       GNUNET_MESSAGE_TYPE_NAMESTORE_MONITOR_NEXT);
+  nm->limit = GNUNET_htonll (limit);
+  GNUNET_MQ_send (zm->mq,
+                  env);
+}
+
+
+/**
  * Stop monitoring a zone for changes.
  *
  * @param zm handle to the monitor activity to stop
diff --git a/src/namestore/plugin_namestore_flat.c b/src/namestore/plugin_namestore_flat.c
index 88b3ce9b4a..bbb9e3c628 100644
--- a/src/namestore/plugin_namestore_flat.c
+++ b/src/namestore/plugin_namestore_flat.c
@@ -140,7 +140,7 @@ database_setup (struct Plugin *plugin)
   char *rvalue;
   char *record_count;
   size_t record_data_size;
-  size_t size;
+  uint64_t size;
   size_t key_len;
   struct GNUNET_HashCode hkey;
   struct GNUNET_DISK_FileHandle *fh;
@@ -218,7 +218,8 @@ database_setup (struct Plugin *plugin)
   if (0 < size)
   {
     line = strtok (buffer, "\n");
-    while (line != NULL) {
+    while (line != NULL)
+    {
       zone_private_key = strtok (line, ",");
       if (NULL == zone_private_key)
         break;
@@ -236,23 +237,33 @@ database_setup (struct Plugin *plugin)
         break;
       line = strtok (NULL, "\n");
       entry = GNUNET_new (struct FlatFileEntry);
-      if (1 != sscanf (rvalue,
-		       "%lu",
-		       &entry->rvalue))
       {
-        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                    "Error parsing entry\n");
-        GNUNET_free (entry);
-        break;
+        unsigned long long ll;
+
+        if (1 != sscanf (rvalue,
+                         "%llu",
+                         &ll))
+        {
+          GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                      "Error parsing entry\n");
+          GNUNET_free (entry);
+          break;
+        }
+        entry->rvalue = (uint64_t) ll;
       }
-      if (1 != sscanf (record_count,
-		       "%u",
-		       &entry->record_count))
       {
-        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                    "Error parsing entry\n");
-        GNUNET_free (entry);
-        break;
+        unsigned int ui;
+
+        if (1 != sscanf (record_count,
+                         "%u",
+                         &ui))
+        {
+          GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                      "Error parsing entry\n");
+          GNUNET_free (entry);
+          break;
+        }
+        entry->record_count = (uint32_t) ui;
       }
       entry->label = GNUNET_strdup (label);
       record_data_size