enable caching private->public key mapping in memory to improve CPU consumption for large zone insertions

5 files changed, 155 insertions, 23 deletions

diff --git a/src/gns/gnunet-gns-proxy.c b/src/gns/gnunet-gns-proxy.c
index 8b9aa599e7..08663a57e9 100644
--- a/src/gns/gnunet-gns-proxy.c
+++ b/src/gns/gnunet-gns-proxy.c
@@ -855,9 +855,9 @@ mhd_content_cb (void *cls,
     return MHD_CONTENT_READER_END_OF_STREAM;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Writing %lu/%lu bytes\n",
-              bytes_to_copy,
-              s5r->io_len);
+              "Writing %llu/%llu bytes\n",
+              (unsigned long long) bytes_to_copy,
+              (unsigned long long) s5r->io_len);
   GNUNET_memcpy (buf,
                  s5r->io_buf,
                  bytes_to_copy);
@@ -1307,12 +1307,12 @@ curl_download_cb (void *ptr,
   if (sizeof (s5r->io_buf) - s5r->io_len < total)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "Pausing CURL `%s%s' download, not enough space %lu %lu %lu\n",
+                "Pausing CURL `%s%s' download, not enough space %llu %llu %llu\n",
                 s5r->domain,
                 s5r->url,
-                sizeof (s5r->io_buf),
-                s5r->io_len,
-                total);
+                (unsigned long long) sizeof (s5r->io_buf),
+                (unsigned long long) s5r->io_len,
+                (unsigned long long) total);
     return CURL_WRITEFUNC_PAUSE; /* not enough space */
   }
   GNUNET_memcpy (&s5r->io_buf[s5r->io_len],
diff --git a/src/gnsrecord/gnsrecord_crypto.c b/src/gnsrecord/gnsrecord_crypto.c
index dd628ea767..27c83b90ef 100644
--- a/src/gnsrecord/gnsrecord_crypto.c
+++ b/src/gnsrecord/gnsrecord_crypto.c
@@ -72,6 +72,7 @@ derive_block_aes_key (struct GNUNET_CRYPTO_SymmetricInitializationVector *iv,
  * Sign name and records
  *
  * @param key the private key
+ * @param pkey associated public key
  * @param expire block expiration
  * @param label the name for the records
  * @param rd record data
@@ -79,17 +80,17 @@ derive_block_aes_key (struct GNUNET_CRYPTO_SymmetricInitializationVector *iv,
  * @return NULL on error (block too large)
  */
 struct GNUNET_GNSRECORD_Block *
-GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
-			       struct GNUNET_TIME_Absolute expire,
-			       const char *label,
-			       const struct GNUNET_GNSRECORD_Data *rd,
-			       unsigned int rd_count)
+block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
+              const struct GNUNET_CRYPTO_EcdsaPublicKey *pkey,
+              struct GNUNET_TIME_Absolute expire,
+              const char *label,
+              const struct GNUNET_GNSRECORD_Data *rd,
+              unsigned int rd_count)
 {
   size_t payload_len = GNUNET_GNSRECORD_records_get_size (rd_count,
                                                           rd);
   char payload[sizeof (uint32_t) + payload_len];
   struct GNUNET_GNSRECORD_Block *block;
-  struct GNUNET_CRYPTO_EcdsaPublicKey pkey;
   struct GNUNET_CRYPTO_EcdsaPrivateKey *dkey;
   struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
   struct GNUNET_CRYPTO_SymmetricSessionKey skey;
@@ -139,12 +140,10 @@ GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
                                                  "gns");
   GNUNET_CRYPTO_ecdsa_key_get_public (dkey,
                                       &block->derived_key);
-  GNUNET_CRYPTO_ecdsa_key_get_public (key,
-                                      &pkey);
   derive_block_aes_key (&iv,
                         &skey,
                         label,
-                        &pkey);
+                        pkey);
   GNUNET_break (payload_len + sizeof (uint32_t) ==
 		GNUNET_CRYPTO_symmetric_encrypt (payload,
                                                  payload_len + sizeof (uint32_t),
@@ -167,6 +166,97 @@ GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 
 
 /**
+ * Sign name and records
+ *
+ * @param key the private key
+ * @param expire block expiration
+ * @param label the name for the records
+ * @param rd record data
+ * @param rd_count number of records
+ * @return NULL on error (block too large)
+ */
+struct GNUNET_GNSRECORD_Block *
+GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
+			       struct GNUNET_TIME_Absolute expire,
+			       const char *label,
+			       const struct GNUNET_GNSRECORD_Data *rd,
+			       unsigned int rd_count)
+{
+  struct GNUNET_CRYPTO_EcdsaPublicKey pkey;
+
+  GNUNET_CRYPTO_ecdsa_key_get_public (key,
+                                      &pkey);
+  return block_create (key,
+                       &pkey,
+                       expire,
+                       label,
+                       rd,
+                       rd_count);
+}
+
+
+/**
+ * Line in cache mapping private keys to public keys.
+ */
+struct KeyCacheLine
+{
+  /**
+   * A private key.
+   */
+  struct GNUNET_CRYPTO_EcdsaPrivateKey key;
+
+  /**
+   * Associated public key.
+   */
+  struct GNUNET_CRYPTO_EcdsaPublicKey pkey;
+
+};
+
+
+/**
+ * Sign name and records, cache derived public key (also keeps the
+ * private key in static memory, so do not use this function if
+ * keeping the private key in the process'es RAM is a major issue).
+ *
+ * @param key the private key
+ * @param expire block expiration
+ * @param label the name for the records
+ * @param rd record data
+ * @param rd_count number of records
+ * @return NULL on error (block too large)
+ */
+struct GNUNET_GNSRECORD_Block *
+GNUNET_GNSRECORD_block_create2 (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
+                                struct GNUNET_TIME_Absolute expire,
+                                const char *label,
+                                const struct GNUNET_GNSRECORD_Data *rd,
+                                unsigned int rd_count)
+{
+#define CSIZE 64
+  static struct KeyCacheLine cache[CSIZE];
+  struct KeyCacheLine *line;
+
+  line = &cache[(*(unsigned int *) key) ^ CSIZE];
+  if (0 != memcmp (&line->key,
+                   key,
+                   sizeof (*key)))
+  {
+    /* cache miss, recompute */
+    line->key = *key;
+    GNUNET_CRYPTO_ecdsa_key_get_public (key,
+                                        &line->pkey);
+  }
+  return block_create (key,
+                       &line->pkey,
+                       expire,
+                       label,
+                       rd,
+                       rd_count);
+}
+
+
+
+/**
  * Check if a signature is valid.  This API is used by the GNS Block
  * to validate signatures received from the network.
  *
diff --git a/src/include/gnunet_gnsrecord_lib.h b/src/include/gnunet_gnsrecord_lib.h
index f07bd3ef30..38346ada3e 100644
--- a/src/include/gnunet_gnsrecord_lib.h
+++ b/src/include/gnunet_gnsrecord_lib.h
@@ -558,6 +558,25 @@ GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 
 
 /**
+ * Sign name and records, cache derived public key (also keeps the
+ * private key in static memory, so do not use this function if
+ * keeping the private key in the process'es RAM is a major issue).
+ *
+ * @param key the private key
+ * @param expire block expiration
+ * @param label the name for the records
+ * @param rd record data
+ * @param rd_count number of records in @a rd
+ */
+struct GNUNET_GNSRECORD_Block *
+GNUNET_GNSRECORD_block_create2 (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
+                                struct GNUNET_TIME_Absolute expire,
+                                const char *label,
+                                const struct GNUNET_GNSRECORD_Data *rd,
+                                unsigned int rd_count);
+
+
+/**
  * Check if a signature is valid.  This API is used by the GNS Block
  * to validate signatures received from the network.
  *
diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c
index f8ac6c31c9..c5a37dcac3 100644
--- a/src/namestore/gnunet-service-namestore.c
+++ b/src/namestore/gnunet-service-namestore.c
@@ -251,6 +251,12 @@ static struct ZoneMonitor *monitor_tail;
  */
 static struct GNUNET_NotificationContext *monitor_nc;
 
+/**
+ * Optimize block insertion by caching map of private keys to
+ * public keys in memory?
+ */
+static int cache_keys;
+
 
 /**
  * Task run during shutdown.
@@ -707,11 +713,18 @@ refresh_block (struct NamestoreClient *nc,
     ? GNUNET_TIME_UNIT_ZERO_ABS
     : GNUNET_GNSRECORD_record_get_expiration_time (res_count,
                                                    res);
-  block = GNUNET_GNSRECORD_block_create (zone_key,
-                                         exp_time,
-                                         name,
-                                         res,
-                                         res_count);
+  if (cache_keys)
+    block = GNUNET_GNSRECORD_block_create2 (zone_key,
+                                            exp_time,
+                                            name,
+                                            res,
+                                            res_count);
+  else
+    block = GNUNET_GNSRECORD_block_create (zone_key,
+                                           exp_time,
+                                           name,
+                                           res,
+                                           res_count);
   GNUNET_assert (NULL != block);
   GNUNET_CRYPTO_ecdsa_key_get_public (zone_key,
                                       &pkey);
@@ -1722,6 +1735,9 @@ run (void *cls,
   monitor_nc = GNUNET_notification_context_create (1);
   namecache = GNUNET_NAMECACHE_connect (cfg);
   /* Loading database plugin */
+  cache_keys = GNUNET_CONFIGURATION_get_value_yesno (cfg,
+                                                     "namestore",
+                                                     "CACHE_KEYS");
   if (GNUNET_OK !=
       GNUNET_CONFIGURATION_get_value_string (cfg,
                                              "namestore",
diff --git a/src/namestore/namestore.conf.in b/src/namestore/namestore.conf.in
index f19ac56439..8b5e440b87 100644
--- a/src/namestore/namestore.conf.in
+++ b/src/namestore/namestore.conf.in
@@ -9,8 +9,17 @@ HOSTNAME = localhost
 BINARY = gnunet-service-namestore
 ACCEPT_FROM = 127.0.0.1;
 ACCEPT_FROM6 = ::1;
+
+# Which database should we use?
 DATABASE = sqlite
 
+# Should we optimize publishing record by caching the mapping
+# from zone private keys to zone public keys in memory?
+# (Set to NO if totally paranoid about keeping private keys
+# in RAM longer than necessary.)
+CACHE_KEYS = YES
+
+
 [namestore-sqlite]
 FILENAME = $GNUNET_DATA_HOME/namestore/sqlite.db
 
@@ -38,5 +47,3 @@ UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-fcfsd.sock
 
 # On what port does the FCFS daemon listen for HTTP clients?
 HTTPPORT = 18080
-
-