Convert MEMFS files to typed arrays before storing to IDBFS for better IDBFS performance. Add testing for -s MEMFS_APPEND_TO_TYPED_ARRAYS=1 option.

5 files changed, 27 insertions, 10 deletions

diff --git a/src/library_idbfs.js b/src/library_idbfs.js
index 91015e77..8082c196 100644
--- a/src/library_idbfs.js
+++ b/src/library_idbfs.js
@@ -135,6 +135,9 @@ mergeInto(LibraryManager.library, {
       if (FS.isDir(stat.mode)) {
         return callback(null, { timestamp: stat.mtime, mode: stat.mode });
       } else if (FS.isFile(stat.mode)) {
+        // Performance consideration: storing a normal JavaScript array to a IndexedDB is much slower than storing a typed array.
+        // Therefore always convert the file contents to a typed array first before writing the data to IndexedDB.
+        node.contents = MEMFS.getFileDataAsTypedArray(node);
         return callback(null, { timestamp: stat.mtime, mode: stat.mode, contents: node.contents });
       } else {
         return callback(new Error('node type not supported'));
diff --git a/src/library_memfs.js b/src/library_memfs.js
index 4b56ebbb..df1d306e 100644
--- a/src/library_memfs.js
+++ b/src/library_memfs.js
@@ -98,6 +98,16 @@ mergeInto(LibraryManager.library, {
       return node.contents; // No-op, the file contents are already in a JS array. Return as-is.
     },
 
+#if USE_TYPED_ARRAYS == 2
+    // Given a file node, returns its file data converted to a typed array.
+    getFileDataAsTypedArray: function(node) {
+      if (node.contents && node.contents.subarray) return node.contents; // No-op get if data already is in a typed array.
+      var ta = new Uint8Array(new ArrayBuffer(node.usedBytes));
+      for(var i = 0; i < node.usedBytes; ++i) ta[i] = node.contents[i];
+      return ta;
+    },
+#endif
+
     // Allocates a new backing store for the given node so that it can fit at least newSize amount of bytes.
     // May allocate more, to provide automatic geometric increase and amortized linear performance appending writes.
     // Never shrinks the storage.
diff --git a/src/settings.js b/src/settings.js
index 7d9d1b57..a8761d90 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -324,8 +324,8 @@ var FS_LOG = 0; // Log all FS operations.  This is especially helpful when you'r
 var CASE_INSENSITIVE_FS = 0; // If set to nonzero, the provided virtual filesystem if treated case-insensitive, like
                              // Windows and OSX do. If set to 0, the VFS is case-sensitive, like on Linux.
 var MEMFS_APPEND_TO_TYPED_ARRAYS = 0; // If set to nonzero, MEMFS will always utilize typed arrays as the backing store 
-                                      // for writing to files. The default behavior is to use typed arrays for files
-                                      // when the file size doesn't change (appending writes), and for files that do
+                                      // for appending data to files. The default behavior is to use typed arrays for files
+                                      // when the file size doesn't change after initial creation, and for files that do
                                       // change size, use normal JS arrays instead.
 
 var USE_BSS = 1; // https://en.wikipedia.org/wiki/.bss
diff --git a/tests/test_browser.py b/tests/test_browser.py
index c8e07b25..0771cb1b 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -1096,9 +1096,10 @@ keydown(100);keyup(100); // trigger the end
     shutil.move('test.html', 'third.html')
 
   def test_fs_idbfs_sync(self):
-    secret = str(time.time())
-    self.btest(path_from_root('tests', 'fs', 'test_idbfs_sync.c'), '1', force_c=True, args=['-DFIRST', '-DSECRET=\'' + secret + '\'', '-s', '''EXPORTED_FUNCTIONS=['_main', '_success']'''])
-    self.btest(path_from_root('tests', 'fs', 'test_idbfs_sync.c'), '1', force_c=True, args=['-DSECRET=\'' + secret + '\'', '-s', '''EXPORTED_FUNCTIONS=['_main', '_success']'''])
+    for mode in [[], ['-s', 'MEMFS_APPEND_TO_TYPED_ARRAYS=1']]:
+      secret = str(time.time())
+      self.btest(path_from_root('tests', 'fs', 'test_idbfs_sync.c'), '1', force_c=True, args=mode + ['-DFIRST', '-DSECRET=\'' + secret + '\'', '-s', '''EXPORTED_FUNCTIONS=['_main', '_success']'''])
+      self.btest(path_from_root('tests', 'fs', 'test_idbfs_sync.c'), '1', force_c=True, args=mode + ['-DSECRET=\'' + secret + '\'', '-s', '''EXPORTED_FUNCTIONS=['_main', '_success']'''])
 
   def test_sdl_pumpevents(self):
     # key events should be detected using SDL_PumpEvents
diff --git a/tests/test_core.py b/tests/test_core.py
index f34ba03e..9541db36 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -4031,8 +4031,8 @@ def process(filename):
 
     mem_file = 'src.cpp.o.js.mem'
     orig_args = self.emcc_args
-    for modes in [[], ['-s', 'MEMFS_APPEND_TO_TYPED_ARRAYS=1']]:
-      self.emcc_args = orig_args + modes
+    for mode in [[], ['-s', 'MEMFS_APPEND_TO_TYPED_ARRAYS=1']]:
+      self.emcc_args = orig_args + mode
       try_delete(mem_file)
       self.do_run(src, ('size: 7\ndata: 100,-56,50,25,10,77,123\nloop: 100 -56 50 25 10 77 123 \ninput:hi there!\ntexto\n$\n5 : 10,30,20,11,88\nother=some data.\nseeked=me da.\nseeked=ata.\nseeked=ta.\nfscanfed: 10 - hello\nok.\ntexte\n', 'size: 7\ndata: 100,-56,50,25,10,77,123\nloop: 100 -56 50 25 10 77 123 \ninput:hi there!\ntexto\ntexte\n$\n5 : 10,30,20,11,88\nother=some data.\nseeked=me da.\nseeked=ata.\nseeked=ta.\nfscanfed: 10 - hello\nok.\n'),
                   post_build=post, extra_emscripten_args=['-H', 'libc/fcntl.h'])
@@ -4080,7 +4080,10 @@ def process(filename):
     test_path = path_from_root('tests', 'core', 'test_fwrite_0')
     src, output = (test_path + s for s in ('.in', '.out'))
 
-    self.do_run_from_file(src, output)
+    orig_args = self.emcc_args
+    for mode in [[], ['-s', 'MEMFS_APPEND_TO_TYPED_ARRAYS=1']]:
+      self.emcc_args = orig_args + mode
+      self.do_run_from_file(src, output)
 
   def test_fgetc_ungetc(self):
     src = open(path_from_root('tests', 'stdio', 'test_fgetc_ungetc.c'), 'r').read()
@@ -4279,8 +4282,8 @@ def process(filename):
     test_path = path_from_root('tests', 'core', 'test_wprintf')
     src, output = (test_path + s for s in ('.c', '.out'))
     orig_args = self.emcc_args
-    for modes in [[], ['-s', 'MEMFS_APPEND_TO_TYPED_ARRAYS=1']]:
-      self.emcc_args = orig_args + modes
+    for mode in [[], ['-s', 'MEMFS_APPEND_TO_TYPED_ARRAYS=1']]:
+      self.emcc_args = orig_args + mode
       self.do_run_from_file(src, output)
 
   def test_direct_string_constant_usage(self):