- added idbfs and nodefs

 - minor coding convention fixes

6 files changed, 471 insertions, 20 deletions

diff --git a/src/library_fs.js b/src/library_fs.js
index 983e0d76..c4b29227 100644
--- a/src/library_fs.js
+++ b/src/library_fs.js
@@ -1,5 +1,5 @@
 mergeInto(LibraryManager.library, {
-  $FS__deps: ['$ERRNO_CODES', '$ERRNO_MESSAGES', '__setErrNo', '$VFS', '$PATH', '$TTY', '$MEMFS', 'stdin', 'stdout', 'stderr', 'fflush'],
+  $FS__deps: ['$ERRNO_CODES', '$ERRNO_MESSAGES', '__setErrNo', '$VFS', '$PATH', '$TTY', '$MEMFS', '$IDBFS', '$NODEFS', 'stdin', 'stdout', 'stderr', 'fflush'],
   $FS__postset: 'FS.staticInit();' +
                 '__ATINIT__.unshift({ func: function() { if (!Module["noFSInit"] && !FS.init.initialized) FS.init() } });' +
                 '__ATMAIN__.push({ func: function() { FS.ignorePermissions = false } });' +
@@ -818,6 +818,8 @@ mergeInto(LibraryManager.library, {
       if ((flags & {{{ cDefine('O_TRUNC')}}})) {
         FS.truncate(node, 0);
       }
+      // we've already handled these, don't pass down to the underlying vfs
+      flags &= ~({{{ cDefine('O_EXCL') }}} | {{{ cDefine('O_TRUNC') }}});
 
       // register the stream with the filesystem
       var stream = FS.createStream({
diff --git a/src/library_idbfs.js b/src/library_idbfs.js
new file mode 100644
index 00000000..9031bad8
--- /dev/null
+++ b/src/library_idbfs.js
@@ -0,0 +1,216 @@
+mergeInto(LibraryManager.library, {
+  $IDBFS__deps: ['$FS', '$MEMFS', '$PATH'],
+  $IDBFS: {
+    dbs: {},
+    indexedDB: function() {
+      return window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB;
+    },
+    DB_VERSION: 20,
+    DB_STORE_NAME: 'FILE_DATA',
+    // reuse all of the core MEMFS functionality
+    mount: function(mount) {
+      return MEMFS.mount.apply(null, arguments);
+    },
+    // the only custom function IDBFS implements is to handle
+    // synchronizing the wrapped MEMFS with a backing IDB instance
+    syncfs: function(mount, populate, callback) {
+      IDBFS.getLocalSet(mount, function(err, local) {
+        if (err) return callback(err);
+
+        IDBFS.getRemoteSet(mount, function(err, remote) {
+          if (err) return callback(err);
+
+          var src = populate ? remote : local;
+          var dst = populate ? local : remote;
+
+          IDBFS.reconcile(src, dst, callback);
+        });
+      });
+    },
+    reconcile: function(src, dst, callback) {
+      var total = 0;
+
+      var create = {};
+      for (var key in src.files) {
+        if (!src.files.hasOwnProperty(key)) continue;
+        var e = src.files[key];
+        var e2 = dst.files[key];
+        if (!e2 || e.timestamp > e2.timestamp) {
+          create[key] = e;
+          total++;
+        }
+      }
+
+      var remove = {};
+      for (var key in dst.files) {
+        if (!dst.files.hasOwnProperty(key)) continue;
+        var e = dst.files[key];
+        var e2 = src.files[key];
+        if (!e2) {
+          remove[key] = e;
+          total++;
+        }
+      }
+
+      if (!total) {
+        // early out
+        return callback(null);
+      }
+
+      var completed = 0;
+      var done = function(err) {
+        if (err) return callback(err);
+        if (++completed >= total) {
+          return callback(null);
+        }
+      };
+
+      // create a single transaction to handle and IDB reads / writes we'll need to do
+      var db = src.type === 'remote' ? src.db : dst.db;
+      var transaction = db.transaction([IDBFS.DB_STORE_NAME], 'readwrite');
+      transaction.onerror = function() { callback(this.error); };
+      var store = transaction.objectStore(IDBFS.DB_STORE_NAME);
+
+      for (var path in create) {
+        if (!create.hasOwnProperty(path)) continue;
+        var entry = create[path];
+
+        if (dst.type === 'local') {
+          // save file to local
+          try {
+            if (FS.isDir(entry.mode)) {
+              FS.mkdir(path, entry.mode);
+            } else if (FS.isFile(entry.mode)) {
+              var stream = FS.open(path, 'w+', 0666);
+              FS.write(stream, entry.contents, 0, entry.contents.length, 0, true /* canOwn */);
+              FS.close(stream);
+            }
+            done(null);
+          } catch (e) {
+            return done(e);
+          }
+        } else {
+          // save file to IDB
+          var req = store.put(entry, path);
+          req.onsuccess = function() { done(null); };
+          req.onerror = function() { done(this.error); };
+        }
+      }
+
+      for (var path in remove) {
+        if (!remove.hasOwnProperty(path)) continue;
+        var entry = remove[path];
+
+        if (dst.type === 'local') {
+          // delete file from local
+          try {
+            if (FS.isDir(entry.mode)) {
+              // TODO recursive delete?
+              FS.rmdir(path);
+            } else if (FS.isFile(entry.mode)) {
+              FS.unlink(path);
+            }
+            done(null);
+          } catch (e) {
+            return done(e);
+          }
+        } else {
+          // delete file from IDB
+          var req = store.delete(path);
+          req.onsuccess = function() { done(null); };
+          req.onerror = function() { done(this.error); };
+        }
+      }
+    },
+    getLocalSet: function(mount, callback) {
+      var files = {};
+
+      var isRealDir = function(p) {
+        return p !== '.' && p !== '..';
+      };
+      var toAbsolute = function(root) {
+        return function(p) {
+          return PATH.join(root, p);
+        }
+      };
+
+      var check = FS.readdir(mount.mountpoint)
+        .filter(isRealDir)
+        .map(toAbsolute(mount.mountpoint));
+
+      while (check.length) {
+        var path = check.pop();
+        var stat, node;
+
+        try {
+          var lookup = FS.lookupPath(path);
+          node = lookup.node;
+          stat = FS.stat(path);
+        } catch (e) {
+          return callback(e);
+        }
+
+        if (FS.isDir(stat.mode)) {
+          check.push.apply(check, FS.readdir(path)
+            .filter(isRealDir)
+            .map(toAbsolute(path)));
+
+          files[path] = { mode: stat.mode, timestamp: stat.mtime };
+        } else if (FS.isFile(stat.mode)) {
+          files[path] = { contents: node.contents, mode: stat.mode, timestamp: stat.mtime };
+        } else {
+          return callback(new Error('node type not supported'));
+        }
+      }
+
+      return callback(null, { type: 'local', files: files });
+    },
+    getDB: function(name, callback) {
+      // look it up in the cache
+      var db = IDBFS.dbs[name];
+      if (db) {
+        return callback(null, db);
+      }
+      var req;
+      try {
+        req = IDBFS.indexedDB().open(name, IDBFS.DB_VERSION);
+      } catch (e) {
+        return onerror(e);
+      }
+      req.onupgradeneeded = function() {
+        db = req.result;
+        db.createObjectStore(IDBFS.DB_STORE_NAME);
+      };
+      req.onsuccess = function() {
+        db = req.result;
+        // add to the cache
+        IDBFS.dbs[name] = db;
+        callback(null, db);
+      };
+      req.onerror = function() {
+        callback(this.error);
+      };
+    },
+    getRemoteSet: function(mount, callback) {
+      var files = {};
+
+      IDBFS.getDB(mount.mountpoint, function(err, db) {
+        if (err) return callback(err);
+
+        var transaction = db.transaction([IDBFS.DB_STORE_NAME], 'readonly');
+        transaction.onerror = function() { callback(this.error); };
+
+        var store = transaction.objectStore(IDBFS.DB_STORE_NAME);
+        store.openCursor().onsuccess = function(event) {
+          var cursor = event.target.result;
+          if (!cursor) {
+            return callback(null, { type: 'remote', db: db, files: files });
+          }
+
+          files[cursor.key] = cursor.value;
+          cursor.continue();
+        };
+      });
+    }
+  }
+});
diff --git a/src/library_memfs.js b/src/library_memfs.js
index 28178d9f..4e56d996 100644
--- a/src/library_memfs.js
+++ b/src/library_memfs.js
@@ -5,18 +5,10 @@ mergeInto(LibraryManager.library, {
     CONTENT_OWNING: 1, // contains a subarray into the heap, and we own it, without copying (note: someone else needs to free() it, if that is necessary)
     CONTENT_FLEXIBLE: 2, // has been modified or never set to anything, and is a flexible js array that can grow/shrink
     CONTENT_FIXED: 3, // contains some fixed-size content written into it, in a typed array
-    ensureFlexible: function(node) {
-      if (node.contentMode !== MEMFS.CONTENT_FLEXIBLE) {
-        var contents = node.contents;
-        node.contents = Array.prototype.slice.call(contents);
-        node.contentMode = MEMFS.CONTENT_FLEXIBLE;
-      }
-    },
-
     mount: function(mount) {
-      return MEMFS.create_node(null, '/', {{{ cDefine('S_IFDIR') }}} | 0777, 0);
+      return MEMFS.createNode(null, '/', {{{ cDefine('S_IFDIR') }}} | 0777, 0);
     },
-    create_node: function(parent, name, mode, dev) {
+    createNode: function(parent, name, mode, dev) {
       if (FS.isBlkdev(mode) || FS.isFIFO(mode)) {
         // no supported
         throw new FS.ErrnoError(ERRNO_CODES.EPERM);
@@ -74,6 +66,13 @@ mergeInto(LibraryManager.library, {
       }
       return node;
     },
+    ensureFlexible: function(node) {
+      if (node.contentMode !== MEMFS.CONTENT_FLEXIBLE) {
+        var contents = node.contents;
+        node.contents = Array.prototype.slice.call(contents);
+        node.contentMode = MEMFS.CONTENT_FLEXIBLE;
+      }
+    },
     node_ops: {
       getattr: function(node) {
         var attr = {};
@@ -121,7 +120,7 @@ mergeInto(LibraryManager.library, {
         throw new FS.ErrnoError(ERRNO_CODES.ENOENT);
       },
       mknod: function(parent, name, mode, dev) {
-        return MEMFS.create_node(parent, name, mode, dev);
+        return MEMFS.createNode(parent, name, mode, dev);
       },
       rename: function(old_node, new_dir, new_name) {
         // if we're overwriting a directory at new_name, make sure it's empty.
@@ -163,7 +162,7 @@ mergeInto(LibraryManager.library, {
         return entries;
       },
       symlink: function(parent, newname, oldpath) {
-        var node = MEMFS.create_node(parent, newname, 0777 | {{{ cDefine('S_IFLNK') }}}, 0);
+        var node = MEMFS.createNode(parent, newname, 0777 | {{{ cDefine('S_IFLNK') }}}, 0);
         node.link = oldpath;
         return node;
       },
diff --git a/src/library_nodefs.js b/src/library_nodefs.js
new file mode 100644
index 00000000..d8df1689
--- /dev/null
+++ b/src/library_nodefs.js
@@ -0,0 +1,234 @@
+mergeInto(LibraryManager.library, {
+  $NODEFS__deps: ['$FS', '$PATH'],
+  $NODEFS__postset: 'if (ENVIRONMENT_IS_NODE) { var fs = require("fs"); }',
+  $NODEFS: {
+    mount: function (mount) {
+      assert(ENVIRONMENT_IS_NODE);
+      return NODEFS.createNode(null, '/', NODEFS.getMode(mount.opts.root), 0);
+    },
+    createNode: function (parent, name, mode, dev) {
+      if (!FS.isDir(mode) && !FS.isFile(mode) && !FS.isLink(mode)) {
+        throw new FS.ErrnoError(ERRNO_CODES.EINVAL);
+      }
+      var node = FS.createNode(parent, name, mode);
+      node.node_ops = NODEFS.node_ops;
+      node.stream_ops = NODEFS.stream_ops;
+      return node;
+    },
+    getMode: function (path) {
+      var stat;
+      try {
+        stat = fs.lstatSync(path);
+      } catch (e) {
+        if (!e.code) throw e;
+        throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+      }
+      return stat.mode;
+    },
+    realPath: function (node) {
+      var parts = [];
+      while (node.parent !== node) {
+        parts.push(node.name);
+        node = node.parent;
+      }
+      parts.push(node.mount.opts.root);
+      parts.reverse();
+      return PATH.join.apply(null, parts);
+    },
+    node_ops: {
+      getattr: function(node) {
+        var path = NODEFS.realPath(node);
+        var stat;
+        try {
+          stat = fs.lstatSync(path);
+        } catch (e) {
+          if (!e.code) throw e;
+          throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+        }
+        return {
+          dev: stat.dev,
+          ino: stat.ino,
+          mode: stat.mode,
+          nlink: stat.nlink,
+          uid: stat.uid,
+          gid: stat.gid,
+          rdev: stat.rdev,
+          size: stat.size,
+          atime: stat.atime,
+          mtime: stat.mtime,
+          ctime: stat.ctime,
+          blksize: stat.blksize,
+          blocks: stat.blocks
+        };
+      },
+      setattr: function(node, attr) {
+        var path = NODEFS.realPath(node);
+        try {
+          if (attr.mode !== undefined) {
+            fs.chmodSync(path, attr.mode);
+            // update the common node structure mode as well
+            node.mode = attr.mode;
+          }
+          if (attr.timestamp !== undefined) {
+            var date = new Date(attr.timestamp);
+            fs.utimesSync(path, date, date);
+          }
+          if (attr.size !== undefined) {
+            fs.truncateSync(path, attr.size);
+          }
+        } catch (e) {
+          if (!e.code) throw e;
+          throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+        }
+      },
+      lookup: function (parent, name) {
+        var path = PATH.join(NODEFS.realPath(parent), name);
+        var mode = NODEFS.getMode(path);
+        return NODEFS.createNode(parent, name, mode);
+      },
+      mknod: function (parent, name, mode, dev) {
+        var node = NODEFS.createNode(parent, name, mode, dev);
+        // create the backing node for this in the fs root as well
+        var path = NODEFS.realPath(node);
+        try {
+          if (FS.isDir(node.mode)) {
+            fs.mkdirSync(path, node.mode);
+          } else {
+            fs.writeFileSync(path, '', { mode: node.mode });
+          }
+        } catch (e) {
+          if (!e.code) throw e;
+          throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+        }
+        return node;
+      },
+      rename: function (oldNode, newDir, newName) {
+        var oldPath = NODEFS.realPath(oldNode);
+        var newPath = PATH.join(NODEFS.realPath(newDir), newName);
+        try {
+          fs.renameSync(oldPath, newPath);
+        } catch (e) {
+          if (!e.code) throw e;
+          throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+        }
+      },
+      unlink: function(parent, name) {
+        var path = PATH.join(NODEFS.realPath(parent), name);
+        try {
+          fs.unlinkSync(path);
+        } catch (e) {
+          if (!e.code) throw e;
+          throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+        }
+      },
+      rmdir: function(parent, name) {
+        var path = PATH.join(NODEFS.realPath(parent), name);
+        try {
+          fs.rmdirSync(path);
+        } catch (e) {
+          if (!e.code) throw e;
+          throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+        }
+      },
+      readdir: function(node) {
+        var path = NODEFS.realPath(node);
+        try {
+          return fs.readdirSync(path);
+        } catch (e) {
+          if (!e.code) throw e;
+          throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+        }
+      },
+      symlink: function(parent, newName, oldPath) {
+        var newPath = PATH.join(NODEFS.realPath(parent), newName);
+        try {
+          fs.symlinkSync(oldPath, newPath);
+        } catch (e) {
+          if (!e.code) throw e;
+          throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+        }
+      },
+      readlink: function(node) {
+        var path = NODEFS.realPath(node);
+        try {
+          return fs.readlinkSync(path);
+        } catch (e) {
+          if (!e.code) throw e;
+          throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+        }
+      },
+    },
+    stream_ops: {
+      open: function (stream) {
+        var path = NODEFS.realPath(stream.node);
+        try {
+          if (FS.isFile(stream.node.mode)) {
+            stream.nfd = fs.openSync(path, stream.flags);
+          }
+        } catch (e) {
+          if (!e.code) throw e;
+          throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+        }
+      },
+      close: function (stream) {
+        try {
+          if (FS.isFile(stream.node.mode)) {
+            fs.closeSync(stream.nfd);
+          }
+        } catch (e) {
+          if (!e.code) throw e;
+          throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+        }
+      },
+      read: function (stream, buffer, offset, length, position) {
+        // FIXME this is terrible.
+        var nbuffer = new Buffer(length);
+        var res;
+        try {
+          res = fs.readSync(stream.nfd, nbuffer, 0, length, position);
+        } catch (e) {
+          throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+        }
+        if (res > 0) {
+          for (var i = 0; i < res; i++) {
+            buffer[offset + i] = nbuffer[i];
+          }
+        }
+        return res;
+      },
+      write: function (stream, buffer, offset, length, position) {
+        // FIXME this is terrible.
+        var nbuffer = new Buffer(buffer.subarray(offset, offset + length));
+        var res;
+        try {
+          res = fs.writeSync(stream.nfd, nbuffer, 0, length, position);
+        } catch (e) {
+          throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+        }
+        return res;
+      },
+      llseek: function (stream, offset, whence) {
+        var position = offset;
+        if (whence === 1) {  // SEEK_CUR.
+          position += stream.position;
+        } else if (whence === 2) {  // SEEK_END.
+          if (FS.isFile(stream.node.mode)) {
+            try {
+              var stat = fs.fstatSync(stream.nfd);
+              position += stat.size;
+            } catch (e) {
+              throw new FS.ErrnoError(ERRNO_CODES[e.code]);
+            }
+          }
+        }
+
+        if (position < 0) {
+          throw new FS.ErrnoError(ERRNO_CODES.EINVAL);
+        }
+
+        stream.position = position;
+        return position;
+      }
+    }
+  }
+});
\ No newline at end of file
diff --git a/src/library_sockfs.js b/src/library_sockfs.js
index b11c6495..af29d11b 100644
--- a/src/library_sockfs.js
+++ b/src/library_sockfs.js
@@ -5,12 +5,6 @@ mergeInto(LibraryManager.library, {
     mount: function(mount) {
       return FS.createNode(null, '/', {{{ cDefine('S_IFDIR') }}} | 0777, 0);
     },
-    nextname: function() {
-      if (!SOCKFS.nextname.current) {
-        SOCKFS.nextname.current = 0;
-      }
-      return 'socket[' + (SOCKFS.nextname.current++) + ']';
-    },
     createSocket: function(family, type, protocol) {
       var streaming = type == {{{ cDefine('SOCK_STREAM') }}};
       if (protocol) {
@@ -95,6 +89,12 @@ mergeInto(LibraryManager.library, {
         sock.sock_ops.close(sock);
       }
     },
+    nextname: function() {
+      if (!SOCKFS.nextname.current) {
+        SOCKFS.nextname.current = 0;
+      }
+      return 'socket[' + (SOCKFS.nextname.current++) + ']';
+    },
     // backend-specific stream ops
     websocket_sock_ops: {
       //
diff --git a/src/modules.js b/src/modules.js
index 1029b233..2757c2cb 100644
--- a/src/modules.js
+++ b/src/modules.js
@@ -422,7 +422,7 @@ var LibraryManager = {
   load: function() {
     if (this.library) return;
 
-    var libraries = ['library.js', 'library_path.js', 'library_fs.js', 'library_memfs.js', 'library_sockfs.js', 'library_tty.js', 'library_browser.js', 'library_sdl.js', 'library_gl.js', 'library_glut.js', 'library_xlib.js', 'library_egl.js', 'library_gc.js', 'library_jansson.js', 'library_openal.js', 'library_glfw.js'].concat(additionalLibraries);
+    var libraries = ['library.js', 'library_path.js', 'library_fs.js', 'library_idbfs.js', 'library_memfs.js', 'library_nodefs.js', 'library_sockfs.js', 'library_tty.js', 'library_browser.js', 'library_sdl.js', 'library_gl.js', 'library_glut.js', 'library_xlib.js', 'library_egl.js', 'library_gc.js', 'library_jansson.js', 'library_openal.js', 'library_glfw.js'].concat(additionalLibraries);
     for (var i = 0; i < libraries.length; i++) {
       eval(processMacros(preprocess(read(libraries[i]))));
     }