Merge branch 'fs_destination' of github.com:juj/emscripten into incoming

2 files changed, 126 insertions, 52 deletions

diff --git a/tests/runner.py b/tests/runner.py
index be8a49e5..720f434e 100755
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -11714,7 +11714,9 @@ elif 'browser' in str(sys.argv):
       self.run_browser('page.html', '', '/report_result?1')
 
     def test_preload_file(self):
-      open(os.path.join(self.get_dir(), 'somefile.txt'), 'w').write('''load me right before running the code please''')
+      absolute_src_path = os.path.join(self.get_dir(), 'somefile.txt').replace('\\', '/')
+      open(absolute_src_path, 'w').write('''load me right before running the code please''')
+      
       def make_main(path):
         print path
         open(os.path.join(self.get_dir(), 'main.cpp'), 'w').write(self.with_report_result(r'''
@@ -11733,32 +11735,92 @@ elif 'browser' in str(sys.argv):
             REPORT_RESULT();
             return 0;
           }
-        ''' % path))
-
-      make_main('somefile.txt')
-      Popen([PYTHON, EMCC, os.path.join(self.get_dir(), 'main.cpp'), '--preload-file', 'somefile.txt', '-o', 'page.html']).communicate()
-      self.run_browser('page.html', 'You should see |load me right before|.', '/report_result?1')
+          ''' % path))
+
+      test_cases = [
+       # (source preload-file string, file on target FS to load)
+        ("somefile.txt", "somefile.txt"),
+        ("./somefile.txt", "somefile.txt"),
+        ("somefile.txt@file.txt", "file.txt"),
+        ("./somefile.txt@file.txt", "file.txt"),
+        ("./somefile.txt@./file.txt", "file.txt"),
+        ("somefile.txt@/file.txt", "file.txt"),
+        ("somefile.txt@/", "somefile.txt"), 
+        (absolute_src_path + "@file.txt", "file.txt"),
+        (absolute_src_path + "@/file.txt", "file.txt"),
+        (absolute_src_path + "@/", "somefile.txt"),
+        ("somefile.txt@/directory/file.txt", "/directory/file.txt"),
+        ("somefile.txt@/directory/file.txt", "directory/file.txt"),
+        (absolute_src_path + "@/directory/file.txt", "directory/file.txt")]
+      
+      for test in test_cases:
+        (srcpath, dstpath) = test
+        make_main(dstpath)
+        print srcpath
+        Popen([PYTHON, EMCC, os.path.join(self.get_dir(), 'main.cpp'), '--preload-file', srcpath, '-o', 'page.html']).communicate()
+        self.run_browser('page.html', 'You should see |load me right before|.', '/report_result?1')
 
       # By absolute path
 
-      make_main(os.path.join(self.get_dir(), 'somefile.txt'))
-      Popen([PYTHON, EMCC, os.path.join(self.get_dir(), 'main.cpp'), '--preload-file', os.path.join(self.get_dir(), 'somefile.txt'), '-o', 'page.html']).communicate()
+      make_main(absolute_src_path)
+      Popen([PYTHON, EMCC, os.path.join(self.get_dir(), 'main.cpp'), '--preload-file', absolute_src_path, '-o', 'page.html']).communicate()
       self.run_browser('page.html', 'You should see |load me right before|.', '/report_result?1')
 
-      # By ./path
-
-      make_main('somefile.txt')
-      Popen([PYTHON, EMCC, os.path.join(self.get_dir(), 'main.cpp'), '--preload-file', './somefile.txt', '-o', 'page.html']).communicate()
-      self.run_browser('page.html', 'You should see |load me right before|.', '/report_result?1')
+      # Test subdirectory handling with asset packaging.
+      os.makedirs(os.path.join(self.get_dir(), 'assets/sub/asset1/').replace('\\', '/'))
+      os.makedirs(os.path.join(self.get_dir(), 'assets/sub/asset2/').replace('\\', '/'))
+      open(os.path.join(self.get_dir(), 'assets/sub/asset1/file1.txt'), 'w').write('''load me right before running the code please''')
+      open(os.path.join(self.get_dir(), 'assets/sub/asset2/file2.txt'), 'w').write('''load me right before running the code please''')
+      absolute_assets_src_path = os.path.join(self.get_dir(), 'assets').replace('\\', '/')
+      def make_main_two_files(path1, path2):
+        open(os.path.join(self.get_dir(), 'main.cpp'), 'w').write(self.with_report_result(r'''
+          #include <stdio.h>
+          #include <string.h>
+          #include <emscripten.h>
+          int main() {
+            FILE *f = fopen("%s", "r");
+            char buf[100];
+            fread(buf, 1, 20, f);
+            buf[20] = 0;
+            fclose(f);
+            printf("|%%s|\n", buf);
 
+            int result = !strcmp("load me right before", buf);
+            
+            f = fopen("%s", "r");
+            if (f == NULL)
+              result = 0;
+            fclose(f);
+            REPORT_RESULT();
+            return 0;
+          }
+        ''' % (path1, path2)))
+
+      test_cases = [
+       # (source directory to embed, file1 on target FS to load, file2 on target FS to load)
+        ("assets", "assets/sub/asset1/file1.txt", "assets/sub/asset2/file2.txt"),
+        ("assets/", "assets/sub/asset1/file1.txt", "assets/sub/asset2/file2.txt"),
+        ("assets@/", "/sub/asset1/file1.txt", "/sub/asset2/file2.txt"),
+        ("assets/@/", "/sub/asset1/file1.txt", "/sub/asset2/file2.txt"),
+        ("assets@./", "/sub/asset1/file1.txt", "/sub/asset2/file2.txt"),
+        (absolute_assets_src_path + "@/", "/sub/asset1/file1.txt", "/sub/asset2/file2.txt"),
+        (absolute_assets_src_path + "@/assets", "/assets/sub/asset1/file1.txt", "/assets/sub/asset2/file2.txt")]
+
+      for test in test_cases:
+        (srcpath, dstpath1, dstpath2) = test
+        make_main_two_files(dstpath1, dstpath2)
+        print srcpath
+        Popen([PYTHON, EMCC, os.path.join(self.get_dir(), 'main.cpp'), '--preload-file', srcpath, '-o', 'page.html']).communicate()
+        self.run_browser('page.html', 'You should see |load me right before|.', '/report_result?1')
+        
       # Should still work with -o subdir/..
 
-      make_main(os.path.join(self.get_dir(), 'somefile.txt'))
+      make_main(absolute_src_path)
       try:
         os.mkdir(os.path.join(self.get_dir(), 'dirrey'))
       except:
         pass
-      Popen([PYTHON, EMCC, os.path.join(self.get_dir(), 'main.cpp'), '--preload-file', os.path.join(self.get_dir(), 'somefile.txt'), '-o', 'dirrey/page.html']).communicate()
+      Popen([PYTHON, EMCC, os.path.join(self.get_dir(), 'main.cpp'), '--preload-file', absolute_src_path, '-o', 'dirrey/page.html']).communicate()
       self.run_browser('dirrey/page.html', 'You should see |load me right before|.', '/report_result?1')
 
       # With FS.preloadFile
diff --git a/tools/file_packager.py b/tools/file_packager.py
index f4cdafc8..6a8390ad 100644
--- a/tools/file_packager.py
+++ b/tools/file_packager.py
@@ -112,16 +112,20 @@ for arg in sys.argv[1:]:
     in_preload = False
     in_embed = False
     in_compress = 0
-  elif in_preload:
-    if os.path.isfile(arg) or os.path.isdir(arg):
-      data_files.append({ 'name': arg, 'mode': 'preload' })
+  elif in_preload or in_embed:
+    mode = 'preload'
+    if in_embed:
+      mode = 'embed'
+    if '@' in arg:
+      srcpath, dstpath = arg.split('@') # User is specifying destination filename explicitly.
     else:
-      print >> sys.stderr, 'Warning: ' + arg + ' does not exist, ignoring.'
-  elif in_embed:
-    if os.path.isfile(arg) or os.path.isdir(arg):
-      data_files.append({ 'name': arg, 'mode': 'embed' })
+      srcpath = dstpath = arg # Use source path as destination path.
+      if os.path.isabs(dstpath):
+        print >> sys.stderr, 'Warning: Embedding an absolute file/directory name "' + dstpath + '" to the virtual filesystem. The file will be made available in the path "' + dstpath + '", and not in the root of the generated file system. Use the explicit syntax --preload-file srcpath@dstpath to specify the target location the absolute source path should be directed to.'
+    if os.path.isfile(srcpath) or os.path.isdir(srcpath):
+      data_files.append({ 'srcpath': srcpath, 'dstpath': dstpath, 'mode': mode })
     else:
-      print >> sys.stderr, 'Warning:' + arg + ' does not exist, ignoring.'
+      print >> sys.stderr, 'Warning: ' + arg + ' does not exist, ignoring.'
   elif in_compress:
     if in_compress == 1:
       Compression.encoder = arg
@@ -147,21 +151,26 @@ function assert(check, msg) {
 '''
 
 # Expand directories into individual files
-def add(mode, dirname, names):
+def add(arg, dirname, names):
+  # rootpathsrc: The path name of the root directory on the local FS we are adding to emscripten virtual FS.
+  # rootpathdst: The name we want to make the source path available on the emscripten virtual FS.
+  mode, rootpathsrc, rootpathdst = arg
   for name in names:
     fullname = os.path.join(dirname, name)
     if not os.path.isdir(fullname):
-      data_files.append({ 'name': fullname, 'mode': mode })
+      dstpath = os.path.join(rootpathdst, os.path.relpath(fullname, rootpathsrc)) # Convert source filename relative to root directory of target FS.
+      data_files.append({ 'srcpath': fullname, 'dstpath': dstpath, 'mode': mode })
 
 for file_ in data_files:
-  if os.path.isdir(file_['name']):
-    os.path.walk(file_['name'], add, file_['mode'])
-data_files = filter(lambda file_: not os.path.isdir(file_['name']), data_files)
+  if os.path.isdir(file_['srcpath']):
+    os.path.walk(file_['srcpath'], add, [file_['mode'], file_['srcpath'], file_['dstpath']])
+data_files = filter(lambda file_: not os.path.isdir(file_['srcpath']), data_files)
 
 for file_ in data_files:
-  if file_['name'].startswith('./'): file_['name'] = file_['name'][2:] # remove redundant ./ prefix
-  file_['name'] = file_['name'].replace(os.path.sep, '/') # name in the filesystem, native and emulated
-  file_['localname'] = file_['name'] # name to actually load from local filesystem, after transformations
+  file_['dstpath'] = file_['dstpath'].replace(os.path.sep, '/') # name in the filesystem, native and emulated
+  if file_['dstpath'].endswith('/'): # If user has submitted a directory name as the destination but omitted the destination filename, use the filename from source file
+    file_['dstpath'] = file_['dstpath'] + os.path.basename(file_['srcpath'])
+  if file_['dstpath'].startswith('./'): file_['dstpath'] = file_['dstpath'][2:] # remove redundant ./ prefix
 
 # Remove duplicates (can occur naively, for example preload dir/, preload dir/subdir/)
 seen = {}
@@ -169,7 +178,7 @@ def was_seen(name):
   if seen.get(name): return True
   seen[name] = 1
   return False
-data_files = filter(lambda file_: not was_seen(file_['name']), data_files)
+data_files = filter(lambda file_: not was_seen(file_['dstpath']), data_files)
 
 if AV_WORKAROUND:
   random.shuffle(data_files)
@@ -201,20 +210,24 @@ if crunch:
 '''
 
   for file_ in data_files:
-    if file_['name'].endswith(CRUNCH_INPUT_SUFFIX):
-      # Do not crunch if crunched version exists and is more recent than dds source
-      crunch_name = unsuffixed(file_['name']) + CRUNCH_OUTPUT_SUFFIX
-      file_['localname'] = crunch_name
+    if file_['dstpath'].endswith(CRUNCH_INPUT_SUFFIX):
+      src_dds_name = file_['srcpath']
+      src_crunch_name = unsuffixed(src_dds_name) + CRUNCH_OUTPUT_SUFFIX
+
+      # Preload/embed the .crn version instead of the .dds version, but use the .dds suffix for the target file in the virtual FS.
+      file_['srcpath'] = src_crunch_name
+
       try:
-        crunch_time = os.stat(crunch_name).st_mtime
-        dds_time = os.stat(file_['name']).st_mtime
+        # Do not crunch if crunched version exists and is more recent than dds source
+        crunch_time = os.stat(src_crunch_name).st_mtime
+        dds_time = os.stat(src_dds_name).st_mtime
         if dds_time < crunch_time: continue
       except:
         pass # if one of them does not exist, continue on
 
       # guess at format. this lets us tell crunch to not try to be clever and use odd formats like DXT5_AGBR
       try:
-        format = Popen(['file', file_['name']], stdout=PIPE).communicate()[0]
+        format = Popen(['file', file_['srcpath']], stdout=PIPE).communicate()[0]
         if 'DXT5' in format:
           format = ['-dxt5']
         elif 'DXT1' in format:
@@ -223,23 +236,22 @@ if crunch:
           raise Exception('unknown format')
       except:
         format = []
-      Popen([CRUNCH, '-file', file_['name'], '-quality', crunch] + format, stdout=sys.stderr).communicate()
+      Popen([CRUNCH, '-outsamedir', '-file', src_dds_name, '-quality', crunch] + format, stdout=sys.stderr).communicate()
       #if not os.path.exists(os.path.basename(crunch_name)):
       #  print >> sys.stderr, 'Failed to crunch, perhaps a weird dxt format? Looking for a source PNG for the DDS'
-      #  Popen([CRUNCH, '-file', unsuffixed(file_['name']) + '.png', '-quality', crunch] + format, stdout=sys.stderr).communicate()
-      assert os.path.exists(os.path.basename(crunch_name)), 'crunch failed to generate output'
-      shutil.move(os.path.basename(crunch_name), crunch_name) # crunch places files in the current dir
+      #  Popen([CRUNCH, '-file', unsuffixed(file_['srcpath']) + '.png', '-quality', crunch] + format, stdout=sys.stderr).communicate()
+      assert os.path.exists(os.path.basename(src_crunch_name)), 'crunch failed to generate output'
       # prepend the dds header
-      crunched = open(crunch_name, 'rb').read()
-      c = open(crunch_name, 'wb')
-      c.write(open(file_['name'], 'rb').read()[:DDS_HEADER_SIZE])
+      crunched = open(src_crunch_name, 'rb').read()
+      c = open(src_crunch_name, 'wb')
+      c.write(open(src_dds_name, 'rb').read()[:DDS_HEADER_SIZE])
       c.write(crunched)
       c.close()
 
 # Set up folders
 partial_dirs = []
 for file_ in data_files:
-  dirname = os.path.dirname(file_['name'])
+  dirname = os.path.dirname(file_['dstpath'])
   dirname = dirname.lstrip('/') # absolute paths start with '/', remove that
   if dirname != '':
     parts = dirname.split('/')
@@ -255,10 +267,10 @@ if has_preloaded:
   start = 0
   for file_ in data_files:
     file_['data_start'] = start
-    curr = open(file_['localname'], 'rb').read()
+    curr = open(file_['srcpath'], 'rb').read()
     file_['data_end'] = start + len(curr)
     if AV_WORKAROUND: curr += '\x00'
-    #print >> sys.stderr, 'bundling', file_['name'], file_['localname'], file_['data_start'], file_['data_end']
+    #print >> sys.stderr, 'bundling', file_['srcpath'], file_['dstpath'], file_['data_start'], file_['data_end']
     start += len(curr)
     data.write(curr)
   data.close()
@@ -280,10 +292,10 @@ if has_preloaded:
 
 counter = 0
 for file_ in data_files:
-  filename = file_['name']
+  filename = file_['dstpath']
   if file_['mode'] == 'embed':
     # Embed
-    data = map(ord, open(file_['localname'], 'rb').read())
+    data = map(ord, open(file_['srcpath'], 'rb').read())
     str_data = ''
     chunk_size = 10240
     while len(data) > 0:
@@ -357,7 +369,7 @@ if has_preloaded:
         Module['HEAPU8'].set(data, ptr);
         curr.response = Module['HEAPU8'].subarray(ptr, ptr + %d);
         curr.onload();
-      ''' % (file_['name'], file_['data_start'], file_['data_end'], file_['data_end'] - file_['data_start'], file_['data_end'] - file_['data_start'])
+      ''' % (file_['dstpath'], file_['data_start'], file_['data_end'], file_['data_end'] - file_['data_start'], file_['data_end'] - file_['data_start'])
   use_data += "          Module['removeRunDependency']('datafile_%s');\n" % data_target
 
   if Compression.on: