diff options
-rw-r--r-- | src/jsifier.js | 14 | ||||
-rw-r--r-- | src/library.js | 45 | ||||
-rw-r--r-- | src/modules.js | 3 | ||||
-rw-r--r-- | src/settings.js | 7 | ||||
-rw-r--r-- | system/include/emscripten.h | 13 | ||||
-rw-r--r-- | system/include/libcxx/streambuf | 9 | ||||
-rw-r--r-- | system/include/libcxx/string | 6 | ||||
-rw-r--r-- | tests/runner.py | 101 | ||||
-rw-r--r-- | tools/eliminator/eliminator-test-output.js | 18 | ||||
-rw-r--r-- | tools/eliminator/eliminator-test.js | 20 | ||||
-rw-r--r-- | tools/eliminator/eliminator.coffee | 26 | ||||
-rw-r--r-- | tools/shared.py | 20 |
12 files changed, 248 insertions, 34 deletions
diff --git a/src/jsifier.js b/src/jsifier.js index 91cfbe6a..d8ed589e 100644 --- a/src/jsifier.js +++ b/src/jsifier.js @@ -300,6 +300,7 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) { substrate.addActor('FunctionStub', { processItem: function(item) { var ret = [item]; + if (IGNORED_FUNCTIONS.indexOf(item.ident) >= 0) return null; var shortident = item.ident.substr(1); if (BUILD_AS_SHARED_LIB) { // Shared libraries reuse the runtime of their parents. @@ -433,6 +434,8 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) { // We have this function all reconstructed, go and finalize it's JS! + if (IGNORED_FUNCTIONS.indexOf(func.ident) >= 0) return null; + func.JS = '\nfunction ' + func.ident + '(' + func.paramIdents.join(', ') + ') {\n'; if (PROFILE) { @@ -929,7 +932,14 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) { } generated = generated.concat(itemsDict.function).concat(data.unparsedFunctions); - if (!mainPass) return generated.map(function(item) { return item.JS }).join('\n'); + if (!mainPass) { + Functions.allIdents = Functions.allIdents.concat(itemsDict.function.map(function(func) { + return func.ident; + }).filter(function(func) { + return IGNORED_FUNCTIONS.indexOf(func.ident) < 0; + })); + return generated.map(function(item) { return item.JS }).join('\n'); + } // We are ready to print out the data, but must do so carefully - we are // dealing with potentially *huge* strings. Convenient replacements and @@ -965,6 +975,8 @@ function JSify(data, functionsOnly, givenFunctions, givenGlobalVariables) { print(Functions.generateIndexing()); // done last, as it may rely on aliases set in postsets print(postParts[1]); print(shellParts[1]); + // Print out some useful metadata (for additional optimizations later, like the eliminator) + print('// EMSCRIPTEN_GENERATED_FUNCTIONS: ' + JSON.stringify(Functions.allIdents) + '\n'); return null; } diff --git a/src/library.js b/src/library.js index faa86c68..75010afb 100644 --- a/src/library.js +++ b/src/library.js @@ -5372,6 +5372,21 @@ LibraryManager.library = { }, __01getrlimit64_: 'getrlimit', + // TODO: Implement for real. We just do time used, and no useful data + __rusage_struct_layout: Runtime.generateStructInfo(null, '%struct.rusage'), + getrusage__deps: ['__rusage_struct_layout'], + getrusage: function(resource, rlp) { + // %struct.timeval = type { i32, i32 } + var timeval = Runtime.calculateStructAlignment({ fields: ['i32', 'i32'] }); + + // int getrusage(int resource, struct rusage *rlp); + {{{ makeSetValue('rlp', '___rusage_struct_layout.ru_utime+timeval[0]', '1', 'i32') }}} + {{{ makeSetValue('rlp', '___rusage_struct_layout.ru_utime+timeval[1]', '2', 'i32') }}} + {{{ makeSetValue('rlp', '___rusage_struct_layout.ru_stime+timeval[0]', '3', 'i32') }}} + {{{ makeSetValue('rlp', '___rusage_struct_layout.ru_stime+timeval[1]', '4', 'i32') }}} + return 0; + }, + // ========================================================================== // pthread.h (stubs for mutexes only - no thread support yet!) // ========================================================================== @@ -5417,6 +5432,36 @@ LibraryManager.library = { EMSCRIPTEN_COMMENT__inline: function(param) { param = stripCorrections(param); return '// ' + Variables.globals[param].value.text.replace('\\00', '') + ' '; + }, + + $Profiling: { + max_: 0, + times: null, + invalid: 0, + dump: function() { + if (Profiling.invalid) { + print('Invalid # of calls to Profiling begin and end!'); + return; + } + print('Profiling data:') + for (var i = 0; i < Profiling.max_; i++) { + print('Block ' + i + ': ' + Profiling.times[i]); + } + } + }, + EMSCRIPTEN_PROFILE_INIT__deps: ['$Profiling'], + EMSCRIPTEN_PROFILE_INIT: function(max_) { + Profiling.max_ = max_; + Profiling.times = new Array(max_); + for (var i = 0; i < max_; i++) Profiling.times[i] = 0; + }, + EMSCRIPTEN_PROFILE_BEGIN__inline: function(id) { + return 'Profiling.times[' + id + '] -= Date.now();' + + 'Profiling.invalid++;' + }, + EMSCRIPTEN_PROFILE_END__inline: function(id) { + return 'Profiling.times[' + id + '] += Date.now();' + + 'Profiling.invalid--;' } }; diff --git a/src/modules.js b/src/modules.js index 3b370878..f04731f8 100644 --- a/src/modules.js +++ b/src/modules.js @@ -228,6 +228,9 @@ var Functions = { // All functions that will be implemented in this file implementedFunctions: null, + // All the function idents seen so far + allIdents: [], + indexedFunctions: [0, 0], // Start at a non-0 (even, see below) value // Mark a function as needing indexing, and returns the index diff --git a/src/settings.js b/src/settings.js index 0fdc445d..0e70316f 100644 --- a/src/settings.js +++ b/src/settings.js @@ -116,6 +116,13 @@ PROFILE = 0; // Enables runtime profiling. See test_profiling for a usage exampl EXPORTED_FUNCTIONS = ['_main']; // Functions that are explicitly exported, so they are guaranteed to // be accessible outside of the generated code. +IGNORED_FUNCTIONS = []; // Functions that we should not generate, neither a stub nor a complete function. + // This is useful if your project code includes a function, and you want to replace + // that in the compiled code with your own handwritten JS. (Of course even without + // this option, you could just override the generated function at runtime. However, + // JS engines might optimize better if the function is defined once in a single + // place in your code.) + EXPORTED_GLOBALS = []; // Global non-function variables that are explicitly // exported, so they are guaranteed to be // accessible outside of the generated code. diff --git a/system/include/emscripten.h b/system/include/emscripten.h index 4d321b2c..0ce31e9f 100644 --- a/system/include/emscripten.h +++ b/system/include/emscripten.h @@ -23,6 +23,19 @@ extern void emscripten_run_script(const char *script); */ extern void EMSCRIPTEN_COMMENT(const char *text); +/* + * Profiling tools. + * INIT must be called first, with the maximum identifier that + * will be used. BEGIN will add some code that marks + * the beginning of a section of code whose run time you + * want to measure. END will finish such a section. Note: If you + * call begin but not end, you will get invalid data! + * The profiling data will be written out if you call Profile.dump(). + */ +extern void EMSCRIPTEN_PROFILE_INIT(int max); +extern void EMSCRIPTEN_PROFILE_BEGIN(int id); +extern void EMSCRIPTEN_PROFILE_END(int id); + #ifdef __cplusplus } #endif diff --git a/system/include/libcxx/streambuf b/system/include/libcxx/streambuf index feb62c7e..20ae24f3 100644 --- a/system/include/libcxx/streambuf +++ b/system/include/libcxx/streambuf @@ -111,6 +111,7 @@ protected: #include <__config> #include <iosfwd> #include <ios> +#include <__locale> #pragma GCC system_header @@ -551,11 +552,11 @@ basic_streambuf<_CharT, _Traits>::overflow(int_type __c) return traits_type::eof(); } -extern template class basic_streambuf<char>; -extern template class basic_streambuf<wchar_t>; +//extern template class basic_streambuf<char>; /* XXX EMScripten */ +//extern template class basic_streambuf<wchar_t>; /* XXX EMScripten */ -extern template class basic_ios<char>; -extern template class basic_ios<wchar_t>; +//extern template class basic_ios<char>; /* XXX EMScripten */ +//extern template class basic_ios<wchar_t>; /* XXX EMScripten */ _LIBCPP_END_NAMESPACE_STD diff --git a/system/include/libcxx/string b/system/include/libcxx/string index 2041510f..4f3e0e76 100644 --- a/system/include/libcxx/string +++ b/system/include/libcxx/string @@ -1021,7 +1021,7 @@ __basic_string_common<__b>::__throw_out_of_range() const #endif } -extern template class __basic_string_common<true>; +//extern template class __basic_string_common<true>; /* XXX EMScripten: Comment to export the class */ template<class _CharT, class _Traits, class _Allocator> class _LIBCPP_VISIBLE basic_string @@ -3965,8 +3965,8 @@ getline(basic_istream<_CharT, _Traits>&& __is, #endif // _LIBCPP_HAS_NO_RVALUE_REFERENCES -extern template class basic_string<char>; -extern template class basic_string<wchar_t>; +//extern template class basic_string<char>; /* XXX EMScripten: Comment to export the class */ +//extern template class basic_string<wchar_t>; /* XXX EMScripten: Comment to export the class */ extern template string diff --git a/tests/runner.py b/tests/runner.py index 3fd63574..cd87d3d3 100644 --- a/tests/runner.py +++ b/tests/runner.py @@ -163,6 +163,13 @@ class RunnerCore(unittest.TestCase): def run_native(self, filename, args): Popen([filename+'.native'] + args, stdout=PIPE, stderr=STDOUT).communicate()[0] + def assertIdentical(self, x, y): + if x != y: + raise Exception("Expected to have '%s' == '%s', diff:\n\n%s" % ( + limit_size(x), limit_size(y), + limit_size(''.join([a.rstrip()+'\n' for a in difflib.unified_diff(x.split('\n'), y.split('\n'), fromfile='expected', tofile='actual')])) + )) + def assertContained(self, value, string): if type(value) is not str: value = value() # lazy loading if type(string) is not str: string = string() @@ -236,6 +243,8 @@ if 'benchmark' not in str(sys.argv): js_engines = [SPIDERMONKEY_ENGINE, V8_ENGINE] if Settings.USE_TYPED_ARRAYS == 2: js_engines = [SPIDERMONKEY_ENGINE] # when oh when will v8 support typed arrays in the console + js_engines = filter(lambda engine: os.path.exists(engine[0]), js_engines) + assert len(js_engines) > 0, 'No JS engine present to run this test with. Check ~/.emscripten and the paths therein.' for engine in js_engines: js_output = self.run_generated_code(engine, filename + '.o.js', args) if output_nicerizer is not None: @@ -2910,6 +2919,8 @@ if 'benchmark' not in str(sys.argv): self.do_run(src, expected) CORRECT_SIGNS = 0 + # libc++ tests + def test_iostream(self): src = ''' #include <iostream> @@ -2923,6 +2934,41 @@ if 'benchmark' not in str(sys.argv): self.do_run(src, 'hello world') + def test_stdvec(self): + src = ''' + #include <vector> + #include <stdio.h> + + struct S { + int a; + float b; + }; + + void foo(int a, float b) + { + printf("%d:%.2f\\n", a, b); + } + + int main ( int argc, char *argv[] ) + { + std::vector<S> ar; + S s; + + s.a = 789; + s.b = 123.456f; + ar.push_back(s); + + s.a = 0; + s.b = 100.1f; + ar.push_back(s); + + foo(ar[0].a, ar[0].b); + foo(ar[1].a, ar[1].b); + } + ''' + + self.do_run(src, '789:123.46\n0:100.1') + ### 'Big' tests def test_fannkuch(self): @@ -3194,7 +3240,7 @@ if 'benchmark' not in str(sys.argv): # Combine libraries - combined = os.path.join(self.get_building_dir(), 'combined.bc') + combined = os.path.join(self.get_build_dir(), 'combined.bc') Building.link([freetype, poppler], combined) self.do_ll_run(combined, @@ -3278,7 +3324,7 @@ if 'benchmark' not in str(sys.argv): includes=[path_from_root('tests', 'openjpeg', 'libopenjpeg'), path_from_root('tests', 'openjpeg', 'codec'), path_from_root('tests', 'openjpeg', 'common'), - os.path.join(self.get_building_dir(), 'openjpeg')], + os.path.join(self.get_build_dir(), 'openjpeg')], force_c=True, post_build=post, output_nicerizer=image_compare)# build_ll_hook=self.do_autodebug) @@ -3381,6 +3427,38 @@ if 'benchmark' not in str(sys.argv): self.do_run(src, '*hello slim world*', build_ll_hook=hook) def test_profiling(self): + src = ''' + #include <emscripten.h> + #include <unistd.h> + + int main() + { + EMSCRIPTEN_PROFILE_INIT(3); + EMSCRIPTEN_PROFILE_BEGIN(0); + usleep(10 * 1000); + EMSCRIPTEN_PROFILE_END(0); + EMSCRIPTEN_PROFILE_BEGIN(1); + usleep(50 * 1000); + EMSCRIPTEN_PROFILE_END(1); + EMSCRIPTEN_PROFILE_BEGIN(2); + usleep(250 * 1000); + EMSCRIPTEN_PROFILE_END(2); + return 0; + } + ''' + + def post1(filename): + src = open(filename, 'a') + src.write(''' + Profiling.dump(); + ''') + src.close() + + self.do_run(src, '''Profiling data: +Block 0: ''', post_build=post1) + + # Part 2: old JS version + Settings.PROFILE = 1 Settings.INVOKE_RUN = 0 @@ -3427,7 +3505,6 @@ if 'benchmark' not in str(sys.argv): ''') src.close() - # Using build_ll_hook forces a recompile, which leads to DFE being done even without opts self.do_run(src, ': __Z6inner1i (5000)\n*ok*', post_build=post) ### Integration tests @@ -4108,13 +4185,27 @@ TT = %s input = open(path_from_root('tools', 'eliminator', 'eliminator-test.js')).read() expected = open(path_from_root('tools', 'eliminator', 'eliminator-test-output.js')).read() output = Popen([COFFEESCRIPT, VARIABLE_ELIMINATOR], stdin=PIPE, stdout=PIPE, stderr=PIPE).communicate(input)[0] - self.assertEquals(output, expected) + self.assertIdentical(expected, output) else: # Benchmarks. Run them with argument |benchmark|. To run a specific test, do # |benchmark.test_X|. - print "Running Emscripten benchmarks..." + fingerprint = [time.asctime()] + try: + fingerprint.append('em: ' + Popen(['git', 'show'], stdout=PIPE).communicate()[0].split('\n')[0]) + except: + pass + try: + d = os.getcwd() + os.chdir(os.path.expanduser('~/Dev/mozilla-central')) + fingerprint.append('sm: ' + filter(lambda line: 'changeset' in line, + Popen(['hg', 'tip'], stdout=PIPE).communicate()[0].split('\n'))[0]) + except: + pass + finally: + os.chdir(d) + print 'Running Emscripten benchmarks... [ %s ]' % ' | '.join(fingerprint) sys.argv = filter(lambda x: x != 'benchmark', sys.argv) diff --git a/tools/eliminator/eliminator-test-output.js b/tools/eliminator/eliminator-test-output.js index 2324124e..b7a983cc 100644 --- a/tools/eliminator/eliminator-test-output.js +++ b/tools/eliminator/eliminator-test-output.js @@ -5,7 +5,7 @@ function f() { HEAP[123] = (GLOB[1] + 1) / 2; } -var g = (function(a1, a2) { +function g(a1, a2) { var a = 1; var c = a * 2 - 1; @@ -39,7 +39,7 @@ var g = (function(a1, a2) { unquoted: 3, 4: 5 }; -}); +} function h() { var out; bar(hello); @@ -91,3 +91,17 @@ function py() { var $8 = HEAP[__PyThreadState_Current] + 12; HEAP[$8] = $7; } +function otherPy() { + var $4 = HEAP[__PyThreadState_Current]; + var $5 = $4 + 12; + var $7 = HEAP[$5] + 1; + var $8 = $4 + 12; + HEAP[$8] = $7; +} +var anon = (function(x) { + var $4 = HEAP[__PyThreadState_Current]; + var $5 = $4 + 12; + var $7 = HEAP[$5] + 1; + var $8 = $4 + 12; + HEAP[$8] = $7; +}); diff --git a/tools/eliminator/eliminator-test.js b/tools/eliminator/eliminator-test.js index 8a364c0a..681b6cf7 100644 --- a/tools/eliminator/eliminator-test.js +++ b/tools/eliminator/eliminator-test.js @@ -5,7 +5,7 @@ function f() { var z = y / 2; HEAP[123] = z; } -var g = function (a1, a2) { +function g(a1, a2) { var a = 1; var b = a * 2; var c = b - 1; @@ -39,7 +39,7 @@ var g = function (a1, a2) { unquoted: 3, 4: 5 }; -}; +} function h() { var out; bar(hello); @@ -91,3 +91,19 @@ function py() { var $8 = $4 + 12; HEAP[$8] = $7; } +function otherPy() { + var $4 = HEAP[__PyThreadState_Current]; + var $5 = $4 + 12; + var $7 = HEAP[$5] + 1; + var $8 = $4 + 12; + HEAP[$8] = $7; +} +var anon = function(x) { + var $4 = HEAP[__PyThreadState_Current]; + var $5 = $4 + 12; + var $7 = HEAP[$5] + 1; + var $8 = $4 + 12; + HEAP[$8] = $7; +} +// EMSCRIPTEN_GENERATED_FUNCTIONS: ["f", "g", "h", "py"] + diff --git a/tools/eliminator/eliminator.coffee b/tools/eliminator/eliminator.coffee index c07e5974..c6de8690 100644 --- a/tools/eliminator/eliminator.coffee +++ b/tools/eliminator/eliminator.coffee @@ -20,6 +20,10 @@ uglify = require 'uglify-js' fs = require 'fs' +# Functions which have been generated by Emscripten. We optimize only those. +generatedFunctions = [] +GENERATED_FUNCTIONS_MARKER = '// EMSCRIPTEN_GENERATED_FUNCTIONS:' + # Maximum number of uses to consider a variable not worth eliminating. MAX_USES = 3 @@ -79,6 +83,8 @@ traverse = (node, callback) -> # function/defun node and call run() to apply the optimization (in-place). class Eliminator constructor: (func) -> + @ident = func[1] + # The statements of the function to analyze. @body = func[3] @@ -107,7 +113,7 @@ class Eliminator # @returns: The number of variables eliminated, or undefined if skipped. run: -> # Our optimization does not account for closures. - if @hasClosures @body then return undefined + if not @isGenerated() then return undefined @calculateBasicVarStats() @analyzeInitialValues() @@ -128,16 +134,8 @@ class Eliminator return eliminated # Determines if a function is Emscripten-generated. - hasClosures: -> - closureFound = false - - traverse @body, (node, type) -> - if type in ['defun', 'function', 'with'] - closureFound = true - return false - return undefined - - return closureFound + isGenerated: -> + return @ident in generatedFunctions # Runs the basic variable scan pass. Fills the following member variables: # isLocal @@ -337,6 +335,12 @@ class Eliminator main = -> # Get the parse tree. src = fs.readFileSync('/dev/stdin').toString() + + throw 'Cannot identify generated functions' if GENERATED_FUNCTIONS_MARKER in src + generatedFunctionsLine = src.split('\n').filter (line) -> + return line.indexOf(GENERATED_FUNCTIONS_MARKER) == 0 + generatedFunctions = eval(generatedFunctionsLine[0].replace(GENERATED_FUNCTIONS_MARKER, '')) + ast = uglify.parser.parse src # Run the eliminator on all functions. diff --git a/tools/shared.py b/tools/shared.py index 38089122..d5629b50 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -73,7 +73,7 @@ def timeout_run(proc, timeout, note): raise Exception("Timed out: " + note) return proc.communicate()[0] -def run_js(engine, filename, args, check_timeout=False, stdout=PIPE, stderr=STDOUT, cwd=None): +def run_js(engine, filename, args=[], check_timeout=False, stdout=PIPE, stderr=STDOUT, cwd=None): return timeout_run(Popen(engine + [filename] + (['--'] if 'd8' in engine[0] else []) + args, stdout=stdout, stderr=stderr, cwd=cwd), 15*60 if check_timeout else None, 'Execution') @@ -220,7 +220,7 @@ class Building: COMPILER_TEST_OPTS = [] @staticmethod - def build_library(name, build_dir, output_dir, generated_libs, configure=['./configure'], configure_args=[], make=['make'], make_args=['-j', '2'], cache=None, cache_name=None, copy_project=False): + def build_library(name, build_dir, output_dir, generated_libs, configure=['./configure'], configure_args=[], make=['make'], make_args=['-j', '2'], cache=None, cache_name=None, copy_project=False, env_init={}): ''' Build a library into a .bc file. We build the .bc file once and cache it for all our tests. (We cache in memory since the test directory is destroyed and recreated for each test. Note that we cache separately for different compilers) ''' @@ -238,20 +238,28 @@ class Building: except: old_dir = None os.chdir(project_dir) + generated_libs = map(lambda lib: os.path.join(project_dir, lib), generated_libs) + #for lib in generated_libs: + # try: + # os.unlink(lib) # make sure compilation completed successfully + # except: + # pass env = os.environ.copy() env['RANLIB'] = env['AR'] = env['CXX'] = env['CC'] = env['LIBTOOL'] = EMMAKEN env['EMMAKEN_COMPILER'] = Building.COMPILER env['EMSCRIPTEN_TOOLS'] = path_from_root('tools') env['CFLAGS'] = env['EMMAKEN_CFLAGS'] = ' '.join(COMPILER_OPTS + Building.COMPILER_TEST_OPTS) # Normal CFLAGS is ignored by some configure's. + for k, v in env_init.iteritems(): + env[k] = v if configure: # Useful in debugging sometimes to comment this out (and the lines below up to and including the |make| call) env['EMMAKEN_JUST_CONFIGURE'] = '1' - Popen(configure + configure_args, stdout=open(os.path.join(output_dir, 'configure'), 'w'), + Popen(configure + configure_args, stdout=open(os.path.join(output_dir, 'configure_'), 'w'), stderr=open(os.path.join(output_dir, 'configure_err'), 'w'), env=env).communicate()[0] del env['EMMAKEN_JUST_CONFIGURE'] - Popen(make + make_args, stdout=open(os.path.join(output_dir, 'make'), 'w'), + Popen(make + make_args, stdout=open(os.path.join(output_dir, 'make_'), 'w'), stderr=open(os.path.join(output_dir, 'make_err'), 'w'), env=env).communicate()[0] bc_file = os.path.join(project_dir, 'bc.bc') - Building.link(map(lambda lib: os.path.join(project_dir, lib), generated_libs), bc_file) + Building.link(generated_libs, bc_file) if cache is not None: cache[cache_name] = open(bc_file, 'rb').read() if old_dir: @@ -261,7 +269,7 @@ class Building: @staticmethod def link(files, target): output = Popen([LLVM_LINK] + files + ['-o', target], stdout=PIPE, stderr=STDOUT).communicate()[0] - assert output is None or 'Could not open input file' not in output, 'Linking error: ' + output + assert os.path.exists(target) and (output is None or 'Could not open input file' not in output), 'Linking error: ' + output # Emscripten optimizations that we run on the .ll file @staticmethod |