Merge pull request #1187 from imvu/scons-embind-build-system

SCons/embind build system

8 files changed, 462 insertions, 1 deletions

diff --git a/demos/scons-embind/SConstruct b/demos/scons-embind/SConstruct
new file mode 100644
index 00000000..8afc3e27
--- /dev/null
+++ b/demos/scons-embind/SConstruct
@@ -0,0 +1,23 @@
+env = Environment(
+    toolpath=['../../scons-tools'],
+    tools=['cc', 'c++', 'ar', 'emscripten', 'llvm', 'closure'],
+    LLVM_ROOT='/opt/local/bin',
+    CLANG='clang-mp-3.2',
+    CLANGXX='clang++-mp-3.2',
+    LLVM_LINK='llvm-link-mp-3.2',
+    LLVM_OPT='opt-mp-3.2',
+    LLVM_DIS='llvm-dis-mp-3.2',
+    EMSCRIPTEN_VERSION_FILE=File('build/version_file'),
+    EMSCRIPTEN_SETTINGS={
+        'ASM_JS': 0,
+    })
+env['BUILDERS']['WrapInModule'] = Builder(
+    action='cp $SOURCE $TARGET',
+)
+
+env.Append()
+a1 = env.Object('build/foo.bc', 'foo.cpp')
+a2 = env.Object('build/bar.bc', 'bar.cpp')
+total = env.LLVMLink('build/thelibrary.bc', [a1, a2])
+
+env.emscripten('build/thelibrary.js', total)
diff --git a/demos/scons-embind/bar.cpp b/demos/scons-embind/bar.cpp
new file mode 100644
index 00000000..a3908014
--- /dev/null
+++ b/demos/scons-embind/bar.cpp
@@ -0,0 +1,2 @@
+void foo() {
+}
diff --git a/demos/scons-embind/foo.cpp b/demos/scons-embind/foo.cpp
new file mode 100644
index 00000000..61be501a
--- /dev/null
+++ b/demos/scons-embind/foo.cpp
@@ -0,0 +1,11 @@
+#include <stdio.h>
+#include <emscripten/bind.h>
+
+void print_some_stuff(int a, float b, const std::string& s) {
+    printf("print_some_stuff: %d, %f, %s\n", a, b, s.c_str());
+}
+
+EMSCRIPTEN_BINDINGS(foo) {
+    emscripten::function("print_some_stuff", &print_some_stuff);
+}
+
diff --git a/demos/scons-embind/test.js b/demos/scons-embind/test.js
new file mode 100644
index 00000000..a6252fd8
--- /dev/null
+++ b/demos/scons-embind/test.js
@@ -0,0 +1,2 @@
+var thelibrary = require('./build/thelibrary.js');
+thelibrary.Module.print_some_stuff(1, 2, 'hello world');
diff --git a/scons-tools/closure.py b/scons-tools/closure.py
new file mode 100644
index 00000000..8f53e507
--- /dev/null
+++ b/scons-tools/closure.py
@@ -0,0 +1,28 @@
+import os.path
+from SCons.Builder import Builder
+
+def generate(env):
+    def depend_on_closure_compiler(target, source, env):
+        env.Depends(target, env['CLOSURE_COMPILER'])
+        return target, source
+
+    ClosureCompiler = Builder(
+        action='$JAVA $JAVAFLAGS -jar $CLOSURE_COMPILER $CLOSURE_FLAGS --js_output_file $TARGET $SOURCES',
+        emitter=depend_on_closure_compiler
+    )
+
+    closure = os.path.join(
+        os.path.dirname(__file__),
+        '..',
+        'third_party',
+        'closure-compiler',
+        'compiler.jar')
+    closure = env.File(closure)
+
+    env['JAVA'] = 'java'
+    env['CLOSURE_COMPILER'] = closure
+    env.Append(
+        BUILDERS={'ClosureCompiler':ClosureCompiler})
+
+def exists(_env):
+    return True
diff --git a/scons-tools/emscripten.py b/scons-tools/emscripten.py
new file mode 100755
index 00000000..473c51ad
--- /dev/null
+++ b/scons-tools/emscripten.py
@@ -0,0 +1,359 @@
+import hashlib

+import json

+import sys

+import os

+from SCons.Defaults import Delete

+from SCons.Builder import Builder

+from SCons.Scanner import Scanner

+

+def exists(env):

+    return True

+

+def _expand_settings_flags(settings, env):

+    return [

+        ('-s%s=%s' % (KEY, json.dumps(VALUE).replace('"', '\\"')))

+        for KEY, VALUE in settings.items() ]

+

+emscripten_version_files = {}

+

+def build_version_file(env):

+    if not env.subst('$EMSCRIPTEN_VERSION_FILE'):

+        raise AssertionError('Must set EMSCRIPTEN_VERSION_FILE in environment')

+    if not env.subst('$EMSCRIPTEN_TEMP_DIR'):

+        raise AssertionError('Must set EMSCRIPTEN_TEMP_DIR in environment')

+    

+    EMSCRIPTEN_DEPENDENCIES = [

+        env.Glob('${EMSCRIPTEN_HOME}/src/*.js'),

+        env.Glob('${EMSCRIPTEN_HOME}/tools/*.py'),

+        '${EMSCRIPTEN_HOME}/emscripten.py',

+    ]

+    if env.subst('$EMSCRIPTEN_SHELL'):

+        EMSCRIPTEN_DEPENDENCIES.append('$EMSCRIPTEN_SHELL')

+

+    def touch_file(target, source, env):

+        m = hashlib.md5()

+        for s in source:

+            m.update(file(s.abspath, 'rb').read())

+        for t in target:

+            file(t.abspath, 'wb').write(m.hexdigest())

+

+    [emscripten_version_file] = env.Command(

+        '$EMSCRIPTEN_VERSION_FILE',

+        EMSCRIPTEN_DEPENDENCIES,

+        touch_file)

+

+    env.AddPostAction(

+        emscripten_version_file,

+        Delete(env.Dir('$EMSCRIPTEN_TEMP_DIR').abspath))

+

+    return emscripten_version_file

+

+def get_emscripten_version_file(env):

+    EMSCRIPTEN_HOME = env.Dir('$EMSCRIPTEN_HOME').abspath

+    try:

+        version_file = emscripten_version_files[EMSCRIPTEN_HOME]

+    except KeyError:

+        version_file = build_version_file(env)

+        emscripten_version_files[EMSCRIPTEN_HOME] = version_file

+    return version_file

+

+def depend_on_emscripten(node, env, path):

+    return [get_emscripten_version_file(env)]

+

+EmscriptenScanner = Scanner(

+    name='emscripten',

+    function=depend_on_emscripten)

+

+def setExtension(filename, extension):

+    return os.path.splitext(filename)[0] + '.' + extension

+

+def emscripten(env, target_js, source_bc):

+    env = env.Clone()

+    def buildName(extension):

+        return setExtension(target_js, extension)

+

+    # for debugging and reading generated code.

+    # not in critical path, uses spare cores.

+    env.LLVMDis(buildName('ll'), source_bc)

+

+    [opt_ll] = env.LLVMOpt(

+        buildName('opt.ll'),

+        source_bc,

+        LLVM_OPT_FLAGS=['-S'])

+

+    [raw_emscripten_js] = env.Emscripten(

+        buildName('raw.js'),

+        [opt_ll])

+

+    [optimized_js] = env.JSOptimizer(

+        buildName('opt.js'),

+        raw_emscripten_js)

+

+    prejs = [

+        env['EMSCRIPTEN_PREJS'],

+        '${EMSCRIPTEN_HOME}/src/embind/emval.js',

+        '${EMSCRIPTEN_HOME}/src/embind/embind.js' ]

+

+    [concatenated_js] = env.Concatenate(

+        buildName('concat.js'),

+        [ prejs,

+          optimized_js,

+          env['EMSCRIPTEN_POSTJS'] ])

+

+    DISABLE_EMSCRIPTEN_WARNINGS = [

+        '--jscomp_error', 'ambiguousFunctionDecl',

+        '--jscomp_error', 'checkDebuggerStatement',

+        '--jscomp_off', 'checkTypes',

+        '--jscomp_off', 'checkVars',

+        '--jscomp_error', 'deprecated',

+        '--jscomp_off', 'duplicate',

+        #'--jscomp_error', 'es5strict',

+        '--jscomp_off', 'missingProperties', # TODO: fix emscripten and turn this one on

+        '--jscomp_error', 'undefinedNames',

+        '--jscomp_off', 'undefinedVars', # TODO: fix emscripten and turn this one on

+        '--jscomp_off', 'uselessCode',

+        '--jscomp_off', 'globalThis',

+    ]

+

+    [iter_global_emscripten_js] = env.Concatenate(

+        buildName('iter.js'),

+        [ prejs,

+          raw_emscripten_js,

+          env['EMSCRIPTEN_POSTJS'] ])

+

+    [global_cc_emscripten_js] = env.ClosureCompiler(

+        buildName('global.closure.js'),

+        concatenated_js,

+        CLOSURE_FLAGS=['--language_in', 'ECMASCRIPT5']+DISABLE_EMSCRIPTEN_WARNINGS+['--formatting', 'PRETTY_PRINT', '--compilation_level', 'SIMPLE_OPTIMIZATIONS'])

+

+    #env.Append(

+    #    NODEJSFLAGS=['--max-stack-size=1000000000'],

+    #    UGLIFYJSFLAGS=['--stats', '-c', 'warnings=false', '-b'])

+    #env.UglifyJS(

+    #    buildName('global.uglify.js'),

+    #    concatenated_js)

+

+    [closure_js] = env.ClosureCompiler(

+        buildName('closure.js'),

+        concatenated_js,

+        CLOSURE_FLAGS=['--language_in', 'ECMASCRIPT5']+DISABLE_EMSCRIPTEN_WARNINGS+['--formatting', 'PRETTY_PRINT', '--compilation_level', 'ADVANCED_OPTIMIZATIONS'])

+

+    [global_emscripten_min_js] = env.JSOptimizer(

+        buildName('global.min.js'),

+        closure_js,

+        JS_OPTIMIZER_PASSES=['simplifyExpressionsPost', 'compress', 'last'])

+

+    [emscripten_iteration_js] = env.WrapInModule(

+        buildName('iteration.js'),

+        iter_global_emscripten_js)

+

+    [emscripten_js] = env.WrapInModule(

+        buildName('debug.js'),

+        global_cc_emscripten_js)

+

+    [emscripten_min_js] = env.WrapInModule(

+        buildName('min.js'),

+        global_emscripten_min_js)

+

+    env.InstallAs(buildName('js'), emscripten_js)

+

+    return [emscripten_iteration_js, emscripten_js, emscripten_min_js]

+

+LIBC_SOURCES = [

+    'system/lib/dlmalloc.c',

+    'system/lib/libc/musl/src/string/wmemset.c',

+    'system/lib/libc/musl/src/string/wmemcpy.c',

+]

+

+LIBCXX_SOURCES = [os.path.join('system/lib/libcxx', x) for x in [

+    'algorithm.cpp',

+    'bind.cpp',

+    #'chrono.cpp',

+    #'condition_variable.cpp',

+    #'debug.cpp',

+    #'exception.cpp',

+    'future.cpp',

+    'hash.cpp',

+    #'ios.cpp',

+    #'iostream.cpp',

+    'memory.cpp',

+    'mutex.cpp',

+    'new.cpp',

+    'random.cpp',

+    'regex.cpp',

+    'stdexcept.cpp',

+    'string.cpp',

+    'strstream.cpp',

+    'system_error.cpp',

+    #'thread.cpp',

+    'typeinfo.cpp',

+    'utility.cpp',

+    'valarray.cpp',

+]]

+

+LIBCXXABI_SOURCES = [os.path.join('system/lib/libcxxabi/src', x) for x in [

+    'private_typeinfo.cpp'

+]]

+

+# MAJOR HACK ALERT

+# ugh, SCons imports tool .py files multiple times, meaning that global variables aren't really global

+# store our "globals" "privately" on the SCons object :(

+import SCons

+

+def build_libembind(env):

+    emscripten_temp_dir = env.Dir('$EMSCRIPTEN_TEMP_DIR').abspath

+    try:

+        libembind_cache = SCons.__emscripten_libembind_cache

+    except AttributeError:

+        libembind_cache = {}

+        SCons.__emscripten_libembind_cache = libembind_cache

+    try:

+        return libembind_cache[emscripten_temp_dir]

+    except KeyError:

+        pass

+

+    libembind = env.Object(

+        '$EMSCRIPTEN_TEMP_DIR/internal_libs/bind',

+        '$EMSCRIPTEN_HOME/system/lib/embind/bind.cpp')

+    env.Depends(libembind, get_emscripten_version_file(env))

+    libembind_cache[emscripten_temp_dir] = libembind

+    return libembind

+

+def build_libcxx(env):

+    emscripten_temp_dir = env.Dir('$EMSCRIPTEN_TEMP_DIR').abspath

+    try:

+        libcxx_cache = SCons.__emscripten_libcxx_cache

+    except AttributeError:

+        libcxx_cache = {}

+        SCons.__emscripten_libcxx_cache = libcxx_cache

+    try:

+        return libcxx_cache[emscripten_temp_dir]

+    except KeyError:

+        pass

+

+    env = env.Clone()

+    env['CXXFLAGS'] = filter(lambda e: e not in ('-Werror', '-Wall'), env['CXXFLAGS'])

+    env['CCFLAGS'] = filter(lambda e: e not in ('-Werror', '-Wall'), env['CCFLAGS'])

+

+    objs = [

+        env.Object(

+            '${EMSCRIPTEN_TEMP_DIR}/libcxx_objects/' + os.path.splitext(o)[0] + '.bc',

+            '${EMSCRIPTEN_HOME}/' + o)

+        for o in LIBC_SOURCES + LIBCXXABI_SOURCES + LIBCXX_SOURCES]

+    env.Depends(objs, get_emscripten_version_file(env))

+

+    libcxx = env.Library('${EMSCRIPTEN_TEMP_DIR}/internal_libs/libcxx', objs)

+    libcxx_cache[emscripten_temp_dir] = libcxx

+    return libcxx

+

+def generate(env):

+    env.SetDefault(

+        PYTHON=sys.executable,

+        NODEJS='node',

+        JS_ENGINE='$NODEJS',

+        EMSCRIPTEN_FLAGS=['-v', '-j', '--suppressUsageWarning'],

+        EMSCRIPTEN_TEMP_DIR=env.Dir('#/emscripten.tmp'),

+        _expand_settings_flags=_expand_settings_flags,

+        EMSCRIPTEN_PREJS=[],

+        EMSCRIPTEN_POSTJS=[],

+        EMSCRIPTEN_SETTINGS={},

+        _EMSCRIPTEN_SETTINGS_FLAGS='${_expand_settings_flags(EMSCRIPTEN_SETTINGS, __env__)}',

+        JS_OPTIMIZER_PASSES=[],

+        LLVM_OPT_PASSES=['-std-compile-opts', '-std-link-opts'],

+

+        EMSCRIPTEN_HOME=env.Dir(os.path.join(os.path.dirname(__file__), '..')),

+    )

+

+    env.Replace(

+        CC='${LLVM_ROOT}/${CLANG}',

+        CXX='${LLVM_ROOT}/${CLANGXX}',

+        AR='${LLVM_ROOT}/${LLVM_LINK}',

+        ARCOM='$AR -o $TARGET $SOURCES',

+        OBJSUFFIX='.bc',

+        LIBPREFIX='',

+        LIBSUFFIX='.bc',

+        RANLIBCOM='',

+        CCFLAGS=[

+            '-U__STRICT_ANSI__',

+            '-target', 'le32-unknown-nacl',

+            '-nostdinc',

+            '-Wno-#warnings',

+            '-Wno-error=unused-variable',

+            '-Werror',

+            '-Os',

+            '-fno-threadsafe-statics',

+            '-fvisibility=hidden',

+            '-fvisibility-inlines-hidden',

+            '-Xclang', '-nostdinc++',

+            '-Xclang', '-nobuiltininc',

+            '-Xclang', '-nostdsysteminc',

+            '-Xclang', '-isystem$EMSCRIPTEN_HOME/system/include',

+            '-Xclang', '-isystem$EMSCRIPTEN_HOME/system/include/libc',

+            '-Xclang', '-isystem$EMSCRIPTEN_HOME/system/include/libcxx',

+            '-Xclang', '-isystem$EMSCRIPTEN_HOME/system/include/bsd',

+            '-emit-llvm'],

+        CXXFLAGS=['-std=c++11', '-fno-exceptions'],

+    )

+    env.Append(CPPDEFINES=[

+        'EMSCRIPTEN',

+        '__EMSCRIPTEN__',

+        '__STDC__',

+        '__IEEE_LITTLE_ENDIAN',

+    ])

+    

+    env['BUILDERS']['Emscripten'] = Builder(

+        action='$PYTHON ${EMSCRIPTEN_HOME}/emscripten.py $EMSCRIPTEN_FLAGS $_EMSCRIPTEN_SETTINGS_FLAGS --temp-dir=$EMSCRIPTEN_TEMP_DIR --compiler $JS_ENGINE --relooper=third-party/relooper.js $SOURCE > $TARGET',

+        target_scanner=EmscriptenScanner)

+

+    env['BUILDERS']['JSOptimizer'] = Builder(

+        action='$JS_ENGINE ${EMSCRIPTEN_HOME}/tools/js-optimizer.js $SOURCE $JS_OPTIMIZER_PASSES > $TARGET',

+        target_scanner=EmscriptenScanner)

+

+    def depend_on_embedder(target, source, env):

+        env.Depends(target, env['JS_EMBEDDER'])

+        return target, source

+

+    def embed_files_in_js(target, source, env, for_signature):

+        return '$PYTHON $JS_EMBEDDER $SOURCE.srcpath > $TARGET'

+    

+    def get_files_in_tree(node, env, path):

+        tree_paths = []

+        for root, dirs, files in os.walk(str(node)):

+            tree_paths += [os.path.join(root, f) for f in files]

+        return [env.File(p) for p in tree_paths]

+

+    env.SetDefault(

+        JS_EMBEDDER=env.File('#/bin/embed_files_in_js.py'))

+    

+    FileTreeScanner = Scanner(

+        function=get_files_in_tree,

+        name='FileTreeScanner',

+        recursive=False)

+

+    env['BUILDERS']['EmbedFilesInJS'] = Builder(

+        generator=embed_files_in_js,

+        emitter=depend_on_embedder,

+        source_scanner=FileTreeScanner)

+

+    env.AddMethod(emscripten)

+    

+    def ConcatenateAction(target, source, env):

+        [target] = target

+        total = ''.join(file(str(s), 'rb').read() for s in source)

+        file(str(target), 'wb').write(total)

+    env['BUILDERS']['Concatenate'] = Builder(action=ConcatenateAction)

+

+    libembind = build_libembind(env)

+    libcxx = build_libcxx(env)

+

+    # should embind be its own tool?

+    env.Append(

+        CPPPATH=[

+            '${EMSCRIPTEN_HOME}/system/include' ],

+        LIBPATH=['$EMSCRIPTEN_TEMP_DIR/internal_libs'],

+        LIBS=[

+            libembind,

+            libcxx,

+        ],

+    )

+

diff --git a/scons-tools/llvm.py b/scons-tools/llvm.py
new file mode 100755
index 00000000..f272bd16
--- /dev/null
+++ b/scons-tools/llvm.py
@@ -0,0 +1,33 @@
+from SCons.Scanner.Prog import scan
+from SCons.Builder import Builder
+
+def exists(env):
+    return True
+
+def add_libraries(target, source, env):
+    unique = []
+    lib_nodes = set()
+    for x in scan(None, env, tuple(map(env.Dir, env['LIBPATH']))):
+        if x in lib_nodes:
+            continue
+        lib_nodes.add(x)
+        unique.append(x)
+    return (target, source + unique)
+
+def generate(env):
+    env.SetDefault(
+        CLANG='clang',
+        CLANGXX='clang++',
+        LLVM_DIS='llvm-dis',
+        LLVM_OPT='opt',
+        LLVM_LINK='llvm-link')
+    
+    env['BUILDERS']['LLVMDis'] = Builder(
+        action='${LLVM_ROOT}/$LLVM_DIS -o=$TARGET $SOURCE')
+
+    env['BUILDERS']['LLVMOpt'] = Builder(
+        action='${LLVM_ROOT}/$LLVM_OPT $LLVM_OPT_FLAGS $LLVM_OPT_PASSES -o=$TARGET $SOURCE')
+
+    env['BUILDERS']['LLVMLink'] = Builder(
+        action='${LLVM_ROOT}/$LLVM_LINK -o=$TARGET $SOURCES',
+        emitter=add_libraries)
diff --git a/tools/cache.py b/tools/cache.py
index 13a53fe9..c316a1fd 100644
--- a/tools/cache.py
+++ b/tools/cache.py
@@ -49,7 +49,10 @@ class JCache:
   def ensure(self):
     self.cache.ensure()
     if not os.path.exists(self.dirname):
-      os.makedirs(self.dirname)
+      try:
+        os.makedirs(self.dirname)
+      except (IOError, OSError):
+        pass
 
   def get_shortkey(self, keys):
     if type(keys) not in [list, tuple]: