aboutsummaryrefslogtreecommitdiff
path: root/emcc
diff options
context:
space:
mode:
Diffstat (limited to 'emcc')
-rwxr-xr-xemcc331
1 files changed, 55 insertions, 276 deletions
diff --git a/emcc b/emcc
index 6471f8f3..6f68a355 100755
--- a/emcc
+++ b/emcc
@@ -77,15 +77,7 @@ emcc can be influenced by a few environment variables:
import os, sys, shutil, tempfile, subprocess, shlex
from subprocess import PIPE, STDOUT
from tools import shared
-
-def execute(cmd, *args, **kw):
- try:
- return subprocess.Popen(cmd, *args, **kw).communicate() # let compiler frontend print directly, so colors are saved (PIPE kills that)
- except:
- if not isinstance(cmd, str):
- cmd = ' '.join(cmd)
- print >> sys.stderr, 'Invoking Process failed: <<< ' + cmd + ' >>>'
- raise
+from tools.shared import Compression, execute, suffix, unsuffixed, unsuffixed_basename
# Mapping of emcc opt levels to llvm opt levels. We use llvm opt level 3 in emcc opt
# levels 2 and 3 (emcc 3 is unsafe opts, so unsuitable for the only level to get
@@ -220,9 +212,14 @@ Options that are modified or new in %s include:
compiled code asynchronously. Otherwise
similar to --embed-file, except that this
option is only relevant when generating
- HTML (it uses asynchronous binary XHRs).
+ HTML (it uses asynchronous binary XHRs),
+ or JS that will be used in a web page.
If a directory is passed here, its entire
contents will be preloaded.
+ Preloaded files are stored in filename.data,
+ where filename.html is the main file you
+ are compiling to. To run your code, you
+ will need both the .html and the .data.
--compression <codec> Compress both the compiled code and embedded/
preloaded files. <codec> should be a triple,
@@ -276,6 +273,11 @@ Options that are modified or new in %s include:
-v to Clang, and also enable EMCC_DEBUG
to details emcc's operations
+ --remove-duplicates If set, will remove duplicate symbols when
+ linking. This can be useful because
+ llvm-link's behavior is not as permissive
+ as ld is.
+
The target file, if specified (-o <target>), defines what will
be generated:
@@ -332,31 +334,17 @@ if EMMAKEN_CFLAGS: CC_ADDITIONAL_ARGS += shlex.split(EMMAKEN_CFLAGS)
# ---------------- Utilities ---------------
SOURCE_SUFFIXES = ('.c', '.cpp', '.cxx', '.cc')
-BITCODE_SUFFIXES = ('.bc', '.o')
+BITCODE_SUFFIXES = ('.bc', '.o', '.obj')
DYNAMICLIB_SUFFIXES = ('.dylib', '.so', '.dll')
STATICLIB_SUFFIXES = ('.a',)
ASSEMBLY_SUFFIXES = ('.ll',)
LIB_PREFIXES = ('', 'lib')
-IMAGE_SUFFIXES = ('.jpg', '.png', '.bmp')
-AUDIO_SUFFIXES = ('.ogg', '.wav', '.mp3')
-AUDIO_MIMETYPES = { 'ogg': 'audio/ogg', 'wav': 'audio/wav', 'mp3': 'audio/mpeg' }
-
-def suffix(name):
- return name.split('.')[-1]
-
-def unsuffixed(name):
- return '.'.join(name.split('.')[:-1])
-
-def unsuffixed_basename(name):
- return os.path.basename(unsuffixed(name))
-
seen_names = {}
-def unsuffixed_uniquename(name):
- ret = unsuffixed_basename(name)
+def uniquename(name):
if name not in seen_names:
seen_names[name] = str(len(seen_names))
- return ret + '_' + seen_names[name]
+ return unsuffixed(name) + '_' + seen_names[name] + (('.' + suffix(name)) if suffix(name) else '')
# ---------------- End configs -------------
@@ -411,22 +399,7 @@ else:
temp_dir = tempfile.mkdtemp()
def in_temp(name):
- return os.path.join(temp_dir, name)
-
-class Compression:
- on = False
-
- @staticmethod
- def compressed_name(filename):
- return filename + '.compress'
-
- @staticmethod
- def compress(filename):
- execute(Compression.encoder, stdin=open(filename, 'rb'), stdout=open(Compression.compressed_name(filename), 'wb'))
-
- @staticmethod
- def worth_it(original, compressed):
- return compressed < original - 1500 # save at least one TCP packet or so
+ return os.path.join(temp_dir, os.path.basename(name))
try:
call = CXX if use_cxx else CC
@@ -440,14 +413,16 @@ try:
llvm_lto = None
closure = None
js_transform = None
- pre_js = None
- post_js = None
+ pre_js = ''
+ post_js = ''
minify_whitespace = None
- data_files = []
+ preload_files = []
+ embed_files = []
compression = None
ignore_dynamic_linking = False
shell_path = shared.path_from_root('src', 'shell.html')
js_libraries = []
+ remove_duplicates = False
def check_bad_eq(arg):
assert '=' not in arg, 'Invalid parameter (do not use "=" with "--" options)'
@@ -482,12 +457,12 @@ try:
newargs[i+1] = ''
elif newargs[i].startswith('--pre-js'):
check_bad_eq(newargs[i])
- pre_js = open(newargs[i+1]).read()
+ pre_js += open(newargs[i+1]).read() + '\n'
newargs[i] = ''
newargs[i+1] = ''
elif newargs[i].startswith('--post-js'):
check_bad_eq(newargs[i])
- post_js = open(newargs[i+1]).read()
+ post_js += open(newargs[i+1]).read() + '\n'
newargs[i] = ''
newargs[i+1] = ''
elif newargs[i].startswith('--minify'):
@@ -497,12 +472,12 @@ try:
newargs[i+1] = ''
elif newargs[i].startswith('--embed-file'):
check_bad_eq(newargs[i])
- data_files.append({ 'name': newargs[i+1], 'mode': 'embed' })
+ embed_files.append(newargs[i+1])
newargs[i] = ''
newargs[i+1] = ''
elif newargs[i].startswith('--preload-file'):
check_bad_eq(newargs[i])
- data_files.append({ 'name': newargs[i+1], 'mode': 'preload' })
+ preload_files.append(newargs[i+1])
newargs[i] = ''
newargs[i+1] = ''
elif newargs[i].startswith('--compression'):
@@ -534,6 +509,9 @@ try:
js_libraries.append(newargs[i+1])
newargs[i] = ''
newargs[i+1] = ''
+ elif newargs[i] == '--remove-duplicates':
+ remove_duplicates = True
+ newargs[i] = ''
newargs = [ arg for arg in newargs if arg is not '' ]
if llvm_opts is None: llvm_opts = LLVM_OPT_LEVEL[opt_level]
@@ -564,7 +542,8 @@ try:
input_files = []
has_source_inputs = False
- lib_dirs = [shared.path_from_root('system', 'lib')]
+ lib_dirs = [shared.path_from_root('system', 'local', 'lib'),
+ shared.path_from_root('system', 'lib')]
libs = []
for i in range(len(newargs)): # find input files XXX this a simple heuristic. we should really analyze based on a full understanding of gcc params,
# right now we just assume that what is left contains no more |-x OPT| things
@@ -605,6 +584,8 @@ try:
libs.append(arg[2:])
newargs[i] = ''
+ original_input_files = input_files[:]
+
newargs = [ arg for arg in newargs if arg is not '' ]
# Find library files
@@ -670,7 +651,7 @@ try:
for input_file in input_files:
if input_file.endswith(SOURCE_SUFFIXES):
if DEBUG: print >> sys.stderr, 'emcc: compiling source file: ', input_file
- output_file = in_temp(unsuffixed_uniquename(input_file) + '.o')
+ output_file = in_temp(unsuffixed(uniquename(input_file)) + '.o')
temp_files.append(output_file)
args = newargs + ['-emit-llvm', '-c', input_file, '-o', output_file]
if DEBUG: print >> sys.stderr, "emcc running:", call, ' '.join(args)
@@ -681,12 +662,12 @@ try:
else: # bitcode
if input_file.endswith(BITCODE_SUFFIXES):
if DEBUG: print >> sys.stderr, 'emcc: copying bitcode file: ', input_file
- temp_file = in_temp(unsuffixed_uniquename(input_file) + '.o')
+ temp_file = in_temp(unsuffixed(uniquename(input_file)) + '.o')
shutil.copyfile(input_file, temp_file)
temp_files.append(temp_file)
elif input_file.endswith(DYNAMICLIB_SUFFIXES) or shared.Building.is_ar(input_file):
if DEBUG: print >> sys.stderr, 'emcc: copying library file: ', input_file
- temp_file = in_temp(os.path.basename(input_file))
+ temp_file = in_temp(uniquename(input_file))
shutil.copyfile(input_file, temp_file)
temp_files.append(temp_file)
else: #.ll
@@ -694,7 +675,7 @@ try:
# Note that by assembling the .ll file, then disassembling it later, we will
# remove annotations which is a good thing for compilation time
if DEBUG: print >> sys.stderr, 'emcc: assembling assembly file: ', input_file
- temp_file = in_temp(unsuffixed_uniquename(input_file) + '.o')
+ temp_file = in_temp(unsuffixed(uniquename(input_file)) + '.o')
shared.Building.llvm_as(input_file, temp_file)
temp_files.append(temp_file)
@@ -706,25 +687,22 @@ try:
print >> sys.stderr, 'emcc: warning: -Ox flags ignored, since not generating JavaScript'
if not specified_target:
for input_file in input_files:
- shutil.move(in_temp(unsuffixed_uniquename(input_file) + '.o'), unsuffixed_basename(input_file) + '.' + final_suffix)
+ shutil.move(in_temp(unsuffixed(uniquename(input_file)) + '.o'), unsuffixed_basename(input_file) + '.' + final_suffix)
else:
if len(input_files) == 1:
- shutil.move(in_temp(unsuffixed_uniquename(input_files[0]) + '.o'), specified_target)
+ shutil.move(in_temp(unsuffixed(uniquename(input_files[0])) + '.o'), specified_target)
else:
- assert not has_dash_c, 'fatal error: cannot specify -o with -c with multiple files' + str(sys.argv)
+ assert len(original_input_files) == 1 or not has_dash_c, 'fatal error: cannot specify -o with -c with multiple files' + str(sys.argv) + ':' + str(original_input_files)
# We have a specified target (-o <target>), which is not JavaScript or HTML, and
# we have multiple files: Link them
if DEBUG: print >> sys.stderr, 'emcc: link: ' + str(temp_files)
- shared.Building.link(temp_files, specified_target)
+ shared.Building.link(temp_files, specified_target, remove_duplicates=remove_duplicates)
exit(0)
## Continue on to create JavaScript
if DEBUG: print >> sys.stderr, 'emcc: will generate JavaScript'
- if final_suffix == 'html':
- shared.Settings.GENERATING_HTML = 1
-
extra_files_to_link = []
if not LEAVE_INPUTS_RAW and not AUTODEBUG:
@@ -764,7 +742,7 @@ try:
assert shared.Settings.QUANTUM_SIZE == 4, 'We do not support libc++ with QUANTUM_SIZE == 1'
# libcxx might need corrections, so turn them all on. TODO: check which are actually needed
shared.Settings.CORRECT_SIGNS = shared.Settings.CORRECT_OVERFLOWS = shared.Settings.CORRECT_ROUNDINGS = 1
- print >> sys.stderr, 'emcc: warning: using libcxx turns on CORRECT_* options'
+ #print >> sys.stderr, 'emcc: info: using libcxx turns on CORRECT_* options'
libcxx_symbols = map(lambda line: line.strip().split(' ')[1], open(shared.path_from_root('system', 'lib', 'libcxx', 'symbols')).readlines())
libcxx_symbols = filter(lambda symbol: symbol not in dlmalloc_symbols, libcxx_symbols)
libcxx_symbols = set(libcxx_symbols)
@@ -776,7 +754,7 @@ try:
return os.path.join(shared.EMSCRIPTEN_TEMP_DIR, 'libcxxabi', 'libcxxabi.bc')
def fix_libcxxabi():
assert shared.Settings.QUANTUM_SIZE == 4, 'We do not support libc++abi with QUANTUM_SIZE == 1'
- print >> sys.stderr, 'emcc: warning: using libcxxabi, this may need CORRECT_* options'
+ #print >> sys.stderr, 'emcc: info: using libcxxabi, this may need CORRECT_* options'
#shared.Settings.CORRECT_SIGNS = shared.Settings.CORRECT_OVERFLOWS = shared.Settings.CORRECT_ROUNDINGS = 1
libcxxabi_symbols = map(lambda line: line.strip().split(' ')[1], open(shared.path_from_root('system', 'lib', 'libcxxabi', 'symbols')).readlines())
libcxxabi_symbols = filter(lambda symbol: symbol not in dlmalloc_symbols, libcxxabi_symbols)
@@ -810,8 +788,7 @@ try:
(not LEAVE_INPUTS_RAW and not (suffix(temp_files[0]) in BITCODE_SUFFIXES or suffix(temp_files[0]) in DYNAMICLIB_SUFFIXES) and shared.Building.is_ar(temp_files[0])):
linker_inputs = temp_files + extra_files_to_link
if DEBUG: print >> sys.stderr, 'emcc: linking: ', linker_inputs
- shared.Building.link(linker_inputs,
- in_temp(target_basename + '.bc'))
+ shared.Building.link(linker_inputs, in_temp(target_basename + '.bc'), remove_duplicates=remove_duplicates)
final = in_temp(target_basename + '.bc')
else:
if not LEAVE_INPUTS_RAW:
@@ -871,216 +848,18 @@ try:
if DEBUG: save_intermediate('original')
# Embed and preload files
- if len(data_files) > 0:
+ if len(preload_files) + len(embed_files) > 0:
if DEBUG: print >> sys.stderr, 'emcc: setting up files'
- code = ''
-
- if final_suffix == 'html':
- code += '''
- var BlobBuilder = typeof MozBlobBuilder != "undefined" ? MozBlobBuilder : (typeof WebKitBlobBuilder != "undefined" ? WebKitBlobBuilder : console.log("warning: cannot build blobs"));
- var URLObject = typeof window != "undefined" ? (window.URL ? window.URL : window.webkitURL) : console.log("warning: cannot create object URLs");
- var hasBlobConstructor;
- try {
- new Blob();
- hasBlobConstructor = true;
- } catch(e) {
- hasBlobConstructor = false;
- console.log("warning: no blob constructor, cannot create blobs with mimetypes");
- }
-'''
-
- code += 'var preloadedImages = {}; // maps url to image data\n'
- code += 'var preloadedAudios = {}; // maps url to audio data\n'
-
- # Expand directories into individual files
- def add(mode, dirname, names):
- for name in names:
- fullname = os.path.join(dirname, name)
- if not os.path.isdir(fullname):
- data_files.append({ 'name': fullname, 'mode': mode })
-
- for file_ in data_files:
- if os.path.isdir(file_['name']):
- os.path.walk(file_['name'], add, file_['mode'])
- data_files = filter(lambda file_: not os.path.isdir(file_['name']), data_files)
-
- for file_ in data_files:
- file_['name'] = file_['name'].replace(os.path.sep, '/')
- file_['net_name'] = file_['name']
-
- data_target = unsuffixed(target) + '.data'
-
- # Set up folders
- partial_dirs = []
- for file_ in data_files:
- dirname = os.path.dirname(file_['name'])
- dirname = dirname.lstrip('/') # absolute paths start with '/', remove that
- if dirname != '':
- parts = dirname.split('/')
- for i in range(len(parts)):
- partial = '/'.join(parts[:i+1])
- if partial not in partial_dirs:
- code += '''FS.createFolder('/%s', '%s', true, false);\n''' % ('/'.join(parts[:i]), parts[i])
- partial_dirs.append(partial)
-
- if final_suffix == 'html':
- # Bundle all datafiles into one archive. Avoids doing lots of simultaneous XHRs which has overhead.
- data = open(data_target, 'wb')
- start = 0
- for file_ in data_files:
- file_['data_start'] = start
- curr = open(file_['name'], 'rb').read()
- file_['data_end'] = start + len(curr)
- start += len(curr)
- data.write(curr)
- data.close()
- if Compression.on:
- Compression.compress(data_target)
-
- # Data requests - for getting a block of data out of the big archive - have a similar API to XHRs
- code += '''
- function DataRequest() {}
- DataRequest.prototype = {
- requests: {},
- open: function(mode, name) {
- this.requests[name] = this;
- },
- send: function() {}
- };
- '''
-
- counter = 0
- for file_ in data_files:
- filename = file_['name']
- if file_['mode'] == 'embed':
- # Embed
- code += '''FS.createDataFile('/', '%s', %s, true, true);\n''' % (os.path.basename(filename), str(map(ord, open(filename, 'rb').read())))
- elif file_['mode'] == 'preload':
- # Preload
- assert final_suffix == 'html', 'Can only preload files when generating HTML'
-
- varname = 'filePreload%d' % counter
- counter += 1
- image = filename.endswith(IMAGE_SUFFIXES)
- audio = filename.endswith(AUDIO_SUFFIXES)
-
- if image:
- finish = '''
- var bb = new BlobBuilder();
- bb.append(byteArray.buffer);
- var b = bb.getBlob();
- var url = URLObject.createObjectURL(b);
- var img = new Image();
- img.onload = function() {
- assert(img.complete, 'Image %(filename)s could not be decoded');
- var canvas = document.createElement('canvas');
- canvas.width = img.width;
- canvas.height = img.height;
- var ctx = canvas.getContext('2d');
- ctx.drawImage(img, 0, 0);
- preloadedImages['%(filename)s'] = canvas;
- URLObject.revokeObjectURL(url);
- removeRunDependency();
- };
- img.onerror = function(event) {
- console.log('Image %(filename)s could not be decoded');
- };
- img.src = url;
-''' % { 'filename': filename }
- elif audio:
- # Need actual blob constructor here, to set the mimetype or else audios fail to decode
- finish = '''
- if (hasBlobConstructor) {
- var b = new Blob([byteArray.buffer], { type: '%(mimetype)s' });
- var url = URLObject.createObjectURL(b); // XXX we never revoke this!
- var audio = new Audio();
- audio.removedDependency = false;
- audio['oncanplaythrough'] = function() { // XXX string for closure
- audio['oncanplaythrough'] = null;
- preloadedAudios['%(filename)s'] = audio;
- if (!audio.removedDependency) {
- removeRunDependency();
- audio.removedDependency = true;
- }
- };
- audio.onerror = function(event) {
- if (!audio.removedDependency) {
- console.log('Audio %(filename)s could not be decoded or timed out trying to decode');
- removeRunDependency();
- audio.removedDependency = true;
- }
- };
- setTimeout(audio.onerror, 2000); // workaround for chromium bug 124926 (still no audio with this, but at least we don't hang)
- audio.src = url;
- } else {
- preloadedAudios['%(filename)s'] = new Audio(); // empty shim
- removeRunDependency();
- }
-''' % { 'filename': filename, 'mimetype': AUDIO_MIMETYPES[suffix(filename)] }
- else:
- finish = 'removeRunDependency();\n'
-
- code += '''
- var %(varname)s = new %(request)s();
- %(varname)s.open('GET', '%(netname)s', true);
- %(varname)s.responseType = 'arraybuffer';
- %(varname)s.onload = function() {
- var arrayBuffer = %(varname)s.response;
- assert(arrayBuffer, 'Loading file %(filename)s failed.');
- var byteArray = arrayBuffer.byteLength ? new Uint8Array(arrayBuffer) : arrayBuffer;
- FS.createDataFile('/%(dirname)s', '%(basename)s', byteArray, true, true);
- %(finish)s
- };
- addRunDependency();
- %(varname)s.send(null);
-''' % {
- 'request': 'DataRequest', # In the past we also supported XHRs here
- 'varname': varname,
- 'filename': filename,
- 'netname': file_['net_name'],
- 'dirname': os.path.dirname(filename),
- 'basename': os.path.basename(filename),
- 'finish': finish
- }
- else:
- assert 0
-
- if final_suffix == 'html':
- # Get the big archive and split it up
- use_data = ''
- for file_ in data_files:
- if file_['mode'] == 'preload':
- use_data += '''
- curr = DataRequest.prototype.requests['%s'];
- curr.response = byteArray.subarray(%d,%d);
- curr.onload();
- ''' % (file_['name'], file_['data_start'], file_['data_end'])
- use_data += ' removeRunDependency();\n'
-
- if Compression.on:
- use_data = '''
- Module["decompress"](byteArray, function(decompressed) {
- byteArray = new Uint8Array(decompressed);
- %s
- });
- ''' % use_data
-
- code += '''
- var dataFile = new XMLHttpRequest();
- dataFile.open('GET', '%s', true);
- dataFile.responseType = 'arraybuffer';
- dataFile.onload = function() {
- var arrayBuffer = dataFile.response;
- assert(arrayBuffer, 'Loading data file failed.');
- var byteArray = new Uint8Array(arrayBuffer);
- var curr;
- %s
- };
- addRunDependency();
- dataFile.send(null);
- if (Module['setStatus']) Module['setStatus']('Downloading...');
- ''' % (Compression.compressed_name(data_target) if Compression.on else data_target, use_data)
-
+ file_args = []
+ if len(preload_files) > 0:
+ file_args.append('--preload')
+ file_args += preload_files
+ if len(embed_files) > 0:
+ file_args.append('--embed')
+ file_args += embed_files
+ if Compression.on:
+ file_args += ['--compress', Compression.encoder, Compression.decoder, Compression.js_name]
+ code = execute(shared.ENV_PREFIX + ['python', shared.FILE_PACKAGER, unsuffixed(target) + '.data'] + file_args, stdout=PIPE)[0]
src = open(final).read().replace('// {{PRE_RUN_ADDITIONS}}', code)
final += '.files.js'
open(final, 'w').write(src)
@@ -1091,7 +870,7 @@ try:
if DEBUG: print >> sys.stderr, 'emcc: applying pre/postjses'
src = open(final).read()
final += '.pp.js'
- open(final, 'w').write((pre_js or '') + src + (post_js or ''))
+ open(final, 'w').write(pre_js + src + post_js)
if DEBUG: save_intermediate('pre-post')
# Apply a source code transformation, if requested
generated by cgit v1.2.3-18-g5258 (git 2.32.0) at 2025-07-21 19:29:50 +0000