1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
|
import os, sys, subprocess, multiprocessing, re
import shared
temp_files = shared.make_temp_files()
__rootpath__ = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
def path_from_root(*pathelems):
return os.path.join(__rootpath__, *pathelems)
JS_OPTIMIZER = path_from_root('tools', 'js-optimizer.js')
NUM_CHUNKS_PER_CORE = 1.5
MIN_CHUNK_SIZE = int(os.environ.get('EMCC_JSOPT_MIN_CHUNK_SIZE') or 1024*1024) # configuring this is just for debugging purposes
MAX_CHUNK_SIZE = 20*1024*1024
WINDOWS = sys.platform.startswith('win')
DEBUG = os.environ.get('EMCC_DEBUG')
def run_on_chunk(command):
filename = command[2] # XXX hackish
#print >> sys.stderr, 'running js optimizer command', ' '.join(command), '""""', open(filename).read()
output = subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0]
assert len(output) > 0 and not output.startswith('Assertion failed'), 'Error in js optimizer: ' + output
filename = temp_files.get(os.path.basename(filename) + '.jo.js').name
f = open(filename, 'w')
f.write(output)
f.close()
return filename
def run_on_js(filename, passes, js_engine, jcache):
if jcache: shared.JCache.ensure()
if type(passes) == str:
passes = [passes]
js = open(filename).read()
if os.linesep != '\n':
js = js.replace(os.linesep, '\n') # we assume \n in the splitting code
# Find suffix
suffix_marker = '// EMSCRIPTEN_GENERATED_FUNCTIONS'
suffix_start = js.find(suffix_marker)
suffix = ''
if suffix_start >= 0:
suffix = js[suffix_start:js.find('\n', suffix_start)] + '\n'
# if there is metadata, we will run only on the generated functions. If there isn't, we will run on everything.
generated = set(eval(suffix[len(suffix_marker)+1:]))
if not suffix and jcache:
# JCache cannot be used without metadata, since it might reorder stuff, and that's dangerous since only generated can be reordered
# This means jcache does not work after closure compiler runs, for example. But you won't get much benefit from jcache with closure
# anyhow (since closure is likely the longest part of the build).
if DEBUG: print >>sys.stderr, 'js optimizer: no metadata, so disabling jcache'
jcache = False
# If we process only generated code, find that and save the rest on the side
func_sig = re.compile('( *)function (_[\w$]+)\(')
if suffix:
pos = 0
gen_start = 0
gen_end = 0
while 1:
m = func_sig.search(js, pos)
if not m: break
pos = m.end()
indent = m.group(1)
ident = m.group(2)
if ident in generated:
if not gen_start:
gen_start = m.start()
assert gen_start
gen_end = js.find('\n%s}\n' % indent, m.end()) + (3 + len(indent))
assert gen_end > gen_start
pre = js[:gen_start]
post = js[gen_end:]
if 'last' in passes:
post = post.replace(suffix, '') # no need to write out the metadata - nothing after us needs it
js = js[gen_start:gen_end]
else:
pre = ''
post = ''
# Pick where to split into chunks, so that (1) they do not oom in node/uglify, and (2) we can run them in parallel
# If we have metadata, we split only the generated code, and save the pre and post on the side (and do not optimize them)
parts = map(lambda part: part, js.split('\n}\n'))
funcs = []
for i in range(len(parts)):
func = parts[i]
if i < len(parts)-1: func += '\n}\n' # last part needs no }
m = func_sig.search(func)
if m:
ident = m.group(2)
else:
if suffix: continue # ignore whitespace
ident = 'anon_%d' % i
assert ident
funcs.append((ident, func))
parts = None
total_size = len(js)
js = None
cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count())
intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE))
chunk_size = min(MAX_CHUNK_SIZE, max(MIN_CHUNK_SIZE, total_size / intended_num_chunks))
chunks = shared.chunkify(funcs, chunk_size, jcache.get_cachename('jsopt') if jcache else None)
if jcache:
# load chunks from cache where we can # TODO: ignore small chunks
cached_outputs = []
def load_from_cache(chunk):
keys = [chunk]
shortkey = shared.JCache.get_shortkey(keys) # TODO: share shortkeys with later code
out = shared.JCache.get(shortkey, keys)
if out:
cached_outputs.append(out)
return False
return True
chunks = filter(load_from_cache, chunks)
if len(cached_outputs) > 0:
if DEBUG: print >> sys.stderr, ' loading %d jsfuncchunks from jcache' % len(cached_outputs)
else:
cached_outputs = []
if len(chunks) > 0:
def write_chunk(chunk, i):
temp_file = temp_files.get('.jsfunc_%d.js' % i).name
f = open(temp_file, 'w')
f.write(chunk)
f.write(suffix)
f.close()
return temp_file
filenames = [write_chunk(chunks[i], i) for i in range(len(chunks))]
else:
filenames = []
if len(filenames) > 0:
# XXX Use '--nocrankshaft' to disable crankshaft to work around v8 bug 1895, needed for older v8/node (node 0.6.8+ should be ok)
commands = map(lambda filename: js_engine + [JS_OPTIMIZER, filename, 'noPrintMetadata'] + passes, filenames)
#print [' '.join(command) for command in commands]
cores = min(cores, filenames)
if len(chunks) > 1 and cores >= 2:
# We can parallelize
if DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks of size %d, using %d cores (total: %.2f MB)' % (len(chunks), chunk_size, cores, total_size/(1024*1024.))
pool = multiprocessing.Pool(processes=cores)
filenames = pool.map(run_on_chunk, commands, chunksize=1)
else:
# We can't parallize, but still break into chunks to avoid uglify/node memory issues
if len(chunks) > 1 and DEBUG: print >> sys.stderr, 'splitting up js optimization into %d chunks of size %d' % (len(chunks), chunk_size)
filenames = [run_on_chunk(command) for command in commands]
else:
filenames = []
for filename in filenames: temp_files.note(filename)
filename += '.jo.js'
f = open(filename, 'w')
f.write(pre);
for out_file in filenames:
f.write(open(out_file).read())
f.write('\n')
if jcache:
for cached in cached_outputs:
f.write(cached); # TODO: preserve order
f.write('\n')
f.write(post);
# No need to write suffix: if there was one, it is inside post which exists when suffix is there
f.write('\n')
f.close()
if jcache:
# save chunks to cache
for i in range(len(chunks)):
chunk = chunks[i]
keys = [chunk]
shortkey = shared.JCache.get_shortkey(keys)
shared.JCache.set(shortkey, keys, open(filenames[i]).read())
if DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d jsfuncchunks to jcache' % len(chunks)
return filename
def run(filename, passes, js_engine, jcache):
return temp_files.run_and_clean(lambda: run_on_js(filename, passes, js_engine, jcache))
|
