aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/library.js93
-rw-r--r--src/settings.js4
-rw-r--r--src/shell.js26
-rw-r--r--tests/checksummer.c71
-rwxr-xr-xtests/runner.py108
5 files changed, 293 insertions, 9 deletions
diff --git a/src/library.js b/src/library.js
index a89f1251..69642151 100644
--- a/src/library.js
+++ b/src/library.js
@@ -289,7 +289,85 @@ LibraryManager.library = {
// XHR, which is not possible in browsers except in a web worker! Use preloading,
// either --preload-file in emcc or FS.createPreloadedFile
createLazyFile: function(parent, name, url, canRead, canWrite) {
- var properties = {isDevice: false, url: url};
+
+ if (typeof XMLHttpRequest !== 'undefined') {
+ if (!ENVIRONMENT_IS_WORKER) throw 'Cannot do synchronous binary XHRs outside webworkers in modern browsers. Use --embed-file or --preload-file in emcc';
+
+ // Lazy chunked Uint8Array (implements get and length from Uint8Array). Actual getting is abstracted away for eventual reuse.
+ var LazyUint8Array = function(chunkSize, length) {
+ this.length = length;
+ this.chunkSize = chunkSize;
+ this.chunks = []; // Loaded chunks. Index is the chunk number
+ this.isLazyUint8Array = true;
+ }
+ LazyUint8Array.prototype.get = function(idx) {
+ if (idx > this.length-1 || idx < 0) {
+ return undefined;
+ }
+ var chunkOffset = idx % chunkSize;
+ var chunkNum = Math.floor(idx / chunkSize);
+ return this.getter(chunkNum)[chunkOffset];
+ }
+ LazyUint8Array.prototype.setDataGetter = function(getter) {
+ this.getter = getter;
+ }
+
+ // Find length
+ var xhr = new XMLHttpRequest();
+ xhr.open('HEAD', url, false);
+ xhr.send(null);
+ if (!(xhr.status >= 200 && xhr.status < 300 || xhr.status === 304)) throw new Error("Couldn't load " + url + ". Status: " + xhr.status);
+ var datalength = Number(xhr.getResponseHeader("Content-length"));
+ var header;
+ var hasByteServing = (header = xhr.getResponseHeader("Accept-Ranges")) && header === "bytes";
+#if SMALL_CHUNKS
+ var chunkSize = 1024; // Chunk size in bytes
+#else
+ var chunkSize = 1024*1024; // Chunk size in bytes
+#endif
+ if (!hasByteServing) chunkSize = datalength;
+
+ // Function to get a range from the remote URL.
+ var doXHR = (function(from, to) {
+ if (from > to) throw new Error("invalid range (" + from + ", " + to + ") or no bytes requested!");
+ if (to > datalength-1) throw new Error("only " + datalength + " bytes available! programmer error!");
+
+ // TODO: Use mozResponseArrayBuffer, responseStream, etc. if available.
+ var xhr = new XMLHttpRequest();
+ xhr.open('GET', url, false);
+ if (datalength !== chunkSize) xhr.setRequestHeader("Range", "bytes=" + from + "-" + to);
+
+ // Some hints to the browser that we want binary data.
+ if (typeof Uint8Array != 'undefined') xhr.responseType = 'arraybuffer';
+ if (xhr.overrideMimeType) {
+ xhr.overrideMimeType('text/plain; charset=x-user-defined');
+ }
+
+ xhr.send(null);
+ if (!(xhr.status >= 200 && xhr.status < 300 || xhr.status === 304)) throw new Error("Couldn't load " + url + ". Status: " + xhr.status);
+ if (xhr.response !== undefined) {
+ return new Uint8Array(xhr.response || []);
+ } else {
+ return intArrayFromString(xhr.responseText || '', true);
+ }
+ });
+
+ var lazyArray = new LazyUint8Array(chunkSize, datalength);
+ lazyArray.setDataGetter(function(chunkNum) {
+ var start = chunkNum * lazyArray.chunkSize;
+ var end = (chunkNum+1) * lazyArray.chunkSize - 1; // including this byte
+ end = Math.min(end, datalength-1); // if datalength-1 is selected, this is the last block
+ if (typeof(lazyArray.chunks[chunkNum]) === "undefined") {
+ lazyArray.chunks[chunkNum] = doXHR(start, end);
+ }
+ if (typeof(lazyArray.chunks[chunkNum]) === "undefined") throw new Error("doXHR failed!");
+ return lazyArray.chunks[chunkNum];
+ });
+ var properties = { isDevice: false, contents: lazyArray };
+ } else {
+ var properties = { isDevice: false, url: url };
+ }
+
return FS.createFile(parent, name, properties, canRead, canWrite);
},
// Preloads a file asynchronously. You can call this before run, for example in
@@ -360,8 +438,7 @@ LibraryManager.library = {
if (obj.isDevice || obj.isFolder || obj.link || obj.contents) return true;
var success = true;
if (typeof XMLHttpRequest !== 'undefined') {
- // Browser.
- throw 'Cannot do synchronous binary XHRs in modern browsers. Use --embed-file or --preload-file in emcc';
+ throw new Error("Lazy loading should have been performed (contents set) in createLazyFile, but it was not. Lazy loading only works in web workers. Use --embed-file or --preload-file in emcc on the main thread.");
} else if (Module['read']) {
// Command-line.
try {
@@ -1631,8 +1708,14 @@ LibraryManager.library = {
}
var contents = stream.object.contents;
var size = Math.min(contents.length - offset, nbyte);
- for (var i = 0; i < size; i++) {
- {{{ makeSetValue('buf', 'i', 'contents[offset + i]', 'i8') }}}
+ if (contents.isLazyUint8Array) {
+ for (var i = 0; i < size; i++) {
+ {{{ makeSetValue('buf', 'i', 'contents.get(offset + i)', 'i8') }}}
+ }
+ } else {
+ for (var i = 0; i < size; i++) {
+ {{{ makeSetValue('buf', 'i', 'contents[offset + i]', 'i8') }}}
+ }
}
bytesRead += size;
return bytesRead;
diff --git a/src/settings.js b/src/settings.js
index 5b1968b9..4017ad98 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -255,6 +255,10 @@ var WARN_ON_UNDEFINED_SYMBOLS = 0; // If set to 1, we will warn on any undefined
// the existing buildsystem), and (2) functions might be
// implemented later on, say in --pre-js
+var SMALL_CHUNKS = 0; // Use small chunk size for binary synchronous XHR's in Web Workers.
+ // Used for testing.
+ // See test_chunked_synchronous_xhr in runner.py and library.js.
+
// Compiler debugging options
var DEBUG_TAGS_SHOWING = [];
// Some useful items:
diff --git a/src/shell.js b/src/shell.js
index 891a6328..3fb6cdbb 100644
--- a/src/shell.js
+++ b/src/shell.js
@@ -44,7 +44,9 @@ if (ENVIRONMENT_IS_NODE) {
if (!Module['arguments']) {
Module['arguments'] = process['argv'].slice(2);
}
-} else if (ENVIRONMENT_IS_SHELL) {
+}
+
+if (ENVIRONMENT_IS_SHELL) {
Module['print'] = print;
if (typeof printErr != 'undefined') Module['printErr'] = printErr; // not present in v8 or older sm
@@ -62,7 +64,9 @@ if (ENVIRONMENT_IS_NODE) {
Module['arguments'] = arguments;
}
}
-} else if (ENVIRONMENT_IS_WEB) {
+}
+
+if (ENVIRONMENT_IS_WEB && !ENVIRONMENT_IS_WORKER) {
if (!Module['print']) {
Module['print'] = function(x) {
console.log(x);
@@ -74,7 +78,9 @@ if (ENVIRONMENT_IS_NODE) {
console.log(x);
};
}
+}
+if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) {
Module['read'] = function(url) {
var xhr = new XMLHttpRequest();
xhr.open('GET', url, false);
@@ -87,12 +93,24 @@ if (ENVIRONMENT_IS_NODE) {
Module['arguments'] = arguments;
}
}
-} else if (ENVIRONMENT_IS_WORKER) {
+}
+
+if (ENVIRONMENT_IS_WORKER) {
// We can do very little here...
+ var TRY_USE_DUMP = false;
+ if (!Module['print']) {
+ Module['print'] = (TRY_USE_DUMP && (typeof(dump) !== "undefined") ? (function(x) {
+ dump(x);
+ }) : (function(x) {
+ self.postMessage(x);
+ }));
+ }
Module['load'] = importScripts;
+}
-} else {
+if (!ENVIRONMENT_IS_WORKER && !ENVIRONMENT_IS_WEB && !ENVIRONMENT_IS_NODE && !ENVIRONMENT_IS_SHELL) {
+ // Unreachable because SHELL is dependant on the others
throw 'Unknown runtime environment. Where are we?';
}
diff --git a/tests/checksummer.c b/tests/checksummer.c
new file mode 100644
index 00000000..c3eb1eea
--- /dev/null
+++ b/tests/checksummer.c
@@ -0,0 +1,71 @@
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+const int MOD_ADLER = 65521;
+
+uint64_t adler32(unsigned char *data, int32_t len) /* where data is the location of the data in physical memory and
+ len is the length of the data in bytes */
+{
+ uint64_t a = 1, b = 0;
+ int32_t index;
+
+ /* Process each byte of the data in order */
+ for (index = 0; index < len; ++index)
+ {
+ a = (a + data[index]) % MOD_ADLER;
+ b = (b + a) % MOD_ADLER;
+ }
+
+ return (b << 16) | a;
+}
+
+int main(int argc, char* argv[]) {
+ long bufsize;
+
+ if (argc != 2) {
+ fputs("Need 1 argument\n", stderr);
+ return (EXIT_FAILURE);
+ }
+
+ unsigned char *source = NULL;
+ FILE *fp = fopen(argv[1], "rb");
+ if (fp != NULL) {
+ /* Go to the end of the file. */
+ if (fseek(fp, 0L, SEEK_END) == 0) {
+ /* Get the size of the file. */
+ bufsize = ftell(fp);
+ if (bufsize == -1) { fputs("Couldn't get size\n", stderr); return (EXIT_FAILURE); }
+
+ /* Allocate our buffer to that size. */
+ source = malloc(sizeof(char) * (bufsize + 1));
+ if (source == NULL) { fputs("Couldn't allocate\n", stderr); return (EXIT_FAILURE); }
+
+ /* Go back to the start of the file. */
+ if (fseek(fp, 0L, SEEK_SET) == -1) { fputs("Couldn't seek\n", stderr); return (EXIT_FAILURE); }
+
+ /* Read the entire file into memory. */
+ size_t newLen = fread(source, sizeof(char), bufsize, fp);
+ if (newLen == 0) {
+ fputs("Error reading file\n", stderr);
+ //return (EXIT_FAILURE);
+ } else {
+ source[++newLen] = '\0'; /* Just to be safe. */
+ }
+ } else {
+ fputs("Couldn't seek to end\n", stderr);
+ return (EXIT_FAILURE);
+ }
+ fclose(fp);
+ } else {
+ fputs("Couldn't open\n", stderr);
+ return (EXIT_FAILURE);
+ }
+
+ printf("%d\n", (uint32_t) adler32(source, bufsize));
+
+ free(source); /* Don't forget to call free() later! */
+
+ return (EXIT_SUCCESS);
+
+}
diff --git a/tests/runner.py b/tests/runner.py
index 1d35fbc8..a419ff0d 100755
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -8780,6 +8780,114 @@ elif 'browser' in str(sys.argv):
html_file.close()
self.run_browser('main.html', 'You should see that the worker was called, and said "hello from worker!"', '/report_result?hello%20from%20worker!')
+ def test_chunked_synchronous_xhr(self):
+ main = 'chunked_sync_xhr.html'
+ worker_filename = "download_and_checksum_worker.js"
+
+ html_file = open(main, 'w')
+ html_file.write(r"""
+ <!doctype html>
+ <html>
+ <head><meta charset="utf-8"><title>Chunked XHR</title></head>
+ <html>
+ <body>
+ Chunked XHR Web Worker Test
+ <script>
+ var worker = new Worker(""" + json.dumps(worker_filename) + r""");
+ var buffer = [];
+ worker.onmessage = function(event) {
+ if (event.data.channel === "stdout") {
+ var xhr = new XMLHttpRequest();
+ xhr.open('GET', 'http://localhost:8888/report_result?' + event.data.line);
+ xhr.send();
+ setTimeout(function() { window.close() }, 1000);
+ } else {
+ if (event.data.trace) event.data.trace.split("\n").map(function(v) { console.error(v); });
+ if (event.data.line) {
+ console.error(event.data.line);
+ } else {
+ var v = event.data.char;
+ if (v == 10) {
+ var line = buffer.splice(0);
+ console.error(line = line.map(function(charCode){return String.fromCharCode(charCode);}).join(''));
+ } else {
+ buffer.push(v);
+ }
+ }
+ }
+ };
+ </script>
+ </body>
+ </html>
+ """)
+ html_file.close()
+
+ c_source_filename = "checksummer.c"
+
+ prejs_filename = "worker_prejs.js"
+ prejs_file = open(prejs_filename, 'w')
+ prejs_file.write(r"""
+ if (typeof(Module) === "undefined") Module = {};
+ Module["arguments"] = ["/bigfile"];
+ Module["preInit"] = function() {
+ FS.createLazyFile('/', "bigfile", "http://localhost:11111/bogus_file_path", true, false);
+ };
+ var doTrace = true;
+ Module["print"] = function(s) { self.postMessage({channel: "stdout", line: s}); };
+ Module["stderr"] = function(s) { self.postMessage({channel: "stderr", char: s, trace: ((doTrace && s === 10) ? new Error().stack : null)}); doTrace = false; };
+ """)
+ prejs_file.close()
+ # vs. os.path.join(self.get_dir(), filename)
+ # vs. path_from_root('tests', 'hello_world_gles.c')
+ Popen(['python', EMCC, path_from_root('tests', c_source_filename), '-g', '-s', 'SMALL_CHUNKS=1', '-o', worker_filename,
+ '--pre-js', prejs_filename]).communicate()
+
+ chunkSize = 1024
+ data = os.urandom(10*chunkSize+1) # 10 full chunks and one 1 byte chunk
+ expectedConns = 11
+ import zlib
+ checksum = zlib.adler32(data)
+
+ def chunked_server(support_byte_ranges):
+ class ChunkedServerHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+ @staticmethod
+ def sendheaders(s, extra=[], length=len(data)):
+ s.send_response(200)
+ s.send_header("Content-Length", str(length))
+ s.send_header("Access-Control-Allow-Origin", "http://localhost:8888")
+ s.send_header("Access-Control-Expose-Headers", "Content-Length, Accept-Ranges")
+ s.send_header("Content-type", "application/octet-stream")
+ if support_byte_ranges:
+ s.send_header("Accept-Ranges", "bytes")
+ for i in extra:
+ s.send_header(i[0], i[1])
+ s.end_headers()
+
+ def do_HEAD(s):
+ ChunkedServerHandler.sendheaders(s)
+
+ def do_GET(s):
+ if not support_byte_ranges:
+ ChunkedServerHandler.sendheaders(s)
+ s.wfile.write(data)
+ else:
+ (start, end) = s.headers.get("range").split("=")[1].split("-")
+ start = int(start)
+ end = int(end)
+ end = min(len(data)-1, end)
+ length = end-start+1
+ ChunkedServerHandler.sendheaders(s,[],length)
+ s.wfile.write(data[start:end+1])
+ s.wfile.close()
+ httpd = BaseHTTPServer.HTTPServer(('localhost', 11111), ChunkedServerHandler)
+ for i in range(expectedConns+1):
+ httpd.handle_request()
+
+ server = multiprocessing.Process(target=chunked_server, args=(True,))
+ server.start()
+ self.run_browser(main, 'Chunked binary synchronous XHR in Web Workers!', '/report_result?' + str(checksum))
+ server.terminate()
+
def test_glgears(self):
self.reftest(path_from_root('tests', 'gears.png'))
Popen(['python', EMCC, path_from_root('tests', 'hello_world_gles.c'), '-o', 'something.html',
generated by cgit v1.2.3-18-g5258 (git 2.32.0) at 2025-07-23 19:58:48 +0000