Skip to content

Commit ebc785e

Browse files
committed
[wasm2js] Remove all handling for external memory file
We have a lot of support code and complexity in emscripten for handling of external memory files. However, its only ever used in wasm2js mode which is a legacy mode. The only reason we continue to support it here IIUC is because its slightly more space efficient than embedding the data as base64. For small programs like hello_world this is an over codesize win. For larger programs there is a regression in overall size in proportion to the amount of static data in the program.
1 parent 75892b3 commit ebc785e

14 files changed

+63
-288
lines changed

ChangeLog.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ See docs/process.md for more on how version tagging works.
2020

2121
3.1.54 (in development)
2222
-----------------------
23+
- Emscripten no longer supported extracting static data and serving it as a
24+
separate `.mem` data file. The feature was already only available under
25+
wasm2js (`-sWASM=0`) so this change will only effect users of this settings.
2326

2427
3.1.53 - 01/29/24
2528
-----------------

emcc.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,6 @@ def __init__(self):
144144
self.emrun = False
145145
self.cpu_profiler = False
146146
self.memory_profiler = False
147-
self.memory_init_file = None
148147
self.use_preload_cache = False
149148
self.use_preload_plugins = False
150149
self.valid_abspaths = []
@@ -1309,7 +1308,7 @@ def consume_arg_file():
13091308
ports.show_ports()
13101309
should_exit = True
13111310
elif check_arg('--memory-init-file'):
1312-
options.memory_init_file = int(consume_arg())
1311+
exit_with_error('--memory-init-file is no longer supported')
13131312
elif check_flag('--proxy-to-worker'):
13141313
settings_changes.append('PROXY_TO_WORKER=1')
13151314
elif check_arg('--valid-abspath'):

src/postamble_minimal.js

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -221,13 +221,6 @@ WebAssembly.instantiate(Module['wasm'], imports).then((output) => {
221221
updateMemoryViews();
222222
#endif
223223

224-
#if !MEM_INIT_IN_WASM && !SINGLE_FILE
225-
#if ASSERTIONS
226-
if (!Module['mem']) throw 'Must load memory initializer as an ArrayBuffer in to variable Module.mem before adding compiled output .js script to the DOM';
227-
#endif
228-
HEAPU8.set(new Uint8Array(Module['mem']), {{{ GLOBAL_BASE }}});
229-
#endif
230-
231224
initRuntime(wasmExports);
232225
#if PTHREADS
233226
// Export Wasm module for pthread creation to access.

src/preamble.js

Lines changed: 0 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -990,9 +990,6 @@ function createWasm() {
990990
#endif
991991
updateMemoryViews();
992992
#endif
993-
#if !MEM_INIT_IN_WASM
994-
runMemoryInitializer();
995-
#endif
996993

997994
#if '$wasmTable' in addedLibraryItems && !RELOCATABLE
998995
wasmTable = wasmExports['__indirect_function_table'];
@@ -1147,88 +1144,6 @@ function getCompilerSetting(name) {
11471144
}
11481145
#endif // RETAIN_COMPILER_SETTINGS
11491146

1150-
#if !MEM_INIT_IN_WASM
1151-
var memoryInitializer = <<< MEM_INITIALIZER >>>;
1152-
1153-
function runMemoryInitializer() {
1154-
#if PTHREADS
1155-
if (ENVIRONMENT_IS_PTHREAD) return;
1156-
#endif
1157-
if (!isDataURI(memoryInitializer)) {
1158-
memoryInitializer = locateFile(memoryInitializer);
1159-
}
1160-
if (ENVIRONMENT_IS_NODE || ENVIRONMENT_IS_SHELL) {
1161-
var data = readBinary(memoryInitializer);
1162-
HEAPU8.set(data, {{{ GLOBAL_BASE }}});
1163-
} else {
1164-
addRunDependency('memory initializer');
1165-
var applyMemoryInitializer = (data) => {
1166-
if (data.byteLength) data = new Uint8Array(data);
1167-
#if ASSERTIONS
1168-
for (var i = 0; i < data.length; i++) {
1169-
assert(HEAPU8[{{{ GLOBAL_BASE }}} + i] === 0, "area for memory initializer should not have been touched before it's loaded");
1170-
}
1171-
#endif
1172-
HEAPU8.set(data, {{{ GLOBAL_BASE }}});
1173-
// Delete the typed array that contains the large blob of the memory initializer request response so that
1174-
// we won't keep unnecessary memory lying around. However, keep the XHR object itself alive so that e.g.
1175-
// its .status field can still be accessed later.
1176-
if (Module['memoryInitializerRequest']) delete Module['memoryInitializerRequest'].response;
1177-
removeRunDependency('memory initializer');
1178-
};
1179-
var doBrowserLoad = () => {
1180-
readAsync(memoryInitializer, applyMemoryInitializer, () => {
1181-
var e = new Error('could not load memory initializer ' + memoryInitializer);
1182-
#if MODULARIZE
1183-
readyPromiseReject(e);
1184-
#else
1185-
throw e;
1186-
#endif
1187-
});
1188-
};
1189-
#if SUPPORT_BASE64_EMBEDDING
1190-
var memoryInitializerBytes = tryParseAsDataURI(memoryInitializer);
1191-
if (memoryInitializerBytes) {
1192-
applyMemoryInitializer(memoryInitializerBytes.buffer);
1193-
} else
1194-
#endif
1195-
if (Module['memoryInitializerRequest']) {
1196-
// a network request has already been created, just use that
1197-
var useRequest = () => {
1198-
var request = Module['memoryInitializerRequest'];
1199-
var response = request.response;
1200-
if (request.status !== 200 && request.status !== 0) {
1201-
#if SUPPORT_BASE64_EMBEDDING
1202-
var data = tryParseAsDataURI(Module['memoryInitializerRequestURL']);
1203-
if (data) {
1204-
response = data.buffer;
1205-
} else {
1206-
#endif
1207-
// If you see this warning, the issue may be that you are using locateFile and defining it in JS. That
1208-
// means that the HTML file doesn't know about it, and when it tries to create the mem init request early, does it to the wrong place.
1209-
// Look in your browser's devtools network console to see what's going on.
1210-
console.warn('a problem seems to have happened with Module.memoryInitializerRequest, status: ' + request.status + ', retrying ' + memoryInitializer);
1211-
doBrowserLoad();
1212-
return;
1213-
#if SUPPORT_BASE64_EMBEDDING
1214-
}
1215-
#endif
1216-
}
1217-
applyMemoryInitializer(response);
1218-
};
1219-
if (Module['memoryInitializerRequest'].response) {
1220-
setTimeout(useRequest, 0); // it's already here; but, apply it asynchronously
1221-
} else {
1222-
Module['memoryInitializerRequest'].addEventListener('load', useRequest); // wait for it
1223-
}
1224-
} else {
1225-
// fetch it from the network ourselves
1226-
doBrowserLoad();
1227-
}
1228-
}
1229-
}
1230-
#endif // MEM_INIT_IN_WASM == 0
1231-
12321147
#if MAIN_MODULE && ASYNCIFY
12331148
// With MAIN_MODULE + ASYNCIFY the normal method of placing stub functions in
12341149
// wasmImports for as-yet-undefined symbols doesn't work since ASYNCIFY then

src/settings_internal.js

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -143,11 +143,6 @@ var AUDIO_WORKLET_FILE = '';
143143
// Base URL the source mapfile, if relevant
144144
var SOURCE_MAP_BASE = '';
145145

146-
// When this is false we use an external memory init file
147-
// See --memory-init-file. When not using wasm2js this flag is ignored, and
148-
// this setting will always be true.
149-
var MEM_INIT_IN_WASM = true;
150-
151146
// If set to 1, src/base64Utils.js will be included in the bundle.
152147
// This is set internally when needed (SINGLE_FILE)
153148
var SUPPORT_BASE64_EMBEDDING = false;

src/shell_minimal.js

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ if (ENVIRONMENT_IS_NODE && ENVIRONMENT_IS_SHELL) {
8282
#endif
8383

8484
#if !SINGLE_FILE
85-
#if ENVIRONMENT_MAY_BE_NODE && ((WASM == 1 && (!WASM2JS || !MEM_INIT_IN_WASM)) || WASM == 2)
85+
#if ENVIRONMENT_MAY_BE_NODE && ((WASM == 1 && !WASM2JS) || WASM == 2)
8686
// Wasm or Wasm2JS loading:
8787

8888
if (ENVIRONMENT_IS_NODE) {
@@ -95,13 +95,10 @@ if (ENVIRONMENT_IS_NODE) {
9595
Module['wasm'] = fs.readFileSync(__dirname + '/{{{ TARGET_BASENAME }}}.wasm');
9696
#endif
9797
#endif
98-
#if !MEM_INIT_IN_WASM
99-
Module['mem'] = fs.readFileSync(__dirname + '/{{{ TARGET_BASENAME }}}.mem');
100-
#endif
10198
}
10299
#endif
103100

104-
#if ENVIRONMENT_MAY_BE_SHELL && ((WASM == 1 && (!WASM2JS || !MEM_INIT_IN_WASM)) || WASM == 2)
101+
#if ENVIRONMENT_MAY_BE_SHELL && ((WASM == 1 && !WASM2JS) || WASM == 2)
105102
if (ENVIRONMENT_IS_SHELL) {
106103
#if WASM == 2
107104
if (typeof WebAssembly != 'undefined') Module['wasm'] = read('{{{ TARGET_BASENAME }}}.wasm', 'binary');
@@ -111,9 +108,6 @@ if (ENVIRONMENT_IS_SHELL) {
111108
Module['wasm'] = read('{{{ TARGET_BASENAME }}}.wasm', 'binary');
112109
#endif
113110
#endif
114-
#if !MEM_INIT_IN_WASM
115-
Module['mem'] = read('{{{ TARGET_BASENAME }}}.mem', 'binary');
116-
#endif
117111
}
118112
#endif
119113

test/code_size/hello_world_wasm2js.js

Lines changed: 35 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,30 @@
1-
var c = Module, g, h, k = new TextDecoder("utf8"), l;
1+
var d = Module, g, h, k = new TextDecoder("utf8"), l;
22

3-
function d(b) {
4-
this.exports = function(f) {
5-
function m(e) {
6-
e.set = function(a, n) {
7-
this[a] = n;
3+
function e(b) {
4+
this.exports = function(r) {
5+
function u(c) {
6+
c.set = function(a, f) {
7+
this[a] = f;
88
};
9-
e.get = function(a) {
9+
c.get = function(a) {
1010
return this[a];
1111
};
12-
return e;
12+
return c;
1313
}
14-
return function(e) {
15-
var a = new ArrayBuffer(16777216), n = e.a.a;
16-
e = m([]);
14+
function x(c, a, f) {
15+
for (var v, p = 0, t = a, w = f.length, y = a + (3 * w >> 2) - ("=" == f[w - 2]) - ("=" == f[w - 1]); p < w; p += 4) a = m[f.charCodeAt(p + 1)],
16+
v = m[f.charCodeAt(p + 2)], c[t++] = m[f.charCodeAt(p)] << 2 | a >> 4, t < y && (c[t++] = a << 4 | v >> 2),
17+
t < y && (c[t++] = v << 6 | m[f.charCodeAt(p + 3)]);
18+
}
19+
for (var q, m = new Uint8Array(123), n = 25; 0 <= n; --n) m[48 + n] = 52 + n, m[65 + n] = n,
20+
m[97 + n] = 26 + n;
21+
m[43] = 62;
22+
m[47] = 63;
23+
return function(c) {
24+
var a = new ArrayBuffer(16777216), f = new Uint8Array(a), v = c.a.a;
25+
q = f;
26+
x(q, 1024, "aGVsbG8h");
27+
c = u([]);
1728
return {
1829
b: Object.create(Object.prototype, {
1930
grow: {},
@@ -24,41 +35,40 @@ function d(b) {
2435
}
2536
}),
2637
c: function() {},
27-
d: function(p, q) {
28-
n(1024);
38+
d: function(p, t) {
39+
v(1024);
2940
return 0;
3041
},
31-
e: e
42+
e: c
3243
};
33-
}(f);
44+
}(r);
3445
}(b);
3546
}
3647

37-
(function(b, f) {
48+
(function(b, r) {
3849
return {
39-
then: function(m) {
40-
m({
41-
instance: new d(f)
50+
then: function(u) {
51+
u({
52+
instance: new e(r)
4253
});
4354
}
4455
};
45-
})(c.wasm, {
56+
})(d.wasm, {
4657
a: {
4758
a: b => {
48-
var f = console, m = f.log;
59+
var r = console, u = r.log;
4960
if (b) {
50-
for (var e = b + void 0, a = b; !(a >= e) && g[a]; ) ++a;
51-
b = k.decode(g.subarray(b, a));
61+
for (var x = b + void 0, q = b; !(q >= x) && g[q]; ) ++q;
62+
b = k.decode(g.subarray(b, q));
5263
} else b = "";
53-
m.call(f, b);
64+
u.call(r, b);
5465
}
5566
}
5667
}).then((b => {
5768
b = b.instance.exports;
5869
l = b.d;
5970
h = b.b;
6071
g = new Uint8Array(h.buffer);
61-
g.set(new Uint8Array(c.mem), 1024);
6272
b.c();
6373
l();
6474
}));
Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,8 @@
11
{
2-
"a.html": 671,
3-
"a.html.gz": 430,
4-
"a.js": 708,
5-
"a.js.gz": 444,
6-
"a.mem": 6,
7-
"a.mem.gz": 32,
8-
"total": 1385,
9-
"total_gz": 906
2+
"a.html": 323,
3+
"a.html.gz": 253,
4+
"a.js": 1060,
5+
"a.js.gz": 636,
6+
"total": 1383,
7+
"total_gz": 889
108
}

test/common.py

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -812,16 +812,6 @@ def setup_node_pthreads(self):
812812
self.js_engines = [nodejs]
813813
self.node_args += shared.node_pthread_flags(nodejs)
814814

815-
def uses_memory_init_file(self):
816-
if self.get_setting('SIDE_MODULE') or (self.is_wasm() and not self.get_setting('WASM2JS')):
817-
return False
818-
elif '--memory-init-file' in self.emcc_args:
819-
return int(self.emcc_args[self.emcc_args.index('--memory-init-file') + 1])
820-
else:
821-
# side modules handle memory differently; binaryen puts the memory in the wasm module
822-
opt_supports = any(opt in self.emcc_args for opt in ('-O2', '-O3', '-Os', '-Oz'))
823-
return opt_supports
824-
825815
def set_temp_dir(self, temp_dir):
826816
self.temp_dir = temp_dir
827817
self.canonical_temp_dir = get_canonical_temp_dir(self.temp_dir)
@@ -1069,11 +1059,6 @@ def build(self, filename, libraries=None, includes=None, force_c=False, js_outfi
10691059
self.run_process(cmd, stderr=self.stderr_redirect if not DEBUG else None)
10701060
self.assertExists(output)
10711061

1072-
if js_outfile and self.uses_memory_init_file():
1073-
src = read_file(output)
1074-
# side memory init file, or an empty one in the js
1075-
assert ('/* memory initializer */' not in src) or ('/* memory initializer */ allocate([]' in src)
1076-
10771062
return output
10781063

10791064
def get_func(self, src, name):

test/test_browser.py

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -2412,10 +2412,6 @@ def test_runtimelink(self):
24122412
self.run_process([EMCC, 'supp.c', '-o', 'supp.wasm', '-sSIDE_MODULE', '-O2'] + self.get_emcc_args())
24132413
self.btest_exit('main.c', args=['-sMAIN_MODULE=2', '-O2', 'supp.wasm'])
24142414

2415-
@parameterized({
2416-
'': ([],),
2417-
'memfile': (['-sWASM=0', '--memory-init-file=1'],)
2418-
})
24192415
def test_pre_run_deps(self, args):
24202416
if args:
24212417
self.require_wasm2js()
@@ -3000,10 +2996,6 @@ def test_glfw3_hi_dpi_aware(self):
30002996

30012997
@requires_graphics_hardware
30022998
@no_wasm64('SDL2 + wasm64')
3003-
@parameterized({
3004-
'': ([],),
3005-
'memfile': (['-sWASM=0', '--memory-init-file=1'],)
3006-
})
30072999
def test_sdl2_image(self, args):
30083000
# load an image file, get pixel data. Also O2 coverage for --preload-file, and memory-init
30093001
shutil.copyfile(test_file('screenshot.jpg'), 'screenshot.jpg')
@@ -4460,27 +4452,6 @@ def test_vanilla_html_when_proxying(self):
44604452
create_file('test.html', '<script src="test.js"></script>')
44614453
self.run_browser('test.html', '/report_result?0')
44624454

4463-
@requires_wasm2js
4464-
@parameterized({
4465-
'O0': [('-O0',), '0'],
4466-
'O1': [('-O1',), '0'],
4467-
'O2': [('-O2',), '1'],
4468-
})
4469-
def test_in_flight_memfile_request(self, args, expected):
4470-
# test the XHR for an asm.js mem init file being in flight already
4471-
self.emcc_args += args
4472-
self.set_setting('WASM', 0)
4473-
4474-
print('plain html')
4475-
self.compile_btest('in_flight_memfile_request.c', ['-o', 'test.js'])
4476-
create_file('test.html', '<script src="test.js"></script>')
4477-
# never when we provide our own HTML like this.
4478-
self.run_browser('test.html', '/report_result?0')
4479-
4480-
print('default html')
4481-
# should happen when there is a mem init file (-O2+)
4482-
self.btest('in_flight_memfile_request.c', expected=expected)
4483-
44844455
def test_async_compile(self):
44854456
# notice when we use async compilation
44864457
script = '''

0 commit comments

Comments
 (0)