Wasm-bindgen: Porting Bindings to Other Language?

[tooolbox]

Summary

I'm wondering if it would be feasible to take the JS bindings for a compiled wasm module and port them to another language if it has a library for running a wasm VM.

Additional Details

The swc project used wasm-bindgen to create an npm module. The bindings are relatively straightforward.

let imports = {};
imports['__wbindgen_placeholder__'] = module.exports;
let wasm;
const { TextDecoder } = require(String.raw`util`);

let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });

cachedTextDecoder.decode();

let cachegetUint8Memory0 = null;
function getUint8Memory0() {
    if (cachegetUint8Memory0 === null || cachegetUint8Memory0.buffer !== wasm.memory.buffer) {
        cachegetUint8Memory0 = new Uint8Array(wasm.memory.buffer);
    }
    return cachegetUint8Memory0;
}

function getStringFromWasm0(ptr, len) {
    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
}

const heap = new Array(32).fill(undefined);

heap.push(undefined, null, true, false);

let heap_next = heap.length;

function addHeapObject(obj) {
    if (heap_next === heap.length) heap.push(heap.length + 1);
    const idx = heap_next;
    heap_next = heap[idx];

    heap[idx] = obj;
    return idx;
}

function getObject(idx) { return heap[idx]; }

let WASM_VECTOR_LEN = 0;

let cachegetNodeBufferMemory0 = null;
function getNodeBufferMemory0() {
    if (cachegetNodeBufferMemory0 === null || cachegetNodeBufferMemory0.buffer !== wasm.memory.buffer) {
        cachegetNodeBufferMemory0 = Buffer.from(wasm.memory.buffer);
    }
    return cachegetNodeBufferMemory0;
}

function passStringToWasm0(arg, malloc) {

    const len = Buffer.byteLength(arg);
    const ptr = malloc(len);
    getNodeBufferMemory0().write(arg, ptr, len);
    WASM_VECTOR_LEN = len;
    return ptr;
}

let cachegetInt32Memory0 = null;
function getInt32Memory0() {
    if (cachegetInt32Memory0 === null || cachegetInt32Memory0.buffer !== wasm.memory.buffer) {
        cachegetInt32Memory0 = new Int32Array(wasm.memory.buffer);
    }
    return cachegetInt32Memory0;
}

function dropObject(idx) {
    if (idx < 36) return;
    heap[idx] = heap_next;
    heap_next = idx;
}

function takeObject(idx) {
    const ret = getObject(idx);
    dropObject(idx);
    return ret;
}
/**
* @param {string} s
* @param {any} opts
* @returns {any}
*/
module.exports.parseSync = function(s, opts) {
    var ptr0 = passStringToWasm0(s, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
    var len0 = WASM_VECTOR_LEN;
    var ret = wasm.parseSync(ptr0, len0, addHeapObject(opts));
    return takeObject(ret);
};

/**
* @param {any} s
* @param {any} opts
* @returns {any}
*/
module.exports.printSync = function(s, opts) {
    var ret = wasm.printSync(addHeapObject(s), addHeapObject(opts));
    return takeObject(ret);
};

/**
* @param {string} s
* @param {any} opts
* @returns {any}
*/
module.exports.transformSync = function(s, opts) {
    var ptr0 = passStringToWasm0(s, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
    var len0 = WASM_VECTOR_LEN;
    var ret = wasm.transformSync(ptr0, len0, addHeapObject(opts));
    return takeObject(ret);
};

module.exports.__wbindgen_json_parse = function(arg0, arg1) {
    var ret = JSON.parse(getStringFromWasm0(arg0, arg1));
    return addHeapObject(ret);
};

module.exports.__wbindgen_json_serialize = function(arg0, arg1) {
    const obj = getObject(arg1);
    var ret = JSON.stringify(obj === undefined ? null : obj);
    var ptr0 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
    var len0 = WASM_VECTOR_LEN;
    getInt32Memory0()[arg0 / 4 + 1] = len0;
    getInt32Memory0()[arg0 / 4 + 0] = ptr0;
};

module.exports.__wbindgen_string_new = function(arg0, arg1) {
    var ret = getStringFromWasm0(arg0, arg1);
    return addHeapObject(ret);
};

module.exports.__wbindgen_object_drop_ref = function(arg0) {
    takeObject(arg0);
};

module.exports.__wbg_new_59cb74e423758ede = function() {
    var ret = new Error();
    return addHeapObject(ret);
};

module.exports.__wbg_stack_558ba5917b466edd = function(arg0, arg1) {
    var ret = getObject(arg1).stack;
    var ptr0 = passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
    var len0 = WASM_VECTOR_LEN;
    getInt32Memory0()[arg0 / 4 + 1] = len0;
    getInt32Memory0()[arg0 / 4 + 0] = ptr0;
};

module.exports.__wbg_error_4bb6c2a97407129a = function(arg0, arg1) {
    try {
        console.error(getStringFromWasm0(arg0, arg1));
    } finally {
        wasm.__wbindgen_free(arg0, arg1);
    }
};

module.exports.__wbindgen_rethrow = function(arg0) {
    throw takeObject(arg0);
};

const path = require('path').join(__dirname, 'wasm_bg.wasm');
const bytes = require('fs').readFileSync(path);

const wasmModule = new WebAssembly.Module(bytes);
const wasmInstance = new WebAssembly.Instance(wasmModule, imports);
wasm = wasmInstance.exports;
module.exports.__wasm = wasm;

I did see #1628 and #1634 and I understand that creating bindings for other languages is out of scope for this project.

My question is, on a basic level, is there anything preventing me from taking those specific JS bindings and porting them to a different language, to work with a wasm VM library?

It seems like the necessary steps are:

1. Implement the heap construct in native code.
2. Implement the helper methods natively and inject them into the VM so the wasm module can access them.
3. Create native wrappers that call into the VM to utilize the wasm module's exported functionality.

I gave this a shot but I can't seem to execute __wbindgen_malloc within the VM (it gets called but blows up the VM) so I'm wondering if there's some secret sauce that isn't obvious from the bindings, some JS-specific ingredient or detail about the wasm's compilation such that it won't work outside the browser.

I understand this may be a little out of the norm but any pointers are appreciated :)

Additional notes:

• I tried this with both Life (which uses Wagon) and Gasm and was looking at WAKit next.
• Yes, I did provide to the VM all methods that the wasm module wanted, with the correct signatures.
• Once I understood what the bindings were doing and how they were getting data to and from the wasm module, I was quite impressed--very nice job! :)

[alexcrichton]

Thanks for the report! I unfortunately think, though, that wasm-bindgen literally-as-is is not a great choice for other languages. The idea of wasm-bindgen I think is a great one to take running for other languages, but the implementation as-is today is very JS-specific. Lots of runtime support in the wasm-bindgen crate as well as various options in the macros are all JS-specific and don't always have other clear meanings in other ecosystems

In essence I think wasm-bindgen is too ambitious about JS interop to be that successful in other languages. I think the general framework of what wasm-bindgen does is a great idea to try to mirror in other languages (and many Rust bindings to other languages already do this), however.

[tooolbox]

Hey @alexcrichton I appreciate you taking the time to swing by and drop a note!

It makes sense to me that wasm-bindgen won't output to other languages and how it's built very JS-specific. I'm sure it would be a lot of work to refactor things to target a new language, and there's no incentive for anyone here to do that, and my Rust knowledge is far too small to attempt such a thing.

The thing that I really admire is the communication layer that this project creates. Where normally you can only pass ints between the VM and the host, wasm-bindgen provides a way to communicate complex types between the two layers. It was pretty mind-blowing when I grokked how that was happening.

Rather than trying to to change wasm-bindgen to target another language, I wanted to find out the feasibility of porting the generated bindings to another host environment besides Node or the browser. Wasm-bindgen has injected certain vital functionality into the wasm itself, so in basic theory, all that's needed is to take that single JS bindings file and port it to another language/host environment.

I'm definitely aware that this would be one-off handwritten port, which gives it more of a science experiment status than anything else, but my curiosity has gotten the better of me. There's a number of languages that have wasm runtimes, and by rewriting the JS bindings file to, say, Python, to work with py-wasm, you could then run that particular Rust program inside Python!

What I'm trying to find out is if this is entirely unfeasible or roadblocked in some way. If I carefully rewrote a particular set of bindings to work with a different host environment, would it work? You mention "runtime support" and "JS-specific macros" and I'm wondering if there's any magic going on besides the straight-up JS that's in the generated bindings file. I can't think of anything myself, but I wanted to ask someone familiar with the wasm-bindgen project.

Thanks!

[alexcrichton]

The language-neutral version of wasm-bindgen is "interface types", which is a proposal for WebAssembly. We're working in wasmtime to get this up and running and provide a way to compile down to interface types to get a wasm-bindgen-like-feeling in host runtimes too!

[tooolbox]

Hm, that is neat info, and something to look forward to, but you're not answering my actual question. Maybe I haven't made myself clear, or maybe you're not reading my whole comment before responding, or maybe you don't know the answer. The question was about feasibility of porting generated bindings to another host environment, not wasm-bindgen's capability for targeting other languages or host environments.

I appreciate that you took time out of your day to respond, and I acknowledge that the question is unusual, but it would have been nice to get an answer.

[alexcrichton]

Sorry I don't mean to sound like I'm dodging the question. The question you're asking is pretty massive and could probably have at least an hour talk about simply it. I'm trying to be succinct and point you in the right direction.

My first point still stands, it will not work today in the limit. There are a lot of intrinsics that assume JS is the host language. Disregarding all that then you get to mostly the features defined by interface types. Wasm-bindgen currently does a bit more (like custom structs) but that's not going to be true in the long run.

The short answer is that it will be very difficult to port the JS file to Python, for example, and have it work there. That's not what wasm-bindgen was designed for.