Protobuf.js: Problem encoding map<string, struct>

[nikunjy]

This is the proto file

syntax = "proto3";
package benchmark;

message KeyStringArrMessage {
  string key = 1;
  repeated string values = 2;
}

message MyMapStringArrayMessage {
  map<string, KeyStringArrMessage> value = 1;
};

I made the struct like this

    var mapLength = 10;
    var arrayLength = 20;
    var myMap = new Object();
    for (var len = 0; len < mapLength; len++) {
      // generating again so that not be fixed
      var arrVal = generateStringArray(arrayLength, 20);
      var subMessage = KeyStringArrayMessage.create({
        key: randomString(20), values: arrVal
      });
      // fixed key
      myMap[randomString(20)] = subMessage;
    }
    // Create a new message
    var message = MyMapStringArrayMessage.create({
      value: myMap
    });
    // Encode a message
    fs.writeFileSync(my_map_string_array.binary, MyMapStringArrayMessage.encode(message).finish());

I am loading this file in golang but I don't get the same struct. Which is odd because this approach works fine for other data types. Worked for regular datatypes, string array, and well defined structs.

This leads me to believe that there is something wrong with map<> in the encoding ?

[nikunjy]

Example of what I encode

{"value":{"ge7aVmfK0V8Z0qwvds5V":{"key":"HisE6NuR5EyvHdNPIcuq","values":["98Hdr1NejWgwrUQ2v5Kc","RR7XWF40B8AGGXa42e0T","h12twraif6G6J6D95fd0","6JIBs9gh00ajinEWuIu1","ImDWnPhC3S1SwkoT5P15","gxctc4zShBOXVoJo2ugh","V8vq1MNszPm7GX7uwNzT","XjWyTYL10B9tGDnPtU3Z","nFswanqruvyhEaBvl7PP","eWRR072ktdkZkzJGzO99"]},"SWtko93xZCvMI2Sh1WfI":{"key":"qjz4eS6k95kNBifFX8mM","values":["N1AE8xo9jJceNVTArVU6","Sy4WoM4aUVjBrwA0dh8L","9GJQj7cymYUGigTFe5ql","cbJnyyh4128nOZVzU713","yB1NgqtZI4SXK2OLXWcB","U9TvdcnRm1V73qZDbVfj","jHM2WseAXIZ4I0gDTOOz","lSNb2mefhHObjZOA3yF0","0wDXZvtGrX2TweJAuw4k","SAqoo7pTTEtk5oi8ZB7A"]},"temfoo33B2kK4B9LrKgs":{"key":"T4YWHdlUmLdRYN256HsA","values":["zI4eUTbes5u9m2kvcHHm","TaTEKitxkJCKpCy3ORki","bH8EFxZU1xlwdmdTnMtu","oN6sokgbX0y701FPfQZ6","PytS6K0Ile54ysIv1qAq","AJC4wjjwofGgV0AKegcZ","lg2DUiFHZ6DtX5TlxXkU","ADUGIT7ZWQoPbLDvDwlp","29pV9aRXBx16vyo2C1OH","UZ3RW6RSKzATVo4nYvRi"]},"abwIhUZJs7BbLC9fR0Ja":{"key":"qaOA4mJTM2xkuTTJg7BG","values":["UORFAkE0ExxmJ5wCS4o1","v2eGzA3ejPyqeopJL1CC","fEe2Ll3qB4325LmreiNo","7lhCobjbY3x88uCjpJ3i","KJmGItAGizFG8joxOEGl","e2OEXhsrh2HRiu2397OF","85J3gHXxiycKWDCZgtkZ","Urap40R1Ptm8WSa2JCkV","cU3ScdYsRNX6IPmDCHTa","QQv6swLC5nAUUQdcNnVA"]},"sDumOdXyipQrAEJbjvql":{"key":"Qj4izjXpLIQD0hTWzpYi","values":["zrFO2WjFegzykjDRciHc","maYm31RlTHGFrBjQ30GH","yWfV7N6XD8Rt4QO6gp2f","lllrrz2FoaqtePdXk2Jp","wyHrZ0uhaAnkUbYxYTZd","IB1ATeQfiyAUZBjtBDdc","7tBhzvEKovUrl8ErAZdC","eDgKmr2YBlqXXCOtYPb7","Eab2QMjdtOK3WpZwrUqr","WAHuZgQ34UL1nI68QNGa"]},"Z1og3lPUjZtsHUnFjPrP":{"key":"eClK6SFnLeVt9QQd3RI5","values":["UbcUq8VOyNHERxDiSETf","eOZQzvRj2aq1iLpk0jmN","jy39ePajWGEFKliMV1bM","lQ1mmk9DZ5ALtggKFlXd","0FzjO4GssyOWmxVRXVZO","eW0K5tt99k8kB2HNlFhP","v2HaHtRNQjzLFQ1GXE6C","n9Tl9CSGQRzAupjO8hQt","jmTTZEwLEf6IHpwvSLzl","63BCF9Y0EhYwWx8y1vLW"]},"jgWuJaeL527Yp7IriLGx":{"key":"Z9cuCgWAYuVLsoeZXLuc","values":["JDGpMIdgXAvPfedpwf6M","pfKhelARD5rHKqLVUJCS","1GoK8cRcy80KG6BihJyy","H8lFXUdN9WAun1qTCApn","nicuaUIwyLgAEGhr8A9S","o2T8ZAKYTfMWdFo4fU45","ruKJBCNbrQYmEkW3n1sK","KVtkvtRraMiA6C5RXHvb","F2VLcA9scMLLCy3sEnmi","6RoduXeSxdFibar3AsMR"]},"cNJqBq3yPcZugy5XFiRZ":{"key":"r6RLGDorJgMKTszB02jq","values":["TtE1C2tORrjv0NgUGl5N","gH4DLaycm7wlxMqlXZfm","DxgXG9DEj1PVSRhQJuRf","Eg4g9tQokHAswYoIGHXY","VwvT2Xbpt6WIm0ZUgRHk","kq4U5iw4EcP5cFaleeLD","IeJcvqFOSHcCDzTAE3ly","PsLMDIhoLH402HcsC8hD","RqRrFqlrRwJ4N11iDRPM","TXRtGaEklbQeYratPT0J"]},"QlsTs9ea5ttZRapwYijT":{"key":"QE6wbaCA1N96YKCRnKkp","values":["hI6kSv9Ycm7DTZ8CKMf8","IUssaitSPrTJI7WfPBEc","XFbSfT2qqWHo8rKLpPTD","GsZ5lkJIsbhXm46LjTZV","qWTJlHrr8nEQqbMw560y","YqZAUSmyPLKc9DslAK2n","wvqjz4S0fbpX5D4YcObf","JMOUKq367RXlOuSqcybR","IfPjBwysggtQUB1du0pE","yTOMVWxmTPAScF8vr3jq"]},"veVWRO9M1VbLd7SkqEMs":{"key":"27TzVyFHrvLAVPmXNgJb","values":["DZTTuYdpJmeeW4tymmnh","BYjKD0pIkops0sYVjf7b","pJMs3hdZXar4aLViRGHa","WPyYJz0wK2OowHG6wY8k","18wfzol2v1aSpbWADm8D","Tm8iKtHf9jV8kxHXYrLA","lGtpXsSbMTeK8bghArU2","aYXC4BQVYfyBH0DwXYXl","ODlMlhJf6jo5pRJOZ9pK","ov9FkNfOX7xjiJUSHz8j"]}}}

Example of what I get in golang decode

{"value":{"veVWRO9M1VbLd7SkqEMs":{"key":"27TzVyFHrvLAVPmXNgJb","values":["98Hdr1NejWgwrUQ2v5Kc","RR7XWF40B8AGGXa42e0T","h12twraif6G6J6D95fd0","6JIBs9gh00ajinEWuIu1","ImDWnPhC3S1SwkoT5P15","gxctc4zShBOXVoJo2ugh","V8vq1MNszPm7GX7uwNzT","XjWyTYL10B9tGDnPtU3Z","nFswanqruvyhEaBvl7PP","eWRR072ktdkZkzJGzO99","N1AE8xo9jJceNVTArVU6","Sy4WoM4aUVjBrwA0dh8L","9GJQj7cymYUGigTFe5ql","cbJnyyh4128nOZVzU713","yB1NgqtZI4SXK2OLXWcB","U9TvdcnRm1V73qZDbVfj","jHM2WseAXIZ4I0gDTOOz","lSNb2mefhHObjZOA3yF0","0wDXZvtGrX2TweJAuw4k","SAqoo7pTTEtk5oi8ZB7A","zI4eUTbes5u9m2kvcHHm","TaTEKitxkJCKpCy3ORki","bH8EFxZU1xlwdmdTnMtu","oN6sokgbX0y701FPfQZ6","PytS6K0Ile54ysIv1qAq","AJC4wjjwofGgV0AKegcZ","lg2DUiFHZ6DtX5TlxXkU","ADUGIT7ZWQoPbLDvDwlp","29pV9aRXBx16vyo2C1OH","UZ3RW6RSKzATVo4nYvRi","UORFAkE0ExxmJ5wCS4o1","v2eGzA3ejPyqeopJL1CC","fEe2Ll3qB4325LmreiNo","7lhCobjbY3x88uCjpJ3i","KJmGItAGizFG8joxOEGl","e2OEXhsrh2HRiu2397OF","85J3gHXxiycKWDCZgtkZ","Urap40R1Ptm8WSa2JCkV","cU3ScdYsRNX6IPmDCHTa","QQv6swLC5nAUUQdcNnVA","zrFO2WjFegzykjDRciHc","maYm31RlTHGFrBjQ30GH","yWfV7N6XD8Rt4QO6gp2f","lllrrz2FoaqtePdXk2Jp","wyHrZ0uhaAnkUbYxYTZd","IB1ATeQfiyAUZBjtBDdc","7tBhzvEKovUrl8ErAZdC","eDgKmr2YBlqXXCOtYPb7","Eab2QMjdtOK3WpZwrUqr","WAHuZgQ34UL1nI68QNGa","UbcUq8VOyNHERxDiSETf","eOZQzvRj2aq1iLpk0jmN","jy39ePajWGEFKliMV1bM","lQ1mmk9DZ5ALtggKFlXd","0FzjO4GssyOWmxVRXVZO","eW0K5tt99k8kB2HNlFhP","v2HaHtRNQjzLFQ1GXE6C","n9Tl9CSGQRzAupjO8hQt","jmTTZEwLEf6IHpwvSLzl","63BCF9Y0EhYwWx8y1vLW","JDGpMIdgXAvPfedpwf6M","pfKhelARD5rHKqLVUJCS","1GoK8cRcy80KG6BihJyy","H8lFXUdN9WAun1qTCApn","nicuaUIwyLgAEGhr8A9S","o2T8ZAKYTfMWdFo4fU45","ruKJBCNbrQYmEkW3n1sK","KVtkvtRraMiA6C5RXHvb","F2VLcA9scMLLCy3sEnmi","6RoduXeSxdFibar3AsMR","TtE1C2tORrjv0NgUGl5N","gH4DLaycm7wlxMqlXZfm","DxgXG9DEj1PVSRhQJuRf","Eg4g9tQokHAswYoIGHXY","VwvT2Xbpt6WIm0ZUgRHk","kq4U5iw4EcP5cFaleeLD","IeJcvqFOSHcCDzTAE3ly","PsLMDIhoLH402HcsC8hD","RqRrFqlrRwJ4N11iDRPM","TXRtGaEklbQeYratPT0J","hI6kSv9Ycm7DTZ8CKMf8","IUssaitSPrTJI7WfPBEc","XFbSfT2qqWHo8rKLpPTD","GsZ5lkJIsbhXm46LjTZV","qWTJlHrr8nEQqbMw560y","YqZAUSmyPLKc9DslAK2n","wvqjz4S0fbpX5D4YcObf","JMOUKq367RXlOuSqcybR","IfPjBwysggtQUB1du0pE","yTOMVWxmTPAScF8vr3jq","DZTTuYdpJmeeW4tymmnh","BYjKD0pIkops0sYVjf7b","pJMs3hdZXar4aLViRGHa","WPyYJz0wK2OowHG6wY8k","18wfzol2v1aSpbWADm8D","Tm8iKtHf9jV8kxHXYrLA","lGtpXsSbMTeK8bghArU2","aYXC4BQVYfyBH0DwXYXl","ODlMlhJf6jo5pRJOZ9pK","ov9FkNfOX7xjiJUSHz8j"]}}}

[nikunjy]

Here is my golang decode

func TestOne(t *testing.T) {
    fileName := getFullPath("small_map_string_array.binary")
    data, err := ioutil.ReadFile(fileName)
    assert.NoError(t, err)
    protoMsg := &MyMapStringArrayMessage{}
    err = proto.Unmarshal(data, protoMsg)
    assert.NoError(t, err)
    data, err = json.Marshal(*protoMsg)
    fmt.Println(string(data))
}

[dcodeIO]

I added a test mostly identical to your use case which I'll commit later on but it didn't show any issues. Once the test is available, please take a look if I am doing something different.

[nikunjy]

@dcodeIO did you add a test for golang ?
I am able to decode the message encoded by the library back into javascript. I guess another way to state my problem is that bytes spitted out by protobuf serialization is different for my data in node (your library) and golang (google protobuf library)

[dcodeIO]

Hmm, well, it would help if you could make a test case that produces a minimal buffer that's different, so that I can compare the two buffers by hand.

One thing I could think of is that the go library does not preserve the order of the map keys, while the JS implementation does. Just a shot in the dark, of course.

[dcodeIO]

Test case

[nikunjy]

Let me comment back with a better test, code and output

[nikunjy]

Golang test

func TestMyOut(t *testing.T) {
    protoMsg := &MyMapStringArrayMessage{
        Value: map[string]*KeyStringArrMessage{
            "alpha": &KeyStringArrMessage{
                Key:    "alpha 1.1",
                Values: []string{"a", "b"},
            },
            "beta": &KeyStringArrMessage{
                Key:    "beta 1.1",
                Values: []string{"c", "d"},
            },
        },
    }
    data, err := json.Marshal(*protoMsg)
    assert.NoError(t, err)
    fmt.Println(string(data))
    data, _ = proto.Marshal(protoMsg)
    ioutil.WriteFile(getFullPath("small_map_string_array_golang.binary"), data, 0777)
}

Golang Output

xxd -b small_map_string_array_golang.binary

0000000: 00001010 00011000 00001010 00000100 01100010 01100101  ....be
0000006: 01110100 01100001 00010010 00010000 00001010 00001000  ta....
000000c: 01100010 01100101 01110100 01100001 00100000 00110001  beta 1
0000012: 00101110 00110001 00010010 00000001 01100011 00010010  .1..c.
0000018: 00000001 01100100 00001010 00011010 00001010 00000101  .d....
000001e: 01100001 01101100 01110000 01101000 01100001 00010010  alpha.
0000024: 00010001 00001010 00001001 01100001 01101100 01110000  ...alp
000002a: 01101000 01100001 00100000 00110001 00101110 00110001  ha 1.1
0000030: 00010010 00000001 01100001 00010010 00000001 01100010  ..a..b

Node test

xxd -b small_map_string_array_node.binary

protobuf.load("map_string_array.proto", function(err, root) {
  var MyMapStringArrayMessage = root.lookup("benchmark.MyMapStringArrayMessage");
  var KeyStringArrayMessage = root.lookup("benchmark.KeyStringArrMessage");
  var message = MyMapStringArrayMessage.create({
    value: {
      "alpha": KeyStringArrayMessage.create({
        key: "alpha 1.1", values: ["a", "b"]
      }),
      "beta": KeyStringArrayMessage.create({
        key: "beta 1.1", values: ["c", "d"]
      })
    }
  });
  var fileName = format("bench_gen/{}_map_string_array_node.binary", "small");
  console.log("Writing file ", fileName);
  fs.writeFileSync(fileName, MyMapStringArrayMessage.encode(message).finish());
  var jsonFileName = format("bench_gen/{}_map_string_array_node.json", "small");
  console.log("Writing file ", jsonFileName);
  fs.writeFileSync(jsonFileName, JSON.stringify(message));
});

Node Output

0000000: 00001010 00110010 00001010 00000101 01100001 01101100  .2..al
0000006: 01110000 01101000 01100001 00010010 00010001 00001010  pha...
000000c: 00001001 01100001 01101100 01110000 01101000 01100001  .alpha
0000012: 00100000 00110001 00101110 00110001 00010010 00000001   1.1..
0000018: 01100001 00010010 00000001 01100010 00001010 00000100  a..b..
000001e: 01100010 01100101 01110100 01100001 00010010 00010000  beta..
0000024: 00001010 00001000 01100010 01100101 01110100 01100001  ..beta
000002a: 00100000 00110001 00101110 00110001 00010010 00000001   1.1..
0000030: 01100011 00010010 00000001 01100100                    c..d

If you were to load the node binary output in golang you would get
{"value":{"beta":{"key":"beta 1.1","values":["a","b","c","d"]}}}

[dcodeIO]

Seems that my shot in the dark hit something: Either the Go library encodes maps in a different order, or the interop layer causes this. The reason might be that the underlying map data type used in Go does not guarantee to retain order of keys - a lot of languages actually don't guarantee this with default data types for performance reasons - but JS's internals do this.

You can test this theory by using different values, and finding a pair that serializes in declaration order.

Edit: see

[nikunjy]

If it was just keys being in different order i'd be okay. But the whole object is different as you can see.

If you were to parse node binary output with the golang library, you don't even get the key "alpha"

Notice the extra "2" in the node binary output ?

Edit: notice this If you were to load the node binary output in golang you would get {"value":{"beta":{"key":"beta 1.1","values":["a","b","c","d"]}}}

[dcodeIO]

0A  id 1, wireType 2 (ldelim, field Outer.value)
32* length 50 (total map length) - WRONG
0A  id 1, wireType 2 (ldelim, field Inner.key)
04  length 4
62  b
65  e
74  t
61  a
...

[dcodeIO]

Please let me know if this commit fixes your issue!

[nikunjy]

Thanks a lot @dcodeIO
This was an awesome turnaround time. It fixed my issue as well.

Going to tag my dependency with this commit hash. When do you make the release ?

[dcodeIO]

It's up as 6.1.1.