Tensorboardx: ONNX export failed on ATen operator sort because torch.onnx.symbolic.sort does not exist

[alistairewj]

Ran into this issue when trying to save a graph from an LSTM. Here's a simple example which reproduces it on my setup:

# generate dummy data

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
# write out the model
from tensorboardX import SummaryWriter

NUM_FEATURES = 2
SEQ_LEN = 8
BATCH_SIZE = 4

# prepare dataset
torch.manual_seed(5)
x = torch.ones([BATCH_SIZE, SEQ_LEN, NUM_FEATURES])
x = x.cumsum(dim=2)
L = torch.ones(4, dtype=torch.long)
for i in range(x.shape[0]):
    idx = 2*(i+1)
    if idx>=x.shape[1]:
        L[i]=x.shape[1]
    else:
        x[i, idx:, :] = 0
        L[i] = idx

y = torch.ones([4, 1], dtype=torch.long)
y[0:2] = 0

class LSTM(nn.Module):
    def __init__(self):
        super(LSTM, self).__init__()
        self.hidden_dim = 10
        self.input_size = NUM_FEATURES
        self.target_size = 2
        self.hidden = None
        self.bidirectional = False

        if self.bidirectional:
            self.num_directions = 2
        else:
            self.num_directions = 1

        self.lstm = nn.LSTM(self.input_size, self.hidden_dim, 1,
        bidirectional=self.bidirectional)
        self.lstm.retain_variables = False

        self.hidden2target = nn.Linear(self.num_directions*self.hidden_dim, self.target_size)

    def init_hidden(self, batch_size):
        return (torch.zeros([self.num_directions, batch_size, self.hidden_dim], requires_grad=True, dtype=torch.float32), \
                torch.zeros([self.num_directions, batch_size, self.hidden_dim], requires_grad=True, dtype=torch.float32))

    def forward(self, seqs, T):
        # seqs is seq_len x batch_size x num_features
        # axes for input sequence:
        #   - The first axis is the sequence itself
        #   - the second indexes instances in the mini-batch
        #   - the third indexes elements of the input

        # reshape to match expected LSTM input
        # original: [batch, seq_len, input_feat]
        # after: [seq_len, batch, input_feat]
        #seqs = seqs.permute(1, 0, 2)

        # initialize hidden state
        self.hidden = self.init_hidden(seqs.size(0))

        # sort the batch by sequence length
        T, idx = T.sort(0, descending=True)
        seqs = seqs.index_select(0, idx)

        # pack the sequences
        seqs_packed = pack_padded_sequence(seqs, T.data, batch_first=True)

        lstm_out_packed, self.hidden = self.lstm(seqs_packed, self.hidden)

        # unpack the output
        lstm_out, _ = pad_packed_sequence(lstm_out_packed)

        # apply forward model to the final hidden state of seq - the [-1]
        targets = self.hidden2target(lstm_out[-1])

        return targets

model = LSTM()
model(x, L)

writer = SummaryWriter(comment='testing')
# below throws error
writer.add_graph(model, (x, L), verbose=False)

Error:

/home/alistairewj/.virtualenvs/torch-0.4.0-py3/lib/python3.5/site-packages/torch/onnx/utils.py:365: UserWarning: ONNX export failed on ATen operator sort because torch.onnx.symbolic.sort does not exist
  .format(op_name, op_name))

---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
<ipython-input-117-1d66713a60c5> in <module>()
     88 writer = SummaryWriter(comment='testing-packed')
     89 # below throws error
---> 90 writer.add_graph(model, (x, L), verbose=False)

~/.virtualenvs/torch-0.4.0-py3/lib/python3.5/site-packages/tensorboardX/writer.py in add_graph(self, model, input_to_model, verbose)
    433                 print('add_graph() only supports PyTorch v0.2.')
    434                 return
--> 435         self.file_writer.add_graph(graph(model, input_to_model, verbose))
    436 
    437     @staticmethod

~/.virtualenvs/torch-0.4.0-py3/lib/python3.5/site-packages/tensorboardX/graph.py in graph(model, args, verbose)
     98     if verbose:
     99         print(graph)
--> 100     list_of_nodes = parse(graph)
    101     nodes = []
    102     node_stats = []

~/.virtualenvs/torch-0.4.0-py3/lib/python3.5/site-packages/tensorboardX/graph.py in parse(graph)
     20 
     21         uname = next(iter(n.outputs())).uniqueName()
---> 22         assert n.scopeName() != '', '{} has empty scope name'.format(n)
     23         scope[uname] = n.scopeName()
     24     if LooseVersion(torch.__version__) >= LooseVersion("0.4"):

AssertionError: %30 : Dynamic = onnx::Shape(%12)
 has empty scope name

Seems related to #145 - is this also not implemented in onnx? I'm using pytorch==0.4.0 and the latest version of tensorboardX (installed from GitHub).

[lanpa]

if you use writer.add_graph(model, (x, L), verbose=True), you can see that many nodes missing their ID and scope name. Not sure if they are the underlining operation of sort. I will check this tomorrow.
ps. Your code sample is helpful. May I add it to the unit test?

  %30 : Dynamic = onnx::Shape(%12)
  %31 : Dynamic = onnx::Constant[value={1}]()
  %32 : Dynamic = onnx::Gather(%30, %31)
  %33 : Dynamic = onnx::Unsqueeze[axes=[0]](%32)
  %34 : Dynamic = onnx::Constant[value={10}]()
  %35 : Dynamic = onnx::Constant[value={1}]()
  %36 : Dynamic = onnx::Unsqueeze[axes=[0]](%35)
  %37 : Dynamic = onnx::Concat[axis=0](%36, %33, %34)
  %38 : Dynamic = onnx::ConstantFill[input_as_shape=1](%37)
  %39 : Dynamic = onnx::Shape(%12)
  %40 : Dynamic = onnx::Constant[value={1}]()
  %41 : Dynamic = onnx::Gather(%39, %40)
  %42 : Dynamic = onnx::Unsqueeze[axes=[0]](%41)
  %43 : Dynamic = onnx::Constant[value={10}]()
  %44 : Dynamic = onnx::Constant[value={1}]()
  %45 : Dynamic = onnx::Unsqueeze[axes=[0]](%44)
  %46 : Dynamic = onnx::Concat[axis=0](%45, %42, %43)
  %47 : Dynamic = onnx::ConstantFill[input_as_shape=1](%46)

[alistairewj]

Please feel free to use it!

[danakianfar]

Any update on this issue? @lanpa

[villmow]

I got the same problem with an LSTM and Pytorch 0.4. Any updates yet?

  %54 : Dynamic = onnx::Shape(%17)
  %55 : Dynamic = onnx::Constant[value={1}]()
  %56 : Dynamic = onnx::Gather(%54, %55)
  %57 : Dynamic = onnx::Unsqueeze[axes=[0]](%56)
  %58 : Dynamic = onnx::Constant[value={100}]()
  %59 : Dynamic = onnx::Constant[value={2}]()
  %60 : Dynamic = onnx::Unsqueeze[axes=[0]](%59)
  %61 : Dynamic = onnx::Concat[axis=0](%60, %57, %58)
  %62 : Dynamic = onnx::ConstantFill[input_as_shape=1](%61)
  %63 : Dynamic = onnx::Shape(%17)
  %64 : Dynamic = onnx::Constant[value={1}]()
  %65 : Dynamic = onnx::Gather(%63, %64)
  %66 : Dynamic = onnx::Unsqueeze[axes=[0]](%65)
  %67 : Dynamic = onnx::Constant[value={100}]()
  %68 : Dynamic = onnx::Constant[value={2}]()
  %69 : Dynamic = onnx::Unsqueeze[axes=[0]](%68)
  %70 : Dynamic = onnx::Concat[axis=0](%69, %66, %67)
  %71 : Dynamic = onnx::ConstantFill[input_as_shape=1](%70)

Traceback (most recent call last):
  File "main.py", line 427, in <module>
    main()
  File "main.py", line 408, in main
    writer.add_graph(model, (dummy_abstract.t(), dummy_candidate.t()), verbose=True)
  File "venv_dke/lib/python3.6/site-packages/tensorboardX/writer.py", line 514, in add_graph
    self.file_writer.add_graph(graph(model, input_to_model, verbose))
  File "venv_dke/lib/python3.6/site-packages/tensorboardX/pytorch_graph.py", line 102, in graph
    list_of_nodes = parse(graph)
  File "venv_dke/lib/python3.6/site-packages/tensorboardX/pytorch_graph.py", line 24, in parse
    assert n.scopeName() != '', '{} has empty scope name'.format(n)
AssertionError: %54 : Dynamic = onnx::Shape(%17)
 has empty scope name

[lanpa]

@danakianfar @villmow Sorry for late reply. Since pytorch changes fast and I don't have much time, I would like to wait for the stable release and then look into it.

[miguelvr]

having the same issue with LSTM (no sorting operation)... any news on this? pytorch 0.4.1 seems like a stable version, I think it's probably worth it to sort this out

[Sunshine352]

comment the line _---> 22 assert n.scopeName() != '', '{} has empty scope name'.format(n)_, problem can be solved.

[lanpa]

seems working in pytorch 1.0.0 + https://github.com/lanpa/tensorboardX/tree/pytorch-1.0-alpha

Need furthur check...