Dvc: RecursionError from Repo('.').graph()[0].reverse()

[PeterFogh]

Please provide information about your setup
DVC version(i.e. dvc --version), Platform and method of installation (pip, homebrew, pkg Mac, exe (Windows), DEB(Linux), RPM(Linux))

dvc --version
> 0.51.1

Installed via pip, on Ubuntu 16.04.6 LTS.

The code:

from dvc.repo import Repo
Repo('.').graph()[0].reverse()

fails with the following error:

---------------------------------------------------------------------------
RecursionError                            Traceback (most recent call last)
<ipython-input-7-90086e6e0e64> in <module>
----> 1 r.graph()[0].reverse()

~/miniconda3/envs/py36_v11/lib/python3.6/site-packages/networkx/classes/digraph.py in reverse(self, copy)
   1195             H = self.__class__()
   1196             H.graph.update(deepcopy(self.graph))
-> 1197             H.add_nodes_from((n, deepcopy(d)) for n, d in self.node.items())
   1198             H.add_edges_from((v, u, deepcopy(d)) for u, v, d
   1199                              in self.edges(data=True))

~/miniconda3/envs/py36_v11/lib/python3.6/site-packages/networkx/classes/digraph.py in add_nodes_from(self, nodes_for_adding, **attr)
    470 
    471         """
--> 472         for n in nodes_for_adding:
    473             # keep all this inside try/except because
    474             # CPython throws TypeError on n not in self._succ,

~/miniconda3/envs/py36_v11/lib/python3.6/site-packages/networkx/classes/digraph.py in <genexpr>(.0)
   1195             H = self.__class__()
   1196             H.graph.update(deepcopy(self.graph))
-> 1197             H.add_nodes_from((n, deepcopy(d)) for n, d in self.node.items())
   1198             H.add_edges_from((v, u, deepcopy(d)) for u, v, d
   1199                              in self.edges(data=True))

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
    148     copier = _deepcopy_dispatch.get(cls)
    149     if copier:
--> 150         y = copier(x, memo)
    151     else:
    152         try:

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
    238     memo[id(x)] = y
    239     for key, value in x.items():
--> 240         y[deepcopy(key, memo)] = deepcopy(value, memo)
    241     return y
    242 d[dict] = _deepcopy_dict

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
    178                     y = x
    179                 else:
--> 180                     y = _reconstruct(x, memo, *rv)
    181 
    182     # If is its own copy, don't memoize.

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    278     if state is not None:
    279         if deep:
--> 280             state = deepcopy(state, memo)
    281         if hasattr(y, '__setstate__'):
    282             y.__setstate__(state)

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
    148     copier = _deepcopy_dispatch.get(cls)
    149     if copier:
--> 150         y = copier(x, memo)
    151     else:
    152         try:

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
    238     memo[id(x)] = y
    239     for key, value in x.items():
--> 240         y[deepcopy(key, memo)] = deepcopy(value, memo)
    241     return y
    242 d[dict] = _deepcopy_dict

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
    178                     y = x
    179                 else:
--> 180                     y = _reconstruct(x, memo, *rv)
    181 
    182     # If is its own copy, don't memoize.

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    278     if state is not None:
    279         if deep:
--> 280             state = deepcopy(state, memo)
    281         if hasattr(y, '__setstate__'):
    282             y.__setstate__(state)

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
    148     copier = _deepcopy_dispatch.get(cls)
    149     if copier:
--> 150         y = copier(x, memo)
    151     else:
    152         try:

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
    238     memo[id(x)] = y
    239     for key, value in x.items():
--> 240         y[deepcopy(key, memo)] = deepcopy(value, memo)
    241     return y
    242 d[dict] = _deepcopy_dict

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
    178                     y = x
    179                 else:
--> 180                     y = _reconstruct(x, memo, *rv)
    181 
    182     # If is its own copy, don't memoize.

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    278     if state is not None:
    279         if deep:
--> 280             state = deepcopy(state, memo)
    281         if hasattr(y, '__setstate__'):
    282             y.__setstate__(state)

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
    148     copier = _deepcopy_dispatch.get(cls)
    149     if copier:
--> 150         y = copier(x, memo)
    151     else:
    152         try:

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
    238     memo[id(x)] = y
    239     for key, value in x.items():
--> 240         y[deepcopy(key, memo)] = deepcopy(value, memo)
    241     return y
    242 d[dict] = _deepcopy_dict

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
    178                     y = x
    179                 else:
--> 180                     y = _reconstruct(x, memo, *rv)
    181 
    182     # If is its own copy, don't memoize.

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    278     if state is not None:
    279         if deep:
--> 280             state = deepcopy(state, memo)
    281         if hasattr(y, '__setstate__'):
    282             y.__setstate__(state)

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
    148     copier = _deepcopy_dispatch.get(cls)
    149     if copier:
--> 150         y = copier(x, memo)
    151     else:
    152         try:

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in _deepcopy_dict(x, memo, deepcopy)
    238     memo[id(x)] = y
    239     for key, value in x.items():
--> 240         y[deepcopy(key, memo)] = deepcopy(value, memo)
    241     return y
    242 d[dict] = _deepcopy_dict

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in deepcopy(x, memo, _nil)
    178                     y = x
    179                 else:
--> 180                     y = _reconstruct(x, memo, *rv)
    181 
    182     # If is its own copy, don't memoize.

~/miniconda3/envs/py36_v11/lib/python3.6/copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    279         if deep:
    280             state = deepcopy(state, memo)
--> 281         if hasattr(y, '__setstate__'):
    282             y.__setstate__(state)
    283         else:

~/miniconda3/envs/py36_v11/lib/python3.6/site-packages/dvc/path_info.py in __getattr__(self, name)
    185 
    186     def __getattr__(self, name):
--> 187         return getattr(self.parsed, name)
    188 
    189     @cached_property

... last 1 frames repeated, from the frame below ...

~/miniconda3/envs/py36_v11/lib/python3.6/site-packages/dvc/path_info.py in __getattr__(self, name)
    185 
    186     def __getattr__(self, name):
--> 187         return getattr(self.parsed, name)
    188 
    189     @cached_property

RecursionError: maximum recursion depth exceeded

[efiop]

Hi @PeterFogh !

Thanks for reporting it! That part of the python API is not officially released yet, so there definitely will be some bugs 🙁 Are you using it in one of your scripts? Could you describe the scenario? E.g. how big is the graph and what networkx version are you using?

[PeterFogh]

You are welcome. Yaah. I know, I'm pushing DVC to the experimental level 😜.
Your scenario is to save the DVC pipeline graph as a PNG image file - which we store in MLflow as an artefact.

The pipeline looks like this by dvc pipeline show --ascii:

+-------------------+                        +---------------------+           +------------------------------+           +---------------------------+
| sql_extration.dvc |                        | dawiki_download.dvc |           | landbrugsinfo_extraction.dvc | **********| lkontoplan_extraction.dvc |
+-------------------+                        +---------------------+***        +------------------------------+*      ****+---------------------------+
          *                                                            ******************** *                 ********               *
          *                                       *********************      ******         *         ********                      *
          *                  *********************                                 ***     *      ****                             *
  +---------------+**********                                                 +-----------------------+                           *
  | splitting.dvc |*******                                                    | pretrain_word2vec.dvc |                           *
  +---------------+       ***************                                     +-----------------------+                           *
          *                              ****************                                 *                                       *
          *                                              ***************                  *                                       *
          *                                                             ********          *                                       *
          *                                                                    +--------------------+                             *
          *                                                             *******| train_word2vec.dvc |                             *
          *                                              ***************       +--------------------+                             *
          *                                **************                                 *                                       *
          *                 ***************                                               *                                       *
          *         ********                                                              *                                       *
+-------------------+                               +--------------+                    ***                                       *
| featurization.dvc |***                        ****| training.dvc |**              ****                                          *
+-------------------+   *********       ********    +--------------+  ******   *****                                              *
          *                     *********                    *            ********                                                *
          *             ********         *********            *       ****        ******                                          *
          *        *****                          *****       *    ***                  ******                                    *
  +----------------+                              +-----------------------+                   ****                            *****
  | prediction.dvc |******                        | model_to_azure_sa.dvc |                  ***                     *********
  +----------------+      ***********             +-----------------------+                **               *********
                                     ***********                                        ***        *********
                                                ***********                          ***  *********
                                                           ***********             *******
                                                                      ****+---------+
                                                                          | Dvcfile |
                                                                          +---------+

networkx version is:

conda list -n py36_v11 | grep networkx
> networkx                  2.3                        py_0

[shcheklein]

@PeterFogh if you don't need to do anything with the Repo object after that, or if you can initialize it again, try to run reverse(copy=False).

I wonder if it's networkx bug? Or something it's something related to us due to deepcopy calls.

[shcheklein]

k, was able to reproduce it this way:

from dvc.path_info import URLInfo
u = URLInfo("ssh://[email protected]:/test1/test2/test3")
copy.deepcopy(u)

so, it something related to our UrlInfo implementation.