Conan: imp.load_source replacement

[memsharded]

That function is already deprecated/undocumented.

https://github.com/conan-io/conan/pull/3340 tried to replace it with __import__, but failing in OSX Py3.6.
This has to be revisited and probably fixed for 2.0.

[danimtb]

Could be also replace with:

from importlib.machinery import SourceFileLoader

loaded = SourceFileLoader(filename, conanfile_path).load_module()

When we drop support for python 2

[danimtb]

SourceFileLoader().load_module() seems to be deprecated too.
https://docs.python.org/3/library/importlib.html#importlib.abc.Loader.load_module

This could be the solution for python 3:

from importlib.machinery import SourceFileLoader
import types
loader = SourceFileLoader(filename, plugin_path)
loaded = types.ModuleType(loader.name)
loader.exec_module(loaded)

[alalazo]

@memsharded Just stumbled on this issue. As I faced exactly the same problem contributing to a different package manager, was wondering if this snippet might be useful to you:

https://github.com/epfl-scitas/spack/blob/af6a3556c4c861148b8e1adc2637685932f4b08a/lib/spack/llnl/util/lang.py#L595-L622

[sztomi]

I've run into this while trying to parallelize recipe exporting with the following script:

import sys
from pathlib import Path
from concurrent.futures import as_completed, ThreadPoolExecutor

from conans.client.conan_api import ConanAPIV1, _get_conanfile_path, cmd_export


def export_all(path: Path, namespace: str):
  conan = ConanAPIV1()
  conan.create_app()

  user, channel = namespace.split("/")
  sys.path.insert(0, str(Path.cwd().absolute() / "plexconantool"))

  futures = []
  path = Path(path)
  conanfiles = list(sorted(
    _get_conanfile_path(str(subdir.absolute()), cwd=None, py=True)
    for subdir in path.iterdir()
  ))

  with ThreadPoolExecutor(max_workers=12) as pool:
    for cf in conanfiles:
      ftr = pool.submit(
        cmd_export,
        conan.app,
        cf,
        name=None,
        version=None,
        user=user,
        channel=channel,
        keep_source=False,
        graph_lock=None,
        ignore_dirty=False,
      )
      futures.append(ftr)

  for ftr in as_completed(futures):
    ftr.result()


if __name__ == "__main__":
  from argparse import ArgumentParser

  parser = ArgumentParser()
  parser.add_argument("directory", type=Path)
  parser.add_argument("namespace")
  args = parser.parse_args()
  export_all(args.directory, args.namespace)

I got this traceback, which to me appears to be a concurrency bug in imp:

Traceback (most recent call last):                                                                                                                                                        
  File "/home/tamas/src/conan/conans/client/loader.py", line 379, in _parse_conanfile                                                                                                     
    loaded = imp.load_source(module_id, conan_file_path)                                                                                                                                  
  File "/home/tamas/.pyenv/versions/3.6.4/lib/python3.6/imp.py", line 172, in load_source                                                                                                 
    module = _load(spec)                                                                                                                                                                  
  File "<frozen importlib._bootstrap>", line 684, in _load                                                                                                                                
  File "<frozen importlib._bootstrap>", line 670, in _load_unlocked                                                                                                                       
KeyError: '8489a127-1207-11eb-8dd4-b7f7f2789788'                                                                                                                                          

During handling of the above exception, another exception occurred:                                                                                                                       

Traceback (most recent call last):                                                                                                                                                        
  File "/home/tamas/src/conan/conans/client/loader.py", line 57, in load_basic_module
    self._generator_manager)
  File "/home/tamas/src/conan/conans/client/loader.py", line 343, in parse_conanfile
    module, filename = _parse_conanfile(conanfile_path)
  File "/home/tamas/src/conan/conans/client/loader.py", line 412, in _parse_conanfile
    '\n'.join(trace[3:])))
conans.errors.ConanException: Unable to load conanfile in /home/tamas/work/conan/packages/opusfile/conanfile.py
  File "/home/tamas/.pyenv/versions/3.6.4/lib/python3.6/imp.py", line 172, in load_source
    module = _load(spec)
  File "<frozen importlib._bootstrap>", line 684, in _load
  File "<frozen importlib._bootstrap>", line 670, in _load_unlocked
KeyError: '8489a127-1207-11eb-8dd4-b7f7f2789788'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "fastexport.py", line 49, in <module>
    export_all(args.directory, args.namespace)
  File "fastexport.py", line 39, in export_all
    ftr.result()
  File "/home/tamas/.pyenv/versions/3.6.4/lib/python3.6/concurrent/futures/_base.py", line 425, in result
    return self.__get_result()
  File "/home/tamas/.pyenv/versions/3.6.4/lib/python3.6/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/home/tamas/.pyenv/versions/3.6.4/lib/python3.6/concurrent/futures/thread.py", line 56, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/home/tamas/src/conan/conans/client/cmd/export.py", line 68, in cmd_export
    conanfile = loader.load_export(conanfile_path, name, version, user, channel)
  File "/home/tamas/src/conan/conans/client/loader.py", line 155, in load_export
    lock_python_requires)
  File "/home/tamas/src/conan/conans/client/loader.py", line 123, in load_named
    conanfile, _ = self.load_basic_module(conanfile_path, lock_python_requires, user, channel)
  File "/home/tamas/src/conan/conans/client/loader.py", line 89, in load_basic_module
    raise ConanException("Error loading conanfile at '{}': {}".format(conanfile_path, e))
conans.errors.ConanException: Error loading conanfile at '/home/tamas/work/conan/packages/opusfile/conanfile.py': Unable to load conanfile in /home/tamas/work/conan/packages/opusfile/conanfile.py
  File "/home/tamas/.pyenv/versions/3.6.4/lib/python3.6/imp.py", line 172, in load_source
    module = _load(spec)
  File "<frozen importlib._bootstrap>", line 684, in _load
  File "<frozen importlib._bootstrap>", line 670, in _load_unlocked
KeyError: '8489a127-1207-11eb-8dd4-b7f7f2789788'

If I replace the load_source call with the snippet above, I get the following:

Traceback (most recent call last):                                                                                                                                                        
  File "/home/tamas/src/conan/conans/client/hook_manager.py", line 102, in _load_module_from_file                                                                                         
    loaded = __import__(filename)                                                                                                                                                         
ModuleNotFoundError: No module named 'attribute_checker'                                                                                                                                  

During handling of the above exception, another exception occurred:                                                                                                                       

Traceback (most recent call last):                                                                                                                                                        
  File "/home/tamas/src/conan/conans/client/hook_manager.py", line 76, in _load_hook
    hook = HookManager._load_module_from_file(hook_path)
  File "/home/tamas/src/conan/conans/client/hook_manager.py", line 126, in _load_module_from_file
    '\n'.join(trace[3:])))
conans.errors.ConanException: Unable to load Hook in /home/tamas/work/conan/conan_home/.conan/hooks/attribute_checker.py
ModuleNotFoundError: No module named 'attribute_checker'


During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "fastexport.py", line 49, in <module>
    export_all(args.directory, args.namespace) 
  File "fastexport.py", line 39, in export_all 
    ftr.result()
  File "/home/tamas/.pyenv/versions/3.6.4/lib/python3.6/concurrent/futures/_base.py", line 425, in result
    return self.__get_result()
  File "/home/tamas/.pyenv/versions/3.6.4/lib/python3.6/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/home/tamas/.pyenv/versions/3.6.4/lib/python3.6/concurrent/futures/thread.py", line 56, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/home/tamas/src/conan/conans/client/cmd/export.py", line 110, in cmd_export
    reference=package_layout.ref)
  File "/home/tamas/src/conan/conans/client/hook_manager.py", line 54, in execute
    self.load_hooks()
  File "/home/tamas/src/conan/conans/client/hook_manager.py", line 69, in load_hooks
    self._load_hook(name)
  File "/home/tamas/src/conan/conans/client/hook_manager.py", line 86, in _load_hook
    raise ConanException("Error loading hook '%s': %s" % (hook_path, str(e)))
conans.errors.ConanException: Error loading hook '/home/tamas/work/conan/conan_home/.conan/hooks/attribute_checker.py': Unable to load Hook in /home/tamas/work/conan/conan_home/.conan/hooks/attribute_checker.py
ModuleNotFoundError: No module named 'attribute_checker'

If I deactivate the attribute_checker hook, the issue disappears. I'm not sure why - __import__ is odd. I think the same snippet could be used for loading the hooks as above.