Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Import Deep Dive

Import Deep Dive

Petr Viktorin

July 24, 2015
Tweet

More Decks by Petr Viktorin

Other Decks in Programming

Transcript

  1. Prior art David Beazley — Modules and Packages: Live and

    Let Die! http://pyvideo.org/video/3387
  2. import random random = __import__('random') from ..spam import foo foo

    = __import__('spam', globals(), locals(), ['foo'], 2).foo https://docs.python.org/3/library/functions.html#__import__
  3. The Import Machinery Photo © Les Chatfield, https://www.flickr.com/photos/elsie/8229790 sans: •

    Locking • Caching • Error handling • Pythons older than 3.4 • Backwards compatibility shims
  4. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> import random >>> old = random >>> import random >>> random is old True
  5. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> import random >>> old = random >>> import random >>> random is old True
  6. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> import random >>> old = random >>> import random >>> random is old True
  7. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> import random >>> old = random >>> del sys.modules['random'] >>> import random >>> random is old False >>> (random.sample is ... old.sample) False >>> sys.modules['impostor'] = 'Not a module, is it?' >>> import impostor >>> impostor[:12] 'Not a module'
  8. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> import random >>> old = random >>> del sys.modules['random'] >>> import random >>> random is old False >>> (random.sample is ... old.sample) False >>> sys.modules['impostor'] = 'Not a module, is it?' >>> import impostor >>> impostor[:12] 'Not a module'
  9. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib64/python3.4', ...] >>> importlib.util.find_spec('antigravity') ModuleSpec(name='antigravity', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/antigravity.py') >>> import io >>> io.__spec__ ModuleSpec(name='io', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/io.py')
  10. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib64/python3.4', ...] >>> importlib.util.find_spec('antigravity') ModuleSpec(name='antigravity', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/antigravity.py') >>> import io >>> io.__spec__ ModuleSpec(name='io', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/io.py')
  11. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib64/python3.4', ...] >>> importlib.util.find_spec('antigravity') ModuleSpec(name='antigravity', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/antigravity.py') >>> import io >>> io.__spec__ ModuleSpec(name='io', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/io.py')
  12. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib64/python3.4', ...] >>> importlib.util.find_spec('antigravity') ModuleSpec(name='antigravity', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/antigravity.py') >>> import io >>> io.__spec__ ModuleSpec(name='io', loader=<_frozen_importlib.SourceFileLoader ...>, origin='/usr/lib64/python3.4/io.py')
  13. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module • Put module in sys.modules • Initialize the module
  14. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
  15. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
  16. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
  17. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
  18. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module foo.py: import bar bar.do_bar() def do_foo(): print('Fooing!') bar.py: import foo foo.do_foo() def do_bar(): print('Barring!') • Put module in sys.modules • Initialize the module
  19. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module Filename Module name random.py random urllib/ __init__.py urllib parse.py urllib.parse request.py urllib.request response.py urllib.response Top-level module Package Parent of the below Submodule
  20. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module Filename Module name random.py random urllib/ __init__.py urllib parse.py urllib.parse request.py urllib.request response.py urllib.response Top-level module Package Parent of the below Submodule
  21. import_module(name): sys.modules[name]? return it spec = find_spec(name, path) load(spec) module

    = sys.modules[name] return module import urllib.parse >>> urllib.__path__ ['/usr/lib64/python3.4/urllib'] Filename Module name random.py random urllib/ __init__.py urllib parse.py urllib.parse request.py urllib.request response.py urllib.response Top-level module Package Parent of the below Submodule
  22. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module import urllib.parse >>> urllib.__path__ ['/usr/lib64/python3.4/urllib'] Filename Module name random.py random urllib/ __init__.py urllib parse.py urllib.parse request.py urllib.request response.py urllib.response Top-level module Package Parent of the below Submodule
  23. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module • Put module in sys.modules • Initialize the module
  24. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
  25. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
  26. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
  27. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
  28. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
  29. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
  30. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module foo/__init__.py: from foo import main from foo import consts foo/main.py: import foo use(foo.consts.value) foo/consts.py: value = 42 • Put module in sys.modules • Initialize the module
  31. __init__ should: import from submodules set __all__ nothing else submodules

    should: not import directly from __init__ not have internal import cycles
  32. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module ?
  33. Where's the module? >>> import random >>> random <module 'random'

    from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
  34. Where's the module? >>> import random >>> random <module 'random'

    from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
  35. Where's the module? >>> import random >>> random <module 'random'

    from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
  36. Where's the module? >>> import random >>> random <module 'random'

    from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
  37. Where's the module? >>> import random >>> random <module 'random'

    from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
  38. Where's the module? >>> import random >>> random <module 'random'

    from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
  39. Where's the module? >>> import random >>> random <module 'random'

    from '/usr/lib64/python3.4/random.py'> >>> random.__file__ '/usr/lib64/python3.4/random.py' >>> import sys >>> sys <module 'sys' (built-in)> >>> sys.__file__ Traceback (most recent call last): File "<stdin>", line 1, in <module> AttributeError: 'module' object has no attribute '__file__' /usr/bin/python3
  40. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path)

    return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>]
  41. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path)

    return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>] >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib/python3.4', ...]
  42. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path)

    return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>] >>> sys.path_hooks [<zipimporter>, <path_hook_for_FileFinder>] >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib/python3.4', ...]
  43. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path)

    return spec if successful PathFinder.find_spec(name, path): for directory in path: get sys.path_hooks entry call its find_spec(name) return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>] >>> sys.path_hooks [<zipimporter>, <path_hook_for_FileFinder>] >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib/python3.4', ...]
  44. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path)

    return spec if successful PathFinder.find_spec(name, path): for directory in path: get sys.path_hooks entry call its find_spec(name) return spec if successful >>> sys.meta_path [<BuiltinImporter>, <FrozenImporter>, <PathFinder>] >>> sys.path_hooks [<zipimporter>, <path_hook_for_FileFinder>] >>> sys.path ['', '/usr/lib64/python34.zip', '/usr/lib/python3.4', ...] random.cpython-34m.so (Extension module) random.abi3.so (Extension module) random.so (Extension module) random.py (Source module) random.pyc (Sourceless module)
  45. find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path)

    return spec if successful PathFinder.find_spec(name, path): for directory in path: get sys.path_hooks entry call its find_spec(name) return spec if successful ModuleSpec: name random origin /usr/lib64/python3.4/random.py cached /usr/lib64/python3.4/__pycache__/random.cpython-34.pyc loader importlib.machinery.SourceFileLoader loader_state, parent, submodule_search_locations
  46. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module ?
  47. load(spec): module = spec.loader.create_module(spec) if module is None: module =

    types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec)
  48. load(spec): module = spec.loader.create_module(spec) if module is None: module =

    types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec)
  49. load(spec): module = spec.loader.create_module(spec) if module is None: module =

    types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) spec → __spec__ spec.name → __name__ spec.loader → __loader__ spec.parent → __package__ spec.origin → __file__ spec.cached → __cached__ spec.submodule_search_locations → __path__
  50. load(spec): module = spec.loader.create_module(spec) if module is None: module =

    types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) spec → __spec__ spec.name → __name__ spec.loader → __loader__ spec.parent → __package__ spec.origin → __file__ spec.cached → __cached__ spec.submodule_search_locations → __path__ >>> import __main__ >>> a = 'hello' >>> __main__.a 'hello' >>> __main__.a = 'world' >>> a 'world'
  51. load(spec): module = spec.loader.create_module(spec) if module is None: module =

    types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) spec → __spec__ spec.name → __name__ spec.loader → __loader__ spec.parent → __package__ spec.origin → __file__ spec.cached → __cached__ spec.submodule_search_locations → __path__ >>> import __main__ >>> a = 'hello' >>> __main__.a 'hello' >>> __main__.a = 'world' >>> a 'world' if __name__ == '__main__'
  52. load(spec): module = spec.loader.create_module(spec) if module is None: module =

    types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) spec → __spec__ spec.name → __name__ spec.loader → __loader__ spec.parent → __package__ spec.origin → __file__ spec.cached → __cached__ spec.submodule_search_locations → __path__ >>> import __main__ >>> a = 'hello' >>> __main__.a 'hello' >>> __main__.a = 'world' >>> a 'world' if __name__ == '__main__'
  53. load(spec): module = spec.loader.create_module(spec) if module is None: module =

    types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) • Put module in sys.modules • Initialize the module
  54. get_code(module, spec): if spec.cached exists, and matches origin stats, return

    it! load source from origin and compile it write bytecode to spec.cached ModuleSpec: name random origin /usr/lib64/python3.4/random.py cached /usr/lib64/python3.4/__pycache__/random.cpython-34.pyc ...
  55. get_code(module, spec): if spec.cached exists, and matches origin stats, return

    it! load source from origin and compile it write bytecode to spec.cached Python 2 somemodule.py somemodule.pyc
  56. get_code(module, spec): if spec.cached exists, and matches origin stats, return

    it! load source from origin and compile it write bytecode to spec.cached Python 2 somemodule.py somemodule.pyc
  57. get_code(module, spec): if spec.cached exists, and matches origin stats, return

    it! load source from origin and compile it write bytecode to spec.cached Python 2 somemodule.py somemodule.pyc Python 3 somemodule.py __pycache__/ somemodule.cpython-34.pyc
  58. get_code(module, spec): if spec.cached exists, and matches origin stats, return

    it! load source from origin and compile it write bytecode to spec.cached Python 2 somemodule.py somemodule.pyc Python 3 somemodule.py __pycache__/ somemodule.cpython-34.pyc Or: somemodule.py somemodule.pyc
  59. import_module(name): sys.modules[name]? return it submodules: load parent module sys.modules[name]? return

    it spec = find_spec(name, path) load(spec) module = sys.modules[name] submodules: set module as attribute of parent return module find_spec(name, path): for finder in sys.meta_path: call its find_spec(name, path) return spec if successful PathFinder.find_spec(name, path): for directory in path: get sys.path_hooks entry call its find_spec(name) return spec if successful load(spec): module = spec.loader.create_module(spec) if module is None: module = types.ModuleType(spec.name) set initial module attributes sys.modules[spec.name] = module spec.loader.exec_module(module, spec) get_code(module, spec): if spec.cached exists, and matches origin stats, return it! load source from origin and compile it write bytecode to spec.cached