Formed in 2009, the Archive Team (not to be confused with the archive.org Archive-It Team) is a rogue archivist collective dedicated to saving copies of rapidly dying or deleted websites for the sake of history and digital heritage. The group is 100% composed of volunteers and interested parties, and has expanded into a large amount of related projects for saving online and digital history.
Permalink
Cannot retrieve contributors at this time
Name already in use
A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
cpython/Lib/pkgutil.py /
Go to fileThis commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| """Utilities to support packages.""" | |
| from collections import namedtuple | |
| from functools import singledispatch as simplegeneric | |
| import importlib | |
| import importlib.util | |
| import importlib.machinery | |
| import os | |
| import os.path | |
| import re | |
| import sys | |
| from types import ModuleType | |
| import warnings | |
| __all__ = [ | |
| 'get_importer', 'iter_importers', 'get_loader', 'find_loader', | |
| 'walk_packages', 'iter_modules', 'get_data', | |
| 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', | |
| 'ModuleInfo', | |
| ] | |
| ModuleInfo = namedtuple('ModuleInfo', 'module_finder name ispkg') | |
| ModuleInfo.__doc__ = 'A namedtuple with minimal info about a module.' | |
| def _get_spec(finder, name): | |
| """Return the finder-specific module spec.""" | |
| # Works with legacy finders. | |
| try: | |
| find_spec = finder.find_spec | |
| except AttributeError: | |
| loader = finder.find_module(name) | |
| if loader is None: | |
| return None | |
| return importlib.util.spec_from_loader(name, loader) | |
| else: | |
| return find_spec(name) | |
| def read_code(stream): | |
| # This helper is needed in order for the PEP 302 emulation to | |
| # correctly handle compiled files | |
| import marshal | |
| magic = stream.read(4) | |
| if magic != importlib.util.MAGIC_NUMBER: | |
| return None | |
| stream.read(12) # Skip rest of the header | |
| return marshal.load(stream) | |
| def walk_packages(path=None, prefix='', onerror=None): | |
| """Yields ModuleInfo for all modules recursively | |
| on path, or, if path is None, all accessible modules. | |
| 'path' should be either None or a list of paths to look for | |
| modules in. | |
| 'prefix' is a string to output on the front of every module name | |
| on output. | |
| Note that this function must import all *packages* (NOT all | |
| modules!) on the given path, in order to access the __path__ | |
| attribute to find submodules. | |
| 'onerror' is a function which gets called with one argument (the | |
| name of the package which was being imported) if any exception | |
| occurs while trying to import a package. If no onerror function is | |
| supplied, ImportErrors are caught and ignored, while all other | |
| exceptions are propagated, terminating the search. | |
| Examples: | |
| # list all modules python can access | |
| walk_packages() | |
| # list all submodules of ctypes | |
| walk_packages(ctypes.__path__, ctypes.__name__+'.') | |
| """ | |
| def seen(p, m={}): | |
| if p in m: | |
| return True | |
| m[p] = True | |
| for info in iter_modules(path, prefix): | |
| yield info | |
| if info.ispkg: | |
| try: | |
| __import__(info.name) | |
| except ImportError: | |
| if onerror is not None: | |
| onerror(info.name) | |
| except Exception: | |
| if onerror is not None: | |
| onerror(info.name) | |
| else: | |
| raise | |
| else: | |
| path = getattr(sys.modules[info.name], '__path__', None) or [] | |
| # don't traverse path items we've seen before | |
| path = [p for p in path if not seen(p)] | |
| yield from walk_packages(path, info.name+'.', onerror) | |
| def iter_modules(path=None, prefix=''): | |
| """Yields ModuleInfo for all submodules on path, | |
| or, if path is None, all top-level modules on sys.path. | |
| 'path' should be either None or a list of paths to look for | |
| modules in. | |
| 'prefix' is a string to output on the front of every module name | |
| on output. | |
| """ | |
| if path is None: | |
| importers = iter_importers() | |
| elif isinstance(path, str): | |
| raise ValueError("path must be None or list of paths to look for " | |
| "modules in") | |
| else: | |
| importers = map(get_importer, path) | |
| yielded = {} | |
| for i in importers: | |
| for name, ispkg in iter_importer_modules(i, prefix): | |
| if name not in yielded: | |
| yielded[name] = 1 | |
| yield ModuleInfo(i, name, ispkg) | |
| @simplegeneric | |
| def iter_importer_modules(importer, prefix=''): | |
| if not hasattr(importer, 'iter_modules'): | |
| return [] | |
| return importer.iter_modules(prefix) | |
| # Implement a file walker for the normal importlib path hook | |
| def _iter_file_finder_modules(importer, prefix=''): | |
| if importer.path is None or not os.path.isdir(importer.path): | |
| return | |
| yielded = {} | |
| import inspect | |
| try: | |
| filenames = os.listdir(importer.path) | |
| except OSError: | |
| # ignore unreadable directories like import does | |
| filenames = [] | |
| filenames.sort() # handle packages before same-named modules | |
| for fn in filenames: | |
| modname = inspect.getmodulename(fn) | |
| if modname=='__init__' or modname in yielded: | |
| continue | |
| path = os.path.join(importer.path, fn) | |
| ispkg = False | |
| if not modname and os.path.isdir(path) and '.' not in fn: | |
| modname = fn | |
| try: | |
| dircontents = os.listdir(path) | |
| except OSError: | |
| # ignore unreadable directories like import does | |
| dircontents = [] | |
| for fn in dircontents: | |
| subname = inspect.getmodulename(fn) | |
| if subname=='__init__': | |
| ispkg = True | |
| break | |
| else: | |
| continue # not a package | |
| if modname and '.' not in modname: | |
| yielded[modname] = 1 | |
| yield prefix + modname, ispkg | |
| iter_importer_modules.register( | |
| importlib.machinery.FileFinder, _iter_file_finder_modules) | |
| def _import_imp(): | |
| global imp | |
| with warnings.catch_warnings(): | |
| warnings.simplefilter('ignore', DeprecationWarning) | |
| imp = importlib.import_module('imp') | |
| class ImpImporter: | |
| """PEP 302 Finder that wraps Python's "classic" import algorithm | |
| ImpImporter(dirname) produces a PEP 302 finder that searches that | |
| directory. ImpImporter(None) produces a PEP 302 finder that searches | |
| the current sys.path, plus any modules that are frozen or built-in. | |
| Note that ImpImporter does not currently support being used by placement | |
| on sys.meta_path. | |
| """ | |
| def __init__(self, path=None): | |
| global imp | |
| warnings.warn("This emulation is deprecated, use 'importlib' instead", | |
| DeprecationWarning) | |
| _import_imp() | |
| self.path = path | |
| def find_module(self, fullname, path=None): | |
| # Note: we ignore 'path' argument since it is only used via meta_path | |
| subname = fullname.split(".")[-1] | |
| if subname != fullname and self.path is None: | |
| return None | |
| if self.path is None: | |
| path = None | |
| else: | |
| path = [os.path.realpath(self.path)] | |
| try: | |
| file, filename, etc = imp.find_module(subname, path) | |
| except ImportError: | |
| return None | |
| return ImpLoader(fullname, file, filename, etc) | |
| def iter_modules(self, prefix=''): | |
| if self.path is None or not os.path.isdir(self.path): | |
| return | |
| yielded = {} | |
| import inspect | |
| try: | |
| filenames = os.listdir(self.path) | |
| except OSError: | |
| # ignore unreadable directories like import does | |
| filenames = [] | |
| filenames.sort() # handle packages before same-named modules | |
| for fn in filenames: | |
| modname = inspect.getmodulename(fn) | |
| if modname=='__init__' or modname in yielded: | |
| continue | |
| path = os.path.join(self.path, fn) | |
| ispkg = False | |
| if not modname and os.path.isdir(path) and '.' not in fn: | |
| modname = fn | |
| try: | |
| dircontents = os.listdir(path) | |
| except OSError: | |
| # ignore unreadable directories like import does | |
| dircontents = [] | |
| for fn in dircontents: | |
| subname = inspect.getmodulename(fn) | |
| if subname=='__init__': | |
| ispkg = True | |
| break | |
| else: | |
| continue # not a package | |
| if modname and '.' not in modname: | |
| yielded[modname] = 1 | |
| yield prefix + modname, ispkg | |
| class ImpLoader: | |
| """PEP 302 Loader that wraps Python's "classic" import algorithm | |
| """ | |
| code = source = None | |
| def __init__(self, fullname, file, filename, etc): | |
| warnings.warn("This emulation is deprecated, use 'importlib' instead", | |
| DeprecationWarning) | |
| _import_imp() | |
| self.file = file | |
| self.filename = filename | |
| self.fullname = fullname | |
| self.etc = etc | |
| def load_module(self, fullname): | |
| self._reopen() | |
| try: | |
| mod = imp.load_module(fullname, self.file, self.filename, self.etc) | |
| finally: | |
| if self.file: | |
| self.file.close() | |
| # Note: we don't set __loader__ because we want the module to look | |
| # normal; i.e. this is just a wrapper for standard import machinery | |
| return mod | |
| def get_data(self, pathname): | |
| with open(pathname, "rb") as file: | |
| return file.read() | |
| def _reopen(self): | |
| if self.file and self.file.closed: | |
| mod_type = self.etc[2] | |
| if mod_type==imp.PY_SOURCE: | |
| self.file = open(self.filename, 'r') | |
| elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION): | |
| self.file = open(self.filename, 'rb') | |
| def _fix_name(self, fullname): | |
| if fullname is None: | |
| fullname = self.fullname | |
| elif fullname != self.fullname: | |
| raise ImportError("Loader for module %s cannot handle " | |
| "module %s" % (self.fullname, fullname)) | |
| return fullname | |
| def is_package(self, fullname): | |
| fullname = self._fix_name(fullname) | |
| return self.etc[2]==imp.PKG_DIRECTORY | |
| def get_code(self, fullname=None): | |
| fullname = self._fix_name(fullname) | |
| if self.code is None: | |
| mod_type = self.etc[2] | |
| if mod_type==imp.PY_SOURCE: | |
| source = self.get_source(fullname) | |
| self.code = compile(source, self.filename, 'exec') | |
| elif mod_type==imp.PY_COMPILED: | |
| self._reopen() | |
| try: | |
| self.code = read_code(self.file) | |
| finally: | |
| self.file.close() | |
| elif mod_type==imp.PKG_DIRECTORY: | |
| self.code = self._get_delegate().get_code() | |
| return self.code | |
| def get_source(self, fullname=None): | |
| fullname = self._fix_name(fullname) | |
| if self.source is None: | |
| mod_type = self.etc[2] | |
| if mod_type==imp.PY_SOURCE: | |
| self._reopen() | |
| try: | |
| self.source = self.file.read() | |
| finally: | |
| self.file.close() | |
| elif mod_type==imp.PY_COMPILED: | |
| if os.path.exists(self.filename[:-1]): | |
| with open(self.filename[:-1], 'r') as f: | |
| self.source = f.read() | |
| elif mod_type==imp.PKG_DIRECTORY: | |
| self.source = self._get_delegate().get_source() | |
| return self.source | |
| def _get_delegate(self): | |
| finder = ImpImporter(self.filename) | |
| spec = _get_spec(finder, '__init__') | |
| return spec.loader | |
| def get_filename(self, fullname=None): | |
| fullname = self._fix_name(fullname) | |
| mod_type = self.etc[2] | |
| if mod_type==imp.PKG_DIRECTORY: | |
| return self._get_delegate().get_filename() | |
| elif mod_type in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION): | |
| return self.filename | |
| return None | |
| try: | |
| import zipimport | |
| from zipimport import zipimporter | |
| def iter_zipimport_modules(importer, prefix=''): | |
| dirlist = sorted(zipimport._zip_directory_cache[importer.archive]) | |
| _prefix = importer.prefix | |
| plen = len(_prefix) | |
| yielded = {} | |
| import inspect | |
| for fn in dirlist: | |
| if not fn.startswith(_prefix): | |
| continue | |
| fn = fn[plen:].split(os.sep) | |
| if len(fn)==2 and fn[1].startswith('__init__.py'): | |
| if fn[0] not in yielded: | |
| yielded[fn[0]] = 1 | |
| yield prefix + fn[0], True | |
| if len(fn)!=1: | |
| continue | |
| modname = inspect.getmodulename(fn[0]) | |
| if modname=='__init__': | |
| continue | |
| if modname and '.' not in modname and modname not in yielded: | |
| yielded[modname] = 1 | |
| yield prefix + modname, False | |
| iter_importer_modules.register(zipimporter, iter_zipimport_modules) | |
| except ImportError: | |
| pass | |
| def get_importer(path_item): | |
| """Retrieve a finder for the given path item | |
| The returned finder is cached in sys.path_importer_cache | |
| if it was newly created by a path hook. | |
| The cache (or part of it) can be cleared manually if a | |
| rescan of sys.path_hooks is necessary. | |
| """ | |
| path_item = os.fsdecode(path_item) | |
| try: | |
| importer = sys.path_importer_cache[path_item] | |
| except KeyError: | |
| for path_hook in sys.path_hooks: | |
| try: | |
| importer = path_hook(path_item) | |
| sys.path_importer_cache.setdefault(path_item, importer) | |
| break | |
| except ImportError: | |
| pass | |
| else: | |
| importer = None | |
| return importer | |
| def iter_importers(fullname=""): | |
| """Yield finders for the given module name | |
| If fullname contains a '.', the finders will be for the package | |
| containing fullname, otherwise they will be all registered top level | |
| finders (i.e. those on both sys.meta_path and sys.path_hooks). | |
| If the named module is in a package, that package is imported as a side | |
| effect of invoking this function. | |
| If no module name is specified, all top level finders are produced. | |
| """ | |
| if fullname.startswith('.'): | |
| msg = "Relative module name {!r} not supported".format(fullname) | |
| raise ImportError(msg) | |
| if '.' in fullname: | |
| # Get the containing package's __path__ | |
| pkg_name = fullname.rpartition(".")[0] | |
| pkg = importlib.import_module(pkg_name) | |
| path = getattr(pkg, '__path__', None) | |
| if path is None: | |
| return | |
| else: | |
| yield from sys.meta_path | |
| path = sys.path | |
| for item in path: | |
| yield get_importer(item) | |
| def get_loader(module_or_name): | |
| """Get a "loader" object for module_or_name | |
| Returns None if the module cannot be found or imported. | |
| If the named module is not already imported, its containing package | |
| (if any) is imported, in order to establish the package __path__. | |
| """ | |
| if module_or_name in sys.modules: | |
| module_or_name = sys.modules[module_or_name] | |
| if module_or_name is None: | |
| return None | |
| if isinstance(module_or_name, ModuleType): | |
| module = module_or_name | |
| loader = getattr(module, '__loader__', None) | |
| if loader is not None: | |
| return loader | |
| if getattr(module, '__spec__', None) is None: | |
| return None | |
| fullname = module.__name__ | |
| else: | |
| fullname = module_or_name | |
| return find_loader(fullname) | |
| def find_loader(fullname): | |
| """Find a "loader" object for fullname | |
| This is a backwards compatibility wrapper around | |
| importlib.util.find_spec that converts most failures to ImportError | |
| and only returns the loader rather than the full spec | |
| """ | |
| if fullname.startswith('.'): | |
| msg = "Relative module name {!r} not supported".format(fullname) | |
| raise ImportError(msg) | |
| try: | |
| spec = importlib.util.find_spec(fullname) | |
| except (ImportError, AttributeError, TypeError, ValueError) as ex: | |
| # This hack fixes an impedance mismatch between pkgutil and | |
| # importlib, where the latter raises other errors for cases where | |
| # pkgutil previously raised ImportError | |
| msg = "Error while finding loader for {!r} ({}: {})" | |
| raise ImportError(msg.format(fullname, type(ex), ex)) from ex | |
| return spec.loader if spec is not None else None | |
| def extend_path(path, name): | |
| """Extend a package's path. | |
| Intended use is to place the following code in a package's __init__.py: | |
| from pkgutil import extend_path | |
| __path__ = extend_path(__path__, __name__) | |
| This will add to the package's __path__ all subdirectories of | |
| directories on sys.path named after the package. This is useful | |
| if one wants to distribute different parts of a single logical | |
| package as multiple directories. | |
| It also looks for *.pkg files beginning where * matches the name | |
| argument. This feature is similar to *.pth files (see site.py), | |
| except that it doesn't special-case lines starting with 'import'. | |
| A *.pkg file is trusted at face value: apart from checking for | |
| duplicates, all entries found in a *.pkg file are added to the | |
| path, regardless of whether they are exist the filesystem. (This | |
| is a feature.) | |
| If the input path is not a list (as is the case for frozen | |
| packages) it is returned unchanged. The input path is not | |
| modified; an extended copy is returned. Items are only appended | |
| to the copy at the end. | |
| It is assumed that sys.path is a sequence. Items of sys.path that | |
| are not (unicode or 8-bit) strings referring to existing | |
| directories are ignored. Unicode items of sys.path that cause | |
| errors when used as filenames may cause this function to raise an | |
| exception (in line with os.path.isdir() behavior). | |
| """ | |
| if not isinstance(path, list): | |
| # This could happen e.g. when this is called from inside a | |
| # frozen package. Return the path unchanged in that case. | |
| return path | |
| sname_pkg = name + ".pkg" | |
| path = path[:] # Start with a copy of the existing path | |
| parent_package, _, final_name = name.rpartition('.') | |
| if parent_package: | |
| try: | |
| search_path = sys.modules[parent_package].__path__ | |
| except (KeyError, AttributeError): | |
| # We can't do anything: find_loader() returns None when | |
| # passed a dotted name. | |
| return path | |
| else: | |
| search_path = sys.path | |
| for dir in search_path: | |
| if not isinstance(dir, str): | |
| continue | |
| finder = get_importer(dir) | |
| if finder is not None: | |
| portions = [] | |
| if hasattr(finder, 'find_spec'): | |
| spec = finder.find_spec(final_name) | |
| if spec is not None: | |
| portions = spec.submodule_search_locations or [] | |
| # Is this finder PEP 420 compliant? | |
| elif hasattr(finder, 'find_loader'): | |
| _, portions = finder.find_loader(final_name) | |
| for portion in portions: | |
| # XXX This may still add duplicate entries to path on | |
| # case-insensitive filesystems | |
| if portion not in path: | |
| path.append(portion) | |
| # XXX Is this the right thing for subpackages like zope.app? | |
| # It looks for a file named "zope.app.pkg" | |
| pkgfile = os.path.join(dir, sname_pkg) | |
| if os.path.isfile(pkgfile): | |
| try: | |
| f = open(pkgfile) | |
| except OSError as msg: | |
| sys.stderr.write("Can't open %s: %s\n" % | |
| (pkgfile, msg)) | |
| else: | |
| with f: | |
| for line in f: | |
| line = line.rstrip('\n') | |
| if not line or line.startswith('#'): | |
| continue | |
| path.append(line) # Don't check for existence! | |
| return path | |
| def get_data(package, resource): | |
| """Get a resource from a package. | |
| This is a wrapper round the PEP 302 loader get_data API. The package | |
| argument should be the name of a package, in standard module format | |
| (foo.bar). The resource argument should be in the form of a relative | |
| filename, using '/' as the path separator. The parent directory name '..' | |
| is not allowed, and nor is a rooted name (starting with a '/'). | |
| The function returns a binary string, which is the contents of the | |
| specified resource. | |
| For packages located in the filesystem, which have already been imported, | |
| this is the rough equivalent of | |
| d = os.path.dirname(sys.modules[package].__file__) | |
| data = open(os.path.join(d, resource), 'rb').read() | |
| If the package cannot be located or loaded, or it uses a PEP 302 loader | |
| which does not support get_data(), then None is returned. | |
| """ | |
| spec = importlib.util.find_spec(package) | |
| if spec is None: | |
| return None | |
| loader = spec.loader | |
| if loader is None or not hasattr(loader, 'get_data'): | |
| return None | |
| # XXX needs test | |
| mod = (sys.modules.get(package) or | |
| importlib._bootstrap._load(spec)) | |
| if mod is None or not hasattr(mod, '__file__'): | |
| return None | |
| # Modify the resource name to be compatible with the loader.get_data | |
| # signature - an os.path format "filename" starting with the dirname of | |
| # the package's __file__ | |
| parts = resource.split('/') | |
| parts.insert(0, os.path.dirname(mod.__file__)) | |
| resource_name = os.path.join(*parts) | |
| return loader.get_data(resource_name) | |
| _DOTTED_WORDS = r'(?!\d)(\w+)(\.(?!\d)(\w+))*' | |
| _NAME_PATTERN = re.compile(f'^(?P<pkg>{_DOTTED_WORDS})(?P<cln>:(?P<obj>{_DOTTED_WORDS})?)?$', re.U) | |
| del _DOTTED_WORDS | |
| def resolve_name(name): | |
| """ | |
| Resolve a name to an object. | |
| It is expected that `name` will be a string in one of the following | |
| formats, where W is shorthand for a valid Python identifier and dot stands | |
| for a literal period in these pseudo-regexes: | |
| W(.W)* | |
| W(.W)*:(W(.W)*)? | |
| The first form is intended for backward compatibility only. It assumes that | |
| some part of the dotted name is a package, and the rest is an object | |
| somewhere within that package, possibly nested inside other objects. | |
| Because the place where the package stops and the object hierarchy starts | |
| can't be inferred by inspection, repeated attempts to import must be done | |
| with this form. | |
| In the second form, the caller makes the division point clear through the | |
| provision of a single colon: the dotted name to the left of the colon is a | |
| package to be imported, and the dotted name to the right is the object | |
| hierarchy within that package. Only one import is needed in this form. If | |
| it ends with the colon, then a module object is returned. | |
| The function will return an object (which might be a module), or raise one | |
| of the following exceptions: | |
| ValueError - if `name` isn't in a recognised format | |
| ImportError - if an import failed when it shouldn't have | |
| AttributeError - if a failure occurred when traversing the object hierarchy | |
| within the imported package to get to the desired object. | |
| """ | |
| m = _NAME_PATTERN.match(name) | |
| if not m: | |
| raise ValueError(f'invalid format: {name!r}') | |
| gd = m.groupdict() | |
| if gd.get('cln'): | |
| # there is a colon - a one-step import is all that's needed | |
| mod = importlib.import_module(gd['pkg']) | |
| parts = gd.get('obj') | |
| parts = parts.split('.') if parts else [] | |
| else: | |
| # no colon - have to iterate to find the package boundary | |
| parts = name.split('.') | |
| modname = parts.pop(0) | |
| # first part *must* be a module/package. | |
| mod = importlib.import_module(modname) | |
| while parts: | |
| p = parts[0] | |
| s = f'{modname}.{p}' | |
| try: | |
| mod = importlib.import_module(s) | |
| parts.pop(0) | |
| modname = s | |
| except ImportError: | |
| break | |
| # if we reach this point, mod is the module, already imported, and | |
| # parts is the list of parts in the object hierarchy to be traversed, or | |
| # an empty list if just the module is wanted. | |
| result = mod | |
| for p in parts: | |
| result = getattr(result, p) | |
| return result |