使用 imp.load_source 动态加载 python 模块和包

Question

我正在尝试从 python 2.7 中的任意文件夹位置动态加载模块和包。它适用于裸露的单文件模块。但是尝试加载一个包有点困难。

我能想到的最好的办法是在包（文件夹）中加载 init.py 文件。但是比如说我有这个：

root:
  mod.py
  package:
    __init__.py
    sub.py

如果 mod.py 包含：

from package import sub

使用我当前的加载代码（如下），它将失败，说明没有名为“sub”的包，除非我将以下内容添加到package/__init__.py

import sub

我不得不想象这是因为当您导入一个包时，它通常还会扫描其中的所有其他子文件。我是否也只需要手动执行此操作，还是有类似于 imp.load_source 的方法也可以处理包文件夹？

加载代码：

import md5
import sys
import os.path
import imp
import traceback
import glob

def load_package(path, base):
    try:
        try:
            sys.path.append(path + "/" + base)
            init = path + "/" + base + "/__init__.py"
            if not os.path.exists(init):
                return None

            fin = open(init, 'rb')

            return  (base, imp.load_source(base, init, fin))
        finally:
            try: fin.close()
            except: pass
    except ImportError, x:
        traceback.print_exc(file = sys.stderr)
        raise
    except:
        traceback.print_exc(file = sys.stderr)
        raise

def load_module(path):
    try:
        try:
            code_dir = os.path.dirname(path)
            code_file = os.path.basename(path)
            base = code_file.replace(".py", "")

            fin = open(path, 'rb')

            hash = md5.new(path).hexdigest() + "_" + code_file
            return  (base, imp.load_source(base, path, fin))
        finally:
            try: fin.close()
            except: pass
    except ImportError, x:
        traceback.print_exc(file = sys.stderr)
        raise
    except:
        traceback.print_exc(file = sys.stderr)
        raise

def load_folder(dir):
    sys.path.append(dir)
    mods = {}

    for p in glob.glob(dir + "/*/"):
        base = p.replace("\\", "").replace("/", "")
        base = base.replace(dir.replace("\\", "").replace("/", ""), "")
        package = load_package(dir, base) 
        if package:
            hash, pack = package
            mods[hash] = pack

    for m in glob.glob(dir + "/*.py"):
        hash, mod = load_module(m) 
        mods[hash] = mod

    return mods

Answer 1

下面的代码在功能上等同于您的代码以 traceback.print_exc 为模（您应该让客户端处理 - 如果不处理，异常最终会打印出来）：

def _load_package(path, base):
    sys.path.append(path + "/" + base)
    init = path + "/" + base + "/__init__.py"
    if not os.path.exists(init):
        return None, None
    with open(init, 'rb') as fin:
        return base, imp.load_source(base, init, fin)

def _load_module(path):
    code_file = os.path.basename(path)
    base = code_file.replace(".py", "")
    with open(path, 'rb') as fin:
        return base, imp.load_source(base, path, fin)

def load_folder(dir):
    sys.path.append(dir)
    mods = {}
    for p in glob.glob(dir + "/*/"):
        base = p.replace("\\", "").replace("/", "")
        base = base.replace(dir.replace("\\", "").replace("/", ""), "")
        hash, pack = _load_package(dir, base)
        if hash: mods[hash] = pack
    for m in glob.glob(dir + "/*.py"): ##: /*/*.py
        hash, mod = _load_module(m)
        mods[hash] = mod
    return mods

## My added code
print('Python %s on %s' % (sys.version, sys.platform))

root_ = r'C:\Dropbox\eclipse_workspaces\python\sandbox\root'

def depyc(root, _indent=''): # deletes .pyc which will end up being imported
    if not _indent: print '\nListing', root
    for p in os.listdir(root):
        name = _indent + p
        abspath = os.path.join(root, p)
        if os.path.isdir(abspath):
            print name + ':'
            depyc(abspath, _indent=_indent + '  ')
        else:
            name_ = name[-4:]
            if name_ == '.pyc':
                os.remove(abspath)
                continue
            print name
    if not _indent: print

depyc(root_)
load_folder(root_)

打印：

Python 2.7.10 (default, May 23 2015, 09:40:32) [MSC v.1500 32 bit (Intel)] on win32

Listing C:\Dropbox\eclipse_workspaces\python\sandbox\root
mod.py
package:
  sub.py
  __init__.py

C:\Dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py imported!
C:\Dropbox\eclipse_workspaces\python\sandbox\root\mod.py imported!

mod.py、sub.py 和 __init__.py 只包含

print(__file__ + u' imported!')

现在将mod.py 修改为：

from package import sub
print(__file__ + u' imported!')

我们确实得到了：

Listing....

C:\Dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py imported! <### this may move around ###>
Traceback (most recent call last):
  File "C:/Users/MrD/.PyCharm40/config/scratches/load_folder.py", line 57, in <module>
    load_folder(root_)
  File "C:/Users/MrD/.PyCharm40/config/scratches/load_folder.py", line 31, in load_folder
    hash, mod = _load_module(m)
  File "C:/Users/MrD/.PyCharm40/config/scratches/load_folder.py", line 20, in _load_module
    return base, imp.load_source(base, path, fin)
  File "C:\Dropbox\eclipse_workspaces\python\sandbox\root\mod.py", line 1, in <module>
    from package import sub
ImportError: cannot import name sub

请注意，错误是“无法导入名称 sub”，而不是“没有名为“sub”的包”。那为什么不能呢？

修改__init__.py:

# package/__init__.py    
print(__file__ + u' imported!')

print '__name__', '->', __name__
print '__package__', '->', __package__
print '__path__', '->', __path__

打印：

Listing...

C:\Dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py imported! <### not really ###>
__name__ -> package
__package__ -> None
__path__ ->
Traceback (most recent call last):
  File "C:/Users/MrD/.PyCharm40/config/scratches/load_folder.py", line 59, in <module>
    load_folder(root_)
  File "C:/Users/MrD/.PyCharm40/config/scratches/load_folder.py", line 30, in load_folder
    hash, pack = _load_package(dir, base)
  File "C:/Users/MrD/.PyCharm40/config/scratches/load_folder.py", line 14, in _load_package
    init = imp.load_source(base, init, fin)
  File "C:\Dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py", line 5, in <module>
    print '__path__', '->', __path__
NameError: name '__path__' is not defined

直接导入时会打印：

>>> sys.path.extend([r'C:\Dropbox\eclipse_workspaces\python\sandbox\root'])
>>> import package
C:\Dropbox\eclipse_workspaces\python\sandbox\root\package\__init__.py imported!
__name__ -> package
__package__ -> None
__path__ -> ['C:\\Dropbox\\eclipse_workspaces\\python\\sandbox\\root\\package']

所以修改_load_package为：

def _load_package(path, base):
    pkgDir = os.path.abspath(os.path.join(path, base))
    init = os.path.join(pkgDir, "__init__.py")
    if not os.path.exists(init):
        return None, None
    file, pathname, description = imp.find_module(base, [path])
    print file, pathname, description # None, pkgDir, ('', '', 5)
    pack = sys.modules.get(base, None) # load_module will reload - yak!
    if pack is None:
        sys.modules[base] = pack = imp.load_module(base, file, pathname, description)
    return base, pack

按原样解决：

...
    if pack is None:
        sys.modules[base] = pack = imp.load_module(base, None, '', description)
        pack.__path__ = [pkgDir]

或在您的原始代码中：

with open(init, 'rb') as fin:
    source = imp.load_source(base, init, fin)
    source.__path__ = path + "/" + base
    return base, source

所以发生的事情是该包依赖其__path __ 属性才能正常运行。

---

不断破解并想出了：

import sys
import os.path
import imp

def _load_(root, name):
    file_object, pathname, description = imp.find_module(name, [root])
    pack = sys.modules.get(name, None)
    try:
        if pack is None:
            pack = imp.load_module(name, file_object, pathname, description)
        else:
            print 'In cache', pack
    finally:
        if file_object is not None: file_object.close()
    return name, pack

def load_folder(root):
    # sys.path.append(root)
    mods = {}
    paths = [(item, os.path.join(root, item)) for item in os.listdir(root)]
    packages = filter(lambda path_tuple: os.path.exists(
        os.path.join((path_tuple[1]), "__init__.py")), paths)
    py_files = filter(lambda path_tuple: path_tuple[0][-3:] == '.py', paths)
    del paths
    # first import packages as in original - modules may import from them
    for path, _abspath in packages:
        print 'Importing', _abspath
        key, mod = _load_(root, name=path) # will use pyc if available!
        mods[key] = mod
    # then modules
    for path, _abspath in py_files:
        print 'Importing', _abspath
        key, mod = _load_(root, name=path[:-3])
        mods[key] = mod
    return mods

我合并了包和模块加载代码删除imp.load_source（一个不太棘手的功能）并依赖 imp.load_module 代替。我不直接与 sys.path 混淆，因为imp.load_module will reload [!] 我检查了sys.modules 缓存。返回的 mods dict 完全未经测试 - 您必须以某种方式实现哈希（_abspath 就足够了）。

运行方式：

def depyc(root, rmpyc, _indent=''):
    if not _indent: print '\nListing', root
    for p in os.listdir(root):
        name = _indent + p
        abspath = os.path.join(root, p)
        if os.path.isdir(abspath):
            print name + ':'
            depyc(abspath, rmpyc, _indent=_indent + '  ')
        else:
            if rmpyc and name[-4:] == '.pyc':
                os.remove(abspath)
                continue
            print name
    if not _indent: print

## Run ##
print('Python %s on %s' % (sys.version, sys.platform))
root_ = os.path.join(os.getcwdu(), u'root')
depyc(root_, False) # False will end up importing the pyc files !
load_folder(root_)

测试各种场景 -

带有示例root/ dir 的代码是here