Refactored flask.ext process to not swallow exceptions on weird Pythons.

flask/ext/__init__.py

@@ -19,91 +19,11 @@
 """
 
 
-class _ExtensionImporter(object):
-    """This importer redirects imports from this submodule to other locations.
-    This makes it possible to transition from the old flaskext.name to the
-    newer flask_name without people having a hard time.
-    """
-    _module_choices = ['flask_%s', 'flaskext.%s']
+def setup():
+    from ..exthook import ExtensionImporter
+    importer = ExtensionImporter(['flask_%s', 'flaskext.%s'], __name__)
+    importer.install()
 
-    def __init__(self):
-        from sys import meta_path
-        self.prefix = __name__ + '.'
-        self.prefix_cutoff = __name__.count('.') + 1
 
-        # since people might reload the flask.ext module (by accident or
-        # intentionally) we have to make sure to not add more than one
-        # import hook.  We can't check class types here either since a new
-        # class will be created on reload.  As a result of that we check
-        # the name of the class and remove stale instances.
-        def _name(x):
-            cls = type(x)
-            return cls.__module__ + '.' + cls.__name__
-        this = _name(self)
-        meta_path[:] = [x for x in meta_path if _name(x) != this] + [self]
-
-    def find_module(self, fullname, path=None):
-        if fullname.startswith(self.prefix):
-            return self
-
-    def load_module(self, fullname):
-        from sys import modules, exc_info
-        if fullname in modules:
-            return modules[fullname]
-        modname = fullname.split('.', self.prefix_cutoff)[self.prefix_cutoff]
-        for path in self._module_choices:
-            realname = path % modname
-            try:
-                __import__(realname)
-            except ImportError:
-                exc_type, exc_value, tb = exc_info()
-                # since we only establish the entry in sys.modules at the
-                # very this seems to be redundant, but if recursive imports
-                # happen we will call into the move import a second time.
-                # On the second invocation we still don't have an entry for
-                # fullname in sys.modules, but we will end up with the same
-                # fake module name and that import will succeed since this
-                # one already has a temporary entry in the modules dict.
-                # Since this one "succeeded" temporarily that second
-                # invocation now will have created a fullname entry in
-                # sys.modules which we have to kill.
-                modules.pop(fullname, None)
-                if self.is_important_traceback(realname, tb):
-                    raise exc_type, exc_value, tb
-                continue
-            module = modules[fullname] = modules[realname]
-            if '.' not in modname:
-                setattr(modules[__name__], modname, module)
-            return module
-        raise ImportError('No module named %s' % fullname)
-
-    def is_important_traceback(self, important_module, tb):
-        """Walks a traceback's frames and checks if any of the frames
-        originated in the given important module.  If that is the case
-        then we were able to import the module itself but apparently
-        something went wrong when the module was imported.  (Eg: import
-        of an import failed).
-        """
-        # Why can we access f_globals' __name__ here and the value is
-        # not None?  I honestly don't know but here is my thinking.
-        # The module owns a reference to globals and the frame has one.
-        # Each function only keeps a reference to the globals not do the
-        # module which normally causes the problem that when the module
-        # shuts down all globals are set to None.  Now however when the
-        # import system fails Python takes the short way out and does not
-        # actually properly shut down the module by Noneing the values
-        # but by just removing the entry from sys.modules.  This means
-        # that the regular reference based cleanup kicks in.
-        #
-        # The good thing: At worst we will swallow an exception we should
-        # not and the error message will be messed up.  However I think
-        # this should be sufficiently reliable.
-        while tb is not None:
-            if tb.tb_frame.f_globals.get('__name__') == important_module:
-                return True
-            tb = tb.tb_next
-        return False
-
-
-_ExtensionImporter()
-del _ExtensionImporter
+setup()
+del setup

flask/exthook.py

@@ -0,0 +1,119 @@
+# -*- coding: utf-8 -*-
+"""
+    flask.exthook
+    ~~~~~~~~~~~~~
+
+    Redirect imports for extensions.  This module basically makes it possible
+    for us to transition from flaskext.foo to flask_foo without having to
+    force all extensions to upgrade at the same time.
+
+    When a user does ``from flask.ext.foo import bar`` it will attempt to
+    import ``from flask_foo import bar`` first and when that fails it will
+    try to import ``from flaskext.foo import bar``.
+
+    We're switching from namespace packages because it was just too painful for
+    everybody involved.
+
+    This is used by `flask.ext`.
+
+    :copyright: (c) 2011 by Armin Ronacher.
+    :license: BSD, see LICENSE for more details.
+"""
+import sys
+import os
+
+
+class ExtensionImporter(object):
+    """This importer redirects imports from this submodule to other locations.
+    This makes it possible to transition from the old flaskext.name to the
+    newer flask_name without people having a hard time.
+    """
+
+    def __init__(self, module_choices, wrapper_module):
+        self.module_choices = module_choices
+        self.wrapper_module = wrapper_module
+        self.prefix = wrapper_module + '.'
+        self.prefix_cutoff = wrapper_module.count('.') + 1
+
+    def __eq__(self, other):
+        return self.__class__.__module__ == other.__class__.__module__ and \
+               self.__class__.__name__ == other.__class__.__name__ and \
+               self.wrapper_module == other.wrapper_module and \
+               self.module_choices == other.module_choices
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def install(self):
+        sys.meta_path[:] = [x for x in sys.meta_path if self != x] + [self]
+
+    def find_module(self, fullname, path=None):
+        if fullname.startswith(self.prefix):
+            return self
+
+    def load_module(self, fullname):
+        if fullname in sys.modules:
+            return sys.modules[fullname]
+        modname = fullname.split('.', self.prefix_cutoff)[self.prefix_cutoff]
+        for path in self.module_choices:
+            realname = path % modname
+            try:
+                __import__(realname)
+            except ImportError:
+                exc_type, exc_value, tb = sys.exc_info()
+                # since we only establish the entry in sys.modules at the
+                # very this seems to be redundant, but if recursive imports
+                # happen we will call into the move import a second time.
+                # On the second invocation we still don't have an entry for
+                # fullname in sys.modules, but we will end up with the same
+                # fake module name and that import will succeed since this
+                # one already has a temporary entry in the modules dict.
+                # Since this one "succeeded" temporarily that second
+                # invocation now will have created a fullname entry in
+                # sys.modules which we have to kill.
+                sys.modules.pop(fullname, None)
+
+                # If it's an important traceback we reraise it, otherwise
+                # we swallow it and try the next choice.  The skipped frame
+                # is the one from __import__ above which we don't care about
+                if self.is_important_traceback(realname, tb):
+                    raise exc_type, exc_value, tb.tb_next
+                continue
+            module = sys.modules[fullname] = sys.modules[realname]
+            if '.' not in modname:
+                setattr(sys.modules[self.wrapper_module], modname, module)
+            return module
+        raise ImportError('No module named %s' % fullname)
+
+    def is_important_traceback(self, important_module, tb):
+        """Walks a traceback's frames and checks if any of the frames
+        originated in the given important module.  If that is the case then we
+        were able to import the module itself but apparently something went
+        wrong when the module was imported.  (Eg: import of an import failed).
+        """
+        while tb is not None:
+            if self.is_important_frame(important_module, tb):
+                return True
+            tb = tb.tb_next
+        return False
+
+    def is_important_frame(self, important_module, tb):
+        """Checks a single frame if it's important."""
+        g = tb.tb_frame.f_globals
+        if '__name__' not in g:
+            return False
+
+        module_name = g['__name__']
+
+        # Python 2.7 Behavior.  Modules are cleaned up late so the
+        # name shows up properly here.  Success!
+        if module_name == important_module:
+            return True
+
+        # Some python verisons will will clean up modules so early that the
+        # module name at that point is no longer set.  Try guessing from
+        # the filename then.
+        filename = os.path.abspath(tb.tb_frame.f_code.co_filename)
+        test_string = os.path.sep + important_module.replace('.', os.path.sep)
+        return test_string + '.py' in filename or \
+               test_string + os.path.sep + '__init__.py' in filename

flask/testsuite/ext.py

@@ -35,8 +35,8 @@ class ExtImportHookTestCase(FlaskTestCase):
         import_hooks = 0
         for item in sys.meta_path:
             cls = type(item)
-            if cls.__module__ == 'flask.ext' and \
-               cls.__name__ == '_ExtensionImporter':
+            if cls.__module__ == 'flask.exthook' and \
+               cls.__name__ == 'ExtensionImporter':
                 import_hooks += 1
         self.assert_equal(import_hooks, 1)
 
@@ -104,6 +104,18 @@ class ExtImportHookTestCase(FlaskTestCase):
             with self.assert_raises(ImportError):
                 import flask.ext.broken
 
+    def test_no_error_swallowing(self):
+        try:
+            import flask.ext.broken
+        except ImportError:
+            exc_type, exc_value, tb = sys.exc_info()
+            self.assert_(exc_type is ImportError)
+            self.assert_equal(str(exc_value), 'No module named missing_module')
+            self.assert_(tb.tb_frame.f_globals is globals())
+
+            next = tb.tb_next
+            self.assert_('flask_broken/__init__.py' in next.tb_frame.f_code.co_filename)
+
 
 def suite():
     suite = unittest.TestSuite()

scripts/flaskext_compat.py

@@ -15,22 +15,33 @@
     :license: BSD, see LICENSE for more details.
 """
 import sys
+import os
 import imp
 
 
-ext_module = imp.new_module('flask.ext')
-ext_module.__path__ = []
-ext_module.__package__ = ext_module.__name__
+class ExtensionImporter(object):
+    """This importer redirects imports from this submodule to other locations.
+    This makes it possible to transition from the old flaskext.name to the
+    newer flask_name without people having a hard time.
+    """
 
+    def __init__(self, module_choices, wrapper_module):
+        self.module_choices = module_choices
+        self.wrapper_module = wrapper_module
+        self.prefix = wrapper_module + '.'
+        self.prefix_cutoff = wrapper_module.count('.') + 1
 
-class _ExtensionImporter(object):
-    """This importer redirects imports from the flask.ext module to other
-    locations.  For implementation details see the code in Flask 0.8
-    that does the same.
-    """
-    _module_choices = ['flask_%s', 'flaskext.%s']
-    prefix = ext_module.__name__ + '.'
-    prefix_cutoff = prefix.count('.')
+    def __eq__(self, other):
+        return self.__class__.__module__ == other.__class__.__module__ and \
+               self.__class__.__name__ == other.__class__.__name__ and \
+               self.wrapper_module == other.wrapper_module and \
+               self.module_choices == other.module_choices
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def install(self):
+        sys.meta_path[:] = [x for x in sys.meta_path if self != x] + [self]
 
     def find_module(self, fullname, path=None):
         if fullname.startswith(self.prefix):
@@ -40,34 +51,74 @@ class _ExtensionImporter(object):
         if fullname in sys.modules:
             return sys.modules[fullname]
         modname = fullname.split('.', self.prefix_cutoff)[self.prefix_cutoff]
-        for path in self._module_choices:
+        for path in self.module_choices:
             realname = path % modname
             try:
                 __import__(realname)
             except ImportError:
                 exc_type, exc_value, tb = sys.exc_info()
+                # since we only establish the entry in sys.modules at the
+                # very this seems to be redundant, but if recursive imports
+                # happen we will call into the move import a second time.
+                # On the second invocation we still don't have an entry for
+                # fullname in sys.modules, but we will end up with the same
+                # fake module name and that import will succeed since this
+                # one already has a temporary entry in the modules dict.
+                # Since this one "succeeded" temporarily that second
+                # invocation now will have created a fullname entry in
+                # sys.modules which we have to kill.
                 sys.modules.pop(fullname, None)
+
+                # If it's an important traceback we reraise it, otherwise
+                # we swallow it and try the next choice.  The skipped frame
+                # is the one from __import__ above which we don't care about
                 if self.is_important_traceback(realname, tb):
-                    raise exc_type, exc_value, tb
+                    raise exc_type, exc_value, tb.tb_next
                 continue
             module = sys.modules[fullname] = sys.modules[realname]
             if '.' not in modname:
-                setattr(ext_module, modname, module)
+                setattr(sys.modules[self.wrapper_module], modname, module)
             return module
         raise ImportError('No module named %s' % fullname)
 
     def is_important_traceback(self, important_module, tb):
+        """Walks a traceback's frames and checks if any of the frames
+        originated in the given important module.  If that is the case then we
+        were able to import the module itself but apparently something went
+        wrong when the module was imported.  (Eg: import of an import failed).
+        """
         while tb is not None:
-            if tb.tb_frame.f_globals.get('__name__') == important_module:
+            if self.is_important_frame(important_module, tb):
                 return True
             tb = tb.tb_next
         return False
 
+    def is_important_frame(self, important_module, tb):
+        """Checks a single frame if it's important."""
+        g = tb.tb_frame.f_globals
+        if '__name__' not in g:
+            return False
+
+        module_name = g['__name__']
+
+        # Python 2.7 Behavior.  Modules are cleaned up late so the
+        # name shows up properly here.  Success!
+        if module_name == important_module:
+            return True
+
+        # Some python verisons will will clean up modules so early that the
+        # module name at that point is no longer set.  Try guessing from
+        # the filename then.
+        filename = os.path.abspath(tb.tb_frame.f_code.co_filename)
+        test_string = os.path.sep + important_module.replace('.', os.path.sep)
+        return test_string + '.py' in filename or \
+               test_string + os.path.sep + '__init__.py' in filename
+
 
 def activate():
-    """Activates the compatibility system."""
     import flask
-    if hasattr(flask, 'ext'):
-        return
-    sys.modules['flask.ext'] = flask.ext = ext_module
-    sys.meta_path.append(_ExtensionImporter())
+    ext_module = imp.new_module('flask.ext')
+    ext_module.__path__ = []
+    flask.ext = sys.modules['flask.ext'] = ext_module
+    importer = ExtensionImporter(['flask_%s', 'flaskext.%s'], 'flask.ext')
+    importer.install()