diff --git a/tests/test_apps/importerror.py b/tests/test_apps/importerror.py
deleted file mode 100644
index 3970e3e7..00000000
--- a/tests/test_apps/importerror.py
+++ /dev/null
@@ -1,2 +0,0 @@
-# TestNoImports
-raise NotImplementedError
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
index 2ae72a0f..bdc4c282 100644
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -577,7 +577,8 @@ class TestNoImports(object):
     imp modules in the Python standard library.
     """
 
-    def test_name_with_import_error(self):
+    def test_name_with_import_error(self, apps_tmpdir):
+        apps_tmpdir.join('importerror.py').write('raise NotImplementedError()')
         try:
             flask.Flask('importerror')
         except NotImplementedError: