diff --git a/CodeSniffer/File.php b/CodeSniffer/File.php
index 36cc44f89e..d9a04d6701 100644
--- a/CodeSniffer/File.php
+++ b/CodeSniffer/File.php
@@ -532,6 +532,7 @@ public function start($contents=null)
                 if (isset($this->_ignoredListeners[$listenerData['class']]) === true
                     || (isset($listenerIgnoreTo[$listenerData['class']]) === true
                     && $listenerIgnoreTo[$listenerData['class']] > $stackPtr)
+                    || empty($this->_listeners) === true
                 ) {
                     // This sniff is ignoring past this token, or the whole file.
                     continue;