moved directives from Preprocessor to Tokenizer

firewave · firewave · commit 44cb133d0de4 · 2024-04-10T15:35:05.000+02:00
diff --git a/lib/cppcheck.cpp b/lib/cppcheck.cpp
@@ -796,7 +796,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
         }
 
         // Get directives
-        preprocessor.setDirectives(tokens1);
+        std::list<Directive> directives = preprocessor.createDirectives(tokens1);
         preprocessor.simplifyPragmaAsm(&tokens1);
 
         preprocessor.setPlatformInfo(&tokens1);
@@ -821,7 +821,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
         // Run define rules on raw code
         if (hasRule("define")) {
             std::string code;
-            for (const Directive &dir : preprocessor.getDirectives()) {
+            for (const Directive &dir : directives) {
                 if (startsWith(dir.str,"#define ") || startsWith(dir.str,"#include "))
                     code += "#line " + std::to_string(dir.linenr) + " \"" + dir.file + "\"\n" + dir.str + '\n';
             }
@@ -890,6 +890,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
             Tokenizer tokenizer(mSettings, this, &preprocessor);
             if (mSettings.showtime != SHOWTIME_MODES::SHOWTIME_NONE)
                 tokenizer.setTimerResults(&s_timerResults);
+            tokenizer.setDirectives(directives); // TODO: how to avoid repeated copies?
 
             try {
                 // Create tokens, skip rest of iteration if failed
diff --git a/lib/preprocessor.cpp b/lib/preprocessor.cpp
@@ -308,10 +308,10 @@ void Preprocessor::inlineSuppressions(const simplecpp::TokenList &tokens, Suppre
     }
 }
 
-void Preprocessor::setDirectives(const simplecpp::TokenList &tokens)
+std::list<Directive> Preprocessor::createDirectives(const simplecpp::TokenList &tokens) const
 {
     // directive list..
-    mDirectives.clear();
+    std::list<Directive> directives;
 
     std::vector<const simplecpp::TokenList *> list;
     list.reserve(1U + mTokenLists.size());
@@ -337,9 +337,11 @@ void Preprocessor::setDirectives(const simplecpp::TokenList &tokens)
                 else
                     directive.str += tok2->str();
             }
-            mDirectives.push_back(std::move(directive));
+            directives.push_back(std::move(directive));
         }
     }
+
+    return directives;
 }
 
 static std::string readcondition(const simplecpp::Token *iftok, const std::set<std::string> &defined, const std::set<std::string> &undefined)
@@ -912,17 +914,6 @@ void Preprocessor::dump(std::ostream &out) const
 {
     // Create a xml dump.
 
-    out << "  <directivelist>" << std::endl;
-    for (const Directive &dir : mDirectives) {
-        out << "    <directive "
-            << "file=\"" << ErrorLogger::toxml(dir.file) << "\" "
-            << "linenr=\"" << dir.linenr << "\" "
-            // str might contain characters such as '"', '<' or '>' which
-            // could result in invalid XML, so run it through toxml().
-            << "str=\"" << ErrorLogger::toxml(dir.str) << "\"/>" << std::endl;
-    }
-    out << "  </directivelist>" << std::endl;
-
     if (!mMacroUsage.empty()) {
         out << "  <macro-usage>" << std::endl;
         for (const simplecpp::MacroUsage &macroUsage: mMacroUsage) {
diff --git a/lib/preprocessor.h b/lib/preprocessor.h
@@ -92,12 +92,7 @@ class CPPCHECKLIB Preprocessor {
 
     void inlineSuppressions(const simplecpp::TokenList &tokens, SuppressionList &suppressions);
 
-    void setDirectives(const simplecpp::TokenList &tokens);
-
-    /** list of all directives met while preprocessing file */
-    const std::list<Directive> &getDirectives() const {
-        return mDirectives;
-    }
+    std::list<Directive> createDirectives(const simplecpp::TokenList &tokens) const;
 
     std::set<std::string> getConfigs(const simplecpp::TokenList &tokens) const;
 
@@ -143,15 +138,10 @@ class CPPCHECKLIB Preprocessor {
 
     static bool hasErrors(const simplecpp::OutputList &outputList);
 
-    void setDirectives(const std::list<Directive> &directives) {
-        mDirectives = directives;
-    }
-
     const Settings& mSettings;
     ErrorLogger *mErrorLogger;
 
     /** list of all directives met while preprocessing file */
-    std::list<Directive> mDirectives;
 
     std::map<std::string, simplecpp::TokenList *> mTokenLists;
 
diff --git a/lib/tokenize.cpp b/lib/tokenize.cpp
@@ -5913,6 +5913,26 @@ void Tokenizer::dump(std::ostream &out) const
 
     std::set<const Library::Container*> containers;
 
+    outs += "  <directivelist>";
+    outs += '\n';
+    for (const Directive &dir : mDirectives) {
+        outs += "    <directive ";
+        outs += "file=\"";
+        outs += ErrorLogger::toxml(dir.file);
+        outs += "\" ";
+        outs += "linenr=\"";
+        outs += std::to_string(dir.linenr);
+        outs += "\" ";
+        // str might contain characters such as '"', '<' or '>' which
+        // could result in invalid XML, so run it through toxml().
+        outs += "str=\"";
+        outs += ErrorLogger::toxml(dir.str);
+        outs +="\"/>";
+        outs += '\n';
+    }
+    outs += "  </directivelist>";
+    outs += '\n';
+
     // tokens..
     outs += "  <tokenlist>";
     outs += '\n';
@@ -10649,12 +10669,17 @@ void Tokenizer::simplifyNamespaceAliases()
     }
 }
 
-// TODO: how to move the Preprocessor dependency out of here?
+void Tokenizer::setDirectives(std::list<Directive> directives)
+{
+    mDirectives = std::move(directives);
+}
+
 bool Tokenizer::hasIfdef(const Token *start, const Token *end) const
 {
     assert(mPreprocessor);
 
-    return std::any_of(mPreprocessor->getDirectives().cbegin(), mPreprocessor->getDirectives().cend(), [&](const Directive& d) {
+    const auto& directives = mDirectives;
+    return std::any_of(directives.cbegin(), directives.cend(), [&](const Directive& d) {
         return startsWith(d.str, "#if") &&
         d.linenr >= start->linenr() &&
         d.linenr <= end->linenr() &&
@@ -10667,7 +10692,7 @@ bool Tokenizer::isPacked(const Token * bodyStart) const
 {
     assert(mPreprocessor);
 
-    const auto& directives = mPreprocessor->getDirectives();
+    const auto& directives = mDirectives;
     // TODO: should this return true if the #pragma exists in any line before the start token?
     return std::any_of(directives.cbegin(), directives.cend(), [&](const Directive& d) {
         return d.linenr < bodyStart->linenr() && d.str == "#pragma pack(1)" && d.file == list.getFiles().front();
diff --git a/lib/tokenize.h b/lib/tokenize.h
@@ -37,6 +37,7 @@ class Token;
 class TemplateSimplifier;
 class ErrorLogger;
 class Preprocessor;
+struct Directive;
 enum class Severity;
 
 /// @addtogroup Core
@@ -624,6 +625,8 @@ class CPPCHECKLIB Tokenizer {
     /** Disable assignment operator */
     Tokenizer &operator=(const Tokenizer &) = delete;
 
+    void setDirectives(std::list<Directive> directives);
+
 private:
     const Token *processFunc(const Token *tok2, bool inOperator) const;
     Token *processFunc(Token *tok2, bool inOperator);
@@ -666,6 +669,8 @@ class CPPCHECKLIB Tokenizer {
     };
     std::vector<TypedefInfo> mTypedefInfo;
 
+    std::list<Directive> mDirectives;
+
     /** variable count */
     nonneg int mVarId{};
 
diff --git a/test/helpers.cpp b/test/helpers.cpp
@@ -124,7 +124,6 @@ std::string PreprocessorHelper::getcode(Preprocessor &preprocessor, const std::s
     tokens1.removeComments();
     preprocessor.simplifyPragmaAsm(&tokens1);
     preprocessor.removeComments();
-    preprocessor.setDirectives(tokens1);
 
     preprocessor.reportOutput(outputList, true);
 
@@ -179,5 +178,6 @@ void PreprocessorHelper::preprocess(Preprocessor &preprocessor, const char code[
     // Tokenizer..
     tokenizer.list.createTokens(std::move(tokens2));
 
-    preprocessor.setDirectives(tokens1);
+    std::list<Directive> directives = preprocessor.createDirectives(tokens1);
+    tokenizer.setDirectives(std::move(directives));
 }
diff --git a/test/testpreprocessor.cpp b/test/testpreprocessor.cpp
@@ -245,10 +245,6 @@ class TestPreprocessor : public TestFixture {
 
         TEST_CASE(wrongPathOnErrorDirective);
 
-        TEST_CASE(testDirectiveIncludeTypes);
-        TEST_CASE(testDirectiveIncludeLocations);
-        TEST_CASE(testDirectiveIncludeComments);
-
         TEST_CASE(testMissingInclude);
         TEST_CASE(testMissingInclude2);
         TEST_CASE(testMissingInclude3);
@@ -275,7 +271,7 @@ class TestPreprocessor : public TestFixture {
         tokens.removeComments();
         preprocessor0.simplifyPragmaAsm(&tokens);
         preprocessor0.removeComments();
-        preprocessor0.setDirectives(tokens);
+        std::list<Directive> directives = preprocessor0.createDirectives(tokens); // TODO
 
         preprocessor0.reportOutput(outputList, true);
 
@@ -2313,88 +2309,6 @@ class TestPreprocessor : public TestFixture {
         ASSERT_EQUALS("[test.c:1]: (error) #error hello world!\n", errout_str());
     }
 
-    void testDirectiveIncludeTypes() {
-        const char filedata[] = "#define macro some definition\n"
-                                "#undef macro\n"
-                                "#ifdef macro\n"
-                                "#elif some (complex) condition\n"
-                                "#else\n"
-                                "#endif\n"
-                                "#if some other condition\n"
-                                "#pragma some proprietary content\n"
-                                "#\n" /* may appear in old C code */
-                                "#ident some text\n" /* may appear in old C code */
-                                "#unknownmacro some unpredictable text\n"
-                                "#warning some warning message\n"
-                                "#error some error message\n";
-        const char dumpdata[] = "  <directivelist>\n"
-
-                                "    <directive file=\"test.c\" linenr=\"1\" str=\"#define macro some definition\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"2\" str=\"#undef macro\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"3\" str=\"#ifdef macro\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"4\" str=\"#elif some (complex) condition\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"5\" str=\"#else\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"6\" str=\"#endif\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"7\" str=\"#if some other condition\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"8\" str=\"#pragma some proprietary content\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"9\" str=\"#\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"10\" str=\"#ident some text\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"11\" str=\"#unknownmacro some unpredictable text\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"12\" str=\"#warning some warning message\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"13\" str=\"#error some error message\"/>\n"
-                                "  </directivelist>\n";
-
-        std::ostringstream ostr;
-        Preprocessor preprocessor(settings0, this);
-        PreprocessorHelper::getcode(preprocessor, filedata, "", "test.c");
-        preprocessor.dump(ostr);
-        ASSERT_EQUALS(dumpdata, ostr.str());
-    }
-
-    void testDirectiveIncludeLocations() {
-        const char filedata[] = "#define macro1 val\n"
-                                "#file \"inc1.h\"\n"
-                                "#define macro2 val\n"
-                                "#file \"inc2.h\"\n"
-                                "#define macro3 val\n"
-                                "#endfile\n"
-                                "#define macro4 val\n"
-                                "#endfile\n"
-                                "#define macro5 val\n";
-        const char dumpdata[] = "  <directivelist>\n"
-                                "    <directive file=\"test.c\" linenr=\"1\" str=\"#define macro1 val\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"2\" str=\"#include &quot;inc1.h&quot;\"/>\n"
-                                "    <directive file=\"inc1.h\" linenr=\"1\" str=\"#define macro2 val\"/>\n"
-                                "    <directive file=\"inc1.h\" linenr=\"2\" str=\"#include &quot;inc2.h&quot;\"/>\n"
-                                "    <directive file=\"inc2.h\" linenr=\"1\" str=\"#define macro3 val\"/>\n"
-                                "    <directive file=\"inc1.h\" linenr=\"3\" str=\"#define macro4 val\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"3\" str=\"#define macro5 val\"/>\n"
-                                "  </directivelist>\n";
-
-        std::ostringstream ostr;
-        Preprocessor preprocessor(settings0, this);
-        PreprocessorHelper::getcode(preprocessor, filedata, "", "test.c");
-        preprocessor.dump(ostr);
-        ASSERT_EQUALS(dumpdata, ostr.str());
-    }
-
-    void testDirectiveIncludeComments() {
-        const char filedata[] = "#ifdef macro2 /* this will be removed */\n"
-                                "#else /* this will be removed too */\n"
-                                "#endif /* this will also be removed */\n";
-        const char dumpdata[] = "  <directivelist>\n"
-                                "    <directive file=\"test.c\" linenr=\"1\" str=\"#ifdef macro2\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"2\" str=\"#else\"/>\n"
-                                "    <directive file=\"test.c\" linenr=\"3\" str=\"#endif\"/>\n"
-                                "  </directivelist>\n";
-
-        std::ostringstream ostr;
-        Preprocessor preprocessor(settings0, this);
-        PreprocessorHelper::getcode(preprocessor, filedata, "", "test.c");
-        preprocessor.dump(ostr);
-        ASSERT_EQUALS(dumpdata, ostr.str());
-    }
-
     // test for existing local include
     void testMissingInclude() {
         /*const*/ Settings settings;
diff --git a/test/testtokenize.cpp b/test/testtokenize.cpp
diff --git a/test/testunusedvar.cpp b/test/testunusedvar.cpp

Original file line number	Diff line number	Diff line change
`@@ -308,10 +308,10 @@ void Preprocessor::inlineSuppressions(const simplecpp::TokenList &tokens, Suppre`
`308`	`308`	`}`
`309`	`309`	`}`
`310`	`310`
`311`		`-void Preprocessor::setDirectives(const simplecpp::TokenList &tokens)`
	`311`	`+std::list<Directive> Preprocessor::createDirectives(const simplecpp::TokenList &tokens) const`
`312`	`312`	`{`
`313`	`313`	`// directive list..`
`314`		`- mDirectives.clear();`
	`314`	`+ std::list<Directive> directives;`
`315`	`315`
`316`	`316`	`std::vector<const simplecpp::TokenList *> list;`
`317`	`317`	`list.reserve(1U + mTokenLists.size());`
`@@ -337,9 +337,11 @@ void Preprocessor::setDirectives(const simplecpp::TokenList &tokens)`
`337`	`337`	`else`
`338`	`338`	`directive.str += tok2->str();`
`339`	`339`	`}`
`340`		`- mDirectives.push_back(std::move(directive));`
	`340`	`+ directives.push_back(std::move(directive));`
`341`	`341`	`}`
`342`	`342`	`}`
	`343`	`+`
	`344`	`+ return directives;`
`343`	`345`	`}`
`344`	`346`
`345`	`347`	`static std::string readcondition(const simplecpp::Token *iftok, const std::set<std::string> &defined, const std::set<std::string> &undefined)`
`@@ -912,17 +914,6 @@ void Preprocessor::dump(std::ostream &out) const`
`912`	`914`	`{`
`913`	`915`	`// Create a xml dump.`
`914`	`916`
`915`		`- out << " <directivelist>" << std::endl;`
`916`		`- for (const Directive &dir : mDirectives) {`
`917`		`- out << " <directive "`
`918`		`- << "file=\"" << ErrorLogger::toxml(dir.file) << "\" "`
`919`		`- << "linenr=\"" << dir.linenr << "\" "`
`920`		`- // str might contain characters such as '"', '<' or '>' which`
`921`		`- // could result in invalid XML, so run it through toxml().`
`922`		`- << "str=\"" << ErrorLogger::toxml(dir.str) << "\"/>" << std::endl;`
`923`		`- }`
`924`		`- out << " </directivelist>" << std::endl;`
`925`		`-`
`926`	`917`	`if (!mMacroUsage.empty()) {`
`927`	`918`	`out << " <macro-usage>" << std::endl;`
`928`	`919`	`for (const simplecpp::MacroUsage &macroUsage: mMacroUsage) {`