make sure the language for a TokenList/Token is always determined

firewave · firewave · commit cc2d9c9a355d · 2024-01-20T00:33:29.000+01:00
diff --git a/lib/clangimport.cpp b/lib/clangimport.cpp
@@ -330,8 +330,12 @@ namespace clangimport {
         void dumpAst(int num = 0, int indent = 0) const;
         void createTokens1(TokenList &tokenList) {
             //dumpAst();
-            if (!tokenList.back())
+            if (!tokenList.back()) {
                 setLocations(tokenList, 0, 1, 1);
+                // FIXME: treat as C++ if no filename (i.e. no lang) is specified for now
+                if (tokenList.getSourceFilePath().empty())
+                    tokenList.setLang(Standards::Language::CPP);
+            }
             else
                 setLocations(tokenList, tokenList.back()->fileIndex(), tokenList.back()->linenr(), 1);
             createTokens(tokenList);
@@ -626,6 +630,7 @@ void clangimport::AstNode::setValueType(Token *tok)
             continue;
 
         TokenList decl(nullptr);
+        decl.setLang(tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
         addTypeTokens(decl, type, tok->scope());
         if (!decl.front())
             break;
diff --git a/lib/cppcheck.cpp b/lib/cppcheck.cpp
@@ -633,12 +633,14 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
 
     try {
         if (mSettings.library.markupFile(filename)) {
+            // this is not a real source file - we just want to tokenize it. treat it as C anyways as the language needs to be determined.
             Tokenizer tokenizer(mSettings, this);
-            if (fileStream)
-                tokenizer.list.createTokens(*fileStream, filename);
+            if (fileStream) {
+                tokenizer.list.createTokens(*fileStream, Standards::Language::C);
+            }
             else {
                 std::ifstream in(filename);
-                tokenizer.list.createTokens(in, filename);
+                tokenizer.list.createTokens(in, Standards::Language::C);
             }
             checkUnusedFunctions.getFileInfo(&tokenizer, &mSettings);
             return EXIT_SUCCESS;
@@ -802,7 +804,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
             }
             Tokenizer tokenizer2(mSettings, this);
             std::istringstream istr2(code);
-            tokenizer2.list.createTokens(istr2);
+            tokenizer2.list.createTokens(istr2, Path::identify(*files.begin()));
             executeRules("define", tokenizer2);
         }
 #endif
diff --git a/lib/importproject.cpp b/lib/importproject.cpp
@@ -560,11 +560,11 @@ namespace {
             replaceAll(c, "$(Configuration)", p.configuration);
             replaceAll(c, "$(Platform)", p.platformStr);
 
-            // TODO : Better evaluation
-            Settings s;
-            std::istringstream istr(c);
+            // TODO: evaluate without using the Tokenizer
+            const Settings s;
             Tokenizer tokenizer(s);
-            tokenizer.tokenize(istr,"vcxproj");
+            std::istringstream istr(c);
+            tokenizer.tokenize(istr,"vcxproj.c");
             for (const Token *tok = tokenizer.tokens(); tok; tok = tok->next()) {
                 if (tok->str() == "(" && tok->astOperand1() && tok->astOperand2()) {
                     if (tok->astOperand1()->expressionString() == "Configuration.Contains")
diff --git a/lib/library.cpp b/lib/library.cpp
@@ -53,10 +53,10 @@ static std::vector<std::string> getnames(const char *names)
     return ret;
 }
 
-static void gettokenlistfromvalid(const std::string& valid, TokenList& tokenList)
+static void gettokenlistfromvalid(const std::string& valid, bool cpp, TokenList& tokenList)
 {
     std::istringstream istr(valid + ',');
-    tokenList.createTokens(istr);
+    tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C);
     for (Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (Token::Match(tok,"- %num%")) {
             tok->str("-" + tok->strAt(1));
@@ -927,7 +927,7 @@ bool Library::isIntArgValid(const Token *ftok, int argnr, const MathLib::bigint
     if (ac->valid.find('.') != std::string::npos)
         return isFloatArgValid(ftok, argnr, argvalue);
     TokenList tokenList(nullptr);
-    gettokenlistfromvalid(ac->valid, tokenList);
+    gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
     for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (tok->isNumber() && argvalue == MathLib::toBigNumber(tok->str()))
             return true;
@@ -947,7 +947,7 @@ bool Library::isFloatArgValid(const Token *ftok, int argnr, double argvalue) con
     if (!ac || ac->valid.empty())
         return true;
     TokenList tokenList(nullptr);
-    gettokenlistfromvalid(ac->valid, tokenList);
+    gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
     for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (Token::Match(tok, "%num% : %num%") && argvalue >= MathLib::toDoubleNumber(tok->str()) && argvalue <= MathLib::toDoubleNumber(tok->strAt(2)))
             return true;
@@ -1736,13 +1736,14 @@ bool Library::hasAnyTypeCheck(const std::string& typeName) const
 
 std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
                                                  const Settings* settings,
+                                                 bool cpp,
                                                  std::unordered_map<nonneg int, const Token*>* lookupVarId)
 {
     std::shared_ptr<TokenList> tokenList = std::make_shared<TokenList>(settings);
     {
         const std::string code = "return " + returnValue + ";";
         std::istringstream istr(code);
-        if (!tokenList->createTokens(istr))
+        if (!tokenList->createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C))
             return nullptr;
     }
 
diff --git a/lib/library.h b/lib/library.h
@@ -603,6 +603,7 @@ CPPCHECKLIB const Library::Container * getLibraryContainer(const Token * tok);
 
 std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
                                                  const Settings* settings,
+                                                 bool cpp,
                                                  std::unordered_map<nonneg int, const Token*>* lookupVarId = nullptr);
 
 /// @}
diff --git a/lib/programmemory.cpp b/lib/programmemory.cpp
@@ -1595,7 +1595,7 @@ namespace {
                                     arg_map[argn] = v;
                                 argn++;
                             }
-                            return evaluateLibraryFunction(arg_map, returnValue, settings);
+                            return evaluateLibraryFunction(arg_map, returnValue, settings, ftok->isCpp());
                         }
                     }
                 }
@@ -1751,15 +1751,16 @@ std::vector<ValueFlow::Value> execute(const Scope* scope, ProgramMemory& pm, con
 
 ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
                                          const std::string& returnValue,
-                                         const Settings* settings)
+                                         const Settings* settings,
+                                         bool cpp)
 {
     thread_local static std::unordered_map<std::string,
                                            std::function<ValueFlow::Value(const std::unordered_map<nonneg int, ValueFlow::Value>& arg)>>
     functions = {};
     if (functions.count(returnValue) == 0) {
 
         std::unordered_map<nonneg int, const Token*> lookupVarId;
-        std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, &lookupVarId);
+        std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, cpp, &lookupVarId);
 
         functions[returnValue] =
             [lookupVarId, expr, settings](const std::unordered_map<nonneg int, ValueFlow::Value>& xargs) {
diff --git a/lib/programmemory.h b/lib/programmemory.h
@@ -211,7 +211,8 @@ ProgramMemory getProgramMemory(const Token* tok, const Token* expr, const ValueF
 
 ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
                                          const std::string& returnValue,
-                                         const Settings* settings);
+                                         const Settings* settings,
+                                         bool cpp);
 
 #endif
 
diff --git a/lib/symboldatabase.cpp b/lib/symboldatabase.cpp
@@ -7487,7 +7487,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
                     ValueType valuetype;
                     TokenList tokenList(&mSettings);
                     std::istringstream istr(typestr+";");
-                    tokenList.createTokens(istr);
+                    tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C);
                     tokenList.simplifyStdType();
                     if (parsedecl(tokenList.front(), &valuetype, mDefaultSignedness, mSettings, mIsCpp)) {
                         valuetype.originalTypeName = typestr;
@@ -7577,7 +7577,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
                 }
                 TokenList tokenList(&mSettings);
                 std::istringstream istr(typestr+";");
-                if (tokenList.createTokens(istr)) {
+                if (tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C)) {
                     ValueType vt;
                     tokenList.simplifyPlatformTypes();
                     tokenList.simplifyStdType();
diff --git a/lib/token.cpp b/lib/token.cpp
@@ -2741,5 +2741,6 @@ bool Token::isCpp() const
     if (mTokensFrontBack && mTokensFrontBack->list) {
         return mTokensFrontBack->list->isCPP();
     }
+    assert(false);
     return true; // assume C++ by default
 }
diff --git a/lib/token.h b/lib/token.h
@@ -151,7 +151,7 @@ class CPPCHECKLIB Token {
     friend class TestToken;
 
 private:
-    TokensFrontBack* mTokensFrontBack{};
+    TokensFrontBack* const mTokensFrontBack{};
 
 public:
     Token(const Token &) = delete;
diff --git a/lib/tokenlist.cpp b/lib/tokenlist.cpp
@@ -30,6 +30,7 @@
 #include "standards.h"
 #include "token.h"
 
+#include <cassert>
 #include <cctype>
 #include <exception>
 #include <functional>
@@ -83,7 +84,7 @@ void TokenList::deallocateTokens()
 
 void TokenList::determineCppC()
 {
-    // only try to determine it if it wasn't enforced
+    // only try to determine if it wasn't enforced
     if (mLang == Standards::Language::None) {
         mLang = Path::identify(getSourceFilePath());
     }
@@ -317,8 +318,27 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n)
 
 bool TokenList::createTokens(std::istream &code, const std::string& file0)
 {
+    assert(!file0.empty());
+
     appendFileIfNew(file0);
 
+    return createTokensInternal(code, file0);
+}
+
+//---------------------------------------------------------------------------
+
+bool TokenList::createTokens(std::istream &code, Standards::Language lang)
+{
+    if (mLang == Standards::Language::None)
+        mLang = lang;
+
+    return createTokensInternal(code, "");
+}
+
+//---------------------------------------------------------------------------
+
+bool TokenList::createTokensInternal(std::istream &code, const std::string& file0)
+{
     simplecpp::OutputList outputList;
     simplecpp::TokenList tokens(code, mFiles, file0, &outputList);
 
@@ -2096,3 +2116,25 @@ bool TokenList::isKeyword(const std::string &str) const
     static const auto& latest_c_keywords = Keywords::getAll(Standards::cstd_t::CLatest);
     return latest_c_keywords.find(str) != latest_c_keywords.end();
 }
+
+bool TokenList::isC() const
+{
+    assert(mLang != Standards::Language::None);
+
+    return mLang == Standards::Language::C;
+}
+
+bool TokenList::isCPP() const
+{
+    assert(mLang != Standards::Language::None);
+
+    return mLang == Standards::Language::CPP;
+}
+
+void TokenList::setLang(Standards::Language lang)
+{
+    assert(lang != Standards::Language::None);
+    assert(mLang == Standards::Language::None);
+
+    mLang = lang;
+}
diff --git a/lib/tokenlist.h b/lib/tokenlist.h
@@ -61,14 +61,12 @@ class CPPCHECKLIB TokenList {
     const std::string& getSourceFilePath() const;
 
     /** Is the code C. Used for bailouts */
-    bool isC() const {
-        return mLang == Standards::Language::C;
-    }
+    bool isC() const;
 
     /** Is the code CPP. Used for bailouts */
-    bool isCPP() const {
-        return mLang == Standards::Language::CPP;
-    }
+    bool isCPP() const;
+
+    void setLang(Standards::Language lang);
 
     /**
      * Delete all tokens in given token list
@@ -104,7 +102,8 @@ class CPPCHECKLIB TokenList {
      * @param code input stream for code
      * @param file0 source file name
      */
-    bool createTokens(std::istream &code, const std::string& file0 = emptyString);
+    bool createTokens(std::istream &code, const std::string& file0);
+    bool createTokens(std::istream &code, Standards::Language lang);
 
     void createTokens(simplecpp::TokenList&& tokenList);
 
@@ -202,6 +201,8 @@ class CPPCHECKLIB TokenList {
 private:
     void determineCppC();
 
+    bool createTokensInternal(std::istream &code, const std::string& file0);
+
     /** Token list */
     TokensFrontBack mTokensFrontBack;
 
diff --git a/lib/valueflow.cpp b/lib/valueflow.cpp
@@ -3738,20 +3738,20 @@ static bool isNotEqual(std::pair<const Token*, const Token*> x, std::pair<const
     start2 = skipCVRefs(start2, y.second);
     return !(start1 == x.second && start2 == y.second);
 }
-static bool isNotEqual(std::pair<const Token*, const Token*> x, const std::string& y)
+static bool isNotEqual(std::pair<const Token*, const Token*> x, const std::string& y, bool cpp)
 {
     TokenList tokenList(nullptr);
     std::istringstream istr(y);
-    tokenList.createTokens(istr);
+    tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C);
     return isNotEqual(x, std::make_pair(tokenList.front(), tokenList.back()));
 }
-static bool isNotEqual(std::pair<const Token*, const Token*> x, const ValueType* y)
+static bool isNotEqual(std::pair<const Token*, const Token*> x, const ValueType* y, bool cpp)
 {
     if (y == nullptr)
         return false;
     if (y->originalTypeName.empty())
         return false;
-    return isNotEqual(x, y->originalTypeName);
+    return isNotEqual(x, y->originalTypeName, cpp);
 }
 
 static bool isDifferentType(const Token* src, const Token* dst)
@@ -3766,9 +3766,9 @@ static bool isDifferentType(const Token* src, const Token* dst)
         std::pair<const Token*, const Token*> parentdecl = Token::typeDecl(dst);
         if (isNotEqual(decl, parentdecl))
             return true;
-        if (isNotEqual(decl, dst->valueType()))
+        if (isNotEqual(decl, dst->valueType(), dst->isCpp()))
             return true;
-        if (isNotEqual(parentdecl, src->valueType()))
+        if (isNotEqual(parentdecl, src->valueType(), src->isCpp()))
             return true;
     }
     return false;
@@ -7590,7 +7590,7 @@ static void valueFlowLibraryFunction(Token *tok, const std::string &returnValue,
     if (returnValue.find("arg") != std::string::npos && argValues.empty())
         return;
     productParams(settings, argValues, [&](const std::unordered_map<nonneg int, ValueFlow::Value>& arg) {
-        ValueFlow::Value value = evaluateLibraryFunction(arg, returnValue, &settings);
+        ValueFlow::Value value = evaluateLibraryFunction(arg, returnValue, &settings, tok->isCpp());
         if (value.isUninitValue())
             return;
         ValueFlow::Value::ValueKind kind = ValueFlow::Value::ValueKind::Known;
@@ -9084,11 +9084,11 @@ static bool getMinMaxValues(const ValueType *vt, const Platform &platform, MathL
     return true;
 }
 
-static bool getMinMaxValues(const std::string &typestr, const Settings &settings, MathLib::bigint &minvalue, MathLib::bigint &maxvalue)
+static bool getMinMaxValues(const std::string &typestr, const Settings &settings, bool cpp, MathLib::bigint &minvalue, MathLib::bigint &maxvalue)
 {
     TokenList typeTokens(&settings);
     std::istringstream istr(typestr+";");
-    if (!typeTokens.createTokens(istr))
+    if (!typeTokens.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C))
         return false;
     typeTokens.simplifyPlatformTypes();
     typeTokens.simplifyStdType();
@@ -9208,7 +9208,7 @@ static void valueFlowUnknownFunctionReturn(TokenList &tokenlist, const Settings
         // Get min/max values for return type
         const std::string &typestr = settings.library.returnValueType(tok->previous());
         MathLib::bigint minvalue, maxvalue;
-        if (!getMinMaxValues(typestr, settings, minvalue, maxvalue))
+        if (!getMinMaxValues(typestr, settings, tok->isCpp(), minvalue, maxvalue))
             continue;
 
         for (MathLib::bigint value : unknownValues) {
diff --git a/test/testlibrary.cpp b/test/testlibrary.cpp
diff --git a/test/testsimplifytypedef.cpp b/test/testsimplifytypedef.cpp

Original file line number	Diff line number	Diff line change
`@@ -1595,7 +1595,7 @@ namespace {`
`1595`	`1595`	`arg_map[argn] = v;`
`1596`	`1596`	`argn++;`
`1597`	`1597`	`}`
`1598`		`- return evaluateLibraryFunction(arg_map, returnValue, settings);`
	`1598`	`+ return evaluateLibraryFunction(arg_map, returnValue, settings, ftok->isCpp());`
`1599`	`1599`	`}`
`1600`	`1600`	`}`
`1601`	`1601`	`}`
`@@ -1751,15 +1751,16 @@ std::vector<ValueFlow::Value> execute(const Scope* scope, ProgramMemory& pm, con`
`1751`	`1751`
`1752`	`1752`	`ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,`
`1753`	`1753`	`const std::string& returnValue,`
`1754`		`- const Settings* settings)`
	`1754`	`+ const Settings* settings,`
	`1755`	`+ bool cpp)`
`1755`	`1756`	`{`
`1756`	`1757`	`thread_local static std::unordered_map<std::string,`
`1757`	`1758`	`std::function<ValueFlow::Value(const std::unordered_map<nonneg int, ValueFlow::Value>& arg)>>`
`1758`	`1759`	`functions = {};`
`1759`	`1760`	`if (functions.count(returnValue) == 0) {`
`1760`	`1761`
`1761`	`1762`	`std::unordered_map<nonneg int, const Token*> lookupVarId;`
`1762`		`- std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, &lookupVarId);`
	`1763`	`+ std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, cpp, &lookupVarId);`
`1763`	`1764`
`1764`	`1765`	`functions[returnValue] =`
`1765`	`1766`	`[lookupVarId, expr, settings](const std::unordered_map<nonneg int, ValueFlow::Value>& xargs) {`
Original file line number	Diff line number	Diff line change
`@@ -2741,5 +2741,6 @@ bool Token::isCpp() const`
`2741`	`2741`	`if (mTokensFrontBack && mTokensFrontBack->list) {`
`2742`	`2742`	`return mTokensFrontBack->list->isCPP();`
`2743`	`2743`	`}`
	`2744`	`+ assert(false);`
`2744`	`2745`	`return true; // assume C++ by default`
`2745`	`2746`	`}`