Skip to content

Commit c78d7c9

Browse files
firewavefirewave
committed
make sure the language for a TokenList is always determined [skip ci]
1 parent c09158d commit c78d7c9

12 files changed

Lines changed: 97 additions & 46 deletions

‎lib/clangimport.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,7 @@ namespace clangimport {
334334
setLocations(tokenList, 0, 1, 1);
335335
else
336336
setLocations(tokenList, tokenList->back()->fileIndex(), tokenList->back()->linenr(), 1);
337+
// TODO: language might not be set for the tokens if no file is specified in the AST dump
337338
createTokens(tokenList);
338339
if (nodeType == VarDecl || nodeType == RecordDecl || nodeType == TypedefDecl)
339340
addtoken(tokenList, ";");
@@ -626,6 +627,7 @@ void clangimport::AstNode::setValueType(Token *tok)
626627
continue;
627628

628629
TokenList decl(nullptr);
630+
decl.setLang(tok->isCpp() ? Settings::Language::CPP : Settings::Language::C);
629631
addTypeTokens(&decl, type, tok->scope());
630632
if (!decl.front())
631633
break;

‎lib/cppcheck.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -794,7 +794,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
794794
}
795795
Tokenizer tokenizer2(&mSettings, this);
796796
std::istringstream istr2(code);
797-
tokenizer2.list.createTokens(istr2);
797+
tokenizer2.list.createTokens(istr2, Path::isCPP(*files.begin()) ? Settings::Language::CPP : Settings::Language::C);
798798
executeRules("define", tokenizer2);
799799
}
800800
#endif

‎lib/library.cpp‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,10 @@ static std::vector<std::string> getnames(const char *names)
5353
return ret;
5454
}
5555

56-
static void gettokenlistfromvalid(const std::string& valid, TokenList& tokenList)
56+
static void gettokenlistfromvalid(const std::string& valid, bool cpp, TokenList& tokenList)
5757
{
5858
std::istringstream istr(valid + ',');
59-
tokenList.createTokens(istr);
59+
tokenList.createTokens(istr, cpp ? Settings::Language::CPP : Settings::Language::C);
6060
for (Token *tok = tokenList.front(); tok; tok = tok->next()) {
6161
if (Token::Match(tok,"- %num%")) {
6262
tok->str("-" + tok->strAt(1));
@@ -920,7 +920,7 @@ bool Library::isIntArgValid(const Token *ftok, int argnr, const MathLib::bigint
920920
if (ac->valid.find('.') != std::string::npos)
921921
return isFloatArgValid(ftok, argnr, argvalue);
922922
TokenList tokenList(nullptr);
923-
gettokenlistfromvalid(ac->valid, tokenList);
923+
gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
924924
for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
925925
if (tok->isNumber() && argvalue == MathLib::toBigNumber(tok->str()))
926926
return true;
@@ -940,7 +940,7 @@ bool Library::isFloatArgValid(const Token *ftok, int argnr, double argvalue) con
940940
if (!ac || ac->valid.empty())
941941
return true;
942942
TokenList tokenList(nullptr);
943-
gettokenlistfromvalid(ac->valid, tokenList);
943+
gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
944944
for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
945945
if (Token::Match(tok, "%num% : %num%") && argvalue >= MathLib::toDoubleNumber(tok->str()) && argvalue <= MathLib::toDoubleNumber(tok->strAt(2)))
946946
return true;
@@ -1725,13 +1725,14 @@ bool Library::hasAnyTypeCheck(const std::string& typeName) const
17251725

17261726
std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
17271727
const Settings* settings,
1728+
bool cpp,
17281729
std::unordered_map<nonneg int, const Token*>* lookupVarId)
17291730
{
17301731
std::shared_ptr<TokenList> tokenList = std::make_shared<TokenList>(settings);
17311732
{
17321733
const std::string code = "return " + returnValue + ";";
17331734
std::istringstream istr(code);
1734-
if (!tokenList->createTokens(istr))
1735+
if (!tokenList->createTokens(istr, cpp ? Settings::Language::CPP : Settings::Language::C))
17351736
return nullptr;
17361737
}
17371738

‎lib/library.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -603,6 +603,7 @@ CPPCHECKLIB const Library::Container * getLibraryContainer(const Token * tok);
603603

604604
std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
605605
const Settings* settings,
606+
bool cpp,
606607
std::unordered_map<nonneg int, const Token*>* lookupVarId = nullptr);
607608

608609
/// @}

‎lib/programmemory.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1596,7 +1596,7 @@ namespace {
15961596
arg_map[argn] = v;
15971597
argn++;
15981598
}
1599-
return evaluateLibraryFunction(arg_map, returnValue, settings);
1599+
return evaluateLibraryFunction(arg_map, returnValue, settings, ftok->isCpp());
16001600
}
16011601
}
16021602
}
@@ -1752,15 +1752,16 @@ std::vector<ValueFlow::Value> execute(const Scope* scope, ProgramMemory& pm, con
17521752

17531753
ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
17541754
const std::string& returnValue,
1755-
const Settings* settings)
1755+
const Settings* settings,
1756+
bool cpp)
17561757
{
17571758
thread_local static std::unordered_map<std::string,
17581759
std::function<ValueFlow::Value(const std::unordered_map<nonneg int, ValueFlow::Value>& arg)>>
17591760
functions = {};
17601761
if (functions.count(returnValue) == 0) {
17611762

17621763
std::unordered_map<nonneg int, const Token*> lookupVarId;
1763-
std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, &lookupVarId);
1764+
std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, cpp, &lookupVarId);
17641765

17651766
functions[returnValue] =
17661767
[lookupVarId, expr, settings](const std::unordered_map<nonneg int, ValueFlow::Value>& xargs) {

‎lib/programmemory.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,8 @@ ProgramMemory getProgramMemory(const Token* tok, const Token* expr, const ValueF
211211

212212
ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
213213
const std::string& returnValue,
214-
const Settings* settings);
214+
const Settings* settings,
215+
bool cpp);
215216

216217
#endif
217218

‎lib/symboldatabase.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7479,7 +7479,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
74797479
ValueType valuetype;
74807480
TokenList tokenList(&mSettings);
74817481
std::istringstream istr(typestr+";");
7482-
tokenList.createTokens(istr);
7482+
tokenList.createTokens(istr, tok->isCpp() ? Settings::Language::CPP : Settings::Language::C);
74837483
tokenList.simplifyStdType();
74847484
if (parsedecl(tokenList.front(), &valuetype, mDefaultSignedness, mSettings, mIsCpp)) {
74857485
valuetype.originalTypeName = typestr;
@@ -7569,7 +7569,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
75697569
}
75707570
TokenList tokenList(&mSettings);
75717571
std::istringstream istr(typestr+";");
7572-
if (tokenList.createTokens(istr)) {
7572+
if (tokenList.createTokens(istr, tok->isCpp() ? Settings::Language::CPP : Settings::Language::C)) {
75737573
ValueType vt;
75747574
tokenList.simplifyPlatformTypes();
75757575
tokenList.simplifyStdType();

‎lib/tokenlist.cpp‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include "standards.h"
3131
#include "token.h"
3232

33+
#include <cassert>
3334
#include <cctype>
3435
#include <exception>
3536
#include <functional>
@@ -89,6 +90,8 @@ void TokenList::determineCppC()
8990
mLang = Settings::Language::C;
9091
else if (Path::isCPP(getSourceFilePath()))
9192
mLang = Settings::Language::CPP;
93+
94+
assert(mLang != Settings::Language::None);
9295
}
9396
}
9497

@@ -320,8 +323,27 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n)
320323

321324
bool TokenList::createTokens(std::istream &code, const std::string& file0)
322325
{
326+
assert(!file0.empty());
327+
323328
appendFileIfNew(file0);
324329

330+
return createTokensInternal(code, file0);
331+
}
332+
333+
//---------------------------------------------------------------------------
334+
335+
bool TokenList::createTokens(std::istream &code, Settings::Language lang)
336+
{
337+
if (mLang == Settings::Language::None)
338+
mLang = lang;
339+
340+
return createTokensInternal(code, "");
341+
}
342+
343+
//---------------------------------------------------------------------------
344+
345+
bool TokenList::createTokensInternal(std::istream &code, const std::string& file0)
346+
{
325347
simplecpp::OutputList outputList;
326348
simplecpp::TokenList tokens(code, mFiles, file0, &outputList);
327349

@@ -2090,3 +2112,25 @@ bool TokenList::isKeyword(const std::string &str) const
20902112
static const auto& latest_c_keywords = Keywords::getAll(Standards::cstd_t::CLatest);
20912113
return latest_c_keywords.find(str) != latest_c_keywords.end();
20922114
}
2115+
2116+
bool TokenList::isC() const
2117+
{
2118+
assert(mLang != Settings::Language::None);
2119+
2120+
return mLang == Settings::Language::C;
2121+
}
2122+
2123+
bool TokenList::isCPP() const
2124+
{
2125+
assert(mLang != Settings::Language::None);
2126+
2127+
return mLang == Settings::Language::CPP;
2128+
}
2129+
2130+
void TokenList::setLang(Settings::Language lang)
2131+
{
2132+
assert(lang != Settings::Language::None);
2133+
assert(mLang == Settings::Language::None);
2134+
2135+
mLang = lang;
2136+
}

‎lib/tokenlist.h‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -60,14 +60,12 @@ class CPPCHECKLIB TokenList {
6060
const std::string& getSourceFilePath() const;
6161

6262
/** Is the code C. Used for bailouts */
63-
bool isC() const {
64-
return mLang == Settings::Language::C;
65-
}
63+
bool isC() const;
6664

6765
/** Is the code CPP. Used for bailouts */
68-
bool isCPP() const {
69-
return mLang == Settings::Language::CPP;
70-
}
66+
bool isCPP() const;
67+
68+
void setLang(Settings::Language lang);
7169

7270
/**
7371
* Delete all tokens in given token list
@@ -103,7 +101,8 @@ class CPPCHECKLIB TokenList {
103101
* @param code input stream for code
104102
* @param file0 source file name
105103
*/
106-
bool createTokens(std::istream &code, const std::string& file0 = emptyString);
104+
bool createTokens(std::istream &code, const std::string& file0);
105+
bool createTokens(std::istream &code, Settings::Language lang);
107106

108107
void createTokens(simplecpp::TokenList&& tokenList);
109108

@@ -201,6 +200,8 @@ class CPPCHECKLIB TokenList {
201200
private:
202201
void determineCppC();
203202

203+
bool createTokensInternal(std::istream &code, const std::string& file0);
204+
204205
/** Token list */
205206
TokensFrontBack mTokensFrontBack;
206207

‎lib/valueflow.cpp‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3738,20 +3738,20 @@ static bool isNotEqual(std::pair<const Token*, const Token*> x, std::pair<const
37383738
start2 = skipCVRefs(start2, y.second);
37393739
return !(start1 == x.second && start2 == y.second);
37403740
}
3741-
static bool isNotEqual(std::pair<const Token*, const Token*> x, const std::string& y)
3741+
static bool isNotEqual(std::pair<const Token*, const Token*> x, const std::string& y, bool cpp)
37423742
{
37433743
TokenList tokenList(nullptr);
37443744
std::istringstream istr(y);
3745-
tokenList.createTokens(istr);
3745+
tokenList.createTokens(istr, cpp ? Settings::Language::CPP : Settings::Language::C);
37463746
return isNotEqual(x, std::make_pair(tokenList.front(), tokenList.back()));
37473747
}
3748-
static bool isNotEqual(std::pair<const Token*, const Token*> x, const ValueType* y)
3748+
static bool isNotEqual(std::pair<const Token*, const Token*> x, const ValueType* y, bool cpp)
37493749
{
37503750
if (y == nullptr)
37513751
return false;
37523752
if (y->originalTypeName.empty())
37533753
return false;
3754-
return isNotEqual(x, y->originalTypeName);
3754+
return isNotEqual(x, y->originalTypeName, cpp);
37553755
}
37563756

37573757
static bool isDifferentType(const Token* src, const Token* dst)
@@ -3766,9 +3766,9 @@ static bool isDifferentType(const Token* src, const Token* dst)
37663766
std::pair<const Token*, const Token*> parentdecl = Token::typeDecl(dst);
37673767
if (isNotEqual(decl, parentdecl))
37683768
return true;
3769-
if (isNotEqual(decl, dst->valueType()))
3769+
if (isNotEqual(decl, dst->valueType(), dst->isCpp()))
37703770
return true;
3771-
if (isNotEqual(parentdecl, src->valueType()))
3771+
if (isNotEqual(parentdecl, src->valueType(), src->isCpp()))
37723772
return true;
37733773
}
37743774
return false;
@@ -7592,7 +7592,7 @@ static void valueFlowLibraryFunction(Token *tok, const std::string &returnValue,
75927592
if (returnValue.find("arg") != std::string::npos && argValues.empty())
75937593
return;
75947594
productParams(settings, argValues, [&](const std::unordered_map<nonneg int, ValueFlow::Value>& arg) {
7595-
ValueFlow::Value value = evaluateLibraryFunction(arg, returnValue, settings);
7595+
ValueFlow::Value value = evaluateLibraryFunction(arg, returnValue, settings, tok->isCpp());
75967596
if (value.isUninitValue())
75977597
return;
75987598
ValueFlow::Value::ValueKind kind = ValueFlow::Value::ValueKind::Known;
@@ -9086,11 +9086,11 @@ static bool getMinMaxValues(const ValueType *vt, const Platform &platform, MathL
90869086
return true;
90879087
}
90889088

9089-
static bool getMinMaxValues(const std::string &typestr, const Settings *settings, MathLib::bigint &minvalue, MathLib::bigint &maxvalue)
9089+
static bool getMinMaxValues(const std::string &typestr, const Settings *settings, bool cpp, MathLib::bigint &minvalue, MathLib::bigint &maxvalue)
90909090
{
90919091
TokenList typeTokens(settings);
90929092
std::istringstream istr(typestr+";");
9093-
if (!typeTokens.createTokens(istr))
9093+
if (!typeTokens.createTokens(istr, cpp ? Settings::Language::CPP : Settings::Language::C))
90949094
return false;
90959095
typeTokens.simplifyPlatformTypes();
90969096
typeTokens.simplifyStdType();
@@ -9210,7 +9210,7 @@ static void valueFlowUnknownFunctionReturn(TokenList &tokenlist, const Settings
92109210
// Get min/max values for return type
92119211
const std::string &typestr = settings->library.returnValueType(tok->previous());
92129212
MathLib::bigint minvalue, maxvalue;
9213-
if (!getMinMaxValues(typestr, settings, minvalue, maxvalue))
9213+
if (!getMinMaxValues(typestr, settings, tok->isCpp(), minvalue, maxvalue))
92149214
continue;
92159215

92169216
for (MathLib::bigint value : unknownValues) {

0 commit comments

Comments
 (0)