Skip to content

Commit 838bb4f

Browse files
firewavefirewave
committed
Tokenizer: removed Preprocessor dependency
1 parent 44cb133 commit 838bb4f

12 files changed

Lines changed: 36 additions & 100 deletions

‎lib/cppcheck.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -506,8 +506,7 @@ unsigned int CppCheck::checkClang(const std::string &path)
506506
}
507507

508508
try {
509-
Preprocessor preprocessor(mSettings, this);
510-
Tokenizer tokenizer(mSettings, this, &preprocessor);
509+
Tokenizer tokenizer(mSettings, this);
511510
tokenizer.list.appendFileIfNew(path);
512511
std::istringstream ast(output2);
513512
clangimport::parseClangAstDump(tokenizer, ast);
@@ -887,7 +886,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
887886
continue;
888887
}
889888

890-
Tokenizer tokenizer(mSettings, this, &preprocessor);
889+
Tokenizer tokenizer(mSettings, this);
891890
if (mSettings.showtime != SHOWTIME_MODES::SHOWTIME_NONE)
892891
tokenizer.setTimerResults(&s_timerResults);
893892
tokenizer.setDirectives(directives); // TODO: how to avoid repeated copies?

‎lib/preprocessor.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ struct CPPCHECKLIB Directive {
6868
* The preprocessor has special functionality for extracting the various ifdef
6969
* configurations that exist in a source file.
7070
*/
71-
class CPPCHECKLIB Preprocessor {
71+
class CPPCHECKLIB WARN_UNUSED Preprocessor {
7272
// TODO: get rid of this
7373
friend class PreprocessorHelper;
7474
friend class TestPreprocessor;

‎lib/tokenize.cpp‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -152,12 +152,11 @@ static bool isClassStructUnionEnumStart(const Token * tok)
152152

153153
//---------------------------------------------------------------------------
154154

155-
Tokenizer::Tokenizer(const Settings &settings, ErrorLogger *errorLogger, const Preprocessor *preprocessor) :
155+
Tokenizer::Tokenizer(const Settings &settings, ErrorLogger *errorLogger) :
156156
list(&settings),
157157
mSettings(settings),
158158
mErrorLogger(errorLogger),
159-
mTemplateSimplifier(new TemplateSimplifier(*this)),
160-
mPreprocessor(preprocessor)
159+
mTemplateSimplifier(new TemplateSimplifier(*this))
161160
{}
162161

163162
Tokenizer::~Tokenizer()
@@ -10676,8 +10675,6 @@ void Tokenizer::setDirectives(std::list<Directive> directives)
1067610675

1067710676
bool Tokenizer::hasIfdef(const Token *start, const Token *end) const
1067810677
{
10679-
assert(mPreprocessor);
10680-
1068110678
const auto& directives = mDirectives;
1068210679
return std::any_of(directives.cbegin(), directives.cend(), [&](const Directive& d) {
1068310680
return startsWith(d.str, "#if") &&
@@ -10690,8 +10687,6 @@ bool Tokenizer::hasIfdef(const Token *start, const Token *end) const
1069010687

1069110688
bool Tokenizer::isPacked(const Token * bodyStart) const
1069210689
{
10693-
assert(mPreprocessor);
10694-
1069510690
const auto& directives = mDirectives;
1069610691
// TODO: should this return true if the #pragma exists in any line before the start token?
1069710692
return std::any_of(directives.cbegin(), directives.cend(), [&](const Directive& d) {

‎lib/tokenize.h‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ class TimerResults;
3636
class Token;
3737
class TemplateSimplifier;
3838
class ErrorLogger;
39-
class Preprocessor;
4039
struct Directive;
4140
enum class Severity;
4241

@@ -54,7 +53,7 @@ class CPPCHECKLIB Tokenizer {
5453
friend class TestTokenizer;
5554

5655
public:
57-
explicit Tokenizer(const Settings & settings, ErrorLogger *errorLogger, const Preprocessor *preprocessor = nullptr);
56+
explicit Tokenizer(const Settings & settings, ErrorLogger *errorLogger);
5857
~Tokenizer();
5958

6059
void setTimerResults(TimerResults *tr) {
@@ -681,8 +680,6 @@ class CPPCHECKLIB Tokenizer {
681680
* TimerResults
682681
*/
683682
TimerResults* mTimerResults{};
684-
685-
const Preprocessor * const mPreprocessor;
686683
};
687684

688685
/// @}

‎test/helpers.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@
4545
class SuppressionList;
4646

4747
const Settings SimpleTokenizer::s_settings;
48-
const Preprocessor SimpleTokenizer::s_preprocessor{s_settings, nullptr}; // TODO: provide ErrorLogger
4948

5049
// TODO: better path-only usage
5150
ScopedFile::ScopedFile(std::string name, const std::string &content, std::string path)

‎test/helpers.h‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -42,18 +42,14 @@ namespace simplecpp {
4242
class SimpleTokenizer : public Tokenizer {
4343
public:
4444
SimpleTokenizer(ErrorLogger& errorlogger, const char code[], bool cpp = true)
45-
: Tokenizer{s_settings, &errorlogger, &s_preprocessor}
45+
: Tokenizer{s_settings, &errorlogger}
4646
{
4747
if (!tokenize(code, cpp))
4848
throw std::runtime_error("creating tokens failed");
4949
}
5050

5151
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger)
52-
: Tokenizer{settings, &errorlogger, &s_preprocessor}
53-
{}
54-
55-
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, const Preprocessor* preprocessor)
56-
: Tokenizer{settings, &errorlogger, preprocessor}
52+
: Tokenizer{settings, &errorlogger}
5753
{}
5854

5955
/*
@@ -87,7 +83,6 @@ class SimpleTokenizer : public Tokenizer {
8783
private:
8884
// TODO. find a better solution
8985
static const Settings s_settings;
90-
static const Preprocessor s_preprocessor;
9186
};
9287

9388
class SimpleTokenList

‎test/testclass.cpp‎

Lines changed: 18 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
#include "errortypes.h"
2222
#include "fixture.h"
2323
#include "helpers.h"
24-
#include "preprocessor.h"
2524
#include "settings.h"
2625
#include "tokenize.h"
2726

@@ -248,10 +247,8 @@ class TestClass : public TestFixture {
248247
void checkCopyCtorAndEqOperator_(const char code[], const char* file, int line) {
249248
const Settings settings = settingsBuilder().severity(Severity::warning).build();
250249

251-
Preprocessor preprocessor(settings);
252-
253250
// Tokenize..
254-
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
251+
SimpleTokenizer tokenizer(settings, *this);
255252
ASSERT_LOC(tokenizer.tokenize(code), file, line);
256253

257254
// Check..
@@ -350,10 +347,8 @@ class TestClass : public TestFixture {
350347

351348
#define checkExplicitConstructors(code) checkExplicitConstructors_(code, __FILE__, __LINE__)
352349
void checkExplicitConstructors_(const char code[], const char* file, int line) {
353-
Preprocessor preprocessor(settings0);
354-
355350
// Tokenize..
356-
SimpleTokenizer tokenizer(settings0, *this, &preprocessor);
351+
SimpleTokenizer tokenizer(settings0, *this);
357352
ASSERT_LOC(tokenizer.tokenize(code), file, line);
358353

359354
// Check..
@@ -500,10 +495,8 @@ class TestClass : public TestFixture {
500495

501496
#define checkDuplInheritedMembers(code) checkDuplInheritedMembers_(code, __FILE__, __LINE__)
502497
void checkDuplInheritedMembers_(const char code[], const char* file, int line) {
503-
Preprocessor preprocessor(settings1);
504-
505498
// Tokenize..
506-
SimpleTokenizer tokenizer(settings1, *this, &preprocessor);
499+
SimpleTokenizer tokenizer(settings1, *this);
507500
ASSERT_LOC(tokenizer.tokenize(code), file, line);
508501

509502
// Check..
@@ -718,10 +711,8 @@ class TestClass : public TestFixture {
718711

719712
#define checkCopyConstructor(code) checkCopyConstructor_(code, __FILE__, __LINE__)
720713
void checkCopyConstructor_(const char code[], const char* file, int line) {
721-
Preprocessor preprocessor(settings3);
722-
723714
// Tokenize..
724-
SimpleTokenizer tokenizer(settings3, *this, &preprocessor);
715+
SimpleTokenizer tokenizer(settings3, *this);
725716
ASSERT_LOC(tokenizer.tokenize(code), file, line);
726717

727718
// Check..
@@ -1163,10 +1154,8 @@ class TestClass : public TestFixture {
11631154
// Check that operator Equal returns reference to this
11641155
#define checkOpertorEqRetRefThis(code) checkOpertorEqRetRefThis_(code, __FILE__, __LINE__)
11651156
void checkOpertorEqRetRefThis_(const char code[], const char* file, int line) {
1166-
Preprocessor preprocessor(settings0);
1167-
11681157
// Tokenize..
1169-
SimpleTokenizer tokenizer(settings0, *this, &preprocessor);
1158+
SimpleTokenizer tokenizer(settings0, *this);
11701159
ASSERT_LOC(tokenizer.tokenize(code), file, line);
11711160

11721161
// Check..
@@ -1635,10 +1624,8 @@ class TestClass : public TestFixture {
16351624
// Check that operator Equal checks for assignment to self
16361625
#define checkOpertorEqToSelf(code) checkOpertorEqToSelf_(code, __FILE__, __LINE__)
16371626
void checkOpertorEqToSelf_(const char code[], const char* file, int line) {
1638-
Preprocessor preprocessor(settings1);
1639-
16401627
// Tokenize..
1641-
SimpleTokenizer tokenizer(settings1, *this, &preprocessor);
1628+
SimpleTokenizer tokenizer(settings1, *this);
16421629
ASSERT_LOC(tokenizer.tokenize(code), file, line);
16431630

16441631
// Check..
@@ -2594,10 +2581,8 @@ class TestClass : public TestFixture {
25942581
void checkVirtualDestructor_(const char* file, int line, const char code[], bool inconclusive = false) {
25952582
const Settings s = settingsBuilder(settings0).certainty(Certainty::inconclusive, inconclusive).severity(Severity::warning).build();
25962583

2597-
Preprocessor preprocessor(s);
2598-
25992584
// Tokenize..
2600-
SimpleTokenizer tokenizer(s, *this, &preprocessor);
2585+
SimpleTokenizer tokenizer(s, *this);
26012586
ASSERT_LOC(tokenizer.tokenize(code), file, line);
26022587

26032588
// Check..
@@ -2932,10 +2917,8 @@ class TestClass : public TestFixture {
29322917
}
29332918

29342919
void checkNoMemset_(const char* file, int line, const char code[], const Settings &settings) {
2935-
Preprocessor preprocessor(settings);
2936-
29372920
// Tokenize..
2938-
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
2921+
SimpleTokenizer tokenizer(settings, *this);
29392922
ASSERT_LOC(tokenizer.tokenize(code), file, line);
29402923

29412924
// Check..
@@ -3579,10 +3562,8 @@ class TestClass : public TestFixture {
35793562

35803563
#define checkThisSubtraction(code) checkThisSubtraction_(code, __FILE__, __LINE__)
35813564
void checkThisSubtraction_(const char code[], const char* file, int line) {
3582-
Preprocessor preprocessor(settings1);
3583-
35843565
// Tokenize..
3585-
SimpleTokenizer tokenizer(settings1, *this, &preprocessor);
3566+
SimpleTokenizer tokenizer(settings1, *this);
35863567
ASSERT_LOC(tokenizer.tokenize(code), file, line);
35873568

35883569
// Check..
@@ -3612,10 +3593,8 @@ class TestClass : public TestFixture {
36123593
void checkConst_(const char* file, int line, const char code[], const Settings *s = nullptr, bool inconclusive = true) {
36133594
const Settings settings = settingsBuilder(s ? *s : settings0).certainty(Certainty::inconclusive, inconclusive).build();
36143595

3615-
Preprocessor preprocessor(settings);
3616-
36173596
// Tokenize..
3618-
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
3597+
SimpleTokenizer tokenizer(settings, *this);
36193598
ASSERT_LOC(tokenizer.tokenize(code), file, line);
36203599

36213600
CheckClass checkClass(&tokenizer, &settings, this);
@@ -7532,11 +7511,8 @@ class TestClass : public TestFixture {
75327511

75337512
#define checkInitializerListOrder(code) checkInitializerListOrder_(code, __FILE__, __LINE__)
75347513
void checkInitializerListOrder_(const char code[], const char* file, int line) {
7535-
// Check..
7536-
Preprocessor preprocessor(settings2);
7537-
75387514
// Tokenize..
7539-
SimpleTokenizer tokenizer(settings2, *this, &preprocessor);
7515+
SimpleTokenizer tokenizer(settings2, *this);
75407516
ASSERT_LOC(tokenizer.tokenize(code), file, line);
75417517

75427518
CheckClass checkClass(&tokenizer, &settings2, this);
@@ -7663,10 +7639,8 @@ class TestClass : public TestFixture {
76637639
// Check..
76647640
const Settings settings = settingsBuilder().severity(Severity::performance).build();
76657641

7666-
Preprocessor preprocessor(settings);
7667-
76687642
// Tokenize..
7669-
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
7643+
SimpleTokenizer tokenizer(settings, *this);
76707644
ASSERT_LOC(tokenizer.tokenize(code), file, line);
76717645

76727646
CheckClass checkClass(&tokenizer, &settings, this);
@@ -7873,10 +7847,8 @@ class TestClass : public TestFixture {
78737847

78747848
#define checkSelfInitialization(code) checkSelfInitialization_(code, __FILE__, __LINE__)
78757849
void checkSelfInitialization_(const char code[], const char* file, int line) {
7876-
Preprocessor preprocessor(settings0);
7877-
78787850
// Tokenize..
7879-
SimpleTokenizer tokenizer(settings0, *this, &preprocessor);
7851+
SimpleTokenizer tokenizer(settings0, *this);
78807852
ASSERT_LOC(tokenizer.tokenize(code), file, line);
78817853

78827854
CheckClass checkClass(&tokenizer, &settings0, this);
@@ -7985,10 +7957,8 @@ class TestClass : public TestFixture {
79857957
// Check..
79867958
const Settings settings = settingsBuilder().severity(Severity::warning).severity(Severity::style).certainty(Certainty::inconclusive, inconclusive).build();
79877959

7988-
Preprocessor preprocessor(settings);
7989-
79907960
// Tokenize..
7991-
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
7961+
SimpleTokenizer tokenizer(settings, *this);
79927962
ASSERT_LOC(tokenizer.tokenize(code), file, line);
79937963

79947964
CheckClass checkClass(&tokenizer, &settings, this);
@@ -8330,10 +8300,8 @@ class TestClass : public TestFixture {
83308300
void checkOverride_(const char code[], const char* file, int line) {
83318301
const Settings settings = settingsBuilder().severity(Severity::style).build();
83328302

8333-
Preprocessor preprocessor(settings);
8334-
83358303
// Tokenize..
8336-
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
8304+
SimpleTokenizer tokenizer(settings, *this);
83378305
ASSERT_LOC(tokenizer.tokenize(code), file, line);
83388306

83398307
// Check..
@@ -8704,10 +8672,8 @@ class TestClass : public TestFixture {
87048672
/*const*/ Settings settings = settingsBuilder().severity(Severity::warning).build();
87058673
settings.safeChecks.classes = true;
87068674

8707-
Preprocessor preprocessor(settings);
8708-
87098675
// Tokenize..
8710-
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
8676+
SimpleTokenizer tokenizer(settings, *this);
87118677
ASSERT_LOC(tokenizer.tokenize(code), file, line);
87128678

87138679
// Check..
@@ -8723,10 +8689,8 @@ class TestClass : public TestFixture {
87238689

87248690
#define checkThisUseAfterFree(code) checkThisUseAfterFree_(code, __FILE__, __LINE__)
87258691
void checkThisUseAfterFree_(const char code[], const char* file, int line) {
8726-
Preprocessor preprocessor(settings1);
8727-
87288692
// Tokenize..
8729-
SimpleTokenizer tokenizer(settings1, *this, &preprocessor);
8693+
SimpleTokenizer tokenizer(settings1, *this);
87308694
ASSERT_LOC(tokenizer.tokenize(code), file, line);
87318695

87328696
// Check..
@@ -8924,10 +8888,8 @@ class TestClass : public TestFixture {
89248888

89258889
#define getFileInfo(code) getFileInfo_(code, __FILE__, __LINE__)
89268890
void getFileInfo_(const char code[], const char* file, int line) {
8927-
Preprocessor preprocessor(settings1);
8928-
89298891
// Tokenize..
8930-
SimpleTokenizer tokenizer(settings1, *this, &preprocessor);
8892+
SimpleTokenizer tokenizer(settings1, *this);
89318893
ASSERT_LOC(tokenizer.tokenize(code), file, line);
89328894

89338895
// Check..

‎test/testcondition.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ class TestCondition : public TestFixture {
129129
void check_(const char* file, int line, const char code[], const Settings &settings, const char* filename = "test.cpp") {
130130
Preprocessor preprocessor(settings);
131131
std::vector<std::string> files(1, filename);
132-
Tokenizer tokenizer(settings, this, &preprocessor);
132+
Tokenizer tokenizer(settings, this);
133133
PreprocessorHelper::preprocess(preprocessor, code, files, tokenizer);
134134

135135
// Tokenizer..

‎test/testgarbage.cpp‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
#include "errortypes.h"
2121
#include "fixture.h"
2222
#include "helpers.h"
23-
#include "preprocessor.h"
2423
#include "settings.h"
2524
#include "token.h"
2625

@@ -283,10 +282,8 @@ class TestGarbage : public TestFixture {
283282
}
284283

285284
std::string checkCodeInternal_(const char code[], bool cpp, const char* file, int line) {
286-
Preprocessor preprocessor(settings);
287-
288285
// tokenize..
289-
SimpleTokenizer tokenizer(settings, *this, &preprocessor);
286+
SimpleTokenizer tokenizer(settings, *this);
290287
ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);
291288

292289
// call all "runChecks" in all registered Check classes

0 commit comments

Comments
 (0)