Skip to content

Commit 390bdfa

Browse files
firewavefirewave
committed
make sure the language for a TokenList is always determined [skip ci]
1 parent bbd2b2a commit 390bdfa

14 files changed

Lines changed: 102 additions & 48 deletions

‎lib/clangimport.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,7 @@ namespace clangimport {
334334
setLocations(tokenList, 0, 1, 1);
335335
else
336336
setLocations(tokenList, tokenList->back()->fileIndex(), tokenList->back()->linenr(), 1);
337+
// TODO: language might not be set for the tokens if no file is specified in the AST dump
337338
createTokens(tokenList);
338339
if (nodeType == VarDecl || nodeType == RecordDecl || nodeType == TypedefDecl)
339340
addtoken(tokenList, ";");
@@ -626,6 +627,7 @@ void clangimport::AstNode::setValueType(Token *tok)
626627
continue;
627628

628629
TokenList decl(nullptr);
630+
decl.setLang(tok->isCpp() ? Settings::Language::CPP : Settings::Language::C);
629631
addTypeTokens(&decl, type, tok->scope());
630632
if (!decl.front())
631633
break;

‎lib/cppcheck.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -794,7 +794,7 @@ unsigned int CppCheck::checkFile(const std::string& filename, const std::string
794794
}
795795
Tokenizer tokenizer2(&mSettings, this);
796796
std::istringstream istr2(code);
797-
tokenizer2.list.createTokens(istr2);
797+
tokenizer2.list.createTokens(istr2, Path::isCPP(*files.begin()) ? Settings::Language::CPP : Settings::Language::C);
798798
executeRules("define", tokenizer2);
799799
}
800800
#endif

‎lib/library.cpp‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,10 +53,10 @@ static std::vector<std::string> getnames(const char *names)
5353
return ret;
5454
}
5555

56-
static void gettokenlistfromvalid(const std::string& valid, TokenList& tokenList)
56+
static void gettokenlistfromvalid(const std::string& valid, bool cpp, TokenList& tokenList)
5757
{
5858
std::istringstream istr(valid + ',');
59-
tokenList.createTokens(istr);
59+
tokenList.createTokens(istr, cpp ? Settings::Language::CPP : Settings::Language::C);
6060
for (Token *tok = tokenList.front(); tok; tok = tok->next()) {
6161
if (Token::Match(tok,"- %num%")) {
6262
tok->str("-" + tok->strAt(1));
@@ -920,7 +920,7 @@ bool Library::isIntArgValid(const Token *ftok, int argnr, const MathLib::bigint
920920
if (ac->valid.find('.') != std::string::npos)
921921
return isFloatArgValid(ftok, argnr, argvalue);
922922
TokenList tokenList(nullptr);
923-
gettokenlistfromvalid(ac->valid, tokenList);
923+
gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
924924
for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
925925
if (tok->isNumber() && argvalue == MathLib::toBigNumber(tok->str()))
926926
return true;
@@ -940,7 +940,7 @@ bool Library::isFloatArgValid(const Token *ftok, int argnr, double argvalue) con
940940
if (!ac || ac->valid.empty())
941941
return true;
942942
TokenList tokenList(nullptr);
943-
gettokenlistfromvalid(ac->valid, tokenList);
943+
gettokenlistfromvalid(ac->valid, ftok->isCpp(), tokenList);
944944
for (const Token *tok = tokenList.front(); tok; tok = tok->next()) {
945945
if (Token::Match(tok, "%num% : %num%") && argvalue >= MathLib::toDoubleNumber(tok->str()) && argvalue <= MathLib::toDoubleNumber(tok->strAt(2)))
946946
return true;
@@ -1725,13 +1725,14 @@ bool Library::hasAnyTypeCheck(const std::string& typeName) const
17251725

17261726
std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
17271727
const Settings* settings,
1728+
bool cpp,
17281729
std::unordered_map<nonneg int, const Token*>* lookupVarId)
17291730
{
17301731
std::shared_ptr<TokenList> tokenList = std::make_shared<TokenList>(settings);
17311732
{
17321733
const std::string code = "return " + returnValue + ";";
17331734
std::istringstream istr(code);
1734-
if (!tokenList->createTokens(istr))
1735+
if (!tokenList->createTokens(istr, cpp ? Settings::Language::CPP : Settings::Language::C))
17351736
return nullptr;
17361737
}
17371738

‎lib/library.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -603,6 +603,7 @@ CPPCHECKLIB const Library::Container * getLibraryContainer(const Token * tok);
603603

604604
std::shared_ptr<Token> createTokenFromExpression(const std::string& returnValue,
605605
const Settings* settings,
606+
bool cpp,
606607
std::unordered_map<nonneg int, const Token*>* lookupVarId = nullptr);
607608

608609
/// @}

‎lib/programmemory.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1595,7 +1595,7 @@ namespace {
15951595
arg_map[argn] = v;
15961596
argn++;
15971597
}
1598-
return evaluateLibraryFunction(arg_map, returnValue, settings);
1598+
return evaluateLibraryFunction(arg_map, returnValue, settings, ftok->isCpp());
15991599
}
16001600
}
16011601
}
@@ -1751,15 +1751,16 @@ std::vector<ValueFlow::Value> execute(const Scope* scope, ProgramMemory& pm, con
17511751

17521752
ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
17531753
const std::string& returnValue,
1754-
const Settings* settings)
1754+
const Settings* settings,
1755+
bool cpp)
17551756
{
17561757
thread_local static std::unordered_map<std::string,
17571758
std::function<ValueFlow::Value(const std::unordered_map<nonneg int, ValueFlow::Value>& arg)>>
17581759
functions = {};
17591760
if (functions.count(returnValue) == 0) {
17601761

17611762
std::unordered_map<nonneg int, const Token*> lookupVarId;
1762-
std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, &lookupVarId);
1763+
std::shared_ptr<Token> expr = createTokenFromExpression(returnValue, settings, cpp, &lookupVarId);
17631764

17641765
functions[returnValue] =
17651766
[lookupVarId, expr, settings](const std::unordered_map<nonneg int, ValueFlow::Value>& xargs) {

‎lib/programmemory.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,8 @@ ProgramMemory getProgramMemory(const Token* tok, const Token* expr, const ValueF
211211

212212
ValueFlow::Value evaluateLibraryFunction(const std::unordered_map<nonneg int, ValueFlow::Value>& args,
213213
const std::string& returnValue,
214-
const Settings* settings);
214+
const Settings* settings,
215+
bool cpp);
215216

216217
#endif
217218

‎lib/symboldatabase.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7479,7 +7479,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
74797479
ValueType valuetype;
74807480
TokenList tokenList(&mSettings);
74817481
std::istringstream istr(typestr+";");
7482-
tokenList.createTokens(istr);
7482+
tokenList.createTokens(istr, tok->isCpp() ? Settings::Language::CPP : Settings::Language::C);
74837483
tokenList.simplifyStdType();
74847484
if (parsedecl(tokenList.front(), &valuetype, mDefaultSignedness, mSettings, mIsCpp)) {
74857485
valuetype.originalTypeName = typestr;
@@ -7569,7 +7569,7 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
75697569
}
75707570
TokenList tokenList(&mSettings);
75717571
std::istringstream istr(typestr+";");
7572-
if (tokenList.createTokens(istr)) {
7572+
if (tokenList.createTokens(istr, tok->isCpp() ? Settings::Language::CPP : Settings::Language::C)) {
75737573
ValueType vt;
75747574
tokenList.simplifyPlatformTypes();
75757575
tokenList.simplifyStdType();

‎lib/token.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2741,5 +2741,6 @@ bool Token::isCpp() const
27412741
if (mTokensFrontBack && mTokensFrontBack->list) {
27422742
return mTokensFrontBack->list->isCPP();
27432743
}
2744+
assert(false);
27442745
return true; // assume C++ by default
27452746
}

‎lib/tokenlist.cpp‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
#include "standards.h"
3131
#include "token.h"
3232

33+
#include <cassert>
3334
#include <cctype>
3435
#include <exception>
3536
#include <functional>
@@ -83,12 +84,16 @@ void TokenList::deallocateTokens()
8384

8485
void TokenList::determineCppC()
8586
{
86-
// only try to determine it if it wasn't enforced
87+
// only try to determine if it wasn't enforced
8788
if (mLang == Settings::Language::None) {
8889
if (Path::isC(getSourceFilePath()))
8990
mLang = Settings::Language::C;
9091
else if (Path::isCPP(getSourceFilePath()))
9192
mLang = Settings::Language::CPP;
93+
else if (Path::isHeader(getSourceFilePath()))
94+
mLang = Settings::Language::C; // FIXME: treat headers as C for now
95+
96+
assert(mLang != Settings::Language::None);
9297
}
9398
}
9499

@@ -320,8 +325,27 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n)
320325

321326
bool TokenList::createTokens(std::istream &code, const std::string& file0)
322327
{
328+
assert(!file0.empty());
329+
323330
appendFileIfNew(file0);
324331

332+
return createTokensInternal(code, file0);
333+
}
334+
335+
//---------------------------------------------------------------------------
336+
337+
bool TokenList::createTokens(std::istream &code, Settings::Language lang)
338+
{
339+
if (mLang == Settings::Language::None)
340+
mLang = lang;
341+
342+
return createTokensInternal(code, "");
343+
}
344+
345+
//---------------------------------------------------------------------------
346+
347+
bool TokenList::createTokensInternal(std::istream &code, const std::string& file0)
348+
{
325349
simplecpp::OutputList outputList;
326350
simplecpp::TokenList tokens(code, mFiles, file0, &outputList);
327351

@@ -2090,3 +2114,25 @@ bool TokenList::isKeyword(const std::string &str) const
20902114
static const auto& latest_c_keywords = Keywords::getAll(Standards::cstd_t::CLatest);
20912115
return latest_c_keywords.find(str) != latest_c_keywords.end();
20922116
}
2117+
2118+
bool TokenList::isC() const
2119+
{
2120+
assert(mLang != Settings::Language::None);
2121+
2122+
return mLang == Settings::Language::C;
2123+
}
2124+
2125+
bool TokenList::isCPP() const
2126+
{
2127+
assert(mLang != Settings::Language::None);
2128+
2129+
return mLang == Settings::Language::CPP;
2130+
}
2131+
2132+
void TokenList::setLang(Settings::Language lang)
2133+
{
2134+
assert(lang != Settings::Language::None);
2135+
assert(mLang == Settings::Language::None);
2136+
2137+
mLang = lang;
2138+
}

‎lib/tokenlist.h‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -60,14 +60,12 @@ class CPPCHECKLIB TokenList {
6060
const std::string& getSourceFilePath() const;
6161

6262
/** Is the code C. Used for bailouts */
63-
bool isC() const {
64-
return mLang == Settings::Language::C;
65-
}
63+
bool isC() const;
6664

6765
/** Is the code CPP. Used for bailouts */
68-
bool isCPP() const {
69-
return mLang == Settings::Language::CPP;
70-
}
66+
bool isCPP() const;
67+
68+
void setLang(Settings::Language lang);
7169

7270
/**
7371
* Delete all tokens in given token list
@@ -103,7 +101,8 @@ class CPPCHECKLIB TokenList {
103101
* @param code input stream for code
104102
* @param file0 source file name
105103
*/
106-
bool createTokens(std::istream &code, const std::string& file0 = emptyString);
104+
bool createTokens(std::istream &code, const std::string& file0);
105+
bool createTokens(std::istream &code, Settings::Language lang);
107106

108107
void createTokens(simplecpp::TokenList&& tokenList);
109108

@@ -201,6 +200,8 @@ class CPPCHECKLIB TokenList {
201200
private:
202201
void determineCppC();
203202

203+
bool createTokensInternal(std::istream &code, const std::string& file0);
204+
204205
/** Token list */
205206
TokensFrontBack mTokensFrontBack;
206207

0 commit comments

Comments
 (0)