Fix #15100 (clang import: internalAstError "invalid AST location: <<built-in>" for simple hello.c) (#8933)

Author: danmar

lib/clangimport.cpp

@@ -123,6 +123,12 @@ static std::string unquote(const std::string &s)
 }
 
 
+// location in a buffer that is generated by clang: "<<built-in>:115:23, file.h:18:19>"
+static bool isGeneratedBufferLocation(const std::string &s, std::string::size_type pos = 0)
+{
+    return s.compare(pos, 12, "<<built-in>:") == 0 || s.compare(pos, 17, "<<scratch space>:") == 0;
+}
+
 static std::vector<std::string> splitString(const std::string &line)
 {
     std::vector<std::string> ret;
@@ -134,9 +140,11 @@ static std::vector<std::string> splitString(const std::string &line)
             pos1 = line.find_first_not_of(' ', pos1 + 1);
             continue;
         }
-        if (line[pos1] == '<')
+        if (line[pos1] == '<') {
             pos2 = line.find('>', pos1);
-        else if (line[pos1] == '\"')
+            if (pos2 != std::string::npos && isGeneratedBufferLocation(line, pos1))
+                pos2 = line.find('>', pos2 + 1);
+        } else if (line[pos1] == '\"')
             pos2 = line.find('\"', pos1+1);
         else if (line[pos1] == '\'') {
             pos2 = line.find('\'', pos1+1);
@@ -560,19 +568,32 @@ void clangimport::AstNode::setLocations(TokenList &tokenList, int file, int line
             if (pos != std::string::npos)
                 col = strToInt<int>(ext.substr(pos+6, ext.find_first_of(":,>", pos+6) - (pos+6)));
         } else if (ext[0] == '<') {
-            const std::string::size_type colon = ext.find(':');
-            if (colon != std::string::npos) {
-                const bool windowsPath = colon == 2 && ext.size() > 3 && ext[2] == ':';
-                const std::string::size_type sep1 = windowsPath ? ext.find(':', 4) : colon;
-                const std::string::size_type sep2 = ext.find(':', sep1 + 1);
-                file = tokenList.appendFileIfNew(ext.substr(1, sep1 - 1));
-                line = strToInt<int>(ext.substr(sep1 + 1, sep2 - sep1 - 1));
+            // start of the file location
+            std::string::size_type pos = 1;
+            if (isGeneratedBufferLocation(ext)) {
+                // Use the end of the range if that is a file location, otherwise the location is unchanged
+                pos = ext.find(", ");
+                if (pos != std::string::npos) {
+                    pos += 2;
+                    if (ext[pos] == '<' || ext.compare(pos, 4, "col:") == 0 || ext.compare(pos, 5, "line:") == 0)
+                        pos = std::string::npos;
+                }
             }
-            else {
-                // "<invalid sloc>" are encountered in every AST dump by some built-in TypedefDecl
-                // an completely empty location block was encountered with a CompoundStmt
-                if (ext != "<<invalid sloc>" && ext != "<>")
-                    throw InternalError(nullptr, "invalid AST location: " + ext, InternalError::AST);
+            if (pos != std::string::npos) {
+                const std::string::size_type colon = ext.find(':', pos);
+                if (colon != std::string::npos) {
+                    const bool windowsPath = colon == pos + 1 && ext.size() > pos + 2;
+                    const std::string::size_type sep1 = windowsPath ? ext.find(':', pos + 3) : colon;
+                    const std::string::size_type sep2 = ext.find(':', sep1 + 1);
+                    file = tokenList.appendFileIfNew(ext.substr(pos, sep1 - pos));
+                    line = strToInt<int>(ext.substr(sep1 + 1, sep2 - sep1 - 1));
+                }
+                else {
+                    // "<invalid sloc>" are encountered in every AST dump by some built-in TypedefDecl
+                    // an completely empty location block was encountered with a CompoundStmt
+                    if (ext != "<<invalid sloc>" && ext != "<>")
+                        throw InternalError(nullptr, "invalid AST location: " + ext, InternalError::AST);
+                }
             }
         }
     }

test/testclangimport.cpp

@@ -117,6 +117,7 @@ class TestClangImport : public TestFixture {
         TEST_CASE(vardecl6);
         TEST_CASE(vardecl7);
         TEST_CASE(vardeclInHeader);
+        TEST_CASE(vardeclBuiltinLocation);
         TEST_CASE(whileStmt1);
         TEST_CASE(whileStmt2);
 
@@ -1051,6 +1052,36 @@ class TestClangImport : public TestFixture {
                       parse(clang));
     }
 
+    void vardeclBuiltinLocation() {
+        // #15100 - the location starts in a buffer that is generated by clang
+        const char clang[] = "|-VarDecl 0x32b8aa0 <1.c:1:1, col:5> col:5 a 'int'\n"
+                             "|-VarDecl 0x32b8b78 <<built-in>:115:23, 1.h:18:19> col:19 b 'unsigned int'\n"
+                             "|-VarDecl 0x32b8c50 <<scratch space>:2:1, col:5> col:5 c 'int'\n"
+                             "`-VarDecl 0x32b8d28 <<scratch space>:41:1> col:5 d 'int'";
+
+        ASSERT_EQUALS("int a@1 ; unsigned int b@2 ; int c@3 ; int d@4 ;",
+                      parse(clang));
+
+        const Settings settings = settingsBuilder().clang().build();
+        TokenList tokenlist{settings, Standards::Language::C};
+        Tokenizer tokenizer(std::move(tokenlist), *this);
+        std::istringstream istr(clang);
+        clangimport::parseClangAstDump(tokenizer, istr);
+
+        // the end of the range is used for "b", the location is unchanged for "c" and "d"
+        ASSERT_EQUALS(2, tokenizer.list.getFiles().size());
+        ASSERT_EQUALS("1.c", tokenizer.list.getFiles()[0]);
+        ASSERT_EQUALS("1.h", tokenizer.list.getFiles()[1]);
+        const Token *tok = Token::findsimplematch(tokenizer.tokens(), "b");
+        ASSERT(tok);
+        ASSERT_EQUALS(1, tok->fileIndex());
+        ASSERT_EQUALS(18, tok->linenr());
+        tok = Token::findsimplematch(tokenizer.tokens(), "d");
+        ASSERT(tok);
+        ASSERT_EQUALS(1, tok->fileIndex());
+        ASSERT_EQUALS(18, tok->linenr());
+    }
+
     void whileStmt1() {
         const char clang[] = "`-FunctionDecl 0x3d45b18 <1.c:1:1, line:3:1> line:1:6 foo 'void ()'\n"
                              "  `-CompoundStmt 0x3d45c48 <col:12, line:3:1>\n"