Highlighting (#301)

* Added more keywords to the C++ syntax, allowed debugging the highligher XPC service, preserving the original entitlements when force-resigning, fixed the double-releasing of failed XPC connections, allowed comments in the lexer settings.

* Moved FileSettingsStorage to a separate file, clang-format

* Added some unit tests for FileSettingsStorage

Source/CUI/source/CommandPopover.mm

@@ -29,7 +29,7 @@ @interface NCCommandPopoverViewController : NSViewController <NSTableViewDataSou
 - (instancetype _Nonnull)initWithPopover:(NCCommandPopover *)_popover andTitle:(NSString *)_title;
 - (void)tableView:(NSTableView *)_table didClickTableRow:(NSInteger)_row;
 - (bool)processKeyDown:(NSEvent *)_event;
-@property (readonly, nonatomic) bool numericHotkeysEnabled;
+@property(readonly, nonatomic) bool numericHotkeysEnabled;
 @end
 
 @interface NCCommandPopoverTableView : NSTableView
@@ -404,9 +404,9 @@ - (NSString *)hotkeyLabelForItemAtIndex:(size_t)_index
 
 - (std::optional<size_t>)itemIndexFromKeyDown:(NSEvent *)_event
 {
-    if(!self.numericHotkeysEnabled)
+    if( !self.numericHotkeysEnabled )
         return {};
-    
+
     // Use only clear keypresses, no modifiers
     if( _event.modifierFlags & (NSEventModifierFlagShift | NSEventModifierFlagControl | NSEventModifierFlagOption |
                                 NSEventModifierFlagCommand) )
@@ -565,7 +565,7 @@ - (NSView *)tableView:(NSTableView *)_table viewForTableColumn:(NSTableColumn *)
             return cv;
         }
 
-        if( [_column.identifier isEqualToString:@"K"] && self.numericHotkeysEnabled) {
+        if( [_column.identifier isEqualToString:@"K"] && self.numericHotkeysEnabled ) {
             NSString *hk = [self hotkeyLabelForItemAtIndex:_row];
             if( hk.length == 0 )
                 return nil;
@@ -759,7 +759,7 @@ - (void)selectFirstSelectableRow
     }
 }
 
-- (bool) numericHotkeysEnabled
+- (bool)numericHotkeysEnabled
 {
     return m_LabelTextField.stringValue == nil || m_LabelTextField.stringValue.length == 0;
 }

Source/NimbleCommander/NimbleCommander.xcodeproj/project.pbxproj

@@ -2961,7 +2961,7 @@
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 			shellPath = /bin/sh;
-			shellScript = "if [[ $CODE_SIGNING_ALLOWED == NO ]] ; then exit ; fi\ncodesign --verbose --sign 'Developer ID Application' --force --deep --timestamp --options runtime \"${TARGET_BUILD_DIR}/${XPCSERVICES_FOLDER_PATH}/Highlighter.xpc/Contents/MacOS/Highlighter\"\n";
+			shellScript = "if [[ $CODE_SIGNING_ALLOWED == NO ]] ; then exit ; fi\ncodesign \\\n --verbose \\\n --sign 'Developer ID Application' \\\n --force \\\n --deep \\\n --timestamp \\\n --entitlements \"${PROJECT_DIR}/../Viewer/resources/Highlighter.entitlements\" \\\n --options runtime \\\n \"${TARGET_BUILD_DIR}/${XPCSERVICES_FOLDER_PATH}/Highlighter.xpc/Contents/MacOS/Highlighter\"\n";
 		};
 		CF9EEB3E17256372001443FA /* ShellScript */ = {
 			isa = PBXShellScriptBuildPhase;

Source/NimbleCommander/NimbleCommander/Bootstrap/AppDelegate.mm

@@ -78,7 +78,7 @@
 #include <Viewer/Log.h>
 #include <Viewer/ViewerViewController.h>
 #include <Viewer/InternalViewerWindowController.h>
-#include <Viewer/Highlighting/SettingsStorage.h>
+#include <Viewer/Highlighting/FileSettingsStorage.h>
 
 #include <Term/Log.h>
 

Source/NimbleCommander/NimbleCommander/Core/Theming/Theme.mm

@@ -61,7 +61,7 @@
     NSColor *m_FilePanelsBriefFocusedActiveItemBackgroundColor;
     NSColor *m_FilePanelsBriefFocusedInactiveItemBackgroundColor;
     NSColor *m_FilePanelsBriefSelectedItemBackgroundColor;
-    
+
     NSFont *m_TerminalFont;
     NSColor *m_TerminalOverlayColor;
     NSColor *m_TerminalForegroundColor;
@@ -85,7 +85,7 @@
     NSColor *m_TerminalAnsiColorD;
     NSColor *m_TerminalAnsiColorE;
     NSColor *m_TerminalAnsiColorF;
-    
+
     NSFont *m_ViewerFont;
     NSColor *m_ViewerOverlayColor;
     NSColor *m_ViewerTextColor;

Source/NimbleCommander/NimbleCommander/Resources/SyntaxHighlighting/C++.json

@@ -1,7 +1,10 @@
 {
     "lexer": "cpp",
-    "wordlists": ["alignas alignof and and_eq asm auto bitand bitor bool break case catch char char8_t char16_t char32_t class compl concept const consteval constexpr constinit const_cast continue co_await co_return co_yield decltype default delete do double dynamic_cast else enum explicit export extern false float for friend goto if inline int long mutable namespace new noexcept not not_eq nullptr operator or or_eq private protected public register reinterpret_cast requires return short signed sizeof static static_assert static_cast struct switch template this thread_local throw true try typedef typeid typename union unsigned using virtual void volatile wchar_t while xor xor_eq"],
-    "properties": {},
+    "wordlists": ["alignas alignof and and_eq asm auto bitand bitor bool break case catch char char8_t char16_t char32_t class compl concept const consteval constexpr constinit const_cast continue co_await co_return co_yield decltype default delete do double dynamic_cast else enum explicit export extern false final float for friend goto if import inline int long module mutable namespace new noexcept not not_eq nullptr operator or or_eq override private protected public register reinterpret_cast requires return short signed sizeof static static_assert static_cast struct switch template this thread_local throw true try typedef typeid typename union unsigned using virtual void volatile wchar_t while xor xor_eq"],
+    "properties": {
+        "styling.within.preprocessor": "1",
+        "lexer.cpp.allow.dollars": "0"
+    },
     "mapping": {
         "SCE_C_DEFAULT": "default",
         "SCE_C_COMMENT": "comment",
@@ -15,6 +18,22 @@
         "SCE_C_PREPROCESSOR": "preprocessor",
         "SCE_C_OPERATOR": "operator",
         "SCE_C_IDENTIFIER": "identifier",
-        "SCE_C_STRINGEOL": "string"
+        "SCE_C_STRINGEOL": "string",
+        "SCE_C_VERBATIM": "string",
+        "SCE_C_REGEX": "string",
+        "SCE_C_REGEX": "string",
+        "SCE_C_COMMENTLINEDOC": "comment",
+        "SCE_C_WORD2": "keyword",
+        "SCE_C_COMMENTDOCKEYWORD": "comment",
+        "SCE_C_COMMENTDOCKEYWORDERROR": "comment",
+        "SCE_C_GLOBALCLASS": "identifier",
+        "SCE_C_STRINGRAW": "string",
+        "SCE_C_TRIPLEVERBATIM": "string",
+        "SCE_C_HASHQUOTEDSTRING": "string",
+        "SCE_C_PREPROCESSORCOMMENT": "comment",
+        "SCE_C_PREPROCESSORCOMMENTDOC": "comment",
+        "SCE_C_USERLITERAL": "identifier",
+        "SCE_C_TASKMARKER": "comment",
+        "SCE_C_ESCAPESEQUENCE": "string"
     }
 }

Source/Viewer/Viewer.xcodeproj/project.pbxproj

@@ -9,6 +9,9 @@
 /* Begin PBXBuildFile section */
 		CF1325622225FD630097F9A1 /* TextModeFrame_UT.cpp in Sources */ = {isa = PBXBuildFile; fileRef = CF13255E2225FB610097F9A1 /* TextModeFrame_UT.cpp */; };
 		CF24E1D3227F0B2A00C166FA /* HexModeLayout_UT.cpp in Sources */ = {isa = PBXBuildFile; fileRef = CF24E1D1227F0B2400C166FA /* HexModeLayout_UT.cpp */; };
+		CF26778A2C1E03FD00EE8F06 /* FileSettingsStorage.h in Headers */ = {isa = PBXBuildFile; fileRef = CF2677892C1E03FD00EE8F06 /* FileSettingsStorage.h */; };
+		CF26778C2C1E041400EE8F06 /* FileSettingsStorage.cpp in Sources */ = {isa = PBXBuildFile; fileRef = CF26778B2C1E041400EE8F06 /* FileSettingsStorage.cpp */; };
+		CF26778E2C1E0A0E00EE8F06 /* hlFileSettingsStorage_UT.cpp in Sources */ = {isa = PBXBuildFile; fileRef = CF26778D2C1E0A0E00EE8F06 /* hlFileSettingsStorage_UT.cpp */; };
 		CF3989B62B41707F006103C1 /* libBase.a in Frameworks */ = {isa = PBXBuildFile; fileRef = CF3989B52B41707F006103C1 /* libBase.a */; };
 		CF46FEFF255EF4480095FC73 /* Internal.h in Headers */ = {isa = PBXBuildFile; fileRef = CF46FEFE255EF4480095FC73 /* Internal.h */; };
 		CF5BF7832BF90FF20057C92E /* Document.h in Headers */ = {isa = PBXBuildFile; fileRef = CF5BF7822BF90FF20057C92E /* Document.h */; };
@@ -132,6 +135,9 @@
 		CF24E1D1227F0B2400C166FA /* HexModeLayout_UT.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = HexModeLayout_UT.cpp; path = tests/HexModeLayout_UT.cpp; sourceTree = "<group>"; };
 		CF24E1D62286E23800C166FA /* PreviewModeView.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; name = PreviewModeView.mm; path = source/PreviewModeView.mm; sourceTree = "<group>"; };
 		CF24E1D82286E24300C166FA /* PreviewModeView.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = PreviewModeView.h; path = include/Viewer/PreviewModeView.h; sourceTree = "<group>"; };
+		CF2677892C1E03FD00EE8F06 /* FileSettingsStorage.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = FileSettingsStorage.h; path = include/Viewer/Highlighting/FileSettingsStorage.h; sourceTree = "<group>"; };
+		CF26778B2C1E041400EE8F06 /* FileSettingsStorage.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = FileSettingsStorage.cpp; path = source/Highlighting/FileSettingsStorage.cpp; sourceTree = "<group>"; };
+		CF26778D2C1E0A0E00EE8F06 /* hlFileSettingsStorage_UT.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = hlFileSettingsStorage_UT.cpp; path = tests/hlFileSettingsStorage_UT.cpp; sourceTree = "<group>"; };
 		CF3989B52B41707F006103C1 /* libBase.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; path = libBase.a; sourceTree = BUILT_PRODUCTS_DIR; };
 		CF46FEFE255EF4480095FC73 /* Internal.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = Internal.h; path = source/Internal.h; sourceTree = "<group>"; };
 		CF46FF03255EF4690095FC73 /* Bundle.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = Bundle.h; path = include/Viewer/Bundle.h; sourceTree = "<group>"; };
@@ -266,6 +272,7 @@
 			children = (
 				CF5BF7B72BFE8B3B0057C92E /* Client.h */,
 				CF5BF7822BF90FF20057C92E /* Document.h */,
+				CF2677892C1E03FD00EE8F06 /* FileSettingsStorage.h */,
 				CF5BF7952BFBD9480057C92E /* Highlighter.h */,
 				CF5BF7932BFBCA030057C92E /* LexerSettings.h */,
 				CF5BF7C72C0B89B40057C92E /* SettingsStorage.h */,
@@ -279,6 +286,7 @@
 			children = (
 				CF5BF7B82BFE8B540057C92E /* Client.cpp */,
 				CF5BF7852BF910100057C92E /* Document.cpp */,
+				CF26778B2C1E041400EE8F06 /* FileSettingsStorage.cpp */,
 				CF5BF7972BFBD9510057C92E /* Highlighter.cpp */,
 				CF5BF7912BFBC9F90057C92E /* LexerSettings.cpp */,
 				CF5BF7AA2BFD48950057C92E /* Service.cpp */,
@@ -410,6 +418,7 @@
 				CF9BF9052269FD7000AD36D9 /* HexModeProcessing_UT.cpp */,
 				CF5BF7C12BFFDD240057C92E /* hlClient_UT.cpp */,
 				CF5BF7872BF9209E0057C92E /* hlDocument_UT.cpp */,
+				CF26778D2C1E0A0E00EE8F06 /* hlFileSettingsStorage_UT.cpp */,
 				CF5BF7992BFBDF1F0057C92E /* hlHighlighter_UT.cpp */,
 				CF5BF79B2BFD39A60057C92E /* hlLexerSettings_UT.cpp */,
 				CF5BF78E2BFA19F50057C92E /* hlStyle_UT.cpp */,
@@ -450,6 +459,7 @@
 			buildActionMask = 2147483647;
 			files = (
 				CF5BF78B2BFA12F70057C92E /* Style.h in Headers */,
+				CF26778A2C1E03FD00EE8F06 /* FileSettingsStorage.h in Headers */,
 				CF5BF7962BFBD9480057C92E /* Highlighter.h in Headers */,
 				CF5BF7C82C0B89B40057C92E /* SettingsStorage.h in Headers */,
 				CF5BF7C42C0B41F90057C92E /* TextModeWorkingSetHighlighting.h in Headers */,
@@ -661,6 +671,7 @@
 				CFA9998D26468A4300F72E93 /* Log.cpp in Sources */,
 				CF5BF7982BFBD9510057C92E /* Highlighter.cpp in Sources */,
 				CF5C1D86255EEA6A00ADE703 /* PreviewModeView.mm in Sources */,
+				CF26778C2C1E041400EE8F06 /* FileSettingsStorage.cpp in Sources */,
 				CF5C1D7F255EEA6A00ADE703 /* HexModeView.mm in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -670,6 +681,7 @@
 			buildActionMask = 2147483647;
 			files = (
 				CF5BF79A2BFBDF1F0057C92E /* hlHighlighter_UT.cpp in Sources */,
+				CF26778E2C1E0A0E00EE8F06 /* hlFileSettingsStorage_UT.cpp in Sources */,
 				CF9BF90D226CF99000AD36D9 /* HexModeFrame_UT.cpp in Sources */,
 				CF61F2FC263D610A009FF900 /* TextMoveView_UT.mm in Sources */,
 				CF9BF9062269FD7000AD36D9 /* HexModeProcessing_UT.cpp in Sources */,

Source/Viewer/include/Viewer/DataBackend.h

@@ -58,7 +58,7 @@ class DataBackend
 
     // Returns a filename component of the underlying VFS file's path
     std::filesystem::path FileName() const;
-    
+
 private:
     void DecodeBuffer(); // called by internal update logic
 

Source/Viewer/include/Viewer/Highlighting/Document.h

@@ -35,7 +35,7 @@ class Document final : public Scintilla::IDocument
     int GetLineIndentation(Sci_Position line) noexcept override;
 
     Sci_Position LineStart(Sci_Position line) const noexcept override;
-    
+
     Sci_Position LineEnd(Sci_Position line) const noexcept override;
 
     Sci_Position GetRelativePosition(Sci_Position positionStart, Sci_Position characterOffset) const noexcept override;
@@ -67,6 +67,7 @@ class Document final : public Scintilla::IDocument
     void ChangeLexerState(Sci_Position start, Sci_Position end) noexcept override;
 
     std::span<const char> Styles() const noexcept;
+
 private:
     std::string_view m_Text;
     std::vector<uint32_t> m_Lines;

Source/Viewer/include/Viewer/Highlighting/FileSettingsStorage.h

@@ -0,0 +1,38 @@
+// Copyright (C) 2024 Michael Kazakov. Subject to GNU General Public License version 3.
+#pragma once
+#include "SettingsStorage.h"
+#include <Base/RobinHoodUtil.h>
+#include <Utility/FileMask.h>
+#include <filesystem>
+#include <vector>
+
+namespace nc::viewer::hl {
+
+class FileSettingsStorage : public SettingsStorage
+{
+public:
+    FileSettingsStorage(const std::filesystem::path &_base_dir, const std::filesystem::path &_overrides_dir);
+
+    std::optional<std::string> Language(std::string_view _filename) noexcept override;
+
+    std::shared_ptr<const std::string> Settings(std::string_view _lang) override;
+
+private:
+    struct Lang {
+        std::string name;
+        std::string settings_filename;
+        nc::utility::FileMask mask;
+    };
+
+    std::vector<Lang> LoadLangs();
+
+    std::filesystem::path m_BaseDir;
+    std::vector<Lang> m_Langs;
+    robin_hood::unordered_flat_map<std::string,
+                                   std::shared_ptr<const std::string>,
+                                   RHTransparentStringHashEqual,
+                                   RHTransparentStringHashEqual>
+        m_Settings;
+};
+
+} // namespace nc::viewer::hl

Source/Viewer/include/Viewer/Highlighting/Highlighter.h

@@ -4,8 +4,7 @@
 #include "Style.h"
 #include "LexerSettings.h"
 
-namespace Scintilla
-{
+namespace Scintilla {
 class ILexer5;
 }
 
@@ -15,15 +14,15 @@ class Highlighter
 {
 public:
     Highlighter(LexerSettings _settings);
-    Highlighter(const Highlighter&) = delete;
+    Highlighter(const Highlighter &) = delete;
     ~Highlighter();
-    Highlighter &operator=(const Highlighter&) = delete;
+    Highlighter &operator=(const Highlighter &) = delete;
 
     std::vector<Style> Highlight(std::string_view _text) const;
-    
+
 private:
     LexerSettings m_Settings;
     Scintilla::ILexer5 *m_Lexer = nullptr;
 };
 
-}
+} // namespace nc::viewer::hl

Source/Viewer/include/Viewer/Highlighting/LexerSettings.h

@@ -14,7 +14,7 @@ struct LexerSettings {
     struct Property {
         std::string key;
         std::string value;
-        auto operator<=>(const Property& ) const noexcept = default;
+        auto operator<=>(const Property &) const noexcept = default;
     };
 
     std::string name;

Source/Viewer/include/Viewer/Highlighting/SettingsStorage.h

@@ -1,13 +1,10 @@
 // Copyright (C) 2024 Michael Kazakov. Subject to GNU General Public License version 3.
 #pragma once
 
-#include <Utility/FileMask.h>
-#include <Base/RobinHoodUtil.h>
 #include <string>
-#include <filesystem>
 #include <string_view>
 #include <memory>
-#include <vector>
+#include <optional>
 
 namespace nc::viewer::hl {
 
@@ -16,44 +13,19 @@ class SettingsStorage
 public:
     virtual ~SettingsStorage() = default;
 
-    virtual std::string Language(std::string_view _filename) = 0;
+    // Returns a name of a predicted language for the given filename.
+    virtual std::optional<std::string> Language(std::string_view _filename) = 0;
 
+    // Returns the syntax settings of the given language name or nullptr if no such language is defined
     virtual std::shared_ptr<const std::string> Settings(std::string_view _lang) = 0;
 };
 
 class DummySettingsStorage : public SettingsStorage
 {
 public:
-    std::string Language(std::string_view _filename) override;
-
-    std::shared_ptr<const std::string> Settings(std::string_view _lang) override;
-};
-
-class FileSettingsStorage : public SettingsStorage
-{
-public:
-    FileSettingsStorage(const std::filesystem::path &_base_dir, const std::filesystem::path &_overrides_dir);
+    std::optional<std::string> Language(std::string_view _filename) override;
 
-    std::string Language(std::string_view _filename) override;
-
     std::shared_ptr<const std::string> Settings(std::string_view _lang) override;
-
-private:
-    struct Lang {
-        std::string name;
-        std::string settings_filename;
-        nc::utility::FileMask mask;
-    };
-
-    void LoadLangs();
-
-    std::filesystem::path m_BaseDir;
-    std::vector<Lang> m_Langs;
-    robin_hood::unordered_flat_map<std::string,
-                                   std::shared_ptr<const std::string>,
-                                   RHTransparentStringHashEqual,
-                                   RHTransparentStringHashEqual>
-        m_Settings;
 };
 
 } // namespace nc::viewer::hl

Source/Viewer/resources/Highlighter.entitlements

@@ -4,5 +4,7 @@
 <dict>
 	<key>com.apple.security.app-sandbox</key>
 	<true/>
+	<key>com.apple.security.get-task-allow</key>
+	<true/>
 </dict>
 </plist>

Source/Viewer/source/DataBackend.cpp

@@ -61,7 +61,7 @@ std::filesystem::path DataBackend::FileName() const
     if( path == nullptr ) {
         return {};
     }
-    
+
     return utility::PathManip::Filename(path);
 }
 

Source/Viewer/source/Highlighting/Client.cpp

@@ -105,7 +105,6 @@ void Client::HighlightAsync(std::string_view _text,
     }
 
     auto handler = ^(xpc_object_t _reply) {
-      auto release_connection = at_scope_end([&] { xpc_release(connection); });
       Log::Trace(SPDLOC, "Got a response from the XPC service");
       const xpc_type_t type = xpc_get_type(_reply);
       if( type == XPC_TYPE_ERROR ) {
@@ -161,6 +160,9 @@ void Client::HighlightAsync(std::string_view _text,
     auto release_message = at_scope_end([&] { xpc_release(message); });
 
     xpc_connection_send_message_with_reply(connection, message, _queue, handler);
+
+    // VVV I seriously don't understand the ownership model of xpc_connection_t and why this seem to be correct...
+    xpc_release(connection);
 }
 
 } // namespace nc::viewer::hl