From 1ed7ed1ee9bd090f530990b0a9b6d5c66d8adee4 Mon Sep 17 00:00:00 2001 From: Nate Mills Date: Sun, 11 Jul 2021 16:42:24 -0700 Subject: [PATCH 1/9] Adding ignore list setting and logic --- highlight_duplicates.sublime-settings | 4 +++- hightlight_duplicates.py | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/highlight_duplicates.sublime-settings b/highlight_duplicates.sublime-settings index 91d755f..14a4855 100644 --- a/highlight_duplicates.sublime-settings +++ b/highlight_duplicates.sublime-settings @@ -10,5 +10,7 @@ "ignore_case" : false, - "min_line_length" : 4 + "min_line_length" : 4, + + "ignore_list" : [] } diff --git a/hightlight_duplicates.py b/hightlight_duplicates.py index 9997c49..67510ae 100644 --- a/hightlight_duplicates.py +++ b/hightlight_duplicates.py @@ -26,7 +26,7 @@ DEFAULT_IS_DISABLED = False DEFAULT_MIN_LINE_LENGTH = 4 -def count_lines(lines, view, minLineLength): +def count_lines(lines, view, minLineLength, ignoreList): '''Counts line occurrences of a view using a hash. The lines are stripped and tested before count. ''' @@ -37,7 +37,7 @@ def count_lines(lines, view, minLineLength): string = string.strip() if ignoreCase(): string = string.lower() - if is_candidate(string, minLineLength): + if is_candidate(string, minLineLength, ignoreList): counts[string].append(line) return counts @@ -74,10 +74,12 @@ def merge_results(countsList): return merged -def is_candidate(string, minLineLength): +def is_candidate(string, minLineLength, ignoreList): '''Tells if a string is a LOC candidate. A candidate is a string long enough after stripping some symbols. ''' + if string in ignoreList: + return False return len(string.strip('{}()[]/')) >= minLineLength @@ -110,7 +112,7 @@ def highlight_duplicates(view): # get all lines lines = view.lines(sublime.Region(0, view.size())) # count and filter out non duplicated lines - duplicates = filter_counts(count_lines(lines, view, getMinLineLength())) + duplicates = filter_counts(count_lines(lines, view, getMinLineLength(), getIgnoreList())) # show duplicated lines show_lines(duplicates.values(), view) @@ -121,7 +123,7 @@ def select_duplicates(view): # get all lines lines = view.lines(sublime.Region(0, view.size())) # count and filter out non duplicated lines - duplicates = filter_counts(count_lines(lines, view, getMinLineLength())) + duplicates = filter_counts(count_lines(lines, view, getMinLineLength(), getIgnoreList())) # select duplicated lines add_lines(duplicates.values(), view) @@ -132,7 +134,7 @@ def remove_duplicates(view, edit): # get all lines lines = view.lines(sublime.Region(0, view.size())) # count and filter out non duplicated lines - duplicates = remove_first(filter_counts(count_lines(lines, view, getMinLineLength()))) + duplicates = remove_first(filter_counts(count_lines(lines, view, getMinLineLength(), getIgnoreList()))) # select duplicated lines merged = merge_results(duplicates.values()) merged.sort(key=lambda elm: elm.begin()) @@ -172,6 +174,11 @@ def getMinLineLength(): else: return DEFAULT_MIN_LINE_LENGTH +def getIgnoreList(): + settings = sublime.load_settings('highlight_duplicates.sublime-settings') + ignoreList = settings.get('ignore_list', []) + return ignoreList + class HighlightDuplicatesCommand(sublime_plugin.WindowCommand): From cf3d672fc6b78c826644a047f4347a5b25b84e83 Mon Sep 17 00:00:00 2001 From: Nate Mills Date: Sun, 11 Jul 2021 16:53:46 -0700 Subject: [PATCH 2/9] Updating readme. --- readme.md | 47 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/readme.md b/readme.md index 410ec16..18384f5 100644 --- a/readme.md +++ b/readme.md @@ -8,7 +8,7 @@ This is a [Sublime Text 3](http://www.sublimetext.com/3) (and ST2) plugin. **Using Package Control** ([installation instructions](https://packagecontrol.io/installation)) -Press ctrl+shift+p (cmd+shift+p for OSX), then use the 'Package Control: Install Package' command. +Press ctrl+shift+p (cmd+shift+p for OSX), then use the 'Package Control: Install Package' command. Search for 'HighlightDuplicates', and press enter to install. @@ -31,7 +31,7 @@ That's it! ## Commands -### Toggle Highlighting +### Toggle Highlighting Using this command will turn the plugin on and off. This allows you to see the highlighted only when you want without having to disable/enable to plugin via the Package Control. @@ -50,17 +50,17 @@ For example... 1: 2: Content 3: -4: -``` +4: +``` Would result in ``` html 1: 2: Content -3: +3: ``` -_Note:_ By default none of these commands have a key binding, and can only be used via the Command Palette. You can set a key binding by adding any of the following lines to the key binding file. (Preferences > Key Bindings) +_Note:_ By default none of these commands have a key binding, and can only be used via the Command Palette. You can set a key binding by adding any of the following lines to the key binding file. (Preferences > Key Bindings) ``` js { "keys": ["alt+shift+h"], "command": "toggle_highlight_duplicates" } @@ -70,7 +70,7 @@ _Note:_ By default none of these commands have a key binding, and can only be us ## Options -### Change The Highlighting Color +### Change The Highlighting Color `Default: "invalid"` The highlighting color can be changed by providing a scope name such @@ -80,41 +80,41 @@ If you'd like to use a custom color, it should be defined as a color scope in your theme file. -### Trim White Space +### Trim White Space `Default: true` If this setting is true, the leading and trailing white space will be removed before being compared to other lines. This setting also affects which lines are selected when using the 'Select Duplicates' command. -For example, if `"trim_white_space" : true` the following 2 lines will be counted as duplicates. +For example, if `"trim_white_space" : true` the following 2 lines will be counted as duplicates. ``` html 1: 2: -``` -However, the following lines would not be counted as duplicates. The reason for this is because there is white space in line 1 that is not leading or trailing, which does not appear in line 2. +``` +However, the following lines would not be counted as duplicates. The reason for this is because there is white space in line 1 that is not leading or trailing, which does not appear in line 2. ``` html 1: 2: ``` -### Ignore Case +### Ignore Case `Default: false` If this setting is true, upper and lower case letters will be considered the same. This setting also affects which lines are selected when using the 'Select Duplicates' command. -For example, if `"ignore_case" : true` the following 2 lines will be counted as duplicates. +For example, if `"ignore_case" : true` the following 2 lines will be counted as duplicates. ``` html 1: 2: ``` -### Min Line Length +### Min Line Length `Default: 4` Lines with fewer characters than specified in this setting, will be ignored for all functions. Setting this to 1 will cause all non empty lines to be possible duplicates. -For example, by default, only lines 7 and 8 will be selected when using the "select duplicate" command. If this setting is set to 2, all the lines except lines 1 and 2 will be selected when using the "select duplicate" command. +For example, by default, only lines 7 and 8 will be selected when using the "select duplicate" command. If this setting is set to 2, all the lines except lines 1 and 2 will be selected when using the "select duplicate" command. ``` html 1: 1 2: 1 @@ -125,3 +125,20 @@ For example, by default, only lines 7 and 8 will be selected when using the "sel 7: 1234 8: 1234 ``` + + + +### Ignore List +`Default: []` + +Lines matching entires in this list, will be ignored for all functions. + +` + "ignore_list": ["This line will be ignored"] +` +``` html +1: This line will be ignored +2: This line will be ignored +3: This line will not be ignored +4: This line will not be ignored +``` From f94ea4a970bc5ee21426e709b03a2003a2cef657 Mon Sep 17 00:00:00 2001 From: Nate Mills Date: Tue, 24 Aug 2021 14:03:15 -0700 Subject: [PATCH 3/9] adding ignore file, removing excesive argument passing, making it so ignore list is always trimmed and lower case --- .gitignore | 1 + hightlight_duplicates.py | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..722d5e7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.vscode diff --git a/hightlight_duplicates.py b/hightlight_duplicates.py index 67510ae..50b5d7a 100644 --- a/hightlight_duplicates.py +++ b/hightlight_duplicates.py @@ -26,7 +26,7 @@ DEFAULT_IS_DISABLED = False DEFAULT_MIN_LINE_LENGTH = 4 -def count_lines(lines, view, minLineLength, ignoreList): +def count_lines(lines, view): '''Counts line occurrences of a view using a hash. The lines are stripped and tested before count. ''' @@ -37,7 +37,7 @@ def count_lines(lines, view, minLineLength, ignoreList): string = string.strip() if ignoreCase(): string = string.lower() - if is_candidate(string, minLineLength, ignoreList): + if is_candidate(string): counts[string].append(line) return counts @@ -74,11 +74,13 @@ def merge_results(countsList): return merged -def is_candidate(string, minLineLength, ignoreList): +def is_candidate(string): '''Tells if a string is a LOC candidate. A candidate is a string long enough after stripping some symbols. ''' - if string in ignoreList: + minLineLength = getMinLineLength() + ignoreList = getIgnoreList() + if string.strip().lower() in ignoreList: return False return len(string.strip('{}()[]/')) >= minLineLength @@ -112,7 +114,7 @@ def highlight_duplicates(view): # get all lines lines = view.lines(sublime.Region(0, view.size())) # count and filter out non duplicated lines - duplicates = filter_counts(count_lines(lines, view, getMinLineLength(), getIgnoreList())) + duplicates = filter_counts(count_lines(lines, view)) # show duplicated lines show_lines(duplicates.values(), view) @@ -123,7 +125,7 @@ def select_duplicates(view): # get all lines lines = view.lines(sublime.Region(0, view.size())) # count and filter out non duplicated lines - duplicates = filter_counts(count_lines(lines, view, getMinLineLength(), getIgnoreList())) + duplicates = filter_counts(count_lines(lines, view)) # select duplicated lines add_lines(duplicates.values(), view) @@ -134,7 +136,7 @@ def remove_duplicates(view, edit): # get all lines lines = view.lines(sublime.Region(0, view.size())) # count and filter out non duplicated lines - duplicates = remove_first(filter_counts(count_lines(lines, view, getMinLineLength(), getIgnoreList()))) + duplicates = remove_first(filter_counts(count_lines(lines, view))) # select duplicated lines merged = merge_results(duplicates.values()) merged.sort(key=lambda elm: elm.begin()) @@ -177,6 +179,9 @@ def getMinLineLength(): def getIgnoreList(): settings = sublime.load_settings('highlight_duplicates.sublime-settings') ignoreList = settings.get('ignore_list', []) + for i in range(len(ignoreList)): + ignoreList[i] = ignoreList[i].strip().lower() + return ignoreList From 927904b0138473d385b8a9edd7c896c678e85de5 Mon Sep 17 00:00:00 2001 From: Nate Mills Date: Tue, 24 Aug 2021 14:08:55 -0700 Subject: [PATCH 4/9] Updating readme, adding release message --- messages/1.5.0.txt | 8 ++++++++ readme.md | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 messages/1.5.0.txt diff --git a/messages/1.5.0.txt b/messages/1.5.0.txt new file mode 100644 index 0000000..3156020 --- /dev/null +++ b/messages/1.5.0.txt @@ -0,0 +1,8 @@ +Highlight Duplicates v1.5.0 + + • Added Ignore List Setting + + Lines matching entires in this list, will be ignored for all functions. Leading and trailing white space, as well as letter case, will be ignored when checking lines against the ignore list. + + Default: + "ignore_list": [] diff --git a/readme.md b/readme.md index 18384f5..4830c2e 100644 --- a/readme.md +++ b/readme.md @@ -131,7 +131,7 @@ For example, by default, only lines 7 and 8 will be selected when using the "sel ### Ignore List `Default: []` -Lines matching entires in this list, will be ignored for all functions. +Lines matching entires in this list, will be ignored for all functions. Leading and trailing white space, as well as letter case, will be ignored when checking lines against the ignore list. ` "ignore_list": ["This line will be ignored"] From d845e089c366f2d58b066633cd0bc86aa202e949 Mon Sep 17 00:00:00 2001 From: Nate Mills Date: Tue, 24 Aug 2021 14:14:56 -0700 Subject: [PATCH 5/9] adding short circuit condition --- hightlight_duplicates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hightlight_duplicates.py b/hightlight_duplicates.py index 50b5d7a..0d3f478 100644 --- a/hightlight_duplicates.py +++ b/hightlight_duplicates.py @@ -80,7 +80,7 @@ def is_candidate(string): ''' minLineLength = getMinLineLength() ignoreList = getIgnoreList() - if string.strip().lower() in ignoreList: + if len(ignoreList) > 0 and string.strip().lower() in ignoreList: return False return len(string.strip('{}()[]/')) >= minLineLength From fb39c733d7449271989dd801739cae0f397e4b3d Mon Sep 17 00:00:00 2001 From: Nate Mills Date: Tue, 24 Aug 2021 14:16:34 -0700 Subject: [PATCH 6/9] typo --- messages/1.5.0.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/messages/1.5.0.txt b/messages/1.5.0.txt index 3156020..70f2728 100644 --- a/messages/1.5.0.txt +++ b/messages/1.5.0.txt @@ -2,7 +2,7 @@ Highlight Duplicates v1.5.0 • Added Ignore List Setting - Lines matching entires in this list, will be ignored for all functions. Leading and trailing white space, as well as letter case, will be ignored when checking lines against the ignore list. + Lines matching entries in this list, will be ignored for all functions. Leading and trailing white space, as well as letter case, will be ignored when checking lines against the ignore list. Default: "ignore_list": [] From 4f89edfddf124e4c3edf8d49a05053f4cebac16b Mon Sep 17 00:00:00 2001 From: Nate Mills Date: Tue, 24 Aug 2021 14:47:55 -0700 Subject: [PATCH 7/9] Updating default settings, adding "min_duplicate_count" option --- highlight_duplicates.sublime-settings | 6 ++++++ hightlight_duplicates.py | 16 +++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/highlight_duplicates.sublime-settings b/highlight_duplicates.sublime-settings index 14a4855..2ae0654 100644 --- a/highlight_duplicates.sublime-settings +++ b/highlight_duplicates.sublime-settings @@ -8,9 +8,15 @@ // If true, whitespace gets trimmed off each line before checking for matching lines "trim_white_space" : true, + // If true, letter casing is ignored for checking for matching lines "ignore_case" : false, + // The minimun length a line needs to be in order to be included as a matching line "min_line_length" : 4, + // The minimum number of duplicate lines needed to be found, in order to be included as a matching line + "min_duplicate_count" : 1, + + // Lines that match entries in this list are ignored as matching lines "ignore_list" : [] } diff --git a/hightlight_duplicates.py b/hightlight_duplicates.py index 0d3f478..d46dc89 100644 --- a/hightlight_duplicates.py +++ b/hightlight_duplicates.py @@ -25,6 +25,7 @@ DEFAULT_IS_ENABLED = True DEFAULT_IS_DISABLED = False DEFAULT_MIN_LINE_LENGTH = 4 +DEFAULT_MIN_DUPLICATE_COUNT = 1 def count_lines(lines, view): '''Counts line occurrences of a view using a hash. @@ -42,13 +43,14 @@ def count_lines(lines, view): return counts -def filter_counts(counts, treshold=1): +def filter_counts(counts): '''Filters the line counts by rejecting every line having a count - lower or equal to the treshold, which defaults to 1. + lower or equal to the "min_duplicate_count" user setting, which defaults to 1. ''' filtered = dict() + threshold = getMinDuplicateCount(); for k, v in counts.items(): - if len(v) > treshold: + if len(v) > threshold: filtered[k] = v return filtered @@ -176,6 +178,14 @@ def getMinLineLength(): else: return DEFAULT_MIN_LINE_LENGTH +def getMinDuplicateCount(): + settings = sublime.load_settings('highlight_duplicates.sublime-settings') + minLength = settings.get('min_duplicate_count', DEFAULT_MIN_DUPLICATE_COUNT) + if isinstance(minLength, int): + return max(DEFAULT_MIN_DUPLICATE_COUNT, minLength) + else: + return DEFAULT_MIN_DUPLICATE_COUNT + def getIgnoreList(): settings = sublime.load_settings('highlight_duplicates.sublime-settings') ignoreList = settings.get('ignore_list', []) From 94787d61522b68e59227652f483ada7e1a4dba1a Mon Sep 17 00:00:00 2001 From: Nate Mills Date: Tue, 24 Aug 2021 14:54:43 -0700 Subject: [PATCH 8/9] Updating readme. --- readme.md | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/readme.md b/readme.md index 4830c2e..9477bca 100644 --- a/readme.md +++ b/readme.md @@ -71,7 +71,7 @@ _Note:_ By default none of these commands have a key binding, and can only be us ## Options ### Change The Highlighting Color -`Default: "invalid"` +`highlight_duplicates_color: "invalid"` The highlighting color can be changed by providing a scope name such as "invalid", "comment", etc... @@ -81,7 +81,7 @@ it should be defined as a color scope in your theme file. ### Trim White Space -`Default: true` +`trim_white_space: true` If this setting is true, the leading and trailing white space will be removed before being compared to other lines. This setting also affects which lines are selected when using the 'Select Duplicates' command. @@ -98,7 +98,7 @@ However, the following lines would not be counted as duplicates. The reason for ### Ignore Case -`Default: false` +`ignore_case: false` If this setting is true, upper and lower case letters will be considered the same. This setting also affects which lines are selected when using the 'Select Duplicates' command. @@ -110,7 +110,7 @@ For example, if `"ignore_case" : true` the following 2 lines will be counted as ### Min Line Length -`Default: 4` +`min_line_length: 4` Lines with fewer characters than specified in this setting, will be ignored for all functions. Setting this to 1 will cause all non empty lines to be possible duplicates. @@ -128,8 +128,25 @@ For example, by default, only lines 7 and 8 will be selected when using the "sel +### Min Duplicate Count +`min_duplicate_count: 1` + +The number of matching lines, beyond the first, that need to be found in order to be counted as duplicates. + +For example, setting this option to `2`, will make it so only lines 3-5 are highlighted below. + +``` html +1: not this +2: not this +3: this +4: this +5: this +``` + + + ### Ignore List -`Default: []` +`ignore_list: []` Lines matching entires in this list, will be ignored for all functions. Leading and trailing white space, as well as letter case, will be ignored when checking lines against the ignore list. From 72e58905f75cffc874735bea9bf7ab6ce14bc09f Mon Sep 17 00:00:00 2001 From: Nate Mills Date: Tue, 24 Aug 2021 14:55:40 -0700 Subject: [PATCH 9/9] Updating release message. --- messages/1.5.0.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/messages/1.5.0.txt b/messages/1.5.0.txt index 70f2728..5927872 100644 --- a/messages/1.5.0.txt +++ b/messages/1.5.0.txt @@ -1,5 +1,12 @@ Highlight Duplicates v1.5.0 + • Added Min Duplicate Count Setting + + The number of matching lines, beyond the first, that need to be found in order to be counted as duplicates. + + Default: + "min_duplicate_count": 1 + • Added Ignore List Setting Lines matching entries in this list, will be ignored for all functions. Leading and trailing white space, as well as letter case, will be ignored when checking lines against the ignore list.