add deduplication feature

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## 2.0.0
 
+- Added
+  - Option to ignore duplicate URLs on open
 - Changed
   - Complete rewrite in Vue.js
 

src/browseraction/BrowserAction.vue

@@ -13,13 +13,15 @@ Promise.all([
   browser.storage.local.get(BrowserStorageKey.lazyload),
   browser.storage.local.get(BrowserStorageKey.random),
   browser.storage.local.get(BrowserStorageKey.reverse),
-  browser.storage.local.get(BrowserStorageKey.preserve)
+  browser.storage.local.get(BrowserStorageKey.preserve),
+  browser.storage.local.get(BrowserStorageKey.deduplicate)
 ]).then((data) => {
   store.urlList = data[0][BrowserStorageKey.urlList] ?? ''
   store.lazyLoadingChecked = data[1][BrowserStorageKey.lazyload] ?? false
   store.loadInRandomOrderChecked = data[2][BrowserStorageKey.random] ?? false
   store.loadInReverseOrderChecked = data[3][BrowserStorageKey.reverse] ?? false
   store.preserveInputChecked = data[4][BrowserStorageKey.preserve] ?? false
+  store.deduplicateURLsChecked = data[5][BrowserStorageKey.deduplicate] ?? false
 
   isStoredValuesLoaded.value = true
 })

src/browseraction/__tests__/BrowserAction.spec.ts

@@ -35,6 +35,7 @@ describe('browser action', () => {
     expect(wrapper.text()).toContain('Load in random order')
     expect(wrapper.text()).toContain('Load in reverse order')
     expect(wrapper.text()).toContain('Preserve input')
+    expect(wrapper.text()).toContain('Ignore duplicate URLs')
   })
 
   describe('features', () => {
@@ -95,33 +96,77 @@ describe('browser action', () => {
       expect(
         (wrapper.find('input[type="checkbox"]#preserve').element as HTMLInputElement).checked
       ).toBeFalsy()
+      expect(
+        (wrapper.find('input[type="checkbox"]#deduplicate').element as HTMLInputElement).checked
+      ).toBeFalsy()
     })
 
     const renderWithStoredValuesTestCases = [
       {
         storeKey: BrowserStorageKey.urlList,
         value: 'foobar',
-        expectedStates: { lazyLoad: false, random: false, reverse: false, preserve: false }
+        expectedStates: {
+          lazyLoad: false,
+          random: false,
+          reverse: false,
+          preserve: false,
+          deduplicate: false
+        }
       },
       {
         storeKey: BrowserStorageKey.lazyload,
         value: 'true',
-        expectedStates: { lazyLoad: true, random: false, reverse: false, preserve: false }
+        expectedStates: {
+          lazyLoad: true,
+          random: false,
+          reverse: false,
+          preserve: false,
+          deduplicate: false
+        }
       },
       {
         storeKey: BrowserStorageKey.random,
         value: 'true',
-        expectedStates: { lazyLoad: false, random: true, reverse: false, preserve: false }
+        expectedStates: {
+          lazyLoad: false,
+          random: true,
+          reverse: false,
+          preserve: false,
+          deduplicate: false
+        }
       },
       {
         storeKey: BrowserStorageKey.reverse,
         value: 'true',
-        expectedStates: { lazyLoad: false, random: false, reverse: true, preserve: false }
+        expectedStates: {
+          lazyLoad: false,
+          random: false,
+          reverse: true,
+          preserve: false,
+          deduplicate: false
+        }
       },
       {
         storeKey: BrowserStorageKey.preserve,
         value: 'true',
-        expectedStates: { lazyLoad: false, random: false, reverse: false, preserve: true }
+        expectedStates: {
+          lazyLoad: false,
+          random: false,
+          reverse: false,
+          preserve: true,
+          deduplicate: false
+        }
+      },
+      {
+        storeKey: BrowserStorageKey.deduplicate,
+        value: 'true',
+        expectedStates: {
+          lazyLoad: false,
+          random: false,
+          reverse: false,
+          preserve: false,
+          deduplicate: true
+        }
       }
     ]
     it.each(renderWithStoredValuesTestCases)(
@@ -150,6 +195,9 @@ describe('browser action', () => {
         expect(
           (wrapper.find('input[type="checkbox"]#preserve').element as HTMLInputElement).checked
         ).toBe(expectedStates.preserve)
+        expect(
+          (wrapper.find('input[type="checkbox"]#deduplicate').element as HTMLInputElement).checked
+        ).toBe(expectedStates.deduplicate)
       }
     )
   })
@@ -172,7 +220,8 @@ describe('browser action', () => {
       { checkboxId: 'lazyLoad', storeKey: BrowserStorageKey.lazyload },
       { checkboxId: 'random', storeKey: BrowserStorageKey.random },
       { checkboxId: 'reverse', storeKey: BrowserStorageKey.reverse },
-      { checkboxId: 'preserve', storeKey: BrowserStorageKey.preserve }
+      { checkboxId: 'preserve', storeKey: BrowserStorageKey.preserve },
+      { checkboxId: 'deduplicate', storeKey: BrowserStorageKey.deduplicate }
     ]
     it.each(storeCheckStateTestCases)(
       'stores $checkboxId check state',

src/browseraction/__tests__/logic/load.spec.ts

@@ -1,6 +1,6 @@
 import browser from 'webextension-polyfill'
 import { afterEach, describe, expect, it, vi } from 'vitest'
-import { getTabCount, loadSites } from '@/browseraction/components/logic/load'
+import { getTabCount, getURLsFromText, loadSites } from '@/browseraction/components/logic/load'
 
 vi.mock('webextension-polyfill', () => ({
   default: { tabs: { create: vi.fn() }, runtime: { getURL: (val: string) => val } }
@@ -16,7 +16,7 @@ describe('load tabs', () => {
   })
 
   it('loads tabs in sequence', async () => {
-    loadSites(urlList, false, false, false)
+    loadSites(urlList, false, false, false, false)
 
     expect(browser.tabs.create).toHaveBeenNthCalledWith(1, {
       url: url1,
@@ -26,10 +26,11 @@ describe('load tabs', () => {
       url: url2,
       active: false
     })
+    expect(browser.tabs.create).toHaveBeenCalledTimes(2)
   })
 
   it('lazy loads tabs', async () => {
-    loadSites(urlList, true, false, false)
+    loadSites(urlList, true, false, false, false)
 
     expect(browser.tabs.create).toHaveBeenCalledWith({
       url: 'lazyloading.html#' + url1,
@@ -39,16 +40,17 @@ describe('load tabs', () => {
       url: 'lazyloading.html#' + url2,
       active: false
     })
+    expect(browser.tabs.create).toHaveBeenCalledTimes(2)
   })
 
   it('loads tabs in random order', async () => {
-    loadSites(urlList, false, true, false)
+    loadSites(urlList, false, true, false, false)
 
     expect(browser.tabs.create).toHaveBeenCalledTimes(2)
   })
 
   it('loads tabs in reverse order', async () => {
-    loadSites(urlList, false, false, true)
+    loadSites(urlList, false, false, true, false)
 
     expect(browser.tabs.create).toHaveBeenNthCalledWith(1, {
       url: url2,
@@ -58,10 +60,25 @@ describe('load tabs', () => {
       url: url1,
       active: false
     })
+    expect(browser.tabs.create).toHaveBeenCalledTimes(2)
+  })
+
+  it('loads tabs and deduplicate', async () => {
+    loadSites(`${urlList}\n${urlList}\n${urlList}\n${urlList}`, false, false, false, true)
+
+    expect(browser.tabs.create).toHaveBeenNthCalledWith(1, {
+      url: url1,
+      active: false
+    })
+    expect(browser.tabs.create).toHaveBeenNthCalledWith(2, {
+      url: url2,
+      active: false
+    })
+    expect(browser.tabs.create).toHaveBeenCalledTimes(2)
   })
 
   it('appends http protocol if protocol does not exist', async () => {
-    loadSites('test.de', false, false, true)
+    loadSites('test.de', false, false, true, false)
 
     expect(browser.tabs.create).toHaveBeenNthCalledWith(1, {
       url: 'http://test.de',
@@ -70,10 +87,24 @@ describe('load tabs', () => {
   })
 
   it('determines tab count correctly', () => {
-    expect(getTabCount('')).toBe('0')
-    expect(getTabCount(url1)).toBe('1')
-    expect(getTabCount(urlList)).toBe('2')
-    expect(getTabCount(`url1\n`.repeat(5000))).toBe('5000')
-    expect(getTabCount(`url1\n`.repeat(5001))).toBe('> 5000')
+    expect(getTabCount('', false)).toBe('0')
+    expect(getTabCount(url1, false)).toBe('1')
+    expect(getTabCount(urlList, false)).toBe('2')
+    expect(getTabCount(`url1\n`.repeat(5000), false)).toBe('5000')
+    expect(getTabCount(`url1\n`.repeat(5001), false)).toBe('> 5000')
+    expect(getTabCount(`${urlList}\n`.repeat(100), true)).toBe('2')
+  })
+
+  it('gets urls from text', () => {
+    expect(getURLsFromText('', false)).toEqual([])
+    expect(getURLsFromText('\n\n', false)).toEqual([])
+    expect(getURLsFromText(urlList, false)).toEqual([url1, url2])
+    expect(getURLsFromText(`\n\n\n${urlList}\n\n\n${urlList}\n\n`, false)).toEqual([
+      url1,
+      url2,
+      url1,
+      url2
+    ])
+    expect(getURLsFromText(`\n\n\n${urlList}\n\n\n${urlList}\n\n`, true)).toEqual([url1, url2])
   })
 })

src/browseraction/components/ActionBar.vue

@@ -32,7 +32,8 @@ export default {
         store.urlList,
         store.lazyLoadingChecked,
         store.loadInRandomOrderChecked,
-        store.loadInReverseOrderChecked
+        store.loadInReverseOrderChecked,
+        store.deduplicateURLsChecked
       )
     },
     setUrlListInputData() {
@@ -41,7 +42,7 @@ export default {
   },
   computed: {
     tabCount: function () {
-      return getTabCount(store.urlList)
+      return getTabCount(store.urlList, store.deduplicateURLsChecked)
     }
   }
 }

src/browseraction/components/OptionBar.vue

@@ -34,6 +34,16 @@ import { store } from '@/browseraction/components/store/store'
       />
       Load in reverse order</label
     >
+    <label class="checkbox"
+      ><input
+        type="checkbox"
+        id="deduplicate"
+        tabindex="5"
+        :checked="store.deduplicateURLsChecked"
+        @change="checkDeduplicateURLs"
+      />
+      Ignore duplicate URLs</label
+    >
   </section>
   <section>
     <label class="checkbox"
@@ -71,6 +81,11 @@ export default {
       this.$nextTick(() => {
         store.setPreserveInputChecked((event?.target as HTMLInputElement).checked)
       })
+    },
+    checkDeduplicateURLs(event: Event) {
+      this.$nextTick(() => {
+        store.setDeduplicateURLsChecked((event?.target as HTMLInputElement).checked)
+      })
     }
   }
 }

src/browseraction/components/logic/load.ts

@@ -15,22 +15,23 @@ const shuffle = (a: string[]) => {
   return a
 }
 
-export const URL_LINE_SPLIT_REGEX = /\r\n?|\n/g
 /**
  * Loads sites in new background tabs
  * @param text Text containing one URL per line
  * @param lazyloading Lazy-load tabs
  * @param random Open tabs in random order
  * @param reverse Open tabs in reverse order
+ * @param deduplicate Ignores duplicate URLs on open
  */
 export const loadSites = (
   text: string,
   lazyloading: boolean,
   random: boolean,
-  reverse: boolean
+  reverse: boolean,
+  deduplicate: boolean
 ): void => {
   const urlschemes = ['http', 'https', 'file', 'view-source']
-  let urls = text.split(URL_LINE_SPLIT_REGEX)
+  let urls = getURLsFromText(text, deduplicate)
 
   if (reverse) {
     urls = urls.reverse()
@@ -65,16 +66,22 @@ export const loadSites = (
   }
 }
 
-export const getTabCount = (text: string) => {
+export const getTabCount = (text: string, deduplicate: boolean) => {
   let tabCount = '0'
   if (text) {
-    const lines = text.split(URL_LINE_SPLIT_REGEX).filter((line) => line !== '')
-    if (lines.length <= 5000) {
+    const urls = getURLsFromText(text, deduplicate)
+    if (urls.length <= 5000) {
       // limit for performance reasons
-      tabCount = String(lines.filter((line: string) => line.trim() !== '').length)
+      tabCount = String(urls.length)
     } else {
       tabCount = '> 5000'
     }
   }
   return tabCount
 }
+
+export const getURLsFromText = (text: string, deduplicate: boolean): string[] => {
+  const urlLineSplitRegex = /\r\n?|\n/g
+  const urls = text.split(urlLineSplitRegex).filter((line) => line.trim() !== '')
+  return deduplicate ? Array.from(new Set(urls)) : urls
+}

src/browseraction/components/store/browser-storage.ts

@@ -3,5 +3,6 @@ export enum BrowserStorageKey {
   lazyload = 'lazyload',
   random = 'random',
   reverse = 'reverse',
-  preserve = 'preserve'
+  preserve = 'preserve',
+  deduplicate = 'deduplicate'
 }

src/browseraction/components/store/store.ts

@@ -8,6 +8,7 @@ export const store = reactive({
   loadInRandomOrderChecked: false,
   loadInReverseOrderChecked: false,
   preserveInputChecked: false,
+  deduplicateURLsChecked: false,
   setUrlList(value: string) {
     this.urlList = value
     if (store.preserveInputChecked) {
@@ -30,5 +31,9 @@ export const store = reactive({
     this.preserveInputChecked = value
     browser.storage.local.set({ [BrowserStorageKey.preserve]: value })
     browser.storage.local.set({ [BrowserStorageKey.urlList]: value ? store.urlList : '' })
+  },
+  setDeduplicateURLsChecked(value: boolean) {
+    this.deduplicateURLsChecked = value
+    browser.storage.local.set({ [BrowserStorageKey.deduplicate]: value })
   }
 })