v7.1.1

- Decode content using detected charset - Update dependencies - Update eslint config - Add node 22 to CI test Related issue: #123
extractus · Apr 26, 2024 · a8fe8ff · a8fe8ff
1 parent 38a50fe
commit a8fe8ff
Show file tree

Hide file tree

Showing 8 changed files with 145 additions and 133 deletions.
diff --git a/.eslintignore b/.eslintignore
diff --git a/.eslintrc.json b/.eslintrc.json
diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        node_version: [18.x, 20.x, 21.x]
+        node_version: [18.x, 20.x, 22.x]
 
     steps:
     - uses: actions/checkout@v4

diff --git a/eslint.config.js b/eslint.config.js
@@ -0,0 +1,129 @@
+// eslint.config.js
+
+import eslintjs from '@eslint/js'
+import globals from 'globals'
+
+export default [
+  eslintjs.configs.recommended,
+  {
+    languageOptions: {
+      ecmaVersion: 'latest',
+      sourceType: 'module',
+      globals: {
+        ...globals.node,
+        ...globals.browser,
+        ...globals.jest,
+        Intl: 'readonly',
+      },
+    },
+    ignores: [
+      'node_modules',
+      'storage',
+      '*.cjs',
+    ],
+    rules: {
+      'arrow-spacing': ['error', { 'before': true, 'after': true }],
+      'block-spacing': ['error', 'always'],
+      'brace-style': ['error', '1tbs', { 'allowSingleLine': true }],
+      'camelcase': ['error', {
+        'allow': ['^UNSAFE_'],
+        'properties': 'never',
+        'ignoreGlobals': true,
+      }],
+      'comma-dangle': ['error', {
+        'arrays': 'always-multiline',
+        'objects': 'always-multiline',
+        'imports': 'never',
+        'exports': 'never',
+        'functions': 'never',
+      }],
+      'comma-spacing': ['error', { 'before': false, 'after': true }],
+      'eol-last': 'error',
+      'eqeqeq': ['error', 'always', { 'null': 'ignore' }],
+      'func-call-spacing': ['error', 'never'],
+      'indent': [
+        'error',
+        2,
+        {
+          'MemberExpression': 1,
+          'FunctionDeclaration': {
+            'body': 1,
+            'parameters': 2,
+          },
+          'SwitchCase': 1,
+          'ignoredNodes': ['TemplateLiteral > *'],
+        },
+      ],
+      'key-spacing': ['error', { 'beforeColon': false, 'afterColon': true }],
+      'keyword-spacing': ['error', { 'before': true, 'after': true }],
+      'lines-between-class-members': ['error', 'always', { 'exceptAfterSingleLine': true }],
+      'max-len': [
+        'error',
+        {
+          'code': 120,
+          'ignoreTrailingComments': true,
+          'ignoreComments': true,
+          'ignoreUrls': true,
+        },
+      ],
+      'max-lines': [
+        'error',
+        {
+          'max': 500,
+          'skipBlankLines': true,
+          'skipComments': false,
+        },
+      ],
+      'max-lines-per-function': [
+        'error',
+        {
+          'max': 200,
+          'skipBlankLines': true,
+        },
+      ],
+      'max-params': ['error', 3],
+      'no-array-constructor': 'error',
+      'no-mixed-spaces-and-tabs': 'error',
+      'no-multi-spaces': 'error',
+      'no-multi-str': 'error',
+      'no-multiple-empty-lines': [
+        'error',
+        {
+          'max': 1,
+          'maxEOF': 0,
+        },
+      ],
+      'no-restricted-syntax': [
+        'error',
+        'WithStatement',
+        'BinaryExpression[operator=\'in\']',
+      ],
+      'no-trailing-spaces': 'error',
+      'no-use-before-define': [
+        'error',
+        {
+          'functions': true,
+          'classes': true,
+          'variables': false,
+        },
+      ],
+      'no-var': 'warn',
+      'object-curly-spacing': ['error', 'always'],
+      'padded-blocks': [
+        'error',
+        {
+          'blocks': 'never',
+          'switches': 'never',
+          'classes': 'never',
+        },
+      ],
+      'quotes': ['error', 'single'],
+      'space-before-blocks': ['error', 'always'],
+      'space-before-function-paren': ['error', 'always'],
+      'space-infix-ops': 'error',
+      'space-unary-ops': ['error', { 'words': true, 'nonwords': false }],
+      'space-in-parens': ['error', 'never'],
+      'semi': ['error', 'never'],
+    },
+  },
+]
diff --git a/package.json b/package.json
@@ -1,5 +1,5 @@
 {
-  "version": "7.1.0",
+  "version": "7.1.1",
   "name": "@extractus/feed-extractor",
   "description": "To read and normalize RSS/ATOM/JSON feed data",
   "homepage": "https://extractor-demos.pages.dev",
@@ -46,7 +46,8 @@
   },
   "devDependencies": {
     "esbuild": "^0.20.2",
-    "eslint": "^8.57.0",
+    "eslint": "^9.1.1",
+    "globals": "^15.0.0",
     "https-proxy-agent": "^7.0.4",
     "jest": "^29.7.0",
     "nock": "^13.5.4"

diff --git a/src/utils/linker.js b/src/utils/linker.js
@@ -4,7 +4,7 @@ export const isValid = (url = '') => {
   try {
     const ourl = new URL(url)
     return ourl !== null && ourl.protocol.startsWith('http')
-  } catch (err) {
+  } catch {
     return false
   }
 }
@@ -13,7 +13,7 @@ export const absolutify = (fullUrl = '', relativeUrl = '') => {
   try {
     const result = new URL(relativeUrl, fullUrl)
     return result.toString()
-  } catch (err) {
+  } catch {
     return ''
   }
 }
@@ -87,7 +87,7 @@ export const purify = (url) => {
     })
 
     return pureUrl.toString().replace(pureUrl.hash, '')
-  } catch (err) {
+  } catch {
     return null
   }
 }
diff --git a/src/utils/normalizer.js b/src/utils/normalizer.js
@@ -16,7 +16,7 @@ import { absolutify, isValid as isValidUrl, purify as purifyUrl } from './linker
 export const toISODateString = (dstr) => {
   try {
     return dstr ? (new Date(dstr)).toISOString() : ''
-  } catch (err) {
+  } catch {
     return ''
   }
 }

diff --git a/src/utils/retrieve.js b/src/utils/retrieve.js
@@ -32,17 +32,22 @@ export default async (url, options = {}) => {
     throw new Error(`Request failed with error code ${status}`)
   }
   const contentType = res.headers.get('content-type')
-  const text = await res.text()
+  const buffer = await res.arrayBuffer()
+  const text = buffer ? Buffer.from(buffer).toString().trim() : ''
 
   if (/(\+|\/)(xml|html)/.test(contentType)) {
-    return { type: 'xml', text: text.trim(), status, contentType }
+    const arr = contentType.split('charset=')
+    const charset = arr.length === 2 ? arr[1].trim() : 'utf8'
+    const decoder = new TextDecoder(charset)
+    const xml = decoder.decode(buffer)
+    return { type: 'xml', text: xml.trim(), status, contentType }
   }
 
   if (/(\+|\/)json/.test(contentType)) {
     try {
       const data = JSON.parse(text)
       return { type: 'json', json: data, status, contentType }
-    } catch (err) {
+    } catch {
       throw new Error('Failed to convert data to JSON object')
     }
   }