Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

v7.1.1 #128

Merged
merged 2 commits into from
Apr 26, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
v7.1.1
- Decode content using detected charset
- Update dependencies
  - Update eslint config
- Add node 22 to CI test

Related issue: #123
ndaidong committed Apr 26, 2024
commit a8fe8fffcaba29ca1766e3d4f6d9638669aab65a
2 changes: 0 additions & 2 deletions .eslintignore

This file was deleted.

121 changes: 0 additions & 121 deletions .eslintrc.json

This file was deleted.

2 changes: 1 addition & 1 deletion .github/workflows/ci-test.yml
Original file line number Diff line number Diff line change
@@ -12,7 +12,7 @@ jobs:

strategy:
matrix:
node_version: [18.x, 20.x, 21.x]
node_version: [18.x, 20.x, 22.x]

steps:
- uses: actions/checkout@v4
129 changes: 129 additions & 0 deletions eslint.config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
// eslint.config.js

import eslintjs from '@eslint/js'
import globals from 'globals'

export default [
eslintjs.configs.recommended,
{
languageOptions: {
ecmaVersion: 'latest',
sourceType: 'module',
globals: {
...globals.node,
...globals.browser,
...globals.jest,
Intl: 'readonly',
},
},
ignores: [
'node_modules',
'storage',
'*.cjs',
],
rules: {
'arrow-spacing': ['error', { 'before': true, 'after': true }],
'block-spacing': ['error', 'always'],
'brace-style': ['error', '1tbs', { 'allowSingleLine': true }],
'camelcase': ['error', {
'allow': ['^UNSAFE_'],
'properties': 'never',
'ignoreGlobals': true,
}],
'comma-dangle': ['error', {
'arrays': 'always-multiline',
'objects': 'always-multiline',
'imports': 'never',
'exports': 'never',
'functions': 'never',
}],
'comma-spacing': ['error', { 'before': false, 'after': true }],
'eol-last': 'error',
'eqeqeq': ['error', 'always', { 'null': 'ignore' }],
'func-call-spacing': ['error', 'never'],
'indent': [
'error',
2,
{
'MemberExpression': 1,
'FunctionDeclaration': {
'body': 1,
'parameters': 2,
},
'SwitchCase': 1,
'ignoredNodes': ['TemplateLiteral > *'],
},
],
'key-spacing': ['error', { 'beforeColon': false, 'afterColon': true }],
'keyword-spacing': ['error', { 'before': true, 'after': true }],
'lines-between-class-members': ['error', 'always', { 'exceptAfterSingleLine': true }],
'max-len': [
'error',
{
'code': 120,
'ignoreTrailingComments': true,
'ignoreComments': true,
'ignoreUrls': true,
},
],
'max-lines': [
'error',
{
'max': 500,
'skipBlankLines': true,
'skipComments': false,
},
],
'max-lines-per-function': [
'error',
{
'max': 200,
'skipBlankLines': true,
},
],
'max-params': ['error', 3],
'no-array-constructor': 'error',
'no-mixed-spaces-and-tabs': 'error',
'no-multi-spaces': 'error',
'no-multi-str': 'error',
'no-multiple-empty-lines': [
'error',
{
'max': 1,
'maxEOF': 0,
},
],
'no-restricted-syntax': [
'error',
'WithStatement',
'BinaryExpression[operator=\'in\']',
],
'no-trailing-spaces': 'error',
'no-use-before-define': [
'error',
{
'functions': true,
'classes': true,
'variables': false,
},
],
'no-var': 'warn',
'object-curly-spacing': ['error', 'always'],
'padded-blocks': [
'error',
{
'blocks': 'never',
'switches': 'never',
'classes': 'never',
},
],
'quotes': ['error', 'single'],
'space-before-blocks': ['error', 'always'],
'space-before-function-paren': ['error', 'always'],
'space-infix-ops': 'error',
'space-unary-ops': ['error', { 'words': true, 'nonwords': false }],
'space-in-parens': ['error', 'never'],
'semi': ['error', 'never'],
},
},
]
5 changes: 3 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"version": "7.1.0",
"version": "7.1.1",
"name": "@extractus/feed-extractor",
"description": "To read and normalize RSS/ATOM/JSON feed data",
"homepage": "https://extractor-demos.pages.dev",
@@ -46,7 +46,8 @@
},
"devDependencies": {
"esbuild": "^0.20.2",
"eslint": "^8.57.0",
"eslint": "^9.1.1",
"globals": "^15.0.0",
"https-proxy-agent": "^7.0.4",
"jest": "^29.7.0",
"nock": "^13.5.4"
6 changes: 3 additions & 3 deletions src/utils/linker.js
Original file line number Diff line number Diff line change
@@ -4,7 +4,7 @@ export const isValid = (url = '') => {
try {
const ourl = new URL(url)
return ourl !== null && ourl.protocol.startsWith('http')
} catch (err) {
} catch {
return false
}
}
@@ -13,7 +13,7 @@ export const absolutify = (fullUrl = '', relativeUrl = '') => {
try {
const result = new URL(relativeUrl, fullUrl)
return result.toString()
} catch (err) {
} catch {
return ''
}
}
@@ -87,7 +87,7 @@ export const purify = (url) => {
})

return pureUrl.toString().replace(pureUrl.hash, '')
} catch (err) {
} catch {
return null
}
}
2 changes: 1 addition & 1 deletion src/utils/normalizer.js
Original file line number Diff line number Diff line change
@@ -16,7 +16,7 @@ import { absolutify, isValid as isValidUrl, purify as purifyUrl } from './linker
export const toISODateString = (dstr) => {
try {
return dstr ? (new Date(dstr)).toISOString() : ''
} catch (err) {
} catch {
return ''
}
}
11 changes: 8 additions & 3 deletions src/utils/retrieve.js
Original file line number Diff line number Diff line change
@@ -32,17 +32,22 @@ export default async (url, options = {}) => {
throw new Error(`Request failed with error code ${status}`)
}
const contentType = res.headers.get('content-type')
const text = await res.text()
const buffer = await res.arrayBuffer()
const text = buffer ? Buffer.from(buffer).toString().trim() : ''

if (/(\+|\/)(xml|html)/.test(contentType)) {
return { type: 'xml', text: text.trim(), status, contentType }
const arr = contentType.split('charset=')
const charset = arr.length === 2 ? arr[1].trim() : 'utf8'
const decoder = new TextDecoder(charset)
const xml = decoder.decode(buffer)
return { type: 'xml', text: xml.trim(), status, contentType }
}

if (/(\+|\/)json/.test(contentType)) {
try {
const data = JSON.parse(text)
return { type: 'json', json: data, status, contentType }
} catch (err) {
} catch {
throw new Error('Failed to convert data to JSON object')
}
}