Merge pull request #128 from extractus/7.1.1

v7.1.1

Author: ndaidong

.eslintignore

@@ -12,7 +12,7 @@ jobs:
 
  strategy:
  matrix:
- node_version: [18.x, 20.x, 21.x]
+ node_version: [18.x, 20.x, 22.x]
 
  steps:
  - uses: actions/checkout@v4
@@ -27,8 +27,6 @@ jobs:
  PROXY_SERVER: ${{ secrets.PROXY_SERVER }}
  run: |
  npm install
- npm run lint
- npm run build --if-present
  npm run test
 
  - name: Coveralls Parallel

.eslintrc.json

@@ -0,0 +1,129 @@
+// eslint.config.js
+
+import eslintjs from '@eslint/js'
+import globals from 'globals'
+
+export default [
+ eslintjs.configs.recommended,
+ {
+ languageOptions: {
+ ecmaVersion: 'latest',
+ sourceType: 'module',
+ globals: {
+ ...globals.node,
+ ...globals.browser,
+ ...globals.jest,
+ Intl: 'readonly',
+ },
+ },
+ ignores: [
+ 'node_modules',
+ 'storage',
+ '*.cjs',
+ ],
+ rules: {
+ 'arrow-spacing': ['error', { 'before': true, 'after': true }],
+ 'block-spacing': ['error', 'always'],
+ 'brace-style': ['error', '1tbs', { 'allowSingleLine': true }],
+ 'camelcase': ['error', {
+ 'allow': ['^UNSAFE_'],
+ 'properties': 'never',
+ 'ignoreGlobals': true,
+ }],
+ 'comma-dangle': ['error', {
+ 'arrays': 'always-multiline',
+ 'objects': 'always-multiline',
+ 'imports': 'never',
+ 'exports': 'never',
+ 'functions': 'never',
+ }],
+ 'comma-spacing': ['error', { 'before': false, 'after': true }],
+ 'eol-last': 'error',
+ 'eqeqeq': ['error', 'always', { 'null': 'ignore' }],
+ 'func-call-spacing': ['error', 'never'],
+ 'indent': [
+ 'error',
+ 2,
+ {
+ 'MemberExpression': 1,
+ 'FunctionDeclaration': {
+ 'body': 1,
+ 'parameters': 2,
+ },
+ 'SwitchCase': 1,
+ 'ignoredNodes': ['TemplateLiteral > *'],
+ },
+ ],
+ 'key-spacing': ['error', { 'beforeColon': false, 'afterColon': true }],
+ 'keyword-spacing': ['error', { 'before': true, 'after': true }],
+ 'lines-between-class-members': ['error', 'always', { 'exceptAfterSingleLine': true }],
+ 'max-len': [
+ 'error',
+ {
+ 'code': 120,
+ 'ignoreTrailingComments': true,
+ 'ignoreComments': true,
+ 'ignoreUrls': true,
+ },
+ ],
+ 'max-lines': [
+ 'error',
+ {
+ 'max': 500,
+ 'skipBlankLines': true,
+ 'skipComments': false,
+ },
+ ],
+ 'max-lines-per-function': [
+ 'error',
+ {
+ 'max': 200,
+ 'skipBlankLines': true,
+ },
+ ],
+ 'max-params': ['error', 3],
+ 'no-array-constructor': 'error',
+ 'no-mixed-spaces-and-tabs': 'error',
+ 'no-multi-spaces': 'error',
+ 'no-multi-str': 'error',
+ 'no-multiple-empty-lines': [
+ 'error',
+ {
+ 'max': 1,
+ 'maxEOF': 0,
+ },
+ ],
+ 'no-restricted-syntax': [
+ 'error',
+ 'WithStatement',
+ 'BinaryExpression[operator=\'in\']',
+ ],
+ 'no-trailing-spaces': 'error',
+ 'no-use-before-define': [
+ 'error',
+ {
+ 'functions': true,
+ 'classes': true,
+ 'variables': false,
+ },
+ ],
+ 'no-var': 'warn',
+ 'object-curly-spacing': ['error', 'always'],
+ 'padded-blocks': [
+ 'error',
+ {
+ 'blocks': 'never',
+ 'switches': 'never',
+ 'classes': 'never',
+ },
+ ],
+ 'quotes': ['error', 'single'],
+ 'space-before-blocks': ['error', 'always'],
+ 'space-before-function-paren': ['error', 'always'],
+ 'space-infix-ops': 'error',
+ 'space-unary-ops': ['error', { 'words': true, 'nonwords': false }],
+ 'space-in-parens': ['error', 'never'],
+ 'semi': ['error', 'never'],
+ },
+ },
+]

.github/workflows/ci-test.yml

@@ -1,5 +1,5 @@
 {
- "version": "7.1.0",
+ "version": "7.1.1",
  "name": "@extractus/feed-extractor",
  "description": "To read and normalize RSS/ATOM/JSON feed data",
  "homepage": "https://extractor-demos.pages.dev",
@@ -46,7 +46,8 @@
  },
  "devDependencies": {
  "esbuild": "^0.20.2",
- "eslint": "^8.57.0",
+ "eslint": "^9.1.1",
+ "globals": "^15.0.0",
  "https-proxy-agent": "^7.0.4",
  "jest": "^29.7.0",
  "nock": "^13.5.4"

eslint.config.js

@@ -4,7 +4,7 @@ export const isValid = (url = '') => {
  try {
  const ourl = new URL(url)
  return ourl !== null && ourl.protocol.startsWith('http')
- } catch (err) {
+ } catch {
  return false
  }
 }
@@ -13,7 +13,7 @@ export const absolutify = (fullUrl = '', relativeUrl = '') => {
  try {
  const result = new URL(relativeUrl, fullUrl)
  return result.toString()
- } catch (err) {
+ } catch {
  return ''
  }
 }
@@ -87,7 +87,7 @@ export const purify = (url) => {
  })
 
  return pureUrl.toString().replace(pureUrl.hash, '')
- } catch (err) {
+ } catch {
  return null
  }
 }

package.json

@@ -16,7 +16,7 @@ import { absolutify, isValid as isValidUrl, purify as purifyUrl } from './linker
 export const toISODateString = (dstr) => {
  try {
  return dstr ? (new Date(dstr)).toISOString() : ''
- } catch (err) {
+ } catch {
  return ''
  }
 }

src/utils/linker.js

@@ -32,17 +32,22 @@ export default async (url, options = {}) => {
  throw new Error(`Request failed with error code ${status}`)
  }
  const contentType = res.headers.get('content-type')
- const text = await res.text()
+ const buffer = await res.arrayBuffer()
+ const text = buffer ? Buffer.from(buffer).toString().trim() : ''
 
  if (/(\+|\/)(xml|html)/.test(contentType)) {
- return { type: 'xml', text: text.trim(), status, contentType }
+ const arr = contentType.split('charset=')
+ const charset = arr.length === 2 ? arr[1].trim() : 'utf8'
+ const decoder = new TextDecoder(charset)
+ const xml = decoder.decode(buffer)
+ return { type: 'xml', text: xml.trim(), status, contentType }
  }
 
  if (/(\+|\/)json/.test(contentType)) {
  try {
  const data = JSON.parse(text)
  return { type: 'json', json: data, status, contentType }
- } catch (err) {
+ } catch {
  throw new Error('Failed to convert data to JSON object')
  }
  }