diff --git a/package-lock.json b/package-lock.json index 6807d794..fa928b20 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,8 +25,7 @@ "react-markdown": "^10.1.0", "react-router-dom": "^7.9.3", "reading-time-estimator": "^1.14.0", - "remark-gfm": "^4.0.1", - "url-metadata": "^5.2.2" + "remark-gfm": "^4.0.1" }, "devDependencies": { "@types/react": "^18.2.43", @@ -2094,12 +2093,6 @@ "baseline-browser-mapping": "dist/cli.js" } }, - "node_modules/boolbase": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", - "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", - "license": "ISC" - }, "node_modules/brace-expansion": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", @@ -2255,79 +2248,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/cheerio": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.1.2.tgz", - "integrity": "sha512-IkxPpb5rS/d1IiLbHMgfPuS0FgiWTtFIm/Nj+2woXDLTZ7fOT2eqzgYbdMlLweqlHbsZjxEChoVK+7iph7jyQg==", - "license": "MIT", - "dependencies": { - "cheerio-select": "^2.1.0", - "dom-serializer": "^2.0.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "encoding-sniffer": "^0.2.1", - "htmlparser2": "^10.0.0", - "parse5": "^7.3.0", - "parse5-htmlparser2-tree-adapter": "^7.1.0", - "parse5-parser-stream": "^7.1.2", - "undici": "^7.12.0", - "whatwg-mimetype": "^4.0.0" - }, - "engines": { - "node": ">=20.18.1" - }, - "funding": { - "url": "https://github.com/cheeriojs/cheerio?sponsor=1" - } - }, - "node_modules/cheerio-select": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", - "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0", - "css-select": "^5.1.0", - "css-what": "^6.1.0", - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/cheerio/node_modules/entities": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", - "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/cheerio/node_modules/htmlparser2": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz", - "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==", - "funding": [ - "https://github.com/fb55/htmlparser2?sponsor=1", - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.1", - "entities": "^6.0.0" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -2396,34 +2316,6 @@ "node": ">= 8" } }, - "node_modules/css-select": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", - "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0", - "css-what": "^6.1.0", - "domhandler": "^5.0.2", - "domutils": "^3.0.1", - "nth-check": "^2.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/css-what": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", - "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", - "license": "BSD-2-Clause", - "engines": { - "node": ">= 6" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, "node_modules/csstype": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", @@ -2596,19 +2488,6 @@ "dev": true, "license": "ISC" }, - "node_modules/encoding-sniffer": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz", - "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==", - "license": "MIT", - "dependencies": { - "iconv-lite": "^0.6.3", - "whatwg-encoding": "^3.1.1" - }, - "funding": { - "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" - } - }, "node_modules/entities": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", @@ -3241,18 +3120,6 @@ "entities": "^4.4.0" } }, - "node_modules/iconv-lite": { - "version": "0.6.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", - "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -3315,15 +3182,6 @@ "integrity": "sha512-0aO8FkhNZlj/ZIbNi7Lxxr12obT7cL1moPfE4tg1LkX7LlLfC6DeX4l2ZEud1ukP9jNQyNnfzQVqwbwmAATY4Q==", "license": "MIT" }, - "node_modules/ipaddr.js": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz", - "integrity": "sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==", - "license": "MIT", - "engines": { - "node": ">= 10" - } - }, "node_modules/is-alphabetical": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", @@ -4531,26 +4389,6 @@ "dev": true, "license": "MIT" }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/node-releases": { "version": "2.0.23", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.23.tgz", @@ -4661,18 +4499,6 @@ "integrity": "sha512-78BTryCLcLYv96ONU8Ws3Q1JzjlAt+43pWQhIl86xZmWeegYCNLPml7yQ+gG3vR6V5h4XGj+TxO+SS5dsThQIA==", "license": "MIT" }, - "node_modules/nth-check": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", - "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0" - }, - "funding": { - "url": "https://github.com/fb55/nth-check?sponsor=1" - } - }, "node_modules/observable-hooks": { "version": "4.2.4", "resolved": "https://registry.npmjs.org/observable-hooks/-/observable-hooks-4.2.4.tgz", @@ -4788,55 +4614,6 @@ "integrity": "sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==", "license": "MIT" }, - "node_modules/parse5": { - "version": "7.3.0", - "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", - "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", - "license": "MIT", - "dependencies": { - "entities": "^6.0.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/parse5-htmlparser2-tree-adapter": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", - "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", - "license": "MIT", - "dependencies": { - "domhandler": "^5.0.3", - "parse5": "^7.0.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/parse5-parser-stream": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", - "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", - "license": "MIT", - "dependencies": { - "parse5": "^7.0.0" - }, - "funding": { - "url": "https://github.com/inikulin/parse5?sponsor=1" - } - }, - "node_modules/parse5/node_modules/entities": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", - "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -5184,15 +4961,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/request-filtering-agent": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/request-filtering-agent/-/request-filtering-agent-2.0.1.tgz", - "integrity": "sha512-QvD3qwthEt9J+2hCdQ3wTn3Z/ZsgyiMECjY9yVJ0F8FtnGfNQG+dRz65eKayYRHIRQ6OGjH8Zuqr1lw7G6pz1Q==", - "license": "MIT", - "dependencies": { - "ipaddr.js": "^2.1.0" - } - }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -5306,12 +5074,6 @@ "tslib": "^2.1.0" } }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" - }, "node_modules/sanitize-html": { "version": "2.17.0", "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.17.0.tgz", @@ -5497,12 +5259,6 @@ "node": ">=8.0" } }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, "node_modules/trim-lines": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", @@ -5582,15 +5338,6 @@ "node": ">=14.17" } }, - "node_modules/undici": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.16.0.tgz", - "integrity": "sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g==", - "license": "MIT", - "engines": { - "node": ">=20.18.1" - } - }, "node_modules/unified": { "version": "11.0.5", "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", @@ -5719,20 +5466,6 @@ "punycode": "^2.1.0" } }, - "node_modules/url-metadata": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/url-metadata/-/url-metadata-5.2.2.tgz", - "integrity": "sha512-/Lq+IIRU7PKh3pa/y1jskR++Wd6v5vYWboUCPpvczCrOJ7dpCBBWIGanE2M6LaIBcvuQgbzoUzUOT/1yokWtuA==", - "license": "MIT", - "dependencies": { - "cheerio": "^1.0.0-rc.12", - "node-fetch": "^2.7.0", - "request-filtering-agent": "2.0.1" - }, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/vfile": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", @@ -5821,43 +5554,6 @@ } } }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "node_modules/whatwg-encoding": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", - "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", - "license": "MIT", - "dependencies": { - "iconv-lite": "0.6.3" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/whatwg-mimetype": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", - "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", diff --git a/package.json b/package.json index b16803d9..8ba26db2 100644 --- a/package.json +++ b/package.json @@ -28,8 +28,7 @@ "react-markdown": "^10.1.0", "react-router-dom": "^7.9.3", "reading-time-estimator": "^1.14.0", - "remark-gfm": "^4.0.1", - "url-metadata": "^5.2.2" + "remark-gfm": "^4.0.1" }, "devDependencies": { "@types/react": "^18.2.43", diff --git a/src/components/AddBookmarkModal.tsx b/src/components/AddBookmarkModal.tsx index c3824350..5d6e7c0c 100644 --- a/src/components/AddBookmarkModal.tsx +++ b/src/components/AddBookmarkModal.tsx @@ -2,13 +2,47 @@ import React, { useState, useEffect, useRef } from 'react' import { FontAwesomeIcon } from '@fortawesome/react-fontawesome' import { faTimes, faSpinner } from '@fortawesome/free-solid-svg-icons' import IconButton from './IconButton' -import urlMetadata from 'url-metadata' +import { fetchReadableContent } from '../services/readerService' interface AddBookmarkModalProps { onClose: () => void onSave: (url: string, title?: string, description?: string, tags?: string[]) => Promise } +// Helper to extract metadata from HTML +function extractMetaTag(html: string, patterns: string[]): string | null { + for (const pattern of patterns) { + const match = html.match(new RegExp(pattern, 'i')) + if (match) return match[1] + } + return null +} + +function extractTags(html: string): string[] { + const tags: string[] = [] + + // Extract keywords meta tag + const keywords = extractMetaTag(html, [ + ' k.trim().toLowerCase()) + .filter(k => k.length > 0 && k.length < 30) + .forEach(k => tags.push(k)) + } + + // Extract article:tag (multiple possible) + const articleTagRegex = / = ({ onClose, onSave }) => { const [url, setUrl] = useState('') const [title, setTitle] = useState('') @@ -48,47 +82,46 @@ const AddBookmarkModal: React.FC = ({ onClose, onSave }) fetchTimeoutRef.current = window.setTimeout(async () => { setIsFetchingMetadata(true) try { - const metadata = await urlMetadata(normalizedUrl) + const content = await fetchReadableContent(normalizedUrl) lastFetchedUrlRef.current = normalizedUrl let extractedAnything = false - // Extract title: prioritize og:title > twitter:title > title - const extractedTitle = metadata['og:title'] || metadata['twitter:title'] || metadata.title - if (extractedTitle && !title) { - setTitle(extractedTitle as string) - extractedAnything = true + // Extract title: prioritize og:title > twitter:title > + if (!title && content.html) { + const extractedTitle = extractMetaTag(content.html, [ + '<meta\\s+property=["\'"]og:title["\'"]\\s+content=["\'"]([^"\']+)["\']', + '<meta\\s+name=["\'"]twitter:title["\'"]\\s+content=["\'"]([^"\']+)["\']' + ]) || content.title + + if (extractedTitle) { + setTitle(extractedTitle) + extractedAnything = true + } } - // Extract description: prioritize og:description > twitter:description > description - if (!description) { - const extractedDesc = metadata['og:description'] || metadata['twitter:description'] || metadata.description + // Extract description: prioritize og:description > twitter:description > meta description + if (!description && content.html) { + const extractedDesc = extractMetaTag(content.html, [ + '<meta\\s+property=["\'"]og:description["\'"]\\s+content=["\'"]([^"\']+)["\']', + '<meta\\s+name=["\'"]twitter:description["\'"]\\s+content=["\'"]([^"\']+)["\']', + '<meta\\s+name=["\'"]description["\'"]\\s+content=["\'"]([^"\']+)["\']' + ]) + if (extractedDesc) { - setDescription(extractedDesc as string) + setDescription(extractedDesc) extractedAnything = true } } // Extract tags from keywords and article:tag (only if user hasn't modified tags) - if (!tagsInput) { - const normalizeTags = (value: string | string[], delimiter = /[,;]/) => { - const arr = Array.isArray(value) ? value : value.split(delimiter) - return arr - .map(t => t.trim().toLowerCase()) - .filter(t => t.length > 0 && t.length < 30) - } - - const extractedTags = [ - ...(metadata.keywords ? normalizeTags(metadata.keywords) : []), - ...(metadata['article:tag'] ? normalizeTags(metadata['article:tag']) : []) - ] - - const uniqueTags = Array.from(new Set(extractedTags)).slice(0, 5) + if (!tagsInput && content.html) { + const extractedTags = extractTags(content.html) // Only add boris tag if we extracted something - if (extractedAnything || uniqueTags.length > 0) { - const allTags = uniqueTags.length > 0 - ? ['boris', ...uniqueTags] + if (extractedAnything || extractedTags.length > 0) { + const allTags = extractedTags.length > 0 + ? ['boris', ...extractedTags] : ['boris'] setTagsInput(allTags.join(', ')) }