twitter-scrapper updated with publish.twitter.com

Author: win

lib/index.js

@@ -1,5 +1,5 @@
 import { ReactTinyLinkType } from '../../ReactTinyLinkTypes';
-declare const _default: (url: string, data: any) => Promise<{
+declare const _default: (url: string, data: any, defaultMedia: any) => Promise<{
  title: any;
  content: string;
  url: string;

lib/index.js.map

@@ -1,14 +1,17 @@
 import { ReactTinyLinkType } from '../../ReactTinyLinkTypes'
+import { isEmpty } from '../utils'
 
-export default async (url: string, data) => {
+export default async (url: string, data, defaultMedia) => {
  const scrappedData = JSON.parse(data)
  return {
  title: scrappedData.title,
  content: url,
  url: url,
  description: scrappedData.title,
  video: null,
- image: [scrappedData.thumbnail_url],
+ image: !defaultMedia
+ ? [scrappedData.thumbnail_url]
+ : [scrappedData.thumbnail_url, defaultMedia].filter(i => !isEmpty(i)),
  type: ReactTinyLinkType.TYPE_IMAGE, // MIME Type
  }
 }

src/rules/Instagram/ScrapInstagram.d.ts

@@ -1,6 +1,6 @@
 import { ReactTinyLinkType } from '../../ReactTinyLinkTypes';
-declare const _default: (url: any, htmlDoc: any, defaultMedia: any) => Promise<{
- title: any;
+declare const _default: (url: any, data: any, htmlDoc: any, defaultMedia: any) => Promise<{
+ title: string;
  description: any;
  url: any;
  video: string[];

src/rules/Instagram/ScrapInstagram.ts

@@ -1,12 +1,17 @@
 import { ReactTinyLinkType } from '../../ReactTinyLinkTypes'
 import { isEmpty, getAttrOfDocElement, fixRelativeUrls, getBaseUrl } from '../utils'
 
-export default async (url, htmlDoc, defaultMedia) => {
+export default async (url, data, htmlDoc, defaultMedia) => {
+ const scrappedData = JSON.parse(data)
+ const htmlElement = document.createElement(`html`)
+ htmlElement.innerHTML = scrappedData.html
+
  let baseUrl = getBaseUrl(htmlDoc, url)
 
  const image = [
  getAttrOfDocElement(htmlDoc, "meta[property='og:image']", 'content'),
  getAttrOfDocElement(htmlDoc, "meta[property='og:image:user_generated']", 'content'),
+ 'https://help.twitter.com/content/dam/help-twitter/twitter-logo.png',
  ]
  .filter(i => !isEmpty(i))
  .map(i => fixRelativeUrls(baseUrl, i))
@@ -17,16 +22,14 @@ export default async (url, htmlDoc, defaultMedia) => {
  ]
  .filter(i => !isEmpty(i))
  .map(i => fixRelativeUrls(baseUrl, i))
- 
+
  return {
- title: getAttrOfDocElement(htmlDoc, "meta[property='og:title']", 'content'),
- description: getAttrOfDocElement(htmlDoc, "meta[property='og:description']", 'content'),
+ title: htmlElement.querySelector('p').textContent,
+ description: scrappedData.author_name,
  url: getAttrOfDocElement(htmlDoc, "meta[property='og:url']", 'content'),
  video: video,
- image: !defaultMedia
- ? image
- : [...image, defaultMedia].filter(i => !isEmpty(i)),
+ image: !defaultMedia ? image : [...image, defaultMedia].filter(i => !isEmpty(i)),
  type: ReactTinyLinkType.TYPE_TWITTER, // MIME Type
- publisher: ["Twitter"]
+ publisher: ['Twitter'],
  }
 }

src/rules/Twitter/ScrapTwitter.d.ts

@@ -32,15 +32,15 @@ export default async (url: string, httpClient, defaultMedia: string[]) => {
  } else if (isImage(mimeType)) {
  return await ScrapImage(url, defaultMedia)
  } else if (isInstagramUrl(url)) {
- return await ScrapInstagram(url, data)
+ return await ScrapInstagram(url, data, defaultMedia)
  } else if (isYoutubeUrl(url)) {
  return await ScrapYoutube(url, htmlDoc, defaultMedia)
  } else if (isAmazonUrl(url)) {
  return await ScrapAmazon(url, htmlDoc, defaultMedia)
  } else if (isBoardGameGeekUrl(url)) {
  return await ScrapBoardGameGeek(url, htmlDoc, defaultMedia)
  } else if (isTwitterUrl(url)) {
- return await ScrapTwitter(url, htmlDoc, defaultMedia)
+ return await ScrapTwitter(url, data, htmlDoc, defaultMedia)
  } else {
  return await ScrapDefault(url, htmlDoc, defaultMedia)
  }

src/rules/Twitter/ScrapTwitter.ts

@@ -6,7 +6,7 @@ import { makeCancelable } from './utils'
 
 // Types
 import { IReactTinyLinkData } from './ReactTinyLinkTypes'
-import { isInstagramUrl } from './rules/utils'
+import { isInstagramUrl, isTwitterUrl } from './rules/utils'
 
 /** current state of the fetch */
 export interface IState<T, E> {
@@ -76,6 +76,9 @@ export function useMountFetch(
  if (isInstagramUrl(url)) {
  const modifiedInstaUrl = `https://api.instagram.com/oembed/?url=${url}`
  client = fetch(proxyUrl ? `${proxyUrl}/${modifiedInstaUrl}` : modifiedInstaUrl, { headers })
+ } else if (isTwitterUrl(url)) {
+ const modifiedInstaUrl = `https://publish.twitter.com/oembed?url=${url}`
+ client = fetch(proxyUrl ? `${proxyUrl}/${modifiedInstaUrl}` : modifiedInstaUrl, { headers })
  } else {
  client = fetch(proxyUrl ? `${proxyUrl}/${url}` : url, { headers })
  }