add staged link previews for everything from signal
Audric Ackermann
5 years ago
parent
f5818817e9
commit
13e02b5bf1
@ -1 +1,3 @@
|
|||||||
|
export function isLinkSafeToPreview(link: string): boolean;
|
||||||
|
|
||||||
export function isLinkSneaky(link: string): boolean;
|
export function isLinkSneaky(link: string): boolean;
|
||||||
|
|||||||
@ -0,0 +1,174 @@
|
|||||||
|
import React, { useEffect, useState } from 'react';
|
||||||
|
import { arrayBufferFromFile, AttachmentType } from '../../../types/Attachment';
|
||||||
|
import { AttachmentUtil, LinkPreviewUtil } from '../../../util';
|
||||||
|
import { StagedLinkPreview } from '../../conversation/StagedLinkPreview';
|
||||||
|
import fetch from 'node-fetch';
|
||||||
|
import { fetchLinkPreviewImage } from '../../../util/linkPreviewFetch';
|
||||||
|
import { AbortController, AbortSignal } from 'abort-controller';
|
||||||
|
|
||||||
|
type Props = {
|
||||||
|
url: string;
|
||||||
|
onClose: () => void;
|
||||||
|
};
|
||||||
|
const LINK_PREVIEW_TIMEOUT = 60 * 1000;
|
||||||
|
|
||||||
|
export interface GetLinkPreviewResultImage {
|
||||||
|
data: ArrayBuffer;
|
||||||
|
size: number;
|
||||||
|
contentType: string;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface GetLinkPreviewResult {
|
||||||
|
title: string;
|
||||||
|
url: string;
|
||||||
|
image?: GetLinkPreviewResultImage;
|
||||||
|
description: string | null;
|
||||||
|
date: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const getPreview = async (
|
||||||
|
url: string,
|
||||||
|
abortSignal: AbortSignal
|
||||||
|
): Promise<null | GetLinkPreviewResult> => {
|
||||||
|
// This is already checked elsewhere, but we want to be extra-careful.
|
||||||
|
if (!window.Signal.LinkPreviews.isLinkSafeToPreview(url)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const linkPreviewMetadata = await LinkPreviewUtil.fetchLinkPreviewMetadata(
|
||||||
|
fetch,
|
||||||
|
url,
|
||||||
|
abortSignal
|
||||||
|
);
|
||||||
|
if (!linkPreviewMetadata) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const { title, imageHref, description, date } = linkPreviewMetadata;
|
||||||
|
|
||||||
|
let image;
|
||||||
|
if (imageHref && window.Signal.LinkPreviews.isLinkSafeToPreview(imageHref)) {
|
||||||
|
let objectUrl: void | string;
|
||||||
|
try {
|
||||||
|
const fullSizeImage = await fetchLinkPreviewImage(
|
||||||
|
fetch,
|
||||||
|
imageHref,
|
||||||
|
abortSignal
|
||||||
|
);
|
||||||
|
if (!fullSizeImage) {
|
||||||
|
throw new Error('Failed to fetch link preview image');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure that this file is either small enough or is resized to meet our
|
||||||
|
// requirements for attachments
|
||||||
|
const withBlob = await AttachmentUtil.autoScale({
|
||||||
|
contentType: fullSizeImage.contentType,
|
||||||
|
file: new Blob([fullSizeImage.data], {
|
||||||
|
type: fullSizeImage.contentType,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const data = await arrayBufferFromFile(withBlob.file);
|
||||||
|
objectUrl = URL.createObjectURL(withBlob.file);
|
||||||
|
|
||||||
|
const dimensions = await window.Signal.Types.VisualAttachment.getImageDimensions(
|
||||||
|
{
|
||||||
|
objectUrl,
|
||||||
|
logger: window.log,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
image = {
|
||||||
|
data,
|
||||||
|
size: data.byteLength,
|
||||||
|
...dimensions,
|
||||||
|
contentType: withBlob.file.type,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
// We still want to show the preview if we failed to get an image
|
||||||
|
window.log.error(
|
||||||
|
'getPreview failed to get image for link preview:',
|
||||||
|
error.message
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
if (objectUrl) {
|
||||||
|
URL.revokeObjectURL(objectUrl);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
title,
|
||||||
|
url,
|
||||||
|
image,
|
||||||
|
description,
|
||||||
|
date,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const SessionStagedLinkPreview = (props: Props) => {
|
||||||
|
const [isLoaded, setIsLoaded] = useState(false);
|
||||||
|
const [title, setTitle] = useState<string | null>(null);
|
||||||
|
const [domain, setDomain] = useState<string | null>(null);
|
||||||
|
const [description, setDescription] = useState<string | null>(null);
|
||||||
|
const [image, setImage] = useState<AttachmentType | undefined>(undefined);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
// Use this abortcontroller to stop current fetch requests when url changed
|
||||||
|
const abortController = new AbortController();
|
||||||
|
setTimeout(() => {
|
||||||
|
abortController.abort();
|
||||||
|
}, LINK_PREVIEW_TIMEOUT);
|
||||||
|
|
||||||
|
setIsLoaded(false);
|
||||||
|
setTitle(null);
|
||||||
|
setDomain(null);
|
||||||
|
setDescription(null);
|
||||||
|
setImage(undefined);
|
||||||
|
|
||||||
|
getPreview(props.url, abortController.signal)
|
||||||
|
.then(ret => {
|
||||||
|
setIsLoaded(true);
|
||||||
|
if (ret) {
|
||||||
|
setTitle(ret.title);
|
||||||
|
if (ret.image?.width) {
|
||||||
|
if (ret.image) {
|
||||||
|
const blob = new Blob([ret.image.data], {
|
||||||
|
type: ret.image.contentType,
|
||||||
|
});
|
||||||
|
const imageAttachment = {
|
||||||
|
...ret.image,
|
||||||
|
url: URL.createObjectURL(blob),
|
||||||
|
fileName: 'preview',
|
||||||
|
};
|
||||||
|
setImage(imageAttachment);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
setDomain(window.Signal.LinkPreviews.getDomain(ret.url));
|
||||||
|
if (ret.description) {
|
||||||
|
setDescription(ret.description);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.catch(err => {
|
||||||
|
abortController.abort();
|
||||||
|
setIsLoaded(true);
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
// Cancel other in-flight link preview requests.
|
||||||
|
abortController.abort();
|
||||||
|
};
|
||||||
|
}, [props.url]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<StagedLinkPreview
|
||||||
|
onClose={props.onClose}
|
||||||
|
isLoaded={isLoaded}
|
||||||
|
title={title}
|
||||||
|
domain={domain}
|
||||||
|
image={image as any}
|
||||||
|
description={description}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
};
|
||||||
@ -0,0 +1,41 @@
|
|||||||
|
import { assert } from 'chai';
|
||||||
|
|
||||||
|
import { isLinkPreviewDateValid } from '../../util/isLinkPreviewDateValid';
|
||||||
|
|
||||||
|
describe('isLinkPreviewDateValid', () => {
|
||||||
|
it('returns false for non-numbers', () => {
|
||||||
|
assert.isFalse(isLinkPreviewDateValid(null));
|
||||||
|
assert.isFalse(isLinkPreviewDateValid(undefined));
|
||||||
|
assert.isFalse(isLinkPreviewDateValid(Date.now().toString()));
|
||||||
|
assert.isFalse(isLinkPreviewDateValid(new Date()));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for zero', () => {
|
||||||
|
assert.isFalse(isLinkPreviewDateValid(0));
|
||||||
|
assert.isFalse(isLinkPreviewDateValid(-0));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for NaN', () => {
|
||||||
|
assert.isFalse(isLinkPreviewDateValid(0 / 0));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for any infinite value', () => {
|
||||||
|
assert.isFalse(isLinkPreviewDateValid(Infinity));
|
||||||
|
assert.isFalse(isLinkPreviewDateValid(-Infinity));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns false for timestamps more than a day from now', () => {
|
||||||
|
const twoDays = 2 * 24 * 60 * 60 * 1000;
|
||||||
|
assert.isFalse(isLinkPreviewDateValid(Date.now() + twoDays));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns true for timestamps before tomorrow', () => {
|
||||||
|
assert.isTrue(isLinkPreviewDateValid(Date.now()));
|
||||||
|
assert.isTrue(isLinkPreviewDateValid(Date.now() + 123));
|
||||||
|
assert.isTrue(isLinkPreviewDateValid(Date.now() - 123));
|
||||||
|
assert.isTrue(isLinkPreviewDateValid(new Date(1995, 3, 20).valueOf()));
|
||||||
|
assert.isTrue(isLinkPreviewDateValid(new Date(1970, 3, 20).valueOf()));
|
||||||
|
assert.isTrue(isLinkPreviewDateValid(new Date(1969, 3, 20).valueOf()));
|
||||||
|
assert.isTrue(isLinkPreviewDateValid(1));
|
||||||
|
});
|
||||||
|
});
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,11 @@
|
|||||||
|
const ONE_DAY = 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
export function isLinkPreviewDateValid(value: unknown): value is number {
|
||||||
|
const maximumLinkPreviewDate = Date.now() + ONE_DAY;
|
||||||
|
return (
|
||||||
|
typeof value === 'number' &&
|
||||||
|
value !== 0 &&
|
||||||
|
Number.isFinite(value) &&
|
||||||
|
value < maximumLinkPreviewDate
|
||||||
|
);
|
||||||
|
}
|
||||||
@ -0,0 +1,569 @@
|
|||||||
|
import { RequestInit, Response } from 'node-fetch';
|
||||||
|
import { AbortSignal } from 'abort-controller';
|
||||||
|
|
||||||
|
import {
|
||||||
|
IMAGE_GIF,
|
||||||
|
IMAGE_ICO,
|
||||||
|
IMAGE_JPEG,
|
||||||
|
IMAGE_PNG,
|
||||||
|
IMAGE_WEBP,
|
||||||
|
MIMEType,
|
||||||
|
} from '../types/MIME';
|
||||||
|
|
||||||
|
const MAX_REQUEST_COUNT_WITH_REDIRECTS = 20;
|
||||||
|
// tslint:disable: prefer-for-of
|
||||||
|
|
||||||
|
// Lifted from the `fetch` spec [here][0].
|
||||||
|
// [0]: https://fetch.spec.whatwg.org/#redirect-status
|
||||||
|
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
||||||
|
|
||||||
|
const MAX_CONTENT_TYPE_LENGTH_TO_PARSE = 100;
|
||||||
|
|
||||||
|
// Though we'll accept HTML of any Content-Length (including no specified length), we
|
||||||
|
// will only load some of the HTML. So we might start loading a 99 gigabyte HTML page
|
||||||
|
// but only parse the first 500 kilobytes. However, if the Content-Length is less than
|
||||||
|
// this, we won't waste space.
|
||||||
|
const MAX_HTML_BYTES_TO_LOAD = 500 * 1024;
|
||||||
|
|
||||||
|
// `<title>x` is 8 bytes. Nothing else (meta tags, etc) will even fit, so we can ignore
|
||||||
|
// it. This is mostly to protect us against empty response bodies.
|
||||||
|
const MIN_HTML_CONTENT_LENGTH = 8;
|
||||||
|
|
||||||
|
// Similar to the above. We don't want to show tiny images (even though the more likely
|
||||||
|
// case is that the Content-Length is 0).
|
||||||
|
const MIN_IMAGE_CONTENT_LENGTH = 8;
|
||||||
|
const MAX_IMAGE_CONTENT_LENGTH = 1024 * 1024;
|
||||||
|
const VALID_IMAGE_MIME_TYPES: Set<MIMEType> = new Set([
|
||||||
|
IMAGE_GIF,
|
||||||
|
IMAGE_ICO,
|
||||||
|
IMAGE_JPEG,
|
||||||
|
IMAGE_PNG,
|
||||||
|
IMAGE_WEBP,
|
||||||
|
]);
|
||||||
|
|
||||||
|
// We want to discard unreasonable dates. Update this in ~950 years. (This may discard
|
||||||
|
// some reasonable dates, which is okay because it is only for link previews.)
|
||||||
|
const MIN_DATE = 0;
|
||||||
|
const MAX_DATE = new Date(3000, 0, 1).valueOf();
|
||||||
|
|
||||||
|
const emptyContentType = { type: null, charset: null };
|
||||||
|
|
||||||
|
type FetchFn = (href: string, init: RequestInit) => Promise<Response>;
|
||||||
|
|
||||||
|
export interface LinkPreviewMetadata {
|
||||||
|
title: string;
|
||||||
|
description: null | string;
|
||||||
|
date: null | number;
|
||||||
|
imageHref: null | string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LinkPreviewImage {
|
||||||
|
data: ArrayBuffer;
|
||||||
|
contentType: MIMEType;
|
||||||
|
}
|
||||||
|
|
||||||
|
type ParsedContentType =
|
||||||
|
| { type: null; charset: null }
|
||||||
|
| { type: MIMEType; charset: null | string };
|
||||||
|
|
||||||
|
// This throws non-helpful errors because (1) it logs (2) it will be immediately caught.
|
||||||
|
async function fetchWithRedirects(
|
||||||
|
fetchFn: FetchFn,
|
||||||
|
href: string,
|
||||||
|
options: RequestInit
|
||||||
|
): Promise<Response> {
|
||||||
|
const urlsSeen = new Set<string>();
|
||||||
|
|
||||||
|
let nextHrefToLoad = href;
|
||||||
|
for (let i = 0; i < MAX_REQUEST_COUNT_WITH_REDIRECTS; i += 1) {
|
||||||
|
if (urlsSeen.has(nextHrefToLoad)) {
|
||||||
|
window.log.warn('fetchWithRedirects: found a redirect loop');
|
||||||
|
throw new Error('redirect loop');
|
||||||
|
}
|
||||||
|
urlsSeen.add(nextHrefToLoad);
|
||||||
|
|
||||||
|
// This `await` is deliberatly inside of a loop.
|
||||||
|
// eslint-disable-next-line no-await-in-loop
|
||||||
|
const response = await fetchFn(nextHrefToLoad, {
|
||||||
|
...options,
|
||||||
|
redirect: 'manual',
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!REDIRECT_STATUSES.has(response.status)) {
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
const location = response.headers.get('location');
|
||||||
|
if (!location) {
|
||||||
|
window.log.warn(
|
||||||
|
'fetchWithRedirects: got a redirect status code but no Location header; bailing'
|
||||||
|
);
|
||||||
|
throw new Error('no location with redirect');
|
||||||
|
}
|
||||||
|
|
||||||
|
const newUrl = maybeParseUrl(location, nextHrefToLoad);
|
||||||
|
if (newUrl?.protocol !== 'https:') {
|
||||||
|
window.log.warn(
|
||||||
|
'fetchWithRedirects: got a redirect status code and an invalid Location header'
|
||||||
|
);
|
||||||
|
throw new Error('invalid location');
|
||||||
|
}
|
||||||
|
|
||||||
|
nextHrefToLoad = newUrl.href;
|
||||||
|
}
|
||||||
|
|
||||||
|
window.log.warn('fetchWithRedirects: too many redirects');
|
||||||
|
throw new Error('too many redirects');
|
||||||
|
}
|
||||||
|
|
||||||
|
function maybeParseUrl(href: string, base: string): null | URL {
|
||||||
|
let result: URL;
|
||||||
|
try {
|
||||||
|
result = new URL(href, base);
|
||||||
|
} catch (err) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
// We never need the hash
|
||||||
|
result.hash = '';
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parses a Content-Type header value. Refer to [RFC 2045][0] for details (though this is
|
||||||
|
* a simplified version for link previews.
|
||||||
|
* [0]: https://tools.ietf.org/html/rfc2045
|
||||||
|
*/
|
||||||
|
const parseContentType = (headerValue: string | null): ParsedContentType => {
|
||||||
|
if (!headerValue || headerValue.length > MAX_CONTENT_TYPE_LENGTH_TO_PARSE) {
|
||||||
|
return emptyContentType;
|
||||||
|
}
|
||||||
|
|
||||||
|
const [rawType, ...rawParameters] = headerValue
|
||||||
|
.toLowerCase()
|
||||||
|
.split(/;/g)
|
||||||
|
.map(part => part.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
if (!rawType) {
|
||||||
|
return emptyContentType;
|
||||||
|
}
|
||||||
|
|
||||||
|
let charset: null | string = null;
|
||||||
|
for (let i = 0; i < rawParameters.length; i += 1) {
|
||||||
|
const rawParameter = rawParameters[i];
|
||||||
|
const parsed = new URLSearchParams(rawParameter);
|
||||||
|
const parsedCharset = parsed.get('charset')?.trim();
|
||||||
|
if (parsedCharset) {
|
||||||
|
charset = parsedCharset;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
type: rawType,
|
||||||
|
charset,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const isInlineContentDisposition = (headerValue: string | null): boolean =>
|
||||||
|
!headerValue || headerValue.split(';', 1)[0] === 'inline';
|
||||||
|
|
||||||
|
const parseContentLength = (headerValue: string | null): number => {
|
||||||
|
// No need to parse gigantic Content-Lengths; only parse the first 10 digits.
|
||||||
|
if (typeof headerValue !== 'string' || !/^\d{1,10}$/g.test(headerValue)) {
|
||||||
|
return Infinity;
|
||||||
|
}
|
||||||
|
const result = parseInt(headerValue, 10);
|
||||||
|
return Number.isNaN(result) ? Infinity : result;
|
||||||
|
};
|
||||||
|
|
||||||
|
const emptyHtmlDocument = (): HTMLDocument =>
|
||||||
|
new DOMParser().parseFromString('', 'text/html');
|
||||||
|
|
||||||
|
// The charset behavior here follows the [W3 guidelines][0]. The priority is BOM, HTTP
|
||||||
|
// header, `http-equiv` meta tag, `charset` meta tag, and finally a UTF-8 fallback.
|
||||||
|
// (This fallback could, perhaps, be smarter based on user locale.)
|
||||||
|
// [0]: https://www.w3.org/International/questions/qa-html-encoding-declarations.en
|
||||||
|
const parseHtmlBytes = (
|
||||||
|
bytes: Readonly<Uint8Array>,
|
||||||
|
httpCharset: string | null
|
||||||
|
): HTMLDocument => {
|
||||||
|
const hasBom = bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf;
|
||||||
|
|
||||||
|
let isSureOfCharset: boolean;
|
||||||
|
let decoder: TextDecoder;
|
||||||
|
if (hasBom) {
|
||||||
|
decoder = new TextDecoder();
|
||||||
|
isSureOfCharset = true;
|
||||||
|
} else if (httpCharset) {
|
||||||
|
try {
|
||||||
|
decoder = new TextDecoder(httpCharset);
|
||||||
|
isSureOfCharset = true;
|
||||||
|
} catch (err) {
|
||||||
|
decoder = new TextDecoder();
|
||||||
|
isSureOfCharset = false;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
decoder = new TextDecoder();
|
||||||
|
isSureOfCharset = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let decoded: string;
|
||||||
|
try {
|
||||||
|
decoded = decoder.decode(bytes);
|
||||||
|
} catch (err) {
|
||||||
|
decoded = '';
|
||||||
|
}
|
||||||
|
|
||||||
|
let document: HTMLDocument;
|
||||||
|
try {
|
||||||
|
document = new DOMParser().parseFromString(decoded, 'text/html');
|
||||||
|
} catch (err) {
|
||||||
|
document = emptyHtmlDocument();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isSureOfCharset) {
|
||||||
|
const httpEquiv = document
|
||||||
|
.querySelector('meta[http-equiv="content-type"]')
|
||||||
|
?.getAttribute('content');
|
||||||
|
if (httpEquiv) {
|
||||||
|
const httpEquivCharset = parseContentType(httpEquiv).charset;
|
||||||
|
if (httpEquivCharset) {
|
||||||
|
return parseHtmlBytes(bytes, httpEquivCharset);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const metaCharset = document
|
||||||
|
.querySelector('meta[charset]')
|
||||||
|
?.getAttribute('charset');
|
||||||
|
if (metaCharset) {
|
||||||
|
return parseHtmlBytes(bytes, metaCharset);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return document;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getHtmlDocument = async (
|
||||||
|
body: AsyncIterable<string | Uint8Array>,
|
||||||
|
contentLength: number,
|
||||||
|
httpCharset: string | null,
|
||||||
|
abortSignal: AbortSignal
|
||||||
|
): Promise<HTMLDocument> => {
|
||||||
|
let result: HTMLDocument = emptyHtmlDocument();
|
||||||
|
|
||||||
|
const maxHtmlBytesToLoad = Math.min(contentLength, MAX_HTML_BYTES_TO_LOAD);
|
||||||
|
const buffer = new Uint8Array(new ArrayBuffer(maxHtmlBytesToLoad));
|
||||||
|
let bytesLoadedSoFar = 0;
|
||||||
|
|
||||||
|
try {
|
||||||
|
// `for ... of` is much cleaner here, so we allow it.
|
||||||
|
/* eslint-disable no-restricted-syntax */
|
||||||
|
for await (let chunk of body) {
|
||||||
|
if (abortSignal.aborted) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// This check exists to satisfy TypeScript; chunk should always be a Buffer.
|
||||||
|
if (typeof chunk === 'string') {
|
||||||
|
if (
|
||||||
|
httpCharset !== null &&
|
||||||
|
httpCharset !== undefined &&
|
||||||
|
Buffer.isEncoding(httpCharset)
|
||||||
|
) {
|
||||||
|
chunk = Buffer.from(chunk, httpCharset);
|
||||||
|
} else {
|
||||||
|
chunk = Buffer.from(chunk, 'utf8');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const truncatedChunk = chunk.slice(
|
||||||
|
0,
|
||||||
|
maxHtmlBytesToLoad - bytesLoadedSoFar
|
||||||
|
);
|
||||||
|
buffer.set(truncatedChunk, bytesLoadedSoFar);
|
||||||
|
bytesLoadedSoFar += truncatedChunk.byteLength;
|
||||||
|
|
||||||
|
result = parseHtmlBytes(buffer.slice(0, bytesLoadedSoFar), httpCharset);
|
||||||
|
|
||||||
|
const hasLoadedMaxBytes = bytesLoadedSoFar >= maxHtmlBytesToLoad;
|
||||||
|
if (hasLoadedMaxBytes) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* eslint-enable no-restricted-syntax */
|
||||||
|
} catch (err) {
|
||||||
|
window.log.warn(
|
||||||
|
'getHtmlDocument: error when reading body; continuing with what we got'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getOpenGraphContent = (
|
||||||
|
document: HTMLDocument,
|
||||||
|
properties: ReadonlyArray<string>
|
||||||
|
): string | null => {
|
||||||
|
for (let i = 0; i < properties.length; i += 1) {
|
||||||
|
const property = properties[i];
|
||||||
|
const content = document
|
||||||
|
.querySelector(`meta[property="${property}"]`)
|
||||||
|
?.getAttribute('content')
|
||||||
|
?.trim();
|
||||||
|
if (content) {
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const getLinkHrefAttribute = (
|
||||||
|
document: HTMLDocument,
|
||||||
|
rels: ReadonlyArray<string>
|
||||||
|
): string | null => {
|
||||||
|
for (let i = 0; i < rels.length; i += 1) {
|
||||||
|
const rel = rels[i];
|
||||||
|
const href = document
|
||||||
|
.querySelector(`link[rel="${rel}"]`)
|
||||||
|
?.getAttribute('href')
|
||||||
|
?.trim();
|
||||||
|
if (href) {
|
||||||
|
return href;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
|
const parseMetadata = (
|
||||||
|
document: HTMLDocument,
|
||||||
|
href: string
|
||||||
|
): LinkPreviewMetadata | null => {
|
||||||
|
const title =
|
||||||
|
getOpenGraphContent(document, ['og:title']) || document.title.trim();
|
||||||
|
if (!title) {
|
||||||
|
window.log.warn(
|
||||||
|
"parseMetadata: HTML document doesn't have a title; bailing"
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const description =
|
||||||
|
getOpenGraphContent(document, ['og:description']) ||
|
||||||
|
document
|
||||||
|
.querySelector('meta[name="description"]')
|
||||||
|
?.getAttribute('content')
|
||||||
|
?.trim() ||
|
||||||
|
null;
|
||||||
|
|
||||||
|
const rawImageHref =
|
||||||
|
getOpenGraphContent(document, ['og:image', 'og:image:url']) ||
|
||||||
|
getLinkHrefAttribute(document, [
|
||||||
|
'shortcut icon',
|
||||||
|
'icon',
|
||||||
|
'apple-touch-icon',
|
||||||
|
]);
|
||||||
|
const imageUrl = rawImageHref ? maybeParseUrl(rawImageHref, href) : null;
|
||||||
|
const imageHref = imageUrl ? imageUrl.href : null;
|
||||||
|
|
||||||
|
let date: number | null = null;
|
||||||
|
const rawDate = getOpenGraphContent(document, [
|
||||||
|
'og:published_time',
|
||||||
|
'article:published_time',
|
||||||
|
'og:modified_time',
|
||||||
|
'article:modified_time',
|
||||||
|
]);
|
||||||
|
if (rawDate) {
|
||||||
|
const parsed = Date.parse(rawDate);
|
||||||
|
if (parsed > MIN_DATE && parsed < MAX_DATE) {
|
||||||
|
date = parsed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
title,
|
||||||
|
description,
|
||||||
|
imageHref,
|
||||||
|
date,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This attempts to fetch link preview metadata, returning `null` if it cannot be found
|
||||||
|
* for any reason.
|
||||||
|
*
|
||||||
|
* NOTE: This does NOT validate the incoming URL for safety. For example, it may fetch an
|
||||||
|
* insecure HTTP href. It also does not offer a timeout; that is up to the caller.
|
||||||
|
*
|
||||||
|
* At a high level, it:
|
||||||
|
*
|
||||||
|
* 1. Makes a GET request, following up to 20 redirects (`fetch`'s default).
|
||||||
|
* 2. Checks the response status code and headers to make sure it's a normal HTML
|
||||||
|
* response.
|
||||||
|
* 3. Streams up to `MAX_HTML_BYTES_TO_LOAD`, stopping when (1) it has loaded all of the
|
||||||
|
* HTML (2) loaded the maximum number of bytes (3) finished loading the `<head>`.
|
||||||
|
* 4. Parses the resulting HTML with `DOMParser`.
|
||||||
|
* 5. Grabs the title, description, image URL, and date.
|
||||||
|
*/
|
||||||
|
export async function fetchLinkPreviewMetadata(
|
||||||
|
fetchFn: FetchFn,
|
||||||
|
href: string,
|
||||||
|
abortSignal: AbortSignal
|
||||||
|
): Promise<null | LinkPreviewMetadata> {
|
||||||
|
let response: Response;
|
||||||
|
try {
|
||||||
|
response = await fetchWithRedirects(fetchFn, href, {
|
||||||
|
headers: {
|
||||||
|
Accept: 'text/html,application/xhtml+xml',
|
||||||
|
'User-Agent': 'WhatsApp',
|
||||||
|
},
|
||||||
|
signal: abortSignal,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
window.log.warn(
|
||||||
|
'fetchLinkPreviewMetadata: failed to fetch link preview HTML; bailing'
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
window.log.warn(
|
||||||
|
`fetchLinkPreviewMetadata: got a ${response.status} status code; bailing`
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!response.body) {
|
||||||
|
window.log.warn('fetchLinkPreviewMetadata: no response body; bailing');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
!isInlineContentDisposition(response.headers.get('Content-Disposition'))
|
||||||
|
) {
|
||||||
|
window.log.warn(
|
||||||
|
'fetchLinkPreviewMetadata: Content-Disposition header is not inline; bailing'
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (abortSignal.aborted) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentLength = parseContentLength(
|
||||||
|
response.headers.get('Content-Length')
|
||||||
|
);
|
||||||
|
if (contentLength < MIN_HTML_CONTENT_LENGTH) {
|
||||||
|
window.log.warn(
|
||||||
|
'fetchLinkPreviewMetadata: Content-Length is too short; bailing'
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentType = parseContentType(response.headers.get('Content-Type'));
|
||||||
|
if (contentType.type !== 'text/html') {
|
||||||
|
window.log.warn(
|
||||||
|
'fetchLinkPreviewMetadata: Content-Type is not HTML; bailing'
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const document = await getHtmlDocument(
|
||||||
|
response.body,
|
||||||
|
contentLength,
|
||||||
|
contentType.charset,
|
||||||
|
abortSignal
|
||||||
|
);
|
||||||
|
|
||||||
|
// [The Node docs about `ReadableStream.prototype[Symbol.asyncIterator]`][0] say that
|
||||||
|
// the stream will be destroyed if you `break` out of the loop, but I could not
|
||||||
|
// reproduce this. Also [`destroy` is a documented method][1] but it is not in the
|
||||||
|
// Node types, which is why we do this cast to `any`.
|
||||||
|
// [0]: https://nodejs.org/docs/latest-v12.x/api/stream.html#stream_readable_symbol_asynciterator
|
||||||
|
// [1]: https://nodejs.org/docs/latest-v12.x/api/stream.html#stream_readable_destroy_error
|
||||||
|
try {
|
||||||
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
|
(response.body as any).destroy();
|
||||||
|
} catch (err) {
|
||||||
|
// Ignored.
|
||||||
|
}
|
||||||
|
|
||||||
|
if (abortSignal.aborted) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return parseMetadata(document, response.url);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This attempts to fetch an image, returning `null` if it fails for any reason.
|
||||||
|
*
|
||||||
|
* NOTE: This does NOT validate the incoming URL for safety. For example, it may fetch an
|
||||||
|
* insecure HTTP href. It also does not offer a timeout; that is up to the caller.
|
||||||
|
*/
|
||||||
|
export async function fetchLinkPreviewImage(
|
||||||
|
fetchFn: FetchFn,
|
||||||
|
href: string,
|
||||||
|
abortSignal: AbortSignal
|
||||||
|
): Promise<null | LinkPreviewImage> {
|
||||||
|
let response: Response;
|
||||||
|
try {
|
||||||
|
response = await fetchWithRedirects(fetchFn, href, {
|
||||||
|
headers: {
|
||||||
|
'User-Agent': 'WhatsApp',
|
||||||
|
},
|
||||||
|
size: MAX_IMAGE_CONTENT_LENGTH,
|
||||||
|
signal: abortSignal,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
window.log.warn('fetchLinkPreviewImage: failed to fetch image; bailing');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (abortSignal.aborted) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
window.log.warn(
|
||||||
|
`fetchLinkPreviewImage: got a ${response.status} status code; bailing`
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const contentLength = parseContentLength(
|
||||||
|
response.headers.get('Content-Length')
|
||||||
|
);
|
||||||
|
if (contentLength < MIN_IMAGE_CONTENT_LENGTH) {
|
||||||
|
window.log.warn(
|
||||||
|
'fetchLinkPreviewImage: Content-Length is too short; bailing'
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (contentLength > MAX_IMAGE_CONTENT_LENGTH) {
|
||||||
|
window.log.warn(
|
||||||
|
'fetchLinkPreviewImage: Content-Length is too large or is unset; bailing'
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const { type: contentType } = parseContentType(
|
||||||
|
response.headers.get('Content-Type')
|
||||||
|
);
|
||||||
|
if (!contentType || !VALID_IMAGE_MIME_TYPES.has(contentType)) {
|
||||||
|
window.log.warn(
|
||||||
|
'fetchLinkPreviewImage: Content-Type is not an image; bailing'
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
let data: ArrayBuffer;
|
||||||
|
try {
|
||||||
|
data = await response.arrayBuffer();
|
||||||
|
} catch (err) {
|
||||||
|
window.log.warn('fetchLinkPreviewImage: failed to read body; bailing');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { data, contentType };
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue