抓取知乎页面 about translators_cn HOT 18 CLOSED

/*!
 * Prototype of Zotero web translator for 知乎 (Zhihu)
 *
 * Language: TypeScript 4.0; ES2020
 * Date: 2020-10-15
 *
 * Lemming <https://github.com/Lemmingh>
 * Licensed under the ISC License.
 */

/// <reference path="types-zotero/index.d.ts" />
/// <reference path="types-zhihu/index.d.ts" />

'use strict';

/* Main. */

async function doWeb(_doc: Document, url: string): Promise<void> {
    const newZoteroItem = new Zotero.Item('document');

    // Remove unnecessary query and fragment string from `url`, before performing other operations.
    url = url.split(/\?|#/)[0];

    // If there is any exception, let it propagate to terminate the process.
    const item: ZhihuItem = await getZhihuItem(url);

    // Generate the snapshot of the Zhihu item.
    const snapshot: HTMLDocument = getSnapshot(item);
    newZoteroItem.attachments.push({
        title: 'Snapshot',
        document: snapshot
    });

    // TODO: Set metadata.
    newZoteroItem.title = item.title;

    // Save.
    newZoteroItem.complete();
}

/**
 * Returns a `Promise<ZhihuItem>`.
 * @param {string} url - URL of the answer or Zhuanlan article.
 */
async function getZhihuItem(url: string): Promise<ZhihuItem> {

    // ! Warning:
    // ! RegExp lookbehind assertions are not available until ECMAScript 2018.
    // ! https://github.com/tc39/proposal-regexp-lookbehind

    // Zhuanlan article URL examples:
    // https://zhuanlan.zhihu.com/p/59589298
    // https://zhuanlan.zhihu.com/p/35295235
    const ZhuanlanArticleIdRegexp = /(?<=p\/)\d+/;

    // Answer URL examples:
    // https://www.zhihu.com/answer/984072342
    // https://www.zhihu.com/question/24952084/answer/984072342
    // https://www.zhihu.com/question/392313958/answer/1198915276
    const AnswerIdRegexp = /(?<=answer\/)\d+/;

    if (url.startsWith('https://zhuanlan.zhihu.com/p/')) {
        return getAsArticle();
    } else if (url.includes('/answer/')) {
        return getAsAnswer();
    } else {
        throw new Error("Not supported");
    }

    /* Sends an HTTP GET request to Zhihu API. */
    // Important:
    // Many Zhihu services, including API, lack cross-origin headers.
    // Well, it doesn't seem to matter, since Zotero web translator seems to run in the same context as the active document in the browser.
    // And `fetch()` can be easily converted to `Zotero.HTTP.request()`.
    // ! `Promise` may throw exceptions that can only be caught by try-catch.
    function fetchJson(aUrl: RequestInfo): Promise<Zhihu.Answer | Zhihu.Article> {
        return fetch(aUrl).then(res => res.json());
    }

    async function getAsAnswer(): Promise<ZhihuItem> {
        // Construct API URL.
        const idStr = url.match(AnswerIdRegexp)![0];
        const apiUrl = `https://www.zhihu.com/api/v4/answers/${idStr}?include=content,excerpt`;

        const res = await fetchJson(apiUrl) as Zhihu.Answer;

        return {
            id: res.id,
            type: res.type as ZhihuItemType,
            author: {
                id: res.author.id,
                name: res.author.name,
            },
            content: res.content!,
            excerpt: res.excerpt!,
            title: res.question.title,
        };

    }

    async function getAsArticle(): Promise<ZhihuItem> {
        // Construct API URL.
        const idStr = url.match(ZhuanlanArticleIdRegexp)![0];
        const apiUrl = `https://zhuanlan.zhihu.com/api/articles/${idStr}`;

        const res = await fetchJson(apiUrl) as Zhihu.Article;

        return {
            id: res.id,
            type: res.type as ZhihuItemType,
            author: {
                id: res.author.id,
                name: res.author.name,
            },
            content: res.content,
            excerpt: new DOMParser().parseFromString(res.excerpt, 'text/html').body.innerText,
            title: res.title,
        };
    }
}

/**
 * Generates the snapshot of a ZhihuItem.
 * @param {ZhihuItem} item
 */
function getSnapshot(item: ZhihuItem): HTMLDocument {
    const doc = new HTMLDocument();

    // Assuming that response from Zhihu is safe.
    doc.body.innerHTML = item.content;

    doc.title = item.title;

    // TODO: Inject other elements if necessary.

    return doc;
}


/* High-level representations of entities on Zhihu, with only properties we are interested in. */

/**
 * A unified representation of items (Zhihu answer or Zhuanlan article) on Zhihu.
 */
interface ZhihuItem {
    /**
     * Item ID.
     */
    id: number;

    /**
     * Item type.
     */
    type: ZhihuItemType;

    author: ZhihuAuthor;

    /**
     * Content.
     * An HTML string.
     */
    content: string;

    /**
     * The first few sentences of the content. Can be used as abstract.
     * A **plain text** string.
     */
    excerpt: string;

    /**
     * The name.
     *
     * For an article, use its title.
     * For an answer, use question title.
     */
    title: string;
}

const enum ZhihuItemType {
    Answer = 'answer',
    Article = 'article',
    People = 'people',
    Question = 'question',
}

/**
 * A user on Zhihu.
 */
interface ZhihuAuthor {
    /**
     * User ID.
     */
    id: string;

    /**
     * Item type.
     */
    // type: ZhihuItemType.People;

    /**
     * 用户名.
     */
    name: string;
}