jadefin/mods/Transcript.js

//@ts-check

import JadefinIntegrity from '../JadefinIntegrity.js';

import Jadefin from "../Jadefin.js";
import JadefinMod from "../JadefinMod.js";
import JadefinModules from "../JadefinModules.js";
import JadefinUtils from "../JadefinUtils.js";

import { rd, rdom, rd$, RDOMListHelper } from "../utils/rdom.js";

const TICKS_PER_MS = 10000;

export default JadefinIntegrity("Transcript", import.meta.url, () => new (class Transcript extends JadefinMod {
    _currentPlayer;
    _currentTrack;

    /** @type {any[] | null} */
    _currentSubsRaw = null;
    /** @type {any[] | null} */
    _currentSubs = null;

    _isEnabled = false;
    _isAutoscroll = true;
    _ignoreScroll = true;

    /** @type {Element | undefined} */
    _lastOsdPage;
    _lastHtmlVideoPlayer;
    _lastOffs = Number.NaN;
    _lastInView = Number.NaN;

    _btnTranscript = rd$()`
        <button is="paper-icon-button-light" class="btnTranscript autoSize paper-icon-button-light" title="Transcript">
            <span class="xlargePaperIconButton material-icons manage_search" aria-hidden="true"></span>
        </button>
    `;

    _osdTranscript = rd$()`
        <div class="osdTranscript">
            <div class="transcriptLogWrap">
                <table class="transcriptLog"></table>
            </div>
        </div>
    `;

    _osdTranscriptLogWrap = JadefinUtils.notnull(this._osdTranscript.querySelector(".transcriptLogWrap"));
    _osdTranscriptLog = JadefinUtils.notnull(this._osdTranscript.querySelector(".transcriptLog"));
    _osdTranscriptLogList = new RDOMListHelper(this._osdTranscriptLog);

    _ssaAssFetching;
    _ssaAssObserver = new MutationObserver(this.onSsaAssCanvasAdded.bind(this));
    _ssaAssObserverConfig = {
        childList: true
    };

    _osdBottomObserver = new ResizeObserver(this.updatePosition.bind(this));

    constructor() {
        super();

        this._btnTranscript.addEventListener("click", () => this.isEnabled = !this.isEnabled);

        this._osdTranscriptLogWrap.addEventListener("scroll", () => {
            if (!this._ignoreScroll) {
                this._isAutoscroll = false;
            } else if (window.onscrollend === undefined) {
                this._ignoreScroll = false;
            }
        });

        this._osdTranscriptLogWrap.addEventListener("scrollend", () => {
            if (!this._ignoreScroll) {
                this._isAutoscroll = false;
            } else {
                this._ignoreScroll = false;
            }
        });

        JadefinUtils.events.addEventListener(JadefinUtils.eventTypes.WORKER_CREATED, e => {
            const detail = e["detail"];

            // TODO: 10.9 should've replaced JSSO / libssa-js with JASSUB, revisit once that's stable.
            if (!detail.args.scriptURL.endsWith("/libraries/subtitles-octopus-worker.js")) {
                return;
            }

            /** @type {Worker} */
            const worker = detail.worker;
            worker.addEventListener("message", this.onSsaAssWorkerMessage.bind(this));
        });

        this.initHookDocumentAddEventListener();
        this.initHookDocumentRemoveEventListener();
    }

    get isEnabled() {
        return this._isEnabled;
    }

    set isEnabled(value) {
        if (!this._isEnabled && value) {
            this._isAutoscroll = true;
            this._lastOffs = Number.NaN;
            this._lastInView = Number.NaN;
        }

        this._isEnabled = value;

        this.update();
    }

    get currentSubsRaw() {
        return JadefinUtils.currentPlayer == this._currentPlayer ? this._currentSubsRaw : null;
    }

    get currentSubs() {
        return JadefinUtils.currentPlayer == this._currentPlayer ? this._currentSubs : null;
    }

    async init(name, url) {
        await super.init(name, url);

        const initing = [
            this.initStyle(),
            this.initHookSetTrackForDisplay(),
            this.initHookFetchSubtitles(),
            this.initHookSetOffset(),
        ];

        document.addEventListener("viewshow", () => {
            this.update();
        });

        await Promise.all(initing);

        this.log.i("Ready");
    }

    async initHookSetTrackForDisplay() {
        await JadefinUtils.waitUntil(() => JadefinModules.htmlVideoPlayer);

        const self = this;
        const orig = this._setTrackForDisplay = JadefinModules.htmlVideoPlayer.prototype.setTrackForDisplay;
        JadefinModules.htmlVideoPlayer.prototype.setTrackForDisplay = function(videoElement, track, targetTextTrackIndex) {
            self.log.i("Setting subtitles");
            self.log.dir(track);

            if (self._currentPlayer == this &&
                self._currentTrack?.DeliveryUrl && track?.DeliveryUrl &&
                self._currentTrack.DeliveryUrl == track.DeliveryUrl) {
                // Subtitle track got "changed" for another reason, f.e. audio track change.
                return orig.call(this, videoElement, track, targetTextTrackIndex);
            }

            self.setSubtitles(track, null, true);

            const codec = track?.Codec?.toLowerCase();
            let format = "";
            let incrementFetchQueue = false;

            if (codec == "ssa" || codec == "ass") {
                format = ".ass";
                incrementFetchQueue = true;
            }

            // Required because ass subs are fetched separately.
            if (incrementFetchQueue) {
                if (self._ssaAssFetching != this) {
                    self.log.v("Manually incrementing fetch queue");
                    self._ssaAssFetching = this;
                    self._ssaAssObserver.disconnect();

                    // @ts-ignore - not entirely correct, but video is non-null here.
                    self._ssaAssObserver.observe(JadefinUtils.video?.parentNode, self._ssaAssObserverConfig);

                    this.incrementFetchQueue();
                }
            } else if (self._ssaAssFetching == this) {
                self.log.v("Manually decrementing fetch queue (sub change)");
                this._ssaAssFetching?.decrementFetchQueue();
                this._ssaAssFetching = null;
                this._ssaAssObserver.disconnect();
            }

            if (format != "") {
                // We need to fetch ass subs ourselves.
                self.log.i("Manually fetching subtitles (non-subrip format)");
                self.log.dir(track);

                let url = JadefinModules.playbackManager.getSubtitleUrl(track, JadefinModules.playbackManager.currentItem(JadefinUtils.currentPlayer));
                url = url.replace(format, ".js");

                this.incrementFetchQueue();

                fetch(url)
                    .then(r => r.json())
                    .then(r => self.setSubtitles(track, r))
                    .finally(() => this.decrementFetchQueue());
            }

            return orig.call(this, videoElement, track, targetTextTrackIndex);
        };
    }

    async initHookFetchSubtitles() {
        await JadefinUtils.waitUntil(() => JadefinModules.htmlVideoPlayer);

        const self = this;
        const orig = this._fetchSubtitles = JadefinModules.htmlVideoPlayer.prototype.fetchSubtitles;
        JadefinModules.htmlVideoPlayer.prototype.fetchSubtitles = function(track, item) {
            self.log.i("Fetching subtitles");
            self.log.dir(track);

            const rv = orig.call(this, track, item);

            rv.then(r => self.setSubtitles(track, r), () => self.setSubtitles(track, null));

            return rv;
        };
    }

    async initHookSetOffset() {
        await JadefinUtils.waitUntil(() => JadefinModules.htmlVideoPlayer);

        const self = this;

        const resetSubtitleOffset = this._resetSubtitleOffset = JadefinModules.htmlVideoPlayer.prototype.resetSubtitleOffset;
        JadefinModules.htmlVideoPlayer.prototype.resetSubtitleOffset = function() {
            var rv = resetSubtitleOffset.call(this);

            self.update();

            return rv;
        };

        const updateCurrentTrackOffset = this._resetSubtitleOffset = JadefinModules.htmlVideoPlayer.prototype.updateCurrentTrackOffset;
        JadefinModules.htmlVideoPlayer.prototype.updateCurrentTrackOffset = function(offsetValue, currentTrackIndex) {
            var rv = updateCurrentTrackOffset.call(this, offsetValue, currentTrackIndex);

            self.update();

            return rv;
        };
    }

    initHookDocumentAddEventListener() {
        const orig = this._addEventListener = document.addEventListener.bind(document);
        document.addEventListener = (type, listener, options) => {
            if (type == "wheel") {
                const listenerStr = listener.toString();

                // Anonymous function in playback-video
                if (listenerStr.indexOf("volumeUp") != -1 &&
                    listenerStr.indexOf("volumeDown") != -1 &&
                    listenerStr.indexOf("deltaY") != -1) {
                    this.log.i("Wrapping playback-video wrap listener");
                    this.log.dir(listener);

                    const origListener = this._playbackWheel = listener;
                    listener = this._playbackWheelWrap = (e) => {
                        if (JadefinUtils.hasParent(e.target, this._osdTranscript)) {
                            return;
                        }

                        return origListener(e);
                    };
                }
            }

            return orig(type, listener, options);
        };
    }

    initHookDocumentRemoveEventListener() {
        const orig = this._removeEventListener = document.removeEventListener.bind(document);
        document.removeEventListener = (type, listener, options) => {
            if (listener == this._playbackWheel) {
                listener = this._playbackWheelWrap;
            }

            return orig(type, listener, options);
        };
    }

    onSsaAssCanvasAdded() {
        const htmlVideoPlayer = this._ssaAssFetching;

        this._ssaAssObserver.disconnect();

        this.log.v("Manual fetch queue progress - SsaAss canvas updated");

        if (!htmlVideoPlayer) {
            this.log.v("Manual fetch queue fail - no htmlVideoPlayer");

            return;
        }

        // The canvas gets added relatively early, therefore hook drawImage.
        // TODO: 10.9 should've replaced JSSO / libssa-js with JASSUB, revisit once that's stable.

        const canvas = JadefinUtils.video.parentNode?.querySelector("canvas");
        const ctx = canvas?.getContext("2d");

        if (!ctx) {
            this.log.v("Manually decrementing fetch queue (no 2d ctx)");
            htmlVideoPlayer.decrementFetchQueue();

            return;
        }

        // TODO: drawImage doesn't fire for ass / ssa scenarios where subs start late.
        const orig = ctx.drawImage.bind(ctx);
        ctx.drawImage = (...args) => {
            if (!this._ssaAssFetching) {
                return orig(...args);
            }

            this.log.v("Manually decrementing fetch queue (drawImage)");

            this._ssaAssFetching.decrementFetchQueue();
            this._ssaAssFetching = null;

            return orig(...args);
        };
    }

    onSsaAssWorkerMessage(e) {
        if (!this._ssaAssFetching) {
            return;
        }

        if (e.data.target == "canvas" && e.data.op == "oneshot-result") {
            this.log.v("Manually decrementing fetch queue (worker message)");

            this._ssaAssFetching.decrementFetchQueue();
            this._ssaAssFetching = null;
        }
    }

    setSubtitles(track, subs, force) {
        if (!force && this._currentTrack != track) {
            this.log.w("Late sub fetch");
            return;
        }

        if (subs) {
            this.log.i("Subtitles updated");
            this.log.dir(subs);
        } else {
            this.log.i("Subtitles reset");
        }

        this._currentPlayer = JadefinUtils.currentPlayer;
        this._currentTrack = track;
        const raw = this._currentSubsRaw = /** @type {any[] | null} */ (subs?.TrackEvents);

        if (!raw) {
            this._currentSubs = null;
        } else {
            const track = JadefinModules.playbackManager.subtitleTracks().find(t => t.Index == JadefinModules.playbackManager.getSubtitleStreamIndex());
            const codec = track?.Codec?.toLowerCase();
            let isAss = false;

            if (codec == "ssa" || codec == "ass") {
                isAss = true;
            }

            let subs = raw.map(l => ({
                id: l.Id,
                timeStartRaw: l.StartPositionTicks,
                timeEndRaw: l.EndPositionTicks,
                timeStart: l.StartPositionTicks / TICKS_PER_MS,
                timeEnd: l.EndPositionTicks / TICKS_PER_MS,
                isPositive: true,
                timeText: "",
                text: l.Text,
                textSpan: /** @type {HTMLElement | null} */ (null),
                active: false,
                el: /** @type {HTMLElement | null} */ (null)
            }));

            for (let line of subs) {
                let text = line.text;

                if (isAss) {
                    // TODO: Filter out SVG in ASS properly
                    text = text.replace(/^(?<!\\){[^}]*}m .+$/g, "");

                    // https://aegisub.org/docs/3.2/ASS_Tags/
                    // Just a basic set of tags - anything more advanced should hopefully only be found in {}
                    text = text.replace(/(?:\\(?:n|N|h|[ius][10]|b-?[\d.]+|[xy]?(?:bord|shad|be|blur|fs|fsc[xy]|fsp|fr[xyz]?|fa[xy]|fe|an?|[kpq])-?[\d.]+|r))+/g, " ");
                }

                text = text.replace(/(?<!\\){[^}]*}/g, "");
                text = text.replace(/\n\n\n+/g, "\n\n");
                line.text = text;

                const span = document.createElement("span");
                span.innerText = text;
                span.innerHTML = span.innerHTML.replace("\n", "<br>");

                line.textSpan = span;
            }

            const seen = [];
            const filtered = [];

            let lastTime = -1;

            for (let line of subs) {
                let time = line.timeStartRaw;
                if (time < lastTime) {
                    continue;
                }

                const text = line.textSpan?.innerText?.trim()?.toLowerCase() ?? "";

                if (text.length == 0) {
                    continue;
                }

                // TODO: Filter out karaoke in ASS properly
                if (isAss && text.indexOf(" ") == -1 && seen.findIndex(t => t.indexOf(text) != -1) != -1) {
                    continue;
                }

                let index = seen.indexOf(text);

                if (index == -1) {
                    filtered.push(line);
                    lastTime = time;

                    if (seen.length >= 10) {
                        seen.splice(0, 1);
                    }
                } else {
                    seen.splice(index, 1);
                }

                seen.push(text);
            }

            this._currentSubs = filtered;
        }

        this._isAutoscroll = true;
        this._lastOffs = Number.NaN;
        this._lastInView = Number.NaN;

        this.update();
    }

    /**
     * @param {ResizeObserverEntry[]} entries
     */
    updatePosition(entries) {
        const entry = entries[0];
        const height = entry.contentRect.height;

        if (height <= 0) {
            return;
        }

        this._osdTranscript.style.bottom = `calc(${height}px + 3em)`;
    }

    updateActive() {
        const video = JadefinUtils.video;
        const currentPlayer = JadefinUtils.currentPlayer;
        const subs = this.isEnabled ? this.currentSubs : null;
        if (!video || !currentPlayer || !subs) {
            return;
        }

        const time =
            video.currentTime * 1000 +
            ((currentPlayer._currentPlayOptions?.transcodingOffsetTicks || 0) / TICKS_PER_MS) +
            currentPlayer.getSubtitleOffset() * 1000;

        /** @type {any} */
        let nextInView = null;

        for (let i in subs) {
            const line = subs[i];
            if (!line.isPositive) {
                continue;
            }

            const active = line.timeStart <= time && time <= line.timeEnd;
            const progress = (time - line.timeStart) / (line.timeEnd - line.timeStart);

            if (progress < 0 && line.timeStart < (subs[nextInView]?.timeStart ?? Number.MAX_VALUE)) {
                nextInView = i;
            }

            if (!line.active && !active) {
                continue;
            }

            if (active) {
                nextInView = i + 1;
            }

            line.active = active;
            line.el?.classList.toggle("active", active);
            line.el?.style.setProperty("--progress", `${Math.max(0, Math.min(progress, 1))}`);
        }

        nextInView = nextInView != null ? Math.max(0, nextInView - 1) : (subs.findIndex(l => l.isPositive) + 1);

        if (this._lastInView != nextInView) {
            const lastInView = subs[this._lastInView];

            if (lastInView && Math.abs(lastInView.el.getBoundingClientRect().bottom - this._osdTranscriptLogWrap.getBoundingClientRect().bottom) < 8) {
                this._isAutoscroll = true;
            }

            this._lastInView = nextInView;

            if (this._isAutoscroll) {
                this._ignoreScroll = true;

                subs[nextInView]?.el?.scrollIntoView({
                    behavior: this._osdTranscript.getAttribute("inert") != null ? "instant" : "smooth",
                    block: "end",
                    inline: "nearest"
                });
            }
        }
    }

    update() {
        if (!JadefinUtils.routePathIsVideo || !this.currentSubs) {
            this._isEnabled = false;
        }

        this.log.v(`Updating, enabled: ${this.isEnabled}`);

        this._updateOsdPage();
        this._updateHtmlVideoPlayer();
        this._updateSubtitleTimes();
        this._updateTranscriptLog();
        this.updateActive();

        this._btnTranscript.classList.toggle("hide", !this.currentSubs);
        this._btnTranscript.classList.toggle("enabled", this.isEnabled);
        this._osdTranscript.classList.toggle("enabled", this.isEnabled);
        this._osdTranscript.toggleAttribute("inert", !this.isEnabled);
    }

    _updateOsdPage() {
        const videoOsdPage = document.querySelector("div#videoOsdPage:not(.hide)");
        if (!videoOsdPage || this._lastOsdPage == videoOsdPage) {
            return;
        }
        this._lastOsdPage = videoOsdPage;

        this.log.i("Adding event listener to videoOsdPage");
        this.log.dir(videoOsdPage);

        videoOsdPage.addEventListener(window.PointerEvent ? "pointerdown" : "click", e => {
            const srcElement = /** @type {HTMLElement} */ (e.target);

            if (JadefinUtils.hasParent(srcElement, this._osdTranscript)) {
                e.stopPropagation();
            }
        }, true);

        const osdBottom = videoOsdPage.querySelector(".videoOsdBottom");
        if (osdBottom) {
            this.log.i("Observing osd bottom size");
            this.log.dir(osdBottom);

            this._osdBottomObserver.disconnect();
            this._osdBottomObserver.observe(osdBottom);
        }

        const buttons = videoOsdPage.querySelector(".osdControls > .buttons");
        if (this._btnTranscript.parentElement != buttons && buttons) {
            this.log.i("Adding transcript button to osd buttons");
            this.log.dir(buttons);

            buttons.insertBefore(this._btnTranscript, buttons.querySelector(".btnSubtitles"));
        }

        if (this._osdTranscript.parentElement != videoOsdPage) {
            this.log.i("Adding transcript to osd page");

            videoOsdPage.insertBefore(this._osdTranscript, videoOsdPage.querySelector(".videoOsdBottom"));
        }
    }

    _updateHtmlVideoPlayer() {
        if (!this.isEnabled) {
            return;
        }

        const currentPlayer = JadefinUtils.currentPlayer;
        if (!currentPlayer || this._lastHtmlVideoPlayer == currentPlayer) {
            return;
        }
        this._lastHtmlVideoPlayer = currentPlayer;

        this.log.i("Adding event listener to htmlVideoPlayer");
        this.log.dir(currentPlayer);

        JadefinModules.Events.on(currentPlayer, "timeupdate", this.updateActive.bind(this));
    }

    _updateSubtitleTimes() {
        const subs = this.isEnabled ? this.currentSubs : null;
        if (!subs) {
            return;
        }

        const offs = (JadefinUtils.htmlVideoPlayer?.getSubtitleOffset() ?? 0) * 1000 * TICKS_PER_MS;
        if (this._lastOffs == offs) {
            return;
        }
        this._lastOffs = offs;

        for (let line of subs) {
            const time = line.timeStartRaw - offs;

            line.timeText = JadefinModules.datetime.getDisplayRunningTime(time);
            line.isPositive = time >= 0;
        }
    }

    _updateTranscriptLog() {
        const elList = this._osdTranscriptLogList;
        const subs = this.isEnabled ? this.currentSubs : null;

        if (!subs) {
            // Don't end - keep the last subs for fadeouts.

            return;
        }

        for (let i in subs) {
            const line = subs[i];

            line.el = elList.add(line.id, el => {
                const old = el;

                el = rd$(el)`
                    <tr class="line" data-index=${i} data-id=${line.id} data-positive=${line.isPositive}>
                        <td class="time">${line.timeText}</td>
                        <td class="text">${line.textSpan}</td>
                    </tr>
                `;

                if (!old) {
                    el.addEventListener("click", () => {
                        if (document.body.getAttribute("input-eaten") != "true") {
                            JadefinModules.playbackManager.seek(el["_data_subLine"].timeStartRaw);
                        }
                    });
                }

                return el;
            });

            line.el?.classList.toggle("active", false);

            line.el["_data_subLine"] = line;
        }

        elList.end();
    }

})());