jadefin/mods/Transcript.js

674 lines
22 KiB
JavaScript

//@ts-check
import JadefinIntegrity from '../JadefinIntegrity.js';
import Jadefin from "../Jadefin.js";
import JadefinMod from "../JadefinMod.js";
import JadefinModules from "../JadefinModules.js";
import JadefinUtils from "../JadefinUtils.js";
import { rd, rdom, rd$, RDOMListHelper } from "../utils/rdom.js";
const TICKS_PER_MS = 10000;
export default JadefinIntegrity("Transcript", import.meta.url, () => new (class Transcript extends JadefinMod {
_currentPlayer;
_currentTrack;
/** @type {any[] | null} */
_currentSubsRaw = null;
/** @type {any[] | null} */
_currentSubs = null;
_isEnabled = false;
_isAutoscroll = true;
_ignoreScroll = true;
/** @type {Element | undefined} */
_lastOsdPage;
_lastHtmlVideoPlayer;
_lastOffs = Number.NaN;
_lastInView = Number.NaN;
_btnTranscript = rd$()`
<button is="paper-icon-button-light" class="btnTranscript autoSize paper-icon-button-light" title="Transcript">
<span class="xlargePaperIconButton material-icons manage_search" aria-hidden="true"></span>
</button>
`;
_osdTranscript = rd$()`
<div class="osdTranscript">
<div class="transcriptLogWrap">
<table class="transcriptLog"></table>
</div>
</div>
`;
_osdTranscriptLogWrap = JadefinUtils.notnull(this._osdTranscript.querySelector(".transcriptLogWrap"));
_osdTranscriptLog = JadefinUtils.notnull(this._osdTranscript.querySelector(".transcriptLog"));
_osdTranscriptLogList = new RDOMListHelper(this._osdTranscriptLog);
_ssaAssFetching;
_ssaAssObserver = new MutationObserver(this.onSsaAssCanvasAdded.bind(this));
_ssaAssObserverConfig = {
childList: true
};
_osdBottomObserver = new ResizeObserver(this.updatePosition.bind(this));
constructor() {
super();
this._btnTranscript.addEventListener("click", () => this.isEnabled = !this.isEnabled);
this._osdTranscriptLogWrap.addEventListener("scroll", () => {
if (!this._ignoreScroll) {
this._isAutoscroll = false;
} else if (window.onscrollend === undefined) {
this._ignoreScroll = false;
}
});
this._osdTranscriptLogWrap.addEventListener("scrollend", () => {
if (!this._ignoreScroll) {
this._isAutoscroll = false;
} else {
this._ignoreScroll = false;
}
});
JadefinUtils.events.addEventListener(JadefinUtils.eventTypes.WORKER_CREATED, e => {
const detail = e["detail"];
// TODO: 10.9 should've replaced JSSO / libssa-js with JASSUB, revisit once that's stable.
if (!detail.args.scriptURL.endsWith("/libraries/subtitles-octopus-worker.js")) {
return;
}
/** @type {Worker} */
const worker = detail.worker;
worker.addEventListener("message", this.onSsaAssWorkerMessage.bind(this));
});
this.initHookDocumentAddEventListener();
this.initHookDocumentRemoveEventListener();
}
get isEnabled() {
return this._isEnabled;
}
set isEnabled(value) {
if (!this._isEnabled && value) {
this._isAutoscroll = true;
this._lastOffs = Number.NaN;
this._lastInView = Number.NaN;
}
this._isEnabled = value;
this.update();
}
get currentSubsRaw() {
return JadefinUtils.currentPlayer == this._currentPlayer ? this._currentSubsRaw : null;
}
get currentSubs() {
return JadefinUtils.currentPlayer == this._currentPlayer ? this._currentSubs : null;
}
async init(name, url) {
await super.init(name, url);
const initing = [
this.initStyle(),
this.initHookSetTrackForDisplay(),
this.initHookFetchSubtitles(),
this.initHookSetOffset(),
];
document.addEventListener("viewshow", () => {
this.update();
});
await Promise.all(initing);
this.log.i("Ready");
}
async initHookSetTrackForDisplay() {
await JadefinUtils.waitUntil(() => JadefinModules.htmlVideoPlayer);
const self = this;
const orig = this._setTrackForDisplay = JadefinModules.htmlVideoPlayer.prototype.setTrackForDisplay;
JadefinModules.htmlVideoPlayer.prototype.setTrackForDisplay = function(videoElement, track, targetTextTrackIndex) {
self.log.i("Setting subtitles");
self.log.dir(track);
if (self._currentPlayer == this &&
self._currentTrack?.DeliveryUrl && track?.DeliveryUrl &&
self._currentTrack.DeliveryUrl == track.DeliveryUrl) {
// Subtitle track got "changed" for another reason, f.e. audio track change.
return orig.call(this, videoElement, track, targetTextTrackIndex);
}
self.setSubtitles(track, null, true);
const codec = track?.Codec?.toLowerCase();
let format = "";
let incrementFetchQueue = false;
if (codec == "ssa" || codec == "ass") {
format = ".ass";
incrementFetchQueue = true;
}
// Required because ass subs are fetched separately.
if (incrementFetchQueue) {
if (self._ssaAssFetching != this) {
self.log.v("Manually incrementing fetch queue");
self._ssaAssFetching = this;
self._ssaAssObserver.disconnect();
// @ts-ignore - not entirely correct, but video is non-null here.
self._ssaAssObserver.observe(JadefinUtils.video?.parentNode, self._ssaAssObserverConfig);
this.incrementFetchQueue();
}
} else if (self._ssaAssFetching == this) {
self.log.v("Manually decrementing fetch queue (sub change)");
this._ssaAssFetching?.decrementFetchQueue();
this._ssaAssFetching = null;
this._ssaAssObserver.disconnect();
}
if (format != "") {
// We need to fetch ass subs ourselves.
self.log.i("Manually fetching subtitles (non-subrip format)");
self.log.dir(track);
let url = JadefinModules.playbackManager.getSubtitleUrl(track, JadefinModules.playbackManager.currentItem(JadefinUtils.currentPlayer));
url = url.replace(format, ".js");
this.incrementFetchQueue();
fetch(url)
.then(r => r.json())
.then(r => self.setSubtitles(track, r))
.finally(() => this.decrementFetchQueue());
}
return orig.call(this, videoElement, track, targetTextTrackIndex);
};
}
async initHookFetchSubtitles() {
await JadefinUtils.waitUntil(() => JadefinModules.htmlVideoPlayer);
const self = this;
const orig = this._fetchSubtitles = JadefinModules.htmlVideoPlayer.prototype.fetchSubtitles;
JadefinModules.htmlVideoPlayer.prototype.fetchSubtitles = function(track, item) {
self.log.i("Fetching subtitles");
self.log.dir(track);
const rv = orig.call(this, track, item);
rv.then(r => self.setSubtitles(track, r), () => self.setSubtitles(track, null));
return rv;
};
}
async initHookSetOffset() {
await JadefinUtils.waitUntil(() => JadefinModules.htmlVideoPlayer);
const self = this;
const resetSubtitleOffset = this._resetSubtitleOffset = JadefinModules.htmlVideoPlayer.prototype.resetSubtitleOffset;
JadefinModules.htmlVideoPlayer.prototype.resetSubtitleOffset = function() {
var rv = resetSubtitleOffset.call(this);
self.update();
return rv;
};
const updateCurrentTrackOffset = this._resetSubtitleOffset = JadefinModules.htmlVideoPlayer.prototype.updateCurrentTrackOffset;
JadefinModules.htmlVideoPlayer.prototype.updateCurrentTrackOffset = function(offsetValue, currentTrackIndex) {
var rv = updateCurrentTrackOffset.call(this, offsetValue, currentTrackIndex);
self.update();
return rv;
};
}
initHookDocumentAddEventListener() {
const orig = this._addEventListener = document.addEventListener.bind(document);
document.addEventListener = (type, listener, options) => {
if (type == "wheel") {
const listenerStr = listener.toString();
// Anonymous function in playback-video
if (listenerStr.indexOf("volumeUp") != -1 &&
listenerStr.indexOf("volumeDown") != -1 &&
listenerStr.indexOf("deltaY") != -1) {
this.log.i("Wrapping playback-video wrap listener");
this.log.dir(listener);
const origListener = this._playbackWheel = listener;
listener = this._playbackWheelWrap = (e) => {
if (JadefinUtils.hasParent(e.target, this._osdTranscript)) {
return;
}
return origListener(e);
};
}
}
return orig(type, listener, options);
};
}
initHookDocumentRemoveEventListener() {
const orig = this._removeEventListener = document.removeEventListener.bind(document);
document.removeEventListener = (type, listener, options) => {
if (listener == this._playbackWheel) {
listener = this._playbackWheelWrap;
}
return orig(type, listener, options);
};
}
onSsaAssCanvasAdded() {
const htmlVideoPlayer = this._ssaAssFetching;
this._ssaAssObserver.disconnect();
this.log.v("Manual fetch queue progress - SsaAss canvas updated");
if (!htmlVideoPlayer) {
this.log.v("Manual fetch queue fail - no htmlVideoPlayer");
return;
}
// The canvas gets added relatively early, therefore hook drawImage.
// TODO: 10.9 should've replaced JSSO / libssa-js with JASSUB, revisit once that's stable.
const canvas = JadefinUtils.video.parentNode?.querySelector("canvas");
const ctx = canvas?.getContext("2d");
if (!ctx) {
this.log.v("Manually decrementing fetch queue (no 2d ctx)");
htmlVideoPlayer.decrementFetchQueue();
return;
}
// TODO: drawImage doesn't fire for ass / ssa scenarios where subs start late.
const orig = ctx.drawImage.bind(ctx);
ctx.drawImage = (...args) => {
if (!this._ssaAssFetching) {
return orig(...args);
}
this.log.v("Manually decrementing fetch queue (drawImage)");
this._ssaAssFetching.decrementFetchQueue();
this._ssaAssFetching = null;
return orig(...args);
};
}
onSsaAssWorkerMessage(e) {
if (!this._ssaAssFetching) {
return;
}
if (e.data.target == "canvas" && e.data.op == "oneshot-result") {
this.log.v("Manually decrementing fetch queue (worker message)");
this._ssaAssFetching.decrementFetchQueue();
this._ssaAssFetching = null;
}
}
setSubtitles(track, subs, force) {
if (!force && this._currentTrack != track) {
this.log.w("Late sub fetch");
return;
}
if (subs) {
this.log.i("Subtitles updated");
this.log.dir(subs);
} else {
this.log.i("Subtitles reset");
}
this._currentPlayer = JadefinUtils.currentPlayer;
this._currentTrack = track;
const raw = this._currentSubsRaw = /** @type {any[] | null} */ (subs?.TrackEvents);
if (!raw) {
this._currentSubs = null;
} else {
const track = JadefinModules.playbackManager.subtitleTracks().find(t => t.Index == JadefinModules.playbackManager.getSubtitleStreamIndex());
const codec = track?.Codec?.toLowerCase();
let isAss = false;
if (codec == "ssa" || codec == "ass") {
isAss = true;
}
let subs = raw.map(l => ({
id: l.Id,
timeStartRaw: l.StartPositionTicks,
timeEndRaw: l.EndPositionTicks,
timeStart: l.StartPositionTicks / TICKS_PER_MS,
timeEnd: l.EndPositionTicks / TICKS_PER_MS,
isPositive: true,
timeText: "",
text: l.Text,
textSpan: /** @type {HTMLElement | null} */ (null),
active: false,
el: /** @type {HTMLElement | null} */ (null)
}));
for (let line of subs) {
let text = line.text;
if (isAss) {
// TODO: Filter out SVG in ASS properly
text = text.replace(/^(?<!\\){[^}]*}m .+$/g, "");
// https://aegisub.org/docs/3.2/ASS_Tags/
// Just a basic set of tags - anything more advanced should hopefully only be found in {}
text = text.replace(/(?:\\(?:n|N|h|[ius][10]|b-?[\d.]+|[xy]?(?:bord|shad|be|blur|fs|fsc[xy]|fsp|fr[xyz]?|fa[xy]|fe|an?|[kpq])-?[\d.]+|r))+/g, " ");
}
text = text.replace(/(?<!\\){[^}]*}/g, "");
text = text.replace(/\n\n\n+/g, "\n\n");
line.text = text;
const span = document.createElement("span");
span.innerText = text;
span.innerHTML = span.innerHTML.replace("\n", "<br>");
line.textSpan = span;
}
const seen = [];
const filtered = [];
let lastTime = -1;
for (let line of subs) {
let time = line.timeStartRaw;
if (time < lastTime) {
continue;
}
const text = line.textSpan?.innerText?.trim()?.toLowerCase() ?? "";
if (text.length == 0) {
continue;
}
// TODO: Filter out karaoke in ASS properly
if (isAss && text.indexOf(" ") == -1 && seen.findIndex(t => t.indexOf(text) != -1) != -1) {
continue;
}
let index = seen.indexOf(text);
if (index == -1) {
filtered.push(line);
lastTime = time;
if (seen.length >= 10) {
seen.splice(0, 1);
}
} else {
seen.splice(index, 1);
}
seen.push(text);
}
this._currentSubs = filtered;
}
this._isAutoscroll = true;
this._lastOffs = Number.NaN;
this._lastInView = Number.NaN;
this.update();
}
/**
* @param {ResizeObserverEntry[]} entries
*/
updatePosition(entries) {
const entry = entries[0];
const height = entry.contentRect.height;
if (height <= 0) {
return;
}
this._osdTranscript.style.bottom = `calc(${height}px + 3em)`;
}
updateActive() {
const video = JadefinUtils.video;
const currentPlayer = JadefinUtils.currentPlayer;
const subs = this.isEnabled ? this.currentSubs : null;
if (!video || !currentPlayer || !subs) {
return;
}
const time =
video.currentTime * 1000 +
((currentPlayer._currentPlayOptions?.transcodingOffsetTicks || 0) / TICKS_PER_MS) +
currentPlayer.getSubtitleOffset() * 1000;
/** @type {any} */
let nextInView = null;
for (let i in subs) {
const line = subs[i];
if (!line.isPositive) {
continue;
}
const active = line.timeStart <= time && time <= line.timeEnd;
const progress = (time - line.timeStart) / (line.timeEnd - line.timeStart);
if (progress < 0 && line.timeStart < (subs[nextInView]?.timeStart ?? Number.MAX_VALUE)) {
nextInView = i;
}
if (!line.active && !active) {
continue;
}
if (active) {
nextInView = i + 1;
}
line.active = active;
line.el?.classList.toggle("active", active);
line.el?.style.setProperty("--progress", `${Math.max(0, Math.min(progress, 1))}`);
}
nextInView = nextInView != null ? Math.max(0, nextInView - 1) : (subs.findIndex(l => l.isPositive) + 1);
if (this._lastInView != nextInView) {
const lastInView = subs[this._lastInView];
if (lastInView && Math.abs(lastInView.el.getBoundingClientRect().bottom - this._osdTranscriptLogWrap.getBoundingClientRect().bottom) < 8) {
this._isAutoscroll = true;
}
this._lastInView = nextInView;
if (this._isAutoscroll) {
this._ignoreScroll = true;
subs[nextInView]?.el?.scrollIntoView({
behavior: this._osdTranscript.getAttribute("inert") != null ? "instant" : "smooth",
block: "end",
inline: "nearest"
});
}
}
}
update() {
if (!JadefinUtils.routePathIsVideo || !this.currentSubs) {
this._isEnabled = false;
}
this.log.v(`Updating, enabled: ${this.isEnabled}`);
this._updateOsdPage();
this._updateHtmlVideoPlayer();
this._updateSubtitleTimes();
this._updateTranscriptLog();
this.updateActive();
this._btnTranscript.classList.toggle("hide", !this.currentSubs);
this._btnTranscript.classList.toggle("enabled", this.isEnabled);
this._osdTranscript.classList.toggle("enabled", this.isEnabled);
this._osdTranscript.toggleAttribute("inert", !this.isEnabled);
}
_updateOsdPage() {
const videoOsdPage = document.querySelector("div#videoOsdPage:not(.hide)");
if (!videoOsdPage || this._lastOsdPage == videoOsdPage) {
return;
}
this._lastOsdPage = videoOsdPage;
this.log.i("Adding event listener to videoOsdPage");
this.log.dir(videoOsdPage);
videoOsdPage.addEventListener(window.PointerEvent ? "pointerdown" : "click", e => {
const srcElement = /** @type {HTMLElement} */ (e.target);
if (JadefinUtils.hasParent(srcElement, this._osdTranscript)) {
e.stopPropagation();
}
}, true);
const osdBottom = videoOsdPage.querySelector(".videoOsdBottom");
if (osdBottom) {
this.log.i("Observing osd bottom size");
this.log.dir(osdBottom);
this._osdBottomObserver.disconnect();
this._osdBottomObserver.observe(osdBottom);
}
const buttons = videoOsdPage.querySelector(".osdControls > .buttons");
if (this._btnTranscript.parentElement != buttons && buttons) {
this.log.i("Adding transcript button to osd buttons");
this.log.dir(buttons);
buttons.insertBefore(this._btnTranscript, buttons.querySelector(".btnSubtitles"));
}
if (this._osdTranscript.parentElement != videoOsdPage) {
this.log.i("Adding transcript to osd page");
videoOsdPage.insertBefore(this._osdTranscript, videoOsdPage.querySelector(".videoOsdBottom"));
}
}
_updateHtmlVideoPlayer() {
if (!this.isEnabled) {
return;
}
const currentPlayer = JadefinUtils.currentPlayer;
if (!currentPlayer || this._lastHtmlVideoPlayer == currentPlayer) {
return;
}
this._lastHtmlVideoPlayer = currentPlayer;
this.log.i("Adding event listener to htmlVideoPlayer");
this.log.dir(currentPlayer);
JadefinModules.Events.on(currentPlayer, "timeupdate", this.updateActive.bind(this));
}
_updateSubtitleTimes() {
const subs = this.isEnabled ? this.currentSubs : null;
if (!subs) {
return;
}
const offs = (JadefinUtils.htmlVideoPlayer?.getSubtitleOffset() ?? 0) * 1000 * TICKS_PER_MS;
if (this._lastOffs == offs) {
return;
}
this._lastOffs = offs;
for (let line of subs) {
const time = line.timeStartRaw - offs;
line.timeText = JadefinModules.datetime.getDisplayRunningTime(time);
line.isPositive = time >= 0;
}
}
_updateTranscriptLog() {
const elList = this._osdTranscriptLogList;
const subs = this.isEnabled ? this.currentSubs : null;
if (!subs) {
// Don't end - keep the last subs for fadeouts.
return;
}
for (let i in subs) {
const line = subs[i];
line.el = elList.add(line.id, el => {
const old = el;
el = rd$(el)`
<tr class="line" data-index=${i} data-id=${line.id} data-positive=${line.isPositive}>
<td class="time">${line.timeText}</td>
<td class="text">${line.textSpan}</td>
</tr>
`;
if (!old) {
el.addEventListener("click", () => {
if (document.body.getAttribute("input-eaten") != "true") {
JadefinModules.playbackManager.seek(el["_data_subLine"].timeStartRaw);
}
});
}
return el;
});
line.el?.classList.toggle("active", false);
line.el["_data_subLine"] = line;
}
elList.end();
}
})());