s or Last Chapter/Next Chapter
s
while (isEmptyOrGarbage(contentEl.firstChild)) {
contentEl.firstChild.remove();
}
while (isEmptyOrGarbage(contentEl.lastChild)) {
contentEl.lastChild.remove();
}
// Remove redundant attributes and style
for (const child of contentEl.children) {
if (child.getAttribute("dir") === "ltr") {
child.removeAttribute("dir");
}
// Only ever appears with align="LEFT" (useless) or align="CENTER" overridden by style="text-align: left;" (also
// useless)
child.removeAttribute("align");
const style = child.getAttribute("style");
if (style === "text-align:left;" || style === "text-align: left;") {
child.removeAttribute("style");
}
// Worm uses 30px; Ward mostly uses 40px but sometimes uses 30px/60px. Let's standardize on 30px.
if (style === "text-align:left;padding-left:30px;" ||
style === "text-align: left;padding-left: 40px;" ||
style === "text-align: left; padding-left: 40px;" ||
style === "padding-left: 40px;") {
child.setAttribute("style", "padding-left: 30px;");
}
}
// Remove empty inline elements.
// Remove style attributes from inline elements, as they're always messed up.
for (const el of contentEl.querySelectorAll("em, i, strong, b")) {
const { textContent } = el;
if (textContent === "") {
el.remove();
} else if (textContent.trim() === "") {
if (el.childElementCount === 0) {
el.replaceWith(" ");
} else if (el.childElementCount === 1 && el.children[0].localName === "br") {
el.outerHTML = "
\n";
}
} else {
el.removeAttribute("style");
}
}
// In https://parahumans.wordpress.com/2013/01/05/monarch-16-13/ there are some
s O_o
for (const address of contentEl.querySelectorAll("address")) {
const p = contentEl.ownerDocument.createElement("p");
p.innerHTML = address.innerHTML;
address.replaceWith(p);
}
// Every except underline ones is pointless at best and frequently messed up. (Weird font size, line spacing,
// etc.)
for (const span of contentEl.querySelectorAll("span")) {
const style = span.getAttribute("style");
if (style === "text-decoration:underline;" || style === "text-decoration: underline;") {
continue;
}
if (span.textContent.trim() === "") {
span.remove();
} else {
const docFrag = contentEl.ownerDocument.createDocumentFragment();
while (span.firstChild) {
docFrag.appendChild(span.firstChild);
}
span.replaceWith(docFrag);
}
}
// In Ward, CloudFlare email protection obfuscates the email addresses:
// https://usamaejaz.com/cloudflare-email-decoding/
for (const emailEl of contentEl.querySelectorAll("[data-cfemail]")) {
const decoded = decodeCloudFlareEmail(emailEl.dataset.cfemail);
emailEl.replaceWith(contentEl.ownerDocument.createTextNode(decoded));
}
// Synthesize a tag to serialize
const bodyEl = contentEl.ownerDocument.createElement("body");
const h1El = contentEl.ownerDocument.createElement("h1");
h1El.textContent = chapter.usedTitle;
bodyEl.append(h1El, "\n\n");
while (contentEl.firstChild) {
bodyEl.append(contentEl.firstChild);
}
bodyEl.append("\n");
const xmlSerializer = new contentEl.ownerDocument.defaultView.XMLSerializer();
let xml = xmlSerializer.serializeToString(bodyEl);
// Fix recurring strange pattern of extra ......
in
\n
\s*<\/em><\/p>/vg, "
([^>]+)<\/em>(!|\?|\.)<\/p>/vg, " $1$2
”/vg, "
“"); xml = xml.replace(/“\s*<\/p>/vg, "”
"); xml = xml.replace(/“\s*<\/em><\/p>/vg, "“\s+/vg, "
“"); xml = xml.replace(/\s+”/vg, "”"); xml = xml.replace(/'/vg, "’"); xml = xml.replace(/’([A-Za-z]+)’/vg, "‘$1’"); xml = xml.replace(/([a-z])”<\/p>/vg, "$1.”
"); fixEms(); xml = xml.replace(/‘([^<]+)<\/em>‘/vg, "‘$1’"); xml = xml.replace(/([a-z]+)!<\/em>/vg, "$1!"); xml = xml.replace(/(?([\w ’]+)([!.?])”<\/em>/vg, "$1$2”"); xml = xml.replace(/([\w ’]+[!.?])”<\/em>/vg, "$1”"); xml = xml.replace(/I”(m|ll)/vg, "I’$1"); xml = xml.replace(/””<\/p>/vg, "”"); xml = xml.replace(/^([^“]+?) ?”(?![ —<])/vgm, "$1 “"); xml = xml.replace(/(?([A-Za-z]+),<\/em>(?!”| +[A-Za-z]+ thought)/vg, "$1,"); xml = xml.replace(/‘([Kk])ay(?!’)/vg, "’$1ay"); xml = xml.replace(/(Why|What|Who|How|Where|When)<\/em>\?/vg, "$1?"); xml = xml.replace(/,<\/em>/vg, ","); xml = xml.replace(/,”<\/p>/vg, ".”"); xml = xml.replace(/(.*),<\/p>/vg, "
$1.
"); xml = xml.replace(/‘(\w+)‘(\w+)’/vg, "‘$1’$2’"); xml = xml.replace(/([a-z]+), ([a-z]+)<\/em>/vg, "$1, $2"); // Similar problems occur in Ward with and as do in Worm with s xml = xml.replace(//vg, ""); xml = xml.replace(/(\s*\s*)<\/b>/vg, "$1"); xml = xml.replace(/(\s*
\s*)<\/strong>/vg, "$1"); xml = xml.replace(/<\/strong>(\s*)/vg, "$1"); xml = xml.replace(/@<\/strong>/vg, "@"); xml = xml.replace(/
(\s*)<\/strong>/vg, "
$1"); xml = xml.replace(/(\s*)<\/strong>/vg, "$1"); xml = xml.replace(/>(.*)<\/strong>:$1:<"); xml = xml.replace(/
\n/vg, "
\n"); // No need for line breaks before paragraph ends or after paragraph starts // These often occur with the
s inside /// fixed above. xml = xml.replace(/
\s*<\/p>/vg, ""); xml = xml.replace(/
\s*/vg, "
"); // This is another quote fix but it needs to happen after the line break deletion... so entangled, ugh. xml = xml.replace(/<\/em>\s*“\s*<\/p>/vg, "
”"); // Fix missing spaces after commas xml = xml.replace(/([a-zA-Z]+),([a-zA-Z]+)/vg, "$1, $2"); // Fix bad periods and spacing/markup surrounding them xml = xml.replace(/\.\.<\/p>/vg, "."); xml = xml.replace(/\.\.”<\/p>/vg, ".”"); xml = xml.replace(/ \. /vg, ". "); xml = xml.replace(/ \.<\/p>/vg, "."); xml = xml.replace(/\.\.\./vg, "…"); xml = xml.replace(/\.\. {2}/vg, ". "); xml = xml.replace(/\.\./vg, "…"); xml = xml.replace(/(?/vg, ""); xml = xml.replace(/([a-z]) ,/vg, "$1,"); // Use actual emojis instead of images xml = xml.replaceAll( `
`,
"🤨"
);
// This needs to happen before other name-related fixes.
xml = standardizeNames(xml);
// Users on messageboards rarely use proper punctuation or standardized spelling, etc. Leave most stuff as-is.
if (!chapter.mostlyMessageboard) {
xml = fixTruncatedWords(xml);
xml = fixDialogueTags(xml);
xml = fixForeignNames(xml);
xml = fixEmDashes(xml);
xml = enDashJointNames(xml);
xml = fixPossessives(xml);
xml = fixCapitalization(xml, bookTitle);
xml = fixMispellings(xml);
xml = standardizeSpellings(xml);
xml = fixHyphens(xml);
xml = fixCaseNumbers(xml);
xml = fixClassNames(xml);
}
xml = cleanSceneBreaks(xml);
xml = fixParahumansOnline(xml);
// One-off fixes
for (const substitution of chapterSubstitutions) {
if (substitution.before) {
const indexOf = xml.indexOf(substitution.before);
if (indexOf === -1) {
warnings.push(`Could not find text "${substitution.before}" in ${chapter.url}. The chapter may have been ` +
`updated at the source, in which case, you should edit the substitutions file.`);
}
if (indexOf !== xml.lastIndexOf(substitution.before)) {
warnings.push(`The text "${substitution.before}" occurred twice, and so the substitution was ambiguous. ` +
`Update the substitutions file for a more precise substitution.`);
}
xml = xml.replace(new RegExp(escapeRegExp(substitution.before), "u"), substitution.after);
} else if (substitution.regExp) {
xml = xml.replace(substitution.regExp, substitution.replacement);
} else {
warnings.push(`Invalid substitution specified for ${chapter.url}`);
}
}
// Serializer inserts extra xmlns for us since it doesn't know we're going to put this into a .
xml = xml.replaceAll(
``,
`\n`
);
return { xml, warnings };
}
function fixTruncatedWords(xml) {
xml = xml.replace(/‘Sup/vg, "’Sup");
xml = xml.replace(/(? “I didn’t get much done,” Greg said, “I got distracted by...
// This should instead be
// > “I didn’t get much done,” Greg said. “I got distracted by...
//
// Our heuristic is to try to automatically fix this if the dialogue tag is two words (X said/admitted/sighed/etc.).
//
// This sometimes overcorrects, as in the following example:
// > “Basically,” Alec said, “For your powers to manifest, ...
// Here instead we should lowercase the "f". We handle that via one-offs in the substitutions file.
//
// This applies to ~800 instances, so although we have to correct back in the substitutions file a decent number of
// times, it definitely pays for itself. Most of the instances we have to correct back we also need to fix the
// capitalization anyway, and that's harder to do automatically, since proper names/"I"/etc. stay capitalized.
xml = xml.replace(/,” ([A-Za-z]+ [A-Za-z]+), “([A-Z])/vg, ",” $1. “$2");
return xml;
}
function fixForeignNames(xml) {
// This is consistently missing diacritics
xml = xml.replace(/Yangban/vg, "Yàngbǎn");
// These are usually not italicized, but sometimes are. Other foreign-language names (like Yàngbǎn) are not
// italicized, so we go in the direction of removing the italics.
xml = xml.replace(/Garama<\/em>/vg, "Garama");
xml = xml.replace(/Thanda<\/em>/vg, "Thanda");
xml = xml.replace(/Vornehm<\/em>/vg, "Vornehm");
xml = xml.replace(/Schwarze Tante<\/em>/vg, "Schwarze Tante");
xml = xml.replace(/Sifara([^<]*)<\/em>/vg, "Sifara$1");
xml = xml.replace(/Moord Nag([^<]*)<\/em>/vg, "Moord Nag$1");
xml = xml.replace(/Califa de Perro([^<]*)<\/em>/vg, "Califa de Perro$1");
xml = xml.replace(/Turanta([^<]*)<\/em>/vg, "Turanta$1");
return xml;
}
function standardizeNames(xml) {
// 197 instances of "Mrs." to 21 of "Ms."
xml = xml.replace(/Ms\. Yamada/vg, "Mrs. Yamada");
// 25 instances of "Amias" to 3 of "Amais"
xml = xml.replace(/Amais/vg, "Amias");
// 185 instances of Juliette to 4 of Juliet
xml = xml.replace(/Juliet(?=\b)/vg, "Juliette");
// Earlier chapters have a space; later ones do not. They're separate words, so side with the earlier chapters.
// One location is missing the "k".
xml = xml.replace(/Crock? o[‘’]Shit/vg, "Crock o’ Shit");
// 5 instances of "Jotun" to 2 of "Jotunn"
xml = xml.replace(/Jotunn/vg, "Jotun");
// 13 instances of Elman to 1 of Elmann
xml = xml.replace(/Elmann/vg, "Elman");
// Thousands of instances of Tattletale to 4 instances of Tatteltale
xml = xml.replace(/Tatteltale/vg, "Tattletale");
// 73 instances of Über to 2 of Uber
xml = xml.replace(/Uber/vg, "Über");
// 5 instances of Johnsonjar to 2 instances of JohnsonJar
xml = xml.replace(/JohnsonJar/vg, "Johnsonjar");
// 4 instances of Flying_Kevin to 2 instances of FlyingKevin
xml = xml.replace(/FlyingKevin/vg, "Flying_Kevin");
// 5 instances of Jean-Paul to 2 instances of Jean-paul
xml = xml.replace(/Jean-paul/vg, "Jean-Paul");
// Dragoncraft names are inexplicably hyphenated and inconsistently capitalized
xml = xml.replace(/Melusine-six/vig, "Melusine Six");
xml = xml.replace(/Glaurung Zero-Model/vig, "Glaurung Zero Model");
xml = xml.replace(/Pythios-Two/vig, "Pythios Two");
xml = xml.replace(/Ladon-two/vig, "Ladon Two");
// 3 instances of Matroyshka to 41 of Matryoshka
xml = xml.replace(/Matroyshka/vg, "Matryoshka");
// 5 instances of Travellers to 179 of Travellers
xml = xml.replace(/Travellers/vg, "Travelers");
return xml;
}
function fixEmDashes(xml) {
xml = xml.replace(/ – /vg, "—");
xml = xml.replace(/“((?:)?)-/vg, "“$1—");
xml = xml.replace(/-[,.]?”/vg, "—”");
xml = xml.replace(/-(!|\?)”/vg, "—$1”");
xml = xml.replace(/-[,.]?<\/([a-z]+)>”/vg, "—”");
xml = xml.replace(/-“/vg, "—”");
xml = xml.replace(/-/vg, "
—"); xml = xml.replace(/-<\/p>/vg, "—
"); xml = xml.replace(/-/vg, "—
"); xml = xml.replace(/-<\/([a-z]+)><\/p>/vg, "—"); xml = xml.replace(/\s?\s?–\s?\s?/vg, "—"); xml = xml.replace(/-\s\s?/vg, "—"); xml = xml.replace(/\s?\s-/vg, "—"); xml = xml.replace(/\s+—”/vg, "—”"); xml = xml.replace(/I-I/vg, "I—I"); xml = xml.replace(/I-uh/vg, "I—uh"); xml = xml.replace(/-\?/vg, "—?"); return xml; } function enDashJointNames(xml) { // Joint names should use en dashes xml = xml.replace(/(A|a)gent-parahuman/vg, "$1gent–parahuman"); xml = xml.replace(/(P|p)arahuman-agent/vg, "$1arahuman–agent"); xml = xml.replace(/(Chinese|Japanese|Asian)-American/vg, "$1–American"); xml = xml.replace(/Alexandria-Pretender/vg, "Alexandria–Pretender"); xml = xml.replace(/Antares-Anelace/vg, "Antares–Anelace"); xml = xml.replace(/Armsmaster-Defiant/vg, "Armsmaster–Defiant"); xml = xml.replace(/Astaroth-Nidhug/vg, "Astaroth–Nidhug"); xml = xml.replace(/Bet-Gimel/vg, "Bet–Gimel"); xml = xml.replace(/Capricorn-Byron/vg, "Capricorn–Byron"); xml = xml.replace(/Capulet-Montague/vg, "Capulet–Montague"); xml = xml.replace(/Challenger-Gallant/vg, "Challenger–Gallant"); xml = xml.replace(/Cheit-Gimel/vg, "Bet–Gimel"); xml = xml.replace(/Crawler-Breed/vg, "Crawler–Breed"); xml = xml.replace(/Creutzfeldt-Jakob/vg, "Creutzfeldt–Jakob"); xml = xml.replace(/Dallon-Pelham/vg, "Dallon–Pelham"); xml = xml.replace(/Damsel-Ashley/vg, "Damsel–Ashley"); xml = xml.replace(/East-West/vg, "east–west"); xml = xml.replace(/G-N/vg, "G–N"); xml = xml.replace(/Gaea-Eden/vg, "Gaea–Eden"); xml = xml.replace(/Gimel-Europe/vg, "Gimel–Europe"); xml = xml.replace(/Imp-Damsel/vg, "Imp–Damsel"); xml = xml.replace(/Matryoshka-Valentin/vg, "Matryoshka–Valentin"); xml = xml.replace(/Night Hag-Nyx/vg, "Night Hag–Nyx"); xml = xml.replace(/Norwalk-Fairfield/vg, "Norwalk–Fairfield"); xml = xml.replace(/Simurgh-Myrddin-plant/vg, "Simurgh–Myrddin–plant"); xml = xml.replace(/Tristan-Byron/vg, "Tristan–Byron"); xml = xml.replace(/Tristan-Capricorn/vg, "Tristan–Capricorn"); xml = xml.replace(/Undersider(s?)-(Breakthrough|Ambassador)/vg, "Undersider$1–$2"); xml = xml.replace(/Weaver-Clockblocker/vg, "Weaver–Clockblocker"); xml = xml.replace(/Winter-Mannequin/vg, "Winter–Mannequin"); return xml; } function fixPossessives(xml) { // Fix possessive of names ending in "s". xml = xml.replace( /(? would be more semantically appropriate, but loses the author's intent. This is // especially the case in Ward, which uses a variety of different scene breaks. xml = xml.replace(/
■
`); xml = xml.replace( /⊙<\/strong><\/p>/vg,
` ⊙ ⊙<\/strong><\/em><\/p>/vg,
` ⊙ ⊙⊙<\/strong><\/p>/vg,
` ⊙ ⊙ *⊙ *⊙ *⊙ *⊙<\/strong><\/p>/vg,
` ⊙ ⊙ ⊙ ⊙ ⊙ Welcome to the Parahumans Online message boards. Welcome to the Parahumans Online message boards. ♦ (.*)<\/strong><\/p>/vg, ` ♦ $1
`,
`
`
);
xml = xml.replace(
/You are currently logged in, ([^<]+)<\/span>/vg,
`You are currently logged in, $1`
);
// Most cases have the colon but some don't.
xml = xml.replace(/(Replied on \w+ \d{1,2}(?:st|nd|rd|th),? ?Y?\d*)
/vg, "$1:
");
// "You have marked yourself as away." has a period, so this one should too.
xml = xml.replace(/(You have marked yourself as back)(?)/vg, "$1.");
// It's inconsistent to exclude the punctuation from the bolding; fix it.
xml = xml.replace(/Welcome back to (.+?)<\/strong>!/vg, "Welcome back to $1!");
xml = xml.replace(/