Improve the rendering pipeline, move admonition markup processing to preproc (#513)

* Split preproc into several files

These files are shorter, and thus easier to navigate than a big monolith.

* Move admonition markup processing to preproc instead of renderer

This lets them work with `mdbook serve` (which hardcodes the HTML renderer),
and at the same time is more robust (no more running regexes against HTML
output!).

The syntax was slightly adjusted to be closer to established VuePress etc.

Author: ISSOtm

CONTRIBUTING.md

@@ -37,12 +37,12 @@ In any case, maintainers will chime in, reviewing what you changed and if necess
 
 Pan Docs uses a custom mdBook preprocessor & renderer to enable some special markup:
 
-### Custom Containers 
+### Custom Containers
 
 Those mimick Vuepress' [custom containers](https://vuepress.vuejs.org/guide/markdown.html#custom-containers) functionality.
 
 ```markdown
-  ::: type HEADING
+  :::type HEADING
 
   Content
 
@@ -58,7 +58,7 @@ These are rendered as "info boxes".
 E.g.
 
 ```markdown
-::: tip SCOPE
+:::tip SCOPE
 
 The information here is targeted at homebrew development.
 Emulator developers may be also interested in the [Game Boy: Complete Technical Reference](https://gekkio.fi/files/gb-docs/gbctr.pdf) document.

preproc/src/admonitions.rs

@@ -0,0 +1,101 @@
+/*
+ * This Source Code Form is subject to the
+ * terms of the Mozilla Public License, v.
+ * 2.0. If a copy of the MPL was not
+ * distributed with this file, You can
+ * obtain one at
+ * http://mozilla.org/MPL/2.0/.
+ */
+
+use std::{iter::Peekable, matches};
+
+use anyhow::Error;
+use mdbook::book::Chapter;
+use pulldown_cmark::{Event, Options, Parser, Tag};
+
+use crate::Pandocs;
+
+impl Pandocs {
+    pub fn process_admonitions(&self, chapter: &mut Chapter) -> Result<(), Error> {
+        let mut buf = String::with_capacity(chapter.content.len());
+        let extensions =
+            Options::ENABLE_TABLES | Options::ENABLE_FOOTNOTES | Options::ENABLE_STRIKETHROUGH;
+
+        let events = AdmonitionsGenerator::new(Parser::new_ext(&chapter.content, extensions));
+
+        pulldown_cmark_to_cmark::cmark(events, &mut buf, None)
+            .map_err(|err| Error::from(err).context("Markdown serialization failed"))?;
+        chapter.content = buf;
+
+        Ok(())
+    }
+}
+
+struct AdmonitionsGenerator<'a, Iter: Iterator<Item = Event<'a>>> {
+    iter: Peekable<Iter>,
+    nesting_level: usize,
+    at_paragraph_start: bool,
+}
+
+impl<'a, Iter: Iterator<Item = Event<'a>>> AdmonitionsGenerator<'a, Iter> {
+    const KINDS: [&'static str; 3] = ["tip", "warning", "danger"];
+
+    fn new(iter: Iter) -> Self {
+        Self {
+            iter: iter.peekable(),
+            nesting_level: 0,
+            at_paragraph_start: false,
+        }
+    }
+}
+
+impl<'a, Iter: Iterator<Item = Event<'a>>> Iterator for AdmonitionsGenerator<'a, Iter> {
+    type Item = Event<'a>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let mut evt = self.iter.next()?;
+
+        match evt {
+            Event::Text(ref text) if self.at_paragraph_start => {
+                if let Some(params) = text.strip_prefix(":::") {
+                    // Check that there is no more text in the paragraph; if there isn't, we'll consume the entire paragraph.
+                    // Note that this intentionally rejects any formatting within the paragraph—serialisation would be too complex.
+                    if matches!(self.iter.peek(), Some(Event::End(Tag::Paragraph))) {
+                        if params.is_empty() {
+                            if self.nesting_level != 0 {
+                                // Ending an admonition.
+                                self.nesting_level -= 1;
+
+                                evt = Event::Html("</div>".into());
+                            }
+                        } else {
+                            let (kind, title) =
+                                match params.split_once(|c: char| c.is_ascii_whitespace()) {
+                                    Some((kind, title)) => (kind, title.trim()),
+                                    None => (params, ""),
+                                };
+                            if Self::KINDS.contains(&kind) {
+                                // Beginning an admonition.
+                                self.nesting_level += 1;
+
+                                evt = Event::Html(
+                                    if title.is_empty() {
+                                        format!("<div class=\"box {kind}\">")
+                                    } else {
+                                        format!("<div class=\"box {kind}\"><p class=\"box-title\">{title}</p>")
+                                    }
+                                    .into(),
+                                );
+                            }
+                        }
+                    }
+                }
+            }
+            _ => {}
+        }
+
+        self.at_paragraph_start = matches!(evt, Event::Start(Tag::Paragraph));
+
+        Some(evt)
+    }
+}

preproc/src/anchors.rs

@@ -0,0 +1,180 @@
+/*
+ * This Source Code Form is subject to the
+ * terms of the Mozilla Public License, v.
+ * 2.0. If a copy of the MPL was not
+ * distributed with this file, You can
+ * obtain one at
+ * http://mozilla.org/MPL/2.0/.
+ */
+
+use std::collections::HashMap;
+use std::io::Write;
+
+use mdbook::book::Chapter;
+use mdbook::errors::Error;
+use pulldown_cmark::{CowStr, Event, LinkType, Options, Parser, Tag};
+use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
+
+use crate::Pandocs;
+
+impl Pandocs {
+    pub fn list_chapter_sections(
+        &self,
+        sections: &mut HashMap<String, (String, bool)>,
+        chapter: &Chapter,
+    ) {
+        let mut events = Parser::new(&chapter.content);
+        while let Some(event) = events.next() {
+            if let Event::Start(Tag::Heading(_)) = event {
+                let mut depth = 1;
+                let mut name = String::new();
+
+                while depth != 0 {
+                    match events.next().expect("Unclosed `Start` tag??") {
+                        Event::Start(_) => depth += 1,
+                        Event::End(_) => depth -= 1,
+                        Event::Text(text) | Event::Code(text) => name.push_str(&text),
+                        event => panic!("Unexpected event in header {:?}", event),
+                    }
+                }
+
+                let chapter_name = name.clone();
+                let mut page_name = chapter
+                    .path
+                    .as_ref()
+                    .expect("Chapter without a source file??")
+                    .clone()
+                    .into_os_string()
+                    .into_string()
+                    .expect("Chapter file path is not valid UTF-8");
+                page_name.truncate(
+                    page_name
+                        .strip_suffix(".md")
+                        .expect("Source file not ending in `.md`??")
+                        .len(),
+                );
+
+                if sections.insert(name, (page_name, false)).is_some() {
+                    // Mark the ambiguity
+                    sections.get_mut(&chapter_name).unwrap().1 = true;
+                }
+            }
+        }
+    }
+
+    pub fn process_internal_anchor_links(
+        &self,
+        chapter: &mut Chapter,
+        sections: &HashMap<String, (String, bool)>,
+    ) -> Result<(), Error> {
+        let mut buf = String::with_capacity(chapter.content.len());
+        let extensions =
+            Options::ENABLE_TABLES | Options::ENABLE_FOOTNOTES | Options::ENABLE_STRIKETHROUGH;
+
+        let events = Parser::new_ext(&chapter.content, extensions).map(|event| match event {
+            Event::Start(Tag::Link(link_type, url, title)) if url.starts_with('#') => {
+                let (link, ok) = translate_anchor_link(sections, link_type, url, title);
+                if !ok {
+                    let mut stderr = StandardStream::stderr(ColorChoice::Auto);
+                    stderr
+                        .set_color(ColorSpec::new().set_fg(Some(Color::Yellow)).set_bold(true))
+                        .unwrap();
+                    write!(&mut stderr, "warning:").unwrap();
+                    stderr.reset().unwrap();
+
+                    if let Tag::Link(_, ref url, _) = link {
+                        eprintln!(
+                            " {}: Internal anchor link \"{}\" not found, keeping as-is",
+                            &chapter.name, url
+                        );
+                    } else {
+                        unreachable!()
+                    }
+                }
+                Event::Start(link)
+            }
+
+            Event::End(Tag::Link(link_type, url, title)) if url.starts_with('#') => {
+                Event::End(translate_anchor_link(sections, link_type, url, title).0)
+            }
+
+            _ => event,
+        });
+
+        pulldown_cmark_to_cmark::cmark(events, &mut buf, None)
+            .map_err(|err| Error::from(err).context("Markdown serialization failed"))?;
+        chapter.content = buf;
+
+        Ok(())
+    }
+}
+
+fn translate_anchor_link<'a>(
+    sections: &HashMap<String, (String, bool)>,
+    link_type: LinkType,
+    url: CowStr<'a>,
+    title: CowStr<'a>,
+) -> (Tag<'a>, bool) {
+    let (url, ok) = if let Some((chapter, multiple)) = sections.get(&url[1..]) {
+        if *multiple {
+            let mut stderr = StandardStream::stderr(ColorChoice::Auto);
+            stderr
+                .set_color(ColorSpec::new().set_fg(Some(Color::Yellow)).set_bold(true))
+                .unwrap();
+            write!(&mut stderr, "warning:").unwrap();
+            stderr.reset().unwrap();
+            eprintln!(
+                " Referencing multiply-defined section \"{}\" (using chapter \"{}\")",
+                &url[1..],
+                &chapter
+            );
+        }
+        (
+            CowStr::Boxed(format!("{}.html#{}", chapter, id_from_name(&url[1..])).into_boxed_str()),
+            true,
+        )
+    } else {
+        (url, false)
+    };
+
+    (Tag::Link(link_type, url, title), ok)
+}
+
+fn id_from_name(name: &str) -> String {
+    let mut content = name.to_string();
+
+    // Skip any tags or html-encoded stuff
+    const REPL_SUB: &[&str] = &[
+        "<em>",
+        "</em>",
+        "<code>",
+        "</code>",
+        "<strong>",
+        "</strong>",
+        "&lt;",
+        "&gt;",
+        "&amp;",
+        "&#39;",
+        "&quot;",
+    ];
+    for sub in REPL_SUB {
+        content = content.replace(sub, "");
+    }
+
+    // Remove spaces and hashes indicating a header
+    let trimmed = content.trim().trim_start_matches('#').trim();
+
+    // Normalize
+    trimmed
+        .chars()
+        .filter_map(|ch| {
+            if ch.is_alphanumeric() || ch == '_' || ch == '-' {
+                Some(ch.to_ascii_lowercase())
+            } else if ch.is_whitespace() {
+                Some('-')
+            } else {
+                None
+            }
+        })
+        .collect::<String>()
+}