dc630ba030
- anchors: 锚点处理增强 - document: 文档核心逻辑 - epub/markdown/pdf: 多格式解析增强 - search: 全文搜索增强 - FFI: 接口绑定更新 + UDL 定义 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
540 lines
20 KiB
Rust
540 lines
20 KiB
Rust
use comrak::nodes::{AstNode, NodeValue};
|
|
use comrak::{Arena, ComrakOptions};
|
|
use uuid::Uuid;
|
|
|
|
use crate::blocks::DocumentBlock;
|
|
use crate::error::DocumentError;
|
|
|
|
fn block_id() -> String {
|
|
Uuid::new_v4().to_string()
|
|
}
|
|
|
|
fn gfm_options() -> ComrakOptions<'static> {
|
|
let mut opts = ComrakOptions::default();
|
|
opts.extension.table = true;
|
|
opts.extension.strikethrough = true;
|
|
opts.extension.tagfilter = true;
|
|
opts.extension.tasklist = true;
|
|
opts
|
|
}
|
|
|
|
/// Parse a Markdown string into a list of DocumentBlock.
|
|
pub fn parse_markdown(md_content: &str) -> Result<Vec<DocumentBlock>, DocumentError> {
|
|
let arena = Arena::new();
|
|
let options = gfm_options();
|
|
let root = comrak::parse_document(&arena, md_content, &options);
|
|
|
|
let mut blocks = Vec::new();
|
|
collect_blocks(root, &mut blocks);
|
|
Ok(blocks)
|
|
}
|
|
|
|
fn collect_blocks<'a>(node: &'a AstNode<'a>, blocks: &mut Vec<DocumentBlock>) {
|
|
for child in node.children() {
|
|
let data = child.data.borrow();
|
|
let value = &data.value;
|
|
|
|
let block = match value {
|
|
NodeValue::Heading(heading) => Some(DocumentBlock::Heading {
|
|
id: block_id(),
|
|
level: heading.level,
|
|
text: collect_text(child),
|
|
}),
|
|
NodeValue::Paragraph => {
|
|
// Check for inline image — extract as standalone Image block
|
|
if let Some(image_block) = extract_image_from_paragraph(child) {
|
|
Some(image_block)
|
|
} else {
|
|
let text = collect_text(child);
|
|
if text.is_empty() {
|
|
None
|
|
} else {
|
|
Some(DocumentBlock::Paragraph {
|
|
id: block_id(),
|
|
text,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
NodeValue::List(list) => {
|
|
let items: Vec<String> = child
|
|
.children()
|
|
.filter_map(|item| {
|
|
if matches!(item.data.borrow().value, NodeValue::Item(_)) {
|
|
Some(collect_text(item))
|
|
} else {
|
|
None
|
|
}
|
|
})
|
|
.collect();
|
|
if items.is_empty() {
|
|
None
|
|
} else {
|
|
Some(DocumentBlock::List {
|
|
id: block_id(),
|
|
ordered: list.list_type == comrak::nodes::ListType::Ordered,
|
|
items,
|
|
})
|
|
}
|
|
}
|
|
NodeValue::CodeBlock(code) => Some(DocumentBlock::CodeBlock {
|
|
id: block_id(),
|
|
language: if code.info.is_empty() {
|
|
None
|
|
} else {
|
|
Some(code.info.clone())
|
|
},
|
|
code: code.literal.clone(),
|
|
}),
|
|
NodeValue::BlockQuote => {
|
|
let text = collect_text(child);
|
|
if text.is_empty() {
|
|
None
|
|
} else {
|
|
Some(DocumentBlock::Quote {
|
|
id: block_id(),
|
|
text,
|
|
})
|
|
}
|
|
}
|
|
NodeValue::Table(_) => {
|
|
let mut headers = Vec::new();
|
|
let mut rows = Vec::new();
|
|
|
|
for row_node in child.children() {
|
|
let cells: Vec<String> = row_node
|
|
.children()
|
|
.map(|cell| collect_text(cell))
|
|
.collect();
|
|
if cells.is_empty() {
|
|
continue;
|
|
}
|
|
// Skip separator rows like |---|---|
|
|
if cells.iter().all(|c| c.chars().all(|ch| ch == '-' || ch == ':' || ch == ' ')) {
|
|
continue;
|
|
}
|
|
if headers.is_empty() {
|
|
headers = cells;
|
|
} else {
|
|
rows.push(cells);
|
|
}
|
|
}
|
|
|
|
if headers.is_empty() {
|
|
None
|
|
} else {
|
|
Some(DocumentBlock::Table {
|
|
id: block_id(),
|
|
headers,
|
|
rows,
|
|
})
|
|
}
|
|
}
|
|
NodeValue::Image(image) => {
|
|
let alt_text = collect_text(child);
|
|
Some(DocumentBlock::Image {
|
|
id: block_id(),
|
|
src: image.url.clone(),
|
|
alt: if alt_text.is_empty() {
|
|
None
|
|
} else {
|
|
Some(alt_text)
|
|
},
|
|
})
|
|
}
|
|
NodeValue::ThematicBreak => Some(DocumentBlock::HorizontalRule {
|
|
id: block_id(),
|
|
}),
|
|
// Recurse into containers: Document, Item, TableCell, TableRow, etc.
|
|
NodeValue::HtmlBlock(block) => {
|
|
let html = block.literal.clone();
|
|
if html.trim().is_empty() {
|
|
None
|
|
} else {
|
|
Some(DocumentBlock::CodeBlock {
|
|
id: block_id(),
|
|
language: Some("html".into()),
|
|
code: html,
|
|
})
|
|
}
|
|
}
|
|
NodeValue::FrontMatter(content) => {
|
|
if content.trim().is_empty() {
|
|
None
|
|
} else {
|
|
Some(DocumentBlock::CodeBlock {
|
|
id: block_id(),
|
|
language: Some("yaml".into()),
|
|
code: content.clone(),
|
|
})
|
|
}
|
|
}
|
|
NodeValue::FootnoteDefinition(_) => {
|
|
let text = collect_text(child);
|
|
if text.is_empty() {
|
|
None
|
|
} else {
|
|
Some(DocumentBlock::Paragraph { id: block_id(), text })
|
|
}
|
|
}
|
|
NodeValue::Document
|
|
| NodeValue::Item(_)
|
|
| NodeValue::TableCell
|
|
| NodeValue::TableRow(_)
|
|
| NodeValue::DescriptionList
|
|
| NodeValue::DescriptionItem(_)
|
|
| NodeValue::DescriptionTerm
|
|
| NodeValue::DescriptionDetails => {
|
|
collect_blocks(child, blocks);
|
|
None
|
|
}
|
|
_ => None,
|
|
};
|
|
|
|
if let Some(b) = block {
|
|
blocks.push(b);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Check if a paragraph contains only an image, and extract it.
|
|
fn extract_image_from_paragraph<'a>(node: &'a AstNode<'a>) -> Option<DocumentBlock> {
|
|
let mut image_node: Option<&AstNode> = None;
|
|
let mut has_other_content = false;
|
|
|
|
for child in node.children() {
|
|
let data = child.data.borrow();
|
|
match &data.value {
|
|
NodeValue::Image(_) => {
|
|
image_node = Some(child);
|
|
}
|
|
NodeValue::Text(t) if t.trim().is_empty() => {}
|
|
NodeValue::SoftBreak | NodeValue::LineBreak => {}
|
|
_ => {
|
|
has_other_content = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
if let Some(img) = image_node {
|
|
if !has_other_content {
|
|
let data = img.data.borrow();
|
|
if let NodeValue::Image(image) = &data.value {
|
|
let alt_text = collect_text(img);
|
|
return Some(DocumentBlock::Image {
|
|
id: block_id(),
|
|
src: image.url.clone(),
|
|
alt: if alt_text.is_empty() {
|
|
None
|
|
} else {
|
|
Some(alt_text)
|
|
},
|
|
});
|
|
}
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
/// Extract plain text from all text nodes within a subtree.
|
|
fn collect_text<'a>(node: &'a AstNode<'a>) -> String {
|
|
let mut text = String::new();
|
|
collect_text_inner(node, &mut text);
|
|
text.trim().to_string()
|
|
}
|
|
|
|
fn collect_text_inner<'a>(node: &'a AstNode<'a>, buf: &mut String) {
|
|
for child in node.children() {
|
|
let data = child.data.borrow();
|
|
match &data.value {
|
|
NodeValue::Text(t) => buf.push_str(t),
|
|
NodeValue::SoftBreak => buf.push(' '),
|
|
NodeValue::LineBreak => buf.push('\n'),
|
|
NodeValue::Code(code) => buf.push_str(&code.literal),
|
|
_ => collect_text_inner(child, buf),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_empty_markdown() {
|
|
let blocks = parse_markdown("").unwrap();
|
|
assert!(blocks.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn test_headings() {
|
|
let md = "# Title\n\n## Section\n\n### Sub";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
let headings: Vec<_> = blocks
|
|
.iter()
|
|
.filter(|b| matches!(b, DocumentBlock::Heading { .. }))
|
|
.collect();
|
|
assert_eq!(headings.len(), 3);
|
|
}
|
|
|
|
#[test]
|
|
fn test_paragraph() {
|
|
let md = "Hello world.\n\nThis is a paragraph.";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
let paras: Vec<_> = blocks
|
|
.iter()
|
|
.filter(|b| matches!(b, DocumentBlock::Paragraph { .. }))
|
|
.collect();
|
|
assert_eq!(paras.len(), 2);
|
|
}
|
|
|
|
#[test]
|
|
fn test_unordered_list() {
|
|
let md = "- one\n- two\n- three";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
if let Some(DocumentBlock::List { items, ordered, .. }) = blocks.first() {
|
|
assert!(!ordered);
|
|
assert_eq!(items.len(), 3);
|
|
assert_eq!(items[0], "one");
|
|
} else {
|
|
panic!("expected a list block");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_ordered_list() {
|
|
let md = "1. first\n2. second\n3. third";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
if let Some(DocumentBlock::List { items, ordered, .. }) = blocks.first() {
|
|
assert!(*ordered);
|
|
assert_eq!(items.len(), 3);
|
|
} else {
|
|
panic!("expected an ordered list block");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_code_block() {
|
|
let md = "```rust\nfn main() {}\n```";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
if let Some(DocumentBlock::CodeBlock { language, code, .. }) = blocks.first() {
|
|
assert_eq!(language.as_deref(), Some("rust"));
|
|
assert!(code.contains("fn main()"));
|
|
} else {
|
|
panic!("expected a code block");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_blockquote() {
|
|
let md = "> This is a quote";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
if let Some(DocumentBlock::Quote { text, .. }) = blocks.first() {
|
|
assert!(text.contains("This is a quote"));
|
|
} else {
|
|
panic!("expected a quote");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_table() {
|
|
// GFM table with separator row
|
|
let md = "| A | B |\n| --- | --- |\n| 1 | 2 |\n| 3 | 4 |";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
let tables: Vec<_> = blocks
|
|
.iter()
|
|
.filter(|b| matches!(b, DocumentBlock::Table { .. }))
|
|
.collect();
|
|
assert!(!tables.is_empty(), "should have at least one table, got {:?}", blocks);
|
|
}
|
|
|
|
#[test]
|
|
fn test_horizontal_rule() {
|
|
let md = "---";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
assert!(matches!(
|
|
blocks.first(),
|
|
Some(DocumentBlock::HorizontalRule { .. })
|
|
));
|
|
}
|
|
|
|
#[test]
|
|
fn test_image() {
|
|
let md = "";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
if let Some(DocumentBlock::Image { src, alt, .. }) = blocks.first() {
|
|
assert_eq!(src, "https://example.com/img.png");
|
|
assert_eq!(alt.as_deref(), Some("alt text"));
|
|
} else {
|
|
panic!("expected an image block");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_complex_document() {
|
|
let md = "# Heading\n\nParagraph text.\n\n- item 1\n- item 2\n\n```rs\nlet x = 1;\n```";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
assert!(blocks.len() >= 4);
|
|
assert!(blocks.iter().any(|b| matches!(b, DocumentBlock::Heading { .. })));
|
|
assert!(blocks.iter().any(|b| matches!(b, DocumentBlock::Paragraph { .. })));
|
|
assert!(blocks.iter().any(|b| matches!(b, DocumentBlock::List { .. })));
|
|
assert!(blocks.iter().any(|b| matches!(b, DocumentBlock::CodeBlock { .. })));
|
|
}
|
|
|
|
#[test]
|
|
fn test_fixture_complex_markdown() {
|
|
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
|
.parent().unwrap().parent().unwrap();
|
|
let path = root.join("fixtures/markdown/complex_markdown.md");
|
|
let md = std::fs::read_to_string(&path).expect("fixture not found");
|
|
let blocks = parse_markdown(&md).unwrap();
|
|
assert!(!blocks.is_empty(), "should parse at least some blocks");
|
|
|
|
// Verify all 8 block types are present
|
|
let has_heading = blocks.iter().any(|b| matches!(b, DocumentBlock::Heading { .. }));
|
|
let has_para = blocks.iter().any(|b| matches!(b, DocumentBlock::Paragraph { .. }));
|
|
let has_list = blocks.iter().any(|b| matches!(b, DocumentBlock::List { .. }));
|
|
let has_code = blocks.iter().any(|b| matches!(b, DocumentBlock::CodeBlock { .. }));
|
|
let has_quote = blocks.iter().any(|b| matches!(b, DocumentBlock::Quote { .. }));
|
|
let has_table = blocks.iter().any(|b| matches!(b, DocumentBlock::Table { .. }));
|
|
let has_image = blocks.iter().any(|b| matches!(b, DocumentBlock::Image { .. }));
|
|
let has_rule = blocks.iter().any(|b| matches!(b, DocumentBlock::HorizontalRule { .. }));
|
|
|
|
assert!(has_heading, "missing heading");
|
|
assert!(has_para, "missing paragraph");
|
|
assert!(has_list, "missing list");
|
|
assert!(has_code, "missing code block");
|
|
assert!(has_quote, "missing quote");
|
|
assert!(has_table, "missing table");
|
|
assert!(has_image, "missing image");
|
|
assert!(has_rule, "missing horizontal rule");
|
|
|
|
// Verify heading level
|
|
let headings: Vec<_> = blocks.iter().filter_map(|b| {
|
|
if let DocumentBlock::Heading { level, text, .. } = b {
|
|
Some((*level, text.clone()))
|
|
} else { None }
|
|
}).collect();
|
|
assert!(headings.iter().any(|(l, _)| *l == 1), "should have h1");
|
|
assert!(headings.iter().any(|(l, _)| *l == 2), "should have h2");
|
|
assert!(headings.iter().any(|(l, _)| *l == 3), "should have h3");
|
|
|
|
// Verify code block has language
|
|
let code_blocks: Vec<_> = blocks.iter().filter_map(|b| {
|
|
if let DocumentBlock::CodeBlock { language, code, .. } = b {
|
|
Some((language.clone(), code.clone()))
|
|
} else { None }
|
|
}).collect();
|
|
assert!(code_blocks.iter().any(|(lang, _)| lang.as_deref() == Some("rust")), "should have rust code block");
|
|
|
|
// Verify table structure
|
|
for b in &blocks {
|
|
if let DocumentBlock::Table { headers, rows, .. } = b {
|
|
assert!(!headers.is_empty(), "table should have headers");
|
|
assert!(!rows.is_empty(), "table should have rows");
|
|
assert_eq!(headers.len(), rows[0].len(), "table row width should match headers");
|
|
}
|
|
}
|
|
|
|
// Verify image has src and alt
|
|
for b in &blocks {
|
|
if let DocumentBlock::Image { src, alt, .. } = b {
|
|
assert!(!src.is_empty(), "image should have src");
|
|
assert!(alt.is_some(), "image should have alt text");
|
|
}
|
|
}
|
|
|
|
// Verify block_ids are non-empty and unique
|
|
let ids: Vec<&str> = blocks.iter().map(|b| match b {
|
|
DocumentBlock::Heading { id, .. } => id.as_str(),
|
|
DocumentBlock::Paragraph { id, .. } => id.as_str(),
|
|
DocumentBlock::List { id, .. } => id.as_str(),
|
|
DocumentBlock::CodeBlock { id, .. } => id.as_str(),
|
|
DocumentBlock::Quote { id, .. } => id.as_str(),
|
|
DocumentBlock::Table { id, .. } => id.as_str(),
|
|
DocumentBlock::Image { id, .. } => id.as_str(),
|
|
DocumentBlock::HorizontalRule { id, .. } => id.as_str(),
|
|
}).collect();
|
|
for id in &ids {
|
|
assert!(!id.is_empty(), "block_id should not be empty");
|
|
}
|
|
let unique: std::collections::HashSet<_> = ids.iter().collect();
|
|
assert_eq!(unique.len(), ids.len(), "all block_ids should be unique");
|
|
}
|
|
|
|
#[test]
|
|
fn test_nested_lists() {
|
|
let md = "- Item 1\n - Sub 1.1\n - Sub 1.2\n- Item 2\n";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
// Nested list items are flattened — at minimum we should have > 0 blocks
|
|
assert!(!blocks.is_empty(), "nested list should parse");
|
|
}
|
|
|
|
#[test]
|
|
fn test_unicode_and_emoji() {
|
|
let md = "# 中文标题 🎉\n\n包含 emoji 和 Unicode 的段落。\n\n## 日本語見出し\n\n日本語の段落。";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
assert!(blocks.iter().any(|b| matches!(b, DocumentBlock::Heading { .. })));
|
|
}
|
|
|
|
#[test]
|
|
fn test_front_matter_handled() {
|
|
let md = "---\ntitle: Test\n---\n\n# Content\n\nBody text.";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
assert!(blocks.iter().any(|b| matches!(b, DocumentBlock::Heading { .. })));
|
|
}
|
|
|
|
#[test]
|
|
fn test_empty_blocks_filtered() {
|
|
let md = "# Heading\n\n\n\nParagraph\n\n\n";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
let paras = blocks.iter().filter(|b| matches!(b, DocumentBlock::Paragraph { .. })).count();
|
|
assert_eq!(paras, 1, "multiple blank lines should not create empty paragraphs");
|
|
}
|
|
|
|
#[test]
|
|
fn test_very_long_document() {
|
|
let mut md = String::from("# Long\n\n");
|
|
for i in 0..500 {
|
|
md.push_str(&format!("Para {i}. Content here.\n\n"));
|
|
}
|
|
let blocks = parse_markdown(&md).unwrap();
|
|
assert!(blocks.len() > 500, "should handle 500 paragraphs");
|
|
}
|
|
|
|
#[test]
|
|
fn test_mixed_inline_formatting() {
|
|
let md = "# Title\n\nText with `code` and **bold** and *italic* and ~~strike~~.\n\n- `coded` item\n- **bold** item";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
assert!(!blocks.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn test_task_list() {
|
|
let md = "- [x] Done task\n- [ ] Pending task\n- Normal item";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
assert!(blocks.iter().any(|b| matches!(b, DocumentBlock::List { .. })));
|
|
}
|
|
|
|
#[test]
|
|
fn test_all_eight_block_types() {
|
|
let md = "# H\n\nPara.\n\n- item\n\n```rs\nlet x=1;\n```\n\n> Quote\n\n|A|B|\n|---|---|\n|1|2|\n\n\n\n---";
|
|
let blocks = parse_markdown(md).unwrap();
|
|
let has = |t: &str| -> bool {
|
|
blocks.iter().any(|b| match (t, b) {
|
|
("heading", DocumentBlock::Heading { .. }) => true,
|
|
("para", DocumentBlock::Paragraph { .. }) => true,
|
|
("list", DocumentBlock::List { .. }) => true,
|
|
("code", DocumentBlock::CodeBlock { .. }) => true,
|
|
("quote", DocumentBlock::Quote { .. }) => true,
|
|
("table", DocumentBlock::Table { .. }) => true,
|
|
("image", DocumentBlock::Image { .. }) => true,
|
|
("rule", DocumentBlock::HorizontalRule { .. }) => true,
|
|
_ => false,
|
|
})
|
|
};
|
|
assert!(has("heading")); assert!(has("para")); assert!(has("list")); assert!(has("code"));
|
|
assert!(has("quote")); assert!(has("table")); assert!(has("image")); assert!(has("rule"));
|
|
}
|
|
}
|