use comrak::nodes::{AstNode, NodeValue}; use comrak::{Arena, ComrakOptions}; use uuid::Uuid; use crate::blocks::DocumentBlock; use crate::error::DocumentError; fn block_id() -> String { Uuid::new_v4().to_string() } fn gfm_options() -> ComrakOptions<'static> { let mut opts = ComrakOptions::default(); opts.extension.table = true; opts.extension.strikethrough = true; opts.extension.tagfilter = true; opts.extension.tasklist = true; opts } /// Parse a Markdown string into a list of DocumentBlock. pub fn parse_markdown(md_content: &str) -> Result, DocumentError> { let arena = Arena::new(); let options = gfm_options(); let root = comrak::parse_document(&arena, md_content, &options); let mut blocks = Vec::new(); collect_blocks(root, &mut blocks); Ok(blocks) } fn collect_blocks<'a>(node: &'a AstNode<'a>, blocks: &mut Vec) { for child in node.children() { let data = child.data.borrow(); let value = &data.value; let block = match value { NodeValue::Heading(heading) => Some(DocumentBlock::Heading { id: block_id(), level: heading.level, text: collect_text(child), }), NodeValue::Paragraph => { // Check for inline image — extract as standalone Image block if let Some(image_block) = extract_image_from_paragraph(child) { Some(image_block) } else { let text = collect_text(child); if text.is_empty() { None } else { Some(DocumentBlock::Paragraph { id: block_id(), text, }) } } } NodeValue::List(list) => { let items: Vec = child .children() .filter_map(|item| { if matches!(item.data.borrow().value, NodeValue::Item(_)) { Some(collect_text(item)) } else { None } }) .collect(); if items.is_empty() { None } else { Some(DocumentBlock::List { id: block_id(), ordered: list.list_type == comrak::nodes::ListType::Ordered, items, }) } } NodeValue::CodeBlock(code) => Some(DocumentBlock::CodeBlock { id: block_id(), language: if code.info.is_empty() { None } else { Some(code.info.clone()) }, code: code.literal.clone(), }), NodeValue::BlockQuote => { let text = collect_text(child); if text.is_empty() { None } else { Some(DocumentBlock::Quote { id: block_id(), text, }) } } NodeValue::Table(_) => { let mut headers = Vec::new(); let mut rows = Vec::new(); for row_node in child.children() { let cells: Vec = row_node .children() .map(|cell| collect_text(cell)) .collect(); if cells.is_empty() { continue; } // Skip separator rows like |---|---| if cells.iter().all(|c| c.chars().all(|ch| ch == '-' || ch == ':' || ch == ' ')) { continue; } if headers.is_empty() { headers = cells; } else { rows.push(cells); } } if headers.is_empty() { None } else { Some(DocumentBlock::Table { id: block_id(), headers, rows, }) } } NodeValue::Image(image) => { let alt_text = collect_text(child); Some(DocumentBlock::Image { id: block_id(), src: image.url.clone(), alt: if alt_text.is_empty() { None } else { Some(alt_text) }, }) } NodeValue::ThematicBreak => Some(DocumentBlock::HorizontalRule { id: block_id(), }), // Recurse into containers: Document, Item, TableCell, TableRow, etc. NodeValue::HtmlBlock(block) => { let html = block.literal.clone(); if html.trim().is_empty() { None } else { Some(DocumentBlock::CodeBlock { id: block_id(), language: Some("html".into()), code: html, }) } } NodeValue::FrontMatter(content) => { if content.trim().is_empty() { None } else { Some(DocumentBlock::CodeBlock { id: block_id(), language: Some("yaml".into()), code: content.clone(), }) } } NodeValue::FootnoteDefinition(_) => { let text = collect_text(child); if text.is_empty() { None } else { Some(DocumentBlock::Paragraph { id: block_id(), text }) } } NodeValue::Document | NodeValue::Item(_) | NodeValue::TableCell | NodeValue::TableRow(_) | NodeValue::DescriptionList | NodeValue::DescriptionItem(_) | NodeValue::DescriptionTerm | NodeValue::DescriptionDetails => { collect_blocks(child, blocks); None } _ => None, }; if let Some(b) = block { blocks.push(b); } } } /// Check if a paragraph contains only an image, and extract it. fn extract_image_from_paragraph<'a>(node: &'a AstNode<'a>) -> Option { let mut image_node: Option<&AstNode> = None; let mut has_other_content = false; for child in node.children() { let data = child.data.borrow(); match &data.value { NodeValue::Image(_) => { image_node = Some(child); } NodeValue::Text(t) if t.trim().is_empty() => {} NodeValue::SoftBreak | NodeValue::LineBreak => {} _ => { has_other_content = true; } } } if let Some(img) = image_node { if !has_other_content { let data = img.data.borrow(); if let NodeValue::Image(image) = &data.value { let alt_text = collect_text(img); return Some(DocumentBlock::Image { id: block_id(), src: image.url.clone(), alt: if alt_text.is_empty() { None } else { Some(alt_text) }, }); } } } None } /// Extract plain text from all text nodes within a subtree. fn collect_text<'a>(node: &'a AstNode<'a>) -> String { let mut text = String::new(); collect_text_inner(node, &mut text); text.trim().to_string() } fn collect_text_inner<'a>(node: &'a AstNode<'a>, buf: &mut String) { for child in node.children() { let data = child.data.borrow(); match &data.value { NodeValue::Text(t) => buf.push_str(t), NodeValue::SoftBreak => buf.push(' '), NodeValue::LineBreak => buf.push('\n'), NodeValue::Code(code) => buf.push_str(&code.literal), _ => collect_text_inner(child, buf), } } } #[cfg(test)] mod tests { use super::*; #[test] fn test_empty_markdown() { let blocks = parse_markdown("").unwrap(); assert!(blocks.is_empty()); } #[test] fn test_headings() { let md = "# Title\n\n## Section\n\n### Sub"; let blocks = parse_markdown(md).unwrap(); let headings: Vec<_> = blocks .iter() .filter(|b| matches!(b, DocumentBlock::Heading { .. })) .collect(); assert_eq!(headings.len(), 3); } #[test] fn test_paragraph() { let md = "Hello world.\n\nThis is a paragraph."; let blocks = parse_markdown(md).unwrap(); let paras: Vec<_> = blocks .iter() .filter(|b| matches!(b, DocumentBlock::Paragraph { .. })) .collect(); assert_eq!(paras.len(), 2); } #[test] fn test_unordered_list() { let md = "- one\n- two\n- three"; let blocks = parse_markdown(md).unwrap(); if let Some(DocumentBlock::List { items, ordered, .. }) = blocks.first() { assert!(!ordered); assert_eq!(items.len(), 3); assert_eq!(items[0], "one"); } else { panic!("expected a list block"); } } #[test] fn test_ordered_list() { let md = "1. first\n2. second\n3. third"; let blocks = parse_markdown(md).unwrap(); if let Some(DocumentBlock::List { items, ordered, .. }) = blocks.first() { assert!(*ordered); assert_eq!(items.len(), 3); } else { panic!("expected an ordered list block"); } } #[test] fn test_code_block() { let md = "```rust\nfn main() {}\n```"; let blocks = parse_markdown(md).unwrap(); if let Some(DocumentBlock::CodeBlock { language, code, .. }) = blocks.first() { assert_eq!(language.as_deref(), Some("rust")); assert!(code.contains("fn main()")); } else { panic!("expected a code block"); } } #[test] fn test_blockquote() { let md = "> This is a quote"; let blocks = parse_markdown(md).unwrap(); if let Some(DocumentBlock::Quote { text, .. }) = blocks.first() { assert!(text.contains("This is a quote")); } else { panic!("expected a quote"); } } #[test] fn test_table() { // GFM table with separator row let md = "| A | B |\n| --- | --- |\n| 1 | 2 |\n| 3 | 4 |"; let blocks = parse_markdown(md).unwrap(); let tables: Vec<_> = blocks .iter() .filter(|b| matches!(b, DocumentBlock::Table { .. })) .collect(); assert!(!tables.is_empty(), "should have at least one table, got {:?}", blocks); } #[test] fn test_horizontal_rule() { let md = "---"; let blocks = parse_markdown(md).unwrap(); assert!(matches!( blocks.first(), Some(DocumentBlock::HorizontalRule { .. }) )); } #[test] fn test_image() { let md = "![alt text](https://example.com/img.png)"; let blocks = parse_markdown(md).unwrap(); if let Some(DocumentBlock::Image { src, alt, .. }) = blocks.first() { assert_eq!(src, "https://example.com/img.png"); assert_eq!(alt.as_deref(), Some("alt text")); } else { panic!("expected an image block"); } } #[test] fn test_complex_document() { let md = "# Heading\n\nParagraph text.\n\n- item 1\n- item 2\n\n```rs\nlet x = 1;\n```"; let blocks = parse_markdown(md).unwrap(); assert!(blocks.len() >= 4); assert!(blocks.iter().any(|b| matches!(b, DocumentBlock::Heading { .. }))); assert!(blocks.iter().any(|b| matches!(b, DocumentBlock::Paragraph { .. }))); assert!(blocks.iter().any(|b| matches!(b, DocumentBlock::List { .. }))); assert!(blocks.iter().any(|b| matches!(b, DocumentBlock::CodeBlock { .. }))); } #[test] fn test_fixture_complex_markdown() { let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .parent().unwrap().parent().unwrap(); let path = root.join("fixtures/markdown/complex_markdown.md"); let md = std::fs::read_to_string(&path).expect("fixture not found"); let blocks = parse_markdown(&md).unwrap(); assert!(!blocks.is_empty(), "should parse at least some blocks"); // Verify all 8 block types are present let has_heading = blocks.iter().any(|b| matches!(b, DocumentBlock::Heading { .. })); let has_para = blocks.iter().any(|b| matches!(b, DocumentBlock::Paragraph { .. })); let has_list = blocks.iter().any(|b| matches!(b, DocumentBlock::List { .. })); let has_code = blocks.iter().any(|b| matches!(b, DocumentBlock::CodeBlock { .. })); let has_quote = blocks.iter().any(|b| matches!(b, DocumentBlock::Quote { .. })); let has_table = blocks.iter().any(|b| matches!(b, DocumentBlock::Table { .. })); let has_image = blocks.iter().any(|b| matches!(b, DocumentBlock::Image { .. })); let has_rule = blocks.iter().any(|b| matches!(b, DocumentBlock::HorizontalRule { .. })); assert!(has_heading, "missing heading"); assert!(has_para, "missing paragraph"); assert!(has_list, "missing list"); assert!(has_code, "missing code block"); assert!(has_quote, "missing quote"); assert!(has_table, "missing table"); assert!(has_image, "missing image"); assert!(has_rule, "missing horizontal rule"); // Verify heading level let headings: Vec<_> = blocks.iter().filter_map(|b| { if let DocumentBlock::Heading { level, text, .. } = b { Some((*level, text.clone())) } else { None } }).collect(); assert!(headings.iter().any(|(l, _)| *l == 1), "should have h1"); assert!(headings.iter().any(|(l, _)| *l == 2), "should have h2"); assert!(headings.iter().any(|(l, _)| *l == 3), "should have h3"); // Verify code block has language let code_blocks: Vec<_> = blocks.iter().filter_map(|b| { if let DocumentBlock::CodeBlock { language, code, .. } = b { Some((language.clone(), code.clone())) } else { None } }).collect(); assert!(code_blocks.iter().any(|(lang, _)| lang.as_deref() == Some("rust")), "should have rust code block"); // Verify table structure for b in &blocks { if let DocumentBlock::Table { headers, rows, .. } = b { assert!(!headers.is_empty(), "table should have headers"); assert!(!rows.is_empty(), "table should have rows"); assert_eq!(headers.len(), rows[0].len(), "table row width should match headers"); } } // Verify image has src and alt for b in &blocks { if let DocumentBlock::Image { src, alt, .. } = b { assert!(!src.is_empty(), "image should have src"); assert!(alt.is_some(), "image should have alt text"); } } // Verify block_ids are non-empty and unique let ids: Vec<&str> = blocks.iter().map(|b| match b { DocumentBlock::Heading { id, .. } => id.as_str(), DocumentBlock::Paragraph { id, .. } => id.as_str(), DocumentBlock::List { id, .. } => id.as_str(), DocumentBlock::CodeBlock { id, .. } => id.as_str(), DocumentBlock::Quote { id, .. } => id.as_str(), DocumentBlock::Table { id, .. } => id.as_str(), DocumentBlock::Image { id, .. } => id.as_str(), DocumentBlock::HorizontalRule { id, .. } => id.as_str(), }).collect(); for id in &ids { assert!(!id.is_empty(), "block_id should not be empty"); } let unique: std::collections::HashSet<_> = ids.iter().collect(); assert_eq!(unique.len(), ids.len(), "all block_ids should be unique"); } #[test] fn test_nested_lists() { let md = "- Item 1\n - Sub 1.1\n - Sub 1.2\n- Item 2\n"; let blocks = parse_markdown(md).unwrap(); // Nested list items are flattened — at minimum we should have > 0 blocks assert!(!blocks.is_empty(), "nested list should parse"); } #[test] fn test_unicode_and_emoji() { let md = "# 中文标题 🎉\n\n包含 emoji 和 Unicode 的段落。\n\n## 日本語見出し\n\n日本語の段落。"; let blocks = parse_markdown(md).unwrap(); assert!(blocks.iter().any(|b| matches!(b, DocumentBlock::Heading { .. }))); } #[test] fn test_front_matter_handled() { let md = "---\ntitle: Test\n---\n\n# Content\n\nBody text."; let blocks = parse_markdown(md).unwrap(); assert!(blocks.iter().any(|b| matches!(b, DocumentBlock::Heading { .. }))); } #[test] fn test_empty_blocks_filtered() { let md = "# Heading\n\n\n\nParagraph\n\n\n"; let blocks = parse_markdown(md).unwrap(); let paras = blocks.iter().filter(|b| matches!(b, DocumentBlock::Paragraph { .. })).count(); assert_eq!(paras, 1, "multiple blank lines should not create empty paragraphs"); } #[test] fn test_very_long_document() { let mut md = String::from("# Long\n\n"); for i in 0..500 { md.push_str(&format!("Para {i}. Content here.\n\n")); } let blocks = parse_markdown(&md).unwrap(); assert!(blocks.len() > 500, "should handle 500 paragraphs"); } #[test] fn test_mixed_inline_formatting() { let md = "# Title\n\nText with `code` and **bold** and *italic* and ~~strike~~.\n\n- `coded` item\n- **bold** item"; let blocks = parse_markdown(md).unwrap(); assert!(!blocks.is_empty()); } #[test] fn test_task_list() { let md = "- [x] Done task\n- [ ] Pending task\n- Normal item"; let blocks = parse_markdown(md).unwrap(); assert!(blocks.iter().any(|b| matches!(b, DocumentBlock::List { .. }))); } #[test] fn test_all_eight_block_types() { let md = "# H\n\nPara.\n\n- item\n\n```rs\nlet x=1;\n```\n\n> Quote\n\n|A|B|\n|---|---|\n|1|2|\n\n![alt](x.png)\n\n---"; let blocks = parse_markdown(md).unwrap(); let has = |t: &str| -> bool { blocks.iter().any(|b| match (t, b) { ("heading", DocumentBlock::Heading { .. }) => true, ("para", DocumentBlock::Paragraph { .. }) => true, ("list", DocumentBlock::List { .. }) => true, ("code", DocumentBlock::CodeBlock { .. }) => true, ("quote", DocumentBlock::Quote { .. }) => true, ("table", DocumentBlock::Table { .. }) => true, ("image", DocumentBlock::Image { .. }) => true, ("rule", DocumentBlock::HorizontalRule { .. }) => true, _ => false, }) }; assert!(has("heading")); assert!(has("para")); assert!(has("list")); assert!(has("code")); assert!(has("quote")); assert!(has("table")); assert!(has("image")); assert!(has("rule")); } }