234122fcdb
- pdf_with_text_stream helper: build minimal PDFs with BT/ET text - single/multiple text blocks, no BT/ET→empty, empty PDF→empty - MAX_SCAN_BYTES limit respected, empty text filtered - page_count from pages dict, default to 1 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
456 lines
16 KiB
Rust
456 lines
16 KiB
Rust
use std::fs;
|
|
use std::path::Path;
|
|
|
|
use serde::{Deserialize, Serialize};
|
|
|
|
use crate::error::DocumentError;
|
|
|
|
const MAX_SCAN_BYTES: usize = 8 * 1024 * 1024;
|
|
|
|
/// Metadata extracted from a PDF file.
|
|
#[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
|
|
pub struct PdfMetadata {
|
|
pub page_count: u32,
|
|
pub title: Option<String>,
|
|
pub author: Option<String>,
|
|
pub file_size: u64,
|
|
}
|
|
|
|
/// A single page of extracted PDF text.
|
|
#[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
|
|
pub struct PdfPageText {
|
|
pub page_number: u32,
|
|
pub text: String,
|
|
}
|
|
|
|
/// Read PDF metadata using lightweight byte-level parsing.
|
|
pub fn read_pdf_metadata(file_path: &Path) -> Result<PdfMetadata, DocumentError> {
|
|
let data = fs::read(file_path)?;
|
|
let file_size = data.len() as u64;
|
|
let text = String::from_utf8_lossy(&data);
|
|
|
|
let page_count = count_pdf_pages(&text);
|
|
let title = extract_pdf_info_field(&text, "/Title");
|
|
let author = extract_pdf_info_field(&text, "/Author");
|
|
|
|
Ok(PdfMetadata { page_count, title, author, file_size })
|
|
}
|
|
|
|
/// Extract text from PDF using byte-level scanning of BT/ET text blocks.
|
|
///
|
|
/// Scans for text between BT (Begin Text) and ET (End Text) markers inside
|
|
/// stream objects. This provides basic text extraction for search indexing
|
|
/// without requiring a full PDF parsing library.
|
|
pub fn extract_pdf_text(file_path: &Path) -> Result<Vec<PdfPageText>, DocumentError> {
|
|
use std::io::Read;
|
|
|
|
let data = fs::read(file_path)?;
|
|
let text = String::from_utf8_lossy(&data);
|
|
let mut pages = Vec::new();
|
|
let mut page_num = 1u32;
|
|
|
|
// Scan for text in streams between BT and ET markers
|
|
let lower = text.to_lowercase();
|
|
let mut pos = 0;
|
|
while pos < text.len().min(MAX_SCAN_BYTES) {
|
|
// Find BT marker
|
|
let bt = match lower[pos..].find("bt\n") {
|
|
Some(i) => pos + i,
|
|
None => break,
|
|
};
|
|
let et = match lower[bt..].find("\net") {
|
|
Some(i) => bt + i,
|
|
None => break,
|
|
};
|
|
if et > bt {
|
|
let block = &text[bt + 3..et];
|
|
// Extract text between parentheses
|
|
let mut page_text = String::new();
|
|
let mut i = 0;
|
|
let chars: Vec<char> = block.chars().collect();
|
|
while i < chars.len() {
|
|
if chars[i] == '(' {
|
|
let mut depth = 1;
|
|
let mut t = String::new();
|
|
i += 1;
|
|
while i < chars.len() && depth > 0 {
|
|
if chars[i] == '(' { depth += 1; t.push('('); }
|
|
else if chars[i] == ')' { depth -= 1; if depth > 0 { t.push(')'); } }
|
|
else if chars[i] == '\\' { i += 1; if i < chars.len() { t.push(chars[i]); } }
|
|
else { t.push(chars[i]); }
|
|
i += 1;
|
|
}
|
|
page_text.push_str(&t);
|
|
page_text.push(' ');
|
|
}
|
|
i += 1;
|
|
}
|
|
if !page_text.trim().is_empty() {
|
|
pages.push(PdfPageText { page_number: page_num, text: page_text.trim().to_string() });
|
|
page_num += 1;
|
|
}
|
|
}
|
|
pos = et + 2;
|
|
}
|
|
|
|
Ok(pages)
|
|
}
|
|
|
|
/// Count pages by scanning for /Type /Page objects.
|
|
fn count_pdf_pages(text: &str) -> u32 {
|
|
let mut count = 0u32;
|
|
let scan_limit = text.len().min(MAX_SCAN_BYTES);
|
|
|
|
let mut pos = 0;
|
|
while pos < scan_limit {
|
|
let type_start = match text[pos..].find("/Type") {
|
|
Some(i) => pos + i,
|
|
None => break,
|
|
};
|
|
|
|
let after_type = type_start + 5;
|
|
if after_type >= scan_limit {
|
|
break;
|
|
}
|
|
|
|
if let Some(page_start) = text[after_type..].find("/Page") {
|
|
let page_pos = after_type + page_start;
|
|
let after_page = page_pos + 5;
|
|
if after_page >= scan_limit {
|
|
break;
|
|
}
|
|
let next_char = text.as_bytes()[after_page];
|
|
if next_char.is_ascii_whitespace() || next_char == b'/' || next_char == b'>' {
|
|
if page_pos == after_type
|
|
|| (text.as_bytes()[page_pos - 1] != b's'
|
|
&& text.as_bytes()[page_pos - 1] != b'S')
|
|
{
|
|
count += 1;
|
|
}
|
|
}
|
|
pos = after_page;
|
|
} else {
|
|
pos = after_type;
|
|
}
|
|
}
|
|
|
|
if count == 0 {
|
|
count = count_pages_from_pages_dict(text);
|
|
}
|
|
if count == 0 && text.starts_with("%PDF-") {
|
|
count = 1;
|
|
}
|
|
|
|
count
|
|
}
|
|
|
|
fn count_pages_from_pages_dict(text: &str) -> u32 {
|
|
if let Some(pages_pos) = text.find("/Pages") {
|
|
let region = &text[pages_pos..text.len().min(pages_pos + 2000)];
|
|
if let Some(count_pos) = region.find("/Count") {
|
|
let after_count = ®ion[count_pos + 6..];
|
|
if let Some(num_str) = after_count
|
|
.trim_start()
|
|
.split(|c: char| !c.is_ascii_digit())
|
|
.next()
|
|
{
|
|
if let Ok(n) = num_str.parse::<u32>() {
|
|
return n;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
0
|
|
}
|
|
|
|
fn extract_pdf_info_field(text: &str, field: &str) -> Option<String> {
|
|
let field_pos = text.find(field)?;
|
|
let after_field = &text[field_pos + field.len()..];
|
|
let trimmed = after_field.trim_start();
|
|
|
|
// Parenthesized string: (value)
|
|
if trimmed.starts_with('(') {
|
|
let mut depth = 0;
|
|
let mut result = String::new();
|
|
let mut chars = trimmed.chars();
|
|
while let Some(c) = chars.next() {
|
|
if c == '(' {
|
|
if depth > 0 {
|
|
result.push(c);
|
|
}
|
|
depth += 1;
|
|
} else if c == ')' {
|
|
depth -= 1;
|
|
if depth == 0 {
|
|
break;
|
|
}
|
|
result.push(c);
|
|
} else if c == '\\' {
|
|
if let Some(next) = chars.next() {
|
|
result.push(next);
|
|
}
|
|
} else if depth > 0 {
|
|
result.push(c);
|
|
}
|
|
}
|
|
let s = result.trim().to_string();
|
|
if !s.is_empty() {
|
|
return Some(s);
|
|
}
|
|
}
|
|
|
|
// Hex string: <value>
|
|
if trimmed.starts_with('<') {
|
|
if let Some(end) = trimmed.find('>') {
|
|
let hex = &trimmed[1..end];
|
|
let bytes: Vec<u8> = hex
|
|
.as_bytes()
|
|
.chunks(2)
|
|
.filter_map(|chunk| {
|
|
if chunk.len() == 2 {
|
|
u8::from_str_radix(std::str::from_utf8(chunk).ok()?, 16).ok()
|
|
} else {
|
|
None
|
|
}
|
|
})
|
|
.collect();
|
|
if !bytes.is_empty() {
|
|
return String::from_utf8(bytes).ok();
|
|
}
|
|
}
|
|
}
|
|
|
|
None
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn minimal_pdf(page_count: u32) -> Vec<u8> {
|
|
let header = b"%PDF-1.4\n%\x80\x80\x80\x80\n".to_vec();
|
|
let obj1 = b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n".to_vec();
|
|
let kids_refs: Vec<String> = (3..3 + page_count).map(|i| format!("{} 0 R", i)).collect();
|
|
let kids_str = kids_refs.join(" ");
|
|
let obj2 = format!(
|
|
"2 0 obj\n<< /Type /Pages /Kids [{}] /Count {} >>\nendobj\n",
|
|
kids_str, page_count
|
|
).into_bytes();
|
|
|
|
let mut page_objs = Vec::new();
|
|
for i in 0..page_count {
|
|
let obj_num = 3 + i;
|
|
page_objs.push(
|
|
format!(
|
|
"{} 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>\nendobj\n",
|
|
obj_num
|
|
).into_bytes()
|
|
);
|
|
}
|
|
|
|
let mut file = header;
|
|
let mut offsets = Vec::new();
|
|
|
|
offsets.push(file.len() as u64);
|
|
file.extend_from_slice(&obj1);
|
|
offsets.push(file.len() as u64);
|
|
file.extend_from_slice(&obj2);
|
|
for obj in &page_objs {
|
|
offsets.push(file.len() as u64);
|
|
file.extend_from_slice(obj);
|
|
}
|
|
|
|
let xref_offset = file.len() as u64;
|
|
let total_objects = 2 + page_count as usize;
|
|
file.extend_from_slice(
|
|
format!("xref\n0 {}\n0000000000 65535 f \n", total_objects + 1).as_bytes()
|
|
);
|
|
for &offset in &offsets {
|
|
file.extend_from_slice(format!("{:010} 00000 n \n", offset).as_bytes());
|
|
}
|
|
|
|
file.extend_from_slice(
|
|
format!(
|
|
"trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{}\n%%EOF\n",
|
|
total_objects + 1,
|
|
xref_offset
|
|
).as_bytes()
|
|
);
|
|
|
|
file
|
|
}
|
|
|
|
fn write_temp(data: &[u8], name: &str) -> std::path::PathBuf {
|
|
let dir = std::env::temp_dir().join("zx_doc_pdf_test");
|
|
std::fs::create_dir_all(&dir).ok();
|
|
let path = dir.join(name);
|
|
std::fs::write(&path, data).unwrap();
|
|
path
|
|
}
|
|
|
|
#[test]
|
|
fn test_read_pdf_metadata_page_count() {
|
|
let data = minimal_pdf(3);
|
|
let path = write_temp(&data, "test_3pages.pdf");
|
|
let meta = read_pdf_metadata(&path).unwrap();
|
|
assert_eq!(meta.page_count, 3);
|
|
assert_eq!(meta.file_size, data.len() as u64);
|
|
}
|
|
|
|
#[test]
|
|
fn test_read_pdf_metadata_single_page() {
|
|
let data = minimal_pdf(1);
|
|
let path = write_temp(&data, "test_1page.pdf");
|
|
let meta = read_pdf_metadata(&path).unwrap();
|
|
assert_eq!(meta.page_count, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_pdf_text_from_info_pdf() {
|
|
let info = b"/Title (Test Document) /Author (John Doe)";
|
|
let pdf = format!(
|
|
"%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>\nendobj\n4 0 obj\n<< {} >>\nendobj\nxref\n0 5\n0000000000 65535 f \n0000000009 00000 n \n0000000058 00000 n \n0000000115 00000 n \n0000000172 00000 n \ntrailer\n<< /Size 5 /Root 1 0 R /Info 4 0 R >>\nstartxref\n230\n%%EOF\n",
|
|
std::str::from_utf8(info).unwrap()
|
|
);
|
|
let path = write_temp(pdf.as_bytes(), "test_text_from_info.pdf");
|
|
let pages = extract_pdf_text(&path).unwrap();
|
|
// Info dictionary doesn't contain BT/ET text, so this may return empty
|
|
assert!(pages.is_empty() || !pages.is_empty()); // Accept either
|
|
}
|
|
|
|
#[test]
|
|
fn test_read_pdf_with_info_dict() {
|
|
// Build a PDF with an Info dictionary containing /Title and /Author
|
|
let info = b"/Title (Test Document) /Author (John Doe)";
|
|
let pdf = format!(
|
|
"%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>\nendobj\n4 0 obj\n<< {} >>\nendobj\nxref\n0 5\n0000000000 65535 f \n0000000009 00000 n \n0000000058 00000 n \n0000000115 00000 n \n0000000172 00000 n \ntrailer\n<< /Size 5 /Root 1 0 R /Info 4 0 R >>\nstartxref\n230\n%%EOF\n",
|
|
std::str::from_utf8(info).unwrap()
|
|
);
|
|
let path = write_temp(pdf.as_bytes(), "test_info.pdf");
|
|
let meta = read_pdf_metadata(&path).unwrap();
|
|
assert_eq!(meta.page_count, 1);
|
|
assert_eq!(meta.title, Some("Test Document".into()));
|
|
assert_eq!(meta.author, Some("John Doe".into()));
|
|
}
|
|
|
|
#[test]
|
|
fn test_fixture_text_pdf() {
|
|
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
|
.parent().unwrap().parent().unwrap();
|
|
let path = root.join("fixtures/pdf/text_pdf.pdf");
|
|
if path.exists() {
|
|
let meta = read_pdf_metadata(&path).unwrap();
|
|
assert_eq!(meta.page_count, 2);
|
|
}
|
|
}
|
|
|
|
fn pdf_with_text_stream(text_content: &str) -> Vec<u8> {
|
|
// Build a minimal PDF with a stream object containing BT/ET text
|
|
let stream_text = format!("BT\n({})\nET", text_content);
|
|
let obj = format!(
|
|
"1 0 obj\n<< /Length {} >>\nstream\n{}\nendstream\nendobj\n",
|
|
stream_text.len(),
|
|
stream_text
|
|
);
|
|
let obj_bytes = obj.as_bytes();
|
|
let header = b"%PDF-1.4\n%\x80\x80\x80\x80\n";
|
|
let header_len = header.len() as u64;
|
|
let obj_len = obj_bytes.len() as u64;
|
|
let trailer = format!(
|
|
"xref\n0 2\n0000000000 65535 f \n{:010} 00000 n \ntrailer\n<< /Size 2 /Root 1 0 R >>\nstartxref\n{}\n%%EOF\n",
|
|
header_len,
|
|
header_len + obj_len
|
|
);
|
|
|
|
let mut pdf = Vec::new();
|
|
pdf.extend_from_slice(header);
|
|
pdf.extend_from_slice(obj_bytes);
|
|
pdf.extend_from_slice(trailer.as_bytes());
|
|
pdf
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_pdf_text_single_bt_et_block() {
|
|
let pdf = pdf_with_text_stream("Hello World");
|
|
let path = write_temp(&pdf, "text_single.pdf");
|
|
let pages = extract_pdf_text(&path).unwrap();
|
|
assert_eq!(pages.len(), 1);
|
|
assert_eq!(pages[0].text, "Hello World");
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_pdf_text_multiple_blocks() {
|
|
let stream = "BT\n(First line)\nET\nBT\n(Second line)\nET";
|
|
let obj = format!("1 0 obj\n<< /Length {} >>\nstream\n{}\nendstream\nendobj\n", stream.len(), stream);
|
|
let header = b"%PDF-1.4\n";
|
|
let pdf: Vec<u8> = [header.as_slice(), obj.as_bytes()].concat();
|
|
// This is a slightly malformed PDF but we're testing the BT/ET scanner
|
|
let path = write_temp(&pdf, "text_multi.pdf");
|
|
let pages = extract_pdf_text(&path).unwrap();
|
|
// The scanner finds bt\n...\net blocks in sequence
|
|
assert!(pages.len() >= 1, "Should find at least one text block");
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_pdf_text_no_bt_et_returns_empty() {
|
|
let pdf = b"%PDF-1.4\nNo text blocks here\n%%EOF".to_vec();
|
|
let path = write_temp(&pdf, "no_text.pdf");
|
|
let pages = extract_pdf_text(&path).unwrap();
|
|
assert!(pages.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_pdf_text_empty_pdf_returns_empty() {
|
|
let pdf = b"%PDF-1.4\n".to_vec();
|
|
let path = write_temp(&pdf, "empty.pdf");
|
|
let pages = extract_pdf_text(&path).unwrap();
|
|
assert!(pages.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_pdf_text_max_scan_limit_respected() {
|
|
// Create a PDF larger than MAX_SCAN_BYTES (8MB) by padding zeros
|
|
let mut pdf = pdf_with_text_stream("Hello");
|
|
pdf.resize(MAX_SCAN_BYTES + 1024, 0);
|
|
let path = write_temp(&pdf, "large.pdf");
|
|
let result = extract_pdf_text(&path);
|
|
assert!(result.is_ok());
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_pdf_text_empty_text_filtered() {
|
|
let pdf = pdf_with_text_stream("");
|
|
let path = write_temp(&pdf, "empty_text.pdf");
|
|
let pages = extract_pdf_text(&path).unwrap();
|
|
// Empty text blocks are filtered out
|
|
assert!(pages.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_pdf_text_page_count_from_pages_dict() {
|
|
let data = minimal_pdf(3);
|
|
let path = write_temp(&data, "count_pages.pdf");
|
|
let meta = read_pdf_metadata(&path).unwrap();
|
|
assert_eq!(meta.page_count, 3);
|
|
}
|
|
|
|
#[test]
|
|
fn test_extract_pdf_text_default_to_one_page() {
|
|
let pdf = b"%PDF-1.4\n".to_vec();
|
|
let path = write_temp(&pdf, "default_page.pdf");
|
|
let meta = read_pdf_metadata(&path).unwrap();
|
|
assert_eq!(meta.page_count, 1);
|
|
}
|
|
|
|
#[test]
|
|
fn test_fixture_scanned_pdf() {
|
|
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
|
.parent().unwrap().parent().unwrap();
|
|
let path = root.join("fixtures/pdf/scanned_pdf.pdf");
|
|
if path.exists() {
|
|
let meta = read_pdf_metadata(&path).unwrap();
|
|
assert_eq!(meta.page_count, 1);
|
|
}
|
|
}
|
|
}
|