diff --git a/Cargo.lock b/Cargo.lock index b163c37..90d83a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2136,6 +2136,7 @@ dependencies = [ "base64 0.23.1", "ely-platform", "getrandom 0.4.3", + "html5ever", "imap-proto", "log", "mail-builder", @@ -2143,8 +2144,12 @@ dependencies = [ "mail-send", "nom 8.0.0", "rcgen", + "rusqlite", "rustls", "rustls-platform-verifier", + "serde", + "serde_json", + "sha2 0.10.9", "smtp-proto", "tempfile", "tokio", @@ -2554,6 +2559,18 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fancy-regex" version = "0.14.0" @@ -3610,6 +3627,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "heapless" version = "0.9.3" @@ -4386,6 +4412,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linebender_resource_handle" version = "0.1.1" @@ -6673,6 +6710,31 @@ dependencies = [ "lru", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.21", +] + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags 2.13.2", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + [[package]] name = "rust-embed" version = "8.12.0" @@ -7403,6 +7465,18 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -8431,6 +8505,12 @@ dependencies = [ "sval_serde", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index 54cd659..4167a7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,8 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg", semver = { version = "1", features = ["serde"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +rusqlite = { version = "0.40", features = ["bundled"] } +html5ever = "0.39" sha2 = "0.10" tempfile = "3" ureq = "3" diff --git a/agents.md b/agents.md index 3af5b8f..efb84ea 100644 --- a/agents.md +++ b/agents.md @@ -320,4 +320,4 @@ cargo run -p ely-doc 启动一个应用（ElyDoc 先 cargo build -p ely - 2026-10-18：Word 公式（R8.2，Gitea #9）。附件 math.docx 的六类公式逐类核对（本机无 Word，以文件所写为准）：键盘输入、OMML（内核转为公式对象）、MathType（OLE，显示随带的 WMF 预览）、上下标、图片原本就对；差别在两处根因。其一，运行的 `w:position`（升降）原先只给文字，图片与 OLE 公式所在的运行失去它（25 张图片与一个 ½ 浮在行上）：补丁 `0049-writer-inline-objects-move-with-their-run.patch` 把它作为固定升降写到对象的锚字符上（按锚字符自己的字号换算百分比，不扩展到其后的文字），版面让“作为字符”的对象随锚字符的固定升降移动（vertAlign 的自动上下标不动对象，同原样）；各导出本就写出锚字符的升降，DOCX 往返仍为 `w:position`。其二，EQ 域原先只认 `\o`、`\s\do` 与注音，其余（如分数 `\f`）整个消失：补丁 `0050-writer-eq-field-parser.patch`（读法）与 `0051-writer-eq-fields-as-formulas.patch`（导入）把 EQ 域转为公式对象：`\f` 分数、`\r` 根式、`\s` 上下标、`\b` 括号、`\l` 列表、`\a` 阵列、`\i` 积分与求和求积；`\x` 与 `\o` 只留内容（无框、无叠印），`\d` 只前移，`\ai`、`\di` 只是行距（不升降），开关不分大小写；每个命令字符保留其运行的上下标（Word 照此画），同一段上下标里字样变了仍是一个上下标；斜体字母写成标识符（StarMath 与 OMML 都画成斜体，DOCX 往返不丢），斜体的数字与符号写成 `ital` 文字，存为 DOCX 后成正体（OMML 无此写法，已知限制）；没有底的上下标以空文字为底（空的底在 OMML 里是占位符，重开成 ``）；公式取域的字号；读不通的命令以其域代码显示为文字，不消失。只有整个域是 `\s\do N(文字)` 时照原样成为下移的文字，其余都经读法（原先小写的 `\s\do` 读到第一个括号即止，其后丢失）。`client/tests/word.rs` 以 `tests/fixtures/office/word.docx`（由 `.work/r8/gen/eq.py` 生成，不入库）核对公式文本（含开头的空格、大写开关、读不通的域、字样变化的下标、`\ai`、`\di`、小写 `\s\do` 的尾部与嵌套）、DOCX 往返后的斜体、下标与无占位符、锚字符的升降与图块里图片低于文字（去掉版面的升降即失败）。审查三轮：第 3 轮的四处已修并以上述测试核对，未再审查（每条至多三轮）。仍与 Word 不同：编辑区里公式与图片四周有内核的辅助框线（`ShowBoundaries`，视图辅助，不印出）；DOCX 另存后重开行距变宽，因内核导出改写文档默认字体与段距，与公式无关。 - 2026-10-18：工作表过多时标签条（R8.3，Gitea #8）。组件库 `navigation::Tabs` 横排时可横向滚动（滚轮同样），标签可窄于自然宽度；所选标签在所选或标签条宽度变了时滚入视口，在绘制时以刚排好的宽度判定（原先只在首帧滚一次，其后右栏打开、标签条变窄，所选标签被挤出视口）；手动滚动保留到二者之一再变；横排的标签至多与标签条同宽，名字过长以省略号结尾（组件库 e56a343、3a68d1b、7d704d9，同批带入上游列外壳拖动的四个提交）。ElyTable 的标签条随之收缩，添加按钮留在其旁；标签条的状态按工作簿各记一份（元素 id 带标签号），切到另一工作簿时其所选标签照样滚入。`navigation/tests/tabs.rs` 以 30 个标签核对：首帧滚到所选、滚轮回到第一个、新选的滚入、变窄后所选仍在末端（去掉按宽度判定即失败）、长名的所选标签整个在标签条内（去掉宽度上限即失败）；ElyTable 的 `tests/strip.rs` 以 `sheets::strip` 画两个各有 30 张表、都选第 30 张的工作簿，滚回开头后来回切换，所选标签每次都在标签条内（两个工作簿共用一个 id 即失败）。Windows 真窗口（`.work/r8/strip3.png`）：30 张表、第 30 张为当前，1100 宽时标签条末端为它。 - 2026-10-18：ElyMark 公式（R8.4，用户定两项都做）。行内公式照块公式由 RaTeX 排版（原只显示着色的 TeX 原文）：gpui 的文本里放不进图片，组件库 `render/math_text.rs` 把每个行内公式换成等宽的不断行空格（U+00A0，按字体量出个数），排好的公式居中画在其上，行高随最高的公式撑开；空格串与词一样整个折到下一行，画在其所在行（折行处的序号 gpui 读作上一行末）；按下与右键所在的字节经换算读回公式的原文；排不出的公式照旧显示 TeX 原文，其后以危险色写出 RaTeX 给的原因与位置，记警告（不含公式）。`\(…\)` 读作行内公式，`\[…\]` 读作块公式（ChatGPT 等常写的分隔符；GitHub 不认，Typora 默认不认）：解析前逐字节把成对的分隔符换成 `$`（长度不变，位置照旧对得上）。成对只在同一段落、标题或单元格里（跨列表项、CR 空行不成对）；不跨代码、公式、HTML 标签、链接地址与引用标签、图片、脚注引用，``、`` 之间（含嵌套；交叉闭合照可视的读法关闭与重开）的文字照原文；`\\(` 不换。公式里与段落里另有的 `$`（含 `\$`）解析时暂写成文本里没有的控制字节，解析后还原，不与换上的 `$` 配对；相邻的两个公式之间暂插一个空格，解析后去掉，仍是两个公式。公式节点的范围是整对分隔符，值是写下的 TeX 原文。独占一段的 `\[…\]` 为块公式（段首可有一至三个空格）。组件库公开 `documents::markdown_tree`，ElyMark 的 HTML、PDF、纯文本导出都经它解析（`markdown_options` 不再公开），纯文本里独占一段的公式照块公式写其 TeX。同时查出（组件库 e095f06）：RaTeX 的 SVG 以 pt 定尺寸，usvg 按 px 量墨迹，两者差 4/3，每个公式的画面右侧与下方多出约四分之一的空白（块公式因此偏左）、导出的公式比正文大 4/3；现生成时即把根改为 px。`markdown/tex/tests.rs` 与 `tree/tests.rs` 核对分隔符的各种情形（相邻、`\$`、链接地址、``、跨块），`math_text.rs` 的窗口测试核对公式排进并读回原文、折到第二行的公式画在第二行首（改回按起点定位即失败）、排不出的公式写出原因（组件库 a20e712、5fba7b3、de961eb），`math/tests.rs` 核对画面与公式同宽（改回 pt 即失败），ElyMark 的 `visual.rs` 核对导出与纯文本。Windows 真窗口（`.work/r8/mark/visual2.png`）：`$…$`、`\(…\)` 排在行内，`\[…\]` 居中成块，行内代码与转义的 `\\(` 照原文。 -- 2026-10-19（用户定）：ElyMail 的后端改为 Rust crate，取代 MailCore 2（其 Windows 构建是 VS 2013 工程加 2015 年前后的预编译依赖，POP 无 OAuth2）：mail-send、mail-parser、mail-builder，在应用进程的后台任务里运行，不另起 worker；MailCore 2 与 libetpan 的取码、构建与探测随之删去。正文以 html5ever 解析、净化后按块的子集由 GPUI 绘制，不当作完整页面；写信用 Markdown 文档组件，发出纯文本与 HTML。Gmail 与 Microsoft 走 OAuth（授权码加 PKCE，本机回环），客户端 ID 在构建时注入：`ELY_GOOGLE_CLIENT_ID`、`ELY_GOOGLE_CLIENT_SECRET`、`ELY_MICROSOFT_CLIENT_ID`，本机写在项目根的 `oauth.env`（不入库），CI 为仓库机密；缺 ID 的构建里该登录写明不可用，不退回别的方式。iCloud 不给第三方 OAuth，走应用专用密码向导（`imap.mail.me.com:993`、`smtp.mail.me.com:587`，无 POP3）。注册由用户做（Google Desktop 客户端，Entra 公共客户端，委托权限 IMAP、POP、SMTP 与 offline_access）。POP3（C12.1 审查后）由引擎自写，不用 async-pop：它留着点填充、丢末尾 CRLF、吞掉登录后的连接错误、不答 XOAUTH2 的错误挑战、在 trace 日志里写出密码。IMAP（第 2 轮审查后）同样自写，以 imap-proto 解析回复，不用 async-imap：它的 LIST、FETCH 等流式命令不看结尾的 NO 与 BAD（拒绝成了空结果），登录与命令中途的 `* BYE` 被丢掉（只剩断线），trace 日志写出密码与邮件。SMTP 的事务（MAIL、RCPT、DATA）也由引擎自走：mail-send 不给首行的点加点；正文有 8 位字节即带 `BODY=8BITMIME`，服务器不支持即拒绝。读信按 MIME 树与所在容器（C12.2），嵌套（多部分与内嵌邮件都算）深于 64 层即报错，超深的解析结果以迭代拆开再丢（直接丢弃会递归、栈溢出）；mail-parser 开 `full_encoding`；SMTPUTF8 按每一层的头块判定。 +- 2026-10-19（用户定）：ElyMail 的后端改为 Rust crate，取代 MailCore 2（其 Windows 构建是 VS 2013 工程加 2015 年前后的预编译依赖，POP 无 OAuth2）：mail-send、mail-parser、mail-builder，在应用进程的后台任务里运行，不另起 worker；MailCore 2 与 libetpan 的取码、构建与探测随之删去。正文以 html5ever 解析、净化后按块的子集由 GPUI 绘制，不当作完整页面；写信用 Markdown 文档组件，发出纯文本与 HTML。Gmail 与 Microsoft 走 OAuth（授权码加 PKCE，本机回环），客户端 ID 在构建时注入：`ELY_GOOGLE_CLIENT_ID`、`ELY_GOOGLE_CLIENT_SECRET`、`ELY_MICROSOFT_CLIENT_ID`，本机写在项目根的 `oauth.env`（不入库），CI 为仓库机密；缺 ID 的构建里该登录写明不可用，不退回别的方式。iCloud 不给第三方 OAuth，走应用专用密码向导（`imap.mail.me.com:993`、`smtp.mail.me.com:587`，无 POP3）。注册由用户做（Google Desktop 客户端，Entra 公共客户端，委托权限 IMAP、POP、SMTP 与 offline_access）。POP3（C12.1 审查后）由引擎自写，不用 async-pop：它留着点填充、丢末尾 CRLF、吞掉登录后的连接错误、不答 XOAUTH2 的错误挑战、在 trace 日志里写出密码。IMAP（第 2 轮审查后）同样自写，以 imap-proto 解析回复，不用 async-imap：它的 LIST、FETCH 等流式命令不看结尾的 NO 与 BAD（拒绝成了空结果），登录与命令中途的 `* BYE` 被丢掉（只剩断线），trace 日志写出密码与邮件。SMTP 的事务（MAIL、RCPT、DATA）也由引擎自走：mail-send 不给首行的点加点；正文有 8 位字节即带 `BODY=8BITMIME`，服务器不支持即拒绝。读信按 MIME 树与所在容器（C12.2），嵌套（多部分与内嵌邮件都算）深于 64 层即报错，超深的解析结果以迭代拆开再丢（直接丢弃会递归、栈溢出）；mail-parser 开 `full_encoding`；SMTPUTF8 按每一层的头块判定。本地存储（C12.3）：rusqlite 自带的 SQLite，`mail.db` 加内容目录（原文按 SHA-256），FTS5（trigram）可由索引与原文重建；`user_version` 记结构版本，升级前备份，失败即只读；原文文件在删去或替换邮件的事务提交前删去，删不掉即回滚。只有 HTML 的邮件经 html5ever 的树构建器取文字入索引，元素深过 512 层即停读（树构建器的开放元素栈扫描随深度平方增长）。 diff --git a/crates/ely-mail-engine/Cargo.toml b/crates/ely-mail-engine/Cargo.toml index c1e7fa0..26cebf1 100644 --- a/crates/ely-mail-engine/Cargo.toml +++ b/crates/ely-mail-engine/Cargo.toml @@ -9,6 +9,7 @@ publish.workspace = true [dependencies] base64.workspace = true getrandom.workspace = true +html5ever.workspace = true imap-proto.workspace = true log.workspace = true mail-builder.workspace = true @@ -17,6 +18,10 @@ mail-send.workspace = true nom.workspace = true rustls.workspace = true rustls-platform-verifier.workspace = true +rusqlite.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true smtp-proto = "0.2" tokio.workspace = true tokio-rustls.workspace = true diff --git a/crates/ely-mail-engine/src/error.rs b/crates/ely-mail-engine/src/error.rs index 187f207..905a90e 100644 --- a/crates/ely-mail-engine/src/error.rs +++ b/crates/ely-mail-engine/src/error.rs @@ -21,6 +21,10 @@ pub enum Error { Invalid(String), /// A message that cannot be read or written. Mime(String), + /// The local store failed: database, disk or damage. + Store(String), + /// The store refuses writes; the reason says why. + ReadOnly(String), } impl fmt::Display for Error { @@ -34,8 +38,22 @@ impl fmt::Display for Error { Self::Timeout(step) => write!(f, "no answer while {step}"), Self::Invalid(text) => write!(f, "cannot be sent: {text}"), Self::Mime(text) => write!(f, "unreadable message: {text}"), + Self::Store(text) => write!(f, "local store failed: {text}"), + Self::ReadOnly(text) => write!(f, "local store is read-only: {text}"), } } } impl std::error::Error for Error {} + +impl From for Error { + fn from(err: rusqlite::Error) -> Self { + Self::Store(err.to_string()) + } +} + +impl From for Error { + fn from(err: serde_json::Error) -> Self { + Self::Store(format!("a stored field: {err}")) + } +} diff --git a/crates/ely-mail-engine/src/lib.rs b/crates/ely-mail-engine/src/lib.rs index c20a560..d80b757 100644 --- a/crates/ely-mail-engine/src/lib.rs +++ b/crates/ely-mail-engine/src/lib.rs @@ -6,6 +6,7 @@ mod mime; mod net; mod pop; mod smtp; +mod store; mod wire; pub use error::{Error, Result}; @@ -14,4 +15,5 @@ pub use mime::{Built, Draft, Mail, Mailbox, Part, build, read}; pub use net::{Login, Secret, Security, Server, system_tls}; pub use pop::Pop; pub use smtp::Smtp; +pub use store::{AccountId, Fetched, Key, MailboxId, MessageId, Sizes, Store, Stored}; pub use tokio_rustls::TlsConnector; diff --git a/crates/ely-mail-engine/src/mime.rs b/crates/ely-mail-engine/src/mime.rs index 98ef149..3099c0d 100644 --- a/crates/ely-mail-engine/src/mime.rs +++ b/crates/ely-mail-engine/src/mime.rs @@ -4,12 +4,13 @@ use mail_builder::{ mime::MimePart, }; use mail_parser::{Address, HeaderValue, MessageParser}; +use serde::{Deserialize, Serialize}; use crate::{Error, Result, smtp::mailbox}; mod walk; -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct Mailbox { pub name: Option, pub email: String, diff --git a/crates/ely-mail-engine/src/store.rs b/crates/ely-mail-engine/src/store.rs new file mode 100644 index 0000000..75e5414 --- /dev/null +++ b/crates/ely-mail-engine/src/store.rs @@ -0,0 +1,228 @@ +//! The local index, the originals and the search index. + +use std::{ + fs, io, + path::{Path, PathBuf}, + time::{SystemTime, UNIX_EPOCH}, +}; + +use rusqlite::{Connection, OpenFlags, OptionalExtension, params}; + +use crate::{Error, Result}; + +mod html; +mod messages; +mod schema; +mod search; +#[cfg(test)] +mod tests; + +pub use messages::{Fetched, Key, Stored}; + +const DB: &str = "mail.db"; +const CONTENT: &str = "content"; +const BACKUPS: &str = "backups"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct AccountId(pub i64); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct MailboxId(pub i64); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct MessageId(pub i64); + +/// Bytes on disk per kind. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Sizes { + pub index: u64, + pub search: u64, + pub originals: u64, +} + +pub struct Store { + conn: Connection, + dir: PathBuf, + /// Why writes are refused, when they are. + read_only: Option, +} + +impl Store { + pub fn open(dir: &Path) -> Result { + open(dir, schema::STEPS) + } + + pub fn read_only(&self) -> Option<&str> { + self.read_only.as_deref() + } + + fn writable(&self) -> Result<()> { + match &self.read_only { + Some(reason) => Err(Error::ReadOnly(reason.clone())), + None => Ok(()), + } + } + + pub fn add_account(&self, address: &str) -> Result { + self.writable()?; + self.conn + .execute("INSERT INTO accounts (address) VALUES (?1)", [address])?; + Ok(AccountId(self.conn.last_insert_rowid())) + } + + pub fn accounts(&self) -> Result> { + let mut query = self + .conn + .prepare("SELECT id, address FROM accounts ORDER BY id")?; + let rows = query.query_map([], |row| Ok((AccountId(row.get(0)?), row.get(1)?)))?; + Ok(rows.collect::>()?) + } + + /// The mailbox by name, made on first use. + pub fn mailbox(&self, account: AccountId, name: &str) -> Result { + let find = "SELECT id FROM mailboxes WHERE account = ?1 AND name = ?2"; + if let Some(id) = self + .conn + .query_row(find, params![account.0, name], |row| row.get(0)) + .optional()? + { + return Ok(MailboxId(id)); + } + self.writable()?; + self.conn.execute( + "INSERT INTO mailboxes (account, name) VALUES (?1, ?2)", + params![account.0, name], + )?; + Ok(MailboxId(self.conn.last_insert_rowid())) + } + + /// The index counts the WAL and shared-memory files. + pub fn sizes(&self) -> Result { + let mut files = 0; + for name in [DB.to_owned(), format!("{DB}-wal"), format!("{DB}-shm")] { + files += match fs::metadata(self.dir.join(&name)) { + Ok(meta) => meta.len(), + Err(err) if err.kind() == io::ErrorKind::NotFound => 0, + Err(err) => return Err(disk("measuring the index", err)), + }; + } + let search: i64 = self.conn.query_row( + "SELECT coalesce(sum(pgsize), 0) FROM dbstat + WHERE name = 'search' OR name GLOB 'search_*'", + [], + |row| row.get(0), + )?; + let search = u64::try_from(search) + .map_err(|_| Error::Store(format!("a negative size: {search}")))?; + let index = files.checked_sub(search).ok_or_else(|| { + Error::Store(format!("search pages {search} exceed the files {files}")) + })?; + Ok(Sizes { + index, + search, + originals: folder_size(&self.dir.join(CONTENT))?, + }) + } +} + +fn open(dir: &Path, steps: &[&str]) -> Result { + let path = dir.join(DB); + fs::create_dir_all(dir.join(CONTENT)).map_err(|err| disk("making the store", err))?; + let mut conn = Connection::open(&path)?; + let version: i64 = conn.query_row("PRAGMA user_version", [], |row| row.get(0))?; + let version = usize::try_from(version) + .map_err(|_| Error::Store(format!("schema version {version} is negative")))?; + let target = steps.len(); + let refused = if version > target { + Some(format!( + "written by a newer ElyMail (schema v{version}, this one reads v{target})" + )) + } else if version < target { + upgrade(&mut conn, dir, version, steps).err() + } else { + None + }; + if let Some(reason) = refused { + log::warn!("mail: store opened read-only: {reason}"); + drop(conn); + let flags = OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX; + let conn = Connection::open_with_flags(&path, flags)?; + return Ok(Store { + conn, + dir: dir.to_owned(), + read_only: Some(reason), + }); + } + conn.pragma_update(None, "foreign_keys", true)?; + conn.pragma_update(None, "journal_mode", "WAL")?; + log::info!("mail: store opened at schema v{target}"); + Ok(Store { + conn, + dir: dir.to_owned(), + read_only: None, + }) +} + +/// A backup comes first; a failure keeps the old data. +fn upgrade( + conn: &mut Connection, + dir: &Path, + version: usize, + steps: &[&str], +) -> std::result::Result<(), String> { + if version > 0 { + backup(conn, dir, version).map_err(|err| format!("no backup before upgrade: {err}"))?; + } + migrate(conn, version, steps).map_err(|err| { + format!( + "upgrade from schema v{version} to v{} failed: {err}", + steps.len() + ) + })?; + log::info!("mail: store schema v{version} upgraded to v{}", steps.len()); + Ok(()) +} + +fn migrate(conn: &mut Connection, version: usize, steps: &[&str]) -> rusqlite::Result<()> { + let tx = conn.transaction()?; + for step in &steps[version..] { + tx.execute_batch(step)?; + } + tx.pragma_update(None, "user_version", steps.len() as i64)?; + tx.commit() +} + +fn backup(conn: &Connection, dir: &Path, version: usize) -> Result<()> { + let folder = dir.join(BACKUPS); + fs::create_dir_all(&folder).map_err(|err| disk("making the backup folder", err))?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|err| Error::Store(format!("the clock is before 1970: {err}")))? + .as_secs(); + let target = folder.join(format!("mail-v{version}-{now}.db")); + let target = target + .to_str() + .ok_or_else(|| Error::Store("the backup path is not UTF-8".into()))?; + conn.execute("VACUUM INTO ?1", [target])?; + log::info!("mail: store schema v{version} backed up before upgrade"); + Ok(()) +} + +/// Two levels: `content/ab/abcd…`. +fn folder_size(folder: &Path) -> Result { + let mut total = 0; + for shard in fs::read_dir(folder).map_err(|err| disk("reading the originals", err))? { + let shard = shard.map_err(|err| disk("reading the originals", err))?; + for file in fs::read_dir(shard.path()).map_err(|err| disk("reading the originals", err))? { + let meta = file + .and_then(|file| file.metadata()) + .map_err(|err| disk("reading the originals", err))?; + total += meta.len(); + } + } + Ok(total) +} + +fn disk(step: &str, err: io::Error) -> Error { + Error::Store(format!("{step}: {err}")) +} diff --git a/crates/ely-mail-engine/src/store/html.rs b/crates/ely-mail-engine/src/store/html.rs new file mode 100644 index 0000000..6b17ed2 --- /dev/null +++ b/crates/ely-mail-engine/src/store/html.rs @@ -0,0 +1,289 @@ +use std::borrow::Cow; +use std::cell::{Cell, RefCell}; + +use html5ever::{ + Attribute, LocalName, Namespace, ParseOpts, QualName, local_name, parse_document, + tendril::{StrTendril, TendrilSink}, + tree_builder::{ElemName, ElementFlags, NodeOrText, QuirksMode, TreeBuilderOpts, TreeSink}, +}; + +/// Deeper trees stall html5ever. +const DEPTH: usize = 512; +const CHUNK: usize = 4096; + +/// Visible HTML text, and whether it stopped short. +pub(super) fn text(html: &str) -> (String, bool) { + let opts = ParseOpts { + tree_builder: TreeBuilderOpts { + // Mail never runs scripts. + scripting_enabled: false, + ..Default::default() + }, + ..Default::default() + }; + let mut parser = parse_document(Tree::default(), opts); + let mut rest = html; + let mut short = false; + while !rest.is_empty() { + if parser.tokenizer.sink.sink.deep.get() { + short = true; + break; + } + let mut cut = rest.len().min(CHUNK); + while !rest.is_char_boundary(cut) { + cut += 1; + } + let (head, tail) = rest.split_at(cut); + parser.process(StrTendril::from_slice(head)); + rest = tail; + } + (parser.finish().text(), short) +} + +enum Data { + Document, + Element { name: QualName, point: bool }, + Text(String), + Other, +} + +struct Node { + parent: Option, + children: Vec, + data: Data, +} + +/// The tree html5ever's builder makes. +struct Tree { + nodes: RefCell>, + /// An element went past `DEPTH`. + deep: Cell, +} + +impl Default for Tree { + fn default() -> Self { + let document = Node { + parent: None, + children: Vec::new(), + data: Data::Document, + }; + Tree { + nodes: RefCell::new(vec![document]), + deep: Cell::new(false), + } + } +} + +#[derive(Debug)] +pub(super) struct Name(QualName); + +impl ElemName for Name { + fn ns(&self) -> &Namespace { + &self.0.ns + } + + fn local_name(&self) -> &LocalName { + &self.0.local + } +} + +impl Tree { + fn add(&self, data: Data) -> usize { + let mut nodes = self.nodes.borrow_mut(); + nodes.push(Node { + parent: None, + children: Vec::new(), + data, + }); + nodes.len() - 1 + } + + fn detach(&self, node: usize) { + let mut nodes = self.nodes.borrow_mut(); + if let Some(parent) = nodes[node].parent.take() { + nodes[parent].children.retain(|&child| child != node); + } + } + + /// Inserts a child at index `at`. + fn insert(&self, parent: usize, at: usize, child: NodeOrText) { + let node = match child { + NodeOrText::AppendText(text) => { + let mut nodes = self.nodes.borrow_mut(); + let before = at.checked_sub(1).map(|i| nodes[parent].children[i]); + if let Some(Data::Text(held)) = before.map(|i| &mut nodes[i].data) { + held.push_str(&text); + return; + } + drop(nodes); + self.add(Data::Text(text.to_string())) + } + NodeOrText::AppendNode(node) => { + self.detach(node); + node + } + }; + let mut nodes = self.nodes.borrow_mut(); + let at = at.min(nodes[parent].children.len()); + nodes[parent].children.insert(at, node); + nodes[node].parent = Some(parent); + if matches!(nodes[node].data, Data::Element { .. }) { + let mut up = Some(parent); + let mut depth = 0; + while let Some(next) = up.filter(|_| depth <= DEPTH) { + depth += 1; + up = nodes[next].parent; + } + if depth > DEPTH { + self.deep.set(true); + } + } + } + + /// The visible text, read without recursion. + fn text(&self) -> String { + let nodes = self.nodes.borrow(); + let mut out = String::new(); + let mut stack = vec![0]; + while let Some(at) = stack.pop() { + match &nodes[at].data { + Data::Text(text) => out.push_str(text), + Data::Element { name, .. } if hidden(&name.local) => out.push(' '), + Data::Element { .. } | Data::Document => { + // Tags separate words. + out.push(' '); + stack.extend(nodes[at].children.iter().rev()); + } + Data::Other => {} + } + } + out + } +} + +/// Not drawn, per the HTML rendering rules. +fn hidden(name: &LocalName) -> bool { + matches!( + *name, + local_name!("area") + | local_name!("base") + | local_name!("basefont") + | local_name!("datalist") + | local_name!("head") + | local_name!("iframe") + | local_name!("link") + | local_name!("meta") + | local_name!("noembed") + | local_name!("noframes") + | local_name!("param") + | local_name!("rp") + | local_name!("script") + | local_name!("style") + | local_name!("template") + | local_name!("title") + ) +} + +impl TreeSink for Tree { + type Handle = usize; + type Output = Self; + type ElemName<'a> = Name; + + fn finish(self) -> Self { + self + } + + fn parse_error(&self, _msg: Cow<'static, str>) {} + + fn get_document(&self) -> usize { + 0 + } + + fn elem_name(&self, target: &usize) -> Name { + match &self.nodes.borrow()[*target].data { + Data::Element { name, .. } => Name(name.clone()), + _ => unreachable!("html5ever asked a non-element for its name"), + } + } + + fn create_element(&self, name: QualName, _: Vec, flags: ElementFlags) -> usize { + let point = flags.mathml_annotation_xml_integration_point; + let node = self.add(Data::Element { name, point }); + if flags.template { + // Template contents stay out of the tree. + self.add(Data::Other); + } + node + } + + fn create_comment(&self, _text: StrTendril) -> usize { + self.add(Data::Other) + } + + fn create_pi(&self, _target: StrTendril, _data: StrTendril) -> usize { + self.add(Data::Other) + } + + fn append(&self, parent: &usize, child: NodeOrText) { + let end = self.nodes.borrow()[*parent].children.len(); + self.insert(*parent, end, child); + } + + fn append_based_on_parent_node( + &self, + element: &usize, + prev_element: &usize, + child: NodeOrText, + ) { + if self.nodes.borrow()[*element].parent.is_some() { + self.append_before_sibling(element, child); + } else { + self.append(prev_element, child); + } + } + + fn append_doctype_to_document(&self, _: StrTendril, _: StrTendril, _: StrTendril) {} + + fn get_template_contents(&self, target: &usize) -> usize { + target + 1 + } + + fn same_node(&self, x: &usize, y: &usize) -> bool { + x == y + } + + fn set_quirks_mode(&self, _mode: QuirksMode) {} + + fn append_before_sibling(&self, sibling: &usize, child: NodeOrText) { + let (parent, at) = { + let nodes = self.nodes.borrow(); + let parent = nodes[*sibling] + .parent + .expect("html5ever names a sibling with a parent"); + let at = nodes[parent].children.iter().position(|c| c == sibling); + (parent, at.expect("a sibling among its parent's children")) + }; + self.insert(parent, at, child); + } + + fn add_attrs_if_missing(&self, _target: &usize, _attrs: Vec) {} + + fn remove_from_parent(&self, target: &usize) { + self.detach(*target); + } + + fn reparent_children(&self, node: &usize, new_parent: &usize) { + let children = std::mem::take(&mut self.nodes.borrow_mut()[*node].children); + for child in children { + self.nodes.borrow_mut()[child].parent = None; + self.append(new_parent, NodeOrText::AppendNode(child)); + } + } + + fn is_mathml_annotation_xml_integration_point(&self, handle: &usize) -> bool { + matches!( + self.nodes.borrow()[*handle].data, + Data::Element { point: true, .. } + ) + } +} diff --git a/crates/ely-mail-engine/src/store/messages.rs b/crates/ely-mail-engine/src/store/messages.rs new file mode 100644 index 0000000..e121730 --- /dev/null +++ b/crates/ely-mail-engine/src/store/messages.rs @@ -0,0 +1,326 @@ +use std::{fs, io::ErrorKind, path::Path}; + +use rusqlite::{Connection, OptionalExtension, Row, ToSql, Transaction, params}; +use sha2::{Digest, Sha256}; + +use super::{CONTENT, MailboxId, MessageId, Store, disk, search}; +use crate::{Error, Mail, Mailbox, Result, read}; + +/// RFC 3501 §2.3.1.1: identity is mailbox, UIDVALIDITY and UID. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Key { + Imap { + uidvalidity: u32, + uid: u32, + }, + /// POP3 has one mailbox and UIDL names. + Pop { + uidl: String, + }, +} + +/// What a sync brought down. +pub enum Fetched<'a> { + Headers(&'a [u8]), + Whole(&'a [u8]), +} + +/// A message as indexed; the original is read apart. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Stored { + pub id: MessageId, + pub mailbox: MailboxId, + pub key: Key, + pub flags: Vec, + pub message_id: Option, + pub in_reply_to: Vec, + pub references: Vec, + pub subject: Option, + pub from: Vec, + pub to: Vec, + pub cc: Vec, + pub date: Option, + /// None until the whole message is here. + pub attachments: Option, + pub whole: bool, +} + +const COLUMNS: &str = "id, mailbox, uidvalidity, uid, uidl, message_id, in_reply_to, refs, \ + subject, sender, recipients, copies, date, attachments, content"; + +impl Store { + pub fn find(&self, mailbox: MailboxId, key: &Key) -> Result> { + let found = match key { + Key::Imap { uidvalidity, uid } => self.conn.query_row( + "SELECT id FROM messages WHERE mailbox = ?1 AND uidvalidity = ?2 AND uid = ?3", + params![mailbox.0, uidvalidity, uid], + |row| row.get(0), + ), + Key::Pop { uidl } => self.conn.query_row( + "SELECT id FROM messages WHERE mailbox = ?1 AND uidl = ?2", + params![mailbox.0, uidl], + |row| row.get(0), + ), + }; + Ok(found.optional()?.map(MessageId)) + } + + /// One key, one message; headers again only set flags. + pub fn put( + &mut self, + mailbox: MailboxId, + key: &Key, + flags: &[&str], + fetched: Fetched, + ) -> Result { + self.writable()?; + let existing = self.find(mailbox, key)?; + let bytes = match (fetched, existing) { + (Fetched::Headers(_), Some(id)) => { + self.set_flags(id, flags)?; + return Ok(id); + } + (Fetched::Headers(bytes), None) => { + let mail = read(bytes)?; + return self.write(mailbox, key, flags, &mail, None, None); + } + (Fetched::Whole(bytes), _) => bytes, + }; + let mail = read(bytes)?; + let (content, made) = self.keep(bytes)?; + let written = self.write(mailbox, key, flags, &mail, Some(&content), existing); + if written.is_err() + && made + && let Err(gone) = drop_unused(&self.conn, &self.dir, &content) + { + log::error!("mail: an unused original stays after a failed write: {gone}"); + } + written + } + + fn write( + &mut self, + mailbox: MailboxId, + key: &Key, + flags: &[&str], + mail: &Mail, + content: Option<&str>, + existing: Option, + ) -> Result { + let (uidvalidity, uid, uidl) = match key { + Key::Imap { uidvalidity, uid } => (Some(*uidvalidity), Some(*uid), None), + Key::Pop { uidl } => (None, None, Some(uidl.as_str())), + }; + let attachments = content.map(|_| mail.attachments.len() as u32); + let json = [ + serde_json::to_string(&mail.in_reply_to)?, + serde_json::to_string(&mail.references)?, + serde_json::to_string(&mail.from)?, + serde_json::to_string(&mail.to)?, + serde_json::to_string(&mail.cc)?, + ]; + let mut values: Vec<&dyn ToSql> = vec![ + &mail.message_id, + &json[0], + &json[1], + &mail.subject, + &json[2], + &json[3], + &json[4], + &mail.date, + &attachments, + &content, + ]; + let tx = self.conn.transaction()?; + let mut old: Option = None; + let id = match existing { + Some(id) => { + old = tx.query_row( + "SELECT content FROM messages WHERE id = ?1", + [id.0], + |row| row.get(0), + )?; + values.push(&id.0); + tx.execute( + "UPDATE messages SET message_id = ?1, in_reply_to = ?2, refs = ?3, + subject = ?4, sender = ?5, recipients = ?6, copies = ?7, date = ?8, + attachments = ?9, content = ?10 WHERE id = ?11", + values.as_slice(), + )?; + id + } + None => { + values.extend::<[&dyn ToSql; 4]>([&mailbox.0, &uidvalidity, &uid, &uidl]); + tx.execute( + "INSERT INTO messages (message_id, in_reply_to, refs, subject, sender, + recipients, copies, date, attachments, content, mailbox, uidvalidity, + uid, uidl) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, + ?13, ?14)", + values.as_slice(), + )?; + MessageId(tx.last_insert_rowid()) + } + }; + replace_flags(&tx, id, flags)?; + search::index(&tx, id, mail, content.is_some())?; + if let Some(old) = old.filter(|old| Some(old.as_str()) != content) { + drop_unused(&tx, &self.dir, &old)?; + } + tx.commit()?; + Ok(id) + } + + pub fn set_flags(&mut self, id: MessageId, flags: &[&str]) -> Result<()> { + self.writable()?; + self.content(id)?; + let tx = self.conn.transaction()?; + replace_flags(&tx, id, flags)?; + tx.commit()?; + Ok(()) + } + + pub fn get(&self, id: MessageId) -> Result { + let mut stored = self.conn.query_row( + &format!("SELECT {COLUMNS} FROM messages WHERE id = ?1"), + [id.0], + |found| Ok(row(found)), + )??; + let mut query = self + .conn + .prepare("SELECT flag FROM flags WHERE message = ?1 ORDER BY flag")?; + stored.flags = query + .query_map([id.0], |row| row.get(0))? + .collect::>()?; + Ok(stored) + } + + /// The original, checked against its hash; None for headers only. + pub fn original(&self, id: MessageId) -> Result>> { + self.content(id)? + .map(|hash| load_at(&self.dir, &hash)) + .transpose() + } + + pub fn remove(&mut self, id: MessageId) -> Result<()> { + self.writable()?; + let content = self.content(id)?; + let tx = self.conn.transaction()?; + tx.execute("DELETE FROM messages WHERE id = ?1", [id.0])?; + if let Some(hash) = content { + drop_unused(&tx, &self.dir, &hash)?; + } + tx.commit()?; + Ok(()) + } + + fn content(&self, id: MessageId) -> Result> { + self.conn + .query_row( + "SELECT content FROM messages WHERE id = ?1", + [id.0], + |row| row.get(0), + ) + .optional()? + .ok_or_else(|| Error::Store(format!("no message {}", id.0))) + } + + /// The hash, and whether the file is new. + fn keep(&self, bytes: &[u8]) -> Result<(String, bool)> { + let hash = hex(&Sha256::digest(bytes)); + let path = self.dir.join(CONTENT).join(&hash[..2]).join(&hash); + let made = match fs::read(&path) { + Ok(held) if hex(&Sha256::digest(&held)) == hash => return Ok((hash, false)), + Ok(_) => { + log::warn!("mail: original {hash} was damaged; rewritten from the server copy"); + false + } + Err(err) if err.kind() == ErrorKind::NotFound => true, + Err(err) => return Err(disk("checking an original", err)), + }; + let shard = path.parent().expect("a shard folder"); + fs::create_dir_all(shard).map_err(|err| disk("storing an original", err))?; + let staged = shard.join(format!(".{hash}.part")); + if let Err(err) = fs::write(&staged, bytes).and_then(|()| fs::rename(&staged, &path)) { + return Err(match fs::remove_file(&staged) { + Ok(()) => disk("storing an original", err), + Err(gone) if gone.kind() == ErrorKind::NotFound => disk("storing an original", err), + Err(left) => Error::Store(format!( + "storing an original: {err}; the staged copy stayed: {left}" + )), + }); + } + Ok((hash, made)) + } +} + +/// Deletes the file once no row names it; runs before commit. +fn drop_unused(conn: &Connection, dir: &Path, hash: &str) -> Result<()> { + let used: bool = conn.query_row( + "SELECT EXISTS (SELECT 1 FROM messages WHERE content = ?1)", + [hash], + |row| row.get(0), + )?; + if used { + return Ok(()); + } + let path = dir.join(CONTENT).join(&hash[..2]).join(hash); + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(err) if err.kind() == ErrorKind::NotFound => { + log::warn!("mail: original {hash} was already gone"); + Ok(()) + } + Err(err) => Err(disk("deleting an original", err)), + } +} + +fn replace_flags(tx: &Transaction, id: MessageId, flags: &[&str]) -> Result<()> { + tx.execute("DELETE FROM flags WHERE message = ?1", [id.0])?; + for flag in flags { + tx.execute( + "INSERT OR IGNORE INTO flags (message, flag) VALUES (?1, ?2)", + params![id.0, flag], + )?; + } + Ok(()) +} + +fn row(row: &Row) -> Result { + let key = match row.get::<_, Option>(4)? { + Some(uidl) => Key::Pop { uidl }, + None => Key::Imap { + uidvalidity: row.get(2)?, + uid: row.get(3)?, + }, + }; + let json = |at: usize| -> Result { Ok(row.get(at)?) }; + Ok(Stored { + id: MessageId(row.get(0)?), + mailbox: MailboxId(row.get(1)?), + key, + flags: Vec::new(), + message_id: row.get(5)?, + in_reply_to: serde_json::from_str(&json(6)?)?, + references: serde_json::from_str(&json(7)?)?, + subject: row.get(8)?, + from: serde_json::from_str(&json(9)?)?, + to: serde_json::from_str(&json(10)?)?, + cc: serde_json::from_str(&json(11)?)?, + date: row.get(12)?, + attachments: row.get(13)?, + whole: row.get::<_, Option>(14)?.is_some(), + }) +} + +pub(super) fn load_at(dir: &Path, hash: &str) -> Result> { + let path = dir.join(CONTENT).join(&hash[..2]).join(hash); + let bytes = fs::read(&path).map_err(|err| disk("reading an original", err))?; + if hex(&Sha256::digest(&bytes)) != hash { + return Err(Error::Store(format!("original {hash} is damaged"))); + } + Ok(bytes) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} diff --git a/crates/ely-mail-engine/src/store/schema.rs b/crates/ely-mail-engine/src/store/schema.rs new file mode 100644 index 0000000..f1fb405 --- /dev/null +++ b/crates/ely-mail-engine/src/store/schema.rs @@ -0,0 +1,51 @@ +/// Step `n` upgrades version `n`. +pub(super) const STEPS: &[&str] = &[V1]; + +const V1: &str = " +CREATE TABLE accounts ( + id INTEGER PRIMARY KEY, + address TEXT NOT NULL UNIQUE +); +CREATE TABLE mailboxes ( + id INTEGER PRIMARY KEY, + account INTEGER NOT NULL REFERENCES accounts ON DELETE CASCADE, + name TEXT NOT NULL, + UNIQUE (account, name) +); +CREATE TABLE messages ( + id INTEGER PRIMARY KEY, + mailbox INTEGER NOT NULL REFERENCES mailboxes ON DELETE CASCADE, + uidvalidity INTEGER, + uid INTEGER, + uidl TEXT, + message_id TEXT, + in_reply_to TEXT NOT NULL, + refs TEXT NOT NULL, + subject TEXT, + sender TEXT NOT NULL, + recipients TEXT NOT NULL, + copies TEXT NOT NULL, + date INTEGER, + attachments INTEGER, + content TEXT, + CHECK ((uid IS NULL) = (uidvalidity IS NULL) AND (uid IS NULL) <> (uidl IS NULL)) +); +CREATE UNIQUE INDEX messages_imap ON messages (mailbox, uidvalidity, uid) + WHERE uid IS NOT NULL; +CREATE UNIQUE INDEX messages_pop ON messages (mailbox, uidl) WHERE uidl IS NOT NULL; +CREATE INDEX messages_thread ON messages (message_id); +CREATE INDEX messages_content ON messages (content); +CREATE TABLE flags ( + message INTEGER NOT NULL REFERENCES messages ON DELETE CASCADE, + flag TEXT NOT NULL, + PRIMARY KEY (message, flag) +) WITHOUT ROWID; +CREATE INDEX flags_by_flag ON flags (flag, message); +CREATE VIRTUAL TABLE search USING fts5( + subject, people, body, files, + content = '', contentless_delete = 1, tokenize = 'trigram' +); +CREATE TRIGGER messages_gone AFTER DELETE ON messages BEGIN + DELETE FROM search WHERE rowid = old.id; +END; +"; diff --git a/crates/ely-mail-engine/src/store/search.rs b/crates/ely-mail-engine/src/store/search.rs new file mode 100644 index 0000000..1bb5079 --- /dev/null +++ b/crates/ely-mail-engine/src/store/search.rs @@ -0,0 +1,134 @@ +use rusqlite::{Transaction, params}; + +use super::{MessageId, Store, html, messages::load_at}; +use crate::{Mail, Mailbox, Result, read}; + +/// Replaces the message's row in the full-text index. +pub(super) fn index(tx: &Transaction, id: MessageId, mail: &Mail, whole: bool) -> Result<()> { + let people = people([&mail.from, &mail.to, &mail.cc]); + let (body, files) = if whole { + contents(id, mail) + } else { + Default::default() + }; + put(tx, id, mail.subject.as_deref(), &people, &body, &files) +} + +fn put( + tx: &Transaction, + id: MessageId, + subject: Option<&str>, + people: &str, + body: &str, + files: &str, +) -> Result<()> { + tx.execute("DELETE FROM search WHERE rowid = ?1", [id.0])?; + tx.execute( + "INSERT INTO search (rowid, subject, people, body, files) VALUES (?1, ?2, ?3, ?4, ?5)", + params![id.0, subject, people, body, files], + )?; + Ok(()) +} + +impl Store { + /// Rebuilt from the index rows and the originals. + pub fn rebuild_search(&mut self) -> Result { + self.writable()?; + let ids: Vec = { + let mut query = self.conn.prepare("SELECT id FROM messages")?; + query + .query_map([], |row| row.get(0))? + .collect::>()? + }; + let tx = self.conn.transaction()?; + // A damaged index may not delete. + let made: String = tx.query_row( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'search'", + [], + |row| row.get(0), + )?; + tx.execute_batch(&format!("DROP TABLE search; {made};"))?; + for &id in &ids { + let (subject, from, to, cc, content): ( + Option, + String, + String, + String, + Option, + ) = tx.query_row( + "SELECT subject, sender, recipients, copies, content FROM messages + WHERE id = ?1", + [id], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, + )?; + let lists: [Vec; 3] = [ + serde_json::from_str(&from)?, + serde_json::from_str(&to)?, + serde_json::from_str(&cc)?, + ]; + let (body, files) = match content { + Some(hash) => contents(MessageId(id), &read(&load_at(&self.dir, &hash)?)?), + None => Default::default(), + }; + let people = people([&lists[0], &lists[1], &lists[2]]); + put( + &tx, + MessageId(id), + subject.as_deref(), + &people, + &body, + &files, + )?; + } + let count = ids.len(); + tx.commit()?; + log::info!("mail: search index rebuilt for {count} messages"); + Ok(count) + } +} + +fn people(lists: [&[Mailbox]; 3]) -> String { + let mut out = String::new(); + for person in lists.into_iter().flatten() { + if let Some(name) = &person.name { + out.push_str(name); + out.push(' '); + } + out.push_str(&person.email); + out.push('\n'); + } + out +} + +/// The body text and the attachment names. +fn contents(id: MessageId, mail: &Mail) -> (String, String) { + let body = match (&mail.text, &mail.html) { + (Some(text), _) => text.clone(), + (None, Some(html)) => { + let (text, short) = html::text(html); + if short { + log::warn!( + "mail: message {} nests HTML too deep; its later text is not indexed", + id.0 + ); + } + text + } + (None, None) => String::new(), + }; + let files = mail + .attachments + .iter() + .filter_map(|part| part.name.as_deref()) + .collect::>() + .join("\n"); + (body, files) +} diff --git a/crates/ely-mail-engine/src/store/tests.rs b/crates/ely-mail-engine/src/store/tests.rs new file mode 100644 index 0000000..6a52dd8 --- /dev/null +++ b/crates/ely-mail-engine/src/store/tests.rs @@ -0,0 +1,96 @@ +use std::fs; + +use rusqlite::Connection; + +use super::{BACKUPS, DB, open, schema::STEPS}; +use crate::Error; + +const ADD: &str = "ALTER TABLE accounts ADD COLUMN nick TEXT"; +const BROKEN: &str = "ALTER TABLE nowhere ADD COLUMN nick TEXT"; + +fn version(path: &std::path::Path) -> i64 { + Connection::open(path) + .unwrap() + .query_row("PRAGMA user_version", [], |row| row.get(0)) + .unwrap() +} + +fn backups(dir: &std::path::Path) -> Vec { + match fs::read_dir(dir.join(BACKUPS)) { + Ok(entries) => entries.map(|entry| entry.unwrap().path()).collect(), + Err(_) => Vec::new(), + } +} + +#[test] +fn a_new_store_is_at_the_last_schema() { + let dir = tempfile::tempdir().unwrap(); + let store = open(dir.path(), STEPS).unwrap(); + assert_eq!(store.read_only(), None); + assert_eq!(version(&dir.path().join(DB)), STEPS.len() as i64); + assert!(backups(dir.path()).is_empty()); +} + +#[test] +fn an_upgrade_backs_up_first() { + let dir = tempfile::tempdir().unwrap(); + open(dir.path(), &[STEPS[0]]) + .unwrap() + .add_account("ann@example.com") + .unwrap(); + let store = open(dir.path(), &[STEPS[0], ADD]).unwrap(); + assert_eq!(store.read_only(), None); + assert_eq!(version(&dir.path().join(DB)), 2); + let [backup] = backups(dir.path()).try_into().unwrap(); + assert_eq!(version(&backup), 1); + let accounts: i64 = Connection::open(&backup) + .unwrap() + .query_row("SELECT count(*) FROM accounts", [], |row| row.get(0)) + .unwrap(); + assert_eq!(accounts, 1); +} + +#[test] +fn a_failed_upgrade_keeps_the_data_and_refuses_writes() { + let dir = tempfile::tempdir().unwrap(); + open(dir.path(), &[STEPS[0]]) + .unwrap() + .add_account("ann@example.com") + .unwrap(); + let store = open(dir.path(), &[STEPS[0], ADD, BROKEN]).unwrap(); + let reason = store.read_only().unwrap(); + assert!(reason.contains("v1 to v3"), "{reason}"); + assert_eq!(store.accounts().unwrap()[0].1, "ann@example.com"); + let err = store.add_account("bo@example.com").unwrap_err(); + assert!(matches!(err, Error::ReadOnly(_)), "{err}"); + drop(store); + assert_eq!(version(&dir.path().join(DB)), 1); + assert_eq!(backups(dir.path()).len(), 1); + let partial = Connection::open(dir.path().join(DB)) + .unwrap() + .prepare("SELECT nick FROM accounts") + .map(drop); + assert!(partial.is_err(), "the first step was not rolled back"); +} + +#[test] +fn a_newer_schema_is_read_only() { + let dir = tempfile::tempdir().unwrap(); + drop(open(dir.path(), &[STEPS[0], ADD]).unwrap()); + let store = open(dir.path(), &[STEPS[0]]).unwrap(); + let reason = store.read_only().unwrap(); + assert!(reason.contains("newer"), "{reason}"); + assert!(store.accounts().unwrap().is_empty()); +} + +#[test] +fn no_backup_means_no_upgrade() { + let dir = tempfile::tempdir().unwrap(); + drop(open(dir.path(), &[STEPS[0]]).unwrap()); + fs::write(dir.path().join(BACKUPS), b"a file in the way").unwrap(); + let store = open(dir.path(), &[STEPS[0], ADD]).unwrap(); + let reason = store.read_only().unwrap(); + assert!(reason.contains("no backup"), "{reason}"); + drop(store); + assert_eq!(version(&dir.path().join(DB)), 1); +} diff --git a/crates/ely-mail-engine/tests/store.rs b/crates/ely-mail-engine/tests/store.rs new file mode 100644 index 0000000..d42a379 --- /dev/null +++ b/crates/ely-mail-engine/tests/store.rs @@ -0,0 +1,199 @@ +use std::path::Path; + +use ely_mail_engine::{Error, Fetched, Key, MailboxId, MessageId, Store}; +use rusqlite::Connection; +use sha2::{Digest, Sha256}; + +/// The sample as a server sends it: CRLF lines. +fn sample() -> Vec { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../tests/fixtures/mail/sample.eml" + ); + let text = std::fs::read_to_string(path).unwrap(); + text.replace("\r\n", "\n") + .replace('\n', "\r\n") + .into_bytes() +} + +fn headers(raw: &[u8]) -> &[u8] { + let end = raw.windows(4).position(|w| w == b"\r\n\r\n").unwrap(); + &raw[..end + 4] +} + +fn inbox(dir: &Path) -> (Store, MailboxId) { + let store = Store::open(dir).unwrap(); + let account = store.add_account("ann@example.com").unwrap(); + let inbox = store.mailbox(account, "INBOX").unwrap(); + (store, inbox) +} + +fn imap(uidvalidity: u32, uid: u32) -> Key { + Key::Imap { uidvalidity, uid } +} + +/// Rows whose index holds the phrase. +fn found(dir: &Path, phrase: &str) -> Vec { + let conn = Connection::open(dir.join("mail.db")).unwrap(); + let mut rows = conn + .prepare("SELECT rowid FROM search WHERE search MATCH ?1 ORDER BY rowid") + .unwrap(); + rows.query_map([format!("\"{phrase}\"")], |row| row.get(0)) + .unwrap() + .map(Result::unwrap) + .collect() +} + +fn original_path(dir: &Path, raw: &[u8]) -> std::path::PathBuf { + let hash: String = Sha256::digest(raw) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + dir.join("content").join(&hash[..2]).join(hash) +} + +#[test] +fn a_whole_message_is_indexed_and_kept_by_its_hash() { + let dir = tempfile::tempdir().unwrap(); + let (mut store, inbox) = inbox(dir.path()); + let raw = sample(); + let id = store + .put(inbox, &imap(7, 42), &["\\Seen"], Fetched::Whole(&raw)) + .unwrap(); + let stored = store.get(id).unwrap(); + assert_eq!(stored.key, imap(7, 42)); + assert_eq!(stored.subject.as_deref(), Some("Café report")); + assert_eq!(stored.from[0].email, "zhang@example.com"); + assert_eq!(stored.message_id.as_deref(), Some("m1@example.com")); + assert_eq!(stored.flags, ["\\Seen"]); + assert_eq!(stored.attachments, Some(1)); + assert!(stored.whole); + assert_eq!(store.original(id).unwrap().as_deref(), Some(raw.as_slice())); + assert_eq!(std::fs::read(original_path(dir.path(), &raw)).unwrap(), raw); + assert_eq!(found(dir.path(), "numbers"), [id.0]); + assert_eq!(found(dir.path(), "report.pdf"), [id.0]); + let sizes = store.sizes().unwrap(); + assert_eq!(sizes.originals, raw.len() as u64); + assert!(sizes.search > 0 && sizes.index > 0, "{sizes:?}"); +} + +#[test] +fn headers_first_then_the_whole_message() { + let dir = tempfile::tempdir().unwrap(); + let (mut store, inbox) = inbox(dir.path()); + let raw = sample(); + let key = imap(7, 42); + let id = store + .put(inbox, &key, &[], Fetched::Headers(headers(&raw))) + .unwrap(); + let stored = store.get(id).unwrap(); + assert_eq!(stored.subject.as_deref(), Some("Café report")); + assert_eq!((stored.whole, stored.attachments), (false, None)); + assert_eq!(store.original(id).unwrap(), None); + assert_eq!(found(dir.path(), "report"), [id.0]); + assert!(found(dir.path(), "numbers").is_empty()); + + let again = store + .put(inbox, &key, &["\\Flagged"], Fetched::Headers(headers(&raw))) + .unwrap(); + assert_eq!(again, id); + assert_eq!(store.get(id).unwrap().flags, ["\\Flagged"]); + + assert_eq!( + store.put(inbox, &key, &[], Fetched::Whole(&raw)).unwrap(), + id + ); + assert_eq!(store.get(id).unwrap().attachments, Some(1)); + assert_eq!(found(dir.path(), "numbers"), [id.0]); +} + +#[test] +fn uidvalidity_is_part_of_the_identity() { + let dir = tempfile::tempdir().unwrap(); + let (mut store, inbox) = inbox(dir.path()); + let raw = sample(); + let old = store + .put(inbox, &imap(1, 5), &[], Fetched::Headers(headers(&raw))) + .unwrap(); + let new = store + .put(inbox, &imap(2, 5), &[], Fetched::Headers(headers(&raw))) + .unwrap(); + assert_ne!(old, new); + assert_eq!(store.find(inbox, &imap(1, 5)).unwrap(), Some(old)); + let pop = Key::Pop { uidl: "abc".into() }; + let id = store + .put(inbox, &pop, &[], Fetched::Headers(headers(&raw))) + .unwrap(); + assert_eq!(store.get(id).unwrap().key, pop); +} + +#[test] +fn a_shared_original_goes_with_its_last_message() { + let dir = tempfile::tempdir().unwrap(); + let (mut store, inbox) = inbox(dir.path()); + let raw = sample(); + let file = original_path(dir.path(), &raw); + let one = store + .put(inbox, &imap(1, 1), &[], Fetched::Whole(&raw)) + .unwrap(); + let two = store + .put(inbox, &imap(1, 2), &[], Fetched::Whole(&raw)) + .unwrap(); + assert_eq!(store.sizes().unwrap().originals, raw.len() as u64); + store.remove(one).unwrap(); + assert!(file.exists()); + assert_eq!(found(dir.path(), "numbers"), [two.0]); + store.remove(two).unwrap(); + assert!(!file.exists()); + assert!(found(dir.path(), "numbers").is_empty()); + let err = store.remove(MessageId(two.0)).unwrap_err(); + assert!( + matches!(&err, Error::Store(text) if text.contains("no message")), + "{err}" + ); +} + +#[test] +fn a_damaged_original_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let (mut store, inbox) = inbox(dir.path()); + let raw = sample(); + let id = store + .put(inbox, &imap(1, 1), &[], Fetched::Whole(&raw)) + .unwrap(); + std::fs::write(original_path(dir.path(), &raw), b"not the message").unwrap(); + let err = store.original(id).unwrap_err(); + assert!( + matches!(&err, Error::Store(text) if text.contains("damaged")), + "{err}" + ); +} + +#[test] +fn the_search_index_rebuilds_from_the_store() { + let dir = tempfile::tempdir().unwrap(); + let (mut store, inbox) = inbox(dir.path()); + let whole = store + .put(inbox, &imap(1, 1), &[], Fetched::Whole(&sample())) + .unwrap(); + let html = b"Subject: news\r\nContent-Type: text/html\r\n\r\n\ +

Quarterlyresults

\ + "; + let page = store + .put(inbox, &imap(1, 2), &[], Fetched::Whole(html)) + .unwrap(); + assert_eq!(found(dir.path(), "results"), [page.0]); + assert!(found(dir.path(), "tracking").is_empty()); + assert!(found(dir.path(), "hidden").is_empty()); + assert!(found(dir.path(), "Quarterlyresults").is_empty()); + + Connection::open(dir.path().join("mail.db")) + .unwrap() + .execute("DELETE FROM search", []) + .unwrap(); + assert!(found(dir.path(), "numbers").is_empty()); + assert_eq!(store.rebuild_search().unwrap(), 2); + assert_eq!(found(dir.path(), "numbers"), [whole.0]); + assert_eq!(found(dir.path(), "zhang"), [whole.0]); + assert_eq!(found(dir.path(), "results"), [page.0]); +} diff --git a/crates/ely-mail-engine/tests/store_release.rs b/crates/ely-mail-engine/tests/store_release.rs new file mode 100644 index 0000000..699c8dd --- /dev/null +++ b/crates/ely-mail-engine/tests/store_release.rs @@ -0,0 +1,72 @@ +#![cfg(windows)] + +use std::{fs::File, os::windows::fs::OpenOptionsExt, path::Path}; + +use ely_mail_engine::{Fetched, Key, MailboxId, Store}; +use sha2::{Digest, Sha256}; + +const ONE: &[u8] = b"Subject: one\r\n\r\nfirst body\r\n"; +const TWO: &[u8] = b"Subject: two\r\n\r\nsecond body\r\n"; + +fn inbox(dir: &Path) -> (Store, MailboxId) { + let store = Store::open(dir).unwrap(); + let account = store.add_account("ann@example.com").unwrap(); + let inbox = store.mailbox(account, "INBOX").unwrap(); + (store, inbox) +} + +fn imap() -> Key { + Key::Imap { + uidvalidity: 1, + uid: 1, + } +} + +fn original_path(dir: &Path, raw: &[u8]) -> std::path::PathBuf { + let hash: String = Sha256::digest(raw) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + dir.join("content").join(&hash[..2]).join(hash) +} + +/// Read sharing only: delete fails. +fn hold(path: &Path) -> File { + File::options().read(true).share_mode(1).open(path).unwrap() +} + +#[test] +fn a_held_original_keeps_its_message_for_retry() { + let dir = tempfile::tempdir().unwrap(); + let (mut store, inbox) = inbox(dir.path()); + let id = store.put(inbox, &imap(), &[], Fetched::Whole(ONE)).unwrap(); + let path = original_path(dir.path(), ONE); + let held = hold(&path); + assert!(store.remove(id).is_err()); + assert_eq!(store.get(id).unwrap().subject.as_deref(), Some("one")); + assert_eq!(store.original(id).unwrap().as_deref(), Some(ONE)); + drop(held); + store.remove(id).unwrap(); + assert!(store.get(id).is_err()); + assert!(!path.exists()); + assert_eq!(store.sizes().unwrap().originals, 0); +} + +#[test] +fn a_held_old_original_keeps_the_old_message() { + let dir = tempfile::tempdir().unwrap(); + let (mut store, inbox) = inbox(dir.path()); + let id = store.put(inbox, &imap(), &[], Fetched::Whole(ONE)).unwrap(); + let old = original_path(dir.path(), ONE); + let new = original_path(dir.path(), TWO); + let held = hold(&old); + assert!(store.put(inbox, &imap(), &[], Fetched::Whole(TWO)).is_err()); + assert_eq!(store.get(id).unwrap().subject.as_deref(), Some("one")); + assert_eq!(store.original(id).unwrap().as_deref(), Some(ONE)); + assert!(!new.exists()); + drop(held); + store.put(inbox, &imap(), &[], Fetched::Whole(TWO)).unwrap(); + assert_eq!(store.original(id).unwrap().as_deref(), Some(TWO)); + assert!(!old.exists()); + assert_eq!(store.sizes().unwrap().originals, TWO.len() as u64); +} diff --git a/crates/ely-mail-engine/tests/store_repair.rs b/crates/ely-mail-engine/tests/store_repair.rs new file mode 100644 index 0000000..a8076dc --- /dev/null +++ b/crates/ely-mail-engine/tests/store_repair.rs @@ -0,0 +1,238 @@ +use std::path::Path; + +use ely_mail_engine::{Error, Fetched, Key, MailboxId, MessageId, Store}; +use rusqlite::Connection; +use sha2::{Digest, Sha256}; + +fn sample() -> Vec { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../tests/fixtures/mail/sample.eml" + ); + let text = std::fs::read_to_string(path).unwrap(); + text.replace("\r\n", "\n") + .replace('\n', "\r\n") + .into_bytes() +} + +fn inbox(dir: &Path) -> (Store, MailboxId) { + let store = Store::open(dir).unwrap(); + let account = store.add_account("ann@example.com").unwrap(); + let inbox = store.mailbox(account, "INBOX").unwrap(); + (store, inbox) +} + +fn imap(uid: u32) -> Key { + Key::Imap { + uidvalidity: 1, + uid, + } +} + +fn search(dir: &Path, phrase: &str) -> rusqlite::Result> { + let conn = Connection::open(dir.join("mail.db"))?; + let mut rows = conn.prepare("SELECT rowid FROM search WHERE search MATCH ?1")?; + rows.query_map([format!("\"{phrase}\"")], |row| row.get(0))? + .collect() +} + +fn found(dir: &Path, phrase: &str) -> Vec { + search(dir, phrase).unwrap() +} + +fn original_path(dir: &Path, raw: &[u8]) -> std::path::PathBuf { + let hash: String = Sha256::digest(raw) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + dir.join("content").join(&hash[..2]).join(hash) +} + +#[test] +fn a_corrupt_search_index_rebuilds() { + let dir = tempfile::tempdir().unwrap(); + let (mut store, inbox) = inbox(dir.path()); + let id = store + .put(inbox, &imap(1), &[], Fetched::Whole(&sample())) + .unwrap(); + let damaged = Connection::open(dir.path().join("mail.db")) + .unwrap() + .execute("UPDATE search_data SET block = X'00' WHERE id > 10", []) + .unwrap(); + assert!(damaged > 0); + assert!(search(dir.path(), "numbers").is_err()); + assert_eq!(store.rebuild_search().unwrap(), 1); + assert_eq!(found(dir.path(), "numbers"), [id.0]); + drop(store); + Store::open(dir.path()).unwrap(); + assert_eq!(found(dir.path(), "numbers"), [id.0]); +} + +#[test] +fn a_damaged_original_is_rewritten_by_a_new_copy() { + let dir = tempfile::tempdir().unwrap(); + let (mut store, inbox) = inbox(dir.path()); + let raw = sample(); + let id = store + .put(inbox, &imap(1), &[], Fetched::Whole(&raw)) + .unwrap(); + std::fs::write(original_path(dir.path(), &raw), b"damaged").unwrap(); + store + .put(inbox, &imap(2), &[], Fetched::Whole(&raw)) + .unwrap(); + assert_eq!(store.original(id).unwrap().as_deref(), Some(raw.as_slice())); + + std::fs::write(original_path(dir.path(), &raw), b"damaged").unwrap(); + assert_eq!( + store + .put(inbox, &imap(1), &[], Fetched::Whole(&raw)) + .unwrap(), + id + ); + assert_eq!(store.original(id).unwrap().as_deref(), Some(raw.as_slice())); +} + +#[test] +fn a_failed_write_keeps_no_new_original() { + let dir = tempfile::tempdir().unwrap(); + let (mut store, inbox) = inbox(dir.path()); + let raw = sample(); + let nowhere = MailboxId(999); + assert!( + store + .put(nowhere, &imap(1), &[], Fetched::Whole(&raw)) + .is_err() + ); + assert_eq!(store.sizes().unwrap().originals, 0); + + let id = store + .put(inbox, &imap(1), &[], Fetched::Whole(&raw)) + .unwrap(); + assert!( + store + .put(nowhere, &imap(2), &[], Fetched::Whole(&raw)) + .is_err() + ); + assert_eq!(store.original(id).unwrap().as_deref(), Some(raw.as_slice())); +} + +#[test] +fn sizes_count_the_write_ahead_log() { + let dir = tempfile::tempdir().unwrap(); + let (mut store, inbox) = inbox(dir.path()); + let id = store + .put(inbox, &imap(1), &[], Fetched::Whole(&sample())) + .unwrap(); + for turn in 0..600 { + let flag = if turn % 2 == 0 { "\\Seen" } else { "\\Flagged" }; + store.set_flags(id, &[flag]).unwrap(); + } + let disk: u64 = ["mail.db", "mail.db-wal", "mail.db-shm"] + .iter() + .map(|name| std::fs::metadata(dir.path().join(name)).unwrap().len()) + .sum(); + let sizes = store.sizes().unwrap(); + assert!(disk > 1_000_000, "{disk}"); + assert_eq!(sizes.index + sizes.search, disk); +} + +#[test] +fn flags_for_a_missing_message_fail() { + let dir = tempfile::tempdir().unwrap(); + let (mut store, _) = inbox(dir.path()); + let err = store.set_flags(MessageId(5), &[]).unwrap_err(); + assert!( + matches!(&err, Error::Store(text) if text.contains("no message")), + "{err}" + ); +} + +#[test] +fn html_text_follows_the_parser_states() { + let dir = tempfile::tempdir().unwrap(); + let (mut store, inbox) = inbox(dir.path()); + let html = b"Subject: page\r\nContent-Type: text/html\r\n\r\n\ + headline\ + \ + <b>literalmarkup</b>\ + vectorlabel\ + \ + \ + annotationcode()\ +

closingword

"; + let id = store + .put(inbox, &imap(1), &[], Fetched::Whole(html)) + .unwrap(); + for round in 0..2 { + for shown in [ + "ordinarycontent", + "literalmarkup", + "vectorlabel", + "` 中的斜杠不会关闭 HTML 的 script 元素，但此处以 `self_closing` 跳过隐藏，搜索能命中 `trackingpixel`；`` 中应显示的普通文字反而被当脚本丢弃，搜索无结果。修复方向：按 HTML 解析语义处理非空元素及 RCDATA/raw-text 状态，覆盖这两种输入与重建后的搜索结果。证据：`.work/C12.3-r1/probes.log`。 + +- crates/ely-mail-engine/src/store/schema.rs:1、crates/ely-mail-engine/src/store/messages.rs:68 — 两条新增注释均为 12 个空白分隔词，超过 `agents.md` 的注释十词上限。修复方向：将这两条注释各缩至十词以内。 diff --git a/tasks/reviews/C12.3-r2.md b/tasks/reviews/C12.3-r2.md new file mode 100644 index 0000000..4eb55e7 --- /dev/null +++ b/tasks/reviews/C12.3-r2.md @@ -0,0 +1,5 @@ +FAIL + +- crates/ely-mail-engine/src/store/search.rs:208 — 第 1 轮的 HTML 解析问题仍未从根因修复：`foreign` 只计未闭合的 svg/math，无法识别 HTML 整合点及退出外来内容的标签。在 `` 内，`` 的脚本被索引，`` 应显示的文字却未被索引；`put` 与 `rebuild_search` 后均如此。html5ever 的树构建器实测把这里的 script、textarea 建为 HTML 元素，并把后者内部的标签当普通文字。修复方向：依据树构建器的当前命名空间、整合点及插入模式决定取文字和分词状态，覆盖这两种输入及重建。证据：`.work/C12.3-r2/probe.log`、`.work/C12.3-r2/html-oracle.log`。 + +- crates/ely-mail-engine/src/store/messages.rs:242 — 暂存原文写入或改名失败时，没有删除 `.part` 文件。Windows 上先保存一封 35 字节原文，再把内容文件损坏并设为只读，重新收取时暂存写入成功、改名报拒绝访问；恢复文件权限后删除该邮件，`sizes().originals` 仍为 35，全部消息已删除却留下完整暂存原文，且没有回收路径。第 1 轮新增的数据库失败清理覆盖不到 `keep` 内部失败。修复方向：在暂存写入或发布失败时清理本次暂存文件，处理并暴露清理错误，补上发布失败且删除最后引用后无残留的回归测试。证据：`.work/C12.3-r2/probe.log`、`.work/C12.3-r2/probe-source.log`。 diff --git a/tasks/reviews/C12.3-r3.md b/tasks/reviews/C12.3-r3.md new file mode 100644 index 0000000..20cb9d3 --- /dev/null +++ b/tasks/reviews/C12.3-r3.md @@ -0,0 +1,5 @@ +FAIL + +- crates/ely-mail-engine/src/store/messages.rs:207 — 删除消息先提交数据库删除，再删除最后一份原文；原文删除失败后失去可重试的引用，留下永久残留。Windows 上存入一封 35 字节邮件，用 `OpenOptionsExt::share_mode(1)` 打开原文并保持句柄，再调用 `remove`：返回文件占用错误，但 `get` 已报无行；关闭句柄后重试 `remove` 只报 `no message 1`，`sizes().originals` 仍为 35。现有测试只覆盖删除成功。修复方向：协调消息删除与内容清理，失败时保留或恢复可重试的引用，或持久记录待清理内容；增加文件占用、解除后重试且无残留的回归测试。证据：`.work/C12.3-r3/remove-probe.log`、`.work/C12.3-r3/probe-source.log`。 + +- crates/ely-mail-engine/src/store/html.rs:14、crates/ely-mail-engine/src/store/html.rs:108 — 新增的两条注释分别为 12 和 11 个空白分隔词，超过 `agents.md` 的十词上限。修复方向：分别缩至十词以内。