C12.3: ElyMail local store
SQLite index, content-addressed originals, rebuildable FTS5 search, schema versions with a backup before each upgrade and read-only on failure, and size reporting. Three review rounds; the round 3 fixes are checked by tests, not reviewed again. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
ZacharyZhang-NYandClaude Opus 5.5 committed
1 parent
fe90221c11
commit
7c5bf48589
20 files changed
+1767
-3
No files matched your search
Generated
+80
@@ -2136,6 +2136,7 @@ dependencies = [
|
||||
"base64 0.23.1",
|
||||
"ely-platform",
|
||||
"getrandom 0.4.3",
|
||||
"html5ever",
|
||||
"imap-proto",
|
||||
"log",
|
||||
"mail-builder",
|
||||
@@ -2143,8 +2144,12 @@ dependencies = [
|
||||
"mail-send",
|
||||
"nom 8.0.0",
|
||||
"rcgen",
|
||||
"rusqlite",
|
||||
"rustls",
|
||||
"rustls-platform-verifier",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"smtp-proto",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
@@ -2554,6 +2559,18 @@ dependencies = [
|
||||
"zune-inflate",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
|
||||
|
||||
[[package]]
|
||||
name = "fallible-streaming-iterator"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
||||
|
||||
[[package]]
|
||||
name = "fancy-regex"
|
||||
version = "0.14.0"
|
||||
@@ -3610,6 +3627,15 @@ dependencies = [
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
version = "0.12.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb"
|
||||
dependencies = [
|
||||
"hashbrown 0.17.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heapless"
|
||||
version = "0.9.3"
|
||||
@@ -4386,6 +4412,17 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libsqlite3-sys"
|
||||
version = "0.38.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linebender_resource_handle"
|
||||
version = "0.1.1"
|
||||
@@ -6673,6 +6710,31 @@ dependencies = [
|
||||
"lru",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rsqlite-vfs"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c"
|
||||
dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusqlite"
|
||||
version = "0.40.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"fallible-iterator",
|
||||
"fallible-streaming-iterator",
|
||||
"hashlink",
|
||||
"libsqlite3-sys",
|
||||
"smallvec",
|
||||
"sqlite-wasm-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust-embed"
|
||||
version = "8.12.0"
|
||||
@@ -7403,6 +7465,18 @@ dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlite-wasm-rs"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"js-sys",
|
||||
"rsqlite-vfs",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.1"
|
||||
@@ -8431,6 +8505,12 @@ dependencies = [
|
||||
"sval_serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
|
||||
@@ -39,6 +39,8 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg",
|
||||
semver = { version = "1", features = ["serde"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
rusqlite = { version = "0.40", features = ["bundled"] }
|
||||
html5ever = "0.39"
|
||||
sha2 = "0.10"
|
||||
tempfile = "3"
|
||||
ureq = "3"
|
||||
|
||||
@@ -320,4 +320,4 @@ cargo run -p ely-doc 启动一个应用（ElyDoc 先 cargo build -p ely
|
||||
- 2026-10-18：Word 公式（R8.2，Gitea #9）。附件 math.docx 的六类公式逐类核对（本机无 Word，以文件所写为准）：键盘输入、OMML（内核转为公式对象）、MathType（OLE，显示随带的 WMF 预览）、上下标、图片原本就对；差别在两处根因。其一，运行的 `w:position`（升降）原先只给文字，图片与 OLE 公式所在的运行失去它（25 张图片与一个 ½ 浮在行上）：补丁 `0049-writer-inline-objects-move-with-their-run.patch` 把它作为固定升降写到对象的锚字符上（按锚字符自己的字号换算百分比，不扩展到其后的文字），版面让“作为字符”的对象随锚字符的固定升降移动（vertAlign 的自动上下标不动对象，同原样）；各导出本就写出锚字符的升降，DOCX 往返仍为 `w:position`。其二，EQ 域原先只认 `\o`、`\s\do` 与注音，其余（如分数 `\f`）整个消失：补丁 `0050-writer-eq-field-parser.patch`（读法）与 `0051-writer-eq-fields-as-formulas.patch`（导入）把 EQ 域转为公式对象：`\f` 分数、`\r` 根式、`\s` 上下标、`\b` 括号、`\l` 列表、`\a` 阵列、`\i` 积分与求和求积；`\x` 与 `\o` 只留内容（无框、无叠印），`\d` 只前移，`\ai`、`\di` 只是行距（不升降），开关不分大小写；每个命令字符保留其运行的上下标（Word 照此画），同一段上下标里字样变了仍是一个上下标；斜体字母写成标识符（StarMath 与 OMML 都画成斜体，DOCX 往返不丢），斜体的数字与符号写成 `ital` 文字，存为 DOCX 后成正体（OMML 无此写法，已知限制）；没有底的上下标以空文字为底（空的底在 OMML 里是占位符，重开成 `<?>`）；公式取域的字号；读不通的命令以其域代码显示为文字，不消失。只有整个域是 `\s\do N(文字)` 时照原样成为下移的文字，其余都经读法（原先小写的 `\s\do` 读到第一个括号即止，其后丢失）。`client/tests/word.rs` 以 `tests/fixtures/office/word.docx`（由 `.work/r8/gen/eq.py` 生成，不入库）核对公式文本（含开头的空格、大写开关、读不通的域、字样变化的下标、`\ai`、`\di`、小写 `\s\do` 的尾部与嵌套）、DOCX 往返后的斜体、下标与无占位符、锚字符的升降与图块里图片低于文字（去掉版面的升降即失败）。审查三轮：第 3 轮的四处已修并以上述测试核对，未再审查（每条至多三轮）。仍与 Word 不同：编辑区里公式与图片四周有内核的辅助框线（`ShowBoundaries`，视图辅助，不印出）；DOCX 另存后重开行距变宽，因内核导出改写文档默认字体与段距，与公式无关。
|
||||
- 2026-10-18：工作表过多时标签条（R8.3，Gitea #8）。组件库 `navigation::Tabs` 横排时可横向滚动（滚轮同样），标签可窄于自然宽度；所选标签在所选或标签条宽度变了时滚入视口，在绘制时以刚排好的宽度判定（原先只在首帧滚一次，其后右栏打开、标签条变窄，所选标签被挤出视口）；手动滚动保留到二者之一再变；横排的标签至多与标签条同宽，名字过长以省略号结尾（组件库 e56a343、3a68d1b、7d704d9，同批带入上游列外壳拖动的四个提交）。ElyTable 的标签条随之收缩，添加按钮留在其旁；标签条的状态按工作簿各记一份（元素 id 带标签号），切到另一工作簿时其所选标签照样滚入。`navigation/tests/tabs.rs` 以 30 个标签核对：首帧滚到所选、滚轮回到第一个、新选的滚入、变窄后所选仍在末端（去掉按宽度判定即失败）、长名的所选标签整个在标签条内（去掉宽度上限即失败）；ElyTable 的 `tests/strip.rs` 以 `sheets::strip` 画两个各有 30 张表、都选第 30 张的工作簿，滚回开头后来回切换，所选标签每次都在标签条内（两个工作簿共用一个 id 即失败）。Windows 真窗口（`.work/r8/strip3.png`）：30 张表、第 30 张为当前，1100 宽时标签条末端为它。
|
||||
- 2026-10-18：ElyMark 公式（R8.4，用户定两项都做）。行内公式照块公式由 RaTeX 排版（原只显示着色的 TeX 原文）：gpui 的文本里放不进图片，组件库 `render/math_text.rs` 把每个行内公式换成等宽的不断行空格（U+00A0，按字体量出个数），排好的公式居中画在其上，行高随最高的公式撑开；空格串与词一样整个折到下一行，画在其所在行（折行处的序号 gpui 读作上一行末）；按下与右键所在的字节经换算读回公式的原文；排不出的公式照旧显示 TeX 原文，其后以危险色写出 RaTeX 给的原因与位置，记警告（不含公式）。`\(…\)` 读作行内公式，`\[…\]` 读作块公式（ChatGPT 等常写的分隔符；GitHub 不认，Typora 默认不认）：解析前逐字节把成对的分隔符换成 `$`（长度不变，位置照旧对得上）。成对只在同一段落、标题或单元格里（跨列表项、CR 空行不成对）；不跨代码、公式、HTML 标签、链接地址与引用标签、图片、脚注引用，`<code>`、`<kbd>` 之间（含嵌套；交叉闭合照可视的读法关闭与重开）的文字照原文；`\\(` 不换。公式里与段落里另有的 `$`（含 `\$`）解析时暂写成文本里没有的控制字节，解析后还原，不与换上的 `$` 配对；相邻的两个公式之间暂插一个空格，解析后去掉，仍是两个公式。公式节点的范围是整对分隔符，值是写下的 TeX 原文。独占一段的 `\[…\]` 为块公式（段首可有一至三个空格）。组件库公开 `documents::markdown_tree`，ElyMark 的 HTML、PDF、纯文本导出都经它解析（`markdown_options` 不再公开），纯文本里独占一段的公式照块公式写其 TeX。同时查出（组件库 e095f06）：RaTeX 的 SVG 以 pt 定尺寸，usvg 按 px 量墨迹，两者差 4/3，每个公式的画面右侧与下方多出约四分之一的空白（块公式因此偏左）、导出的公式比正文大 4/3；现生成时即把根改为 px。`markdown/tex/tests.rs` 与 `tree/tests.rs` 核对分隔符的各种情形（相邻、`\$`、链接地址、`<code>`、跨块），`math_text.rs` 的窗口测试核对公式排进并读回原文、折到第二行的公式画在第二行首（改回按起点定位即失败）、排不出的公式写出原因（组件库 a20e712、5fba7b3、de961eb），`math/tests.rs` 核对画面与公式同宽（改回 pt 即失败），ElyMark 的 `visual.rs` 核对导出与纯文本。Windows 真窗口（`.work/r8/mark/visual2.png`）：`$…$`、`\(…\)` 排在行内，`\[…\]` 居中成块，行内代码与转义的 `\\(` 照原文。
|
||||
- 2026-10-19（用户定）：ElyMail 的后端改为 Rust crate，取代 MailCore 2（其 Windows 构建是 VS 2013 工程加 2015 年前后的预编译依赖，POP 无 OAuth2）：mail-send、mail-parser、mail-builder，在应用进程的后台任务里运行，不另起 worker；MailCore 2 与 libetpan 的取码、构建与探测随之删去。正文以 html5ever 解析、净化后按块的子集由 GPUI 绘制，不当作完整页面；写信用 Markdown 文档组件，发出纯文本与 HTML。Gmail 与 Microsoft 走 OAuth（授权码加 PKCE，本机回环），客户端 ID 在构建时注入：`ELY_GOOGLE_CLIENT_ID`、`ELY_GOOGLE_CLIENT_SECRET`、`ELY_MICROSOFT_CLIENT_ID`，本机写在项目根的 `oauth.env`（不入库），CI 为仓库机密；缺 ID 的构建里该登录写明不可用，不退回别的方式。iCloud 不给第三方 OAuth，走应用专用密码向导（`imap.mail.me.com:993`、`smtp.mail.me.com:587`，无 POP3）。注册由用户做（Google Desktop 客户端，Entra 公共客户端，委托权限 IMAP、POP、SMTP 与 offline_access）。POP3（C12.1 审查后）由引擎自写，不用 async-pop：它留着点填充、丢末尾 CRLF、吞掉登录后的连接错误、不答 XOAUTH2 的错误挑战、在 trace 日志里写出密码。IMAP（第 2 轮审查后）同样自写，以 imap-proto 解析回复，不用 async-imap：它的 LIST、FETCH 等流式命令不看结尾的 NO 与 BAD（拒绝成了空结果），登录与命令中途的 `* BYE` 被丢掉（只剩断线），trace 日志写出密码与邮件。SMTP 的事务（MAIL、RCPT、DATA）也由引擎自走：mail-send 不给首行的点加点；正文有 8 位字节即带 `BODY=8BITMIME`，服务器不支持即拒绝。读信按 MIME 树与所在容器（C12.2），嵌套（多部分与内嵌邮件都算）深于 64 层即报错，超深的解析结果以迭代拆开再丢（直接丢弃会递归、栈溢出）；mail-parser 开 `full_encoding`；SMTPUTF8 按每一层的头块判定。
|
||||
- 2026-10-19（用户定）：ElyMail 的后端改为 Rust crate，取代 MailCore 2（其 Windows 构建是 VS 2013 工程加 2015 年前后的预编译依赖，POP 无 OAuth2）：mail-send、mail-parser、mail-builder，在应用进程的后台任务里运行，不另起 worker；MailCore 2 与 libetpan 的取码、构建与探测随之删去。正文以 html5ever 解析、净化后按块的子集由 GPUI 绘制，不当作完整页面；写信用 Markdown 文档组件，发出纯文本与 HTML。Gmail 与 Microsoft 走 OAuth（授权码加 PKCE，本机回环），客户端 ID 在构建时注入：`ELY_GOOGLE_CLIENT_ID`、`ELY_GOOGLE_CLIENT_SECRET`、`ELY_MICROSOFT_CLIENT_ID`，本机写在项目根的 `oauth.env`（不入库），CI 为仓库机密；缺 ID 的构建里该登录写明不可用，不退回别的方式。iCloud 不给第三方 OAuth，走应用专用密码向导（`imap.mail.me.com:993`、`smtp.mail.me.com:587`，无 POP3）。注册由用户做（Google Desktop 客户端，Entra 公共客户端，委托权限 IMAP、POP、SMTP 与 offline_access）。POP3（C12.1 审查后）由引擎自写，不用 async-pop：它留着点填充、丢末尾 CRLF、吞掉登录后的连接错误、不答 XOAUTH2 的错误挑战、在 trace 日志里写出密码。IMAP（第 2 轮审查后）同样自写，以 imap-proto 解析回复，不用 async-imap：它的 LIST、FETCH 等流式命令不看结尾的 NO 与 BAD（拒绝成了空结果），登录与命令中途的 `* BYE` 被丢掉（只剩断线），trace 日志写出密码与邮件。SMTP 的事务（MAIL、RCPT、DATA）也由引擎自走：mail-send 不给首行的点加点；正文有 8 位字节即带 `BODY=8BITMIME`，服务器不支持即拒绝。读信按 MIME 树与所在容器（C12.2），嵌套（多部分与内嵌邮件都算）深于 64 层即报错，超深的解析结果以迭代拆开再丢（直接丢弃会递归、栈溢出）；mail-parser 开 `full_encoding`；SMTPUTF8 按每一层的头块判定。本地存储（C12.3）：rusqlite 自带的 SQLite，`mail.db` 加内容目录（原文按 SHA-256），FTS5（trigram）可由索引与原文重建；`user_version` 记结构版本，升级前备份，失败即只读；原文文件在删去或替换邮件的事务提交前删去，删不掉即回滚。只有 HTML 的邮件经 html5ever 的树构建器取文字入索引，元素深过 512 层即停读（树构建器的开放元素栈扫描随深度平方增长）。
|
||||
@@ -9,6 +9,7 @@ publish.workspace = true
|
||||
[dependencies]
|
||||
base64.workspace = true
|
||||
getrandom.workspace = true
|
||||
html5ever.workspace = true
|
||||
imap-proto.workspace = true
|
||||
log.workspace = true
|
||||
mail-builder.workspace = true
|
||||
@@ -17,6 +18,10 @@ mail-send.workspace = true
|
||||
nom.workspace = true
|
||||
rustls.workspace = true
|
||||
rustls-platform-verifier.workspace = true
|
||||
rusqlite.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
sha2.workspace = true
|
||||
smtp-proto = "0.2"
|
||||
tokio.workspace = true
|
||||
tokio-rustls.workspace = true
|
||||
|
||||
@@ -21,6 +21,10 @@ pub enum Error {
|
||||
Invalid(String),
|
||||
/// A message that cannot be read or written.
|
||||
Mime(String),
|
||||
/// The local store failed: database, disk or damage.
|
||||
Store(String),
|
||||
/// The store refuses writes; the reason says why.
|
||||
ReadOnly(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for Error {
|
||||
@@ -34,8 +38,22 @@ impl fmt::Display for Error {
|
||||
Self::Timeout(step) => write!(f, "no answer while {step}"),
|
||||
Self::Invalid(text) => write!(f, "cannot be sent: {text}"),
|
||||
Self::Mime(text) => write!(f, "unreadable message: {text}"),
|
||||
Self::Store(text) => write!(f, "local store failed: {text}"),
|
||||
Self::ReadOnly(text) => write!(f, "local store is read-only: {text}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Error {}
|
||||
|
||||
impl From<rusqlite::Error> for Error {
|
||||
fn from(err: rusqlite::Error) -> Self {
|
||||
Self::Store(err.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<serde_json::Error> for Error {
|
||||
fn from(err: serde_json::Error) -> Self {
|
||||
Self::Store(format!("a stored field: {err}"))
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ mod mime;
|
||||
mod net;
|
||||
mod pop;
|
||||
mod smtp;
|
||||
mod store;
|
||||
mod wire;
|
||||
|
||||
pub use error::{Error, Result};
|
||||
@@ -14,4 +15,5 @@ pub use mime::{Built, Draft, Mail, Mailbox, Part, build, read};
|
||||
pub use net::{Login, Secret, Security, Server, system_tls};
|
||||
pub use pop::Pop;
|
||||
pub use smtp::Smtp;
|
||||
pub use store::{AccountId, Fetched, Key, MailboxId, MessageId, Sizes, Store, Stored};
|
||||
pub use tokio_rustls::TlsConnector;
|
||||
@@ -4,12 +4,13 @@ use mail_builder::{
|
||||
mime::MimePart,
|
||||
};
|
||||
use mail_parser::{Address, HeaderValue, MessageParser};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{Error, Result, smtp::mailbox};
|
||||
|
||||
mod walk;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Mailbox {
|
||||
pub name: Option<String>,
|
||||
pub email: String,
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
//! The local index, the originals and the search index.
|
||||
|
||||
use std::{
|
||||
fs, io,
|
||||
path::{Path, PathBuf},
|
||||
time::{SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use rusqlite::{Connection, OpenFlags, OptionalExtension, params};
|
||||
|
||||
use crate::{Error, Result};
|
||||
|
||||
mod html;
|
||||
mod messages;
|
||||
mod schema;
|
||||
mod search;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub use messages::{Fetched, Key, Stored};
|
||||
|
||||
const DB: &str = "mail.db";
|
||||
const CONTENT: &str = "content";
|
||||
const BACKUPS: &str = "backups";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct AccountId(pub i64);
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct MailboxId(pub i64);
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
||||
pub struct MessageId(pub i64);
|
||||
|
||||
/// Bytes on disk per kind.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct Sizes {
|
||||
pub index: u64,
|
||||
pub search: u64,
|
||||
pub originals: u64,
|
||||
}
|
||||
|
||||
pub struct Store {
|
||||
conn: Connection,
|
||||
dir: PathBuf,
|
||||
/// Why writes are refused, when they are.
|
||||
read_only: Option<String>,
|
||||
}
|
||||
|
||||
impl Store {
|
||||
pub fn open(dir: &Path) -> Result<Self> {
|
||||
open(dir, schema::STEPS)
|
||||
}
|
||||
|
||||
pub fn read_only(&self) -> Option<&str> {
|
||||
self.read_only.as_deref()
|
||||
}
|
||||
|
||||
fn writable(&self) -> Result<()> {
|
||||
match &self.read_only {
|
||||
Some(reason) => Err(Error::ReadOnly(reason.clone())),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_account(&self, address: &str) -> Result<AccountId> {
|
||||
self.writable()?;
|
||||
self.conn
|
||||
.execute("INSERT INTO accounts (address) VALUES (?1)", [address])?;
|
||||
Ok(AccountId(self.conn.last_insert_rowid()))
|
||||
}
|
||||
|
||||
pub fn accounts(&self) -> Result<Vec<(AccountId, String)>> {
|
||||
let mut query = self
|
||||
.conn
|
||||
.prepare("SELECT id, address FROM accounts ORDER BY id")?;
|
||||
let rows = query.query_map([], |row| Ok((AccountId(row.get(0)?), row.get(1)?)))?;
|
||||
Ok(rows.collect::<rusqlite::Result<_>>()?)
|
||||
}
|
||||
|
||||
/// The mailbox by name, made on first use.
|
||||
pub fn mailbox(&self, account: AccountId, name: &str) -> Result<MailboxId> {
|
||||
let find = "SELECT id FROM mailboxes WHERE account = ?1 AND name = ?2";
|
||||
if let Some(id) = self
|
||||
.conn
|
||||
.query_row(find, params![account.0, name], |row| row.get(0))
|
||||
.optional()?
|
||||
{
|
||||
return Ok(MailboxId(id));
|
||||
}
|
||||
self.writable()?;
|
||||
self.conn.execute(
|
||||
"INSERT INTO mailboxes (account, name) VALUES (?1, ?2)",
|
||||
params![account.0, name],
|
||||
)?;
|
||||
Ok(MailboxId(self.conn.last_insert_rowid()))
|
||||
}
|
||||
|
||||
/// The index counts the WAL and shared-memory files.
|
||||
pub fn sizes(&self) -> Result<Sizes> {
|
||||
let mut files = 0;
|
||||
for name in [DB.to_owned(), format!("{DB}-wal"), format!("{DB}-shm")] {
|
||||
files += match fs::metadata(self.dir.join(&name)) {
|
||||
Ok(meta) => meta.len(),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => 0,
|
||||
Err(err) => return Err(disk("measuring the index", err)),
|
||||
};
|
||||
}
|
||||
let search: i64 = self.conn.query_row(
|
||||
"SELECT coalesce(sum(pgsize), 0) FROM dbstat
|
||||
WHERE name = 'search' OR name GLOB 'search_*'",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
let search = u64::try_from(search)
|
||||
.map_err(|_| Error::Store(format!("a negative size: {search}")))?;
|
||||
let index = files.checked_sub(search).ok_or_else(|| {
|
||||
Error::Store(format!("search pages {search} exceed the files {files}"))
|
||||
})?;
|
||||
Ok(Sizes {
|
||||
index,
|
||||
search,
|
||||
originals: folder_size(&self.dir.join(CONTENT))?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn open(dir: &Path, steps: &[&str]) -> Result<Store> {
|
||||
let path = dir.join(DB);
|
||||
fs::create_dir_all(dir.join(CONTENT)).map_err(|err| disk("making the store", err))?;
|
||||
let mut conn = Connection::open(&path)?;
|
||||
let version: i64 = conn.query_row("PRAGMA user_version", [], |row| row.get(0))?;
|
||||
let version = usize::try_from(version)
|
||||
.map_err(|_| Error::Store(format!("schema version {version} is negative")))?;
|
||||
let target = steps.len();
|
||||
let refused = if version > target {
|
||||
Some(format!(
|
||||
"written by a newer ElyMail (schema v{version}, this one reads v{target})"
|
||||
))
|
||||
} else if version < target {
|
||||
upgrade(&mut conn, dir, version, steps).err()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(reason) = refused {
|
||||
log::warn!("mail: store opened read-only: {reason}");
|
||||
drop(conn);
|
||||
let flags = OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX;
|
||||
let conn = Connection::open_with_flags(&path, flags)?;
|
||||
return Ok(Store {
|
||||
conn,
|
||||
dir: dir.to_owned(),
|
||||
read_only: Some(reason),
|
||||
});
|
||||
}
|
||||
conn.pragma_update(None, "foreign_keys", true)?;
|
||||
conn.pragma_update(None, "journal_mode", "WAL")?;
|
||||
log::info!("mail: store opened at schema v{target}");
|
||||
Ok(Store {
|
||||
conn,
|
||||
dir: dir.to_owned(),
|
||||
read_only: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// A backup comes first; a failure keeps the old data.
|
||||
fn upgrade(
|
||||
conn: &mut Connection,
|
||||
dir: &Path,
|
||||
version: usize,
|
||||
steps: &[&str],
|
||||
) -> std::result::Result<(), String> {
|
||||
if version > 0 {
|
||||
backup(conn, dir, version).map_err(|err| format!("no backup before upgrade: {err}"))?;
|
||||
}
|
||||
migrate(conn, version, steps).map_err(|err| {
|
||||
format!(
|
||||
"upgrade from schema v{version} to v{} failed: {err}",
|
||||
steps.len()
|
||||
)
|
||||
})?;
|
||||
log::info!("mail: store schema v{version} upgraded to v{}", steps.len());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate(conn: &mut Connection, version: usize, steps: &[&str]) -> rusqlite::Result<()> {
|
||||
let tx = conn.transaction()?;
|
||||
for step in &steps[version..] {
|
||||
tx.execute_batch(step)?;
|
||||
}
|
||||
tx.pragma_update(None, "user_version", steps.len() as i64)?;
|
||||
tx.commit()
|
||||
}
|
||||
|
||||
fn backup(conn: &Connection, dir: &Path, version: usize) -> Result<()> {
|
||||
let folder = dir.join(BACKUPS);
|
||||
fs::create_dir_all(&folder).map_err(|err| disk("making the backup folder", err))?;
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_err(|err| Error::Store(format!("the clock is before 1970: {err}")))?
|
||||
.as_secs();
|
||||
let target = folder.join(format!("mail-v{version}-{now}.db"));
|
||||
let target = target
|
||||
.to_str()
|
||||
.ok_or_else(|| Error::Store("the backup path is not UTF-8".into()))?;
|
||||
conn.execute("VACUUM INTO ?1", [target])?;
|
||||
log::info!("mail: store schema v{version} backed up before upgrade");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Two levels: `content/ab/abcd…`.
|
||||
fn folder_size(folder: &Path) -> Result<u64> {
|
||||
let mut total = 0;
|
||||
for shard in fs::read_dir(folder).map_err(|err| disk("reading the originals", err))? {
|
||||
let shard = shard.map_err(|err| disk("reading the originals", err))?;
|
||||
for file in fs::read_dir(shard.path()).map_err(|err| disk("reading the originals", err))? {
|
||||
let meta = file
|
||||
.and_then(|file| file.metadata())
|
||||
.map_err(|err| disk("reading the originals", err))?;
|
||||
total += meta.len();
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
fn disk(step: &str, err: io::Error) -> Error {
|
||||
Error::Store(format!("{step}: {err}"))
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
use std::borrow::Cow;
|
||||
use std::cell::{Cell, RefCell};
|
||||
|
||||
use html5ever::{
|
||||
Attribute, LocalName, Namespace, ParseOpts, QualName, local_name, parse_document,
|
||||
tendril::{StrTendril, TendrilSink},
|
||||
tree_builder::{ElemName, ElementFlags, NodeOrText, QuirksMode, TreeBuilderOpts, TreeSink},
|
||||
};
|
||||
|
||||
/// Deeper trees stall html5ever.
|
||||
const DEPTH: usize = 512;
|
||||
const CHUNK: usize = 4096;
|
||||
|
||||
/// Visible HTML text, and whether it stopped short.
|
||||
pub(super) fn text(html: &str) -> (String, bool) {
|
||||
let opts = ParseOpts {
|
||||
tree_builder: TreeBuilderOpts {
|
||||
// Mail never runs scripts.
|
||||
scripting_enabled: false,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
let mut parser = parse_document(Tree::default(), opts);
|
||||
let mut rest = html;
|
||||
let mut short = false;
|
||||
while !rest.is_empty() {
|
||||
if parser.tokenizer.sink.sink.deep.get() {
|
||||
short = true;
|
||||
break;
|
||||
}
|
||||
let mut cut = rest.len().min(CHUNK);
|
||||
while !rest.is_char_boundary(cut) {
|
||||
cut += 1;
|
||||
}
|
||||
let (head, tail) = rest.split_at(cut);
|
||||
parser.process(StrTendril::from_slice(head));
|
||||
rest = tail;
|
||||
}
|
||||
(parser.finish().text(), short)
|
||||
}
|
||||
|
||||
enum Data {
|
||||
Document,
|
||||
Element { name: QualName, point: bool },
|
||||
Text(String),
|
||||
Other,
|
||||
}
|
||||
|
||||
struct Node {
|
||||
parent: Option<usize>,
|
||||
children: Vec<usize>,
|
||||
data: Data,
|
||||
}
|
||||
|
||||
/// The tree html5ever's builder makes.
|
||||
struct Tree {
|
||||
nodes: RefCell<Vec<Node>>,
|
||||
/// An element went past `DEPTH`.
|
||||
deep: Cell<bool>,
|
||||
}
|
||||
|
||||
impl Default for Tree {
|
||||
fn default() -> Self {
|
||||
let document = Node {
|
||||
parent: None,
|
||||
children: Vec::new(),
|
||||
data: Data::Document,
|
||||
};
|
||||
Tree {
|
||||
nodes: RefCell::new(vec![document]),
|
||||
deep: Cell::new(false),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(super) struct Name(QualName);
|
||||
|
||||
impl ElemName for Name {
|
||||
fn ns(&self) -> &Namespace {
|
||||
&self.0.ns
|
||||
}
|
||||
|
||||
fn local_name(&self) -> &LocalName {
|
||||
&self.0.local
|
||||
}
|
||||
}
|
||||
|
||||
impl Tree {
|
||||
fn add(&self, data: Data) -> usize {
|
||||
let mut nodes = self.nodes.borrow_mut();
|
||||
nodes.push(Node {
|
||||
parent: None,
|
||||
children: Vec::new(),
|
||||
data,
|
||||
});
|
||||
nodes.len() - 1
|
||||
}
|
||||
|
||||
fn detach(&self, node: usize) {
|
||||
let mut nodes = self.nodes.borrow_mut();
|
||||
if let Some(parent) = nodes[node].parent.take() {
|
||||
nodes[parent].children.retain(|&child| child != node);
|
||||
}
|
||||
}
|
||||
|
||||
/// Inserts a child at index `at`.
|
||||
fn insert(&self, parent: usize, at: usize, child: NodeOrText<usize>) {
|
||||
let node = match child {
|
||||
NodeOrText::AppendText(text) => {
|
||||
let mut nodes = self.nodes.borrow_mut();
|
||||
let before = at.checked_sub(1).map(|i| nodes[parent].children[i]);
|
||||
if let Some(Data::Text(held)) = before.map(|i| &mut nodes[i].data) {
|
||||
held.push_str(&text);
|
||||
return;
|
||||
}
|
||||
drop(nodes);
|
||||
self.add(Data::Text(text.to_string()))
|
||||
}
|
||||
NodeOrText::AppendNode(node) => {
|
||||
self.detach(node);
|
||||
node
|
||||
}
|
||||
};
|
||||
let mut nodes = self.nodes.borrow_mut();
|
||||
let at = at.min(nodes[parent].children.len());
|
||||
nodes[parent].children.insert(at, node);
|
||||
nodes[node].parent = Some(parent);
|
||||
if matches!(nodes[node].data, Data::Element { .. }) {
|
||||
let mut up = Some(parent);
|
||||
let mut depth = 0;
|
||||
while let Some(next) = up.filter(|_| depth <= DEPTH) {
|
||||
depth += 1;
|
||||
up = nodes[next].parent;
|
||||
}
|
||||
if depth > DEPTH {
|
||||
self.deep.set(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The visible text, read without recursion.
|
||||
fn text(&self) -> String {
|
||||
let nodes = self.nodes.borrow();
|
||||
let mut out = String::new();
|
||||
let mut stack = vec![0];
|
||||
while let Some(at) = stack.pop() {
|
||||
match &nodes[at].data {
|
||||
Data::Text(text) => out.push_str(text),
|
||||
Data::Element { name, .. } if hidden(&name.local) => out.push(' '),
|
||||
Data::Element { .. } | Data::Document => {
|
||||
// Tags separate words.
|
||||
out.push(' ');
|
||||
stack.extend(nodes[at].children.iter().rev());
|
||||
}
|
||||
Data::Other => {}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Not drawn, per the HTML rendering rules.
|
||||
fn hidden(name: &LocalName) -> bool {
|
||||
matches!(
|
||||
*name,
|
||||
local_name!("area")
|
||||
| local_name!("base")
|
||||
| local_name!("basefont")
|
||||
| local_name!("datalist")
|
||||
| local_name!("head")
|
||||
| local_name!("iframe")
|
||||
| local_name!("link")
|
||||
| local_name!("meta")
|
||||
| local_name!("noembed")
|
||||
| local_name!("noframes")
|
||||
| local_name!("param")
|
||||
| local_name!("rp")
|
||||
| local_name!("script")
|
||||
| local_name!("style")
|
||||
| local_name!("template")
|
||||
| local_name!("title")
|
||||
)
|
||||
}
|
||||
|
||||
impl TreeSink for Tree {
|
||||
type Handle = usize;
|
||||
type Output = Self;
|
||||
type ElemName<'a> = Name;
|
||||
|
||||
fn finish(self) -> Self {
|
||||
self
|
||||
}
|
||||
|
||||
fn parse_error(&self, _msg: Cow<'static, str>) {}
|
||||
|
||||
fn get_document(&self) -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
fn elem_name(&self, target: &usize) -> Name {
|
||||
match &self.nodes.borrow()[*target].data {
|
||||
Data::Element { name, .. } => Name(name.clone()),
|
||||
_ => unreachable!("html5ever asked a non-element for its name"),
|
||||
}
|
||||
}
|
||||
|
||||
fn create_element(&self, name: QualName, _: Vec<Attribute>, flags: ElementFlags) -> usize {
|
||||
let point = flags.mathml_annotation_xml_integration_point;
|
||||
let node = self.add(Data::Element { name, point });
|
||||
if flags.template {
|
||||
// Template contents stay out of the tree.
|
||||
self.add(Data::Other);
|
||||
}
|
||||
node
|
||||
}
|
||||
|
||||
fn create_comment(&self, _text: StrTendril) -> usize {
|
||||
self.add(Data::Other)
|
||||
}
|
||||
|
||||
fn create_pi(&self, _target: StrTendril, _data: StrTendril) -> usize {
|
||||
self.add(Data::Other)
|
||||
}
|
||||
|
||||
fn append(&self, parent: &usize, child: NodeOrText<usize>) {
|
||||
let end = self.nodes.borrow()[*parent].children.len();
|
||||
self.insert(*parent, end, child);
|
||||
}
|
||||
|
||||
fn append_based_on_parent_node(
|
||||
&self,
|
||||
element: &usize,
|
||||
prev_element: &usize,
|
||||
child: NodeOrText<usize>,
|
||||
) {
|
||||
if self.nodes.borrow()[*element].parent.is_some() {
|
||||
self.append_before_sibling(element, child);
|
||||
} else {
|
||||
self.append(prev_element, child);
|
||||
}
|
||||
}
|
||||
|
||||
fn append_doctype_to_document(&self, _: StrTendril, _: StrTendril, _: StrTendril) {}
|
||||
|
||||
fn get_template_contents(&self, target: &usize) -> usize {
|
||||
target + 1
|
||||
}
|
||||
|
||||
fn same_node(&self, x: &usize, y: &usize) -> bool {
|
||||
x == y
|
||||
}
|
||||
|
||||
fn set_quirks_mode(&self, _mode: QuirksMode) {}
|
||||
|
||||
fn append_before_sibling(&self, sibling: &usize, child: NodeOrText<usize>) {
|
||||
let (parent, at) = {
|
||||
let nodes = self.nodes.borrow();
|
||||
let parent = nodes[*sibling]
|
||||
.parent
|
||||
.expect("html5ever names a sibling with a parent");
|
||||
let at = nodes[parent].children.iter().position(|c| c == sibling);
|
||||
(parent, at.expect("a sibling among its parent's children"))
|
||||
};
|
||||
self.insert(parent, at, child);
|
||||
}
|
||||
|
||||
fn add_attrs_if_missing(&self, _target: &usize, _attrs: Vec<Attribute>) {}
|
||||
|
||||
fn remove_from_parent(&self, target: &usize) {
|
||||
self.detach(*target);
|
||||
}
|
||||
|
||||
fn reparent_children(&self, node: &usize, new_parent: &usize) {
|
||||
let children = std::mem::take(&mut self.nodes.borrow_mut()[*node].children);
|
||||
for child in children {
|
||||
self.nodes.borrow_mut()[child].parent = None;
|
||||
self.append(new_parent, NodeOrText::AppendNode(child));
|
||||
}
|
||||
}
|
||||
|
||||
fn is_mathml_annotation_xml_integration_point(&self, handle: &usize) -> bool {
|
||||
matches!(
|
||||
self.nodes.borrow()[*handle].data,
|
||||
Data::Element { point: true, .. }
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
use std::{fs, io::ErrorKind, path::Path};
|
||||
|
||||
use rusqlite::{Connection, OptionalExtension, Row, ToSql, Transaction, params};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::{CONTENT, MailboxId, MessageId, Store, disk, search};
|
||||
use crate::{Error, Mail, Mailbox, Result, read};
|
||||
|
||||
/// RFC 3501 §2.3.1.1: identity is mailbox, UIDVALIDITY and UID.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Key {
|
||||
Imap {
|
||||
uidvalidity: u32,
|
||||
uid: u32,
|
||||
},
|
||||
/// POP3 has one mailbox and UIDL names.
|
||||
Pop {
|
||||
uidl: String,
|
||||
},
|
||||
}
|
||||
|
||||
/// What a sync brought down.
|
||||
pub enum Fetched<'a> {
|
||||
Headers(&'a [u8]),
|
||||
Whole(&'a [u8]),
|
||||
}
|
||||
|
||||
/// A message as indexed; the original is read apart.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Stored {
|
||||
pub id: MessageId,
|
||||
pub mailbox: MailboxId,
|
||||
pub key: Key,
|
||||
pub flags: Vec<String>,
|
||||
pub message_id: Option<String>,
|
||||
pub in_reply_to: Vec<String>,
|
||||
pub references: Vec<String>,
|
||||
pub subject: Option<String>,
|
||||
pub from: Vec<Mailbox>,
|
||||
pub to: Vec<Mailbox>,
|
||||
pub cc: Vec<Mailbox>,
|
||||
pub date: Option<i64>,
|
||||
/// None until the whole message is here.
|
||||
pub attachments: Option<u32>,
|
||||
pub whole: bool,
|
||||
}
|
||||
|
||||
const COLUMNS: &str = "id, mailbox, uidvalidity, uid, uidl, message_id, in_reply_to, refs, \
|
||||
subject, sender, recipients, copies, date, attachments, content";
|
||||
|
||||
impl Store {
|
||||
pub fn find(&self, mailbox: MailboxId, key: &Key) -> Result<Option<MessageId>> {
|
||||
let found = match key {
|
||||
Key::Imap { uidvalidity, uid } => self.conn.query_row(
|
||||
"SELECT id FROM messages WHERE mailbox = ?1 AND uidvalidity = ?2 AND uid = ?3",
|
||||
params![mailbox.0, uidvalidity, uid],
|
||||
|row| row.get(0),
|
||||
),
|
||||
Key::Pop { uidl } => self.conn.query_row(
|
||||
"SELECT id FROM messages WHERE mailbox = ?1 AND uidl = ?2",
|
||||
params![mailbox.0, uidl],
|
||||
|row| row.get(0),
|
||||
),
|
||||
};
|
||||
Ok(found.optional()?.map(MessageId))
|
||||
}
|
||||
|
||||
/// One key, one message; headers again only set flags.
|
||||
pub fn put(
|
||||
&mut self,
|
||||
mailbox: MailboxId,
|
||||
key: &Key,
|
||||
flags: &[&str],
|
||||
fetched: Fetched,
|
||||
) -> Result<MessageId> {
|
||||
self.writable()?;
|
||||
let existing = self.find(mailbox, key)?;
|
||||
let bytes = match (fetched, existing) {
|
||||
(Fetched::Headers(_), Some(id)) => {
|
||||
self.set_flags(id, flags)?;
|
||||
return Ok(id);
|
||||
}
|
||||
(Fetched::Headers(bytes), None) => {
|
||||
let mail = read(bytes)?;
|
||||
return self.write(mailbox, key, flags, &mail, None, None);
|
||||
}
|
||||
(Fetched::Whole(bytes), _) => bytes,
|
||||
};
|
||||
let mail = read(bytes)?;
|
||||
let (content, made) = self.keep(bytes)?;
|
||||
let written = self.write(mailbox, key, flags, &mail, Some(&content), existing);
|
||||
if written.is_err()
|
||||
&& made
|
||||
&& let Err(gone) = drop_unused(&self.conn, &self.dir, &content)
|
||||
{
|
||||
log::error!("mail: an unused original stays after a failed write: {gone}");
|
||||
}
|
||||
written
|
||||
}
|
||||
|
||||
fn write(
|
||||
&mut self,
|
||||
mailbox: MailboxId,
|
||||
key: &Key,
|
||||
flags: &[&str],
|
||||
mail: &Mail,
|
||||
content: Option<&str>,
|
||||
existing: Option<MessageId>,
|
||||
) -> Result<MessageId> {
|
||||
let (uidvalidity, uid, uidl) = match key {
|
||||
Key::Imap { uidvalidity, uid } => (Some(*uidvalidity), Some(*uid), None),
|
||||
Key::Pop { uidl } => (None, None, Some(uidl.as_str())),
|
||||
};
|
||||
let attachments = content.map(|_| mail.attachments.len() as u32);
|
||||
let json = [
|
||||
serde_json::to_string(&mail.in_reply_to)?,
|
||||
serde_json::to_string(&mail.references)?,
|
||||
serde_json::to_string(&mail.from)?,
|
||||
serde_json::to_string(&mail.to)?,
|
||||
serde_json::to_string(&mail.cc)?,
|
||||
];
|
||||
let mut values: Vec<&dyn ToSql> = vec![
|
||||
&mail.message_id,
|
||||
&json[0],
|
||||
&json[1],
|
||||
&mail.subject,
|
||||
&json[2],
|
||||
&json[3],
|
||||
&json[4],
|
||||
&mail.date,
|
||||
&attachments,
|
||||
&content,
|
||||
];
|
||||
let tx = self.conn.transaction()?;
|
||||
let mut old: Option<String> = None;
|
||||
let id = match existing {
|
||||
Some(id) => {
|
||||
old = tx.query_row(
|
||||
"SELECT content FROM messages WHERE id = ?1",
|
||||
[id.0],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
values.push(&id.0);
|
||||
tx.execute(
|
||||
"UPDATE messages SET message_id = ?1, in_reply_to = ?2, refs = ?3,
|
||||
subject = ?4, sender = ?5, recipients = ?6, copies = ?7, date = ?8,
|
||||
attachments = ?9, content = ?10 WHERE id = ?11",
|
||||
values.as_slice(),
|
||||
)?;
|
||||
id
|
||||
}
|
||||
None => {
|
||||
values.extend::<[&dyn ToSql; 4]>([&mailbox.0, &uidvalidity, &uid, &uidl]);
|
||||
tx.execute(
|
||||
"INSERT INTO messages (message_id, in_reply_to, refs, subject, sender,
|
||||
recipients, copies, date, attachments, content, mailbox, uidvalidity,
|
||||
uid, uidl) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12,
|
||||
?13, ?14)",
|
||||
values.as_slice(),
|
||||
)?;
|
||||
MessageId(tx.last_insert_rowid())
|
||||
}
|
||||
};
|
||||
replace_flags(&tx, id, flags)?;
|
||||
search::index(&tx, id, mail, content.is_some())?;
|
||||
if let Some(old) = old.filter(|old| Some(old.as_str()) != content) {
|
||||
drop_unused(&tx, &self.dir, &old)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
pub fn set_flags(&mut self, id: MessageId, flags: &[&str]) -> Result<()> {
|
||||
self.writable()?;
|
||||
self.content(id)?;
|
||||
let tx = self.conn.transaction()?;
|
||||
replace_flags(&tx, id, flags)?;
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn get(&self, id: MessageId) -> Result<Stored> {
|
||||
let mut stored = self.conn.query_row(
|
||||
&format!("SELECT {COLUMNS} FROM messages WHERE id = ?1"),
|
||||
[id.0],
|
||||
|found| Ok(row(found)),
|
||||
)??;
|
||||
let mut query = self
|
||||
.conn
|
||||
.prepare("SELECT flag FROM flags WHERE message = ?1 ORDER BY flag")?;
|
||||
stored.flags = query
|
||||
.query_map([id.0], |row| row.get(0))?
|
||||
.collect::<rusqlite::Result<_>>()?;
|
||||
Ok(stored)
|
||||
}
|
||||
|
||||
/// The original, checked against its hash; None for headers only.
|
||||
pub fn original(&self, id: MessageId) -> Result<Option<Vec<u8>>> {
|
||||
self.content(id)?
|
||||
.map(|hash| load_at(&self.dir, &hash))
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub fn remove(&mut self, id: MessageId) -> Result<()> {
|
||||
self.writable()?;
|
||||
let content = self.content(id)?;
|
||||
let tx = self.conn.transaction()?;
|
||||
tx.execute("DELETE FROM messages WHERE id = ?1", [id.0])?;
|
||||
if let Some(hash) = content {
|
||||
drop_unused(&tx, &self.dir, &hash)?;
|
||||
}
|
||||
tx.commit()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn content(&self, id: MessageId) -> Result<Option<String>> {
|
||||
self.conn
|
||||
.query_row(
|
||||
"SELECT content FROM messages WHERE id = ?1",
|
||||
[id.0],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()?
|
||||
.ok_or_else(|| Error::Store(format!("no message {}", id.0)))
|
||||
}
|
||||
|
||||
/// The hash, and whether the file is new.
|
||||
fn keep(&self, bytes: &[u8]) -> Result<(String, bool)> {
|
||||
let hash = hex(&Sha256::digest(bytes));
|
||||
let path = self.dir.join(CONTENT).join(&hash[..2]).join(&hash);
|
||||
let made = match fs::read(&path) {
|
||||
Ok(held) if hex(&Sha256::digest(&held)) == hash => return Ok((hash, false)),
|
||||
Ok(_) => {
|
||||
log::warn!("mail: original {hash} was damaged; rewritten from the server copy");
|
||||
false
|
||||
}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => true,
|
||||
Err(err) => return Err(disk("checking an original", err)),
|
||||
};
|
||||
let shard = path.parent().expect("a shard folder");
|
||||
fs::create_dir_all(shard).map_err(|err| disk("storing an original", err))?;
|
||||
let staged = shard.join(format!(".{hash}.part"));
|
||||
if let Err(err) = fs::write(&staged, bytes).and_then(|()| fs::rename(&staged, &path)) {
|
||||
return Err(match fs::remove_file(&staged) {
|
||||
Ok(()) => disk("storing an original", err),
|
||||
Err(gone) if gone.kind() == ErrorKind::NotFound => disk("storing an original", err),
|
||||
Err(left) => Error::Store(format!(
|
||||
"storing an original: {err}; the staged copy stayed: {left}"
|
||||
)),
|
||||
});
|
||||
}
|
||||
Ok((hash, made))
|
||||
}
|
||||
}
|
||||
|
||||
/// Deletes the file once no row names it; runs before commit.
|
||||
fn drop_unused(conn: &Connection, dir: &Path, hash: &str) -> Result<()> {
|
||||
let used: bool = conn.query_row(
|
||||
"SELECT EXISTS (SELECT 1 FROM messages WHERE content = ?1)",
|
||||
[hash],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
if used {
|
||||
return Ok(());
|
||||
}
|
||||
let path = dir.join(CONTENT).join(&hash[..2]).join(hash);
|
||||
match fs::remove_file(path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {
|
||||
log::warn!("mail: original {hash} was already gone");
|
||||
Ok(())
|
||||
}
|
||||
Err(err) => Err(disk("deleting an original", err)),
|
||||
}
|
||||
}
|
||||
|
||||
fn replace_flags(tx: &Transaction, id: MessageId, flags: &[&str]) -> Result<()> {
|
||||
tx.execute("DELETE FROM flags WHERE message = ?1", [id.0])?;
|
||||
for flag in flags {
|
||||
tx.execute(
|
||||
"INSERT OR IGNORE INTO flags (message, flag) VALUES (?1, ?2)",
|
||||
params![id.0, flag],
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn row(row: &Row) -> Result<Stored> {
|
||||
let key = match row.get::<_, Option<String>>(4)? {
|
||||
Some(uidl) => Key::Pop { uidl },
|
||||
None => Key::Imap {
|
||||
uidvalidity: row.get(2)?,
|
||||
uid: row.get(3)?,
|
||||
},
|
||||
};
|
||||
let json = |at: usize| -> Result<String> { Ok(row.get(at)?) };
|
||||
Ok(Stored {
|
||||
id: MessageId(row.get(0)?),
|
||||
mailbox: MailboxId(row.get(1)?),
|
||||
key,
|
||||
flags: Vec::new(),
|
||||
message_id: row.get(5)?,
|
||||
in_reply_to: serde_json::from_str(&json(6)?)?,
|
||||
references: serde_json::from_str(&json(7)?)?,
|
||||
subject: row.get(8)?,
|
||||
from: serde_json::from_str(&json(9)?)?,
|
||||
to: serde_json::from_str(&json(10)?)?,
|
||||
cc: serde_json::from_str(&json(11)?)?,
|
||||
date: row.get(12)?,
|
||||
attachments: row.get(13)?,
|
||||
whole: row.get::<_, Option<String>>(14)?.is_some(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn load_at(dir: &Path, hash: &str) -> Result<Vec<u8>> {
|
||||
let path = dir.join(CONTENT).join(&hash[..2]).join(hash);
|
||||
let bytes = fs::read(&path).map_err(|err| disk("reading an original", err))?;
|
||||
if hex(&Sha256::digest(&bytes)) != hash {
|
||||
return Err(Error::Store(format!("original {hash} is damaged")));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
bytes.iter().map(|byte| format!("{byte:02x}")).collect()
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
/// Step `n` upgrades version `n`.
|
||||
pub(super) const STEPS: &[&str] = &[V1];
|
||||
|
||||
const V1: &str = "
|
||||
CREATE TABLE accounts (
|
||||
id INTEGER PRIMARY KEY,
|
||||
address TEXT NOT NULL UNIQUE
|
||||
);
|
||||
CREATE TABLE mailboxes (
|
||||
id INTEGER PRIMARY KEY,
|
||||
account INTEGER NOT NULL REFERENCES accounts ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
UNIQUE (account, name)
|
||||
);
|
||||
CREATE TABLE messages (
|
||||
id INTEGER PRIMARY KEY,
|
||||
mailbox INTEGER NOT NULL REFERENCES mailboxes ON DELETE CASCADE,
|
||||
uidvalidity INTEGER,
|
||||
uid INTEGER,
|
||||
uidl TEXT,
|
||||
message_id TEXT,
|
||||
in_reply_to TEXT NOT NULL,
|
||||
refs TEXT NOT NULL,
|
||||
subject TEXT,
|
||||
sender TEXT NOT NULL,
|
||||
recipients TEXT NOT NULL,
|
||||
copies TEXT NOT NULL,
|
||||
date INTEGER,
|
||||
attachments INTEGER,
|
||||
content TEXT,
|
||||
CHECK ((uid IS NULL) = (uidvalidity IS NULL) AND (uid IS NULL) <> (uidl IS NULL))
|
||||
);
|
||||
CREATE UNIQUE INDEX messages_imap ON messages (mailbox, uidvalidity, uid)
|
||||
WHERE uid IS NOT NULL;
|
||||
CREATE UNIQUE INDEX messages_pop ON messages (mailbox, uidl) WHERE uidl IS NOT NULL;
|
||||
CREATE INDEX messages_thread ON messages (message_id);
|
||||
CREATE INDEX messages_content ON messages (content);
|
||||
CREATE TABLE flags (
|
||||
message INTEGER NOT NULL REFERENCES messages ON DELETE CASCADE,
|
||||
flag TEXT NOT NULL,
|
||||
PRIMARY KEY (message, flag)
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX flags_by_flag ON flags (flag, message);
|
||||
CREATE VIRTUAL TABLE search USING fts5(
|
||||
subject, people, body, files,
|
||||
content = '', contentless_delete = 1, tokenize = 'trigram'
|
||||
);
|
||||
CREATE TRIGGER messages_gone AFTER DELETE ON messages BEGIN
|
||||
DELETE FROM search WHERE rowid = old.id;
|
||||
END;
|
||||
";
|
||||
@@ -0,0 +1,134 @@
|
||||
use rusqlite::{Transaction, params};
|
||||
|
||||
use super::{MessageId, Store, html, messages::load_at};
|
||||
use crate::{Mail, Mailbox, Result, read};
|
||||
|
||||
/// Replaces the message's row in the full-text index.
|
||||
pub(super) fn index(tx: &Transaction, id: MessageId, mail: &Mail, whole: bool) -> Result<()> {
|
||||
let people = people([&mail.from, &mail.to, &mail.cc]);
|
||||
let (body, files) = if whole {
|
||||
contents(id, mail)
|
||||
} else {
|
||||
Default::default()
|
||||
};
|
||||
put(tx, id, mail.subject.as_deref(), &people, &body, &files)
|
||||
}
|
||||
|
||||
fn put(
|
||||
tx: &Transaction,
|
||||
id: MessageId,
|
||||
subject: Option<&str>,
|
||||
people: &str,
|
||||
body: &str,
|
||||
files: &str,
|
||||
) -> Result<()> {
|
||||
tx.execute("DELETE FROM search WHERE rowid = ?1", [id.0])?;
|
||||
tx.execute(
|
||||
"INSERT INTO search (rowid, subject, people, body, files) VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
params![id.0, subject, people, body, files],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// Rebuilt from the index rows and the originals.
|
||||
pub fn rebuild_search(&mut self) -> Result<usize> {
|
||||
self.writable()?;
|
||||
let ids: Vec<i64> = {
|
||||
let mut query = self.conn.prepare("SELECT id FROM messages")?;
|
||||
query
|
||||
.query_map([], |row| row.get(0))?
|
||||
.collect::<rusqlite::Result<_>>()?
|
||||
};
|
||||
let tx = self.conn.transaction()?;
|
||||
// A damaged index may not delete.
|
||||
let made: String = tx.query_row(
|
||||
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'search'",
|
||||
[],
|
||||
|row| row.get(0),
|
||||
)?;
|
||||
tx.execute_batch(&format!("DROP TABLE search; {made};"))?;
|
||||
for &id in &ids {
|
||||
let (subject, from, to, cc, content): (
|
||||
Option<String>,
|
||||
String,
|
||||
String,
|
||||
String,
|
||||
Option<String>,
|
||||
) = tx.query_row(
|
||||
"SELECT subject, sender, recipients, copies, content FROM messages
|
||||
WHERE id = ?1",
|
||||
[id],
|
||||
|row| {
|
||||
Ok((
|
||||
row.get(0)?,
|
||||
row.get(1)?,
|
||||
row.get(2)?,
|
||||
row.get(3)?,
|
||||
row.get(4)?,
|
||||
))
|
||||
},
|
||||
)?;
|
||||
let lists: [Vec<Mailbox>; 3] = [
|
||||
serde_json::from_str(&from)?,
|
||||
serde_json::from_str(&to)?,
|
||||
serde_json::from_str(&cc)?,
|
||||
];
|
||||
let (body, files) = match content {
|
||||
Some(hash) => contents(MessageId(id), &read(&load_at(&self.dir, &hash)?)?),
|
||||
None => Default::default(),
|
||||
};
|
||||
let people = people([&lists[0], &lists[1], &lists[2]]);
|
||||
put(
|
||||
&tx,
|
||||
MessageId(id),
|
||||
subject.as_deref(),
|
||||
&people,
|
||||
&body,
|
||||
&files,
|
||||
)?;
|
||||
}
|
||||
let count = ids.len();
|
||||
tx.commit()?;
|
||||
log::info!("mail: search index rebuilt for {count} messages");
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
|
||||
fn people(lists: [&[Mailbox]; 3]) -> String {
|
||||
let mut out = String::new();
|
||||
for person in lists.into_iter().flatten() {
|
||||
if let Some(name) = &person.name {
|
||||
out.push_str(name);
|
||||
out.push(' ');
|
||||
}
|
||||
out.push_str(&person.email);
|
||||
out.push('\n');
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The body text and the attachment names.
|
||||
fn contents(id: MessageId, mail: &Mail) -> (String, String) {
|
||||
let body = match (&mail.text, &mail.html) {
|
||||
(Some(text), _) => text.clone(),
|
||||
(None, Some(html)) => {
|
||||
let (text, short) = html::text(html);
|
||||
if short {
|
||||
log::warn!(
|
||||
"mail: message {} nests HTML too deep; its later text is not indexed",
|
||||
id.0
|
||||
);
|
||||
}
|
||||
text
|
||||
}
|
||||
(None, None) => String::new(),
|
||||
};
|
||||
let files = mail
|
||||
.attachments
|
||||
.iter()
|
||||
.filter_map(|part| part.name.as_deref())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
(body, files)
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
use std::fs;
|
||||
|
||||
use rusqlite::Connection;
|
||||
|
||||
use super::{BACKUPS, DB, open, schema::STEPS};
|
||||
use crate::Error;
|
||||
|
||||
const ADD: &str = "ALTER TABLE accounts ADD COLUMN nick TEXT";
|
||||
const BROKEN: &str = "ALTER TABLE nowhere ADD COLUMN nick TEXT";
|
||||
|
||||
fn version(path: &std::path::Path) -> i64 {
|
||||
Connection::open(path)
|
||||
.unwrap()
|
||||
.query_row("PRAGMA user_version", [], |row| row.get(0))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn backups(dir: &std::path::Path) -> Vec<std::path::PathBuf> {
|
||||
match fs::read_dir(dir.join(BACKUPS)) {
|
||||
Ok(entries) => entries.map(|entry| entry.unwrap().path()).collect(),
|
||||
Err(_) => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_new_store_is_at_the_last_schema() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let store = open(dir.path(), STEPS).unwrap();
|
||||
assert_eq!(store.read_only(), None);
|
||||
assert_eq!(version(&dir.path().join(DB)), STEPS.len() as i64);
|
||||
assert!(backups(dir.path()).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_upgrade_backs_up_first() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
open(dir.path(), &[STEPS[0]])
|
||||
.unwrap()
|
||||
.add_account("ann@example.com")
|
||||
.unwrap();
|
||||
let store = open(dir.path(), &[STEPS[0], ADD]).unwrap();
|
||||
assert_eq!(store.read_only(), None);
|
||||
assert_eq!(version(&dir.path().join(DB)), 2);
|
||||
let [backup] = backups(dir.path()).try_into().unwrap();
|
||||
assert_eq!(version(&backup), 1);
|
||||
let accounts: i64 = Connection::open(&backup)
|
||||
.unwrap()
|
||||
.query_row("SELECT count(*) FROM accounts", [], |row| row.get(0))
|
||||
.unwrap();
|
||||
assert_eq!(accounts, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_upgrade_keeps_the_data_and_refuses_writes() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
open(dir.path(), &[STEPS[0]])
|
||||
.unwrap()
|
||||
.add_account("ann@example.com")
|
||||
.unwrap();
|
||||
let store = open(dir.path(), &[STEPS[0], ADD, BROKEN]).unwrap();
|
||||
let reason = store.read_only().unwrap();
|
||||
assert!(reason.contains("v1 to v3"), "{reason}");
|
||||
assert_eq!(store.accounts().unwrap()[0].1, "ann@example.com");
|
||||
let err = store.add_account("bo@example.com").unwrap_err();
|
||||
assert!(matches!(err, Error::ReadOnly(_)), "{err}");
|
||||
drop(store);
|
||||
assert_eq!(version(&dir.path().join(DB)), 1);
|
||||
assert_eq!(backups(dir.path()).len(), 1);
|
||||
let partial = Connection::open(dir.path().join(DB))
|
||||
.unwrap()
|
||||
.prepare("SELECT nick FROM accounts")
|
||||
.map(drop);
|
||||
assert!(partial.is_err(), "the first step was not rolled back");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_newer_schema_is_read_only() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
drop(open(dir.path(), &[STEPS[0], ADD]).unwrap());
|
||||
let store = open(dir.path(), &[STEPS[0]]).unwrap();
|
||||
let reason = store.read_only().unwrap();
|
||||
assert!(reason.contains("newer"), "{reason}");
|
||||
assert!(store.accounts().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_backup_means_no_upgrade() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
drop(open(dir.path(), &[STEPS[0]]).unwrap());
|
||||
fs::write(dir.path().join(BACKUPS), b"a file in the way").unwrap();
|
||||
let store = open(dir.path(), &[STEPS[0], ADD]).unwrap();
|
||||
let reason = store.read_only().unwrap();
|
||||
assert!(reason.contains("no backup"), "{reason}");
|
||||
drop(store);
|
||||
assert_eq!(version(&dir.path().join(DB)), 1);
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
use std::path::Path;
|
||||
|
||||
use ely_mail_engine::{Error, Fetched, Key, MailboxId, MessageId, Store};
|
||||
use rusqlite::Connection;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
/// The sample as a server sends it: CRLF lines.
|
||||
fn sample() -> Vec<u8> {
|
||||
let path = concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../tests/fixtures/mail/sample.eml"
|
||||
);
|
||||
let text = std::fs::read_to_string(path).unwrap();
|
||||
text.replace("\r\n", "\n")
|
||||
.replace('\n', "\r\n")
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
fn headers(raw: &[u8]) -> &[u8] {
|
||||
let end = raw.windows(4).position(|w| w == b"\r\n\r\n").unwrap();
|
||||
&raw[..end + 4]
|
||||
}
|
||||
|
||||
fn inbox(dir: &Path) -> (Store, MailboxId) {
|
||||
let store = Store::open(dir).unwrap();
|
||||
let account = store.add_account("ann@example.com").unwrap();
|
||||
let inbox = store.mailbox(account, "INBOX").unwrap();
|
||||
(store, inbox)
|
||||
}
|
||||
|
||||
fn imap(uidvalidity: u32, uid: u32) -> Key {
|
||||
Key::Imap { uidvalidity, uid }
|
||||
}
|
||||
|
||||
/// Rows whose index holds the phrase.
|
||||
fn found(dir: &Path, phrase: &str) -> Vec<i64> {
|
||||
let conn = Connection::open(dir.join("mail.db")).unwrap();
|
||||
let mut rows = conn
|
||||
.prepare("SELECT rowid FROM search WHERE search MATCH ?1 ORDER BY rowid")
|
||||
.unwrap();
|
||||
rows.query_map([format!("\"{phrase}\"")], |row| row.get(0))
|
||||
.unwrap()
|
||||
.map(Result::unwrap)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn original_path(dir: &Path, raw: &[u8]) -> std::path::PathBuf {
|
||||
let hash: String = Sha256::digest(raw)
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect();
|
||||
dir.join("content").join(&hash[..2]).join(hash)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_whole_message_is_indexed_and_kept_by_its_hash() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let raw = sample();
|
||||
let id = store
|
||||
.put(inbox, &imap(7, 42), &["\\Seen"], Fetched::Whole(&raw))
|
||||
.unwrap();
|
||||
let stored = store.get(id).unwrap();
|
||||
assert_eq!(stored.key, imap(7, 42));
|
||||
assert_eq!(stored.subject.as_deref(), Some("Café report"));
|
||||
assert_eq!(stored.from[0].email, "zhang@example.com");
|
||||
assert_eq!(stored.message_id.as_deref(), Some("m1@example.com"));
|
||||
assert_eq!(stored.flags, ["\\Seen"]);
|
||||
assert_eq!(stored.attachments, Some(1));
|
||||
assert!(stored.whole);
|
||||
assert_eq!(store.original(id).unwrap().as_deref(), Some(raw.as_slice()));
|
||||
assert_eq!(std::fs::read(original_path(dir.path(), &raw)).unwrap(), raw);
|
||||
assert_eq!(found(dir.path(), "numbers"), [id.0]);
|
||||
assert_eq!(found(dir.path(), "report.pdf"), [id.0]);
|
||||
let sizes = store.sizes().unwrap();
|
||||
assert_eq!(sizes.originals, raw.len() as u64);
|
||||
assert!(sizes.search > 0 && sizes.index > 0, "{sizes:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn headers_first_then_the_whole_message() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let raw = sample();
|
||||
let key = imap(7, 42);
|
||||
let id = store
|
||||
.put(inbox, &key, &[], Fetched::Headers(headers(&raw)))
|
||||
.unwrap();
|
||||
let stored = store.get(id).unwrap();
|
||||
assert_eq!(stored.subject.as_deref(), Some("Café report"));
|
||||
assert_eq!((stored.whole, stored.attachments), (false, None));
|
||||
assert_eq!(store.original(id).unwrap(), None);
|
||||
assert_eq!(found(dir.path(), "report"), [id.0]);
|
||||
assert!(found(dir.path(), "numbers").is_empty());
|
||||
|
||||
let again = store
|
||||
.put(inbox, &key, &["\\Flagged"], Fetched::Headers(headers(&raw)))
|
||||
.unwrap();
|
||||
assert_eq!(again, id);
|
||||
assert_eq!(store.get(id).unwrap().flags, ["\\Flagged"]);
|
||||
|
||||
assert_eq!(
|
||||
store.put(inbox, &key, &[], Fetched::Whole(&raw)).unwrap(),
|
||||
id
|
||||
);
|
||||
assert_eq!(store.get(id).unwrap().attachments, Some(1));
|
||||
assert_eq!(found(dir.path(), "numbers"), [id.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uidvalidity_is_part_of_the_identity() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let raw = sample();
|
||||
let old = store
|
||||
.put(inbox, &imap(1, 5), &[], Fetched::Headers(headers(&raw)))
|
||||
.unwrap();
|
||||
let new = store
|
||||
.put(inbox, &imap(2, 5), &[], Fetched::Headers(headers(&raw)))
|
||||
.unwrap();
|
||||
assert_ne!(old, new);
|
||||
assert_eq!(store.find(inbox, &imap(1, 5)).unwrap(), Some(old));
|
||||
let pop = Key::Pop { uidl: "abc".into() };
|
||||
let id = store
|
||||
.put(inbox, &pop, &[], Fetched::Headers(headers(&raw)))
|
||||
.unwrap();
|
||||
assert_eq!(store.get(id).unwrap().key, pop);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_shared_original_goes_with_its_last_message() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let raw = sample();
|
||||
let file = original_path(dir.path(), &raw);
|
||||
let one = store
|
||||
.put(inbox, &imap(1, 1), &[], Fetched::Whole(&raw))
|
||||
.unwrap();
|
||||
let two = store
|
||||
.put(inbox, &imap(1, 2), &[], Fetched::Whole(&raw))
|
||||
.unwrap();
|
||||
assert_eq!(store.sizes().unwrap().originals, raw.len() as u64);
|
||||
store.remove(one).unwrap();
|
||||
assert!(file.exists());
|
||||
assert_eq!(found(dir.path(), "numbers"), [two.0]);
|
||||
store.remove(two).unwrap();
|
||||
assert!(!file.exists());
|
||||
assert!(found(dir.path(), "numbers").is_empty());
|
||||
let err = store.remove(MessageId(two.0)).unwrap_err();
|
||||
assert!(
|
||||
matches!(&err, Error::Store(text) if text.contains("no message")),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_damaged_original_is_refused() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let raw = sample();
|
||||
let id = store
|
||||
.put(inbox, &imap(1, 1), &[], Fetched::Whole(&raw))
|
||||
.unwrap();
|
||||
std::fs::write(original_path(dir.path(), &raw), b"not the message").unwrap();
|
||||
let err = store.original(id).unwrap_err();
|
||||
assert!(
|
||||
matches!(&err, Error::Store(text) if text.contains("damaged")),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_search_index_rebuilds_from_the_store() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let whole = store
|
||||
.put(inbox, &imap(1, 1), &[], Fetched::Whole(&sample()))
|
||||
.unwrap();
|
||||
let html = b"Subject: news\r\nContent-Type: text/html\r\n\r\n\
|
||||
<style>.hidden{color:red}</style><p>Quarterly<b>results</b></p>\
|
||||
<script>trackingpixel()</script>";
|
||||
let page = store
|
||||
.put(inbox, &imap(1, 2), &[], Fetched::Whole(html))
|
||||
.unwrap();
|
||||
assert_eq!(found(dir.path(), "results"), [page.0]);
|
||||
assert!(found(dir.path(), "tracking").is_empty());
|
||||
assert!(found(dir.path(), "hidden").is_empty());
|
||||
assert!(found(dir.path(), "Quarterlyresults").is_empty());
|
||||
|
||||
Connection::open(dir.path().join("mail.db"))
|
||||
.unwrap()
|
||||
.execute("DELETE FROM search", [])
|
||||
.unwrap();
|
||||
assert!(found(dir.path(), "numbers").is_empty());
|
||||
assert_eq!(store.rebuild_search().unwrap(), 2);
|
||||
assert_eq!(found(dir.path(), "numbers"), [whole.0]);
|
||||
assert_eq!(found(dir.path(), "zhang"), [whole.0]);
|
||||
assert_eq!(found(dir.path(), "results"), [page.0]);
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
#![cfg(windows)]
|
||||
|
||||
use std::{fs::File, os::windows::fs::OpenOptionsExt, path::Path};
|
||||
|
||||
use ely_mail_engine::{Fetched, Key, MailboxId, Store};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const ONE: &[u8] = b"Subject: one\r\n\r\nfirst body\r\n";
|
||||
const TWO: &[u8] = b"Subject: two\r\n\r\nsecond body\r\n";
|
||||
|
||||
fn inbox(dir: &Path) -> (Store, MailboxId) {
|
||||
let store = Store::open(dir).unwrap();
|
||||
let account = store.add_account("ann@example.com").unwrap();
|
||||
let inbox = store.mailbox(account, "INBOX").unwrap();
|
||||
(store, inbox)
|
||||
}
|
||||
|
||||
fn imap() -> Key {
|
||||
Key::Imap {
|
||||
uidvalidity: 1,
|
||||
uid: 1,
|
||||
}
|
||||
}
|
||||
|
||||
fn original_path(dir: &Path, raw: &[u8]) -> std::path::PathBuf {
|
||||
let hash: String = Sha256::digest(raw)
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect();
|
||||
dir.join("content").join(&hash[..2]).join(hash)
|
||||
}
|
||||
|
||||
/// Read sharing only: delete fails.
|
||||
fn hold(path: &Path) -> File {
|
||||
File::options().read(true).share_mode(1).open(path).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_original_keeps_its_message_for_retry() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let id = store.put(inbox, &imap(), &[], Fetched::Whole(ONE)).unwrap();
|
||||
let path = original_path(dir.path(), ONE);
|
||||
let held = hold(&path);
|
||||
assert!(store.remove(id).is_err());
|
||||
assert_eq!(store.get(id).unwrap().subject.as_deref(), Some("one"));
|
||||
assert_eq!(store.original(id).unwrap().as_deref(), Some(ONE));
|
||||
drop(held);
|
||||
store.remove(id).unwrap();
|
||||
assert!(store.get(id).is_err());
|
||||
assert!(!path.exists());
|
||||
assert_eq!(store.sizes().unwrap().originals, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_held_old_original_keeps_the_old_message() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let id = store.put(inbox, &imap(), &[], Fetched::Whole(ONE)).unwrap();
|
||||
let old = original_path(dir.path(), ONE);
|
||||
let new = original_path(dir.path(), TWO);
|
||||
let held = hold(&old);
|
||||
assert!(store.put(inbox, &imap(), &[], Fetched::Whole(TWO)).is_err());
|
||||
assert_eq!(store.get(id).unwrap().subject.as_deref(), Some("one"));
|
||||
assert_eq!(store.original(id).unwrap().as_deref(), Some(ONE));
|
||||
assert!(!new.exists());
|
||||
drop(held);
|
||||
store.put(inbox, &imap(), &[], Fetched::Whole(TWO)).unwrap();
|
||||
assert_eq!(store.original(id).unwrap().as_deref(), Some(TWO));
|
||||
assert!(!old.exists());
|
||||
assert_eq!(store.sizes().unwrap().originals, TWO.len() as u64);
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
use std::path::Path;
|
||||
|
||||
use ely_mail_engine::{Error, Fetched, Key, MailboxId, MessageId, Store};
|
||||
use rusqlite::Connection;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
fn sample() -> Vec<u8> {
|
||||
let path = concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../tests/fixtures/mail/sample.eml"
|
||||
);
|
||||
let text = std::fs::read_to_string(path).unwrap();
|
||||
text.replace("\r\n", "\n")
|
||||
.replace('\n', "\r\n")
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
fn inbox(dir: &Path) -> (Store, MailboxId) {
|
||||
let store = Store::open(dir).unwrap();
|
||||
let account = store.add_account("ann@example.com").unwrap();
|
||||
let inbox = store.mailbox(account, "INBOX").unwrap();
|
||||
(store, inbox)
|
||||
}
|
||||
|
||||
fn imap(uid: u32) -> Key {
|
||||
Key::Imap {
|
||||
uidvalidity: 1,
|
||||
uid,
|
||||
}
|
||||
}
|
||||
|
||||
fn search(dir: &Path, phrase: &str) -> rusqlite::Result<Vec<i64>> {
|
||||
let conn = Connection::open(dir.join("mail.db"))?;
|
||||
let mut rows = conn.prepare("SELECT rowid FROM search WHERE search MATCH ?1")?;
|
||||
rows.query_map([format!("\"{phrase}\"")], |row| row.get(0))?
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn found(dir: &Path, phrase: &str) -> Vec<i64> {
|
||||
search(dir, phrase).unwrap()
|
||||
}
|
||||
|
||||
fn original_path(dir: &Path, raw: &[u8]) -> std::path::PathBuf {
|
||||
let hash: String = Sha256::digest(raw)
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect();
|
||||
dir.join("content").join(&hash[..2]).join(hash)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_corrupt_search_index_rebuilds() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let id = store
|
||||
.put(inbox, &imap(1), &[], Fetched::Whole(&sample()))
|
||||
.unwrap();
|
||||
let damaged = Connection::open(dir.path().join("mail.db"))
|
||||
.unwrap()
|
||||
.execute("UPDATE search_data SET block = X'00' WHERE id > 10", [])
|
||||
.unwrap();
|
||||
assert!(damaged > 0);
|
||||
assert!(search(dir.path(), "numbers").is_err());
|
||||
assert_eq!(store.rebuild_search().unwrap(), 1);
|
||||
assert_eq!(found(dir.path(), "numbers"), [id.0]);
|
||||
drop(store);
|
||||
Store::open(dir.path()).unwrap();
|
||||
assert_eq!(found(dir.path(), "numbers"), [id.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_damaged_original_is_rewritten_by_a_new_copy() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let raw = sample();
|
||||
let id = store
|
||||
.put(inbox, &imap(1), &[], Fetched::Whole(&raw))
|
||||
.unwrap();
|
||||
std::fs::write(original_path(dir.path(), &raw), b"damaged").unwrap();
|
||||
store
|
||||
.put(inbox, &imap(2), &[], Fetched::Whole(&raw))
|
||||
.unwrap();
|
||||
assert_eq!(store.original(id).unwrap().as_deref(), Some(raw.as_slice()));
|
||||
|
||||
std::fs::write(original_path(dir.path(), &raw), b"damaged").unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.put(inbox, &imap(1), &[], Fetched::Whole(&raw))
|
||||
.unwrap(),
|
||||
id
|
||||
);
|
||||
assert_eq!(store.original(id).unwrap().as_deref(), Some(raw.as_slice()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_write_keeps_no_new_original() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let raw = sample();
|
||||
let nowhere = MailboxId(999);
|
||||
assert!(
|
||||
store
|
||||
.put(nowhere, &imap(1), &[], Fetched::Whole(&raw))
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(store.sizes().unwrap().originals, 0);
|
||||
|
||||
let id = store
|
||||
.put(inbox, &imap(1), &[], Fetched::Whole(&raw))
|
||||
.unwrap();
|
||||
assert!(
|
||||
store
|
||||
.put(nowhere, &imap(2), &[], Fetched::Whole(&raw))
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(store.original(id).unwrap().as_deref(), Some(raw.as_slice()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sizes_count_the_write_ahead_log() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let id = store
|
||||
.put(inbox, &imap(1), &[], Fetched::Whole(&sample()))
|
||||
.unwrap();
|
||||
for turn in 0..600 {
|
||||
let flag = if turn % 2 == 0 { "\\Seen" } else { "\\Flagged" };
|
||||
store.set_flags(id, &[flag]).unwrap();
|
||||
}
|
||||
let disk: u64 = ["mail.db", "mail.db-wal", "mail.db-shm"]
|
||||
.iter()
|
||||
.map(|name| std::fs::metadata(dir.path().join(name)).unwrap().len())
|
||||
.sum();
|
||||
let sizes = store.sizes().unwrap();
|
||||
assert!(disk > 1_000_000, "{disk}");
|
||||
assert_eq!(sizes.index + sizes.search, disk);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flags_for_a_missing_message_fail() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, _) = inbox(dir.path());
|
||||
let err = store.set_flags(MessageId(5), &[]).unwrap_err();
|
||||
assert!(
|
||||
matches!(&err, Error::Store(text) if text.contains("no message")),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_text_follows_the_parser_states() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let html = b"Subject: page\r\nContent-Type: text/html\r\n\r\n\
|
||||
<title>headline</title><script/>trackingpixel()</script>\
|
||||
<textarea><script>ordinarycontent</script></textarea>\
|
||||
<xmp><b>literalmarkup</b></xmp><iframe>fallbacktext</iframe>\
|
||||
<svg><style/>vectorlabel<script>vectorcode()</script></svg>\
|
||||
<svg><foreignObject><script/>trackingforeign()</script>\
|
||||
<textarea><script>ordinaryforeign</script></textarea></foreignObject></svg>\
|
||||
<math><annotation-xml encoding=\"text/html\"><script/>annotationcode()</script>\
|
||||
</annotation-xml></math><p>closingword</p>";
|
||||
let id = store
|
||||
.put(inbox, &imap(1), &[], Fetched::Whole(html))
|
||||
.unwrap();
|
||||
for round in 0..2 {
|
||||
for shown in [
|
||||
"ordinarycontent",
|
||||
"<b>literalmarkup",
|
||||
"vectorlabel",
|
||||
"<script>ordinaryforeign",
|
||||
"closingword",
|
||||
] {
|
||||
assert_eq!(found(dir.path(), shown), [id.0], "{shown} {round}");
|
||||
}
|
||||
for hidden in [
|
||||
"headline",
|
||||
"trackingpixel",
|
||||
"fallbacktext",
|
||||
"vectorcode",
|
||||
"trackingforeign",
|
||||
"annotationcode",
|
||||
] {
|
||||
assert!(found(dir.path(), hidden).is_empty(), "{hidden} {round}");
|
||||
}
|
||||
store.rebuild_search().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn a_failed_publish_leaves_no_staged_copy() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let raw = sample();
|
||||
let id = store
|
||||
.put(inbox, &imap(1), &[], Fetched::Whole(&raw))
|
||||
.unwrap();
|
||||
let path = original_path(dir.path(), &raw);
|
||||
std::fs::write(&path, b"damaged").unwrap();
|
||||
let mut locked = std::fs::metadata(&path).unwrap().permissions();
|
||||
locked.set_readonly(true);
|
||||
std::fs::set_permissions(&path, locked.clone()).unwrap();
|
||||
assert!(
|
||||
store
|
||||
.put(inbox, &imap(1), &[], Fetched::Whole(&raw))
|
||||
.is_err()
|
||||
);
|
||||
#[allow(clippy::permissions_set_readonly_false)]
|
||||
locked.set_readonly(false);
|
||||
std::fs::set_permissions(&path, locked).unwrap();
|
||||
store.remove(id).unwrap();
|
||||
assert_eq!(store.sizes().unwrap().originals, 0);
|
||||
let left: Vec<_> = std::fs::read_dir(path.parent().unwrap()).unwrap().collect();
|
||||
assert!(left.is_empty(), "{left:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_html_stops_at_its_depth_limit() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (mut store, inbox) = inbox(dir.path());
|
||||
let mut html = b"Subject: deep
|
||||
Content-Type: text/html
|
||||
|
||||
"
|
||||
.to_vec();
|
||||
html.extend("<div>".repeat(500).bytes());
|
||||
html.extend(b"shallowword");
|
||||
html.extend("<div>".repeat(200_000).bytes());
|
||||
html.extend(b"bottomword");
|
||||
let started = std::time::Instant::now();
|
||||
let id = store
|
||||
.put(inbox, &imap(1), &[], Fetched::Whole(&html))
|
||||
.unwrap();
|
||||
assert!(started.elapsed().as_secs() < 5, "{:?}", started.elapsed());
|
||||
assert_eq!(found(dir.path(), "shallowword"), [id.0]);
|
||||
assert!(found(dir.path(), "bottomword").is_empty());
|
||||
}
|
||||
+1
-1
@@ -136,7 +136,7 @@
|
||||
- [x] C12.0 选型修订：PRD §1.1、§12.1、§17.3、§21.2、§23 与 `agents.md` 改为上述选型；删去 MailCore 2 与 libetpan 的取码、构建与探测脚本；`oauth.env` 不入库；`references.sh` 随之（并修上游钉与工作区不一致的组件库钉） — R1 ok
|
||||
- [x] C12.1 `crates/ely-mail-engine` 与连通（MAIL-01 前半）：IMAP、POP3、SMTP 经 TLS（rustls，系统根证书）或 STARTTLS 连接，密码与 XOAUTH2 认证，失败带服务器原话；替身服务器测三协议的登录、拒绝、列邮箱、取信、发信。已做：`Imap`、`Pop`、`Smtp` 三个会话，`Server`（主机、端口、`Security::Tls` 或 `StartTls`，无明文）、`Login`（密码或 OAuth 令牌）；系统信任库经 rustls-platform-verifier，测试换入只信测试证书的连接器；每步 60 秒超时；`Error` 分连接、TLS、凭据被拒（服务器原话）、命令被拒、读不懂、超时、发不出（信封地址或登录带换行等）。IMAP 由引擎自写，以 imap-proto 解析（`wire.rs` 收一个个完整回复，含字面量），换下 async-imap（它的 LIST、FETCH 不看结尾的 NO 与 BAD，拒绝成了空结果与“无此 UID”；登录与命令中途的 `* BYE` 被丢掉，只剩断线；trace 日志写出密码与邮件）：每条命令看结尾的状态，NO 与 BAD 带原话报出（登录为凭据被拒，其余为命令被拒）；问候或任何时候收到 `* BYE`，此后断线即带其原话；非 ASCII 或带换行的参数以字面量发出，等服务器的 `+`；STARTTLS 之后若已有明文字节即拒绝；LIST 里带引号的邮箱名去掉转义（imap-proto 照原样留着），字面量照原样，列出的名字可直接打开；日志只记标签与命令名。POP3 由引擎自写（RFC 1939：问候、STLS、USER/PASS、AUTH XOAUTH2、UIDL、RETR、QUIT），换下 async-pop（它留着点填充、丢末尾 CRLF、吞掉登录后 CAPA 的连接错误、不答 XOAUTH2 的错误挑战、在 trace 日志里写出密码与邮件）；STLS 之后若已有明文字节即拒绝；用户名与密码带换行即拒绝。SMTP 的 XOAUTH2 与事务由引擎自走（mail-send 遇错误挑战会重发令牌；它的 DATA 不给首行的点加点，且自加末尾 CRLF）：信封地址照 RFC 5321 的 Mailbox 核对（点串或带引号的本地部分，域名或地址字面量；地址或信头有 UTF-8 时，服务器报 SMTPUTF8 才发出并带此参数，否则拒绝（RFC 6531）），不合即拒绝、不发任何命令；DATA 每行开头的点（含首行）加一个点，裸 CR 或 LF 即拒绝。服务器拒绝事务中的一步后发 RSET，会话可再发信。三协议的 XOAUTH2 被拒都以空行答错误挑战、留下服务器最后的原话。34 个测试（`cargo test -p ely-mail-engine`，含以真日志器在 trace 下跑三协议、日志里无密码、令牌与正文），真服务器未连。审查三轮：第 3 轮的两处（带引号的邮箱名、信头的 UTF-8）已修并以测试核对，未再审查（每条至多三轮）
|
||||
- [x] C12.2 MIME（MAIL-04、05 的基础）：mail-parser 读出头、纯文本、HTML、cid 内嵌图片、附件与原始头；mail-builder 写出多部分邮件、附件、Message-ID、In-Reply-To 与 References；样本 EML 往返测试。已做：`read` 得 `Mail`（主题、发件人、收件人、抄送（分组展开）、日期、Message-ID、In-Reply-To、References、纯文本、HTML、`inline`、`attachments`、原始头块）；按 MIME 树与所在容器读（不用 mail-parser 摊平的列表）：正文只取发件人写的部分（mail-parser 会把 HTML 转成文字或反之顶替缺的一种，不取）；alternative 里纯文本与 HTML 各取最后一个（RFC 2046 §5.1.4），其余版本不并入，没有正文的版本（如日历）为附件；related 的根照 `start`，无 `start` 取首个（RFC 2387），其余带 Content-ID 的即 cid 目标，不论处置；mixed 等的部分依次并成正文；处置为 attachment 的实体整个是附件，多部分实体连同其头（RFC 2183 §2.9），子孙不进正文；附件字节只解传输编码，文字附件不按字符集转换；不在 related 里的部分，带 Content-ID 且不是 attachment 的为 cid 目标；无 Content-Type 照 RFC 2045 为 text/plain，digest 的子部分为 message/rfc822；解码有问题的部分与无地址的收件人记警告。`build` 由 `Draft` 写出（密送不进信头，留在 SMTP 信封）：Message-ID 为 16 字节随机数加发件人的域（mail-builder 默认写本机主机名，不用），交回供发件记录；地址照 SMTP 的 Mailbox 核对，主题、姓名、Message-ID、文件名、类型带换行即拒绝；附件须有文件名，Content-ID 照写（带换行即拒绝）；正文与附件自己组成 mixed 与 alternative。`tests/fixtures/mail/sample.eml` 读全；由它回复写出再读回，各字段与附件（名、类型、Content-ID、含 0 至 255 每个字节）相同；第 1 轮审查后另 7 个结构测试（`tests/mime_tree.rs`：Latin-1 文字附件、`start` 在图后、related 里标为附件的图、附件化的多部分、同型 alternative、digest），去掉各自的修复即失败；第 2 轮审查后：嵌套深于 64 层即报错（先以迭代算出深度，原递归在 2048 层时栈溢出、进程退出）；mail-parser 开 `full_encoding`（原 GB2312 等多字节字符集读成替换字符）；`Part::params` 带 Content-Type 的参数（不含 name），写出时照写，转发的 Latin-1 附件收件方读出 Café；多部分与 message 附件照原样写出（不编码，7bit 或 8bit，多部分带原分界，RFC 2045 §6.4），SMTP 遇 8 位正文即带 `BODY=8BITMIME`，服务器不支持即在发出前拒绝；related 的根不看处置，其余每个资源整个是一个实体（多部分资源带其 Content-ID 与文件名）；带 Content-ID 的正文文字部分另为 cid 目标（related 的根除外）；未知处置照附件（RFC 2183 §2.8）。另 9 个测试（`tests/mime_edges.rs`、`tests/mime_forward.rs`，SMTP 一个），去掉各自的修复即失败；第 3 轮审查（最后一轮，修完不再审查）后：深度也数内嵌的邮件（mail-parser 以迭代读出层层相套的 message/rfc822，丢弃时却递归，8192 层即栈溢出），超深的解析结果以迭代逐层拆开再丢；带 Content-ID 的多部分实体（related 的根除外）连同其头另为 cid 目标，子孙照常读；SMTP 按每一层的头块判定是否需 SMTPUTF8（转发的 message/global 里的中文主题），只深入未经传输编码的内嵌邮件；每次发送都先解析邮件，不是邮件即拒绝。另 5 个测试（`tests/mime_deep.rs` 四个、`tests/smtp_utf8.rs` 一个；SMTP 的编码测试与辅助函数移入 `tests/smtp_utf8.rs` 与 `tests/fake/smtp.rs`），去掉各自的修复即失败或栈溢出；原往返测试按子串查 bcc，随机的 Message-ID 约四十次一次含 bcc 而误报，改为查 `Bcc:` 头行；引擎共 61 个测试
|
||||
- [ ] C12.3 本地存储（§12.3、§18.1、§18.3）：SQLite 索引（账号、邮箱、消息身份为账号加邮箱加 UIDVALIDITY 加 UID、标记、头字段）、原文按内容存进内容目录、FTS5 全文索引（可重建）、结构版本与升级前备份、升级失败只读；各类大小可查
|
||||
- [x] C12.3 本地存储（§12.3、§18.1、§18.3）：SQLite 索引（账号、邮箱、消息身份为账号加邮箱加 UIDVALIDITY 加 UID、标记、头字段）、原文按内容存进内容目录、FTS5 全文索引（可重建）、结构版本与升级前备份、升级失败只读；各类大小可查。已做：`Store`（rusqlite，自带 SQLite 3.53.2，FTS5 与 dbstat）在给定文件夹开 `mail.db`（WAL，外键开）；表 accounts、mailboxes（账号加名字唯一）、messages（身份为邮箱加 UIDVALIDITY 加 UID，POP3 为邮箱加 UIDL，两者恰居其一，各有部分唯一索引；头字段：Message-ID、In-Reply-To、References、主题、发件人、收件人、抄送（JSON）、日期；附件数在整封到达前为空）、flags（每标记一行）；原文按 SHA-256 存在 `content/前两位/散列`（暂存后改名，同内容只存一份，读时核对散列，不符即报“已损坏”，最后一封引用它的邮件删去时删文件）；同一身份再来头部只改标记，整封到达即补原文、附件数与正文索引；FTS5 表 `search`（主题、人、正文、附件名，无内容副本，trigram 分词，删邮件的触发器同删索引行），HTML 独有的正文以 html5ever 分词器取文字（略去 script 与 style，标签处断词）；`rebuild_search` 由索引行与原文重建；`PRAGMA user_version` 记结构版本，升级前以 `VACUUM INTO` 备份到 `backups/`，各步在一个事务里，备份或升级失败即以只读重开、写入报 `ReadOnly` 并带原因，更新的版本写的库同样只读；`sizes` 给索引、全文索引（dbstat）与原文三类字节数。测试：结构版本 5 个（新库、升级先备份、失败回滚且只读、更新的版本只读、无备份不升级），存储 6 个（整封、先头后整封、UIDVALIDITY 与 POP 身份、共用原文、损坏原文、HTML 正文与重建），去掉各自的规则即失败；引擎共 72 个测试。第 1 轮审查后：重建先删去整张全文表再按库里记的定义重建（损坏的段删不掉）；存原文时已有的文件先核对散列，坏了即以本次收到的字节改写；写库失败时删去本次新建且无人引用的原文；`sizes` 的索引按 `mail.db`、`-wal`、`-shm` 的实际字节减去全文页；HTML 取文字照树构建器给分词器的状态（script 为脚本数据，style、iframe、noembed、noframes 为原始文本，title 与 textarea 为 RCDATA，xmp 与 plaintext 照显示，自闭合在 HTML 元素上无效；SVG 与 MathML 里自闭合有效、script 与 style 的文字略去）；给不存在的邮件设标记即报错。`tests/store_repair.rs` 6 个，去掉各自的修复即失败；引擎共 78 个测试。第 2 轮审查后：HTML 取文字改走 html5ever 的整个树构建器（`store/html.rs`，自建的节点表作 `TreeSink`，脚本关闭），命名空间、整合点与插入模式全由它定，不再自仿分词器状态；不画的元素照 HTML 渲染规则（script、style、title、head、iframe、template 等）略去；树构建器每个开始标签都扫一遍开放元素栈，嵌套越深越慢（实测 release：十万层 div 28 秒），故按 4 KB 分块喂入，元素深过 512 层（同 Blink 的上限）即停，已读的文字照常入索引，警告写明哪封邮件；暂存原文写入或改名失败时删去暂存文件，删不掉即连同原因报错。`tests/store_repair.rs` 共 8 个；去掉清理、不画元素、MathML 整合点、深度上限即各自失败。第 3 轮审查后（最后一轮，修完不再审查）：删除与替换邮件时，原文文件在同一事务提交之前删去，删不掉即整步回滚，邮件与原文都留着、可重试（原先先提交再删文件，失败后失去引用，文件永久残留）；提交在删文件之后失败时，下次清理遇文件已不在只记警告；两处超长注释缩短。`tests/store_release.rs`（Windows，以只读共享占住文件）2 个，删去与替换各一，改回先提交即失败；引擎共 82 个测试
|
||||
- [ ] C12.4 凭据与 OAuth（MAIL-02，§18.2）：密码与刷新令牌只进系统凭据库（不可用即只留到本次退出并写明）；Gmail 与 Microsoft 的授权码加 PKCE、本机回环、换令牌与刷新；客户端 ID 编进构建，缺 ID 即写明不可用；刷新被拒转为“需重新登录”；替身授权端点测试
|
||||
- [ ] C12.5 账号（MAIL-01、02）：ElyMail 主窗口与添加账号对话框：Gmail、Microsoft、iCloud（应用专用密码向导）、其他 IMAP/SMTP、POP3/SMTP；服务器、端口、TLS 方式、认证方式；连通性检查逐项报结果；多账号；删除账号先写明删去的本地缓存、草稿与凭据，不动服务端
|
||||
- [ ] C12.6 IMAP 同步（MAIL-14，§12.3，QA-18）：文件夹列表与特殊用途；邮件头全部、正文只取 90 天内；UIDVALIDITY 变化即重建该邮箱；服务端删除与标记变化随之；后台任务执行，界面显示最后同步时间、下载范围、未缓存的数量与失败原因
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
FAIL
|
||||
|
||||
- crates/ely-mail-engine/src/store/search.rs:54 — 损坏的全文索引没有真正重建。存入一封邮件后，以 `UPDATE search_data SET block = X'00' WHERE id > 10` 损坏一个 FTS 段；搜索报 `SQLITE_CORRUPT_VTAB`，但 `rebuild_search()` 返回 `Ok(1)`，关闭、重开后搜索仍报同一损坏错误，原文读取正常。这违反 §18.1 的全文索引可重建要求；现有测试只清空健康索引，覆盖不到此场景。修复方向：重建时清除并重新建立底层 FTS 数据，而非逐行删除后插入；增加实际损坏索引、重建并重新搜索的回归测试。证据：`.work/C12.3-r1/probes.log`、`.work/C12.3-r1/sizes.log`。
|
||||
|
||||
- crates/ely-mail-engine/src/store/messages.rs:221 — 原文路径存在即直接认定保存成功，没有核对已有文件。先存入整封邮件，再把对应内容文件改成 `damaged`，随后重新 `put` 同一封完整原文，得到 `Ok(MessageId(1))`，但 `original` 仍报原文损坏；新身份引用相同原文也会关联到坏文件。修复方向：复用已有内容文件前验证完整性，损坏时明确报错或以本次收到的正确字节原子修复，并测试重新收取与跨消息去重。证据：`.work/C12.3-r1/probes.log`。
|
||||
|
||||
- crates/ely-mail-engine/src/store/messages.rs:95 — 数据库写入失败后，之前由 `keep` 写入的原文永久遗留。对不存在的 `MailboxId(999)` 存入 80 字节整封邮件，数据库正确报外键错误，但 `sizes().originals` 变为 80；没有消息行引用该文件，也没有回收路径，反复失败会累积不可清理的缓存。修复方向：协调原文发布与数据库事务，失败时清理本次新增且无人引用的内容，保留其他消息共用的文件，并覆盖写入失败的回归测试。证据：`.work/C12.3-r1/probes.log`。
|
||||
|
||||
- crates/ely-mail-engine/src/store.rs:99 — 容量统计遗漏 WAL 与共享内存文件，把逻辑页大小当成磁盘占用。存入一封邮件并交替设置标记 600 次后，`index + search` 仍仅报 65,536 字节，实际 `mail.db`、`mail.db-wal`、`mail.db-shm` 合计 4,222,456 字节；WAL 在连接运行期间持续占用磁盘，容量管理会明显低估缓存。修复方向：将数据库附属文件的实际占用计入统计，并测试未关闭连接、重复写入后的占用。证据：`.work/C12.3-r1/sizes.log`。
|
||||
|
||||
- crates/ely-mail-engine/src/store/search.rs:157 — HTML 取文字没有遵守 HTML 元素的解析状态，导致正文漏索引和脚本被索引。`<script/>trackingpixel()</script>` 中的斜杠不会关闭 HTML 的 script 元素，但此处以 `self_closing` 跳过隐藏，搜索能命中 `trackingpixel`；`<textarea><script>ordinarycontent</script></textarea>` 中应显示的普通文字反而被当脚本丢弃，搜索无结果。修复方向：按 HTML 解析语义处理非空元素及 RCDATA/raw-text 状态，覆盖这两种输入与重建后的搜索结果。证据：`.work/C12.3-r1/probes.log`。
|
||||
|
||||
- crates/ely-mail-engine/src/store/schema.rs:1、crates/ely-mail-engine/src/store/messages.rs:68 — 两条新增注释均为 12 个空白分隔词，超过 `agents.md` 的注释十词上限。修复方向：将这两条注释各缩至十词以内。
|
||||
@@ -0,0 +1,5 @@
|
||||
FAIL
|
||||
|
||||
- crates/ely-mail-engine/src/store/search.rs:208 — 第 1 轮的 HTML 解析问题仍未从根因修复：`foreign` 只计未闭合的 svg/math，无法识别 HTML 整合点及退出外来内容的标签。在 `<svg><foreignObject>` 内，`<script/>trackingforeign()</script>` 的脚本被索引，`<textarea><script>ordinaryforeign</script></textarea>` 应显示的文字却未被索引；`put` 与 `rebuild_search` 后均如此。html5ever 的树构建器实测把这里的 script、textarea 建为 HTML 元素，并把后者内部的标签当普通文字。修复方向：依据树构建器的当前命名空间、整合点及插入模式决定取文字和分词状态，覆盖这两种输入及重建。证据：`.work/C12.3-r2/probe.log`、`.work/C12.3-r2/html-oracle.log`。
|
||||
|
||||
- crates/ely-mail-engine/src/store/messages.rs:242 — 暂存原文写入或改名失败时，没有删除 `.part` 文件。Windows 上先保存一封 35 字节原文，再把内容文件损坏并设为只读，重新收取时暂存写入成功、改名报拒绝访问；恢复文件权限后删除该邮件，`sizes().originals` 仍为 35，全部消息已删除却留下完整暂存原文，且没有回收路径。第 1 轮新增的数据库失败清理覆盖不到 `keep` 内部失败。修复方向：在暂存写入或发布失败时清理本次暂存文件，处理并暴露清理错误，补上发布失败且删除最后引用后无残留的回归测试。证据：`.work/C12.3-r2/probe.log`、`.work/C12.3-r2/probe-source.log`。
|
||||
@@ -0,0 +1,5 @@
|
||||
FAIL
|
||||
|
||||
- crates/ely-mail-engine/src/store/messages.rs:207 — 删除消息先提交数据库删除，再删除最后一份原文；原文删除失败后失去可重试的引用，留下永久残留。Windows 上存入一封 35 字节邮件，用 `OpenOptionsExt::share_mode(1)` 打开原文并保持句柄，再调用 `remove`：返回文件占用错误，但 `get` 已报无行；关闭句柄后重试 `remove` 只报 `no message 1`，`sizes().originals` 仍为 35。现有测试只覆盖删除成功。修复方向：协调消息删除与内容清理，失败时保留或恢复可重试的引用，或持久记录待清理内容；增加文件占用、解除后重试且无残留的回归测试。证据：`.work/C12.3-r3/remove-probe.log`、`.work/C12.3-r3/probe-source.log`。
|
||||
|
||||
- crates/ely-mail-engine/src/store/html.rs:14、crates/ely-mail-engine/src/store/html.rs:108 — 新增的两条注释分别为 12 和 11 个空白分隔词，超过 `agents.md` 的十词上限。修复方向：分别缩至十词以内。
|
||||
Reference in new issue
Block a user