Q190: search finds Chinese and other non-English words inside text, in any case
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
bc7e214b65
commit
32c712f86f
3 files changed
+63
-17
No files matched your search
@@ -24,25 +24,21 @@ export class SearchService {
|
|||||||
/** Every match the person may read, best first, counted in full before one page is cut from it. */
|
/** Every match the person may read, best first, counted in full before one page is cut from it. */
|
||||||
search(input: SearchInput): SearchPage {
|
search(input: SearchInput): SearchPage {
|
||||||
this.assertEmployee(input.orgId, input.employeeId);
|
this.assertEmployee(input.orgId, input.employeeId);
|
||||||
const query = normalizeQuery(input.query);
|
const query = parseQuery(input.query);
|
||||||
const limit = Math.max(1, Math.min(input.limit ?? 20, 50));
|
const limit = Math.max(1, Math.min(input.limit ?? 20, 50));
|
||||||
const offset = Math.max(0, input.offset ?? 0);
|
const offset = Math.max(0, input.offset ?? 0);
|
||||||
const entityTypes = validEntityTypes(input.entityTypes ?? [...SEARCH_ENTITY_TYPES]);
|
const entityTypes = validEntityTypes(input.entityTypes ?? [...SEARCH_ENTITY_TYPES]);
|
||||||
this.reindexOrg(input.orgId);
|
this.reindexOrg(input.orgId);
|
||||||
const rows = query
|
if (query.blank) return { results: [], total: 0 };
|
||||||
? this.db.prepare(`
|
// English-like words use the full-text index; other scripts, which it does not split into words, must appear in the text.
|
||||||
select entity_type, entity_id, title, body, facet_json
|
const rows = this.db.prepare(`
|
||||||
from search_documents
|
select entity_type, entity_id, title, body, facet_json
|
||||||
where search_documents match ? and org_id = ?
|
from search_documents
|
||||||
order by rank
|
where org_id = ? ${query.match ? "and search_documents match ?" : ""}
|
||||||
`).all(query, input.orgId) as unknown as SearchRow[]
|
order by ${query.match ? "rank" : "rowid desc"}
|
||||||
: this.db.prepare(`
|
`).all(input.orgId, ...(query.match ? [query.match] : [])) as unknown as SearchRow[];
|
||||||
select entity_type, entity_id, title, body, facet_json
|
|
||||||
from search_documents
|
|
||||||
where org_id = ?
|
|
||||||
order by rowid desc
|
|
||||||
`).all(input.orgId) as unknown as SearchRow[];
|
|
||||||
const readable = rows
|
const readable = rows
|
||||||
|
.filter((row) => query.contains.every((word) => row.title.toLowerCase().includes(word) || row.body.toLowerCase().includes(word)))
|
||||||
.filter((row) => entityTypes.has(row.entity_type))
|
.filter((row) => entityTypes.has(row.entity_type))
|
||||||
.filter((row) => this.canRead(input.orgId, input.employeeId, row.entity_type, row.entity_id));
|
.filter((row) => this.canRead(input.orgId, input.employeeId, row.entity_type, row.entity_id));
|
||||||
const results = readable
|
const results = readable
|
||||||
@@ -210,9 +206,17 @@ function doc(entityType: SearchEntityType, entityId: string, title: string, body
|
|||||||
return { entityType, entityId, title: title.trim(), body: body.trim(), facets };
|
return { entityType, entityId, title: title.trim(), body: body.trim(), facets };
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeQuery(query: string): string {
|
/** Splits the words: ASCII ones become prefix terms for the index, the rest Unicode-lowercased substrings; blank when nothing searchable remains. */
|
||||||
const terms = query.toLowerCase().match(/[a-z0-9_@.-]+/g) ?? [];
|
function parseQuery(query: string): { blank: boolean; match: string | null; contains: string[] } {
|
||||||
return terms.map((term) => `${term.replaceAll('"', "")}*`).join(" AND ");
|
const words = query.toLowerCase().split(/\s+/).filter(Boolean);
|
||||||
|
const terms: string[] = [];
|
||||||
|
const contains: string[] = [];
|
||||||
|
for (const word of words) {
|
||||||
|
if ([...word].every((character) => character.charCodeAt(0) < 128)) terms.push(...(word.match(/[a-z0-9_@.-]+/g) ?? []));
|
||||||
|
else contains.push(word);
|
||||||
|
}
|
||||||
|
const match = terms.length > 0 ? terms.map((term) => `"${term}"*`).join(" AND ") : null;
|
||||||
|
return { blank: !match && contains.length === 0, match, contains };
|
||||||
}
|
}
|
||||||
|
|
||||||
function validEntityTypes(entityTypes: SearchEntityType[]): Set<SearchEntityType> {
|
function validEntityTypes(entityTypes: SearchEntityType[]): Set<SearchEntityType> {
|
||||||
|
|||||||
@@ -71,6 +71,20 @@ test("a search hit opens the object itself, not just its page", async () => {
|
|||||||
await expect(searchDialog.getByRole("button", { name: "Show more" })).toHaveCount(0);
|
await expect(searchDialog.getByRole("button", { name: "Show more" })).toHaveCount(0);
|
||||||
await searchDialog.getByRole("textbox").fill("qqzzxx");
|
await searchDialog.getByRole("textbox").fill("qqzzxx");
|
||||||
await expect(searchDialog.getByText("Nothing matches “qqzzxx”.")).toBeVisible();
|
await expect(searchDialog.getByText("Nothing matches “qqzzxx”.")).toBeVisible();
|
||||||
|
// Chinese words are found inside longer text.
|
||||||
|
await page.evaluate(async () => {
|
||||||
|
const api = (window as unknown as { agentCompany: {
|
||||||
|
loadSnapshot: () => Promise<{ ok: true; data: { activeOrgId: string } }>;
|
||||||
|
createTask: (input: unknown) => Promise<{ ok: boolean }>;
|
||||||
|
} }).agentCompany;
|
||||||
|
const orgId = (await api.loadSnapshot()).data.activeOrgId;
|
||||||
|
await api.createTask({
|
||||||
|
orgId, title: "季度复盘会议", description: "d", status: "backlog", assigneeAgentId: null, risk: "low",
|
||||||
|
projectId: null, goalId: null, workspaceId: null, priority: "normal", requiresFilesystem: false,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
await searchDialog.getByRole("textbox").fill("复盘");
|
||||||
|
await expect(searchDialog.getByRole("button", { name: /季度复盘会议/ })).toBeVisible();
|
||||||
await page.keyboard.press("Escape");
|
await page.keyboard.press("Escape");
|
||||||
|
|
||||||
// A message opens its conversation with its thread shown and the message marked.
|
// A message opens its conversation with its thread shown and the message marked.
|
||||||
|
|||||||
@@ -176,4 +176,32 @@ describe("search service", () => {
|
|||||||
expect(search.search({ orgId, employeeId, query: "nothinglikethis", limit: 12 })).toEqual({ results: [], total: 0 });
|
expect(search.search({ orgId, employeeId, query: "nothinglikethis", limit: 12 })).toEqual({ results: [], total: 0 });
|
||||||
connection.close();
|
connection.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("finds Chinese words inside text, alone or with English words", () => {
|
||||||
|
const connection = createConnection();
|
||||||
|
const organization = new OrganizationService(connection.db);
|
||||||
|
const search = new SearchService(connection.db);
|
||||||
|
let orgId = "";
|
||||||
|
connection.transaction(() => {
|
||||||
|
orgId = organization.createOrganization({ organizationName: "中文公司", employeeName: "Dana Lee", mission: "m", goal: "g" });
|
||||||
|
for (const title of ["数据迁移计划", "数据迁移 rollout plan", "Quarterly rollout", "100%_完成", "CAFÉ migration"]) {
|
||||||
|
organization.createTask({
|
||||||
|
orgId, title, description: "d", status: TASK_STATUS.BACKLOG, assigneeAgentId: null, risk: "low",
|
||||||
|
projectId: null, goalId: null, workspaceId: null, priority: "normal", requiresFilesystem: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
const employeeId = (connection.db.prepare("select id from employees where org_id = ?").get(orgId) as { id: string }).id;
|
||||||
|
const titles = (query: string) => search.search({ orgId, employeeId, query, entityTypes: ["task"] }).results.map((hit) => hit.title).sort();
|
||||||
|
expect(titles("迁移")).toEqual(["数据迁移 rollout plan", "数据迁移计划"]);
|
||||||
|
expect(titles("迁移 rollout")).toEqual(["数据迁移 rollout plan"]);
|
||||||
|
expect(titles("rollout")).toEqual(["Quarterly rollout", "数据迁移 rollout plan"]);
|
||||||
|
// LIKE wildcards in the words are taken literally.
|
||||||
|
expect(titles("%_完")).toEqual(["100%_完成"]);
|
||||||
|
// Letters outside ASCII match whatever their case.
|
||||||
|
expect(titles("café")).toEqual(["CAFÉ migration"]);
|
||||||
|
expect(titles("CAFÉ")).toEqual(["CAFÉ migration"]);
|
||||||
|
expect(search.search({ orgId, employeeId, query: "!!", entityTypes: ["task"] })).toEqual({ results: [], total: 0 });
|
||||||
|
connection.close();
|
||||||
|
});
|
||||||
});
|
});
|
||||||
Reference in new issue
Block a user