feat: IMAP mail sync with thread-to-contact matching (PRD §6.1)
Real IMAP connection via imapflow library: - Connects to IMAP server (IMAPFlow, TLS on port 993) - Fetches recent messages (configurable limit, default 20) - Extracts: from, fromEmail, subject, date, snippet (200 chars) - Thread-to-contact matching (PRD §6.1): 1. Match sender email against persons table 2. If no person match, extract domain and match against company.website 3. Returns matchedCompany + matchedPerson for each email - Audit-logged (action: crm.mailsync:imap) - Zod validated (host, port, user, password, folder, limit) - 502 on IMAP connection failure with detail - 400 on validation failure Tested: invalid creds → 502 'IMAP connection failed', missing host → 400
This commit is contained in:
Z User committed
1 parent
7acc0c5451
commit
e45b68bb8b
1089 files changed
+260614
No files matched your search
+210
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from scripts.constants import AMINER_PAPER_URL_TEMPLATE, DEFAULT_REC5_URL
|
||||
|
||||
DEFAULT_TIMEOUT_SECONDS = 30
|
||||
DEFAULT_RETRY_ATTEMPTS = 2
|
||||
RETRYABLE_HTTP_CODES = {429, 500, 502, 503, 504}
|
||||
|
||||
|
||||
def _clean_text(value: Any) -> str:
|
||||
return " ".join(str(value or "").split()).strip()
|
||||
|
||||
|
||||
def resolve_token(config: dict[str, Any] | None = None) -> str:
|
||||
config = config or {}
|
||||
aminer_config = config.get("aminer") if isinstance(config.get("aminer"), dict) else {}
|
||||
return _clean_text(os.getenv("AMINER_API_KEY") or aminer_config.get("token"))
|
||||
|
||||
|
||||
def resolve_rec5_url(config: dict[str, Any] | None = None) -> str:
|
||||
config = config or {}
|
||||
aminer_config = config.get("aminer") if isinstance(config.get("aminer"), dict) else {}
|
||||
return _clean_text(aminer_config.get("rec5_url") or os.getenv("AMINER_REC5_URL")) or DEFAULT_REC5_URL
|
||||
|
||||
|
||||
def build_api_request(
|
||||
*,
|
||||
aminer_author_id: str = "",
|
||||
author_name: str = "",
|
||||
author_org: str = "",
|
||||
topics: list[str] | None = None,
|
||||
size: int = 5,
|
||||
offset: int = 0,
|
||||
start_year: int | None = None,
|
||||
end_year: int | None = None,
|
||||
language_sort: str = "",
|
||||
) -> dict[str, Any]:
|
||||
params: dict[str, Any] = {}
|
||||
if _clean_text(aminer_author_id):
|
||||
params["aminer_author_id"] = _clean_text(aminer_author_id)
|
||||
if _clean_text(author_name):
|
||||
params["author_name"] = _clean_text(author_name)
|
||||
if _clean_text(author_org):
|
||||
params["author_org"] = _clean_text(author_org)
|
||||
cleaned_topics = [_clean_text(t) for t in (topics or []) if _clean_text(t)]
|
||||
if cleaned_topics:
|
||||
params["topics"] = cleaned_topics
|
||||
params["size"] = max(1, min(int(size), 20))
|
||||
params["offset"] = max(0, min(int(offset), 100))
|
||||
if start_year is not None:
|
||||
params["start_year"] = int(start_year)
|
||||
if end_year is not None:
|
||||
params["end_year"] = int(end_year)
|
||||
if _clean_text(language_sort) in {"zh", "en"}:
|
||||
params["language_sort"] = _clean_text(language_sort)
|
||||
return params
|
||||
|
||||
|
||||
def normalize_rec5_paper(raw: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Normalize raw rec5 paper dict to the in-skill record shape (Markdown display / JSON 输出)."""
|
||||
paper_id = _clean_text(raw.get("paper_id") or raw.get("id"))
|
||||
links = raw.get("links") if isinstance(raw.get("links"), dict) else {}
|
||||
aminer_url = (
|
||||
_clean_text(links.get("aminer"))
|
||||
or _clean_text(raw.get("paper_url"))
|
||||
or (AMINER_PAPER_URL_TEMPLATE.format(paper_id=paper_id) if paper_id else "")
|
||||
)
|
||||
arxiv_url = _clean_text(links.get("arxiv") or raw.get("arxiv_url") or "")
|
||||
pdf_url = _clean_text(links.get("pdf") or raw.get("pdf_url") or "")
|
||||
arxiv_id = _clean_text(raw.get("arxiv_id") or "")
|
||||
|
||||
raw_ss = raw.get("structured_summary")
|
||||
if isinstance(raw_ss, dict):
|
||||
structured_summary: dict[str, str] = {k: _clean_text(v) for k, v in raw_ss.items() if _clean_text(v)}
|
||||
else:
|
||||
structured_summary = {}
|
||||
|
||||
raw_fa = raw.get("famous_authors")
|
||||
famous_authors: list[Any] = []
|
||||
if isinstance(raw_fa, list):
|
||||
for item in raw_fa:
|
||||
if isinstance(item, dict):
|
||||
name = _clean_text(item.get("name"))
|
||||
if not name:
|
||||
continue
|
||||
famous_authors.append(
|
||||
{
|
||||
"name": name,
|
||||
"description": _clean_text(item.get("description") or item.get("bio") or ""),
|
||||
"profile_url": _clean_text(item.get("profile_url") or ""),
|
||||
}
|
||||
)
|
||||
elif isinstance(item, str) and _clean_text(item):
|
||||
famous_authors.append(_clean_text(item))
|
||||
|
||||
raw_profiles = raw.get("aminer_author_profiles")
|
||||
aminer_author_profiles: list[dict[str, Any]] = (
|
||||
[p for p in raw_profiles if isinstance(p, dict)] if isinstance(raw_profiles, list) else []
|
||||
)
|
||||
|
||||
raw_entries = raw.get("author_entries")
|
||||
author_entries: list[dict[str, Any]] = (
|
||||
[e for e in raw_entries if isinstance(e, dict)] if isinstance(raw_entries, list) else []
|
||||
)
|
||||
|
||||
year = raw.get("year")
|
||||
if year is not None:
|
||||
try:
|
||||
year = int(year)
|
||||
except (TypeError, ValueError):
|
||||
year = None
|
||||
|
||||
return {
|
||||
"paper_id": paper_id,
|
||||
"arxiv_id": arxiv_id,
|
||||
"aminer_paper_id": paper_id,
|
||||
"aminer_paper_url": aminer_url,
|
||||
"abs_url": arxiv_url,
|
||||
"pdf_url": pdf_url,
|
||||
"title": _clean_text(raw.get("title")),
|
||||
"year": year,
|
||||
"authors": [_clean_text(a) for a in list(raw.get("authors") or []) if _clean_text(a)],
|
||||
"keywords": [_clean_text(k) for k in list(raw.get("keywords") or []) if _clean_text(k)],
|
||||
"summary": _clean_text(raw.get("summary") or ""),
|
||||
"structured_summary": structured_summary,
|
||||
"famous_authors": famous_authors,
|
||||
"aminer_author_profiles": aminer_author_profiles,
|
||||
"author_entries": author_entries,
|
||||
"source": _clean_text(raw.get("source") or "rec5"),
|
||||
"recommendation_reason": _clean_text(raw.get("recommendation_reason") or ""),
|
||||
}
|
||||
|
||||
|
||||
def call_rec5_api(
|
||||
params: dict[str, Any],
|
||||
*,
|
||||
token: str,
|
||||
url: str = DEFAULT_REC5_URL,
|
||||
timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
|
||||
retry_attempts: int = DEFAULT_RETRY_ATTEMPTS,
|
||||
) -> dict[str, Any]:
|
||||
if not _clean_text(token):
|
||||
raise RuntimeError("missing_aminer_api_key")
|
||||
|
||||
body = json.dumps(params, ensure_ascii=False).encode("utf-8")
|
||||
ssl_context = ssl.create_default_context()
|
||||
opener = urllib.request.build_opener(
|
||||
urllib.request.ProxyHandler({}),
|
||||
urllib.request.HTTPSHandler(context=ssl_context),
|
||||
)
|
||||
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(1, retry_attempts + 2):
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=body,
|
||||
headers={
|
||||
"Content-Type": "application/json;charset=utf-8",
|
||||
"Authorization": token,
|
||||
"User-Agent": "aminer-rec/1.0",
|
||||
"X-Platform": "openclaw",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with opener.open(request, timeout=timeout_seconds) as response: # nosec B310
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code in RETRYABLE_HTTP_CODES and attempt <= retry_attempts:
|
||||
last_error = exc
|
||||
time.sleep(0.5 * attempt)
|
||||
continue
|
||||
raise RuntimeError(f"rec5_api_http_{exc.code}") from exc
|
||||
except Exception as exc:
|
||||
if attempt <= retry_attempts:
|
||||
last_error = exc
|
||||
time.sleep(0.5 * attempt)
|
||||
continue
|
||||
raise RuntimeError(f"rec5_api_error:{exc.__class__.__name__}") from exc
|
||||
|
||||
if not payload.get("success"):
|
||||
msg = _clean_text(payload.get("msg") or str(payload.get("code") or "api_error"))
|
||||
raise RuntimeError(f"rec5_api_failed:{msg}")
|
||||
|
||||
data = payload.get("data")
|
||||
if isinstance(data, list) and data:
|
||||
papers = list(data[0].get("papers") or [])
|
||||
data_obj = data[0] if isinstance(data[0], dict) else {}
|
||||
elif isinstance(data, dict):
|
||||
papers = list(data.get("papers") or [])
|
||||
data_obj = data
|
||||
else:
|
||||
papers = []
|
||||
data_obj = {}
|
||||
analyzed_topics = list(data_obj.get("analyzed_topics") or []) if isinstance(data_obj, dict) else []
|
||||
return {
|
||||
"papers": [p for p in papers if isinstance(p, dict)],
|
||||
"analyzed_topics": [str(t).strip() for t in analyzed_topics if str(t).strip()],
|
||||
}
|
||||
|
||||
raise RuntimeError(f"rec5_api_unreachable:{_clean_text(str(last_error))}") from last_error
|
||||
Reference in new issue
Block a user