"""PST file parsing utilities for message import.
Broad exception handling (W0718/C0302) is intentional: PST parsing relies on
the pypff C library which can raise arbitrary exceptions on malformed data.
Each field extraction must be individually guarded to maximise data recovery.
"""
# pylint: disable=broad-exception-caught,too-many-lines
import base64
import hashlib
import re
import struct
import uuid
from collections.abc import Callable, Generator
from datetime import datetime, timezone
from email import message_from_string
from django.conf import settings
import pypff
from celery.utils.log import get_task_logger
from jmap_email import (
ComposeError,
compose_email,
is_valid_addr_spec,
is_valid_msg_id,
parse_address,
parse_addresses,
parse_date,
sent_at_to_datetime,
)
from jmap_email.types import EmailAddress
from core.mda.addresses import address_domain
from core.mda.utils import ARCHIVE_COMPOSE_OPTIONS
from core.services.s3_seekable import BUFFER_NONE, S3SeekableReader
from .channel import ImportCancelled
from .utils import (
FAILURE_OVERSIZED,
FAILURE_UNPARSEABLE,
beat,
bump_reason,
deliver,
imports_storage,
run_plan,
)
# A task logger (like the other importer modules) so each per-message line
# carries the ``run_import_task`` id, which ``run_import_task`` maps to its
# import channel when it starts.
logger = get_task_logger(__name__)
# MAPI property tags — message store
PR_CONTAINER_CLASS = 0x3613
PR_MESSAGE_FLAGS = 0x0E07
PR_IPM_SUBTREE_ENTRYID = 0x35E0
PR_IPM_SENTMAIL_ENTRYID = 0x35E4
PR_IPM_WASTEBASKET_ENTRYID = 0x35E3
PR_IPM_OUTBOX_ENTRYID = 0x35E2
# MAPI property tags — sender
PR_SENDER_EMAIL_ADDRESS = 0x0C1F
PR_SENDER_SMTP_ADDRESS = 0x5D01
PR_SENDER_ADDRTYPE = 0x0C1E
# MAPI property tags — sent-representing (delegation / shared mailboxes).
# For internal Exchange messages, PR_SENDER_* may hold the X.500 DN while
# PR_SENT_REPRESENTING_SMTP_ADDRESS carries the resolvable SMTP.
PR_SENT_REPRESENTING_NAME = 0x0042
PR_SENT_REPRESENTING_ADDRTYPE = 0x0064
PR_SENT_REPRESENTING_EMAIL_ADDRESS = 0x0065
PR_SENT_REPRESENTING_SMTP_ADDRESS = 0x5D02
# MAPI property tags — creator / last modifier.
# Last-resort SMTP fallback for messages where every PR_SENDER_* and
# PR_SENT_REPRESENTING_* slot only holds an X.500 DN. For sent items
# composed by a delegate of a shared mailbox, these typically carry the
# delegate's real SMTP — semantically imperfect (the human-facing sender
# is the shared mailbox) but at least a resolvable contact.
PR_CREATOR_SMTP_ADDRESS = 0x5D0A
PR_LAST_MODIFIER_SMTP_ADDRESS = 0x5D0B
# MAPI property tags — recipients
PR_DISPLAY_NAME = 0x3001
PR_ADDRTYPE = 0x3002
PR_EMAIL_ADDRESS = 0x3003
PR_RECIPIENT_TYPE = 0x0C15
PR_SMTP_ADDRESS = 0x39FE
# MAPI property tags — pre-formatted "display" recipient strings.
# Outlook keeps a header-style copy of the To/Cc/Bcc fields as
# semicolon-separated strings, used as a fallback when pypff exposes an
# empty recipient table.
PR_DISPLAY_BCC = 0x0E02
PR_DISPLAY_CC = 0x0E03
PR_DISPLAY_TO = 0x0E04
# MAPI property tags — attachments
PR_ATTACH_LONG_FILENAME = 0x3707
PR_ATTACH_FILENAME = 0x3704
PR_ATTACH_MIME_TAG = 0x370E
PR_ATTACH_CONTENT_ID = 0x3712
PR_ATTACH_METHOD = 0x3705
# MAPI property tags — flags
PR_FLAG_STATUS = 0x1090
# MAPI property tag — Internet Message-ID stored natively on the PST message,
# independent of transport_headers. Outlook populates it for received items
# and often for drafts; reading it lets us dedupe messages whose
# transport_headers were lost or never existed.
PR_INTERNET_MESSAGE_ID = 0x1035
# Attachment method values
ATTACH_BY_VALUE = 1
ATTACH_EMBEDDED_MSG = 5
# Message flag bits
MSGFLAG_READ = 0x1
MSGFLAG_UNSENT = 0x8
# Flag status values
FLAG_STATUS_FOLLOWUP = 2 # Flagged for follow-up
# Container class prefixes for folders that hold mail items (MAPI standard).
# IPF.Note — standard Outlook mail folders.
# IPF.Imap — IMAP accounts archived to PST keep this class on their mail
# folders (e.g. the inbox), so they must be treated as mail too.
EMAIL_CONTAINER_CLASS_PREFIXES = ("IPF.Note", "IPF.Imap")
# Maximum recursion depth for PST folder traversal
MAX_FOLDER_DEPTH = 50
# Folder types
FOLDER_TYPE_NORMAL = "normal"
FOLDER_TYPE_INBOX = "inbox"
FOLDER_TYPE_SENT = "sent"
FOLDER_TYPE_DELETED = "deleted"
FOLDER_TYPE_OUTBOX = "outbox"
FOLDER_TYPE_DRAFTS = "drafts"
# Regex to extract charset from HTML meta tags
_CHARSET_RE = re.compile(rb']+charset=["\']?([^"\';>\s]+)', re.IGNORECASE)
# Part of the UI contract: the imports list matches this marker in the run's
# error to show the dedicated "archive is corrupt, retrying will not help"
# message instead of the generic one (see imports-data-grid.tsx). Shared by
# every ``PSTFileUnreadableError`` raise site so the copies can't drift apart.
PST_UNREADABLE_MARKER = "PST_UNREADABLE"
class PSTFileUnreadableError(RuntimeError):
"""The PST file cannot be parsed.
Raised when the archive is corrupt or its MAPI tree is missing the
structures we need (root folder or message store). The wider importer
pipeline catches this to surface a user-facing message distinct from
the generic processing-error fallback.
"""
class OversizedAttachmentError(RuntimeError):
"""A PST attachment exceeds the per-message size budget.
Raised out of ``reconstruct_eml`` so the whole message is counted as a
visible failure (``_reconstruct_plan_item`` logs it and returns ``None``)
rather than delivered without some attachments as an apparent success.
"""
def assert_pst_readable(pst_file) -> None:
"""Probe the PST for the structures the walk relies on.
Some corrupt archives open without raising, then later fail deep in
the traversal with an AttributeError on a NoneType. Touching the root
folder and message store up-front turns those late, opaque failures
into a single explicit error at task boot.
"""
try:
root = pst_file.get_root_folder()
_ = root.number_of_sub_folders
store = pst_file.get_message_store()
_ = store.number_of_record_sets
except Exception as exc:
raise PSTFileUnreadableError(
f"{PST_UNREADABLE_MARKER}: the archive is missing its root folder "
"or message store (corrupt or truncated file)."
) from exc
def get_mapi_property(item, property_tag):
"""Get a MAPI property entry from an item's record sets by tag."""
for record_set_idx in range(item.number_of_record_sets):
record_set = item.get_record_set(record_set_idx)
for entry_idx in range(record_set.get_number_of_entries()):
entry = record_set.get_entry(entry_idx)
if entry.entry_type == property_tag:
return entry
return None
def get_mapi_property_data(item, property_tag) -> bytes | None:
"""Get raw data bytes for a MAPI property."""
entry = get_mapi_property(item, property_tag)
if entry is not None:
try:
return entry.data
except Exception:
return None
return None
def get_mapi_property_integer(item, property_tag) -> int | None:
"""Get an integer value for a MAPI property."""
entry = get_mapi_property(item, property_tag)
if entry is not None:
try:
return entry.data_as_integer
except Exception:
return None
return None
def get_mapi_property_string(item, property_tag) -> str | None:
"""Get a string value for a MAPI property."""
entry = get_mapi_property(item, property_tag)
if entry is not None:
try:
return entry.data_as_string
except Exception:
# Fall back to raw data decoded as utf-8
try:
raw = entry.data
if isinstance(raw, bytes):
return raw.decode("utf-8", errors="replace").rstrip("\x00")
except Exception:
logger.debug("Failed to decode MAPI property 0x%04X", property_tag)
return None
def _folder_id_from_entry_id(entry_id: bytes | None) -> int | None:
"""Extract the folder identifier from a MAPI entry ID.
PST entry IDs are 24 bytes: 4 flags + 16 UID + 4 folder_id (LE uint32).
The last 4 bytes match folder.get_identifier().
"""
if entry_id and len(entry_id) >= 4:
return struct.unpack_from(" dict:
"""Build a mapping from folder identifiers to folder types.
Reads PR_IPM_SENTMAIL_ENTRYID, PR_IPM_WASTEBASKET_ENTRYID, and
PR_IPM_OUTBOX_ENTRYID from the message store, extracts the folder
identifier from each entry ID, and maps it to a folder type.
"""
special_map = {}
try:
store = pst_file.get_message_store()
for tag, folder_type in [
(PR_IPM_SENTMAIL_ENTRYID, FOLDER_TYPE_SENT),
(PR_IPM_WASTEBASKET_ENTRYID, FOLDER_TYPE_DELETED),
(PR_IPM_OUTBOX_ENTRYID, FOLDER_TYPE_OUTBOX),
]:
entry_id = get_mapi_property_data(store, tag)
folder_id = _folder_id_from_entry_id(entry_id)
if folder_id is not None:
special_map[folder_id] = folder_type
except Exception:
logger.debug("Could not read message store properties for special folder map")
return special_map
def get_store_owner_email(pst_file) -> str | None:
"""Get the mailbox owner's email from the message store PR_DISPLAY_NAME."""
try:
store = pst_file.get_message_store()
except Exception:
return None
display_name = get_mapi_property_string(store, PR_DISPLAY_NAME)
if display_name and "@" in display_name:
return display_name
return None
# SourceWellKnownFolderType — named property from Microsoft's migration tools.
# GUID {9137a2fd-2fa5-4409-91aa-2c3ee697350a}, string name "SourceWellKnownFolderType".
# Present on PSTs exported via Exchange/O365 migration; absent from local Outlook PSTs.
_SOURCE_WKFT_GUID = "9137a2fd-2fa5-4409-91aa-2c3ee697350a"
_SOURCE_WKFT_NAME = "SourceWellKnownFolderType"
_SOURCE_WKFT_VALUES = {
10: FOLDER_TYPE_INBOX,
11: FOLDER_TYPE_SENT,
12: FOLDER_TYPE_OUTBOX,
14: FOLDER_TYPE_DELETED,
17: FOLDER_TYPE_DRAFTS,
}
def _resolve_named_property_tag(pst_file, target_guid: str, target_name: str):
"""Resolve a string-named property to its NPID tag.
Reads the Name-to-ID Map to find the property tag for a given
(GUID, string_name) pair. Returns the NPID (e.g. 0x8022) or None.
"""
try:
name_map = pst_file.get_name_to_id_map()
rs = name_map.get_record_set(0)
entry_stream = guid_stream = string_stream = None
for e_idx in range(rs.get_number_of_entries()):
entry = rs.get_entry(e_idx)
tag = entry.entry_type
if tag == 0x0002:
guid_stream = entry.data
elif tag == 0x0003:
entry_stream = entry.data
elif tag == 0x0004:
string_stream = entry.data
if not entry_stream or not guid_stream or not string_stream:
return None
target_name_lower = target_name.lower()
for i in range(len(entry_stream) // 8):
record = entry_stream[i * 8 : i * 8 + 8]
dw_prop_id = struct.unpack_from("> 1) & 0x7FFF
if n_bit != 1:
continue # Not a string-named property
# Resolve GUID
if w_guid < 3:
continue # PS_MAPI or PS_PUBLIC_STRINGS — not what we want
guid_offset = (w_guid - 3) * 16
if guid_offset + 16 > len(guid_stream):
continue
guid_str = str(
uuid.UUID(bytes_le=guid_stream[guid_offset : guid_offset + 16])
)
if guid_str != target_guid:
continue
# Read string name (4-byte LE length prefix + UTF-16LE)
if dw_prop_id + 4 > len(string_stream):
continue
str_len = struct.unpack_from(" len(string_stream):
continue
prop_name = string_stream[dw_prop_id + 4 : dw_prop_id + 4 + str_len].decode(
"utf-16-le", errors="replace"
)
if prop_name.lower() == target_name_lower:
return 0x8000 + w_prop_idx
except Exception:
logger.debug("Failed to resolve named property %s", target_name)
return None
def build_well_known_folder_map(pst_file, ipm_subtree) -> dict:
"""Build folder map from SourceWellKnownFolderType named property.
This named property is set by Microsoft migration tools on Exchange/O365
PSTs. Returns a dict mapping folder identifier → folder type, or empty
dict if the property is absent.
"""
tag = _resolve_named_property_tag(pst_file, _SOURCE_WKFT_GUID, _SOURCE_WKFT_NAME)
if tag is None:
return {}
wkft_map = {}
try:
for i in range(ipm_subtree.number_of_sub_folders):
folder = ipm_subtree.get_sub_folder(i)
val = get_mapi_property_integer(folder, tag)
if val is not None and val in _SOURCE_WKFT_VALUES:
wkft_map[folder.get_identifier()] = _SOURCE_WKFT_VALUES[val]
except Exception:
logger.debug("Failed to read SourceWellKnownFolderType from folders")
return wkft_map
def _is_email_folder(folder) -> bool:
"""Check if a folder contains email items based on PR_CONTAINER_CLASS."""
entry = get_mapi_property(folder, PR_CONTAINER_CLASS)
if entry is None:
# No container class set — treat as email folder (safe default)
return True
try:
container_class = entry.data_as_string
return container_class.startswith(EMAIL_CONTAINER_CLASS_PREFIXES)
except Exception:
logger.debug("Failed to read container class for folder")
return True
def _get_folder_type(folder, special_folder_map: dict) -> str:
"""Determine the folder type by matching its identifier against the special folder map."""
try:
folder_id = folder.get_identifier()
if folder_id in special_folder_map:
return special_folder_map[folder_id]
except Exception:
logger.debug("Failed to get folder identifier")
return FOLDER_TYPE_NORMAL
# Mapping of known Outlook default folder names (lowercased) to folder types.
# Used as fallback when MAPI entry ID properties are absent from the message store
# (PR_VALID_FOLDER_MASK bits not set). Only applied to direct children of the
# IPM subtree — these are always the standard Outlook special folders.
# fmt: off
_KNOWN_FOLDER_NAMES = {
# Inbox
"inbox": FOLDER_TYPE_INBOX,
"boîte de réception": FOLDER_TYPE_INBOX,
"posteingang": FOLDER_TYPE_INBOX,
"bandeja de entrada": FOLDER_TYPE_INBOX,
"posta in arrivo": FOLDER_TYPE_INBOX,
"postvak in": FOLDER_TYPE_INBOX,
"caixa de entrada": FOLDER_TYPE_INBOX,
"a]receber": FOLDER_TYPE_INBOX,
"входящие": FOLDER_TYPE_INBOX,
"skrzynka odbiorcza": FOLDER_TYPE_INBOX,
"doručená pošta": FOLDER_TYPE_INBOX,
"beérkezett üzenetek": FOLDER_TYPE_INBOX,
"indbakke": FOLDER_TYPE_INBOX,
"innboks": FOLDER_TYPE_INBOX,
"inkorg": FOLDER_TYPE_INBOX,
"saapuneet": FOLDER_TYPE_INBOX,
"gelen kutusu": FOLDER_TYPE_INBOX,
"受信トレイ": FOLDER_TYPE_INBOX,
"收件箱": FOLDER_TYPE_INBOX,
"收件匣": FOLDER_TYPE_INBOX,
"받은 편지함": FOLDER_TYPE_INBOX,
"علبة الوارد": FOLDER_TYPE_INBOX,
"דואר נכנס": FOLDER_TYPE_INBOX,
"вхідні": FOLDER_TYPE_INBOX,
"primite": FOLDER_TYPE_INBOX,
# Sent Items
"sent items": FOLDER_TYPE_SENT,
"sent": FOLDER_TYPE_SENT,
"éléments envoyés": FOLDER_TYPE_SENT,
"gesendete elemente": FOLDER_TYPE_SENT,
"gesendete objekte": FOLDER_TYPE_SENT,
"elementos enviados": FOLDER_TYPE_SENT,
"posta inviata": FOLDER_TYPE_SENT,
"verzonden items": FOLDER_TYPE_SENT,
"itens enviados": FOLDER_TYPE_SENT,
"отправленные": FOLDER_TYPE_SENT,
"отправленные элементы": FOLDER_TYPE_SENT,
"elementy wysłane": FOLDER_TYPE_SENT,
"odeslaná pošta": FOLDER_TYPE_SENT,
"elküldött elemek": FOLDER_TYPE_SENT,
"sendte elementer": FOLDER_TYPE_SENT,
"skickade objekt": FOLDER_TYPE_SENT,
"lähetetyt": FOLDER_TYPE_SENT,
"gönderilmiş öğeler": FOLDER_TYPE_SENT,
"送信済みアイテム": FOLDER_TYPE_SENT,
"已发送邮件": FOLDER_TYPE_SENT,
"寄件備份": FOLDER_TYPE_SENT,
"보낸 편지함": FOLDER_TYPE_SENT,
"العناصر المرسلة": FOLDER_TYPE_SENT,
"פריטים שנשלחו": FOLDER_TYPE_SENT,
"надіслані": FOLDER_TYPE_SENT,
"trimise": FOLDER_TYPE_SENT,
# Drafts
"drafts": FOLDER_TYPE_DRAFTS,
"brouillons": FOLDER_TYPE_DRAFTS,
"entwürfe": FOLDER_TYPE_DRAFTS,
"borradores": FOLDER_TYPE_DRAFTS,
"bozze": FOLDER_TYPE_DRAFTS,
"concepten": FOLDER_TYPE_DRAFTS,
"rascunhos": FOLDER_TYPE_DRAFTS,
"черновики": FOLDER_TYPE_DRAFTS,
"wersje robocze": FOLDER_TYPE_DRAFTS,
"koncepty": FOLDER_TYPE_DRAFTS,
"piszkozatok": FOLDER_TYPE_DRAFTS,
"kladder": FOLDER_TYPE_DRAFTS,
"utkast": FOLDER_TYPE_DRAFTS,
"luonnokset": FOLDER_TYPE_DRAFTS,
"taslaklar": FOLDER_TYPE_DRAFTS,
"下書き": FOLDER_TYPE_DRAFTS,
"草稿": FOLDER_TYPE_DRAFTS,
"임시 보관함": FOLDER_TYPE_DRAFTS,
"مسودات": FOLDER_TYPE_DRAFTS,
"טיוטות": FOLDER_TYPE_DRAFTS,
"чернетки": FOLDER_TYPE_DRAFTS,
"ciorne": FOLDER_TYPE_DRAFTS,
# Outbox
"outbox": FOLDER_TYPE_OUTBOX,
"boîte d'envoi": FOLDER_TYPE_OUTBOX,
"postausgang": FOLDER_TYPE_OUTBOX,
"bandeja de salida": FOLDER_TYPE_OUTBOX,
"posta in uscita": FOLDER_TYPE_OUTBOX,
"postvak uit": FOLDER_TYPE_OUTBOX,
"caixa de saída": FOLDER_TYPE_OUTBOX,
"a]enviar": FOLDER_TYPE_OUTBOX,
"исходящие": FOLDER_TYPE_OUTBOX,
"skrzynka nadawcza": FOLDER_TYPE_OUTBOX,
"pošta k odeslání": FOLDER_TYPE_OUTBOX,
"postázandó üzenetek": FOLDER_TYPE_OUTBOX,
"udbakke": FOLDER_TYPE_OUTBOX,
"utboks": FOLDER_TYPE_OUTBOX,
"utkorgen": FOLDER_TYPE_OUTBOX,
"lähtevät": FOLDER_TYPE_OUTBOX,
"giden kutusu": FOLDER_TYPE_OUTBOX,
"送信トレイ": FOLDER_TYPE_OUTBOX,
"发件箱": FOLDER_TYPE_OUTBOX,
"寄件匣": FOLDER_TYPE_OUTBOX,
"보낼 편지함": FOLDER_TYPE_OUTBOX,
"علبة الصادر": FOLDER_TYPE_OUTBOX,
"דואר יוצא": FOLDER_TYPE_OUTBOX,
"вихідні": FOLDER_TYPE_OUTBOX,
# Deleted Items
"deleted items": FOLDER_TYPE_DELETED,
"trash": FOLDER_TYPE_DELETED,
"éléments supprimés": FOLDER_TYPE_DELETED,
"gelöschte elemente": FOLDER_TYPE_DELETED,
"gelöschte objekte": FOLDER_TYPE_DELETED,
"elementos eliminados": FOLDER_TYPE_DELETED,
"posta eliminata": FOLDER_TYPE_DELETED,
"verwijderde items": FOLDER_TYPE_DELETED,
"itens excluídos": FOLDER_TYPE_DELETED,
"удалённые": FOLDER_TYPE_DELETED,
"удаленные": FOLDER_TYPE_DELETED,
"elementy usunięte": FOLDER_TYPE_DELETED,
"odstraněná pošta": FOLDER_TYPE_DELETED,
"törölt elemek": FOLDER_TYPE_DELETED,
"slettet post": FOLDER_TYPE_DELETED,
"slettede elementer": FOLDER_TYPE_DELETED,
"borttagna objekt": FOLDER_TYPE_DELETED,
"poistetut": FOLDER_TYPE_DELETED,
"silinmiş öğeler": FOLDER_TYPE_DELETED,
"削除済みアイテム": FOLDER_TYPE_DELETED,
"已删除邮件": FOLDER_TYPE_DELETED,
"刪除的郵件": FOLDER_TYPE_DELETED,
"지운 편지함": FOLDER_TYPE_DELETED,
"العناصر المحذوفة": FOLDER_TYPE_DELETED,
"פריטים שנמחקו": FOLDER_TYPE_DELETED,
"видалені": FOLDER_TYPE_DELETED,
}
# fmt: on
def _detect_folder_type_by_name(folder_name: str) -> str:
"""Detect folder type by matching folder name against known Outlook defaults.
Only used as a fallback when MAPI entry ID properties are absent.
Should only be applied to direct children of the IPM subtree.
"""
return _KNOWN_FOLDER_NAMES.get(folder_name.lower().strip(), FOLDER_TYPE_NORMAL)
# When the message was sent or received: the import plan is ordered on these.
_SENT_TIME_ATTRS = ("delivery_time", "client_submit_time")
# The record's own times, a last resort for a ``Date``: they may only date the
# copy into the archive, but still beat having no date at all. Kept out of the
# plan ordering — a resumed import restarts at a positional cursor, so the
# order must not change between versions — and a message lacking a sent time
# (mostly a draft) is best imported last anyway: its parents already exist.
_RECORD_TIME_ATTRS = ("creation_time", "modification_time")
def _get_message_timestamp(
message, attrs: tuple[str, ...] = _SENT_TIME_ATTRS
) -> datetime | None:
"""First tz-aware timestamp found among ``attrs``, or ``None``."""
for attr in attrs:
try:
dt = getattr(message, attr)
except Exception:
logger.debug("Failed to read %s", attr)
continue
if isinstance(dt, datetime):
# pypff can return naive datetimes; assume UTC so a mix of naive
# and tz-aware values from one PST stays mutually comparable when
# we sort chronologically (a bare compare would raise TypeError).
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt
return None
def _get_message_flags(message) -> int:
"""Get PR_MESSAGE_FLAGS from a message."""
flags = get_mapi_property_integer(message, PR_MESSAGE_FLAGS)
return flags if flags is not None else 0
def _get_flag_status(message) -> int | None:
"""Get PR_FLAG_STATUS from a message (follow-up flag)."""
return get_mapi_property_integer(message, PR_FLAG_STATUS)
# Outlook renders a one-off SMTP recipient as ``Display Name (addr@domain)``,
# sometimes wrapped in single quotes. RFC 5322 reads a parenthesised part as a
# comment, so ``parse_address`` hands the whole string back as the address,
# which the composer then rejects — losing the message.
# The name ends on a non-space so that only ``\s*`` takes the spaces before
# ``(``: with a bare ``.*?`` both could, and a long run of spaces would be
# rescanned from every position (quadratic on a crafted display name).
_OUTLOOK_DISPLAY_ADDRESS_RE = re.compile(
r"^(?P['\"]?)(?P(?:.*?\S)?)\s*\((?P[^()\s]+)\)(?P=quote)$"
)
def _unquote(value: str) -> str:
"""Drop one pair of matching quotes Outlook wraps around a display name."""
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
return value[1:-1].strip()
return value
def _unquote_address(addr: str) -> str | None:
"""Bare address Outlook wrapped in quotes, or ``None`` if it wasn't.
Outlook quotes some addresses, sometimes twice and unevenly
(``''a@b.fr'``). Such a value still passes as an addr-spec, as quotes are
legal characters. A quote *ending* the address proves the wrapping is an
artefact — no domain ends with one — so every surrounding quote is then
dropped. Without it nothing is touched: a local part may legitimately
start with a quote.
"""
addr = addr.strip()
if not addr.endswith(("'", '"')):
return None
bare = addr.strip("'\"")
return bare if is_valid_addr_spec(bare) else None
def _split_outlook_display(value: str) -> tuple[str, str] | None:
"""Split Outlook's ``Name (addr)`` form into ``(name, addr)``, or ``None``
when ``value`` isn't exactly that shape around a valid address."""
match = _OUTLOOK_DISPLAY_ADDRESS_RE.match(value.strip())
if match and is_valid_addr_spec(match["email"]):
return match["name"].strip(), match["email"]
return None
def _addr_tuple_to_dict(name: str, addr: str) -> EmailAddress:
"""Bridge the ``(name, email)`` shape returned by
:func:`jmap_email.parse_address` to a JMAP ``EmailAddress`` dict, undoing
Outlook's display artefacts.
``addr`` is repaired when Outlook wrapped it in quotes (see
:func:`_unquote_address`) or wrote it as ``Name (addr)``; ``name`` drops
surrounding quotes and the ``(addr)`` suffix.
When ``name`` is empty or is the raw string the address was repaired from,
it takes the name salvaged from it instead. Anything else is passed through
untouched: an address that is still invalid fails the whole message at
compose time rather than being dropped or rewritten.
"""
raw_addr = addr
salvaged_name = ""
unquoted = _unquote_address(addr)
if unquoted:
addr = unquoted
elif not is_valid_addr_spec(addr):
split = _split_outlook_display(addr)
if split:
salvaged_name, addr = split
if (not name or name == raw_addr) and addr != raw_addr:
name = salvaged_name
split = _split_outlook_display(name)
if split:
name = split[0]
return {"name": _unquote(name), "email": addr}
def _resolve_smtp_address(item) -> str | None:
"""Resolve an SMTP email address from a MAPI item (recipient or message).
Handles Exchange "EX" address types by checking PR_SMTP_ADDRESS first.
"""
addr_type = get_mapi_property_string(item, PR_ADDRTYPE)
# PR_ADDRTYPE is a MAPI address *kind* ("EX", "SMTP"), not an address.
if addr_type and addr_type.upper() == "EX": # pylint: disable=address-unicode-case
# Exchange address — try PR_SMTP_ADDRESS first
smtp = get_mapi_property_string(item, PR_SMTP_ADDRESS)
if smtp and "@" in smtp:
return smtp
# Fall back to PR_EMAIL_ADDRESS (may be X.500 DN, but try anyway)
email_addr = get_mapi_property_string(item, PR_EMAIL_ADDRESS)
if email_addr and "@" in email_addr:
return email_addr
return None
# SMTP or unknown — use PR_EMAIL_ADDRESS directly
email_addr = get_mapi_property_string(item, PR_EMAIL_ADDRESS)
if email_addr and "@" in email_addr:
return email_addr
# Also try PR_SMTP_ADDRESS as fallback
smtp = get_mapi_property_string(item, PR_SMTP_ADDRESS)
if smtp and "@" in smtp:
return smtp
return None
def _safe_sender_name(message) -> str:
"""Safely get sender_name from a message, returning empty string on error."""
try:
return message.sender_name or ""
except Exception:
return ""
def _extract_sender_from_mapi(
message,
store_email: str | None = None,
preferred_name: str | None = None,
) -> EmailAddress | None:
"""Extract sender address from MAPI properties on the message itself.
Order of attempts (first hit wins):
1. PR_SENDER_SMTP_ADDRESS (direct, Exchange-resolved)
2. PR_SENDER_EMAIL_ADDRESS if it looks like SMTP
3. PR_SENT_REPRESENTING_SMTP_ADDRESS (shared mailboxes / delegation)
4. PR_SENT_REPRESENTING_EMAIL_ADDRESS if SMTP
5. PR_CREATOR_SMTP_ADDRESS / PR_LAST_MODIFIER_SMTP_ADDRESS — usually
the actual delegate's SMTP for shared-mailbox sent items
6. Parse sender_name as an email address
7. store_email (common for EX sent items where only the store owner
identifies the sender)
``preferred_name`` overrides any name resolved here — used when an upstream
header (e.g. transport_headers' From) carried a valid display name but an
X.500 DN as the address, so the human-readable name is preserved while the
SMTP comes from MAPI.
"""
def _build(
fallback_name: str | None,
smtp: str,
*,
sender_name_fallback: bool = True,
) -> EmailAddress:
# ``sender_name`` is only a valid display-name fallback when the SMTP
# came from a *different* source (PR_SENDER_*, store_email…). When the
# SMTP was itself extracted from ``sender_name``, reusing it as a name
# produces a redundant ``"addr" ``.
name = preferred_name or fallback_name
if not name and sender_name_fallback:
name = _safe_sender_name(message)
return _addr_tuple_to_dict(name or "", smtp)
# 1. Direct sender SMTP (most reliable when present)
smtp = get_mapi_property_string(message, PR_SENDER_SMTP_ADDRESS)
if smtp and "@" in smtp:
return _build(None, smtp)
# 2. Direct sender email — only trusted if it looks like SMTP (excludes
# Exchange X.500 DNs that lack '@')
email_addr = get_mapi_property_string(message, PR_SENDER_EMAIL_ADDRESS)
if email_addr and "@" in email_addr:
return _build(None, email_addr)
# 3. Sent-representing SMTP — shared mailboxes ("au nom de") often carry
# the resolvable SMTP only here while PR_SENDER_* holds the X.500 DN
sr_smtp = get_mapi_property_string(message, PR_SENT_REPRESENTING_SMTP_ADDRESS)
if sr_smtp and "@" in sr_smtp:
sr_name = get_mapi_property_string(message, PR_SENT_REPRESENTING_NAME)
return _build(sr_name, sr_smtp)
# 4. Sent-representing email as SMTP
sr_email = get_mapi_property_string(message, PR_SENT_REPRESENTING_EMAIL_ADDRESS)
if sr_email and "@" in sr_email:
sr_name = get_mapi_property_string(message, PR_SENT_REPRESENTING_NAME)
return _build(sr_name, sr_email)
# 5. Creator / last-modifier SMTP — last resort before heuristics. For
# delegate sends from shared mailboxes this is typically the only
# resolvable SMTP available (the shared mailbox's own SMTP lives only
# in the GAL, which a PST does not export).
for tag in (PR_CREATOR_SMTP_ADDRESS, PR_LAST_MODIFIER_SMTP_ADDRESS):
creator_smtp = get_mapi_property_string(message, tag)
if creator_smtp and "@" in creator_smtp:
return _build(None, creator_smtp)
# 6. Try to parse sender_name as an email address.
try:
if message.sender_name:
parsed_name, addr = parse_address(message.sender_name, lenient=True)
if addr and "@" in addr:
return _build(parsed_name, addr, sender_name_fallback=False)
except Exception:
logger.debug("Failed to parse sender_name as email address")
# 7. Fall back to message store owner email (common for EX sent items)
if store_email:
return _build(None, store_email)
return None
def _extract_recipients_from_mapi(message) -> dict[str, list[EmailAddress]]:
"""Extract To/Cc/Bcc recipients from MAPI recipient table.
Returns dict with ``to`` / ``cc`` / ``bcc`` keys mapping to lists of
JMAP ``EmailAddress`` entries.
"""
result = {"to": [], "cc": [], "bcc": []}
# MAPI recipient types
MAPI_TO = 1 # pylint: disable=invalid-name
MAPI_CC = 2 # pylint: disable=invalid-name
MAPI_BCC = 3 # pylint: disable=invalid-name
try:
num_recipients = int(message.number_of_recipients)
except (AttributeError, TypeError, ValueError, Exception):
return result
for i in range(num_recipients):
try:
recipient = message.get_recipient(i)
except Exception:
logger.debug("Failed to get recipient %d", i)
continue
# Get recipient type
recip_type = get_mapi_property_integer(recipient, PR_RECIPIENT_TYPE)
if recip_type is None:
recip_type = MAPI_TO # Default to To
# Get display name
display_name = get_mapi_property_string(recipient, PR_DISPLAY_NAME) or ""
# Resolve email address (handles EX vs SMTP)
email_addr = _resolve_smtp_address(recipient)
if not email_addr:
# Try display name as email
if "@" in display_name:
email_addr = display_name
else:
continue # Skip recipients without resolvable email
addr_dict = _addr_tuple_to_dict(display_name, email_addr)
if recip_type == MAPI_TO:
result["to"].append(addr_dict)
elif recip_type == MAPI_CC:
result["cc"].append(addr_dict)
elif recip_type == MAPI_BCC:
result["bcc"].append(addr_dict)
else:
result["to"].append(addr_dict)
return result
def _parse_display_recipients(
display_string: str | None,
) -> list[EmailAddress]:
"""Parse Outlook's semicolon-separated To/Cc/Bcc display string.
Returns a list of JMAP ``EmailAddress`` entries for tokens that
contain a usable email address. Name-only entries (no ``@``) are
dropped on purpose — a Contact without an email cannot be created
downstream, and silently inventing one would corrupt the address
book.
"""
if not display_string:
return []
addresses: list[EmailAddress] = []
for raw in display_string.split(";"):
token = raw.strip()
if not token:
continue
try:
name, addr = parse_address(token, lenient=True)
except Exception:
logger.debug("Failed to parse display recipient token")
continue
if addr and "@" in addr:
addresses.append(_addr_tuple_to_dict(name or "", addr))
elif "@" in token:
addresses.append(_addr_tuple_to_dict("", token))
else:
logger.debug("Dropping display recipient with no email")
return addresses
def _extract_display_recipients_from_mapi(
message,
) -> dict[str, list[EmailAddress]]:
"""Fall back to PR_DISPLAY_TO/CC/BCC when the recipient table is empty.
Some PSTs exported from Exchange Online expose ``number_of_recipients=0``
even for messages that clearly went to recipients. The header-style
strings still carry whichever addresses Exchange formatted at compose
time, so they're our last source of truth before giving up.
"""
return {
"to": _parse_display_recipients(
get_mapi_property_string(message, PR_DISPLAY_TO)
),
"cc": _parse_display_recipients(
get_mapi_property_string(message, PR_DISPLAY_CC)
),
"bcc": _parse_display_recipients(
get_mapi_property_string(message, PR_DISPLAY_BCC)
),
}
def sanitize_folder_name(name: str, max_length: int = 255) -> str:
"""Sanitize a PST folder name for use as an IMAP label."""
name = name.strip()
# Remove control characters
name = "".join(c for c in name if c.isprintable())
return name[:max_length] if name else "Unknown"
def _decode_html_bytes(raw_html: bytes) -> str:
"""Decode HTML bytes, detecting encoding from meta charset tag.
Falls back through common Outlook encodings before using utf-8 with replacement.
"""
# Try to detect charset from HTML meta tag
match = _CHARSET_RE.search(raw_html[:4096])
if match:
charset = match.group(1).decode("ascii", errors="ignore")
try:
return raw_html.decode(charset)
except (UnicodeDecodeError, LookupError):
pass
# Try UTF-8 first (most modern)
try:
return raw_html.decode("utf-8")
except UnicodeDecodeError:
pass
# Try Windows-1252 (most common for Outlook)
try:
return raw_html.decode("cp1252")
except UnicodeDecodeError:
pass
# Final fallback
return raw_html.decode("utf-8", errors="replace")
def _apply_recipient_fallback_chain(message, jmap_data: dict) -> None:
"""Fill missing To/Cc/Bcc via MAPI recipient table, then PR_DISPLAY_TO/CC/BCC.
The recipient table is authoritative. Exchange Online exports often leave
it empty even on real messages — PR_DISPLAY_* carries the addresses that
Exchange formatted at compose time and is the last source of truth before
giving up. Common to both transport-headers and MAPI-only branches.
"""
if all(jmap_data.get(key) for key in ("to", "cc", "bcc")):
return
recipients = _extract_recipients_from_mapi(message)
display_recipients = None
for key in ("to", "cc", "bcc"):
if jmap_data.get(key):
continue
if recipients[key]:
jmap_data[key] = recipients[key]
continue
if display_recipients is None:
display_recipients = _extract_display_recipients_from_mapi(message)
if display_recipients[key]:
jmap_data[key] = display_recipients[key]
def _sanitize_message_id(raw: str | None) -> str | None:
"""Return ``raw`` stripped of brackets/whitespace if it's a valid msg-id.
PST archives routinely carry Message-IDs that strict composition
would reject (empty, missing '@', embedded whitespace, nested
brackets). Falling back to MAPI / synthesis here keeps the importer
"lenient parse, strict compose" contract from collapsing on a
malformed archive. The shape predicate is owned by
:func:`jmap_email.is_valid_msg_id` so this path stays in lockstep
with the composer's strict check.
"""
if not raw:
return None
candidate = raw.strip()
if candidate.startswith("<") and candidate.endswith(">"):
candidate = candidate[1:-1].strip()
if not is_valid_msg_id(candidate):
return None
return candidate
def _extract_message_id_from_mapi(message) -> str | None:
"""Read the native Internet Message-ID stored on the PST message.
Returns the value of PR_INTERNET_MESSAGE_ID stripped of surrounding
angle brackets when it parses to a shape compose_email accepts, or
None when the property is absent, empty, or malformed.
"""
return _sanitize_message_id(
get_mapi_property_string(message, PR_INTERNET_MESSAGE_ID)
)
def _synthesize_message_id(message, recipient_email: str | None) -> str:
"""Build a deterministic Message-ID from stable MAPI properties.
Used as last resort when no native Message-ID is available (typical of
drafts and locally-composed items). Two re-imports of the same PST will
produce the same value, allowing the inbound dedup check to skip them.
The hash inputs are picked for stability across libpff reads: delivery
and submit timestamps, sender, subject, body prefix, and attachment
fingerprint. The recipient mailbox's domain is used as the host part so
the synthesized ID stays scoped to this import.
"""
parts: list[str] = []
for attr in ("delivery_time", "client_submit_time"):
try:
value = getattr(message, attr, None)
parts.append(value.isoformat() if value else "")
except Exception:
parts.append("")
try:
parts.append(message.subject or "")
except Exception:
parts.append("")
try:
parts.append(message.sender_name or "")
except Exception:
parts.append("")
parts.append(get_mapi_property_string(message, PR_SENDER_SMTP_ADDRESS) or "")
parts.append(get_mapi_property_string(message, PR_SENDER_EMAIL_ADDRESS) or "")
# Body prefix — 1 KB is enough to disambiguate without making the hash
# sensitive to small late-message edits (signature insertion, etc.).
try:
raw_text = message.plain_text_body
if raw_text:
if isinstance(raw_text, bytes):
raw_text = raw_text.decode("utf-8", errors="replace")
parts.append(raw_text[:1024])
except Exception:
logger.debug("Failed to read plain_text_body for Message-ID synthesis")
# Attachment fingerprint — count + total declared size; reading the full
# buffers would defeat the point of an O(1) Message-ID synthesis.
try:
num_attachments = int(message.number_of_attachments)
total_size = 0
for i in range(num_attachments):
try:
total_size += int(message.get_attachment(i).get_size())
except Exception:
logger.debug(
"Failed to read attachment %d size for Message-ID synthesis", i
)
parts.append(f"{num_attachments}:{total_size}")
except Exception:
parts.append("")
digest = hashlib.sha256("\x00".join(parts).encode("utf-8")).hexdigest()[:32]
domain = (recipient_email and address_domain(recipient_email)) or "localhost"
return f"pst-synth-{digest}@{domain}"
def _archive_date(value: datetime | None) -> datetime | None:
"""``value`` if usable as the Date of an archived message, else None.
None when out of range once in UTC, in the future, or before year 100
(the composer refuses those, which would drop the message): the MAPI
timestamps are better than the import time the inbound step would use.
An offset with seconds (``+05:30:15``, ISO only) has no RFC 5322 form:
such a date is returned in UTC.
"""
if value is None:
return None
if value.tzinfo is None:
value = value.replace(tzinfo=timezone.utc)
try:
utc = value.astimezone(timezone.utc)
except (OverflowError, ValueError):
return None
if utc > datetime.now(timezone.utc) or utc.year < 100 or value.year < 100:
return None
return value if value.utcoffset().total_seconds() % 60 == 0 else utc
def reconstruct_eml(
message,
store_email: str | None = None,
recipient_email: str | None = None,
) -> bytes: # pylint: disable=too-many-branches
"""Convert a pypff message to RFC 5322 bytes.
If transport_headers is available, uses those for threading headers.
Otherwise, constructs headers from MAPI properties.
Uses the ``jmap-email`` library's ``compose_email`` for MIME construction.
``recipient_email`` is the import target mailbox; its domain is used to
synthesize ``unknown-sender@`` when no sender can be extracted,
so the message still composes (``compose_email`` rejects empty ``from``).
"""
# pylint: disable=too-many-locals,too-many-branches,too-many-statements
jmap_data = {}
extra_headers: list[dict[str, str]] = []
# Try to get original transport headers for threading-critical fields
transport_headers = None
try:
transport_headers = message.transport_headers
except Exception:
logger.debug("Failed to read transport_headers")
if transport_headers:
parsed_headers = message_from_string(transport_headers)
# From — Exchange rewrites internal/shared-mailbox senders as X.500
# DNs ("/O=EXCHANGELABS/.../CN=..."). Trust the header only if it
# parses to a real SMTP address; otherwise fall back to MAPI below,
# preserving the human-readable name from the header.
from_str = parsed_headers.get("From", "")
from_name_hint: str | None = None
if from_str:
name, addr = parse_address(from_str, lenient=True)
if addr and "@" in addr:
jmap_data["from"] = [_addr_tuple_to_dict(name, addr)]
else:
from_name_hint = name or None
if "from" not in jmap_data:
sender_dict = _extract_sender_from_mapi(
message,
store_email=store_email,
preferred_name=from_name_hint,
)
if sender_dict:
jmap_data["from"] = [sender_dict]
# To / Cc / Bcc — header values are authoritative when present.
to_str = parsed_headers.get("To", "")
if to_str:
jmap_data["to"] = [
_addr_tuple_to_dict(n, a) for n, a in parse_addresses(to_str)
]
cc_str = parsed_headers.get("Cc", "")
if cc_str:
jmap_data["cc"] = [
_addr_tuple_to_dict(n, a) for n, a in parse_addresses(cc_str)
]
bcc_str = parsed_headers.get("Bcc", "")
if bcc_str:
jmap_data["bcc"] = [
_addr_tuple_to_dict(n, a) for n, a in parse_addresses(bcc_str)
]
# Subject
subject = parsed_headers.get("Subject")
if subject:
jmap_data["subject"] = subject
# sentAt: RFC 5322, or ISO 8601 as some exporters write it.
date_str = str(parsed_headers.get("Date") or "")
sent_at = _archive_date(parse_date(date_str) or sent_at_to_datetime(date_str))
if sent_at is not None:
jmap_data["sentAt"] = sent_at.isoformat()
# Message-ID — header is preferred, but Exchange/O365 exports
# sometimes strip it or carry a malformed value (empty, missing
# '@', embedded whitespace) that compose_email's strict validator
# would reject; fall back to MAPI and synthesis below in those
# cases.
message_id = _sanitize_message_id(parsed_headers.get("Message-ID"))
if message_id:
jmap_data["messageId"] = [message_id]
# In-Reply-To and References — pass as custom headers to preserve
# exact original values (the in_reply_to parameter on compose_email
# would append to References, which we don't want for imports)
in_reply_to_val = parsed_headers.get("In-Reply-To")
if in_reply_to_val:
extra_headers.append({"name": "In-Reply-To", "value": in_reply_to_val})
references = parsed_headers.get("References")
if references:
extra_headers.append({"name": "References", "value": references})
else:
# Build from MAPI properties — sender
sender_dict = _extract_sender_from_mapi(message, store_email=store_email)
if sender_dict:
jmap_data["from"] = [sender_dict]
# Recipients fallback chain (MAPI recipient table → PR_DISPLAY_*).
# Exchange Online sometimes emits transport headers without To/Cc/Bcc
# lines, and PST exports often leave the recipient table empty even on
# real messages — this guarantees we capture every available source.
_apply_recipient_fallback_chain(message, jmap_data)
# Subject — ensure set even if transport_headers didn't include it
if "subject" not in jmap_data:
try:
if message.subject:
jmap_data["subject"] = message.subject
except Exception:
logger.debug("Failed to read message subject")
# Message-ID fallback chain — drafts and locally-composed items have no
# transport_headers, and Exchange exports sometimes drop the header even
# for received items. Without a Message-ID the inbound dedup check is
# skipped and the same message gets imported again on every re-run.
if "messageId" not in jmap_data:
native_id = _extract_message_id_from_mapi(message)
if native_id:
jmap_data["messageId"] = [native_id]
else:
jmap_data["messageId"] = [_synthesize_message_id(message, recipient_email)]
# ``sentAt`` fallback (no transport headers, or headers without a Date):
# the composer is strict-by-design and rejects a missing Date header.
# Take the record's own timestamps, and only when it has none log a
# warning and use the Unix epoch rather than fail the whole message.
if "sentAt" not in jmap_data:
timestamp = _get_message_timestamp(
message, _SENT_TIME_ATTRS + _RECORD_TIME_ATTRS
)
if timestamp is not None:
jmap_data["sentAt"] = timestamp.isoformat()
else:
logger.warning("PST message has no resolvable Date; falling back to epoch")
jmap_data["sentAt"] = "1970-01-01T00:00:00+00:00"
# No sender resolvable: synthesize one using the recipient's domain so
# compose_email accepts the message. inbound_create.py keeps this value
# as-is (it only substitutes when the email is empty).
if "from" not in jmap_data:
fallback_domain = recipient_email and address_domain(recipient_email)
fallback_email = (
f"unknown-sender@{fallback_domain}"
if fallback_domain
else "unknown-sender@localhost"
)
logger.warning(
"PST message has no resolvable sender; using synthesized sender address"
)
jmap_data["from"] = [{"name": "Unknown Sender", "email": fallback_email}]
# Body parts
try:
raw_text = message.plain_text_body
if raw_text:
if isinstance(raw_text, bytes):
raw_text = raw_text.decode("utf-8", errors="replace")
jmap_data["textBody"] = [{"content": raw_text}]
except Exception:
logger.debug("Failed to read plain_text_body")
try:
raw_html = message.html_body
if raw_html:
if isinstance(raw_html, str):
html_str = raw_html
else:
html_str = _decode_html_bytes(raw_html)
jmap_data["htmlBody"] = [{"content": html_str}]
except Exception:
logger.debug("Failed to read html_body")
# Ensure at least an empty text body
if "textBody" not in jmap_data and "htmlBody" not in jmap_data:
jmap_data["textBody"] = [{"content": ""}]
# Attachments
attachments = []
# Bound peak memory: never materialize an attachment (or the running total
# of them) beyond the per-message size limit. A crafted PST could otherwise
# hold a multi-GB attachment that OOMs the worker before ``deliver`` — which
# would reject the oversized message anyway — ever gets to check the size.
attachment_budget = settings.MAX_INCOMING_EMAIL_SIZE
try:
num_attachments = message.number_of_attachments
for i in range(num_attachments):
try:
attachment = message.get_attachment(i)
att_size = attachment.get_size()
if att_size > attachment_budget:
# Fail the whole message visibly (it lands in failure_count
# via ``_reconstruct_plan_item``) rather than silently
# delivering a truncated copy as a "success" — deliver()
# would reject the assembled oversized EML anyway.
raise OversizedAttachmentError(
f"attachment of {att_size} bytes exceeds the remaining "
f"per-message budget ({attachment_budget} bytes)"
)
att_data = attachment.read_buffer(att_size)
attachment_budget -= len(att_data)
# Skip empty / whitespace-only parts. DSN and read-receipt
# reports routinely expose blank diagnostic parts (e.g. an empty
# text/rfc822-headers) that libpff surfaces as attachments;
# importing them yields 0-byte attachments that render as broken
# in the UI while carrying no information.
if not att_data or not att_data.strip():
logger.debug("Skipping empty attachment %d", i)
continue
# Filename from MAPI properties
filename = (
get_mapi_property_string(attachment, PR_ATTACH_LONG_FILENAME)
or get_mapi_property_string(attachment, PR_ATTACH_FILENAME)
or get_mapi_property_string(attachment, PR_DISPLAY_NAME)
or f"attachment_{i}"
)
# MIME type from MAPI property
mime_type = get_mapi_property_string(attachment, PR_ATTACH_MIME_TAG)
if not mime_type:
mime_type = "application/octet-stream"
# text/rfc822-headers (the original-headers part of a DSN/read
# receipt) composes fine but on re-parse the display parser
# used to drop the body of this subtype, surfacing it as a
# 0-byte attachment in the UI. The content is plain RFC822
# header text, so normalize the label to text/plain, which
# round-trips intact.
if mime_type.split(";")[0].strip().lower() == "text/rfc822-headers":
mime_type = "text/plain"
# Content-ID for inline images
content_id = get_mapi_property_string(attachment, PR_ATTACH_CONTENT_ID)
# Attachment method
attach_method = get_mapi_property_integer(attachment, PR_ATTACH_METHOD)
# Determine disposition
disposition = "attachment"
if content_id and attach_method == ATTACH_BY_VALUE:
disposition = "inline"
att_dict = {
"content": base64.b64encode(att_data).decode("ascii"),
"type": mime_type,
"name": filename,
"disposition": disposition,
}
if disposition == "inline" and content_id:
att_dict["cid"] = content_id
attachments.append(att_dict)
except OversizedAttachmentError:
raise
except Exception:
logger.debug("Failed to process attachment %d", i)
except OversizedAttachmentError:
raise
except Exception:
logger.debug("Failed to read attachments")
if attachments:
jmap_data["attachments"] = attachments
if extra_headers:
jmap_data["headers"] = extra_headers
# PST is an archive: the Bcc list was in the original source file and we
# are reconstructing the .eml for storage in the user's own mailbox, not
# retransmitting. Preserve it.
return compose_email(jmap_data, options=ARCHIVE_COMPOSE_OPTIONS)
def _find_ipm_subtree(pst_file):
"""Find the IPM subtree folder (the real top-level mail folder).
Uses PR_IPM_SUBTREE_ENTRYID from the message store to locate the
subtree. Falls back to the root folder if not found.
"""
root = pst_file.get_root_folder()
try:
store = pst_file.get_message_store()
except Exception:
return root
subtree_entry_id = get_mapi_property_data(store, PR_IPM_SUBTREE_ENTRYID)
subtree_id = _folder_id_from_entry_id(subtree_entry_id)
if subtree_id is not None:
try:
for i in range(root.number_of_sub_folders):
child = root.get_sub_folder(i)
if child.get_identifier() == subtree_id:
return child
except Exception:
logger.debug("Failed to locate IPM subtree folder")
return root
def count_pst_messages(pst_file, special_folder_map: dict | None = None) -> int:
"""Recursively count email messages across all email folders in a PST file."""
if special_folder_map is None:
special_folder_map = build_special_folder_map(pst_file)
count = 0
start_folder = _find_ipm_subtree(pst_file)
def _count_folder(folder, depth=0):
nonlocal count
if depth > MAX_FOLDER_DEPTH:
logger.warning("Maximum folder depth exceeded in PST file")
return
if not _is_email_folder(folder):
return
try:
count += folder.number_of_sub_messages
except Exception:
logger.debug("Failed to count sub_messages in folder")
try:
for i in range(folder.number_of_sub_folders):
_count_folder(folder.get_sub_folder(i), depth + 1)
except Exception:
logger.debug("Failed to iterate sub_folders for counting")
try:
for i in range(start_folder.number_of_sub_folders):
_count_folder(start_folder.get_sub_folder(i))
except Exception:
logger.debug("Failed to iterate root sub_folders for counting")
return count
def _collect_pst_plan(
pst_file,
special_folder_map: dict,
on_progress: Callable[[], None] | None = None,
) -> list[tuple]:
"""Collect + chronologically sort every email message's locator in a PST.
Returns ``(folder_ref, msg_idx, folder_type, folder_path, flags,
flag_status)`` tuples, oldest-first (for threading). Only lightweight
per-message metadata is read here — EML reconstruction is deferred to
:func:`_reconstruct_plan_item` so only one message is ever held in memory.
``len()`` of the result is the authoritative message total. ``on_progress``
is invoked periodically so the runner can beat its heartbeat through this
(potentially long) traversal.
"""
start_folder = _find_ipm_subtree(pst_file)
# Build well-known folder type map from SourceWellKnownFolderType named
# property (Exchange/O365 migration PSTs). Merges with entry-ID-based map,
# with entry-ID taking priority.
wkft_map = build_well_known_folder_map(pst_file, start_folder)
merged_map = {**wkft_map, **special_folder_map}
# (timestamp, folder_ref, msg_index, folder_type, folder_path, flags, flag_status)
collected = []
# Keep folder references alive to prevent GC from releasing pypff resources
_folder_refs = []
def _collect_folder(folder, folder_type, folder_path, depth=0):
if depth > MAX_FOLDER_DEPTH:
logger.warning("Maximum folder depth exceeded in PST file")
return
if not _is_email_folder(folder):
return
_folder_refs.append(folder)
ft = _get_folder_type(folder, merged_map)
if ft != FOLDER_TYPE_NORMAL:
folder_type = ft
try:
for i in range(folder.number_of_sub_messages):
if on_progress:
on_progress()
try:
message = folder.get_sub_message(i)
flags = _get_message_flags(message)
flag_status = _get_flag_status(message)
timestamp = _get_message_timestamp(message)
collected.append(
(
timestamp,
folder,
i,
folder_type,
folder_path,
flags,
flag_status,
)
)
except Exception:
logger.debug(
"Failed to read message %d in folder %s", i, folder_path
)
except ImportCancelled:
# ``on_progress`` is the runner's beat(): a cancel raised there
# must unwind the whole pre-scan, not be swallowed as a corrupt
# folder — the worker would otherwise keep scanning (and holding
# the run lock) for the rest of the archive.
raise
except Exception:
logger.debug("Failed to iterate sub_messages in folder %s", folder_path)
# Children inherit the parent's folder_type (so they keep the special
# treatment, e.g. is_import_sender for Sent subfolders) and build
# hierarchical paths. Special folders are called with folder_path=""
# so their children start from just their own name.
try:
for i in range(folder.number_of_sub_folders):
child = folder.get_sub_folder(i)
child_name = None
try:
if child.name:
child_name = sanitize_folder_name(child.name)
except Exception:
logger.debug("Failed to read subfolder name")
if folder_path and child_name:
child_path = f"{folder_path}/{child_name}"
else:
child_path = child_name or folder_path
_collect_folder(child, folder_type, child_path, depth + 1)
except ImportCancelled:
raise
except Exception:
logger.debug("Failed to iterate sub_folders in folder %s", folder_path)
try:
for i in range(start_folder.number_of_sub_folders):
subfolder = start_folder.get_sub_folder(i)
name = "Inbox"
try:
if subfolder.name:
name = sanitize_folder_name(subfolder.name)
except Exception:
logger.debug("Failed to read root subfolder name")
folder_type = _get_folder_type(subfolder, merged_map)
# Fall back to name-based detection for direct children of
# the IPM subtree (standard Outlook special folders).
if folder_type == FOLDER_TYPE_NORMAL:
folder_type = _detect_folder_type_by_name(name)
# Special folders get empty path — their label comes from
# folder_type, not folder_path. Their children build paths
# starting from just their own name.
folder_path = "" if folder_type != FOLDER_TYPE_NORMAL else name
_collect_folder(subfolder, folder_type, folder_path)
except ImportCancelled:
raise
except Exception:
logger.debug("Failed to iterate root sub_folders")
# Sort by timestamp (oldest first), None timestamps go last, then drop the
# sort key — the caller only needs the locator + metadata.
collected.sort(key=lambda x: (x[0] is None, x[0] or 0))
return [item[1:] for item in collected]
def _reconstruct_plan_item(
item,
store_email: str | None = None,
recipient_email: str | None = None,
*,
reasons: dict[str, int] | None = None,
) -> tuple[str, str, int, int | None, bytes | None]:
"""Reconstruct one plan item into ``(folder_type, folder_path, flags,
flag_status, eml_bytes)``. ``eml_bytes`` is ``None`` when the message cannot
be read from libpff or reconstruction raised — the caller counts those as
failures rather than silently dropping them. When ``reasons`` is passed,
each failure bumps its category (same out-param contract as ``deliver``)."""
folder_ref, msg_idx, folder_type, folder_path, flags, flag_status = item
try:
message = folder_ref.get_sub_message(msg_idx)
except Exception:
logger.warning("Failed to read message %d in folder %s", msg_idx, folder_path)
bump_reason(reasons, FAILURE_UNPARSEABLE)
return folder_type, folder_path, flags, flag_status, None
try:
eml_bytes = reconstruct_eml(
message, store_email=store_email, recipient_email=recipient_email
)
except OversizedAttachmentError as exc:
# An expected import outcome (the archive holds a message we refuse to
# deliver), not a bug: count it as a failure without the Sentry-visible
# traceback ``logger.exception`` would emit.
logger.warning(
"Skipping message %d in folder %s: %s", msg_idx, folder_path, exc
)
bump_reason(reasons, FAILURE_OVERSIZED)
return folder_type, folder_path, flags, flag_status, None
except ComposeError as exc:
# Also an expected outcome: the composer refuses a malformed value from
# the archive (recipient address, Content-ID…). Only the error type is
# logged — its message echoes that value, i.e. names and addresses.
logger.warning(
"Skipping message %d in folder %s: rejected by the composer (%s)",
msg_idx,
folder_path,
type(exc).__name__,
)
bump_reason(reasons, FAILURE_UNPARSEABLE)
return folder_type, folder_path, flags, flag_status, None
except Exception:
logger.exception(
"Failed to reconstruct EML for message %d in folder %s",
msg_idx,
folder_path,
)
bump_reason(reasons, FAILURE_UNPARSEABLE)
return folder_type, folder_path, flags, flag_status, None
return folder_type, folder_path, flags, flag_status, eml_bytes
def walk_pst_messages(
pst_file,
special_folder_map: dict,
store_email: str | None = None,
recipient_email: str | None = None,
) -> Generator[tuple[str, str, int, int | None, bytes | None], None, None]:
"""Walk all email messages in a PST, chronologically, yielding reconstructed
EML one at a time. Thin composition of :func:`_collect_pst_plan` (metadata +
sort) and :func:`_reconstruct_plan_item` (per-message reconstruction)."""
for item in _collect_pst_plan(pst_file, special_folder_map):
yield _reconstruct_plan_item(item, store_email, recipient_email)
def _pst_labels_flags(folder_type, folder_path, message_flags, flag_status):
"""Map PST folder/message metadata to IMAP-style labels/flags/sender."""
imap_flags = []
if message_flags & MSGFLAG_READ:
imap_flags.append("\\Seen")
if message_flags & MSGFLAG_UNSENT or folder_type == FOLDER_TYPE_DRAFTS:
imap_flags.append("\\Draft")
if flag_status is not None and flag_status >= FLAG_STATUS_FOLLOWUP:
imap_flags.append("\\Flagged")
imap_labels = []
if folder_type == FOLDER_TYPE_SENT:
imap_labels.append("Sent")
elif folder_type == FOLDER_TYPE_DELETED:
imap_labels.append("Trash")
elif folder_type == FOLDER_TYPE_OUTBOX:
imap_labels.append("OUTBOX")
elif folder_type in (FOLDER_TYPE_INBOX, FOLDER_TYPE_DRAFTS):
pass
elif folder_path:
imap_labels.append(sanitize_folder_name(folder_path))
if folder_path and folder_type != FOLDER_TYPE_NORMAL:
imap_labels.append(sanitize_folder_name(folder_path))
is_sender = folder_type in (FOLDER_TYPE_SENT, FOLDER_TYPE_OUTBOX)
return imap_labels, imap_flags, is_sender
def run_pst(channel, state) -> tuple[int, int, int]:
"""Resumable PST pass: walk folders/messages, deliver from ``cursor``."""
recipient = channel.mailbox
file_key = (channel.settings or {})["import"]["file_key"]
storage, s3_client = imports_storage()
with S3SeekableReader(
s3_client,
storage.bucket_name,
file_key,
buffer_strategy=BUFFER_NONE,
buffer_size=64 * 1024,
buffer_count=2048,
) as reader:
pst = pypff.file()
try:
pst.open_file_object(reader)
except Exception as exc:
# Same UI contract as ``assert_pst_readable``: a file that fails at
# open (bad signature, truncated) must carry the marker too, not a
# raw libpff error.
raise PSTFileUnreadableError(
f"{PST_UNREADABLE_MARKER}: the file is not a readable PST "
"archive (invalid signature or truncated file)."
) from exc
try:
assert_pst_readable(pst)
special_folder_map = build_special_folder_map(pst)
store_email = get_store_owner_email(pst)
# Building the plan is the long pre-scan: beat through it so a live
# run keeps renewing its lock. len(plan) is the authoritative total,
# so the progress bar reaches 100% (unlike a separate count that
# over-counted unreadable messages).
plan = _collect_pst_plan(
pst, special_folder_map, on_progress=lambda: beat(channel)
)
def deliver_item(item, reasons):
folder_type, folder_path, message_flags, flag_status, eml_bytes = (
_reconstruct_plan_item(
item,
store_email=store_email,
recipient_email=str(recipient),
reasons=reasons,
)
)
if eml_bytes is None:
# Couldn't rebuild a .eml from the PST record; the failure
# category was already tallied by _reconstruct_plan_item.
return False
labels, flags, is_sender = _pst_labels_flags(
folder_type, folder_path, message_flags, flag_status
)
return deliver(
eml_bytes,
recipient,
channel,
imap_labels=labels,
imap_flags=flags,
is_sender=is_sender,
reasons=reasons,
)
return run_plan(channel, state, plan, deliver_item)
finally:
pst.close()