"""Shared canonical schemas for RegEngine inter-service communication. This module provides the single source of truth for all data contracts across NLP, Review, and Graph services, preventing schema drift. """ import re from datetime import datetime, timezone from enum import Enum from typing import Any, Dict, List, Literal, Optional from urllib.parse import unquote from uuid import UUID, uuid4 from pydantic import BaseModel, Field, field_validator, model_validator from .fsma_validation import validate_tlc # Example constants to avoid duplication in schema examples EX_SUBJECT = "food facilities on the FTL" EX_ACTION = "must maintain" EX_OBJECT = "traceability records" EX_TEXT = "Food facilities on the FTL must maintain traceability records for at least 2 years" class ObligationType(str, Enum): """Standardized obligation types for regulatory requirements.""" MUST = "MUST" MUST_NOT = "MUST_NOT" SHOULD = "SHOULD" MAY = "MAY" RECORDKEEPING = "RECORDKEEPING" REPORTING = "REPORTING" class JurisdictionScope(str, Enum): """Scope of jurisdiction granularity.""" FEDERAL = "federal" STATE = "state" MUNICIPAL = "municipal" class Jurisdiction(BaseModel): """Hierarchical jurisdiction model using ISO-3166 style codes. Examples: - US (federal) - US-NY (state) - US-TX-AUS (municipal: Austin, TX) """ code: str = Field(..., description="Jurisdiction code (e.g., US, US-NY, US-TX-AUS)") name: str = Field(..., description="Human-readable name (e.g., United States, New York, Austin)") parent_code: Optional[str] = Field( None, description="Parent jurisdiction code (e.g., US for US-NY, US-TX for US-TX-AUS)" ) scope: JurisdictionScope = Field(..., description="Jurisdiction scope: federal/state/municipal") class Threshold(BaseModel): """Normalized threshold representation for regulatory limits.""" value: float = Field(..., description="Numeric value of the threshold") unit: str = Field(..., description="Unit of measurement (USD, percent, days, etc.)") operator: Literal["gt", "lt", "eq", "gte", "lte", "<", ">", "<=", ">=", "=="] = Field( ..., description="Comparison operator" ) context: Optional[str] = Field(None, description="Additional context for threshold") model_config = { "json_schema_extra": { "example": { "value": 2.0, "unit": "years", "operator": "gte", "context": "record retention period", } } } class ExtractionPayload(BaseModel): """NLP extraction output - canonical format for regulatory provisions.""" provision_id: Optional[str] = Field(None, description="Unique provision identifier") subject: str = Field(..., description="Subject of the obligation (e.g., 'food facilities on the FTL')") action: str = Field(..., description="Required action (e.g., 'must maintain', 'shall report')") object: Optional[str] = Field(None, description="Object of the obligation") obligation_type: ObligationType = Field(..., description="Type of regulatory obligation") thresholds: List[Threshold] = Field(default_factory=list, description="Associated thresholds") effective_date: Optional[str] = Field(None, description="When obligation becomes effective") jurisdiction: Optional[str] = Field( None, description="Regulatory jurisdiction (deprecated; use jurisdiction_codes)" ) jurisdiction_codes: List[str] = Field( default_factory=list, description="One or more jurisdiction codes the provision applies to (e.g., ['US', 'US-NY'])", ) # Citation/versioning metadata rule_version: Optional[str] = Field(None, description="Regulatory rule/version identifier (e.g., EU-2016/679)") source_uri: Optional[str] = Field(None, description="Canonical source URI for the cited rule") jurisdiction_code: Optional[str] = Field(None, description="Primary jurisdiction code for citation context") attributes: Dict[str, Any] = Field(default_factory=dict, description="Additional metadata") confidence_score: float = Field(..., ge=0.0, le=1.0, description="Extraction confidence (0-1)") source_text: str = Field(..., description="Original text span from document") source_offset: int = Field(..., description="Character offset in source document") entities: List[Dict[str, Any]] = Field(default_factory=list, description="Raw extracted entities") model_config = { "json_schema_extra": { "example": { "subject": EX_SUBJECT, "action": EX_ACTION, "object": EX_OBJECT, "obligation_type": "MUST", "thresholds": [{"value": 2.0, "unit": "years", "operator": "gte"}], "confidence_score": 0.92, "source_text": EX_TEXT, "source_offset": 1024, "jurisdiction": "US-FDA", "jurisdiction_codes": ["US"], } } } class GraphEvent(BaseModel): """Event sent to Graph service for knowledge graph ingestion.""" event_id: str = Field(default_factory=lambda: str(uuid4()), description="Unique event ID") event_type: Literal["create_document", "create_provision", "approve_provision"] = Field( ..., description="Type of graph operation" ) tenant_id: Optional[UUID] = Field(None, description="Tenant identifier for multi-tenancy") doc_hash: str = Field(..., description="Content hash of source document") document_id: str = Field(..., description="Document identifier") text_clean: str = Field(..., description="Cleaned provision text") extraction: ExtractionPayload = Field(..., description="Extracted provision data") provenance: Dict[str, Any] = Field( default_factory=dict, description="Provenance metadata (source URL, page, offset, request_id)", ) embedding: Optional[List[float]] = Field(None, description="768-dimensional sentence-transformer embedding") status: Literal["APPROVED", "APPROVED_BY_HUMAN", "PENDING_REVIEW"] = Field( default="PENDING_REVIEW", description="Approval status" ) reviewer_id: Optional[str] = Field(None, description="User ID of human reviewer (if applicable)") timestamp: datetime = Field( default_factory=lambda: datetime.now(timezone.utc), description="Event timestamp", ) @field_validator("embedding") @classmethod def validate_embedding_dimension(cls, v): """Ensure embedding is 768-dimensional for sentence-transformers compatibility.""" if v is not None and len(v) != 768: raise ValueError(f"Embedding must be 768-dimensional, got {len(v)}") return v model_config = { "json_schema_extra": { "example": { "event_type": "approve_provision", "tenant_id": "550e8400-e29b-41d4-a716-446655440000", "doc_hash": "abc123def456", "document_id": "doc_001", "text_clean": EX_TEXT, "extraction": { "subject": EX_SUBJECT, "action": EX_ACTION, "object": EX_OBJECT, "obligation_type": "MUST", "thresholds": [{"value": 2.0, "unit": "years", "operator": "gte"}], "confidence_score": 0.92, "source_text": EX_TEXT, "source_offset": 1024, }, "provenance": {"source_url": "https://example.com/fsma204.pdf", "page": 5}, "status": "APPROVED", } } } class LegacyGraphEvent(BaseModel): """Legacy-format event accepted on ``graph.update`` for backward compat (#1216). Used when a producer emits the pre-``GraphEvent`` shape. Validates the fields the graph consumer actually reads so a malformed / malicious message is DLQ'd instead of crashing the consumer or leaking junk into Neo4j. Additional fields are permitted (Pydantic default is ``ignore``) so this model does not block forward-compatible additions. """ document_id: str = Field(..., min_length=1, description="Document identifier") entities: List[Dict[str, Any]] = Field(default_factory=list, description="Extracted entities to upsert") source_url: Optional[str] = Field(None, description="Origin URL for the document") tenant_id: Optional[str] = Field(None, description="Tenant UUID (validated downstream before DB routing)") @field_validator("entities") @classmethod def _entities_must_be_dicts(cls, v): if not isinstance(v, list): raise ValueError("entities must be a list of dicts") for i, item in enumerate(v): if not isinstance(item, dict): raise ValueError(f"entities[{i}] must be a dict, got {type(item).__name__}") return v class ReviewItem(BaseModel): """Review queue item for human-in-the-loop validation.""" id: UUID = Field(default_factory=uuid4, description="Review item ID") tenant_id: Optional[UUID] = Field(None, description="Tenant identifier") document_id: str = Field(..., description="Source document ID") extraction: ExtractionPayload = Field(..., description="Extracted provision awaiting review") status: Literal["pending", "approved", "rejected"] = Field(default="pending", description="Review status") reviewer_id: Optional[str] = Field(None, description="Reviewer user ID") reviewed_at: Optional[datetime] = Field(None, description="Review timestamp") notes: Optional[str] = Field(None, description="Reviewer notes") created_at: datetime = Field( default_factory=lambda: datetime.now(timezone.utc), description="Creation timestamp", ) model_config = { "json_schema_extra": { "example": { "document_id": "doc_001", "extraction": { "subject": EX_SUBJECT, "action": EX_ACTION, "object": EX_OBJECT, "obligation_type": "MUST", "confidence_score": 0.72, "source_text": EX_TEXT, "source_offset": 1024, }, "status": "pending", } } } # ============================================================================= # FSMA 204 Models - Food Safety Modernization Act Section 204 # ============================================================================= class CTEType(str, Enum): """Critical Tracking Event Types per FSMA 204. Used across services to ensure consistent event type handling. """ SHIPPING = "SHIPPING" RECEIVING = "RECEIVING" TRANSFORMATION = "TRANSFORMATION" CREATION = "CREATION" INITIAL_PACKING = "INITIAL_PACKING" class Location(BaseModel): """Location identifier for FSMA traceability (simplified model).""" gln: Optional[str] = Field(None, description="GS1 Global Location Number (13 digits)") name: Optional[str] = Field(None, description="Human-readable location name") address: Optional[str] = Field(None, description="Physical address") model_config = { "json_schema_extra": { "example": { "gln": "1234567890123", "name": "Fresh Farms Distribution Center", "address": "123 Produce Way, Fresno, CA 93706", } } } class ProductDescription(BaseModel): """Product description for FSMA traceability.""" text: str = Field(..., description="Product description text") sku: Optional[str] = Field(None, description="Stock Keeping Unit") gtin: Optional[str] = Field(None, description="GS1 Global Trade Item Number (14 digits)") model_config = { "json_schema_extra": { "example": {"text": "Romaine Lettuce Hearts 12ct", "sku": "ROM-12CT", "gtin": "00012345678901"} } } class KDE(BaseModel): """Key Data Element extracted from a document.""" name: str = Field(..., description="KDE field name") value: str = Field(..., description="KDE value as string") confidence: Optional[float] = Field(None, ge=0.0, le=1.0, description="Extraction confidence") model_config = { "json_schema_extra": { "example": {"name": "traceability_lot_code", "value": "00012345678901-L2025-1105-A", "confidence": 0.92} } } class FSMAEventType(str, Enum): """FSMA event type enum for validated FSMAEvent payloads.""" CREATION = "CREATION" TRANSFORMATION = "TRANSFORMATION" SHIPPING = "SHIPPING" RECEIVING = "RECEIVING" _TLC_PRODUCTION_RE = re.compile(r"^[0-9]{14}[A-Za-z0-9\-\.]+\Z") # GS1 General Specifications Figure 7.11-1: 82 characters, space excluded. # AI (10) is 1 to 20 of those characters after percent-decoding. A well-formed # %HH escape is one character. The structural patterns reject a raw character # that the carrier requires to be percent-encoded, and a % that is not an escape. _GS1_AI_82_CHARS = frozenset("!\"%&'()*+,-./0123456789:;<=>?ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz") _PCT_ESCAPE = r"%[0-9A-Fa-f]{2}" # Digital Link XCHAR literals. Reserved symbols exist only as %HH escapes. _DL_LOT_RAW_CLASS = r'[0-9A-Za-z"._-]' # TDS GS3A3 URI form. Appendix A escapes that must not appear raw: " % & / < > ? _LGTIN_LOT_RAW_CLASS = r"[!'()*+,\-.0-9:;=A-Z_a-z]" # Zero or more custom path segments between the host and /01 (segment = *pchar). _DL_CUSTOM_STEM = r"(?P(?:/[^/\s?#]*)*?)" # Two non-empty digit runs whose lengths sum to 13 (company prefix + indicator/item ref). _LGTIN_DIGIT_PAIRS = "(?:" + "|".join(rf"[0-9]{{{n}}}\.[0-9]{{{13 - n}}}" for n in range(1, 13)) + ")" _TLC_LGTIN_RE = re.compile( rf"^urn:epc:class:lgtin:{_LGTIN_DIGIT_PAIRS}\." rf"(?P(?:{_LGTIN_LOT_RAW_CLASS}|{_PCT_ESCAPE})+)\Z" ) _TLC_DIGITAL_LINK_RE = re.compile( rf"^https?://[^/\s?#]+{_DL_CUSTOM_STEM}/01/[0-9]{{14}}/10/" rf"(?P(?:{_DL_LOT_RAW_CLASS}|{_PCT_ESCAPE})+)" rf"(?:/[^\s?#]*)?(?:\?[^\s#]*)?\Z" ) def _decoded_ai10_lot_within_limit(encoded_lot: str) -> bool: """Return whether an AI (10) lot is 1-20 GS1 characters after unquote. The caller has already matched a carrier pattern. This does not rewrite the TLC; it only accepts or rejects the lot substring. """ index = 0 size = len(encoded_lot) while index < size: if encoded_lot[index] != "%": index += 1 continue hex_pair = encoded_lot[index + 1 : index + 3] if len(hex_pair) != 2 or any(char not in "0123456789ABCDEFabcdef" for char in hex_pair): return False index += 3 decoded = unquote(encoded_lot) return 1 <= len(decoded) <= 20 and all(char in _GS1_AI_82_CHARS for char in decoded) class FSMAEvent(BaseModel): """FSMA event model for Critical Tracking Events. Represents a single validated event extracted from EPCIS, EDI, or other supply chain documents. Contains the required KDEs for FSMA 204 compliance. All extracted data from EPCIS and EDI pipelines is validated against this model before being emitted to Kafka. Required KDEs (rejection triggers): - ``tlc`` – at least 3 characters after stripping whitespace - ``event_time`` – ISO 8601 timestamp (required) """ event_id: str = Field( default_factory=lambda: str(uuid4()), description="Unique event identifier", ) event_type: FSMAEventType = Field( ..., description="FSMA event type: CREATION, TRANSFORMATION, SHIPPING, RECEIVING", ) tlc: str = Field( ..., description="Traceability Lot Code (production-pattern mismatches warn only)", min_length=3, ) product_description: str = Field( default="Unknown Product", description="Human-readable product description", ) quantity: Optional[float] = Field(None, description="Numeric quantity", ge=0) unit_of_measure: Optional[str] = Field(None, description="Unit of measure (cases, lbs, kg, etc.)") location_gln: Optional[str] = Field(None, description="Primary location GLN (13-digit GS1)") event_time: str = Field(..., description="ISO 8601 timestamp of the event", min_length=1) source_gln: Optional[str] = Field(None, description="Source/ship-from GLN") destination_gln: Optional[str] = Field(None, description="Destination/ship-to GLN") reference_document_type: Optional[str] = Field( None, description="Source document type (EPCIS, EDI_856, EDI_850, etc.)", ) reference_document_number: Optional[str] = Field(None, description="Source document reference number") tenant_id: Optional[str] = Field(None, description="Tenant identifier for multi-tenancy") warnings: List[str] = Field(default_factory=list) # Legacy fields retained for backward compatibility with document extraction date: Optional[str] = Field(None, description="Event date in YYYY-MM-DD format") cte_type: Optional[CTEType] = Field(None, description="Legacy CTE type (use event_type instead)") unit: Optional[str] = Field(None, description="Legacy unit field (use unit_of_measure instead)") product: Optional[ProductDescription] = Field(None, description="Product information") ship_from: Optional[Location] = Field(None, description="Ship from location") ship_to: Optional[Location] = Field(None, description="Ship to location") document_source: Optional[str] = Field(None, description="Source document identifier") document_hash: Optional[str] = Field(None, description="Source document hash") raw_row_index: Optional[int] = Field(None, description="Row index in source table (for tabular docs)") kdes: List[KDE] = Field(default_factory=list, description="All extracted Key Data Elements") @field_validator("tlc") @classmethod def validate_tlc_format(cls, v: str) -> str: """Strip surrounding whitespace; reject blank or short TLCs. Production-pattern advisories are collected on each model instance. """ cleaned = v.strip() if not cleaned: raise ValueError("TLC must not be blank") if len(cleaned) < 3: raise ValueError("TLC must be at least 3 characters") return cleaned @model_validator(mode="after") def collect_tlc_warnings(self) -> "FSMAEvent": for warning in validate_tlc(self.tlc, strict=True).warnings: if warning not in self.warnings: self.warnings.append(warning) return self @field_validator("event_time") @classmethod def validate_event_time_nonempty(cls, v: str) -> str: """Reject empty or whitespace-only event_time.""" stripped = v.strip() if not stripped: raise ValueError("event_time is a required KDE and must not be blank") return stripped model_config = { "json_schema_extra": { "example": { "event_id": "550e8400-e29b-41d4-a716-446655440000", "event_type": "SHIPPING", "tlc": "00012345678901-L2025-1105-A", "product_description": "Romaine Lettuce Hearts 12ct", "quantity": 50, "unit_of_measure": "cases", "location_gln": "1234567890123", "event_time": "2025-11-05T14:30:00Z", "source_gln": "1234567890123", "destination_gln": "9876543210987", "reference_document_type": "EPCIS", "reference_document_number": "ASN-2025-001", "tenant_id": "tenant-001", } } } class FSMALocation(BaseModel): """Location identifier for FSMA traceability.""" gln: Optional[str] = Field(None, description="GS1 Global Location Number (13 digits)") fda_registration: Optional[str] = Field(None, description="FDA Facility Registration Number") name: Optional[str] = Field(None, description="Human-readable location name") address: Optional[str] = Field(None, description="Physical address") model_config = { "json_schema_extra": { "example": { "gln": "1234567890123", "fda_registration": "12345678901", "name": "Fresh Farms Distribution Center", "address": "123 Produce Way, Fresno, CA 93706", } } } class FSMAKeyDataElements(BaseModel): """Key Data Elements (KDEs) per FSMA 204 requirements.""" traceability_lot_code: Optional[str] = Field(None, description="GTIN + Lot identifier") product_description: Optional[str] = Field(None, description="Product name/description") quantity: Optional[float] = Field(None, description="Numeric quantity") unit_of_measure: Optional[str] = Field(None, description="Unit (cases, lbs, kg, etc.)") location_identifier: Optional[str] = Field(None, description="GLN or FDA Reg as URN") event_date: Optional[str] = Field(None, description="Date in YYYY-MM-DD format") event_time: Optional[str] = Field(None, description="Time in HH:MM:SSZ format") ship_from_location: Optional[FSMALocation] = None ship_to_location: Optional[FSMALocation] = None # TLC Source fields - required for TRANSFORMATION/INITIAL_PACKING/CREATION events tlc_source_gln: Optional[str] = Field(None, description="GLN of facility that assigned the TLC") tlc_source_fda_reg: Optional[str] = Field(None, description="FDA Registration of TLC source facility") model_config = { "json_schema_extra": { "example": { "traceability_lot_code": "00012345678901-L2025-1105-A", "product_description": "Romaine Lettuce Hearts 12ct", "quantity": 50, "unit_of_measure": "cases", "location_identifier": "urn:gln:1234567890123", "event_date": "2025-11-05", "event_time": "14:30:00Z", "tlc_source_gln": "1234567890123", } } } class FSMACriticalTrackingEvent(BaseModel): """Critical Tracking Event (CTE) per FSMA 204.""" type: str = Field(..., description="SHIPPING, RECEIVING, TRANSFORMATION, or CREATION") kdes: FSMAKeyDataElements confidence: float = Field(..., ge=0.0, le=1.0, description="Extraction confidence score") source_document_id: Optional[str] = None extraction_timestamp: Optional[str] = None model_config = { "json_schema_extra": { "example": { "type": "SHIPPING", "kdes": { "traceability_lot_code": "00012345678901-L2025-1105-A", "product_description": "Romaine Lettuce Hearts 12ct", "quantity": 50, "unit_of_measure": "cases", "location_identifier": "urn:gln:1234567890123", "event_date": "2025-11-05", }, "confidence": 0.89, } } } class FSMAExtractionPayload(BaseModel): """Payload for FSMA extraction results.""" document_id: str document_type: str = Field(..., description="BOL, INVOICE, PRODUCTION_LOG, or UNKNOWN") ctes: List[FSMACriticalTrackingEvent] extraction_timestamp: str warnings: List[str] = Field(default_factory=list) tenant_id: Optional[str] = None model_config = { "json_schema_extra": { "example": { "document_id": "doc-fsma-001", "document_type": "BOL", "ctes": [ { "type": "SHIPPING", "kdes": { "traceability_lot_code": "00012345678901-L2025-1105-A", "quantity": 50, "unit_of_measure": "cases", }, "confidence": 0.89, } ], "extraction_timestamp": "2025-12-02T10:30:00Z", "warnings": [], } } } class FSMATraceRequest(BaseModel): """Request for traceability query.""" lot_id: str = Field(..., description="Traceability Lot Code to trace") direction: str = Field("forward", description="forward or backward") max_depth: int = Field(10, ge=1, le=100, description="Maximum hops in trace") model_config = { "json_schema_extra": { "example": {"lot_id": "00012345678901-L2025-1105-A", "direction": "forward", "max_depth": 10} } } class FSMATraceResult(BaseModel): """Result of traceability query.""" lot_id: str direction: str facilities: List[FSMALocation] events: List[FSMACriticalTrackingEvent] total_quantity: Optional[float] = None query_time_ms: float model_config = { "json_schema_extra": { "example": { "lot_id": "00012345678901-L2025-1105-A", "direction": "forward", "facilities": [ {"gln": "1234567890123", "name": "Fresh Farms"}, {"gln": "9876543210987", "name": "Metro Distribution"}, ], "events": [], "total_quantity": 500, "query_time_ms": 45.2, } } }