import io import os from dataclasses import dataclass from pathlib import Path from typing import Iterable, Sequence, cast from urllib.parse import urlparse import frontmatter import mistletoe import mistletoe.block_token from liquid2 import DictLoader, Environment from mistletoe.block_token import List, Paragraph, Quote from mistletoe.markdown_renderer import Fragment, MarkdownRenderer from mistletoe.span_token import Emphasis, Link, RawText, SpanToken, Strong from mistletoe.token import Token from mistletoe.utils import traverse from plain_parser import concept_utils, loaders, plain_spec from plain_parser.exceptions import ( ImportedModuleWithFunctionalitiesError, MissingFunctionalitiesError, ModuleDoesNotExistError, PlainSyntaxError, UnsupportedBase64Content, ) from plain_parser.liquid_nodes import Plain2CodeIncludeTag, Plain2CodeLoaderMixin from plain_parser.loaders import find_large_base64_blob RESOURCE_MARKER = "[resource]" IMPORT_DIRECTIVE = "import" REQUIRES_DIRECTIVE = "requires" REQUIRED_CONCEPTS_DIRECTIVE = "required_concepts" EXPORTED_CONCEPTS_DIRECTIVE = "exported_concepts" PLAIN_SOURCE_FILE_EXTENSION = ".plain" PLAIN_SOURCE_TEMPLATE = { plain_spec.DEFINITIONS: None, plain_spec.NON_FUNCTIONAL_REQUIREMENTS: None, plain_spec.TEST_REQUIREMENTS: None, } @dataclass class PlainFileParseResult: plain_source: dict plain_source_obj: frontmatter.Post required_modules: list[str] required_concepts: list[str] class PlainRenderer(MarkdownRenderer): def render_link(self, token: Link) -> Iterable[Fragment]: yield from self.embed_span( Fragment(f"`{RESOURCE_MARKER}"), cast("Iterable[SpanToken]", token.children), Fragment("`"), ) def get_filename_from_module_name(module_name: str) -> str: return f"{module_name}{PLAIN_SOURCE_FILE_EXTENSION}" def get_module_name_from_filename(filename: str) -> str: return filename.replace(PLAIN_SOURCE_FILE_EXTENSION, "") def remove_quotes(token): # If the token has no children, there's nothing to remove. if not hasattr(token, "children") or token.children is None: return # Convert children to a list for easy filtering. children_list = list(token.children) # Build a filtered list that excludes Quote tokens. new_children = [] for child in children_list: if not isinstance(child, Quote): # Recursively remove quotes in any nested children. remove_quotes(child) new_children.append(child) # Convert the filtered list back to a tuple (or a list, if you prefer). # Mistletoe tokens often expect a tuple, so tuple is safest. token.children = tuple(new_children) def check_section_for_linked_resources(section, template_dirs): linked_resources = [] for link in traverse(section, klass=Link): parsed_url = urlparse(link.node.target) if parsed_url.scheme != "" or os.path.isabs(link.node.target): raise PlainSyntaxError( f"Plain syntax error: Only relative links are allowed (text: {link.node.children[0].content}, target: {link.node.target})." ) resolved_target = loaders.resolve_linked_resource(template_dirs, link.node.target) if resolved_target is None: searched_dirs = "\n".join(f" {position}. {dir}" for position, dir in enumerate(template_dirs, start=1)) raise PlainSyntaxError( f"Plain syntax error: Link {link.node.target} does not exist. " f"Linked resources are looked up relative to the following directories " f"(highest to lowest precedence):\n{searched_dirs}" ) if not os.path.isfile(resolved_target): raise PlainSyntaxError( f"Plain syntax error: Link {link.node.target} must be a file (resolved to {resolved_target})." ) if len(link.node.children) != 1: raise PlainSyntaxError(f"Plain syntax error: Link must have text specified (link: {link.node.target}).") linked_resource_file_extension = os.path.splitext(os.path.basename(link.node.target))[1] if linked_resource_file_extension == ".plain": raise PlainSyntaxError( f"Referenced resource '{link.node.target}' is a .plain file. " f"Referencing .plain files through Linked Resources is not supported. " "Please use the import section to include definitions or implementation/test requirements from another module." ) linked_resources.append({"text": link.node.children[0].content, "target": link.node.target}) if linked_resources: section.linked_resources = linked_resources def check_for_linked_resources(plain_source, template_dirs): for specification_heading in plain_spec.ALLOWED_SPECIFICATION_HEADINGS: if specification_heading in plain_source and hasattr(plain_source[specification_heading], "children"): for requirement in plain_source[specification_heading].children: check_section_for_linked_resources(requirement, template_dirs) if hasattr(requirement, plain_spec.ACCEPTANCE_TESTS): for acceptance_test in requirement.acceptance_tests: check_section_for_linked_resources(acceptance_test, template_dirs) def process_section_code_variables(section, code_variables): if "markdown" not in section: return code_variables_array = [] for key, value in code_variables.items(): if key not in section["markdown"]: continue variable_name = next(iter(value)) section["markdown"] = section["markdown"].replace(key, f"{{{{ {variable_name} }}}}") code_variables_array.append({"name": variable_name, "value": value[variable_name]}) if code_variables_array: section["code_variables"] = code_variables_array def process_code_variables(plain_source, code_variables): for specification_heading in plain_spec.ALLOWED_SPECIFICATION_HEADINGS: if specification_heading in plain_source: for requirement in plain_source[specification_heading]: process_section_code_variables(requirement, code_variables) def has_functional_specs_section(plain_source) -> bool: """Whether the module declares a ***functional specs*** section (even if it is empty).""" return plain_spec.FUNCTIONAL_REQUIREMENTS in plain_source def count_functionalities(plain_source) -> int: """Number of functionalities under the ***functional specs*** section (0 if absent or empty).""" section = plain_source.get(plain_spec.FUNCTIONAL_REQUIREMENTS) return len(section.children) if section is not None else 0 def validate_functionalities_have_implementation_reqs(plain_source, module_name, has_requires=False) -> None: """Raise if the module has functionalities but no implementation reqs are specified. ``has_requires`` indicates whether the module depends on another module via ``requires``. Unlike ``import``, ``requires`` does not inherit implementation reqs, so a hint is added to steer users who expected inheritance. """ implementation_reqs = plain_source[plain_spec.NON_FUNCTIONAL_REQUIREMENTS] has_implementation_reqs = ( implementation_reqs is not None and hasattr(implementation_reqs, "children") and len(implementation_reqs.children) > 0 ) if not has_implementation_reqs: message = ( f"Plain syntax error: Module '{module_name}' defines functionalities but specifies no " f"{plain_spec.NON_FUNCTIONAL_REQUIREMENTS}. At least one implementation req is required." ) if has_requires: message += ( " Implementation reqs are not inherited through 'requires' — " "add them to this module, or 'import' a module that provides them." ) raise PlainSyntaxError(message) def _is_acceptance_test_heading(token) -> tuple[bool, str | None]: """ Check if token is an 'Acceptance test:' heading. This method is going to evaluate to True when: - The token is a Paragraph - The paragraph has 1 child - The child is an Emphasis (*** ... *** marker) - The Emphasis has 1 child (the raw text content) - The child of the Emphasis is a Strong - The Strong has 1 child - The child of the Strong is a RawText - The RawText content is equal to "Acceptance test:" This is expected structure for the acceptance test heading. """ if not isinstance(token, Paragraph): return False, None # Check for the specific structure, descending one level at a time. mistletoe types # ``children`` as ``Optional[Iterable[Token]]``, so cast each level to a concrete # sequence before indexing/len. children = cast("Sequence[Token]", token.children) if len(children) != 1 or not isinstance(children[0], Emphasis): return False, None emphasis_children = cast("Sequence[Token]", children[0].children) if len(emphasis_children) != 1 or not isinstance(emphasis_children[0], Strong): return False, None strong_children = cast("Sequence[Token]", emphasis_children[0].children) if len(strong_children) != 1 or not isinstance(strong_children[0], RawText): return False, None # Check the actual text content content = strong_children[0].content.strip() if content == plain_spec.ACCEPTANCE_TEST_HEADING: return True, None problem = f"Syntax error at line {token.line_number}: Invalid acceptance test heading (`{content}`). Expected: `{plain_spec.ACCEPTANCE_TEST_HEADING}`." return False, problem def _find_acceptance_test_heading(token): """ Recursively search a token tree for an `acceptance tests` heading. Returns the heading token if one is found anywhere in the subtree, otherwise None. Used to detect acceptance tests nested under sections other than functional specs. """ is_acceptance_test_heading, _ = _is_acceptance_test_heading(token) if is_acceptance_test_heading: return token if getattr(token, "children", None): for child in token.children: found = _find_acceptance_test_heading(child) if found is not None: return found return None def _process_single_acceptance_test_requirement(functional_requirement: mistletoe.block_token.ListItem): """ Process a single functionality to extract acceptance tests. Expected functional_requirement properties: - Is a list item - If acceptance tests are specified, it has 3 children: - List item element with functionality instructions/text - Paragraph with `***Acceptance test:***` heading - List of acceptance tests - If acceptance tests are not specified, it has 1 child: - List item element with functionality instructions/text """ new_children: list = [] functional_requirement_children = iter(functional_requirement.children or ()) acceptance_tests_found_already = False for functional_requirement_child in functional_requirement_children: is_acceptance_test_heading, acceptance_test_heading_problem = _is_acceptance_test_heading( functional_requirement_child ) if acceptance_test_heading_problem: # Handle the case when the heading is not valid. This case includes cases such as: # - Writing `acceptance test` instead of `acceptance tests` (or any other syntax diffs). # - Instead of specifying `acceptance tests` below the functionality, creator of the plain file # might have specified some other building block (e.g. `implementation reqs`) raise PlainSyntaxError(f"Plain syntax error: {acceptance_test_heading_problem}") if is_acceptance_test_heading: if acceptance_tests_found_already: # Handle edge case of duplicated ***acceptance tests*** heading raise PlainSyntaxError( f"Plain syntax error: Syntax error at line {functional_requirement_child.line_number}: Duplicate 'acceptance tests' heading found within the same functionality. Only one block of acceptance tests is allowed per functionality." ) try: # If there is an acceptance test heading, the next token should be a list, with children being list items next_token = next(functional_requirement_children) if isinstance(next_token, List): # Found valid acceptance test list -> Assign it property acceptance_tests and append all list items to it if not hasattr(functional_requirement, plain_spec.ACCEPTANCE_TESTS): functional_requirement.acceptance_tests = [] for list_item in next_token.children: functional_requirement.acceptance_tests.append(list_item) acceptance_tests_found_already = True else: # Not followed by a list, keep both tokens new_children.append(functional_requirement_child) new_children.append(next_token) except StopIteration: # No next token, keep this one new_children.append(functional_requirement_child) else: # Regular token, keep it new_children.append(functional_requirement_child) # Assign the children property to all the children of the functionality from previous, with exception # of those we parsed as acceptance tests container_type = cast(type, type(functional_requirement.children)) functional_requirement.children = container_type(new_children) def process_acceptance_tests(plain_source): # Early returns for cases without functionalities if plain_spec.FUNCTIONAL_REQUIREMENTS not in plain_source: return frs = plain_source[plain_spec.FUNCTIONAL_REQUIREMENTS] if not hasattr(frs, "children"): return # Process each functionality for functional_requirement in frs.children: if not hasattr(functional_requirement, "children"): continue # Process each requirement to extract acceptance tests _process_single_acceptance_test_requirement(functional_requirement) def get_raw_text(token): if isinstance(token, RawText): yield token.content elif hasattr(token, "children"): if token.children is None: return for child in token.children: yield from get_raw_text(child) elif hasattr(token, "content"): yield token.content else: raise Exception(f"Unknown token type: {type(token)}") def marshall_plain_source(input_plain_source): plain_source = {} with PlainRenderer() as renderer: if "ID" in input_plain_source: plain_source["ID"] = input_plain_source["ID"] if "Heading" in input_plain_source: plain_source["Heading"] = input_plain_source["Heading"] for specification_heading in plain_spec.ALLOWED_SPECIFICATION_HEADINGS: if specification_heading in input_plain_source and hasattr( input_plain_source[specification_heading], "children" ): list_of_requirements = [] for requirement in input_plain_source[specification_heading].children: requirement_section = {"markdown": renderer.render(requirement).strip()} if hasattr(requirement, "linked_resources"): requirement_section["linked_resources"] = requirement.linked_resources if hasattr(requirement, plain_spec.ACCEPTANCE_TESTS): requirement_section[plain_spec.ACCEPTANCE_TESTS] = [] for acceptance_test in requirement.acceptance_tests: acceptance_test_section = {"markdown": renderer.render(acceptance_test).strip()} if hasattr(acceptance_test, "linked_resources"): acceptance_test_section["linked_resources"] = acceptance_test.linked_resources requirement_section[plain_spec.ACCEPTANCE_TESTS].append(acceptance_test_section) list_of_requirements.append(requirement_section) plain_source[specification_heading] = list_of_requirements return plain_source class Plain2CodeDictLoader(Plain2CodeLoaderMixin, DictLoader): pass def render_plain_source(plain_source, loaded_templates, code_variables): env = Environment(loader=Plain2CodeDictLoader(loaded_templates)) env.tags["include"] = Plain2CodeIncludeTag(env) env.filters["code_variable"] = plain_spec.code_variable_liquid_filter env.filters["prohibited_chars"] = plain_spec.prohibited_chars_liquid_filter template = env.from_string(plain_source) return template.render(code_variables=code_variables) def process_imports( plain_source: dict, imports: list[str], code_variables: dict, template_dirs: list[str], imported_modules: list[str], modules_trace: list[str], ) -> list[str]: required_concepts = list[str]() for module_name in imports: if module_name in modules_trace: raise PlainSyntaxError(f"Plain syntax error: Circular import detected: {module_name}.") if module_name in imported_modules: continue plain_file_parse_result = parse_plain_file( module_name, code_variables, template_dirs, imported_modules, modules_trace ) if has_functional_specs_section(plain_file_parse_result.plain_source): raise ImportedModuleWithFunctionalitiesError( f"Module '{module_name}' is imported but contains functional specs. " f"Imported modules may only provide definitions, implementation reqs, and test reqs — " f"use 'requires' instead of 'import' if this module's functionalities should be built on." ) for specification_heading in plain_file_parse_result.plain_source: if specification_heading not in plain_spec.ALLOWED_IMPORT_SPECIFICATION_HEADINGS: raise PlainSyntaxError( f"Plain syntax error: Invalid specification heading (`{specification_heading}`). Allowed headings: {', '.join(plain_spec.ALLOWED_IMPORT_SPECIFICATION_HEADINGS)}" ) if plain_source[specification_heading] is None: plain_source[specification_heading] = plain_file_parse_result.plain_source[specification_heading] elif plain_file_parse_result.plain_source[specification_heading] is not None: plain_source[specification_heading].children.extend( plain_file_parse_result.plain_source[specification_heading].children ) if REQUIRED_CONCEPTS_DIRECTIVE in plain_file_parse_result.plain_source_obj.metadata: for item in plain_file_parse_result.plain_source_obj.metadata[REQUIRED_CONCEPTS_DIRECTIVE]: new_concepts, _ = concept_utils.extract_concepts_from_definition(item) required_concepts.extend(new_concepts) required_concepts.extend(plain_file_parse_result.required_concepts) imported_modules.append(module_name) return required_concepts def normalize_line_endings(plain_source_text: str) -> str: return plain_source_text.replace("\r\n", "\n").replace("\r", "\n") def restore_stripped_lines(plain_source_text: str, content: str) -> str: stripped_source = plain_source_text.rstrip() if not content or not stripped_source.endswith(content): return content stripped_line_count = stripped_source[: len(stripped_source) - len(content)].count("\n") return "\n" * stripped_line_count + content def read_plain_source_metadata(plain_source_text): try: plain_source_obj = frontmatter.loads(plain_source_text) except Exception as e: raise PlainSyntaxError(f"Plain syntax error: Invalid frontmatter: {e}") for directive in [IMPORT_DIRECTIVE, REQUIRES_DIRECTIVE, EXPORTED_CONCEPTS_DIRECTIVE, REQUIRED_CONCEPTS_DIRECTIVE]: if directive in plain_source_obj.metadata and not isinstance(plain_source_obj.metadata[directive], list): plain_source_obj.metadata[directive] = [plain_source_obj.metadata[directive]] for directive in [IMPORT_DIRECTIVE, REQUIRES_DIRECTIVE]: for module_name in plain_source_obj.metadata.get(directive, []): if not isinstance(module_name, str): raise PlainSyntaxError(f"Plain syntax error: Invalid {directive} metadata. Expected a module name.") for directive in [EXPORTED_CONCEPTS_DIRECTIVE, REQUIRED_CONCEPTS_DIRECTIVE]: if directive in plain_source_obj.metadata: prepared_metadata = [] for item in plain_source_obj.metadata[directive]: if isinstance(item, dict): for k, v in item.items(): prepared_metadata.append(f"- {k}: {v}") elif isinstance(item, str): prepared_metadata.append(f"- {item}") else: raise PlainSyntaxError( f"Plain syntax error: Invalid {directive} metadata. Expected a dictionary or a string." ) plain_source_obj.metadata[directive] = prepared_metadata return plain_source_obj def parse_plain_source( # noqa: C901 plain_source_text: str, code_variables: dict, template_dirs: list[str], imported_modules: list[str], modules_trace: list[str], module_name: str | None = None, ) -> PlainFileParseResult: plain_source_text = normalize_line_endings(plain_source_text) plain_source_obj = read_plain_source_metadata(plain_source_text) plain_source = PLAIN_SOURCE_TEMPLATE.copy() if IMPORT_DIRECTIVE in plain_source_obj.metadata: required_concepts = process_imports( plain_source, plain_source_obj.metadata[IMPORT_DIRECTIVE], code_variables, template_dirs, imported_modules, modules_trace, ) else: required_concepts = list[str]() [_, loaded_templates] = loaders.get_loaded_templates(template_dirs, plain_source_text, module_name) plain_source_content = restore_stripped_lines(plain_source_text, plain_source_obj.content) plain_source_full_text = render_plain_source(plain_source_content, loaded_templates, code_variables) plain_file = mistletoe.Document(io.StringIO(plain_source_full_text)) remove_quotes(plain_file) current_specification_heading = None processed_specification_headings = set[str]() for token in plain_file.children: token_text = "".join(get_raw_text(token)).strip() if isinstance(token, Paragraph): paragraph_children = cast("Sequence[Token]", token.children) emphasis_children = ( cast("Sequence[Token]", paragraph_children[0].children) if paragraph_children and isinstance(paragraph_children[0], Emphasis) else () ) if not ( len(paragraph_children) == 1 and isinstance(paragraph_children[0], Emphasis) and len(emphasis_children) == 1 and isinstance(emphasis_children[0], Strong) ): raise PlainSyntaxError( f"Plain syntax error: Syntax error at line {token.line_number}: Invalid specification (`{token_text}`)" ) strong_children = cast("Sequence[Token]", emphasis_children[0].children) specification_heading = cast(RawText, strong_children[0]).content if specification_heading == plain_spec.ACCEPTANCE_TEST_HEADING: raise PlainSyntaxError( f"Plain syntax error: Syntax error at line {token.line_number}: {plain_spec.ACCEPTANCE_TEST_HEADING} heading should be nested under specific functional spec." ) if specification_heading not in plain_spec.ALLOWED_SPECIFICATION_HEADINGS: raise PlainSyntaxError( f"Plain syntax error: Syntax error at line {token.line_number}: Invalid specification heading (`{specification_heading}`). Allowed headings: {', '.join(plain_spec.ALLOWED_SPECIFICATION_HEADINGS)}" ) if ( specification_heading == plain_spec.FUNCTIONAL_REQUIREMENTS and specification_heading not in plain_source ): plain_source[specification_heading] = None if specification_heading in processed_specification_headings: raise PlainSyntaxError( f"Plain syntax error: Syntax error at line {token.line_number}: Duplicate specification heading (`{specification_heading}`)" ) if specification_heading == plain_spec.DEFINITIONS and current_specification_heading is not None: raise PlainSyntaxError( f"Plain syntax error: Syntax error at line {token.line_number}: Definitions specification must be the first specification in the section (`{token_text}`)" ) current_specification_heading = specification_heading if plain_source[current_specification_heading] is None: plain_source[current_specification_heading] = mistletoe.Document("") processed_specification_headings.add(current_specification_heading) elif isinstance(token, List): if current_specification_heading is None: raise PlainSyntaxError( f"Plain syntax error: Syntax error at line {token.line_number}: Missing specification heading (`{token_text}`)" ) plain_source[current_specification_heading].children.extend(token.children) else: raise PlainSyntaxError( f"Plain syntax error: Syntax error at line {token.line_number}: Invalid source structure (`{token_text}`)" ) # Acceptance tests are only allowed nested under functional specs. Reject them when # they are nested under any other section (e.g. definitions, implementation reqs, test reqs). for non_functional_heading in ( plain_spec.DEFINITIONS, plain_spec.NON_FUNCTIONAL_REQUIREMENTS, plain_spec.TEST_REQUIREMENTS, ): section = plain_source.get(non_functional_heading) if section is None: continue nested_acceptance_test_heading = _find_acceptance_test_heading(section) if nested_acceptance_test_heading is not None: raise PlainSyntaxError( f"Plain syntax error: Syntax error at line {nested_acceptance_test_heading.line_number}: {plain_spec.ACCEPTANCE_TEST_HEADING} heading should be nested under specific functional spec." ) defined_concepts = set[str]() if plain_source[plain_spec.DEFINITIONS] is not None: with PlainRenderer() as renderer: for token in plain_source[plain_spec.DEFINITIONS].children: rendered_token = renderer.render(token) new_concepts, _ = concept_utils.extract_concepts_from_definition(rendered_token) defined_concepts.update(new_concepts) for concept in new_concepts: if concept in required_concepts: required_concepts.remove(concept) if EXPORTED_CONCEPTS_DIRECTIVE in plain_source_obj.metadata: module_name = modules_trace[-1] if modules_trace else None subject = f"Module '{module_name}'" if module_name else "Module" for entry in plain_source_obj.metadata[EXPORTED_CONCEPTS_DIRECTIVE]: exported_concepts, errors = concept_utils.extract_concepts_from_definition(entry) if errors: raise PlainSyntaxError( f"Plain syntax error: {subject} exports an invalid concept (`{entry}`). " f"Expected a concept token, e.g. :ConceptName:." ) for exported_concept in exported_concepts: if exported_concept in concept_utils.DEFAULT_CONCEPTS: raise PlainSyntaxError( f"Plain syntax error: {subject} cannot export default concept {exported_concept}. " f"Only user-defined concepts can be exported." ) if exported_concept not in defined_concepts: message = f"Plain syntax error: {subject} exports undefined concept {exported_concept}." if REQUIRES_DIRECTIVE in plain_source_obj.metadata: message += ( " An exported concept must be defined in this module's definitions or in an " "imported module; a concept inherited through requires cannot be re-exported." ) raise PlainSyntaxError(message) required_modules = [] if REQUIRES_DIRECTIVE in plain_source_obj.metadata: required_modules = plain_source_obj.metadata[REQUIRES_DIRECTIVE] return PlainFileParseResult( plain_source=plain_source, plain_source_obj=plain_source_obj, required_modules=required_modules, required_concepts=required_concepts, ) def read_module_plain_source(module_name: str, template_dirs: list[str]) -> str: plain_source_text = loaders.open_from(template_dirs, module_name + PLAIN_SOURCE_FILE_EXTENSION) if plain_source_text is None: raise ModuleDoesNotExistError(f"Module does not exist ({module_name}).") blob = find_large_base64_blob(plain_source_text) if blob is not None: raise UnsupportedBase64Content( f"Module '{module_name}' contains a base64-encoded blob ({len(blob)} characters) " "inlined in the specification. This is not supported. " "Remove the base64 data from the .plain file or if necessary, " "include the binary file path in the specification." ) return plain_source_text def parse_plain_file( module_name: str, code_variables: dict, template_dirs: list[str], imported_modules: list[str], modules_trace: list[str], ) -> PlainFileParseResult: # noqa: C901 plain_source_text = read_module_plain_source(module_name, template_dirs) return parse_plain_source( plain_source_text, code_variables, template_dirs, imported_modules, modules_trace + [module_name], module_name=module_name, ) @dataclass class RequiredModule: """A required module as parsed by the requires walk; validated once the top module is complete.""" parse_result: PlainFileParseResult # Definitions exported to this module by the modules it requires. ancestor_exported_definitions: list[mistletoe.block_token.token] code_variables: dict # Definitions this module exports to the modules that require it. exported_definitions: list[mistletoe.block_token.token] # Marshalled plain source tree, set once the module is validated. tree: dict | None = None def process_required_modules( required_modules: list[str], template_dirs: list[str], all_required_modules: list[str], modules_trace: list[str], required: dict[str, RequiredModule], ) -> list[mistletoe.block_token.token]: """Parse every required module, deepest first, without validating it. Records each module in ``required`` the first time it is reached (deepest ancestors first) and returns the exported definitions of the directly required modules. """ exported_definitions = list[mistletoe.block_token.token]() for module_name in required_modules: if module_name in modules_trace: raise PlainSyntaxError(f"Plain syntax error: Circular required module detected: {module_name}.") if len(all_required_modules) > 0 and module_name == all_required_modules[-1]: # Already processed as an earlier entry; its exports are still visible to this module. exported_definitions.extend(required[module_name].exported_definitions) continue code_variables: dict = {} plain_file_parse_result = parse_plain_file( module_name, code_variables, template_dirs, imported_modules=[], modules_trace=[] ) ancestor_exported_definitions: list[mistletoe.block_token.token] = [] if len(plain_file_parse_result.required_modules) == 0: if len(all_required_modules) > 0: # For now we require that there is fixed order how required modules are dependent. # In the future we will support the cases where required modules can be rendered independently # and then merged (somehow). raise PlainSyntaxError( f"Plain syntax error: There must be a fixed order how required modules are dependent ({module_name})." ) else: ancestor_exported_definitions = process_required_modules( plain_file_parse_result.required_modules, template_dirs, all_required_modules, modules_trace + [module_name], required, ) own_exported_definitions = list[mistletoe.block_token.token]() if EXPORTED_CONCEPTS_DIRECTIVE in plain_file_parse_result.plain_source_obj.metadata: exported_concepts = list[str]() for concept in plain_file_parse_result.plain_source_obj.metadata[EXPORTED_CONCEPTS_DIRECTIVE]: exported_concepts.extend(concept_utils.extract_concepts_from_definition(concept)[0]) with PlainRenderer() as renderer: for exported_concept in exported_concepts: own_exported_definitions.extend( concept_utils.find_concept_definitions_in_plain_source( exported_concept, plain_file_parse_result.plain_source, renderer ) ) required[module_name] = RequiredModule( plain_file_parse_result, ancestor_exported_definitions, code_variables, own_exported_definitions ) exported_definitions.extend(own_exported_definitions) all_required_modules.append(module_name) return exported_definitions def validate_required_modules( required_modules: list[str], required: dict[str, RequiredModule], template_dirs: list[str] ) -> None: """Validate recorded required modules top-down, in listed order, each module once.""" for module_name in required_modules: module = required[module_name] if module.tree is not None: continue validate_module_structure(module.parse_result, module_name) module.tree = validate_and_build_module_tree( module.parse_result, module_name, module.ancestor_exported_definitions, module.code_variables, template_dirs ) validate_required_modules(module.parse_result.required_modules, required, template_dirs) def process_exported_definitions(plain_source: dict, exported_definitions: list[mistletoe.block_token.token]) -> None: if len(exported_definitions) == 0: return if plain_source[plain_spec.DEFINITIONS] is None: plain_source[plain_spec.DEFINITIONS] = mistletoe.Document("") with PlainRenderer() as renderer: for exported_definition in exported_definitions: add_defintion = True exported_rendered_definition = renderer.render(exported_definition).strip() for definition in plain_source[plain_spec.DEFINITIONS].children: rendered_definition = renderer.render(definition).strip() if exported_rendered_definition == rendered_definition: add_defintion = False break if add_defintion: plain_source[plain_spec.DEFINITIONS].children.append(exported_definition) def validate_module_structure(plain_file_parse_result: PlainFileParseResult, module_name: str) -> None: """Required concepts defined, functional specs present and non-empty, implementation reqs present.""" if len(plain_file_parse_result.required_concepts) > 0: missing_required_concepts_msg = "Missing required concepts: " missing_required_concepts_msg += ", ".join(plain_file_parse_result.required_concepts) raise PlainSyntaxError( f"Plain syntax error: Not all required concepts were defined. {missing_required_concepts_msg}." ) if not has_functional_specs_section(plain_file_parse_result.plain_source): # No ***functional specs*** section at all: valid as an import, but not renderable. Usage error. raise MissingFunctionalitiesError( f"Module '{module_name}' does not have any functionality specified. " f"At least one functionality is required for rendering." ) if count_functionalities(plain_file_parse_result.plain_source) == 0: # ***functional specs*** section present but empty: invalid in every role. Syntax error. raise PlainSyntaxError( f"Plain syntax error: Module '{module_name}' has an empty " f"'{plain_spec.FUNCTIONAL_REQUIREMENTS}' section. At least one functionality must be specified." ) validate_functionalities_have_implementation_reqs( plain_file_parse_result.plain_source, module_name, has_requires=bool(plain_file_parse_result.required_modules), ) def validate_and_build_module_tree( plain_file_parse_result: PlainFileParseResult, module_name: str, exported_definitions: list[mistletoe.block_token.token], code_variables: dict, template_dirs: list[str], ) -> dict: """Merge exported definitions, validate acceptance tests, linked resources and concepts, and return the marshalled plain source tree.""" process_exported_definitions(plain_file_parse_result.plain_source, exported_definitions) process_acceptance_tests(plain_file_parse_result.plain_source) check_for_linked_resources(plain_file_parse_result.plain_source, template_dirs) marshalled_plain_source = marshall_plain_source(plain_file_parse_result.plain_source) process_code_variables(marshalled_plain_source, code_variables) validation_errors = concept_utils.validate_concepts(marshalled_plain_source) if len(validation_errors) > 0: errors_msg = "\n".join(validation_errors) msg = f"Found {len(validation_errors)} errors in the plain file:\n{errors_msg}" raise PlainSyntaxError(f"Plain syntax error: {msg}") if plain_spec.DEFINITIONS in marshalled_plain_source: concept_utils.sort_definitions(marshalled_plain_source[plain_spec.DEFINITIONS]) return marshalled_plain_source def _parse_module( plain_source_file_name: str, template_dirs: list[str] ) -> tuple[str, dict, list[str], list[tuple[str, dict]]]: plain_source_file_path = Path(plain_source_file_name) if plain_source_file_path.suffix != PLAIN_SOURCE_FILE_EXTENSION: raise PlainSyntaxError( f"Plain syntax error: Invalid plain file extension: {plain_source_file_path.suffix}. Expected: {PLAIN_SOURCE_FILE_EXTENSION}." ) module_name = ( plain_source_file_path.stem if plain_source_file_path.is_absolute() else plain_source_file_path.with_suffix("").as_posix() ) # code_variables are populated when liquid templating is applied and consumed by process_code_variables # once the plain source is marshalled. code_variables: dict = {} plain_file_parse_result = parse_plain_file( module_name, code_variables, template_dirs, imported_modules=[], modules_trace=[], ) # Check the top module's structure, walk its requires, finish the top module's checks, then validate # every required module top-down. validate_module_structure(plain_file_parse_result, module_name) required: dict[str, RequiredModule] = {} exported_definitions = process_required_modules( plain_file_parse_result.required_modules, template_dirs=template_dirs, all_required_modules=[], modules_trace=[], required=required, ) marshalled_plain_source = validate_and_build_module_tree( plain_file_parse_result, module_name, exported_definitions, code_variables, template_dirs ) validate_required_modules(plain_file_parse_result.required_modules, required, template_dirs) # The walk records modules deepest first, which is the chain order. chain = [(chain_module_name, module.tree) for chain_module_name, module in required.items()] return module_name, marshalled_plain_source, plain_file_parse_result.required_modules, chain def plain_file_parser(plain_source_file_name: str, template_dirs: list[str]) -> tuple[str, dict, list[str]]: """Parse a module, validating it and every module in its ``requires`` chain.""" module_name, marshalled_plain_source, required_modules, _ = _parse_module(plain_source_file_name, template_dirs) return module_name, marshalled_plain_source, required_modules def parse_module_chain(plain_file_name: str, template_dirs: list[str]) -> list[tuple[str, dict]]: """Parse a module and every module in its ``requires`` chain. Returns ``(module name, marshalled plain source tree)`` pairs: every required module first, deepest ancestors first, then the module itself. A module reached through more than one ``requires`` path appears once, at its first (deepest) position. """ module_name, marshalled_plain_source, _, chain = _parse_module(plain_file_name, template_dirs) return chain + [(module_name, marshalled_plain_source)]