#!/usr/bin/env python3

# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM
# Exceptions.
# See https://edgcpp.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

import argparse
import functools
import re
import json
import shutil
import subprocess
import sys
import tempfile
import traceback

from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from typing import Any, List, Optional, Set, Tuple

import edgchanges
import edgspelling
import edgutil

TOOL_PATH_ASPELL = shutil.which('aspell')
TOOL_PATH_GET_COMMENTS = edgutil.RequiredToolPath('get-comments')

CODE_FILE_EXTENSIONS = {'.c', '.cpp', '.h', '.hpp'}

# Text and data files where long lines are normal; these skip the line length
# check.
NO_LINE_LENGTH_EXTENSIONS = {'.txt', '.json', '.yml', '.yaml'}

# (pattern, ignore_case, exclude_substrings)
_FIND_THE_THE_CHECKS = [
  ('the the', True, ()),
  ('is is', True, ()),
  ('it it', True, ()),
  ('be be', True, ()),
  ('the its', True, ()),
  ('its the', True, ()),
  ('the a', False, ()),
  ('The a', False, ()),
  ('a the', True, ()),
  ('a an', True, ()),
  ('its a', True, ()),
  ('an member', True, ()),
  ('this it a', True, ()),
  ('of of', True, ()),
  ('the that', True, ()),
  ('in in', True, ('in in-', '-in in')),
  ('if is', True, ('#if is',)),
  ('is has', True, ()),
  ('that for which', True, ()),
  ('the is', True, ()),
  ('and and', True, ()),
  ('on on', True, ()),
  ('with with', True, ()),
]

def _compile_find_the_checks():
  checks = []
  for pattern, ignore_case, excludes in _FIND_THE_THE_CHECKS:
    checks.append(
      (
        re.compile(
          r'(?<!\w)' + re.escape(pattern) + r'(?!\w)',
          re.IGNORECASE if ignore_case else 0
        ),
        excludes
      )
    )
  return checks

_FIND_THE_THE_CHECKS_COMPILED = _compile_find_the_checks()

def find_the_the(path):
  '''Find repeated-word typos like "the the".

  Matches use whole-word boundaries.  Patterns marked ignore_case use.  Some
  patterns also exclude fixed substrings.  Returns matching lines joined by
  newlines, or '' if none.
  '''

  with open(path, 'r', errors = 'replace') as handle:
    lines = handle.read().splitlines()

  return _find_the_the_in_lines(lines)

def _find_the_the_in_lines(lines: List[str]) -> List[str]:
  matches = []
  for regex, excludes in _FIND_THE_THE_CHECKS_COMPILED:
    # grep -w: match must be bounded by non-word characters (or line edges).
    for line in lines:
      if regex.search(line) is None:
        continue
      if any(exclude in line for exclude in excludes):
        continue
      matches.append(line)

  return matches

def _is_aspell_working() -> bool:
  if not TOOL_PATH_ASPELL:
    return False

  completed_process = subprocess.run(
    [TOOL_PATH_ASPELL, 'list'],
    input = 'cat'.encode('utf-8'),
    stdout = subprocess.DEVNULL,
    stderr = subprocess.DEVNULL
  )
  return completed_process.returncode == 0

def _run_spell_command(file_path: Path) -> Set[str]:
  '''Return the words aspell rejects in file_path.

  Only the comments of code files are checked.  The text is preprocessed
  first (see edgspelling.preprocess).
  '''
  with open(file_path, 'rb') as src_file_handle:
    if file_path.suffix in CODE_FILE_EXTENSIONS:
      completed_process = subprocess.run(
        [str(TOOL_PATH_GET_COMMENTS), '-c'],
        stdin = src_file_handle,
        stdout = subprocess.PIPE,
        stderr = subprocess.STDOUT
      )
      spelling_bytes = completed_process.stdout
    else:
      spelling_bytes = src_file_handle.read()

  spelling_text = edgspelling.preprocess(
    spelling_bytes.decode('utf-8', errors = 'replace')
  )
  completed_process = subprocess.run(
    [TOOL_PATH_ASPELL, 'list'],
    input = spelling_text.encode('utf-8'),
    stdout = subprocess.PIPE,
    stderr = subprocess.STDOUT
  )

  return set(completed_process.stdout.decode().splitlines())

def _without_allowed_words(allowlist: edgspelling.Allowlist,
                           words: Set[str],
                           repo_path: str) -> Set[str]:
  '''Drop the words allowlist allows for repo_path.'''
  allowed_words = allowlist.words_for_path(repo_path)
  allowed_names = allowlist.names_for_path(repo_path)
  return {
    word for word in words
    if word.lower() not in allowed_words and word not in allowed_names
  }

def check_spelling(allowlist: edgspelling.Allowlist,
                   repo_path: str,
                   old_file: Path,
                   new_file: Path) -> Optional[List[str]]:
  '''Report new misspellings relative to the old file.

  Words allowlist allows for repo_path aren't reported.
  '''

  if not _is_aspell_working():
    return None

  new_mispelled_words = _run_spell_command(new_file)
  old_mispelled_words = _run_spell_command(old_file)

  newly_mispelled_words = new_mispelled_words - old_mispelled_words
  return sorted(
    _without_allowed_words(allowlist, newly_mispelled_words, repo_path)
  )

@functools.lru_cache(maxsize = None)
def _changes_file_misspellings(changes_path: Path) -> Set[str]:
  '''Return the words aspell rejects in changes_path (empty if there's no
  such file).'''
  if not changes_path.is_file():
    return set()
  return _run_spell_command(changes_path)

def _check_fragment_spelling(allowlist: edgspelling.Allowlist,
                             repo_path: str,
                             new_file: Path) -> Optional[List[str]]:
  '''Report misspellings in a Changes entry.

  The directives and the tag block aren't checked, and words the entry's
  //allowed_words: directives list aren't reported.  Neither are words
  the allowlist allows for the entry's path, nor words
  already in the Changes file the entry will be collected into (so a term
  like "constexpr" needn't be allowed in every entry).
  '''
  if not _is_aspell_working():
    return None

  allowed_words, prose_lines = edgchanges.read_prose(new_file)
  with tempfile.TemporaryDirectory() as tmp_dir_str:
    text_path = Path(tmp_dir_str) / 'prose.txt'
    with open(text_path, 'w', encoding = 'utf-8') as handle:
      handle.write('\n'.join(prose_lines) + '\n')
    mispelled_words = _run_spell_command(text_path)

  # Accept the words the entry allows and those its Changes file already
  # uses.
  target_path = edgchanges.target_changes_path(Path.cwd() / repo_path)
  accepted_words = allowed_words | _changes_file_misspellings(target_path)

  return sorted(
    _without_allowed_words(
      allowlist, mispelled_words - accepted_words, repo_path
    )
  )

def _file_check_config(file_path: Path):
  '''Return spelling/coding/line-length check settings for a path.'''
  suffix = file_path.suffix

  check_for_coding_errors = False
  check_for_overlong_lines = False

  if suffix in CODE_FILE_EXTENSIONS:
    check_for_coding_errors = True
    check_for_overlong_lines = True
  elif suffix in NO_LINE_LENGTH_EXTENSIONS:
    pass
  else:
    check_for_overlong_lines = True

  return check_for_coding_errors, check_for_overlong_lines

ENDIF_WITHOUT_COMMENT_REGEX = re.compile(r'^#endif[ \t]*$')
ELSE_WITHOUT_COMMENT_REGEX = re.compile(r'^#else[ \t]*$')

def _check_coding_errors(new_file: Path) -> Any:
  '''Report missing #endif/#else comments and closing-comment issues.'''
  coding_errors = {
    'missing_endifs': [],
    'missing_elses': [],
    'closing_comments': []
  }

  with open(new_file, 'r', errors = 'replace') as file_handle:
    for line_no, line in enumerate(file_handle, start = 1):
      if ENDIF_WITHOUT_COMMENT_REGEX.match(line.rstrip('\n')):
        coding_errors['missing_endifs'].append(f"{line_no}:{line.rstrip()}")
      if ELSE_WITHOUT_COMMENT_REGEX.match(line.rstrip('\n')):
        coding_errors['missing_elses'].append(f"{line_no}:{line.rstrip()}")

  completed_process = subprocess.run(
    ['check-closing-comments', new_file],
    capture_output = True,
    text = True
  )
  if completed_process.returncode != 0:
    coding_errors['closing_comments'] = completed_process.stdout.splitlines()

  return coding_errors

MD_TABLE_DELIMITER_CHARS = set('|:- \t')
RST_GRID_TABLE_BORDER_REGEX = re.compile(r'^\s*\+([-=]+\+)+\s*$')
RST_SIMPLE_TABLE_BORDER_REGEX = re.compile(r'^\s*=+([ \t]+=+)+\s*$')

def _is_md_table_delimiter_row(stripped_line: str) -> bool:
  '''Return True if stripped_line looks like the delimiter row of a pipe
  table (e.g., "| --- | :-: |").'''
  return (
    '-' in stripped_line
    and set(stripped_line) <= MD_TABLE_DELIMITER_CHARS
  )

def _md_table_line_numbers(lines: List[str]) -> Set[int]:
  '''Return the (1-based) numbers of the lines of the pipe tables in lines.

  This is an approximation: a table is a run of lines starting with "|" (after
  any indentation) that includes a delimiter row.
  '''
  table_line_numbers = set()
  index = 0
  while index < len(lines):
    if not lines[index].lstrip().startswith('|'):
      index += 1
      continue
    run_start = index
    while index < len(lines) and lines[index].lstrip().startswith('|'):
      index += 1
    if any(_is_md_table_delimiter_row(line.strip())
           for line in lines[run_start:index]):
      table_line_numbers.update(range(run_start + 1, index + 1))
  return table_line_numbers

def _is_rst_line_block_line(line: str) -> bool:
  '''Return True if line starts a line of an RST line block ("| text", or a
  lone "|" for a blank line, after any indentation).'''
  stripped_line = line.strip()
  return stripped_line == '|' or line.lstrip().startswith(('| ', '|\t'))

def _indentation(line: str) -> int:
  '''Return the width of the leading whitespace of line.'''
  return len(line.expandtabs()) - len(line.expandtabs().lstrip())

def _rst_unwrappable_line_numbers(lines: List[str]) -> Set[int]:
  '''Return the (1-based) numbers of the lines of the grid tables, simple
  tables, and line blocks in lines.

  This is an approximation: a grid table starts with a "+---+" border and
  continues while lines start with "+" or "|" (after any indentation).  A
  simple table starts with a border of two or more "=" columns and ends at a
  later border that's followed by a blank line or the end of the file.  A
  line block is a run of lines starting with "| " (or a lone "|"), along with
  their continuation lines, which are indented past the "|".  Directive
  tables (list-table, csv-table) aren't recognized; their cells can be
  wrapped.
  '''
  def is_blank_or_end(index: int) -> bool:
    return index >= len(lines) or not lines[index].strip()

  unwrappable_line_numbers = set()
  index = 0
  while index < len(lines):
    line = lines[index]
    if _is_rst_line_block_line(line):
      bar_indentation = _indentation(line)
      unwrappable_line_numbers.add(index + 1)
      index += 1
      while (index < len(lines)
             and lines[index].strip()
             and not _is_rst_line_block_line(lines[index])
             and _indentation(lines[index]) > bar_indentation):
        unwrappable_line_numbers.add(index + 1)
        index += 1
      continue
    if RST_GRID_TABLE_BORDER_REGEX.match(line):
      table_start = index
      index += 1
      while (index < len(lines)
             and lines[index].lstrip().startswith(('+', '|'))):
        index += 1
      unwrappable_line_numbers.update(range(table_start + 1, index + 1))
      continue
    if (RST_SIMPLE_TABLE_BORDER_REGEX.match(line)
        and not is_blank_or_end(index + 1)):
      table_end = next(
        (
          end for end in range(index + 1, len(lines))
          if RST_SIMPLE_TABLE_BORDER_REGEX.match(lines[end])
          and is_blank_or_end(end + 1)
        ),
        None
      )
      if table_end is not None:
        unwrappable_line_numbers.update(range(index + 1, table_end + 2))
        index = table_end + 1
        continue
    index += 1
  return unwrappable_line_numbers

def _unwrappable_line_numbers(suffix: str, lines: List[str]) -> Set[int]:
  '''Return the (1-based) numbers of the lines that can't be wrapped in lines,
  the contents of a file with the given suffix.  Only Markdown tables and
  reStructuredText tables and line blocks are recognized.'''
  if suffix == '.md':
    return _md_table_line_numbers(lines)
  if suffix == '.rst':
    return _rst_unwrappable_line_numbers(lines)
  return set()

def _check_overlong_lines(new_file: Path,
                          ignore_trailing_whitespace: bool = False) -> Any:
  '''Report lines longer than 79 columns after tab expansion.

  The rows of Markdown and reStructuredText tables and the lines of
  reStructuredText line blocks can't be wrapped, so they aren't reported,
  though they still appear in the context of other lines.
  '''
  overlong_lines = {}

  with open(new_file, 'r', errors = 'replace') as file_handle:
    lines = file_handle.readlines()
    unwrappable_line_numbers = _unwrappable_line_numbers(
      new_file.suffix, lines
    )
    for line_no, line in enumerate(lines, start = 1):
      if line_no in unwrappable_line_numbers:
        continue
      checked_line = (
        line.rstrip() if ignore_trailing_whitespace else line.rstrip('\n')
      )
      if len(checked_line.expandtabs()) <= 79:
        continue

      overlong_lines[line_no] = [
         max(1, line_no - 1),
         lines[max(0, line_no - 2):min(len(lines), line_no + 1)]
      ]

  return overlong_lines

def check_changes_fragment(allowlist: edgspelling.Allowlist,
                           repo_path: str,
                           new_file: Path) -> Any:
  '''Check new_file, the contents of the pending Changes entry repo_path.

  Besides the usual checks, its format is checked (see edgchanges).  Its
  spelling is checked against the Changes file it will be collected into, not
  against an old version of the entry (see _check_fragment_spelling), and
  trailing whitespace (stripped when the entry is collected) doesn't count
  toward the line length.
  '''
  _, format_errors = edgchanges.parse_fragment(
    new_file,
    check_line_length = False
  )
  _, prose_lines = edgchanges.read_prose(new_file)
  return {
    'format_errors': format_errors,
    'misspellings': _check_fragment_spelling(
      allowlist, repo_path, new_file
    ),
    'typos': _find_the_the_in_lines(prose_lines),
    'coding_errors': {},
    'overlong_lines': _check_overlong_lines(
      new_file,
      ignore_trailing_whitespace = True
    )
  }

def check_file(allowlist: edgspelling.Allowlist,
               repo_path: str,
               old_file: Path,
               new_file: Path) -> Any:
  '''Check new_file, the new contents of repo_path, against old_file.'''
  if edgchanges.is_fragment_path(repo_path):
    return check_changes_fragment(allowlist, repo_path, new_file)

  check_for_coding_errors, check_for_overlong_lines = (
    _file_check_config(new_file)
  )
  return {
    'format_errors': [],
    'misspellings': check_spelling(
      allowlist, repo_path, old_file, new_file
    ),
    'typos': find_the_the(new_file),
    'coding_errors': (
      _check_coding_errors(new_file) if check_for_coding_errors else {}
    ),
    'overlong_lines': (
      _check_overlong_lines(new_file) if check_for_overlong_lines else []
    )
  }

def parse_args() -> argparse.Namespace:
  parser = argparse.ArgumentParser(
    description = (
      'Perform checking on the files defined by the given input file.'
    )
  )
  parser.add_argument(
    'input_spec',
    type = Path,
    help = 'the json file defining what to check'
  )
  parser.add_argument(
    'output_file',
    type = Path,
    help = 'the json file reporting the results of the check'
  )
  return parser.parse_args()

def main() -> None:
  args = parse_args()

  # Refuse to start with a bad allowlist rather than checking against a
  # partial one.
  try:
    allowlist = edgspelling.Allowlist()
  except edgspelling.AllowlistError as error:
    edgutil.eprint(f"Invalid spelling allowlist: {error}")
    sys.exit(1)

  with open(args.input_spec, 'r', encoding = 'utf-8') as file_handle:
    input_dict = json.load(file_handle)

  # Only code files need get-comments, so checking just Changes entries (as
  # CI does) works without the built tools.
  if any(Path(new_file).suffix in CODE_FILE_EXTENSIONS
         for _, new_file in input_dict.values()):
    TOOL_PATH_GET_COMMENTS.exit_if_not_found()

  output_dict = dict.fromkeys(input_dict)

  with ThreadPoolExecutor() as executor:
    result = {}
    cwd = Path.cwd()

    for key, (old_file, new_file) in input_dict.items():
      output_dict[key] = executor.submit(
        check_file,
        allowlist,
        key,
        cwd.joinpath(old_file),
        cwd.joinpath(new_file)
      )

  for key in list(output_dict.keys()):
    try:
      output_dict[key] = output_dict[key].result()
    except Exception as ex:
      if sys.version_info >= (3, 10):
        ex.add_note(f"while processing file: {key}")
        traceback.print_exception(ex)
      else:
        print(f"{key} generated an exception: {ex}")
      del output_dict[key]

  with open(args.output_file, 'w') as file_handle:
    json.dump(output_dict, file_handle)

main()