#! /usr/bin/env python
#coding=utf8

import json
import os
import re
import sys
import time
from datetime import date, datetime, timedelta
from io import StringIO
from urllib.error import HTTPError, URLError
from urllib.parse import quote, urlencode, urlparse
from urllib.request import Request, urlopen

import ruamel.yaml

from alerts import AlertDeliveryError, send_alert
from atomic_io import atomic_write_many_text
from history_store import (
    append_snapshot,
    field_on_or_before,
    load_history,
    serialize_history,
)

sys.stdout.reconfigure(encoding='utf-8')


GH_CREDENTIALS_PATH = '/home/slisakov/gh_credentials'
GITLAB_CREDENTIALS_PATH = '/home/slisakov/gitlab_credentials'
STARS_DIFF_DAYS = 30
STAR_DROP_WARNING_THRESHOLD = 20
FETCH_ATTEMPTS = 3
RETRYABLE_HTTP_CODES = {429, 500, 502, 503, 504}
MAX_API_RESPONSE_BYTES = 5 * 1024 * 1024


yaml = ruamel.yaml.YAML()
yaml.indent(mapping=4, sequence=4, offset=2)
yaml.preserve_quotes = True


def sources_list(data_item):
    sources = data_item.get('source')
    if not isinstance(sources, list):
        sources = [sources]
    return sources


def github_repo_api_url(source):
    parsed = urlparse(str(source))
    if parsed.netloc not in ('github.com', 'www.github.com'):
        return None

    parts = [part for part in parsed.path.split('/') if part]
    if len(parts) < 2:
        return None

    owner = parts[0]
    repo = re.sub(r'\.git$', '', parts[1])
    return 'https://api.github.com/repos/{}/{}'.format(owner, repo)


def gitlab_repo_api_url(source):
    parsed = urlparse(str(source))
    if parsed.netloc not in ('gitlab.com', 'www.gitlab.com'):
        return None

    parts = [part for part in parsed.path.split('/') if part]
    if '-' in parts:
        parts = parts[:parts.index('-')]
    if len(parts) < 2:
        return None

    parts[-1] = re.sub(r'\.git$', '', parts[-1])
    project_path = '/'.join(parts)
    return '{}://{}/api/v4/projects/{}'.format(
        parsed.scheme or 'https',
        parsed.netloc,
        quote(project_path, safe=''),
    )


def repo_sources(data_item):
    sources = []
    seen = set()
    for source in sources_list(data_item):
        github_api_url = github_repo_api_url(source)
        if github_api_url:
            key = ('github', github_api_url)
            if key in seen:
                raise RuntimeError('duplicate repository source: {}'.format(source))
            sources.append({'provider': 'github', 'api_url': github_api_url, 'url': str(source)})
            seen.add(key)
            continue

        gitlab_api_url = gitlab_repo_api_url(source)
        if gitlab_api_url:
            key = ('gitlab', gitlab_api_url)
            if key in seen:
                raise RuntimeError('duplicate repository source: {}'.format(source))
            sources.append({'provider': 'gitlab', 'api_url': gitlab_api_url, 'url': str(source)})
            seen.add(key)
            continue

        parsed = urlparse(str(source))
        if parsed.netloc.lower() in (
            'github.com',
            'www.github.com',
            'gitlab.com',
            'www.gitlab.com',
        ):
            raise RuntimeError('malformed repository source URL: {}'.format(source))

    return sources


def token_from_file(path):
    if not os.path.exists(path):
        return None

    with open(path, 'r', encoding='utf-8') as f:
        for line in f:
            parts = line.split()
            if len(parts) >= 2:
                return parts[1]
            if len(parts) == 1:
                return parts[0]
    return None


def retry_delay(headers, attempt):
    retry_after = headers.get('Retry-After') if headers else None
    if retry_after and str(retry_after).isdigit():
        return min(max(int(retry_after), 1), 60)
    return 2 ** (attempt - 1)


def fetch_json(url, headers, on_anomaly=None):
    for attempt in range(1, FETCH_ATTEMPTS + 1):
        request = Request(url, headers=headers)
        try:
            with urlopen(request, timeout=60) as response:
                raw_body = response.read(MAX_API_RESPONSE_BYTES + 1)
                if len(raw_body) > MAX_API_RESPONSE_BYTES:
                    raise RuntimeError(
                        'API response for {} exceeds {} bytes'.format(
                            url,
                            MAX_API_RESPONSE_BYTES,
                        )
                    )
                body = raw_body.decode('utf-8')
                final_url = response.geturl()
                try:
                    return json.loads(body), response.headers, final_url
                except json.JSONDecodeError as error:
                    raise RuntimeError('API returned invalid JSON for {}: {}'.format(url, error))
        except HTTPError as error:
            body = error.read(2048).decode('utf-8', errors='replace')
            lower_body = body.lower()
            if (
                (error.headers or {}).get('cf-mitigated', '').lower() == 'challenge'
                or 'challenge.cloudflare.com' in lower_body
                or '<title>just a moment' in lower_body
            ):
                detail = 'Cloudflare challenge page'
            elif '<html' in lower_body:
                detail = 'HTML error page'
            else:
                detail = body.replace('\n', ' ')[:500]
            message = 'API returned {} for {}: {}'.format(
                error.code,
                url,
                detail,
            )
            if error.code not in RETRYABLE_HTTP_CODES or attempt == FETCH_ATTEMPTS:
                raise RuntimeError(message)
            error_headers = error.headers
        except (URLError, TimeoutError) as error:
            message = 'Could not fetch {}: {}'.format(
                url,
                getattr(error, 'reason', error),
            )
            if attempt == FETCH_ATTEMPTS:
                raise RuntimeError(message)
            error_headers = getattr(error, 'headers', None)

        retry_message = 'Temporary API error; retrying {}/{}: {}'.format(
            attempt + 1,
            FETCH_ATTEMPTS,
            message,
        )
        print(retry_message, file=sys.stderr)
        if on_anomaly:
            on_anomaly('API request required a retry', retry_message)
        time.sleep(retry_delay(error_headers, attempt))


def require_api_fields(api_data, fields, url):
    if not isinstance(api_data, dict):
        raise RuntimeError('API returned an unexpected response for {}'.format(url))

    missing = [field for field in fields if field not in api_data]
    if missing:
        raise RuntimeError(
            'API response for {} is missing required fields: {}'.format(
                url,
                ', '.join(missing),
            )
        )


def require_nonnegative_int(api_data, field, url):
    value = api_data.get(field)
    if isinstance(value, bool) or not isinstance(value, int) or value < 0:
        raise RuntimeError(
            'API field {} for {} must be a non-negative integer, got {!r}'.format(
                field,
                url,
                value,
            )
        )
    return value


def require_url(value, field, url):
    parsed = urlparse(str(value))
    if parsed.scheme != 'https' or not parsed.netloc:
        raise RuntimeError(
            'API field {} for {} is not a valid HTTPS URL: {!r}'.format(
                field,
                url,
                value,
            )
        )
    return str(value)


def replace_source_url(item, old_url, new_url):
    sources = item.get('source')
    if isinstance(sources, list):
        for index, source in enumerate(sources):
            if str(source) == old_url:
                sources[index] = new_url
                return True
        return False

    if str(sources) == old_url:
        item['source'] = new_url
        return True
    return False


def github_headers(token):
    headers = {
        'Accept': 'application/vnd.github+json',
        'User-Agent': 'open-source-comments-updater',
    }
    if token:
        headers['Authorization'] = 'token {}'.format(token)
    return headers


def gitlab_headers(token):
    headers = {
        'Accept': 'application/json',
        'User-Agent': 'open-source-comments-updater',
    }
    if token:
        headers['PRIVATE-TOKEN'] = token
    return headers


def api_date(value):
    if not value:
        return 'undefined'
    raw_date = str(value)[:10]
    try:
        datetime.strptime(raw_date, '%Y-%m-%d')
    except ValueError:
        raise RuntimeError('API returned an invalid date: {!r}'.format(value))
    return raw_date.replace('-', '‑')


def github_license(api_data, item):
    license_data = api_data.get('license')
    if not license_data:
        return item.get('license') or 'undefined'

    return license_data.get('spdx_id') or license_data.get('name') or 'undefined'


def gitlab_license(api_data, item):
    license_data = api_data.get('license')
    if isinstance(license_data, dict):
        return license_data.get('key') or license_data.get('nickname') or license_data.get('name') or 'undefined'
    return item.get('license') or 'undefined'


def github_commit_date(api_url, default_branch, token, on_anomaly=None):
    branch = quote(default_branch or 'master', safe='')
    commit_url = '{}/commits/{}'.format(api_url, branch)
    commit_data, _headers, _final_url = fetch_json(
        commit_url,
        github_headers(token),
        on_anomaly=on_anomaly,
    )
    commit_date = commit_data.get('commit', {}).get('committer', {}).get('date')
    if not commit_date:
        raise RuntimeError('API response for {} has no commit date'.format(commit_url))
    return api_date(commit_date)


def gitlab_commit_date(api_url, default_branch, token, on_anomaly=None):
    branch = default_branch or 'master'
    branch_url = '{}/repository/branches/{}'.format(api_url, quote(branch, safe=''))
    branch_data, _headers, _final_url = fetch_json(
        branch_url,
        gitlab_headers(token),
        on_anomaly=on_anomaly,
    )
    if not isinstance(branch_data, dict) or branch_data.get('name') != branch:
        raise RuntimeError('API returned an unexpected branch for {}'.format(branch_url))
    commit_data = branch_data.get('commit')
    if not isinstance(commit_data, dict):
        raise RuntimeError('API response for {} has no branch commit'.format(branch_url))
    commit_date = commit_data.get('committed_date') or commit_data.get('created_at')
    if not commit_date:
        raise RuntimeError('API response for {} has no commit date'.format(branch_url))
    return api_date(commit_date)


def gitlab_open_issue_count(api_url, token, on_anomaly=None):
    url = '{}/issues_statistics?scope=all'.format(api_url)
    data, _headers, _final_url = fetch_json(
        url,
        gitlab_headers(token),
        on_anomaly=on_anomaly,
    )
    value = data.get('statistics', {}).get('counts', {}).get('opened')
    if isinstance(value, bool) or not isinstance(value, int) or value < 0:
        raise RuntimeError(
            'API response for {} has invalid opened issue count: {!r}'.format(url, value)
        )
    return value


def gitlab_open_merge_request_count(api_url, token, on_anomaly=None):
    params = urlencode({'state': 'opened', 'per_page': 1})
    url = '{}/merge_requests?{}'.format(api_url, params)
    _data, headers, _final_url = fetch_json(
        url,
        gitlab_headers(token),
        on_anomaly=on_anomaly,
    )
    total = headers.get('X-Total')
    if total is None or not str(total).isdigit():
        raise RuntimeError('API response for {} has invalid X-Total header: {!r}'.format(url, total))
    return int(total)


def fetch_github_repo(source, token, item, on_anomaly=None):
    api_data, _headers, final_url = fetch_json(
        source['api_url'],
        github_headers(token),
        on_anomaly=on_anomaly,
    )
    require_api_fields(
        api_data,
        ('stargazers_count', 'created_at', 'open_issues_count', 'html_url'),
        source['api_url'],
    )
    stars = require_nonnegative_int(api_data, 'stargazers_count', source['api_url'])
    open_issues = require_nonnegative_int(api_data, 'open_issues_count', source['api_url'])
    canonical_url = require_url(api_data['html_url'], 'html_url', source['api_url'])
    canonical_api_url = github_repo_api_url(canonical_url)
    if not canonical_api_url:
        raise RuntimeError(
            'GitHub returned an invalid canonical repository URL for {}: {}'.format(
                source['api_url'],
                canonical_url,
            )
        )

    api_url = source['api_url']
    if (
        final_url.rstrip('/') != source['api_url'].rstrip('/')
        or canonical_api_url.rstrip('/') != source['api_url'].rstrip('/')
    ):
        api_url = canonical_api_url
        if replace_source_url(item, source['url'], canonical_url):
            move_notice = 'Repository moved: {} -> {}'.format(source['url'], canonical_url)
            print(move_notice)
            if on_anomaly:
                on_anomaly('Repository moved permanently', move_notice)

    default_branch = api_data.get('default_branch') or 'master'
    if not isinstance(default_branch, str):
        raise RuntimeError('GitHub returned an invalid default branch for {}'.format(api_url))
    last_commit = github_commit_date(api_url, default_branch, token, on_anomaly=on_anomaly)

    return {
        'provider': 'github',
        'stars': stars,
        'created': api_date(api_data.get('created_at')),
        'open_issues': open_issues,
        'license': github_license(api_data, item),
        'last_commit': last_commit,
    }


def fetch_gitlab_repo(source, token, item, on_anomaly=None):
    api_data, _headers, _final_url = fetch_json(
        source['api_url'],
        gitlab_headers(token),
        on_anomaly=on_anomaly,
    )
    require_api_fields(
        api_data,
        ('star_count', 'created_at'),
        source['api_url'],
    )
    stars = require_nonnegative_int(api_data, 'star_count', source['api_url'])
    default_branch = api_data.get('default_branch') or 'master'
    if not isinstance(default_branch, str):
        raise RuntimeError('GitLab returned an invalid default branch for {}'.format(source['api_url']))

    open_issues = gitlab_open_issue_count(source['api_url'], token, on_anomaly=on_anomaly)
    open_issues += gitlab_open_merge_request_count(
        source['api_url'],
        token,
        on_anomaly=on_anomaly,
    )
    last_commit = gitlab_commit_date(
        source['api_url'],
        default_branch,
        token,
        on_anomaly=on_anomaly,
    )

    return {
        'provider': 'gitlab',
        'stars': stars,
        'created': api_date(api_data.get('created_at')),
        'open_issues': open_issues,
        'license': gitlab_license(api_data, item),
        'last_commit': last_commit,
    }


def fetch_repo_stats(source, tokens, item, on_anomaly=None):
    if source['provider'] == 'github':
        return fetch_github_repo(
            source,
            tokens.get('github'),
            item,
            on_anomaly=on_anomaly,
        )
    if source['provider'] == 'gitlab':
        return fetch_gitlab_repo(
            source,
            tokens.get('gitlab'),
            item,
            on_anomaly=on_anomaly,
        )
    raise RuntimeError('Unsupported repo provider: {}'.format(source['provider']))


def primary_repo_stats(stats):
    return stats[0]


def display_star_stats(stats):
    winner = max(stats, key=lambda repo: repo['stars'])
    other_repos = [repo for repo in stats if repo is not winner and repo['stars']]
    other_stars = sum(repo['stars'] for repo in other_repos)
    other_providers = [repo['provider'] for repo in other_repos]
    stars_total = sum(repo['stars'] for repo in stats)
    provider_stars = {}
    for repo in stats:
        provider = repo['provider']
        provider_stars[provider] = provider_stars.get(provider, 0) + repo['stars']
    return winner['stars'], other_stars, winner['provider'], other_providers, stars_total, provider_stars


def old_provider_stars(history, name, provider, comparison_date, current_stars, provider_count):
    provider_field = 'stars_{}'.format(provider)
    old_stars = field_on_or_before(history, name, provider_field, comparison_date)
    if old_stars is not None:
        return int(old_stars)

    old_display_stars = field_on_or_before(history, name, 'stars', comparison_date)
    if old_display_stars is not None and (provider == 'github' or provider_count == 1):
        return int(old_display_stars)

    provider_events = history['projects'].get(name, {}).get(provider_field, [])
    if provider_events:
        return int(provider_events[0][1])

    return current_stars


def stars_diff(history, name, comparison_date, provider_stars):
    diff = 0
    for provider, current_stars in provider_stars.items():
        old_stars = old_provider_stars(
            history,
            name,
            provider,
            comparison_date,
            current_stars,
            len(provider_stars),
        )
        if old_stars is None:
            return '?'
        if old_stars == current_stars:
            continue
        if old_stars > 0:
            diff += current_stars - old_stars
            continue
        if current_stars == 0:
            continue
        return '?'

    return diff


class RepositoryEvents:
    def __init__(self, name):
        self.name = name
        self.messages = []
        self.emitted = set()

    def callback(self, source_url):
        def notify(kind, detail):
            self.notify(kind, detail, source_url)
        return notify

    def notify(self, kind, detail, source_url):
        message = '{}: {}'.format(kind, detail)
        if message not in self.messages:
            self.messages.append(message[:1000])
        if kind in self.emitted:
            return
        self.emitted.add(kind)

        try:
            send_alert(
                'open-source-comments repository warning: {}'.format(self.name),
                'Repository: {}\nSource: {}\nEvent: {}\n\n{}'.format(
                    self.name,
                    source_url,
                    kind,
                    detail,
                ),
            )
        except AlertDeliveryError as error:
            delivery_message = 'Alert delivery failed: {}'.format(error)
            if delivery_message not in self.messages:
                self.messages.append(delivery_message[:1000])
            print('ERROR: {}'.format(delivery_message), file=sys.stderr)

    def tooltip(self):
        return ' | '.join(self.messages)[:3000]


def set_update_warning(item, events, snapshot_date):
    if events.messages:
        item['update_warning'] = events.tooltip()
        item['update_warning_at'] = snapshot_date
    else:
        item.pop('update_warning', None)
        item.pop('update_warning_at', None)


def comparable_date(value):
    return str(value).replace('‑', '-') if value else None


def check_repository_changes(history, name, snapshot_date, primary_stats, provider_stars, notify):
    safe_to_update = True
    old_created = field_on_or_before(history, name, 'created', snapshot_date)
    if (
        old_created not in (None, 'undefined')
        and primary_stats['created'] not in (None, 'undefined')
        and old_created != primary_stats['created']
    ):
        message = 'creation date changed from {} to {}'.format(
            old_created,
            primary_stats['created'],
        )
        notify('Repository identity changed', message)
        safe_to_update = False

    old_license = field_on_or_before(history, name, 'license', snapshot_date)
    if (
        old_license not in (None, 'undefined', 'NOASSERTION')
        and primary_stats['license'] not in (None, 'undefined', 'NOASSERTION')
        and old_license != primary_stats['license']
    ):
        notify(
            'Repository license changed',
            'license changed from {} to {}'.format(old_license, primary_stats['license']),
        )

    old_commit = field_on_or_before(history, name, 'last_commit', snapshot_date)
    if (
        old_commit
        and primary_stats['last_commit']
        and comparable_date(primary_stats['last_commit']) < comparable_date(old_commit)
    ):
        notify(
            'Latest commit date moved backwards',
            'latest commit changed from {} to {}'.format(
                old_commit,
                primary_stats['last_commit'],
            ),
        )

    for provider, current_stars in provider_stars.items():
        provider_field = 'stars_{}'.format(provider)
        old_stars = field_on_or_before(history, name, provider_field, snapshot_date)
        if old_stars is None and provider == 'github':
            old_stars = field_on_or_before(history, name, 'stars', snapshot_date)
        if old_stars is None:
            continue

        old_stars = int(old_stars)
        if old_stars - current_stars >= STAR_DROP_WARNING_THRESHOLD:
            notify(
                '{} star count decreased'.format(provider),
                '{} stars decreased from {} to {}'.format(
                    provider,
                    old_stars,
                    current_stars,
                ),
            )
        elif (
            old_stars > 0
            and current_stars - old_stars >= 1000
            and current_stars >= old_stars * 2
        ):
            notify(
                '{} star count jumped unexpectedly'.format(provider),
                '{} stars jumped from {} to {}'.format(
                    provider,
                    old_stars,
                    current_stars,
                ),
            )
    return safe_to_update


def main():
    snapshot_date = str(date.today())
    comparison_date = str(date.today() - timedelta(days=STARS_DIFF_DAYS))
    tokens = {
        'github': token_from_file(GH_CREDENTIALS_PATH),
        'gitlab': token_from_file(GITLAB_CREDENTIALS_PATH),
    }

    with open('data.yaml', 'r', encoding='utf-8') as f:
        data = yaml.load(f)

    history = load_history()
    snapshot = {}
    repository_issues = []

    print('{:<27}{:<8}{:<6}{:<7}{}'.format('Name', 'Stars', 'Δ★', 'I+PR', 'Created'))
    for name, item in data.items():
        if 'pelican_static' in name:
            continue

        events = RepositoryEvents(name)
        try:
            sources = repo_sources(item)
        except Exception as error:
            message = '{}: invalid repository configuration: {}'.format(name, error)
            repository_issues.append(message)
            events.notify(
                'Invalid repository configuration',
                message,
                item.get('source'),
            )
            set_update_warning(item, events, snapshot_date)
            print('ERROR: {}'.format(message), file=sys.stderr)
            continue
        if not sources:
            set_update_warning(item, events, snapshot_date)
            continue

        stats = []
        for source in sources:
            notify = events.callback(source['url'])
            try:
                repo_stats = fetch_repo_stats(
                    source,
                    tokens,
                    item,
                    on_anomaly=notify,
                )
                stats.append(repo_stats)
            except Exception as error:
                message = '{} ({}): {}'.format(name, source['url'], error)
                repository_issues.append(message)
                notify('Repository update failed', message)
                print('ERROR: {}'.format(message), file=sys.stderr)

        if len(stats) != len(sources):
            set_update_warning(item, events, snapshot_date)
            continue

        try:
            primary_stats = primary_repo_stats(stats)
            (
                stars,
                stars_extra,
                stars_provider,
                stars_extra_providers,
                stars_total,
                provider_stars,
            ) = display_star_stats(stats)
            stars_dif = stars_diff(history, name, comparison_date, provider_stars)
            safe_to_update = check_repository_changes(
                history,
                name,
                snapshot_date,
                primary_stats,
                provider_stars,
                events.callback(item.get('source')),
            )
        except Exception as error:
            events.notify(
                'Could not process repository result',
                str(error),
                item.get('source'),
            )
            safe_to_update = False
        if not safe_to_update:
            message = '{}: repository result was not trusted; previous values retained'.format(name)
            repository_issues.append(message)
            set_update_warning(item, events, snapshot_date)
            print('ERROR: {}'.format(message), file=sys.stderr)
            continue

        item['stars'] = stars
        item['stars_dif'] = stars_dif
        item['open_issues'] = primary_stats['open_issues']
        item['created'] = primary_stats['created']
        item['license'] = primary_stats['license']
        item['stars_provider'] = stars_provider
        if stars_extra:
            item['stars_extra'] = stars_extra
            item['stars_extra_provider'] = ','.join(stars_extra_providers)
        else:
            item.pop('stars_extra', None)
            item.pop('stars_extra_provider', None)
        if primary_stats['last_commit']:
            item['last_committed'] = primary_stats['last_commit']
        set_update_warning(item, events, snapshot_date)

        snapshot[name] = {
            'stars': stars,
            'stars_total': stars_total,
            'created': primary_stats['created'],
            'open_issues': primary_stats['open_issues'],
            'license': primary_stats['license'],
        }
        for provider, provider_stars_value in provider_stars.items():
            snapshot[name]['stars_{}'.format(provider)] = provider_stars_value
        if primary_stats['last_commit']:
            snapshot[name]['last_commit'] = primary_stats['last_commit']

        print('{:<27}{:<8}{:<6}{:<7}{}'.format(name, stars, stars_dif, primary_stats['open_issues'], primary_stats['created']))

    if repository_issues:
        print(
            '\nUpdate completed with {} repository warning(s). '
            'Affected repositories retained their previous trusted values and were marked in the table.'.format(
                len(repository_issues)
            ),
            file=sys.stderr,
        )

    append_snapshot(history, snapshot_date, snapshot)
    yaml_output = StringIO()
    yaml.dump(data, stream=yaml_output)
    atomic_write_many_text(
        {
            'data.yaml': yaml_output.getvalue(),
            'apigh/history.json': serialize_history(history),
        }
    )
    return 0


if __name__ == '__main__':
    raise SystemExit(main())