#!/usr/bin/env python

from multiprocessing import ProcessError
from multiprocessing.dummy import Process
import logging as log
import os
import json
import argparse
import subprocess
import tempfile
from shutil import which
from os import path
from collections import OrderedDict
from pathlib import Path
import sys

# priority is: lowest index is highest priority

# de-centralized upstream kernel mirrors carry a lot of data ISIS does not need
KERNEL_FILTERS = [
        '+ calibration/**', # we generally want everything in calibration
        '- source/',
        '- /a_older_versions/',
        '- /former_versions/',
        '- corrupt_files/',
        '- zzarchive/',
        '- /original/',
        '- ck/prime_mission/',
        '- extended_mission/',
        '- /prime_mission/',
        '- ck/GEM/',
        '- ck/save/',
        '- spk/SAVE_SCS_2017-11-22/',
        '- spk/spk_psp_rec*',
        '- release*/',
        '- Archive/',
        '- ek/',
        '- *.lbl',
        '- *.txt',
        '- *.log',
        '- *.temp',
        '- *.history',
        '- misc/',
        '- document/',
        '- *.csv',
        '- toolkit/',
        '- kernels_ORG/',
        '- orbnum/**',
        '- old/**',
        '- SpecialProducts/**'
    ]

TESTDATA_FILTERS = [
        '- print.prt',
        '- ~?'
    ]

CURATED_FILTERS = [
        '- *.log',
        '- *.temp',
        '- *.history'
    ]

FILTER_MAP = {
        'isistestdata': TESTDATA_FILTERS,
        'base': CURATED_FILTERS,
        'legacybase': CURATED_FILTERS,
        'control': CURATED_FILTERS
    }


def filters_for(mission_name):
    return list(FILTER_MAP.get(mission_name, KERNEL_FILTERS))

def find_conf():
    local_path = Path("rclone.conf")
    # this should be installed in scripts folder, so config is one directory up in etc 
    install_path = Path(os.path.dirname(__file__)) / '..' / "etc" / "isis" / 'rclone.conf'
    repo_path = Path(os.path.dirname(__file__)) / '..' / 'config' / 'rclone.conf'
    if local_path.exists():
        return str(local_path)
    elif repo_path.exists():
        return str(repo_path)
    elif install_path.exists():
        return str(install_path)
    else:
        raise RuntimeError("No config found, try passing it in explicitly with --config <path>")


# set log level to debug
def call_subprocess(command, redirect_stdout=True, redirect_stderr=False):
    stdout = subprocess.PIPE if redirect_stdout else None
    stderr = subprocess.PIPE if redirect_stderr else None

    if not which(command[0]):
        raise ProcessLookupError(f"{command[0]} is not installed.")

    log.debug("Invoking : %s", " ".join(command))

    with subprocess.Popen(
            command,
            stdout=stdout,
            stderr=stderr) as proc:
        return proc.communicate()
    

def rclone(command, config=None, extra_args=[], redirect_stdout=True, redirect_stderr=False): 
    try:
        if path.isfile(config):
            log.debug(f"rclone() : using config file {config}")

            # this is probably a config file on disk so pass it through
            config_path = config

            command_with_args = ["rclone", command, f'--config={config_path}', *extra_args]
            log.debug("Invoking : %s", " ".join(command_with_args))
            return call_subprocess(command_with_args, redirect_stdout, redirect_stderr)
        else:
            log.debug(f"rclone() : using config string {config}")

            # most likely a config string
            with tempfile.NamedTemporaryFile() as f:
                config_path = path.realpath(f.name)

                log.debug(f"USING CONFIG:\n{config}")

                f.write(config.encode())

                command_with_args = ["rclone", command, f"--config={config_path}", *extra_args]
                return call_subprocess(command_with_args, redirect_stdout, redirect_stderr)
    except ProcessLookupError as not_found_e:
        log.error("Executable not found. %s", not_found_e)
        raise ProcessLookupError("rclone is not installed, please install with: 'conda install -c conda-forge rclone'")
    except Exception as generic_e:
        message = f"Error running command. Reason: {generic_e}"
        log.exception(message)
        raise Exception(message)

def create_rclone_arguments(destination, mission_name, parsedArgs, rclone_kwargs=[]):
    """
    Parameters
    ----------

    destination : str
        path to location where files will be copied/downloaded too

    set_of_pub : set(str)
        set of kernels that will be downloaded
    """
    log.debug(f"Creating RClone command for {mission_name}")

    mission_dir_name = mission_name
    
    if (mission_name == "legacybase"):
        # We still want things to go into base
        mission_dir_name = "base"

    destination = os.path.join(destination, str(mission_dir_name))

    filter_list = filters_for(mission_name)

    if args.filter:
        filter_list.extend(f"- {arg}" for arg in args.filter)

    # Check for additional include and exclude flags
    if args.include:
        filter_list.extend(f"+ {arg}" for arg in args.include)

    if args.exclude:
        filter_list.extend(f"- {arg}" for arg in args.exclude)

    if args.no_kernels:
        filter_list.extend(f"- {arg}" for arg in ['ck/**','ek/**','fk/**','ik/**','iak/**','lsk/**','mk/**','pck/**','sclk/**','spk/**','tspk/**','dsk/**'])

    # we need to add this to the end
    if args.include:
        filter_list.append("- *")

    filter_args = [f'--filter={item}' for item in filter_list]
    extra_args = [f"{mission_name}:",
                  f"{destination}", 
                  "--progress",
                  f"--checkers={parsedArgs.num_transfers}",
                  f"--transfers={parsedArgs.num_transfers}",
                  "--track-renames",
                  f"--log-level={log.getLevelName(log.getLogger().getEffectiveLevel())}"]
    extra_args.extend(filter_args)
    extra_args.extend(rclone_kwargs)
    log.debug(f"Args created: {extra_args}")
    return extra_args
    
def main(mission, dest, cfg_path, parsedArgs, kwargs):
    """
    Parameters
    ----------

    source str
            path to location where files are being copied/downloaded from

    dest : str
               path to location where files will be copied/downloaded too
    
    kwargs : list
             list of additional args to pass into rclone
          
    """
    log.info("---Script starting---")
            
    log.debug(f"Using config: {cfg_path}")
    result = rclone("listremotes", config=cfg_path)
    config_sources = result[0].decode("utf-8").split("\n")
    if config_sources == ['']:
        log.error("Remote sources came back empty. Get more info by re-running with verbose flag.")
        quit(-1)
    log.debug(f"Remote Sources: {config_sources}")

    supported_missions = []
    for source in config_sources:
        if not any(x in source for x in ["_", "esa", "naif", "jaxa", "sbn"]):
            if source != "":
                supported_missions.append(source.replace(":", ""))

    log.debug(f"Supported missions:\n {supported_missions}")

    if not mission in supported_missions and mission.upper() not in ("ALL", "LEGACYBASE"):
        raise LookupError(f"{mission} is not in the list of supported missions: {supported_missions}")

    if(mission.upper() == "ALL"):
        supported_missions.remove("legacybase")
        # test data is opt-in, it is ~19 GiB
        supported_missions.remove("isistestdata")
        for mission in supported_missions:
            rclone_args = create_rclone_arguments(dest, mission, parsedArgs, kwargs)
            rclone(command=parsedArgs.command, extra_args=rclone_args, config=cfg_path, redirect_stdout=False, redirect_stderr=False)
    else:
        rclone_args = create_rclone_arguments(dest, mission,  parsedArgs, kwargs)
        rclone(command=parsedArgs.command, extra_args=rclone_args, config=cfg_path, redirect_stdout=False, redirect_stderr=False)

if __name__ == '__main__':
    usageString = "usage: downloadIsisData mission dest [-h] [-v] [-n NUM_TRANSFERS] [--config CONFIG] [rclone flags]"
    helpString = (
    '''
    Use to copy ISIS data from USGS sources as well as public end points. 

    This script combines multiple sources using the rclone command under the hood. 
    
    After positional args, accepts arbitrary rclone flags. Run "rclone help flags" to get a full list of rclone flags. 
    
    See the ISIS repo (https://github.com/USGS-Astrogeology/ISIS3) for more info. 
    
    Examples of how to run this program:
        
        # download everything for tgo
        $ downloadIsisData tgo $ISISDATA
    
        # download everything except kernels with debug output
        $ downloadIsisData all $ISISDATA --dry-run --no-kernels -vv
        
        # download only cks and fks
        $ downloadIsisData all $ISISDATA --include="{ck/**,fk/**}"

        # download app test data, not included in "all" as it is ~19 GiB
        $ downloadIsisData isistestdata $ISISTESTDATA
    ''')
    parser = argparse.ArgumentParser(description = helpString,formatter_class=argparse.RawDescriptionHelpFormatter, usage=usageString)
    parser.add_argument('mission', help='mission for files to be downloaded. Use "all" to download everything, "legacybase" to download base data for ISIS<4.1.0.')
    parser.add_argument('dest', help='the destination to download files from source')
    parser.add_argument('-v', '--verbose', action='count', default=0)
    parser.add_argument('-n', '--num-transfers', action='store', default=10)
    parser.add_argument('--config', action='store', default=None)
    parser.add_argument('--filter', help='Additional filters for files', nargs='*')
    parser.add_argument('--include', help='files and patterns to include while downloading', nargs='*')
    parser.add_argument('--exclude', help='files and patterns to ignore while downloading', nargs='*')
    parser.add_argument('--no-kernels', help='Exclude all kernel files from download.  Analogous to --exclude="{ck/**,ek/**,fk/**,ik/**,iak/**,lsk/**,mk/**,pck/**,sclk/**,spk/**,tspk/**,dsk/**}"', default=False, action='store_true')
    parser.add_argument('-c', '--command', choices=["copy", "sync", "ls", "lsd", "size"], help='files and patterns to ignore while downloading', default="copy")

    args, kwargs = parser.parse_known_args()

    if args.config is None:
        args.config = find_conf()
    
    log_kwargs = {
      'format': '%(asctime)s %(levelname)-8s %(message)s',
      'level': log.WARN,
      'datefmt' : '%Y-%m-%d %H:%M:%S'
    }

    if args.verbose == 0:
        log_kwargs['level'] = log.WARN
    elif args.verbose == 1:
        log_kwargs['level'] = log.INFO
    elif args.verbose >= 2:
        log_kwargs['level'] = log.DEBUG

    log.basicConfig(**log_kwargs)
    log.debug("args: ", args)
    log.debug("Additional Args:", *kwargs)

    main(args.mission, args.dest, os.path.expanduser(args.config), args, kwargs)