import crypto from 'crypto';
import type express from 'express';

import {
    DEFAULT_ACCOUNT_ID,
    DEMO_MODE,
    DUPLICATE_LAX_MODE,
    DUPLICATE_THRESHOLD,
} from '../../shared/settings';
import {
    assert,
    asyncErr,
    getErrorCode,
    isAppriseApiEnabled,
    isEmailEnabled,
    KError,
    makeLogger,
    UNKNOWN_TRANSACTION_TYPE,
    unwrap,
} from '../helpers';
import diffAccount from '../lib/diff-accounts';
import diffTransactions from '../lib/diff-transactions';
import { ConfigGhostSettings, getAll as getAllInstanceProperties } from '../lib/instance';
import {
    Access,
    Account,
    Alert,
    AppliedRecurringTransaction,
    Budget,
    Category,
    DuplicatesIgnored,
    type MinimalTransaction,
    RecurringTransaction,
    Setting,
    Transaction,
    TransactionRule,
    User,
    View,
} from '../models';
import runDataMigrations from '../models/data-migrations';
import { bankVendorByUuid, getBankVendors } from '../providers';
import DefaultSettings from '../shared/default-settings';
import { validatePassword } from '../shared/helpers';
import type { DuplicatesPairs } from '../shared/types';
import { type AllData, type ClientAccess, cleanData, type Remapping } from './helpers';
import { isDemoEnabled } from './instance';
import { ofxToKresus } from './ofx';
import type { IdentifiedRequest } from './routes';

const log = makeLogger('controllers/all');

const ERR_MSG_LOADING_ALL = 'Error when loading all Kresus data';

interface GetAllDataOptions {
    isExport?: boolean;
    cleanPassword?: boolean;
}

type StartupTask = () => Promise<void>;
type UserId = number;

// Startup tasks are cleanup tasks that will be run the next time a user will run the /all initial
// request.
const STARTUP_TASKS: Record<UserId, StartupTask[]> = {};

// Registers a new startup task for the given user.
export function registerStartupTask(userId: UserId, f: StartupTask) {
    STARTUP_TASKS[userId] = STARTUP_TASKS[userId] || [];
    STARTUP_TASKS[userId].push(f);
}

// Run startup tasks for the user, if any.
async function runStartupTasks(userId: number) {
    if (STARTUP_TASKS[userId]) {
        while (STARTUP_TASKS[userId].length) {
            const task = unwrap(STARTUP_TASKS[userId].pop());
            await task();
        }
    }
}

async function getAllData(userId: number, options: GetAllDataOptions = {}): Promise<AllData> {
    const { isExport = false, cleanPassword = true } = options;

    let ret: AllData = {
        accounts: [],
        accesses: [],
        alerts: [],
        categories: [],
        transactions: [],
        settings: [],
        instance: {},
        recurringTransactions: [],
        views: [],
        bankVendors: [],
    };

    const accesses = await Access.all(userId);
    for (const access of accesses) {
        const clientAccess: ClientAccess = { ...access };

        // Just keep the name and the value of the field.
        clientAccess.fields = (access.fields || []).map(({ name, value }) => {
            return { name, value };
        });

        if (cleanPassword) {
            clientAccess.fields = clientAccess.fields.filter(f => f.name !== 'password');
            delete clientAccess.session;
        }

        if (!isExport) {
            // Process enabled status only for the /all request.
            clientAccess.enabled = access.isEnabled();
            delete clientAccess.session;
        }

        const bank = bankVendorByUuid(clientAccess.vendorId);
        if (bank && bank.name) {
            clientAccess.label = bank.name;
        }

        ret.accesses.push(clientAccess);
    }

    // On export we don't want the balance to be returned for manual accounts, otherwise
    // on import the balance won't be automatically re-computed on transactions
    // creation/update/deletion.
    ret.accounts = await Account.all(userId, !isExport);

    ret.categories = await Category.all(userId);
    ret.transactions = await Transaction.all(userId);
    ret.settings = await Setting.all(userId);
    ret.recurringTransactions = await RecurringTransaction.all(userId);
    ret.views = await View.all(userId);

    if (isExport) {
        ret.budgets = await Budget.all(userId);

        // This fetches the associated conditions and actions data, so no need
        // to join explicitly here.
        ret.transactionRules = await TransactionRule.allOrdered(userId);

        // We only need to export the applied recurring transactions from the current month since
        // the recurring transactions won't be created for the past at next poll.
        const now = new Date();
        ret.appliedRecurringTransactions = await AppliedRecurringTransaction.byMonthAndYear(
            userId,
            now.getMonth(),
            now.getFullYear()
        );

        const ignoredDuplicatesPairs = (await DuplicatesIgnored.all(userId)).map(pair => [
            pair.transactionId,
            pair.otherTransactionId,
        ]) satisfies DuplicatesPairs;

        ret.duplicates = {
            ignored: ignoredDuplicatesPairs,
        };
    } else {
        ret.bankVendors = getBankVendors();

        ret.instance = await getAllInstanceProperties();

        // Note: the duplicates (detected and ignored ones) are lazy-loaded through the
        // /duplicates endpoint and therefore not part of this payload.

        const user = await User.find(userId);
        if (user) {
            ret.user = user;
        }
    }

    if (isExport || isEmailEnabled() || isAppriseApiEnabled()) {
        ret.alerts = await Alert.all(userId);
    } else {
        ret.alerts = [];
    }

    if (isExport) {
        ret = cleanData(ret);
    }

    return ret;
}

export async function all(req: IdentifiedRequest<any>, res: express.Response) {
    try {
        const { id: userId } = req.user;
        await runStartupTasks(userId);
        const ret = await getAllData(userId);
        res.status(200).json(ret);
    } catch (err) {
        err.code = ERR_MSG_LOADING_ALL;
        asyncErr(res, err, 'when loading all data');
    }
}

const ENCRYPTION_ALGORITHM = 'aes-256-ctr';
const ENCRYPTED_CONTENT_TAG = Buffer.from('KRE');

function encryptData(data: Record<string, unknown>, passphrase: string) {
    assert(process.kresus.salt !== null, 'must have provided a salt');

    const initVector = crypto.randomBytes(16);
    const key = crypto.pbkdf2Sync(passphrase, process.kresus.salt, 100000, 32, 'sha512');
    const cipher = crypto.createCipheriv(ENCRYPTION_ALGORITHM, key, initVector);

    return Buffer.concat([
        initVector,
        ENCRYPTED_CONTENT_TAG,
        cipher.update(JSON.stringify(data)),
        cipher.final(),
    ]).toString('base64');
}

function decryptData(data: string, passphrase: string) {
    assert(process.kresus.salt !== null, 'must have provided a salt');

    const rawData = Buffer.from(data, 'base64');
    const [initVector, tag, encrypted] = [
        rawData.slice(0, 16),
        rawData.slice(16, 16 + 3),
        rawData.slice(16 + 3),
    ];

    if (tag.toString() !== ENCRYPTED_CONTENT_TAG.toString()) {
        throw new KError(
            'submitted file is not a valid kresus encrypted file',
            400,
            getErrorCode('INVALID_ENCRYPTED_EXPORT')
        );
    }

    const key = crypto.pbkdf2Sync(passphrase, process.kresus.salt, 100000, 32, 'sha512');

    const decipher = crypto.createDecipheriv(ENCRYPTION_ALGORITHM, key, initVector);
    return Buffer.concat([decipher.update(encrypted), decipher.final()]).toString();
}

export async function export_(req: IdentifiedRequest<any>, res: express.Response) {
    try {
        const { id: userId } = req.user;

        let passphrase: string | null = null;
        if (req.body.encrypted) {
            if (typeof req.body.passphrase !== 'string' || !req.body.passphrase) {
                throw new KError('missing parameter "passphrase"', 400);
            }

            if (process.kresus.salt === null) {
                throw new KError(
                    "server hasn't been configured for encryption; " +
                        'please ask your administrator to provide a salt'
                );
            }

            passphrase = req.body.passphrase;
            assert(passphrase !== null, 'passphrase must be set here');

            // Check password strength
            if (!validatePassword(passphrase)) {
                throw new KError('submitted passphrase is too weak', 400);
            }
        }

        const data = await getAllData(userId, {
            isExport: true,
            cleanPassword: !passphrase,
        });

        let ret = {};
        if (passphrase) {
            const encryptedData = encryptData(data, passphrase);
            ret = {
                encrypted: true,
                data: encryptedData,
            };
        } else {
            ret = {
                encrypted: false,
                data,
            };
        }

        res.status(200).json(ret);
    } catch (err) {
        err.code = ERR_MSG_LOADING_ALL;
        asyncErr(res, err, 'when exporting data');
    }
}

type AnyObject = { [key: string]: AnyObject } | any;

function applyRenamings(model: any): (arg: AnyObject) => AnyObject {
    if (typeof model.renamings === 'undefined') {
        return obj => obj;
    }
    return (obj: AnyObject): AnyObject => {
        for (const from of Object.keys(model.renamings)) {
            const to = model.renamings[from];
            if (typeof obj[from] !== 'undefined') {
                if (typeof obj[to] === 'undefined') {
                    obj[to] = obj[from];
                }
                delete obj[from];
            }
        }
        return obj;
    };
}

export function parseDate(date: any) {
    switch (typeof date) {
        case 'string': {
            const parsedDate = Date.parse(date);
            if (!isNaN(parsedDate)) {
                return new Date(parsedDate);
            }
            break;
        }

        case 'number':
            if (!isNaN(date) && date > -8640000000000000 && date < 8640000000000000) {
                return new Date(date);
            }
            break;

        default:
            if (date instanceof Date) {
                return date;
            }
    }

    return null;
}

type AccessRemapping = {
    [key: number]: {
        // Remapped ID of the Kresus access in our database.
        id: number;
        // Was the access known, i.e. it exists in Kresus' database?
        wasKnown: boolean;
        // Optional vendor id of the access.
        vendorId?: string;
    };
};

// Import the given data from the `world` object, created by interpreting the JSON created from a
// call to `export_()` in this same file.
//
// If `dontCreateAccess` is true, it is assumed that the access id refers to a valid access that is
// known in the database. Otherwise, then the access is created.
//
// Note: destroy the input `world` argument by changing its fields in place.
export async function importData(userId: number, world: any, dontCreateAccess?: boolean) {
    world.accesses = (world.accesses || []).map(applyRenamings(Access));
    world.accounts = (world.accounts || []).map(applyRenamings(Account));
    world.alerts = (world.alerts || []).map(applyRenamings(Alert));
    world.budgets = (world.budgets || []).map(applyRenamings(Budget));
    world.categories = (world.categories || []).map(applyRenamings(Category));
    // Keep backward compat with 'operations'.
    world.transactions = (world.transactions || world.operations || []).map(
        applyRenamings(Transaction)
    );
    world.settings = (world.settings || []).map(applyRenamings(Setting));
    world.transactionRules = world.transactionRules || [];
    world.recurringTransactions = world.recurringTransactions || [];
    world.appliedRecurringTransactions = world.appliedRecurringTransactions || [];
    world.views = world.views || [];

    // Static data.
    // Keep backward compat with 'operationtypes'.
    world.transactiontypes = world.transactiontypes || world.operationtypes || [];

    // Importing only known settings prevents assertion errors in the client when
    // importing Kresus data in an older version of kresus.
    world.settings = world.settings.filter((s: any) => DefaultSettings.has(s.key)) || [];

    // Exports made before the ignored duplicates were exported don't have a `duplicates` object.
    world.ignoredDuplicates = world.duplicates?.ignored || [];

    log.info(`Importing:
        accesses:          ${world.accesses.length}
        accounts:          ${world.accounts.length}
        alerts:            ${world.alerts.length}
        budgets:           ${world.budgets.length}
        categories:        ${world.categories.length}
        transaction-types: ${world.transactiontypes.length}
        settings:          ${world.settings.length}
        transactions:      ${world.transactions.length}
        rules:             ${world.transactionRules.length}
        recurring-transactions:           ${world.recurringTransactions.length}
        applied-recurring-transactions:           ${world.appliedRecurringTransactions.length}
        views:           ${world.views.length},
        ignoredDuplicates ${world.ignoredDuplicates.length}
    `);

    log.info('Import accesses...');
    const accessMap: AccessRemapping = {};
    for (const access of world.accesses) {
        const accessId = access.id;
        delete access.id;
        delete access.userId;

        // Support legacy "customFields" value.
        if (typeof access.customFields === 'string' && !access.fields) {
            try {
                access.fields = JSON.parse(access.customFields);
            } catch (e) {
                log.error('Invalid JSON customFields, ignoring fields:', e.toString());
            }
        }

        if (!(access.fields instanceof Array)) {
            access.fields = [];
        }

        // Support legacy 'login' and 'password' values.
        if (access.login) {
            if (
                !access.fields.some(
                    (f: unknown) => typeof f === 'object' && f && 'name' in f && f.name === 'login'
                )
            ) {
                access.fields.push({ name: 'login', value: access.login });
            }

            delete access.login;
        }

        if (access.password) {
            if (
                !access.fields.some(
                    (f: unknown) =>
                        typeof f === 'object' && f && 'name' in f && f.name === 'password'
                )
            ) {
                access.fields.push({ name: 'password', value: access.password });
            }

            delete access.password;
        }

        const sanitizedCustomFields: { name: string; value: string }[] = [];

        for (const { name, value } of access.fields) {
            if (typeof name !== 'string') {
                log.warn('Ignoring access field because of non-string "name" property.');
                continue;
            }
            if (typeof value !== 'string') {
                log.warn(
                    `Ignoring access field for key ${name} because of non-string "value" property`
                );
                continue;
            }
            sanitizedCustomFields.push({ name, value });
        }

        access.fields = sanitizedCustomFields;

        const accessLogin = sanitizedCustomFields.find(f => f.name === 'login');

        // Try to match the vendor id and login against an existing pair in the database.
        const foundKnown: Access | null = accessLogin
            ? await Access.byCredentials(userId, {
                  uuid: access.vendorId,
                  login: accessLogin.value,
              })
            : null;

        if (foundKnown !== null) {
            // The access was already known: don't reimport it.
            accessMap[accessId] = {
                id: foundKnown.id,
                wasKnown: true,
                vendorId: foundKnown.vendorId,
            };
            log.info(
                `Ignoring import of access ${access.vendorId} (${
                    accessLogin ? accessLogin.value : 'no login'
                }) as it already exists.`
            );
            continue;
        }

        if (dontCreateAccess) {
            // The access id given from the `world` object is valid, according to this function's
            // contract, so the mapping doesn't need to redirect to another access we created.
            accessMap[accessId] = { id: accessId, wasKnown: true };
            log.info(
                `Not creating new known access ${access.vendorId} (${
                    accessLogin ? accessLogin.value : 'no login'
                }) because it's explicitly marked as known.`
            );
        } else {
            const created = await Access.create(userId, access);
            accessMap[accessId] = { id: created.id, wasKnown: false };
            log.info(
                `Creating new unknown access ${access.vendorId} (${
                    accessLogin ? accessLogin.value : 'no login'
                }).`
            );
        }
    }
    log.info('Done.');

    log.info('Import accounts...');
    const accountIdToAccount: Map<number, number> = new Map();
    const vendorToOwnAccountId: Map<string, number> = new Map();
    for (const account of world.accounts) {
        if (typeof accessMap[account.accessId] === 'undefined') {
            log.warn('Ignoring orphan account:\n', account);
            continue;
        }

        const accountId = account.id;

        // Create a copy of the account. We should use structuredClone but for some reasons our
        // tests are passing models instead of literal objects.
        const accountCopy = JSON.parse(JSON.stringify(account));
        delete accountCopy.id;
        delete accountCopy.userId;

        // For an initial import which does not come from Kresus (ex: a
        // handmade JSON file), there might be no lastCheckDate.
        accountCopy.lastCheckDate = parseDate(accountCopy.lastCheckDate);
        if (accountCopy.lastCheckDate === null) {
            let latestTransactionDate: Date | null = null;
            if (world.transactions) {
                const accountTransactions = world.transactions.filter(
                    (tr: any) => tr.accountId === accountId
                );
                for (const tr of accountTransactions) {
                    const trDate = parseDate(tr.date);
                    if (
                        trDate !== null &&
                        (latestTransactionDate === null || trDate > latestTransactionDate)
                    ) {
                        latestTransactionDate = trDate;
                    }
                }
            }
            accountCopy.lastCheckDate = latestTransactionDate || new Date();
        }

        // If the access containing this account was known, then try to match this account against
        // one known in the database.
        const accessRemap = accessMap[accountCopy.accessId];
        accountCopy.accessId = accessRemap?.id;

        if (accessRemap?.wasKnown) {
            const knownAccounts = await Account.byAccess(userId, {
                id: accessRemap.id,
            });
            if (typeof knownAccounts !== 'undefined') {
                const vendorId = accessRemap.vendorId ?? undefined;

                const diffResult = diffAccount(knownAccounts, [accountCopy], {
                    vendorId,
                });

                // There can be at most one perfect match, since we provided only one account.
                if (diffResult.perfectMatches.length === 1) {
                    const firstMatchPair = diffResult.perfectMatches[0];
                    const knownAccount = firstMatchPair[0];

                    // The account was found as a perfect match: do not import it, and reuse it
                    // instead.
                    accountIdToAccount.set(accountId, knownAccount.id);
                    vendorToOwnAccountId.set(accountCopy.vendorAccountId, knownAccount.id);

                    log.info(
                        `Ignoring import of account ${accountCopy.label} (${accountCopy.vendorAccountId}) as it already exists.`
                    );

                    continue;
                }
            }
        }

        log.info(
            `Importing new unknown account ${accountCopy.label} (${accountCopy.vendorAccountId}).`
        );
        const created = await Account.create(userId, accountCopy);
        accountIdToAccount.set(accountId, created.id);
        vendorToOwnAccountId.set(created.vendorAccountId, created.id);
    }
    log.info('Done.');

    log.info('Import views...');
    const viewsMap: Remapping = {};
    const existingViews = await View.all(userId);
    for (const view of world.views) {
        const viewId = view.id;
        delete view.id;

        if (!view.createdByUser) {
            // Views created by the user are views created automatically by Kresus to group accounts.
            // Since new automatic views were created automatically when importing accounts, we
            // should not recreate them here.
            // However, we still need to map the view ids (for budgets at least).

            // Try to retrieve the corresponding view created automatically.
            const viewAccountsRemapped = view.accounts.map((vAcc: any) =>
                accountIdToAccount.get(vAcc.accountId)
            );

            const correspondingView = existingViews.find(ev => {
                if (ev.accounts.length === view.accounts.length) {
                    const evAccounts = ev.accounts.map(a => a.accountId);

                    for (const accId of viewAccountsRemapped) {
                        if (!evAccounts.includes(accId)) {
                            return false;
                        }
                    }

                    return true;
                }

                return false;
            });

            if (correspondingView) {
                viewsMap[viewId] = correspondingView.id;

                log.warn(
                    'Not re-creating view not created by user as there is already one corresponding view:\n',
                    view
                );

                continue;
            }
        }

        const viewAccounts = [];
        for (const viewAcc of view.accounts) {
            if (
                typeof viewAcc.accountId !== 'undefined' &&
                accountIdToAccount.has(viewAcc.accountId)
            ) {
                viewAcc.accountId = accountIdToAccount.get(viewAcc.accountId);
                viewAccounts.push(viewAcc);
            } else {
                log.warn('Ignoring account in view:\n', view);
            }
        }

        if (viewAccounts.length === 0) {
            log.warn('Ignoring view without accounts:\n', view);
            continue;
        }

        const created = await View.create(userId, Object.assign(view, { accounts: viewAccounts }));
        viewsMap[viewId] = created.id;
    }
    log.info('Done.');

    log.info('Import categories...');
    const existingCategories = await Category.all(userId);
    const existingCategoriesMap = new Map();
    for (const category of existingCategories) {
        existingCategoriesMap.set(category.label, category);
    }

    const categoryMap: Remapping = {};
    for (const category of world.categories) {
        const catId = category.id;
        delete category.id;
        delete category.userId;
        if (existingCategoriesMap.has(category.label)) {
            const existing = existingCategoriesMap.get(category.label);
            categoryMap[catId] = existing.id;
        } else {
            const created = await Category.create(userId, category);
            categoryMap[catId] = created.id;
        }
    }
    log.info('Done.');

    log.info('Import budgets...');
    const makeBudgetKey = (b: Budget) => `${b.viewId}-${b.categoryId}-${b.year}-${b.month}`;

    const existingBudgets = await Budget.all(userId);
    const existingBudgetsMap = new Map();
    for (const budget of existingBudgets) {
        existingBudgetsMap.set(makeBudgetKey(budget), budget);
    }

    let defaultViewId = -1;
    let lookForDefaultView = true;

    for (const importedBudget of world.budgets) {
        if (typeof importedBudget.viewId !== 'number') {
            if (defaultViewId === -1 && lookForDefaultView) {
                // For budgets pre-existing views, there is no viewId set. We need to retrieve
                // it from the defaultAccountId (the one imported) or the first account imported.
                const defaultAccountImportedSetting = world.settings.find(
                    (setting: any) => setting && setting.key === DEFAULT_ACCOUNT_ID
                );
                let defaultImportedAccountId: number | undefined;
                if (defaultAccountImportedSetting) {
                    defaultImportedAccountId = accountIdToAccount.get(
                        defaultAccountImportedSetting.value
                    );
                } else {
                    // Find the first checking account otherwise the first account.
                    const bestGuessAccountFromImport =
                        world.accounts.find((acc: any) => acc.type === 'account-type.checking') ||
                        world.accounts[0];
                    defaultImportedAccountId = accountIdToAccount.get(
                        bestGuessAccountFromImport.id
                    );
                }

                if (typeof defaultImportedAccountId === 'number') {
                    const allViews = await View.all(userId);
                    const defaultAccountAssociatedView = allViews.find(view => {
                        return (
                            view.accounts.length === 1 &&
                            view.accounts[0].accountId === defaultImportedAccountId
                        );
                    });

                    if (defaultAccountAssociatedView) {
                        defaultViewId = defaultAccountAssociatedView.id;
                    } else {
                        lookForDefaultView = false;
                    }
                } else {
                    lookForDefaultView = false;
                }
            }

            if (defaultViewId === -1) {
                // We don't know what to do with this budget without any account/view
                // to associate it with.
                log.warn('No view to associate a budget with, skipping budget import…');
                continue;
            }

            // Set the default view id as the budget's viewId.
            importedBudget.viewId = defaultViewId;
        } else {
            if (!viewsMap[importedBudget.viewId]) {
                log.warn('No view to associate a budget with, skipping budget import…');
                continue;
            }

            importedBudget.viewId = viewsMap[importedBudget.viewId];
        }

        // Note the order here: first map to the actual category id, so the
        // map lookup thereafter uses an existing category id.
        importedBudget.categoryId = categoryMap[importedBudget.categoryId];

        const existingBudget = existingBudgetsMap.get(makeBudgetKey(importedBudget));
        if (existingBudget) {
            if (
                !existingBudget.threshold ||
                existingBudget.threshold !== importedBudget.threshold
            ) {
                await Budget.update(userId, existingBudget.id, {
                    threshold: importedBudget.threshold,
                });
            }
        } else {
            delete importedBudget.id;
            delete importedBudget.userId;

            const newBudget = await Budget.create(userId, importedBudget);

            // There could be duplicates in the import (see #1051), ensure we don't try
            // to import the same budget twice.
            existingBudgetsMap.set(makeBudgetKey(newBudget), newBudget);
        }
    }
    log.info('Done.');

    // No need to import transaction types.

    // importedTypesMap is used to set type to imported transactions (backward compatibility).
    const importedTypes = world.transactiontypes || [];
    const importedTypesMap = new Map();
    for (const type of importedTypes) {
        importedTypesMap.set(type.id.toString(), type.name);
    }

    log.info('Import transactions...');

    let perfectMatchMaxDateThreshold = 0;

    // If the user chose to loosen the duplicate detection algo, use the duplicate threshold for the perfect match algo instead of default 0 value
    const enableDuplicateLaxModeSetting = world.settings.find(
        (s: Setting) => s.key === DUPLICATE_LAX_MODE
    );
    const enableDuplicateLaxMode =
        (enableDuplicateLaxModeSetting
            ? enableDuplicateLaxModeSetting.value
            : unwrap(DefaultSettings.get(DUPLICATE_LAX_MODE))) === 'true';

    if (enableDuplicateLaxMode) {
        const duplicateThresholdSetting = world.settings.find(
            (s: Setting) => s.key === DUPLICATE_THRESHOLD
        );
        // The duplicate threshold is stored in hours, convert it to days.
        const duplicateThreshold = duplicateThresholdSetting
            ? duplicateThresholdSetting.value
            : unwrap(DefaultSettings.get(DUPLICATE_THRESHOLD));
        perfectMatchMaxDateThreshold = Math.round(Number.parseInt(duplicateThreshold, 10) / 24);
    }
    const newByAccountId: Map<number, MinimalTransaction[]> = new Map();

    // Maps the transaction ids as they were in the imported data to the ids they were given when
    // inserted in database. Since the transactions are inserted per account, and only some of them
    // are actually inserted, the previous id is kept on the transaction object itself rather than
    // in a list, so it can't be mismatched.
    const transactionIdsMap: Remapping = {};
    const previousTransactionIds: Map<MinimalTransaction, number> = new Map();

    for (let i = 0; i < world.transactions.length; i++) {
        const tr = world.transactions[i];

        tr.date = parseDate(tr.date);
        tr.debitDate = parseDate(tr.debitDate);
        tr.importDate = parseDate(tr.importDate);

        if (tr.date === null) {
            log.warn('Ignoring transaction without date\n', tr);
            continue;
        }

        if (typeof tr.amount !== 'number' || isNaN(tr.amount)) {
            log.warn('Ignoring transaction without valid amount\n', tr);
            continue;
        }

        // Map transaction to account.
        if (typeof tr.accountId !== 'undefined') {
            if (!accountIdToAccount.has(tr.accountId)) {
                log.warn('Ignoring orphan transaction:\n', tr);
                continue;
            }
            tr.accountId = accountIdToAccount.get(tr.accountId);
        } else {
            if (!vendorToOwnAccountId.has(tr.bankAccount)) {
                log.warn('Ignoring orphan transaction:\n', tr);
                continue;
            }
            tr.accountId = vendorToOwnAccountId.get(tr.bankAccount);
        }

        // Remove bankAccount as the transaction is now linked to account with accountId prop.
        delete tr.bankAccount;

        const categoryId = tr.categoryId;
        if (typeof categoryId !== 'undefined' && categoryId !== null) {
            if (typeof categoryMap[categoryId] === 'undefined') {
                log.warn('Unknown category, unsetting for transaction:\n', tr);
            }
            tr.categoryId = categoryMap[categoryId];
        }

        // Set transaction type base on transactionId.
        // (Maintain 'operation' in name, as it's a deprecated field.)
        if (typeof tr.operationTypeID !== 'undefined') {
            const key = tr.operationTypeID.toString();
            if (importedTypesMap.has(key)) {
                tr.type = importedTypesMap.get(key);
            } else {
                tr.type = UNKNOWN_TRANSACTION_TYPE;
            }
            delete tr.operationTypeID;
        }

        // If there is no import date, set it to now.
        if (tr.importDate === null) {
            tr.importDate = new Date();
        }

        // If there is no label use the rawLabel, and vice-versa.
        if (typeof tr.label === 'undefined') {
            tr.label = tr.rawLabel;
        }
        if (typeof tr.rawLabel === 'undefined') {
            tr.rawLabel = tr.label;
        }
        if (typeof tr.label === 'undefined' && typeof tr.rawLabel === 'undefined') {
            log.warn('Ignoring transaction without label/rawLabel:\n', tr);
            continue;
        }

        // Consider that old imports have the type set by the user, to have a consistent behaviour
        // with the migration.
        if (typeof tr.isUserDefinedType === 'undefined') {
            tr.isUserDefinedType = true;
        }

        // Now that we are sure the transaction is to be imported (we are past all `continue`),
        // store the id.
        if (typeof tr.id === 'number') {
            previousTransactionIds.set(tr, tr.id);
        }

        // Remove contents of deprecated fields, if there were any.
        delete tr.attachments;
        delete tr.binary;

        // Remove ids that differ depending on the instance (due to auto-increment mainly).
        delete tr.id;
        delete tr.userId;

        // Add the transaction to its account group.
        const accountGroup = newByAccountId.get(tr.accountId) || [];
        accountGroup.push(tr);
        newByAccountId.set(tr.accountId, accountGroup);
    }

    for (const [accountId, provided] of newByAccountId) {
        const known = await Transaction.byAccount(userId, accountId);

        const diffResult = diffTransactions(known, provided, { perfectMatchMaxDateThreshold });

        // Ignore perfect matches and knownOrphans. Only import the "provider" (import) orphans
        // and the duplicate candidates, since we're not too sure about those.
        const transactions = diffResult.providerOrphans;
        const candidates = diffResult.duplicateCandidates.map(pair => pair[1]);
        transactions.push(...candidates);

        if (transactions.length > 0) {
            log.info(`Importing ${transactions.length} transactions for account ${accountId}.`);
            const insertedIds = await Transaction.bulkCreate(userId, transactions);

            // There should be the same number of inserted transactions as transactions to create…
            // Otherwise we can't map new ids to old ids.
            if (transactions.length === insertedIds.length) {
                insertedIds.forEach((trId, index) => {
                    const previousId = previousTransactionIds.get(transactions[index]);
                    if (typeof previousId === 'number') {
                        transactionIdsMap[previousId] = trId;
                    }
                });
            } else {
                log.warn(
                    'Could not map the imported transaction ids to the inserted ones, the duplicates pairs to ignore will be lost.'
                );
            }
        } else {
            log.info(`No transactions to import for account ${accountId}.`);
        }
    }

    log.info('Done.');

    log.info('Import duplicates pairs to ignore...');
    for (const pair of world.ignoredDuplicates) {
        const newId = transactionIdsMap[pair[0]];
        const newOtherId = transactionIdsMap[pair[1]];

        if (typeof newId === 'number' && typeof newOtherId === 'number') {
            await DuplicatesIgnored.create(userId, newId, newOtherId);
        } else {
            log.warn(
                `Could not import pair of duplicates to ignore because the ids do not match the ids mapping: ${pair[0]}/${pair[1]}`
            );
        }
    }

    log.info('Done.');

    log.info('Import settings...');
    for (const setting of world.settings) {
        if (ConfigGhostSettings.has(setting.key)) {
            continue;
        }

        // Reset the default account id, if it's set.
        if (
            setting.key === DEFAULT_ACCOUNT_ID &&
            setting.value !== DefaultSettings.get(DEFAULT_ACCOUNT_ID)
        ) {
            const settingAsNumber = Number.parseInt(setting.value, 10);
            if (!accountIdToAccount.has(settingAsNumber)) {
                log.warn(`unknown default account id: ${setting.value}, skipping.`);
                continue;
            }
            setting.value = accountIdToAccount.get(settingAsNumber);

            await Setting.updateByKey(userId, DEFAULT_ACCOUNT_ID, setting.value);
            continue;
        }

        // Overwrite the previous value of the demo-mode, if it was set.
        if (setting.key === DEMO_MODE && setting.value === 'true') {
            const found = await Setting.byKey(userId, DEMO_MODE);
            if (found && found.value !== 'true') {
                await Setting.updateByKey(userId, DEMO_MODE, 'true');
                continue;
            }
        }

        delete setting.userId;

        // Note that former existing values are not overwritten!
        await Setting.findOrCreateByKey(userId, setting.key, setting.value);
    }

    log.info('Done.');

    log.info('Import alerts...');
    const existingAlerts = await Alert.all(userId);

    for (const a of world.alerts) {
        // Map alert to account.
        if (typeof a.accountId !== 'undefined') {
            if (!accountIdToAccount.has(a.accountId)) {
                log.warn('Ignoring orphan alert:\n', a);
                continue;
            }
            a.accountId = accountIdToAccount.get(a.accountId);
        } else {
            if (!vendorToOwnAccountId.has(a.bankAccount)) {
                log.warn('Ignoring orphan alert:\n', a);
                continue;
            }
            a.accountId = vendorToOwnAccountId.get(a.bankAccount);
        }

        // Don't reimport an existing similar alert.
        let foundDuplicate = false;
        for (const known of existingAlerts) {
            if (
                known.accountId === a.accountId &&
                known.type === a.type &&
                known.frequency === a.frequency &&
                known.limit === a.limit &&
                known.order === a.order
            ) {
                foundDuplicate = true;
                break;
            }
        }

        if (foundDuplicate) {
            continue;
        }

        // Remove bankAccount as the alert is now linked to account with accountId prop.
        delete a.bankAccount;
        delete a.id;
        delete a.userId;
        await Alert.create(userId, a);
    }
    log.info('Done.');

    log.info('Import transaction rules...');
    const existingRules = new Set(
        (await TransactionRule.allOrdered(userId)).map(r => TransactionRule.easyHash(r))
    );

    for (const r of world.transactionRules) {
        // Clean up actions: they must refer to existing entities.
        const removeActions = [];
        let i = 0;
        for (const a of r.actions) {
            if (
                typeof a.categoryId === 'undefined' ||
                typeof categoryMap[a.categoryId] === 'undefined'
            ) {
                log.warn('Ignoring unknown category action.');
                removeActions.push(i);
            } else {
                a.categoryId = categoryMap[a.categoryId];
            }
            i += 1;
        }

        removeActions.reverse();
        for (const j of removeActions) {
            r.actions.splice(j, 1);
        }

        for (const a of r.actions) {
            delete a.ruleId;
            delete a.id;
            // Manually fill the user id. Note this may clobber the previous
            // one, if it was existing, which is fine: we should not rely on
            // those in general.
            a.userId = userId;
        }

        for (const c of r.conditions) {
            delete c.ruleId;
            delete c.id;
            // See above.
            c.userId = userId;
        }

        if (r.actions.length === 0) {
            log.warn('Ignoring rule with no actions.');
            return;
        }
        if (r.conditions.length === 0) {
            log.warn('Ignoring rule with no conditions.');
            return;
        }

        delete r.id;
        delete r.userId;

        if (!existingRules.has(TransactionRule.easyHash(r))) {
            await TransactionRule.create(userId, r);
        }
    }
    log.info('Done.');

    log.info('Import recurring transactions...');

    const currentDate = new Date();
    const currentYear = currentDate.getFullYear();
    const currentMonth = currentDate.getMonth();

    const recurringTransactionsMap = new Map<number, number>();

    const existingRecurringTransactionsHashMap = new Map<string, number>();
    const existingRecurringTransactions = await RecurringTransaction.all(userId);
    existingRecurringTransactions.forEach(rt => {
        existingRecurringTransactionsHashMap.set(RecurringTransaction.easyHash(rt), rt.id);
    });

    for (let i = 0; i < world.recurringTransactions.length; i++) {
        const rt = world.recurringTransactions[i];

        if (typeof rt.amount !== 'number' || isNaN(rt.amount)) {
            log.warn('Ignoring recurring transactions without valid amount\n', rt);
            continue;
        }

        // Map recurring transaction to account.
        if (typeof rt.accountId !== 'undefined') {
            if (!accountIdToAccount.has(rt.accountId)) {
                log.warn('Ignoring orphan recurring transaction:\n', rt);
                continue;
            }
            rt.accountId = accountIdToAccount.get(rt.accountId);
        }

        if (typeof rt.label === 'undefined') {
            log.warn('Ignoring recurring transaction without label/rawLabel:\n', rt);
            continue;
        }

        const hash = RecurringTransaction.easyHash(rt);
        const exists = existingRecurringTransactionsHashMap.has(hash);
        if (exists) {
            recurringTransactionsMap.set(rt.id, existingRecurringTransactionsHashMap.get(hash)!);
        } else {
            const created = await RecurringTransaction.create(userId, rt);
            recurringTransactionsMap.set(rt.id, created.id);
        }
    }

    for (let i = 0; i < world.appliedRecurringTransactions.length; i++) {
        const art = world.appliedRecurringTransactions[i];

        // Only import applied recurring transactions from current month.
        if (art.year !== currentYear || art.month !== currentMonth) {
            continue;
        }

        if (
            !accountIdToAccount.has(art.accountId) ||
            !recurringTransactionsMap.has(art.recurringTransactionId)
        ) {
            log.warn('Ignoring orphan applied recurring transaction:\n', art);
            continue;
        }
        art.accountId = accountIdToAccount.get(art.accountId);
        art.recurringTransactionId = recurringTransactionsMap.get(art.recurringTransactionId);

        const exists = await AppliedRecurringTransaction.exists(
            userId,
            art.accountId,
            art.recurringTransactionId,
            art.month,
            art.year
        );
        if (!exists) {
            await AppliedRecurringTransaction.create(userId, art);
        }
    }
    log.info('Done.');

    log.info('Apply banks migrations');
    await runDataMigrations(userId);
    log.info('Done.');
}

export async function import_(req: IdentifiedRequest<any>, res: express.Response) {
    try {
        const { id: userId } = req.user;

        if (await isDemoEnabled(userId)) {
            throw new KError("importing accesses isn't allowed in demo mode", 400);
        }

        if (!req.body.data) {
            throw new KError('missing parameter "data" in the file', 400);
        }

        let world = req.body.data;
        if (req.body.encrypted) {
            if (typeof req.body.data !== 'string') {
                throw new KError('content of an encrypted export should be an encoded string', 400);
            }
            if (typeof req.body.passphrase !== 'string') {
                throw new KError('missing parameter "passphrase"', 400);
            }

            if (process.kresus.salt === null) {
                throw new KError(
                    "server hasn't been configured for encryption; " +
                        'please ask your administrator to provide a salt'
                );
            }

            world = decryptData(world, req.body.passphrase);

            try {
                world = JSON.parse(world);
            } catch (err) {
                throw new KError(
                    'Invalid JSON file or bad passphrase.',
                    400,
                    getErrorCode('INVALID_PASSWORD_JSON_EXPORT')
                );
            }
        } else if (typeof req.body.data !== 'object') {
            throw new KError('content of a JSON export should be a JSON object', 400);
        }

        await importData(userId, world);

        log.info('Import finished with success!');
        res.status(200).end();
    } catch (err) {
        asyncErr(res, err, 'when importing data');
    }
}

export async function importOFX_(req: IdentifiedRequest<any>, res: express.Response) {
    try {
        const { id: userId } = req.user;

        log.info('Parsing OFX file...');

        const userData = JSON.parse(req.body);
        const convertedData = await ofxToKresus(userData.data);
        let dontCreateAccess = false;

        // Set the accessId set by the user.
        if (typeof userData.accessId === 'number' && convertedData) {
            // Make sure the access exists.
            if (!(await Access.exists(userId, userData.accessId))) {
                throw new KError('No existing access for this access id', 400);
            }

            // Replace the accessId in the converted data by this one.
            convertedData.accesses[0].id = userData.accessId;
            convertedData.accounts.forEach(acc => {
                acc.accessId = userData.accessId;
            });
            dontCreateAccess = true;
        }

        await importData(userId, convertedData, dontCreateAccess);

        log.info('Import finished with success!');
        res.status(200).end();
    } catch (err) {
        asyncErr(res, err, 'when importing data');
    }
}

export const testing = {
    ofxToKresus,
    encryptData,
    decryptData,
    getAllData,
};