import { type DataSource, type DeepPartial, type ObjectLiteral, QueryFailedError, type QueryRunner, type Repository, } from 'typeorm'; import type { TableColumnOptions } from 'typeorm/schema-builder/options/TableColumnOptions'; import type { TableForeignKeyOptions } from 'typeorm/schema-builder/options/TableForeignKeyOptions'; import type { BankVendor } from '../../shared/types'; import { makeLogger, UNKNOWN_TRANSACTION_TYPE } from '../helpers'; import { bankVendorByUuid } from '../providers'; import type { AccessField, Transaction } from './'; const log = makeLogger('models/helpers'); const hasCategory = (tr: Transaction): boolean => tr.categoryId !== null; const hasType = (tr: Transaction): boolean => { return typeof tr.type !== 'undefined' && tr.type !== UNKNOWN_TRANSACTION_TYPE; }; const hasCustomLabel = (tr: Transaction): boolean => typeof tr.customLabel === 'string'; const hasBudgetDate = (tr: Transaction): boolean => { return typeof tr.budgetDate !== 'undefined' && tr.budgetDate !== null; }; const hasDebitDate = (tr: Transaction): boolean => { return typeof tr.debitDate !== 'undefined' && tr.debitDate !== null; }; export function mergeWith(target: Transaction, other: Transaction): DeepPartial { const update: DeepPartial = {}; // Always trigger an update for the import date, to avoid duplicate // transactions to appear in reports around the date where the duplicate // has been imported. // This should be always true, but we stay defensive here. if (typeof other.importDate !== 'undefined' && other.importDate !== null) { update.importDate = other.importDate; } if (!hasCategory(target) && hasCategory(other)) { update.categoryId = other.categoryId; } if (!hasType(target) && hasType(other)) { update.type = other.type; update.isUserDefinedType = other.isUserDefinedType; } if (!hasCustomLabel(target)) { if (hasCustomLabel(other)) { update.customLabel = other.customLabel; } else if (other.createdByUser) { // If the transaction was manually created the label is probably better suited. update.customLabel = other.label; } } if (!hasBudgetDate(target) && hasBudgetDate(other)) { update.budgetDate = other.budgetDate; } if (!hasDebitDate(target) && hasDebitDate(other)) { update.debitDate = other.debitDate; } // If the other transaction was not created by the user it means // the current one was probably created as a provisional transaction // and should now be considered as the actual (coming from the bank) // transaction. if (target.createdByUser && !other.createdByUser) { update.createdByUser = false; } // If this is a recurring transaction but not the other, the other is probably // a manual or real transaction, which should have the priority. if (target.isRecurrentTransaction && !other.isRecurrentTransaction) { update.isRecurrentTransaction = false; } return update; } // A hack to make sure that a value read from the database is coerced into a // number, if it's not it in the first place. Useful to support postgres's // numerical type which returns a string. export class ForceNumericColumn { // This direction is fine. to(data: any) { return data; } // Converts from a string to a number. from(data: any) { let ret: number | undefined | null; if (['undefined', 'number'].includes(typeof data) || data === null) { ret = data; } else { ret = Number.parseFloat(data); } if (Number.isNaN(ret)) { log.error('unexpected NaN stored in column'); } return ret; } } // The type to use in entities declarations for datetime (date with a // timestamp), Note it must be used like this: { type: DatetimeType }, because // of limitations of the ORM. export const DatetimeType = Date; // Normalizes the datetime type for the database for migrations. export function datetimeType(queryRunner: QueryRunner): string { return queryRunner.connection.driver.normalizeType({ type: Date }); } // sqlite can't use more than 999 variables in a single query. If an entity has // N fields, then we can't insert more than 999/N entities at once; in this // case, we need to split up the batches into smaller ones. // // 50 ought to be enough for everyone, since it allows up to 19 fields. export const LOW_NUM_ENTITIES_IN_BATCH = 50; // The same issue happens with postgres which can't bind more than 64K features at once. const NUM_ENTITIES_IN_BATCH = 1000; export function isSqlite(connection: DataSource): boolean { const dbType = connection.driver.options.type; return dbType === 'better-sqlite3'; } export function isUniqueConstraintViolation(err: unknown): boolean { if (!(err instanceof QueryFailedError)) { return false; } const code = (err.driverError as { code?: string } | undefined)?.code; // The error codes reported when a UNIQUE constraint is violated: better-sqlite3 uses an extended // sqlite result code, postgres the SQLSTATE for unique_violation. // See https://www.sqlite.org/c3ref/c_abort_rollback.html#:~:text=SQLITE_CONSTRAINT_UNIQUE // See https://www.postgresql.org/docs/current/errcodes-appendix.html#:~:text=23505 return typeof code === 'string' && (code === 'SQLITE_CONSTRAINT_UNIQUE' || code === '23505'); } // Note: doesn't return the inserted entities, only their ids, in the same order as the entities // which were passed as arguments. export async function bulkInsert( repository: Repository, entities: Extract['insert']>[0], Array> ): Promise { // Do not call `repository.insert` without actual entities, that will generate an empty insert // query and throw an error. // See https://github.com/typeorm/typeorm/issues/3111 if (entities.length === 0) { return []; } const insertedIds: number[] = []; let remaining = entities; let batchSize = NUM_ENTITIES_IN_BATCH; if (isSqlite(repository.manager.connection)) { batchSize = LOW_NUM_ENTITIES_IN_BATCH; } log.info(`bulk insert: splitting up batches with a size of ${batchSize}`); while (remaining.length > 0) { const nextRemaining = remaining.splice(batchSize); const result = await repository.insert(remaining); insertedIds.push(...result.identifiers.map(identifier => identifier.id)); remaining = nextRemaining; } return insertedIds; } export async function bulkDelete( repository: Repository, ids: number[] ): Promise { if (ids.length === 0) { return; } let remaining = ids; let batchSize = NUM_ENTITIES_IN_BATCH; if (isSqlite(repository.manager.connection)) { batchSize = LOW_NUM_ENTITIES_IN_BATCH; } log.info(`bulk delete: splitting up batches with a size of ${batchSize}`); while (remaining.length > 0) { const nextRemaining = remaining.splice(batchSize); await repository.delete(remaining); remaining = nextRemaining; } } export function idColumn(): TableColumnOptions { return { name: 'id', type: 'integer', isPrimary: true, isGenerated: true, generationStrategy: 'increment', }; } export function foreignKey( constraintName: string, columnName: string, referencedTableName: string, referencedColumnName: string, cascadeOpts: { onDelete?: string; onUpdate?: string } = { onDelete: 'CASCADE', onUpdate: 'NO ACTION', } ): TableForeignKeyOptions { return { name: constraintName, columnNames: [columnName], referencedColumnNames: [referencedColumnName], referencedTableName, ...cascadeOpts, }; } export function foreignKeyUserId(tableName: string): TableForeignKeyOptions { return foreignKey(`${tableName}_ref_user_id`, 'userId', 'user', 'id'); } // As Partial, but accepting one extra level of Partial-ness for its // attributes. This allows passing Partial to the TransactionRule's // creator for example. // (Inspired from typeorm's DeepPartial) export type PartialOnePlus = { [P in keyof T]?: T[P] extends Array ? Array> : T[P] extends ReadonlyArray ? ReadonlyArray> : Partial | T[P]; }; export const areFieldsComplete = (vendorId: string, fields: AccessField[]) => { let vendorDescriptor: BankVendor; try { vendorDescriptor = bankVendorByUuid(vendorId); } catch { return false; } // Check that for every non-optional field, we do have a corresponding value. if (!vendorDescriptor.customFields) { return vendorDescriptor.noCredentials === true; } const mandatoryFields = vendorDescriptor.customFields .filter(f => !('optional' in f) || f.optional === false) .map(f => f.name); if (fields.length < mandatoryFields.length) { return false; } return mandatoryFields.every(fieldName => fields.find(f => f.name === fieldName && !!f.value)); };