feat(expenses): CSV import with per-row validation
- lib/server/csv-parse.ts: RFC 4180 parser (quoted fields, embedded commas and quotes, CRLF/LF, BOM), no extra dep - services/expenses.ts: importExpenses() is transactional + all-or-nothing — if any row fails Zod-style validation, nothing is inserted - /properties/[id]/expenses/import: upload page with per-line error list, sample CSV download at /template.csv, 5 MB cap - 'Import CSV' button wired on the expenses tab
This commit is contained in:
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* Minimal RFC 4180 CSV parser. Handles:
|
||||
* - comma delimiter (only)
|
||||
* - CRLF or LF line endings
|
||||
* - double-quoted fields (including embedded commas, newlines, quotes)
|
||||
* - escaped quotes inside quoted fields (`""` -> `"`)
|
||||
* - trailing blank lines
|
||||
*
|
||||
* Not supported: alternate delimiters (semicolon, tab), BOM removal beyond
|
||||
* the leading BOM, or streaming. Fine for user-supplied expense imports
|
||||
* where files are <5 MB.
|
||||
*
|
||||
* Returns an array of rows, each a string[]. The first row is the header row
|
||||
* — callers decide how to map it. Pair with `parseCsvDict` below if you want
|
||||
* header-keyed records.
|
||||
*/
|
||||
export function parseCsv(input: string): string[][] {
|
||||
// Strip BOM if present.
|
||||
if (input.charCodeAt(0) === 0xfeff) input = input.slice(1);
|
||||
|
||||
const rows: string[][] = [];
|
||||
let row: string[] = [];
|
||||
let field = '';
|
||||
let inQuotes = false;
|
||||
const n = input.length;
|
||||
let i = 0;
|
||||
|
||||
while (i < n) {
|
||||
const c = input[i];
|
||||
|
||||
if (inQuotes) {
|
||||
if (c === '"') {
|
||||
// Escaped quote ("") or end of quoted field.
|
||||
if (input[i + 1] === '"') {
|
||||
field += '"';
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
inQuotes = false;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
field += c;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Not inside quotes.
|
||||
if (c === '"') {
|
||||
// Opening quote — only valid at field start.
|
||||
inQuotes = true;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c === ',') {
|
||||
row.push(field);
|
||||
field = '';
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c === '\n' || c === '\r') {
|
||||
row.push(field);
|
||||
field = '';
|
||||
// Skip CRLF pair.
|
||||
if (c === '\r' && input[i + 1] === '\n') i += 2;
|
||||
else i++;
|
||||
// Drop empty rows (e.g. blank line at EOF).
|
||||
if (row.length > 1 || row[0] !== '') rows.push(row);
|
||||
row = [];
|
||||
continue;
|
||||
}
|
||||
field += c;
|
||||
i++;
|
||||
}
|
||||
// Flush trailing field / row.
|
||||
if (field !== '' || row.length > 0) {
|
||||
row.push(field);
|
||||
if (row.length > 1 || row[0] !== '') rows.push(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a CSV with a header row and return a list of record objects keyed by
|
||||
* the (trimmed, lowercased) header names. Throws if the CSV is empty or has
|
||||
* inconsistent column counts per row.
|
||||
*/
|
||||
export function parseCsvDict(input: string): Record<string, string>[] {
|
||||
const rows = parseCsv(input);
|
||||
if (rows.length === 0) return [];
|
||||
const headers = rows[0].map((h) => h.trim().toLowerCase());
|
||||
const out: Record<string, string>[] = [];
|
||||
for (let i = 1; i < rows.length; i++) {
|
||||
const r = rows[i];
|
||||
if (r.length !== headers.length) {
|
||||
throw new Error(
|
||||
`Row ${i + 1} has ${r.length} columns but header has ${headers.length}`
|
||||
);
|
||||
}
|
||||
const obj: Record<string, string> = {};
|
||||
for (let k = 0; k < headers.length; k++) obj[headers[k]] = r[k] ?? '';
|
||||
out.push(obj);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
type NewPropertyExpense,
|
||||
type PropertyExpense
|
||||
} from '$lib/server/db/schema/expenses';
|
||||
import type { ExpenseKind } from '$lib/expenses';
|
||||
import { EXPENSE_KINDS, type ExpenseKind } from '$lib/expenses';
|
||||
|
||||
export type { ExpenseKind };
|
||||
|
||||
@@ -279,5 +279,125 @@ export async function summaryForProperty(
|
||||
return { byKind, grandTotal: grand, currency };
|
||||
}
|
||||
|
||||
// --- CSV import --------------------------------------------------------------
|
||||
|
||||
export interface ImportRow {
|
||||
/** 1-based row number in the source CSV (including header = row 1). */
|
||||
line: number;
|
||||
values: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface ImportResult {
|
||||
imported: number;
|
||||
errors: Array<{ line: number; message: string }>;
|
||||
}
|
||||
|
||||
const REQUIRED_COLS = ['date', 'kind', 'amount'] as const;
|
||||
const OPTIONAL_COLS = [
|
||||
'currency',
|
||||
'period_start',
|
||||
'period_end',
|
||||
'vendor',
|
||||
'reference',
|
||||
'notes'
|
||||
] as const;
|
||||
const ALL_COLS = [...REQUIRED_COLS, ...OPTIONAL_COLS];
|
||||
|
||||
function parseDate(raw: string | undefined, field: string): Date | null {
|
||||
if (!raw || !raw.trim()) return null;
|
||||
const s = raw.trim();
|
||||
// Accept YYYY-MM-DD, YYYY/MM/DD, or full ISO 8601.
|
||||
const match = s.match(/^(\d{4})[-/](\d{1,2})[-/](\d{1,2})(?:[T ]\d.+)?$/);
|
||||
if (!match) throw new Error(`${field} is not a valid date (use YYYY-MM-DD)`);
|
||||
const d = new Date(s.replace(/\//g, '-'));
|
||||
if (Number.isNaN(d.getTime())) throw new Error(`${field} is not a valid date`);
|
||||
return d;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate an array of raw CSV records and insert them in a single transaction.
|
||||
* If any row fails validation, NOTHING is inserted and the caller gets back a
|
||||
* list of per-line error messages.
|
||||
*/
|
||||
export async function importExpenses(
|
||||
companyId: string,
|
||||
propertyId: string,
|
||||
createdBy: string,
|
||||
rows: ImportRow[],
|
||||
defaultCurrency: string
|
||||
): Promise<ImportResult> {
|
||||
await assertProperty(companyId, propertyId);
|
||||
if (rows.length === 0) return { imported: 0, errors: [] };
|
||||
|
||||
// --- validation pass -----------------------------------------------------
|
||||
const errors: Array<{ line: number; message: string }> = [];
|
||||
const prepared: NewPropertyExpense[] = [];
|
||||
const validKinds = new Set<string>(EXPENSE_KINDS);
|
||||
const defCur = defaultCurrency.trim().toUpperCase();
|
||||
|
||||
for (const row of rows) {
|
||||
try {
|
||||
const v = row.values;
|
||||
// Reject unknown columns early so a typo'd "dato" doesn't silently drop data.
|
||||
for (const k of Object.keys(v)) {
|
||||
if (!ALL_COLS.includes(k as (typeof ALL_COLS)[number])) {
|
||||
throw new Error(`unknown column '${k}'`);
|
||||
}
|
||||
}
|
||||
const kind = (v.kind ?? '').trim().toLowerCase();
|
||||
if (!kind) throw new Error('kind is required');
|
||||
if (!validKinds.has(kind)) {
|
||||
throw new Error(
|
||||
`kind '${kind}' is not one of: ${EXPENSE_KINDS.join(', ')}`
|
||||
);
|
||||
}
|
||||
const amountStr = (v.amount ?? '').trim();
|
||||
if (!amountStr) throw new Error('amount is required');
|
||||
const amount = Number(amountStr);
|
||||
if (!Number.isFinite(amount)) throw new Error(`amount '${amountStr}' is not a number`);
|
||||
if (amount <= 0) throw new Error('amount must be positive');
|
||||
|
||||
const incurredAt = parseDate(v.date, 'date');
|
||||
if (!incurredAt) throw new Error('date is required');
|
||||
|
||||
const currency = ((v.currency ?? '').trim() || defCur).toUpperCase();
|
||||
if (currency.length !== 3) throw new Error(`currency '${currency}' must be 3 letters`);
|
||||
|
||||
const periodStart = parseDate(v.period_start, 'period_start');
|
||||
const periodEnd = parseDate(v.period_end, 'period_end');
|
||||
|
||||
prepared.push({
|
||||
propertyId,
|
||||
accountId: null,
|
||||
kind: kind as ExpenseKind,
|
||||
amount: String(amount),
|
||||
currency,
|
||||
incurredAt,
|
||||
periodStart,
|
||||
periodEnd,
|
||||
vendor: (v.vendor ?? '').trim() || null,
|
||||
reference: (v.reference ?? '').trim() || null,
|
||||
notes: (v.notes ?? '').trim() || null,
|
||||
createdBy
|
||||
});
|
||||
} catch (e) {
|
||||
errors.push({ line: row.line, message: (e as Error).message });
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) return { imported: 0, errors };
|
||||
|
||||
// --- transactional insert ------------------------------------------------
|
||||
await db.transaction(async (tx) => {
|
||||
// Drizzle has a ~65k parameter cap per statement. Chunk to be safe on
|
||||
// very large imports (12 params per row → ~5k rows per chunk).
|
||||
const CHUNK = 2_000;
|
||||
for (let i = 0; i < prepared.length; i += CHUNK) {
|
||||
await tx.insert(propertyExpenses).values(prepared.slice(i, i + CHUNK));
|
||||
}
|
||||
});
|
||||
return { imported: prepared.length, errors: [] };
|
||||
}
|
||||
|
||||
// Unused import placeholder — asc kept in case we add a low-level listing helper later.
|
||||
void asc;
|
||||
|
||||
@@ -71,10 +71,16 @@
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
<div class="flex shrink-0 items-center gap-2">
|
||||
<a href="expenses/import"
|
||||
class="rounded-md border border-gray-300 px-3 py-1.5 text-sm font-medium text-gray-700 hover:bg-gray-50 dark:border-gray-600 dark:text-gray-200 dark:hover:bg-gray-700">
|
||||
Import CSV
|
||||
</a>
|
||||
<button type="button" onclick={() => { showForm = !showForm; editingId = null; }}
|
||||
class="rounded-md bg-primary-600 px-3 py-1.5 text-sm font-medium text-white hover:bg-primary-700">
|
||||
{showForm ? 'Cancel' : '+ New expense'}
|
||||
</button>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{#if form?.error}
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import { fail } from '@sveltejs/kit';
|
||||
import { and, eq } from 'drizzle-orm';
|
||||
import { requireCompany } from '$lib/server/auth/guards';
|
||||
import { db } from '$lib/server/db/client';
|
||||
import { companies } from '$lib/server/db/schema/tenancy';
|
||||
import { parseCsvDict } from '$lib/server/csv-parse';
|
||||
import { importExpenses, type ImportRow } from '$lib/server/services/expenses';
|
||||
import type { Actions, PageServerLoad } from './$types';
|
||||
|
||||
const MAX_BYTES = 5 * 1024 * 1024; // 5 MB
|
||||
|
||||
interface CompanySettings {
|
||||
default_currency?: string;
|
||||
}
|
||||
function parseSettings(raw: string | null | undefined): CompanySettings {
|
||||
if (!raw) return {};
|
||||
try {
|
||||
return JSON.parse(raw) as CompanySettings;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export const load: PageServerLoad = async ({ locals }) => {
|
||||
const { company } = requireCompany(locals);
|
||||
const [row] = await db
|
||||
.select({ settings: companies.settings })
|
||||
.from(companies)
|
||||
.where(eq(companies.id, company.id))
|
||||
.limit(1);
|
||||
return {
|
||||
defaultCurrency: parseSettings(row?.settings ?? null).default_currency ?? 'USD'
|
||||
};
|
||||
};
|
||||
|
||||
export const actions: Actions = {
|
||||
default: async ({ request, locals, params }) => {
|
||||
const { user, company } = requireCompany(locals);
|
||||
const form = await request.formData();
|
||||
const file = form.get('file');
|
||||
if (!(file instanceof File) || file.size === 0) {
|
||||
return fail(400, { error: 'Pick a CSV file to upload.', rowErrors: [] });
|
||||
}
|
||||
if (file.size > MAX_BYTES) {
|
||||
return fail(413, {
|
||||
error: `File too large (max ${MAX_BYTES / 1024 / 1024} MB).`,
|
||||
rowErrors: []
|
||||
});
|
||||
}
|
||||
const text = await file.text();
|
||||
|
||||
let records: Record<string, string>[];
|
||||
try {
|
||||
records = parseCsvDict(text);
|
||||
} catch (e) {
|
||||
return fail(400, {
|
||||
error: `CSV parse failed: ${(e as Error).message}`,
|
||||
rowErrors: []
|
||||
});
|
||||
}
|
||||
if (records.length === 0) {
|
||||
return fail(400, {
|
||||
error: 'No data rows found (is the file just a header?).',
|
||||
rowErrors: []
|
||||
});
|
||||
}
|
||||
|
||||
const rows: ImportRow[] = records.map((values, idx) => ({
|
||||
// +2 because: human 1-based line numbers, plus header row.
|
||||
line: idx + 2,
|
||||
values
|
||||
}));
|
||||
|
||||
const [companyRow] = await db
|
||||
.select({ settings: companies.settings })
|
||||
.from(companies)
|
||||
.where(eq(companies.id, company.id))
|
||||
.limit(1);
|
||||
const defCur =
|
||||
parseSettings(companyRow?.settings ?? null).default_currency ?? 'USD';
|
||||
|
||||
try {
|
||||
const result = await importExpenses(company.id, params.id, user.id, rows, defCur);
|
||||
if (result.errors.length > 0) {
|
||||
return fail(400, {
|
||||
error: `${result.errors.length} row${result.errors.length === 1 ? '' : 's'} rejected — nothing imported. Fix the CSV and retry.`,
|
||||
rowErrors: result.errors
|
||||
});
|
||||
}
|
||||
return { ok: true, imported: result.imported };
|
||||
} catch (e) {
|
||||
return fail(400, { error: (e as Error).message, rowErrors: [] });
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// silence unused helper
|
||||
void and;
|
||||
@@ -0,0 +1,84 @@
|
||||
<script lang="ts">
|
||||
import { enhance } from '$app/forms';
|
||||
import type { PageData, ActionData } from './$types';
|
||||
|
||||
let { data, form }: { data: PageData; form: ActionData } = $props();
|
||||
let uploading = $state(false);
|
||||
</script>
|
||||
|
||||
<div class="mx-auto max-w-3xl space-y-6">
|
||||
<div>
|
||||
<a href="../" class="text-xs text-gray-500 hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-200">← back to expenses</a>
|
||||
<h1 class="mt-1 text-2xl font-semibold text-gray-900 dark:text-gray-100">Import expenses from CSV</h1>
|
||||
<p class="mt-1 text-sm text-gray-500 dark:text-gray-400">
|
||||
Upload a CSV exported from your bank, accounting tool, or a hand-typed sheet.
|
||||
Validation is all-or-nothing — if any row is bad, nothing is imported and
|
||||
you'll get a list of line numbers to fix.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="rounded-lg border border-gray-200 bg-white p-4 text-sm dark:border-gray-700 dark:bg-gray-800">
|
||||
<p class="font-medium text-gray-900 dark:text-gray-100">Expected format</p>
|
||||
<p class="mt-1 text-gray-600 dark:text-gray-300">
|
||||
First line is the header. Required columns:
|
||||
<code class="font-mono">date</code>,
|
||||
<code class="font-mono">kind</code>,
|
||||
<code class="font-mono">amount</code>.
|
||||
Optional:
|
||||
<code class="font-mono">currency</code> (falls back to <code class="font-mono">{data.defaultCurrency}</code>),
|
||||
<code class="font-mono">period_start</code>,
|
||||
<code class="font-mono">period_end</code>,
|
||||
<code class="font-mono">vendor</code>,
|
||||
<code class="font-mono">reference</code>,
|
||||
<code class="font-mono">notes</code>.
|
||||
</p>
|
||||
<p class="mt-2 text-gray-600 dark:text-gray-300">
|
||||
Dates are <code class="font-mono">YYYY-MM-DD</code>. Valid kinds:
|
||||
<code class="font-mono">water, electricity, gas, internet, phone, cable, waste, maintenance, repair, cleaning, insurance, tax, rent, other</code>.
|
||||
</p>
|
||||
<a href="template.csv" download
|
||||
class="mt-3 inline-block text-sm text-primary-600 hover:underline dark:text-primary-400">
|
||||
Download template CSV
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<form method="post" enctype="multipart/form-data"
|
||||
use:enhance={() => {
|
||||
uploading = true;
|
||||
return ({ update }) => update().finally(() => (uploading = false));
|
||||
}}
|
||||
class="space-y-4 rounded-lg border border-gray-200 bg-white p-6 dark:border-gray-700 dark:bg-gray-800">
|
||||
{#if form?.error}
|
||||
<div class="rounded-md border border-red-200 bg-red-50 p-3 text-sm text-red-700 dark:border-red-700/50 dark:bg-red-900/20 dark:text-red-300">
|
||||
<p class="font-medium">{form.error}</p>
|
||||
{#if form.rowErrors && form.rowErrors.length > 0}
|
||||
<ul class="mt-2 list-disc space-y-0.5 pl-5 text-xs">
|
||||
{#each form.rowErrors as err}
|
||||
<li><span class="font-mono">line {err.line}</span> — {err.message}</li>
|
||||
{/each}
|
||||
</ul>
|
||||
{/if}
|
||||
</div>
|
||||
{:else if form?.ok}
|
||||
<div class="rounded-md border border-emerald-200 bg-emerald-50 p-3 text-sm text-emerald-700 dark:border-emerald-700/50 dark:bg-emerald-900/20 dark:text-emerald-300">
|
||||
Imported {form.imported} expense{form.imported === 1 ? '' : 's'}.
|
||||
<a href="../" class="ml-2 underline">View them →</a>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<label class="block">
|
||||
<span class="block text-sm font-medium text-gray-700 dark:text-gray-300">CSV file <span class="text-red-500">*</span></span>
|
||||
<input name="file" type="file" accept=".csv,text/csv" required
|
||||
class="mt-1 block w-full text-sm text-gray-700 file:mr-3 file:rounded-md file:border-0 file:bg-primary-50 file:px-3 file:py-1.5 file:text-sm file:font-medium file:text-primary-700 hover:file:bg-primary-100 dark:text-gray-300 dark:file:bg-primary-900/30 dark:file:text-primary-300" />
|
||||
<p class="mt-1 text-xs text-gray-400">Max 5 MB.</p>
|
||||
</label>
|
||||
|
||||
<div class="flex justify-end gap-3 border-t border-gray-200 pt-4 dark:border-gray-700">
|
||||
<a href="../" class="text-sm text-gray-600 hover:text-gray-900 dark:text-gray-400 dark:hover:text-gray-100">Cancel</a>
|
||||
<button type="submit" disabled={uploading}
|
||||
class="rounded-md bg-primary-600 px-4 py-2 text-sm font-medium text-white shadow-sm hover:bg-primary-700 disabled:opacity-60">
|
||||
{uploading ? 'Uploading…' : 'Import'}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
@@ -0,0 +1,48 @@
|
||||
import { requireCompany } from '$lib/server/auth/guards';
|
||||
import { csvResponse, toCsv } from '$lib/server/csv';
|
||||
import type { RequestHandler } from './$types';
|
||||
|
||||
// Tiny sample CSV so the user has a well-formed example to edit rather than
|
||||
// guessing the column names and date format.
|
||||
export const GET: RequestHandler = async ({ locals }) => {
|
||||
requireCompany(locals);
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
const body = toCsv(
|
||||
[
|
||||
{
|
||||
date: today,
|
||||
kind: 'electricity',
|
||||
amount: '1234.56',
|
||||
currency: 'THB',
|
||||
period_start: '',
|
||||
period_end: '',
|
||||
vendor: 'MEA',
|
||||
reference: 'INV-2026-0001',
|
||||
notes: ''
|
||||
},
|
||||
{
|
||||
date: today,
|
||||
kind: 'water',
|
||||
amount: '345.00',
|
||||
currency: 'THB',
|
||||
period_start: '',
|
||||
period_end: '',
|
||||
vendor: 'MWA',
|
||||
reference: '',
|
||||
notes: ''
|
||||
}
|
||||
],
|
||||
[
|
||||
'date',
|
||||
'kind',
|
||||
'amount',
|
||||
'currency',
|
||||
'period_start',
|
||||
'period_end',
|
||||
'vendor',
|
||||
'reference',
|
||||
'notes'
|
||||
]
|
||||
);
|
||||
return csvResponse('expenses-template.csv', body);
|
||||
};
|
||||
Reference in New Issue
Block a user