"""Acesso ao SQLite: esquema, migrações leves e helpers de consulta. Estende o esquema original (uploaded_files -> detected_documents -> fiscal_documents) com autenticação (users), lotes de importação (import_batches) e os campos de legibilidade usados pela extração por IA. """ from __future__ import annotations import json import sqlite3 from contextlib import contextmanager from pathlib import Path from typing import Any, Iterator from .config import get_settings SCHEMA = """ CREATE TABLE IF NOT EXISTS users ( id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT NOT NULL UNIQUE, password_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE TABLE IF NOT EXISTS import_batches ( id INTEGER PRIMARY KEY AUTOINCREMENT, status TEXT NOT NULL DEFAULT 'open', created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, confirmed_at TEXT ); CREATE TABLE IF NOT EXISTS uploaded_files ( id INTEGER PRIMARY KEY AUTOINCREMENT, batch_id INTEGER REFERENCES import_batches(id) ON DELETE SET NULL, original_name TEXT NOT NULL, stored_path TEXT NOT NULL, content_type TEXT NOT NULL, size_bytes INTEGER NOT NULL, status TEXT NOT NULL DEFAULT 'pending', detected_count INTEGER NOT NULL DEFAULT 0, message TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE TABLE IF NOT EXISTS detected_documents ( id INTEGER PRIMARY KEY AUTOINCREMENT, upload_id INTEGER NOT NULL REFERENCES uploaded_files(id) ON DELETE CASCADE, batch_id INTEGER REFERENCES import_batches(id) ON DELETE SET NULL, source_file_name TEXT NOT NULL, source_page INTEGER, source_location TEXT NOT NULL, raw_text TEXT NOT NULL DEFAULT '', purchase_date TEXT, supplier_name TEXT, total_paid REAL, confidence TEXT NOT NULL DEFAULT 'low', field_confidence_json TEXT NOT NULL DEFAULT '{}', legible INTEGER NOT NULL DEFAULT 1, uncertain_fields TEXT NOT NULL DEFAULT '[]', extractor TEXT NOT NULL DEFAULT 'local', status TEXT NOT NULL DEFAULT 'staged', created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE TABLE IF NOT EXISTS fiscal_documents ( id INTEGER PRIMARY KEY AUTOINCREMENT, detected_document_id INTEGER REFERENCES detected_documents(id) ON DELETE SET NULL, source_file_name TEXT NOT NULL DEFAULT '', source_location TEXT NOT NULL DEFAULT '', purchase_date TEXT NOT NULL, supplier_name TEXT NOT NULL, total_paid REAL NOT NULL, confidence TEXT NOT NULL DEFAULT 'high', created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX IF NOT EXISTS idx_fiscal_date ON fiscal_documents(purchase_date); CREATE INDEX IF NOT EXISTS idx_detected_batch ON detected_documents(batch_id, status); """ def connect(db_path: Path | None = None) -> sqlite3.Connection: settings = get_settings() settings.ensure_storage() conn = sqlite3.connect(db_path or settings.db_path) conn.row_factory = sqlite3.Row conn.execute("PRAGMA foreign_keys = ON") conn.execute("PRAGMA journal_mode = WAL") return conn def init_db(conn: sqlite3.Connection) -> None: conn.executescript(SCHEMA) conn.commit() @contextmanager def session(db_path: Path | None = None) -> Iterator[sqlite3.Connection]: conn = connect(db_path) try: init_db(conn) yield conn finally: conn.close() # --------------------------------------------------------------------------- # # Usuários # --------------------------------------------------------------------------- # def count_users(conn: sqlite3.Connection) -> int: return int(conn.execute("SELECT COUNT(*) FROM users").fetchone()[0]) def get_user(conn: sqlite3.Connection, username: str) -> sqlite3.Row | None: return conn.execute( "SELECT * FROM users WHERE username = ?", (username,) ).fetchone() def create_user(conn: sqlite3.Connection, username: str, password_hash: str) -> int: cur = conn.execute( "INSERT INTO users (username, password_hash) VALUES (?, ?)", (username, password_hash), ) conn.commit() return int(cur.lastrowid) def update_password(conn: sqlite3.Connection, username: str, password_hash: str) -> None: conn.execute( "UPDATE users SET password_hash = ? WHERE username = ?", (password_hash, username), ) conn.commit() # --------------------------------------------------------------------------- # # Lotes de importação # --------------------------------------------------------------------------- # def create_batch(conn: sqlite3.Connection) -> int: cur = conn.execute("INSERT INTO import_batches (status) VALUES ('open')") conn.commit() return int(cur.lastrowid) def get_batch(conn: sqlite3.Connection, batch_id: int) -> sqlite3.Row | None: return conn.execute( "SELECT * FROM import_batches WHERE id = ?", (batch_id,) ).fetchone() def set_batch_status(conn: sqlite3.Connection, batch_id: int, status: str) -> None: if status == "confirmed": conn.execute( "UPDATE import_batches SET status = ?, confirmed_at = CURRENT_TIMESTAMP WHERE id = ?", (status, batch_id), ) else: conn.execute( "UPDATE import_batches SET status = ? WHERE id = ?", (status, batch_id) ) conn.commit() # --------------------------------------------------------------------------- # # Uploads # --------------------------------------------------------------------------- # def insert_upload( conn: sqlite3.Connection, batch_id: int, original_name: str, stored_path: Path, content_type: str, size_bytes: int, ) -> int: cur = conn.execute( """ INSERT INTO uploaded_files (batch_id, original_name, stored_path, content_type, size_bytes) VALUES (?, ?, ?, ?, ?) """, (batch_id, original_name, str(stored_path), content_type, size_bytes), ) conn.commit() return int(cur.lastrowid) def update_upload_status( conn: sqlite3.Connection, upload_id: int, status: str, detected_count: int, message: str | None = None, ) -> None: conn.execute( """ UPDATE uploaded_files SET status = ?, detected_count = ?, message = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? """, (status, detected_count, message, upload_id), ) conn.commit() def get_upload(conn: sqlite3.Connection, upload_id: int) -> sqlite3.Row | None: return conn.execute( "SELECT * FROM uploaded_files WHERE id = ?", (upload_id,) ).fetchone() # --------------------------------------------------------------------------- # # Documentos detectados (staging) # --------------------------------------------------------------------------- # def insert_detected( conn: sqlite3.Connection, *, upload_id: int, batch_id: int, source_file_name: str, source_page: int | None, source_location: str, raw_text: str, purchase_date: str | None, supplier_name: str | None, total_paid: float | None, confidence: str, field_confidence: dict[str, str], legible: bool, uncertain_fields: list[str], extractor: str, ) -> int: cur = conn.execute( """ INSERT INTO detected_documents ( upload_id, batch_id, source_file_name, source_page, source_location, raw_text, purchase_date, supplier_name, total_paid, confidence, field_confidence_json, legible, uncertain_fields, extractor, status ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'staged') """, ( upload_id, batch_id, source_file_name, source_page, source_location, raw_text, purchase_date, supplier_name, total_paid, confidence, json.dumps(field_confidence, ensure_ascii=False), 1 if legible else 0, json.dumps(uncertain_fields, ensure_ascii=False), extractor, ), ) conn.commit() return int(cur.lastrowid) def get_detected(conn: sqlite3.Connection, detected_id: int) -> sqlite3.Row | None: return conn.execute( "SELECT * FROM detected_documents WHERE id = ?", (detected_id,) ).fetchone() def staged_documents(conn: sqlite3.Connection, batch_id: int) -> list[sqlite3.Row]: return list( conn.execute( """ SELECT * FROM detected_documents WHERE batch_id = ? AND status = 'staged' ORDER BY id ASC """, (batch_id,), ) ) def batch_summary(conn: sqlite3.Connection, batch_id: int) -> dict[str, Any]: row = conn.execute( """ SELECT COUNT(*) AS n, COALESCE(SUM(total_paid), 0) AS total, SUM(CASE WHEN legible = 0 THEN 1 ELSE 0 END) AS ilegiveis, SUM(CASE WHEN supplier_name IS NULL OR TRIM(supplier_name) = '' OR total_paid IS NULL THEN 1 ELSE 0 END) AS incompletos FROM detected_documents WHERE batch_id = ? AND status = 'staged' """, (batch_id,), ).fetchone() # "pendentes" = tudo que impede importar (ilegível OU faltando fornecedor/valor). ilegiveis = int(row["ilegiveis"] or 0) incompletos = int(row["incompletos"] or 0) pendentes = conn.execute( """ SELECT COUNT(*) FROM detected_documents WHERE batch_id = ? AND status = 'staged' AND (legible = 0 OR supplier_name IS NULL OR TRIM(supplier_name) = '' OR total_paid IS NULL) """, (batch_id,), ).fetchone()[0] return { "count": int(row["n"]), "total": float(row["total"]), "ilegiveis": ilegiveis, "incompletos": incompletos, "pendentes": int(pendentes), } def update_staged( conn: sqlite3.Connection, detected_id: int, *, purchase_date: str, supplier_name: str, total_paid: float, legible: bool = True, ) -> None: conn.execute( """ UPDATE detected_documents SET purchase_date = ?, supplier_name = ?, total_paid = ?, legible = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? AND status = 'staged' """, (purchase_date, supplier_name, total_paid, 1 if legible else 0, detected_id), ) conn.commit() def discard_staged(conn: sqlite3.Connection, detected_id: int) -> None: conn.execute( "UPDATE detected_documents SET status = 'discarded', updated_at = CURRENT_TIMESTAMP WHERE id = ?", (detected_id,), ) conn.commit() def confirm_batch(conn: sqlite3.Connection, batch_id: int) -> int: """Promove todos os detectados 'staged' do lote para fiscal_documents.""" rows = staged_documents(conn, batch_id) inserted = 0 for row in rows: if not row["purchase_date"] or not row["supplier_name"] or row["total_paid"] is None: # Sem os três campos essenciais não confirma — permanece staged. continue conn.execute( """ INSERT INTO fiscal_documents ( detected_document_id, source_file_name, source_location, purchase_date, supplier_name, total_paid, confidence ) VALUES (?, ?, ?, ?, ?, ?, ?) """, ( row["id"], row["source_file_name"], row["source_location"], row["purchase_date"], row["supplier_name"], float(row["total_paid"]), row["confidence"], ), ) conn.execute( "UPDATE detected_documents SET status = 'confirmed', updated_at = CURRENT_TIMESTAMP WHERE id = ?", (row["id"],), ) inserted += 1 set_batch_status(conn, batch_id, "confirmed") conn.commit() return inserted # --------------------------------------------------------------------------- # # CRUD de fiscal_documents # --------------------------------------------------------------------------- # def create_fiscal( conn: sqlite3.Connection, *, purchase_date: str, supplier_name: str, total_paid: float, source_file_name: str = "lançamento manual", source_location: str = "manual", confidence: str = "high", detected_document_id: int | None = None, ) -> int: cur = conn.execute( """ INSERT INTO fiscal_documents ( detected_document_id, source_file_name, source_location, purchase_date, supplier_name, total_paid, confidence ) VALUES (?, ?, ?, ?, ?, ?, ?) """, ( detected_document_id, source_file_name, source_location, purchase_date, supplier_name, total_paid, confidence, ), ) conn.commit() return int(cur.lastrowid) def get_fiscal(conn: sqlite3.Connection, doc_id: int) -> sqlite3.Row | None: return conn.execute( "SELECT * FROM fiscal_documents WHERE id = ?", (doc_id,) ).fetchone() def update_fiscal( conn: sqlite3.Connection, doc_id: int, *, purchase_date: str, supplier_name: str, total_paid: float, ) -> None: conn.execute( """ UPDATE fiscal_documents SET purchase_date = ?, supplier_name = ?, total_paid = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? """, (purchase_date, supplier_name, total_paid, doc_id), ) conn.commit() def delete_fiscal(conn: sqlite3.Connection, doc_id: int) -> None: conn.execute("DELETE FROM fiscal_documents WHERE id = ?", (doc_id,)) conn.commit() def list_fiscal( conn: sqlite3.Connection, *, start: str | None = None, end: str | None = None, supplier: str | None = None, ) -> list[sqlite3.Row]: clauses: list[str] = [] params: list[Any] = [] if start: clauses.append("purchase_date >= ?") params.append(start) if end: clauses.append("purchase_date <= ?") params.append(end) if supplier: clauses.append("supplier_name LIKE ?") params.append(f"%{supplier}%") where = f"WHERE {' AND '.join(clauses)}" if clauses else "" return list( conn.execute( f"SELECT * FROM fiscal_documents {where} ORDER BY purchase_date DESC, id DESC", params, ) ) def monthly_totals(conn: sqlite3.Connection) -> list[sqlite3.Row]: return list( conn.execute( """ SELECT substr(purchase_date, 1, 7) AS month, SUM(total_paid) AS total, COUNT(*) AS count FROM fiscal_documents GROUP BY substr(purchase_date, 1, 7) ORDER BY month ASC """ ) ) def supplier_totals(conn: sqlite3.Connection, limit: int = 10) -> list[sqlite3.Row]: return list( conn.execute( """ SELECT supplier_name, SUM(total_paid) AS total, COUNT(*) AS count FROM fiscal_documents GROUP BY supplier_name ORDER BY total DESC LIMIT ? """, (limit,), ) ) def overall_totals(conn: sqlite3.Connection) -> dict[str, Any]: row = conn.execute( "SELECT COUNT(*) AS n, COALESCE(SUM(total_paid), 0) AS total FROM fiscal_documents" ).fetchone() n = int(row["n"]) total = float(row["total"]) return {"count": n, "total": total, "avg": (total / n) if n else 0.0}