885 lines
30 KiB
Python
885 lines
30 KiB
Python
"""Acesso ao SQLite: esquema, migrações leves e helpers de consulta.
|
|
|
|
Estende o esquema original (uploaded_files -> detected_documents -> fiscal_documents)
|
|
com autenticação (users), lotes de importação (import_batches) e os campos de
|
|
legibilidade usados pela extração por IA.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sqlite3
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from typing import Any, Iterator
|
|
|
|
from .config import get_settings
|
|
|
|
|
|
_CATEGORIA_SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS categoria (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
categoria TEXT NOT NULL,
|
|
palavra_chave TEXT NOT NULL
|
|
);
|
|
"""
|
|
|
|
SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS users (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
username TEXT NOT NULL UNIQUE,
|
|
password_hash TEXT NOT NULL,
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS import_batches (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
status TEXT NOT NULL DEFAULT 'open',
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
confirmed_at TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS uploaded_files (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
batch_id INTEGER REFERENCES import_batches(id) ON DELETE SET NULL,
|
|
original_name TEXT NOT NULL,
|
|
stored_path TEXT NOT NULL,
|
|
content_type TEXT NOT NULL,
|
|
size_bytes INTEGER NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
detected_count INTEGER NOT NULL DEFAULT 0,
|
|
message TEXT,
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS detected_documents (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
upload_id INTEGER NOT NULL REFERENCES uploaded_files(id) ON DELETE CASCADE,
|
|
batch_id INTEGER REFERENCES import_batches(id) ON DELETE SET NULL,
|
|
source_file_name TEXT NOT NULL,
|
|
source_page INTEGER,
|
|
source_location TEXT NOT NULL,
|
|
raw_text TEXT NOT NULL DEFAULT '',
|
|
mes INTEGER,
|
|
ano INTEGER,
|
|
supplier_name TEXT,
|
|
total_paid REAL,
|
|
confidence TEXT NOT NULL DEFAULT 'low',
|
|
field_confidence_json TEXT NOT NULL DEFAULT '{}',
|
|
legible INTEGER NOT NULL DEFAULT 1,
|
|
uncertain_fields TEXT NOT NULL DEFAULT '[]',
|
|
extractor TEXT NOT NULL DEFAULT 'local',
|
|
status TEXT NOT NULL DEFAULT 'staged',
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
__CATEGORIA_PLACEHOLDER__
|
|
CREATE TABLE IF NOT EXISTS fiscal_documents (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
detected_document_id INTEGER REFERENCES detected_documents(id) ON DELETE SET NULL,
|
|
source_file_name TEXT NOT NULL DEFAULT '',
|
|
source_location TEXT NOT NULL DEFAULT '',
|
|
mes INTEGER NOT NULL,
|
|
ano INTEGER NOT NULL,
|
|
supplier_name TEXT NOT NULL,
|
|
total_paid REAL NOT NULL,
|
|
confidence TEXT NOT NULL DEFAULT 'high',
|
|
categoria_id INTEGER REFERENCES categoria(id),
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_fiscal_competencia ON fiscal_documents(ano, mes);
|
|
CREATE INDEX IF NOT EXISTS idx_detected_batch ON detected_documents(batch_id, status);
|
|
""".replace("__CATEGORIA_PLACEHOLDER__", _CATEGORIA_SCHEMA)
|
|
|
|
_RESERVED_CATEGORIA_SEED = (
|
|
"INSERT OR IGNORE INTO categoria (id, categoria, palavra_chave) VALUES (1, 'Não Encontrado', '')"
|
|
)
|
|
|
|
|
|
def connect(db_path: Path | None = None) -> sqlite3.Connection:
|
|
settings = get_settings()
|
|
settings.ensure_storage()
|
|
conn = sqlite3.connect(db_path or settings.db_path)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA foreign_keys = ON")
|
|
conn.execute("PRAGMA journal_mode = WAL")
|
|
return conn
|
|
|
|
|
|
def _table_columns(conn: sqlite3.Connection, table: str) -> set[str]:
|
|
return {row["name"] for row in conn.execute(f"PRAGMA table_info({table})")}
|
|
|
|
|
|
def _rebuild_fiscal_documents(conn: sqlite3.Connection, old_columns: set[str]) -> None:
|
|
"""Recria `fiscal_documents` com `mes`/`ano` no lugar de `purchase_date`.
|
|
|
|
SQLite não suporta `DROP COLUMN` em todas as versões-alvo, então o rebuild
|
|
(tabela nova + `INSERT ... SELECT` + `DROP` + `RENAME`) é a técnica
|
|
portável recomendada pela própria documentação do SQLite.
|
|
"""
|
|
categoria_expr = "categoria_id" if "categoria_id" in old_columns else "NULL"
|
|
conn.execute("DROP TABLE IF EXISTS fiscal_documents_new")
|
|
conn.execute(
|
|
"""
|
|
CREATE TABLE fiscal_documents_new (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
detected_document_id INTEGER REFERENCES detected_documents(id) ON DELETE SET NULL,
|
|
source_file_name TEXT NOT NULL DEFAULT '',
|
|
source_location TEXT NOT NULL DEFAULT '',
|
|
mes INTEGER NOT NULL,
|
|
ano INTEGER NOT NULL,
|
|
supplier_name TEXT NOT NULL,
|
|
total_paid REAL NOT NULL,
|
|
confidence TEXT NOT NULL DEFAULT 'high',
|
|
categoria_id INTEGER REFERENCES categoria(id),
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
"""
|
|
)
|
|
conn.execute(
|
|
f"""
|
|
INSERT INTO fiscal_documents_new (
|
|
id, detected_document_id, source_file_name, source_location,
|
|
mes, ano, supplier_name, total_paid, confidence, categoria_id,
|
|
created_at, updated_at
|
|
)
|
|
SELECT id, detected_document_id, source_file_name, source_location,
|
|
CAST(substr(purchase_date, 6, 2) AS INTEGER),
|
|
CAST(substr(purchase_date, 1, 4) AS INTEGER),
|
|
supplier_name, total_paid, confidence, {categoria_expr},
|
|
created_at, updated_at
|
|
FROM fiscal_documents
|
|
"""
|
|
)
|
|
conn.execute("DROP TABLE fiscal_documents")
|
|
conn.execute("ALTER TABLE fiscal_documents_new RENAME TO fiscal_documents")
|
|
|
|
|
|
def _rebuild_detected_documents(conn: sqlite3.Connection) -> None:
|
|
"""Mesmo rebuild que `_rebuild_fiscal_documents`, mas `mes`/`ano` ficam
|
|
nullable — mesmo comportamento opcional que `purchase_date` tinha."""
|
|
conn.execute("DROP TABLE IF EXISTS detected_documents_new")
|
|
conn.execute(
|
|
"""
|
|
CREATE TABLE detected_documents_new (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
upload_id INTEGER NOT NULL REFERENCES uploaded_files(id) ON DELETE CASCADE,
|
|
batch_id INTEGER REFERENCES import_batches(id) ON DELETE SET NULL,
|
|
source_file_name TEXT NOT NULL,
|
|
source_page INTEGER,
|
|
source_location TEXT NOT NULL,
|
|
raw_text TEXT NOT NULL DEFAULT '',
|
|
mes INTEGER,
|
|
ano INTEGER,
|
|
supplier_name TEXT,
|
|
total_paid REAL,
|
|
confidence TEXT NOT NULL DEFAULT 'low',
|
|
field_confidence_json TEXT NOT NULL DEFAULT '{}',
|
|
legible INTEGER NOT NULL DEFAULT 1,
|
|
uncertain_fields TEXT NOT NULL DEFAULT '[]',
|
|
extractor TEXT NOT NULL DEFAULT 'local',
|
|
status TEXT NOT NULL DEFAULT 'staged',
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
"""
|
|
)
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO detected_documents_new (
|
|
id, upload_id, batch_id, source_file_name, source_page, source_location,
|
|
raw_text, mes, ano, supplier_name, total_paid, confidence,
|
|
field_confidence_json, legible, uncertain_fields, extractor, status,
|
|
created_at, updated_at
|
|
)
|
|
SELECT id, upload_id, batch_id, source_file_name, source_page, source_location,
|
|
raw_text,
|
|
CASE WHEN purchase_date IS NOT NULL THEN CAST(substr(purchase_date, 6, 2) AS INTEGER) END,
|
|
CASE WHEN purchase_date IS NOT NULL THEN CAST(substr(purchase_date, 1, 4) AS INTEGER) END,
|
|
supplier_name, total_paid, confidence,
|
|
field_confidence_json, legible, uncertain_fields, extractor, status,
|
|
created_at, updated_at
|
|
FROM detected_documents
|
|
"""
|
|
)
|
|
conn.execute("DROP TABLE detected_documents")
|
|
conn.execute("ALTER TABLE detected_documents_new RENAME TO detected_documents")
|
|
|
|
|
|
def _migrate_competencia(conn: sqlite3.Connection) -> None:
|
|
"""Migra `fiscal_documents`/`detected_documents` de `purchase_date` para
|
|
`mes`/`ano`, uma única vez. Guardado por `PRAGMA table_info`: só roda se
|
|
`purchase_date` ainda existir (tabela de instalação anterior a esta
|
|
mudança); em uma instalação nova, ou já migrada, é um no-op."""
|
|
fiscal_columns = _table_columns(conn, "fiscal_documents")
|
|
detected_columns = _table_columns(conn, "detected_documents")
|
|
needs_fiscal = "purchase_date" in fiscal_columns
|
|
needs_detected = "purchase_date" in detected_columns
|
|
if not needs_fiscal and not needs_detected:
|
|
return
|
|
conn.execute("BEGIN")
|
|
try:
|
|
if needs_fiscal:
|
|
_rebuild_fiscal_documents(conn, fiscal_columns)
|
|
if needs_detected:
|
|
_rebuild_detected_documents(conn)
|
|
except Exception:
|
|
conn.rollback()
|
|
raise
|
|
else:
|
|
conn.commit()
|
|
|
|
|
|
def init_db(conn: sqlite3.Connection) -> None:
|
|
# `categoria` precisa existir antes da migração: `_rebuild_fiscal_documents`
|
|
# recria `fiscal_documents` com `categoria_id REFERENCES categoria(id)`, e com
|
|
# `PRAGMA foreign_keys = ON` o INSERT falha com "no such table" se a tabela
|
|
# referenciada ainda não existir num banco pré-existente sem ela.
|
|
conn.execute(_CATEGORIA_SCHEMA)
|
|
_migrate_competencia(conn)
|
|
conn.executescript(SCHEMA)
|
|
columns = {row["name"] for row in conn.execute("PRAGMA table_info(fiscal_documents)")}
|
|
if "categoria_id" not in columns:
|
|
conn.execute("ALTER TABLE fiscal_documents ADD COLUMN categoria_id INTEGER REFERENCES categoria(id)")
|
|
conn.execute(_RESERVED_CATEGORIA_SEED)
|
|
conn.commit()
|
|
|
|
|
|
@contextmanager
|
|
def session(db_path: Path | None = None) -> Iterator[sqlite3.Connection]:
|
|
conn = connect(db_path)
|
|
try:
|
|
init_db(conn)
|
|
yield conn
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Usuários
|
|
# --------------------------------------------------------------------------- #
|
|
def count_users(conn: sqlite3.Connection) -> int:
|
|
return int(conn.execute("SELECT COUNT(*) FROM users").fetchone()[0])
|
|
|
|
|
|
def get_user(conn: sqlite3.Connection, username: str) -> sqlite3.Row | None:
|
|
return conn.execute(
|
|
"SELECT * FROM users WHERE username = ?", (username,)
|
|
).fetchone()
|
|
|
|
|
|
def create_user(conn: sqlite3.Connection, username: str, password_hash: str) -> int:
|
|
cur = conn.execute(
|
|
"INSERT INTO users (username, password_hash) VALUES (?, ?)",
|
|
(username, password_hash),
|
|
)
|
|
conn.commit()
|
|
return int(cur.lastrowid)
|
|
|
|
|
|
def update_password(conn: sqlite3.Connection, username: str, password_hash: str) -> None:
|
|
conn.execute(
|
|
"UPDATE users SET password_hash = ? WHERE username = ?",
|
|
(password_hash, username),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Lotes de importação
|
|
# --------------------------------------------------------------------------- #
|
|
def create_batch(conn: sqlite3.Connection) -> int:
|
|
cur = conn.execute("INSERT INTO import_batches (status) VALUES ('open')")
|
|
conn.commit()
|
|
return int(cur.lastrowid)
|
|
|
|
|
|
def get_batch(conn: sqlite3.Connection, batch_id: int) -> sqlite3.Row | None:
|
|
return conn.execute(
|
|
"SELECT * FROM import_batches WHERE id = ?", (batch_id,)
|
|
).fetchone()
|
|
|
|
|
|
def set_batch_status(conn: sqlite3.Connection, batch_id: int, status: str) -> None:
|
|
if status == "confirmed":
|
|
conn.execute(
|
|
"UPDATE import_batches SET status = ?, confirmed_at = CURRENT_TIMESTAMP WHERE id = ?",
|
|
(status, batch_id),
|
|
)
|
|
else:
|
|
conn.execute(
|
|
"UPDATE import_batches SET status = ? WHERE id = ?", (status, batch_id)
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Uploads
|
|
# --------------------------------------------------------------------------- #
|
|
def insert_upload(
|
|
conn: sqlite3.Connection,
|
|
batch_id: int,
|
|
original_name: str,
|
|
stored_path: Path,
|
|
content_type: str,
|
|
size_bytes: int,
|
|
*,
|
|
commit: bool = True,
|
|
) -> int:
|
|
cur = conn.execute(
|
|
"""
|
|
INSERT INTO uploaded_files (batch_id, original_name, stored_path, content_type, size_bytes)
|
|
VALUES (?, ?, ?, ?, ?)
|
|
""",
|
|
(batch_id, original_name, str(stored_path), content_type, size_bytes),
|
|
)
|
|
if commit:
|
|
conn.commit()
|
|
return int(cur.lastrowid)
|
|
|
|
|
|
def update_upload_status(
|
|
conn: sqlite3.Connection,
|
|
upload_id: int,
|
|
status: str,
|
|
detected_count: int,
|
|
message: str | None = None,
|
|
*,
|
|
commit: bool = True,
|
|
) -> None:
|
|
conn.execute(
|
|
"""
|
|
UPDATE uploaded_files
|
|
SET status = ?, detected_count = ?, message = ?, updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""",
|
|
(status, detected_count, message, upload_id),
|
|
)
|
|
if commit:
|
|
conn.commit()
|
|
|
|
|
|
def get_upload(conn: sqlite3.Connection, upload_id: int) -> sqlite3.Row | None:
|
|
return conn.execute(
|
|
"SELECT * FROM uploaded_files WHERE id = ?", (upload_id,)
|
|
).fetchone()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Documentos detectados (staging)
|
|
# --------------------------------------------------------------------------- #
|
|
def insert_detected(
|
|
conn: sqlite3.Connection,
|
|
*,
|
|
upload_id: int,
|
|
batch_id: int,
|
|
source_file_name: str,
|
|
source_page: int | None,
|
|
source_location: str,
|
|
raw_text: str,
|
|
mes: int | None,
|
|
ano: int | None,
|
|
supplier_name: str | None,
|
|
total_paid: float | None,
|
|
confidence: str,
|
|
field_confidence: dict[str, str],
|
|
legible: bool,
|
|
uncertain_fields: list[str],
|
|
extractor: str,
|
|
commit: bool = True,
|
|
) -> int:
|
|
cur = conn.execute(
|
|
"""
|
|
INSERT INTO detected_documents (
|
|
upload_id, batch_id, source_file_name, source_page, source_location,
|
|
raw_text, mes, ano, supplier_name, total_paid, confidence,
|
|
field_confidence_json, legible, uncertain_fields, extractor, status
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'staged')
|
|
""",
|
|
(
|
|
upload_id,
|
|
batch_id,
|
|
source_file_name,
|
|
source_page,
|
|
source_location,
|
|
raw_text,
|
|
mes,
|
|
ano,
|
|
supplier_name,
|
|
total_paid,
|
|
confidence,
|
|
json.dumps(field_confidence, ensure_ascii=False),
|
|
1 if legible else 0,
|
|
json.dumps(uncertain_fields, ensure_ascii=False),
|
|
extractor,
|
|
),
|
|
)
|
|
if commit:
|
|
conn.commit()
|
|
return int(cur.lastrowid)
|
|
|
|
|
|
def get_detected(conn: sqlite3.Connection, detected_id: int) -> sqlite3.Row | None:
|
|
return conn.execute(
|
|
"SELECT * FROM detected_documents WHERE id = ?", (detected_id,)
|
|
).fetchone()
|
|
|
|
|
|
def staged_documents(conn: sqlite3.Connection, batch_id: int) -> list[sqlite3.Row]:
|
|
return list(
|
|
conn.execute(
|
|
"""
|
|
SELECT * FROM detected_documents
|
|
WHERE batch_id = ? AND status = 'staged'
|
|
ORDER BY id ASC
|
|
""",
|
|
(batch_id,),
|
|
)
|
|
)
|
|
|
|
|
|
def batch_summary(conn: sqlite3.Connection, batch_id: int) -> dict[str, Any]:
|
|
row = conn.execute(
|
|
"""
|
|
SELECT COUNT(*) AS n,
|
|
COALESCE(SUM(total_paid), 0) AS total,
|
|
SUM(CASE WHEN legible = 0 THEN 1 ELSE 0 END) AS ilegiveis,
|
|
SUM(CASE WHEN supplier_name IS NULL OR TRIM(supplier_name) = ''
|
|
OR total_paid IS NULL OR mes IS NULL OR ano IS NULL THEN 1 ELSE 0 END) AS incompletos
|
|
FROM detected_documents
|
|
WHERE batch_id = ? AND status = 'staged'
|
|
""",
|
|
(batch_id,),
|
|
).fetchone()
|
|
# "pendentes" = tudo que impede importar (ilegível OU faltando fornecedor/valor/competência).
|
|
ilegiveis = int(row["ilegiveis"] or 0)
|
|
incompletos = int(row["incompletos"] or 0)
|
|
pendentes = conn.execute(
|
|
"""
|
|
SELECT COUNT(*) FROM detected_documents
|
|
WHERE batch_id = ? AND status = 'staged'
|
|
AND (legible = 0 OR supplier_name IS NULL OR TRIM(supplier_name) = '' OR total_paid IS NULL
|
|
OR mes IS NULL OR ano IS NULL)
|
|
""",
|
|
(batch_id,),
|
|
).fetchone()[0]
|
|
return {
|
|
"count": int(row["n"]),
|
|
"total": float(row["total"]),
|
|
"ilegiveis": ilegiveis,
|
|
"incompletos": incompletos,
|
|
"pendentes": int(pendentes),
|
|
}
|
|
|
|
|
|
def update_staged(
|
|
conn: sqlite3.Connection,
|
|
detected_id: int,
|
|
*,
|
|
mes: int | None,
|
|
ano: int | None,
|
|
supplier_name: str,
|
|
total_paid: float,
|
|
legible: bool = True,
|
|
) -> None:
|
|
conn.execute(
|
|
"""
|
|
UPDATE detected_documents
|
|
SET mes = ?, ano = ?, supplier_name = ?, total_paid = ?,
|
|
legible = ?, updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ? AND status = 'staged'
|
|
""",
|
|
(mes, ano, supplier_name, total_paid, 1 if legible else 0, detected_id),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def discard_staged(conn: sqlite3.Connection, detected_id: int) -> None:
|
|
conn.execute(
|
|
"UPDATE detected_documents SET status = 'discarded', updated_at = CURRENT_TIMESTAMP WHERE id = ?",
|
|
(detected_id,),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def confirm_batch(conn: sqlite3.Connection, batch_id: int) -> int:
|
|
"""Promove todos os detectados 'staged' do lote para fiscal_documents."""
|
|
# Import local para evitar ciclo: categorization.py importa este módulo no topo.
|
|
from .categorization import categorize_supplier
|
|
|
|
rows = staged_documents(conn, batch_id)
|
|
inserted = 0
|
|
for row in rows:
|
|
mes, ano = row["mes"], row["ano"]
|
|
if (
|
|
mes is None or ano is None or not (1 <= mes <= 12)
|
|
or not row["supplier_name"] or row["total_paid"] is None
|
|
):
|
|
# Sem os campos essenciais (competência, fornecedor, valor) não confirma — permanece staged.
|
|
continue
|
|
categoria_id = categorize_supplier(row["supplier_name"], conn, batch_id)
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO fiscal_documents (
|
|
detected_document_id, source_file_name, source_location,
|
|
mes, ano, supplier_name, total_paid, confidence, categoria_id
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
row["id"],
|
|
row["source_file_name"],
|
|
row["source_location"],
|
|
mes,
|
|
ano,
|
|
row["supplier_name"],
|
|
float(row["total_paid"]),
|
|
row["confidence"],
|
|
categoria_id,
|
|
),
|
|
)
|
|
conn.execute(
|
|
"UPDATE detected_documents SET status = 'confirmed', updated_at = CURRENT_TIMESTAMP WHERE id = ?",
|
|
(row["id"],),
|
|
)
|
|
inserted += 1
|
|
set_batch_status(conn, batch_id, "confirmed")
|
|
conn.commit()
|
|
return inserted
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# CRUD de fiscal_documents
|
|
# --------------------------------------------------------------------------- #
|
|
def create_fiscal(
|
|
conn: sqlite3.Connection,
|
|
*,
|
|
mes: int,
|
|
ano: int,
|
|
supplier_name: str,
|
|
total_paid: float,
|
|
source_file_name: str = "lançamento manual",
|
|
source_location: str = "manual",
|
|
confidence: str = "high",
|
|
detected_document_id: int | None = None,
|
|
categoria_id: int | None = None,
|
|
) -> int:
|
|
cur = conn.execute(
|
|
"""
|
|
INSERT INTO fiscal_documents (
|
|
detected_document_id, source_file_name, source_location,
|
|
mes, ano, supplier_name, total_paid, confidence, categoria_id
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
detected_document_id,
|
|
source_file_name,
|
|
source_location,
|
|
mes,
|
|
ano,
|
|
supplier_name,
|
|
total_paid,
|
|
confidence,
|
|
categoria_id,
|
|
),
|
|
)
|
|
conn.commit()
|
|
return int(cur.lastrowid)
|
|
|
|
|
|
def get_fiscal(conn: sqlite3.Connection, doc_id: int) -> sqlite3.Row | None:
|
|
return conn.execute(
|
|
"SELECT * FROM fiscal_documents WHERE id = ?", (doc_id,)
|
|
).fetchone()
|
|
|
|
|
|
def update_fiscal(
|
|
conn: sqlite3.Connection,
|
|
doc_id: int,
|
|
*,
|
|
mes: int,
|
|
ano: int,
|
|
supplier_name: str,
|
|
total_paid: float,
|
|
categoria_id: int | None = None,
|
|
) -> None:
|
|
conn.execute(
|
|
"""
|
|
UPDATE fiscal_documents
|
|
SET mes = ?, ano = ?, supplier_name = ?, total_paid = ?, categoria_id = ?, updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""",
|
|
(mes, ano, supplier_name, total_paid, categoria_id, doc_id),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def delete_fiscal(conn: sqlite3.Connection, doc_id: int) -> None:
|
|
conn.execute("DELETE FROM fiscal_documents WHERE id = ?", (doc_id,))
|
|
conn.commit()
|
|
|
|
|
|
def _category_clause(category: str | None) -> tuple[str | None, Any]:
|
|
"""Traduz o filtro de categoria (id, 'none' para Sem categoria, ou None/''
|
|
para nenhum filtro) numa clausula SQL + parâmetro."""
|
|
if not category:
|
|
return None, None
|
|
if category == "none":
|
|
return "categoria_id IS NULL", None
|
|
try:
|
|
return "categoria_id = ?", int(category)
|
|
except ValueError:
|
|
return None, None
|
|
|
|
|
|
def _fiscal_where(
|
|
*,
|
|
start: tuple[int, int] | None = None,
|
|
end: tuple[int, int] | None = None,
|
|
supplier: str | None = None,
|
|
category: str | None = None,
|
|
) -> tuple[str, list[Any]]:
|
|
"""`start`/`end` são tuplas `(mes, ano)` delimitando o intervalo de
|
|
competência (inclusive), comparadas via a chave `ano * 12 + mes`."""
|
|
clauses: list[str] = []
|
|
params: list[Any] = []
|
|
if start:
|
|
start_mes, start_ano = start
|
|
clauses.append("(ano * 12 + mes) >= ?")
|
|
params.append(start_ano * 12 + start_mes)
|
|
if end:
|
|
end_mes, end_ano = end
|
|
clauses.append("(ano * 12 + mes) <= ?")
|
|
params.append(end_ano * 12 + end_mes)
|
|
if supplier:
|
|
clauses.append("supplier_name LIKE ?")
|
|
params.append(f"%{supplier}%")
|
|
cat_clause, cat_param = _category_clause(category)
|
|
if cat_clause:
|
|
clauses.append(cat_clause)
|
|
if cat_param is not None:
|
|
params.append(cat_param)
|
|
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
|
|
return where, params
|
|
|
|
|
|
FISCAL_SORT_COLUMNS = {
|
|
"competencia": "f.ano, f.mes",
|
|
"supplier_name": "f.supplier_name COLLATE NOCASE",
|
|
"categoria": "categoria_nome COLLATE NOCASE",
|
|
"total_paid": "f.total_paid",
|
|
}
|
|
|
|
|
|
def list_fiscal(
|
|
conn: sqlite3.Connection,
|
|
*,
|
|
start: tuple[int, int] | None = None,
|
|
end: tuple[int, int] | None = None,
|
|
supplier: str | None = None,
|
|
category: str | None = None,
|
|
sort: str | None = None,
|
|
order: str = "desc",
|
|
limit: int | None = None,
|
|
offset: int = 0,
|
|
) -> list[sqlite3.Row]:
|
|
where, params = _fiscal_where(start=start, end=end, supplier=supplier, category=category)
|
|
limit_sql = ""
|
|
if limit is not None:
|
|
limit_sql = "LIMIT ? OFFSET ?"
|
|
params = [*params, limit, offset]
|
|
sort_col = FISCAL_SORT_COLUMNS.get(sort or "competencia", FISCAL_SORT_COLUMNS["competencia"])
|
|
direction = "ASC" if (order or "").lower() == "asc" else "DESC"
|
|
order_sql = ", ".join(f"{col} {direction}" for col in sort_col.split(", ")) + f", f.id {direction}"
|
|
if sort_col != FISCAL_SORT_COLUMNS["competencia"]:
|
|
order_sql += ", f.ano DESC, f.mes DESC"
|
|
return list(
|
|
conn.execute(
|
|
f"""
|
|
SELECT f.*, c.categoria AS categoria_nome
|
|
FROM fiscal_documents f
|
|
LEFT JOIN categoria c ON c.id = f.categoria_id
|
|
{where}
|
|
ORDER BY {order_sql}
|
|
{limit_sql}
|
|
""",
|
|
params,
|
|
)
|
|
)
|
|
|
|
|
|
def fiscal_summary(
|
|
conn: sqlite3.Connection,
|
|
*,
|
|
start: tuple[int, int] | None = None,
|
|
end: tuple[int, int] | None = None,
|
|
supplier: str | None = None,
|
|
category: str | None = None,
|
|
) -> tuple[int, float]:
|
|
"""Retorna (quantidade, soma de total_paid) para o filtro informado, sem paginação."""
|
|
where, params = _fiscal_where(start=start, end=end, supplier=supplier, category=category)
|
|
row = conn.execute(
|
|
f"SELECT COUNT(*) AS n, COALESCE(SUM(total_paid), 0) AS total FROM fiscal_documents {where}",
|
|
params,
|
|
).fetchone()
|
|
return row["n"], row["total"]
|
|
|
|
|
|
def monthly_totals(conn: sqlite3.Connection, category: str | None = None) -> list[sqlite3.Row]:
|
|
cat_clause, cat_param = _category_clause(category)
|
|
where = f"WHERE {cat_clause}" if cat_clause else ""
|
|
params = [cat_param] if cat_param is not None else []
|
|
return list(
|
|
conn.execute(
|
|
f"""
|
|
SELECT ano, mes, SUM(total_paid) AS total, COUNT(*) AS count
|
|
FROM fiscal_documents
|
|
{where}
|
|
GROUP BY ano, mes
|
|
ORDER BY ano ASC, mes ASC
|
|
""",
|
|
params,
|
|
)
|
|
)
|
|
|
|
|
|
def supplier_totals(conn: sqlite3.Connection, limit: int = 10, category: str | None = None) -> list[sqlite3.Row]:
|
|
cat_clause, cat_param = _category_clause(category)
|
|
where = f"WHERE {cat_clause}" if cat_clause else ""
|
|
params: list[Any] = [cat_param] if cat_param is not None else []
|
|
params.append(limit)
|
|
return list(
|
|
conn.execute(
|
|
f"""
|
|
SELECT supplier_name, SUM(total_paid) AS total, COUNT(*) AS count
|
|
FROM fiscal_documents
|
|
{where}
|
|
GROUP BY supplier_name
|
|
ORDER BY total DESC
|
|
LIMIT ?
|
|
""",
|
|
params,
|
|
)
|
|
)
|
|
|
|
|
|
def category_totals(
|
|
conn: sqlite3.Connection,
|
|
start: tuple[int, int] | None = None,
|
|
end: tuple[int, int] | None = None,
|
|
) -> list[sqlite3.Row]:
|
|
"""Totais agrupados por categoria (LEFT JOIN, inclui 'Não Encontrado'), mais
|
|
um grupo 'Sem categoria' à parte para categoria_id IS NULL (legado).
|
|
|
|
`start`/`end` são tuplas `(mes, ano)` delimitando o intervalo de competência.
|
|
"""
|
|
clauses: list[str] = []
|
|
params: list[Any] = []
|
|
if start:
|
|
start_mes, start_ano = start
|
|
clauses.append("(f.ano * 12 + f.mes) >= ?")
|
|
params.append(start_ano * 12 + start_mes)
|
|
if end:
|
|
end_mes, end_ano = end
|
|
clauses.append("(f.ano * 12 + f.mes) <= ?")
|
|
params.append(end_ano * 12 + end_mes)
|
|
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
|
|
return list(
|
|
conn.execute(
|
|
f"""
|
|
SELECT COALESCE(c.categoria, 'Sem categoria') AS categoria,
|
|
f.categoria_id AS categoria_id,
|
|
SUM(f.total_paid) AS total, COUNT(*) AS count
|
|
FROM fiscal_documents f
|
|
LEFT JOIN categoria c ON c.id = f.categoria_id
|
|
{where}
|
|
GROUP BY COALESCE(c.categoria, 'Sem categoria'), f.categoria_id
|
|
ORDER BY total DESC
|
|
""",
|
|
params,
|
|
)
|
|
)
|
|
|
|
|
|
def overall_totals(conn: sqlite3.Connection, category: str | None = None) -> dict[str, Any]:
|
|
cat_clause, cat_param = _category_clause(category)
|
|
where = f"WHERE {cat_clause}" if cat_clause else ""
|
|
params = [cat_param] if cat_param is not None else []
|
|
row = conn.execute(
|
|
f"SELECT COUNT(*) AS n, COALESCE(SUM(total_paid), 0) AS total FROM fiscal_documents {where}",
|
|
params,
|
|
).fetchone()
|
|
n = int(row["n"])
|
|
total = float(row["total"])
|
|
return {"count": n, "total": total, "avg": (total / n) if n else 0.0}
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# CRUD de categoria
|
|
# --------------------------------------------------------------------------- #
|
|
RESERVED_CATEGORIA_ID = 1
|
|
|
|
|
|
def create_categoria(conn: sqlite3.Connection, categoria: str, palavra_chave: str) -> int:
|
|
cur = conn.execute(
|
|
"INSERT INTO categoria (categoria, palavra_chave) VALUES (?, ?)",
|
|
(categoria, palavra_chave),
|
|
)
|
|
conn.commit()
|
|
return int(cur.lastrowid)
|
|
|
|
|
|
def get_categoria(conn: sqlite3.Connection, categoria_id: int) -> sqlite3.Row | None:
|
|
return conn.execute(
|
|
"SELECT * FROM categoria WHERE id = ?", (categoria_id,)
|
|
).fetchone()
|
|
|
|
|
|
def list_categorias(conn: sqlite3.Connection) -> list[sqlite3.Row]:
|
|
return list(conn.execute("SELECT * FROM categoria ORDER BY id ASC"))
|
|
|
|
|
|
def list_categorias_por_nome(conn: sqlite3.Connection) -> list[sqlite3.Row]:
|
|
return list(conn.execute("SELECT * FROM categoria ORDER BY categoria ASC"))
|
|
|
|
|
|
def update_categoria(conn: sqlite3.Connection, categoria_id: int, categoria: str, palavra_chave: str) -> None:
|
|
conn.execute(
|
|
"UPDATE categoria SET categoria = ?, palavra_chave = ? WHERE id = ?",
|
|
(categoria, palavra_chave, categoria_id),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def delete_categoria(conn: sqlite3.Connection, categoria_id: int) -> bool:
|
|
"""Exclui uma categoria, reatribuindo documentos referenciados para a
|
|
categoria reservada (id=1). Rejeita a exclusão da própria linha reservada."""
|
|
if categoria_id == RESERVED_CATEGORIA_ID:
|
|
return False
|
|
conn.execute(
|
|
"UPDATE fiscal_documents SET categoria_id = ? WHERE categoria_id = ?",
|
|
(RESERVED_CATEGORIA_ID, categoria_id),
|
|
)
|
|
conn.execute("DELETE FROM categoria WHERE id = ?", (categoria_id,))
|
|
conn.commit()
|
|
return True
|
|
|
|
|
|
def find_categoria_by_keyword(conn: sqlite3.Connection, supplier_name: str) -> sqlite3.Row | None:
|
|
"""Primeira categoria (id ASC, excluindo a reservada id=1) cuja palavra_chave
|
|
é substring case-insensitive de supplier_name."""
|
|
supplier = (supplier_name or "").upper()
|
|
rows = conn.execute(
|
|
"SELECT * FROM categoria WHERE id != ? ORDER BY id ASC",
|
|
(RESERVED_CATEGORIA_ID,),
|
|
)
|
|
for row in rows:
|
|
keyword = (row["palavra_chave"] or "").strip()
|
|
if keyword and keyword.upper() in supplier:
|
|
return row
|
|
return None
|