509 lines
15 KiB
Python
509 lines
15 KiB
Python
"""Acesso ao SQLite: esquema, migrações leves e helpers de consulta.
|
|
|
|
Estende o esquema original (uploaded_files -> detected_documents -> fiscal_documents)
|
|
com autenticação (users), lotes de importação (import_batches) e os campos de
|
|
legibilidade usados pela extração por IA.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sqlite3
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from typing import Any, Iterator
|
|
|
|
from .config import get_settings
|
|
|
|
|
|
SCHEMA = """
|
|
CREATE TABLE IF NOT EXISTS users (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
username TEXT NOT NULL UNIQUE,
|
|
password_hash TEXT NOT NULL,
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS import_batches (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
status TEXT NOT NULL DEFAULT 'open',
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
confirmed_at TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS uploaded_files (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
batch_id INTEGER REFERENCES import_batches(id) ON DELETE SET NULL,
|
|
original_name TEXT NOT NULL,
|
|
stored_path TEXT NOT NULL,
|
|
content_type TEXT NOT NULL,
|
|
size_bytes INTEGER NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
detected_count INTEGER NOT NULL DEFAULT 0,
|
|
message TEXT,
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS detected_documents (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
upload_id INTEGER NOT NULL REFERENCES uploaded_files(id) ON DELETE CASCADE,
|
|
batch_id INTEGER REFERENCES import_batches(id) ON DELETE SET NULL,
|
|
source_file_name TEXT NOT NULL,
|
|
source_page INTEGER,
|
|
source_location TEXT NOT NULL,
|
|
raw_text TEXT NOT NULL DEFAULT '',
|
|
purchase_date TEXT,
|
|
supplier_name TEXT,
|
|
total_paid REAL,
|
|
confidence TEXT NOT NULL DEFAULT 'low',
|
|
field_confidence_json TEXT NOT NULL DEFAULT '{}',
|
|
legible INTEGER NOT NULL DEFAULT 1,
|
|
uncertain_fields TEXT NOT NULL DEFAULT '[]',
|
|
extractor TEXT NOT NULL DEFAULT 'local',
|
|
status TEXT NOT NULL DEFAULT 'staged',
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS fiscal_documents (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
detected_document_id INTEGER REFERENCES detected_documents(id) ON DELETE SET NULL,
|
|
source_file_name TEXT NOT NULL DEFAULT '',
|
|
source_location TEXT NOT NULL DEFAULT '',
|
|
purchase_date TEXT NOT NULL,
|
|
supplier_name TEXT NOT NULL,
|
|
total_paid REAL NOT NULL,
|
|
confidence TEXT NOT NULL DEFAULT 'high',
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_fiscal_date ON fiscal_documents(purchase_date);
|
|
CREATE INDEX IF NOT EXISTS idx_detected_batch ON detected_documents(batch_id, status);
|
|
"""
|
|
|
|
|
|
def connect(db_path: Path | None = None) -> sqlite3.Connection:
|
|
settings = get_settings()
|
|
settings.ensure_storage()
|
|
conn = sqlite3.connect(db_path or settings.db_path)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA foreign_keys = ON")
|
|
conn.execute("PRAGMA journal_mode = WAL")
|
|
return conn
|
|
|
|
|
|
def init_db(conn: sqlite3.Connection) -> None:
|
|
conn.executescript(SCHEMA)
|
|
conn.commit()
|
|
|
|
|
|
@contextmanager
|
|
def session(db_path: Path | None = None) -> Iterator[sqlite3.Connection]:
|
|
conn = connect(db_path)
|
|
try:
|
|
init_db(conn)
|
|
yield conn
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Usuários
|
|
# --------------------------------------------------------------------------- #
|
|
def count_users(conn: sqlite3.Connection) -> int:
|
|
return int(conn.execute("SELECT COUNT(*) FROM users").fetchone()[0])
|
|
|
|
|
|
def get_user(conn: sqlite3.Connection, username: str) -> sqlite3.Row | None:
|
|
return conn.execute(
|
|
"SELECT * FROM users WHERE username = ?", (username,)
|
|
).fetchone()
|
|
|
|
|
|
def create_user(conn: sqlite3.Connection, username: str, password_hash: str) -> int:
|
|
cur = conn.execute(
|
|
"INSERT INTO users (username, password_hash) VALUES (?, ?)",
|
|
(username, password_hash),
|
|
)
|
|
conn.commit()
|
|
return int(cur.lastrowid)
|
|
|
|
|
|
def update_password(conn: sqlite3.Connection, username: str, password_hash: str) -> None:
|
|
conn.execute(
|
|
"UPDATE users SET password_hash = ? WHERE username = ?",
|
|
(password_hash, username),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Lotes de importação
|
|
# --------------------------------------------------------------------------- #
|
|
def create_batch(conn: sqlite3.Connection) -> int:
|
|
cur = conn.execute("INSERT INTO import_batches (status) VALUES ('open')")
|
|
conn.commit()
|
|
return int(cur.lastrowid)
|
|
|
|
|
|
def get_batch(conn: sqlite3.Connection, batch_id: int) -> sqlite3.Row | None:
|
|
return conn.execute(
|
|
"SELECT * FROM import_batches WHERE id = ?", (batch_id,)
|
|
).fetchone()
|
|
|
|
|
|
def set_batch_status(conn: sqlite3.Connection, batch_id: int, status: str) -> None:
|
|
if status == "confirmed":
|
|
conn.execute(
|
|
"UPDATE import_batches SET status = ?, confirmed_at = CURRENT_TIMESTAMP WHERE id = ?",
|
|
(status, batch_id),
|
|
)
|
|
else:
|
|
conn.execute(
|
|
"UPDATE import_batches SET status = ? WHERE id = ?", (status, batch_id)
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Uploads
|
|
# --------------------------------------------------------------------------- #
|
|
def insert_upload(
|
|
conn: sqlite3.Connection,
|
|
batch_id: int,
|
|
original_name: str,
|
|
stored_path: Path,
|
|
content_type: str,
|
|
size_bytes: int,
|
|
) -> int:
|
|
cur = conn.execute(
|
|
"""
|
|
INSERT INTO uploaded_files (batch_id, original_name, stored_path, content_type, size_bytes)
|
|
VALUES (?, ?, ?, ?, ?)
|
|
""",
|
|
(batch_id, original_name, str(stored_path), content_type, size_bytes),
|
|
)
|
|
conn.commit()
|
|
return int(cur.lastrowid)
|
|
|
|
|
|
def update_upload_status(
|
|
conn: sqlite3.Connection,
|
|
upload_id: int,
|
|
status: str,
|
|
detected_count: int,
|
|
message: str | None = None,
|
|
) -> None:
|
|
conn.execute(
|
|
"""
|
|
UPDATE uploaded_files
|
|
SET status = ?, detected_count = ?, message = ?, updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""",
|
|
(status, detected_count, message, upload_id),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def get_upload(conn: sqlite3.Connection, upload_id: int) -> sqlite3.Row | None:
|
|
return conn.execute(
|
|
"SELECT * FROM uploaded_files WHERE id = ?", (upload_id,)
|
|
).fetchone()
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Documentos detectados (staging)
|
|
# --------------------------------------------------------------------------- #
|
|
def insert_detected(
|
|
conn: sqlite3.Connection,
|
|
*,
|
|
upload_id: int,
|
|
batch_id: int,
|
|
source_file_name: str,
|
|
source_page: int | None,
|
|
source_location: str,
|
|
raw_text: str,
|
|
purchase_date: str | None,
|
|
supplier_name: str | None,
|
|
total_paid: float | None,
|
|
confidence: str,
|
|
field_confidence: dict[str, str],
|
|
legible: bool,
|
|
uncertain_fields: list[str],
|
|
extractor: str,
|
|
) -> int:
|
|
cur = conn.execute(
|
|
"""
|
|
INSERT INTO detected_documents (
|
|
upload_id, batch_id, source_file_name, source_page, source_location,
|
|
raw_text, purchase_date, supplier_name, total_paid, confidence,
|
|
field_confidence_json, legible, uncertain_fields, extractor, status
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'staged')
|
|
""",
|
|
(
|
|
upload_id,
|
|
batch_id,
|
|
source_file_name,
|
|
source_page,
|
|
source_location,
|
|
raw_text,
|
|
purchase_date,
|
|
supplier_name,
|
|
total_paid,
|
|
confidence,
|
|
json.dumps(field_confidence, ensure_ascii=False),
|
|
1 if legible else 0,
|
|
json.dumps(uncertain_fields, ensure_ascii=False),
|
|
extractor,
|
|
),
|
|
)
|
|
conn.commit()
|
|
return int(cur.lastrowid)
|
|
|
|
|
|
def get_detected(conn: sqlite3.Connection, detected_id: int) -> sqlite3.Row | None:
|
|
return conn.execute(
|
|
"SELECT * FROM detected_documents WHERE id = ?", (detected_id,)
|
|
).fetchone()
|
|
|
|
|
|
def staged_documents(conn: sqlite3.Connection, batch_id: int) -> list[sqlite3.Row]:
|
|
return list(
|
|
conn.execute(
|
|
"""
|
|
SELECT * FROM detected_documents
|
|
WHERE batch_id = ? AND status = 'staged'
|
|
ORDER BY id ASC
|
|
""",
|
|
(batch_id,),
|
|
)
|
|
)
|
|
|
|
|
|
def batch_summary(conn: sqlite3.Connection, batch_id: int) -> dict[str, Any]:
|
|
row = conn.execute(
|
|
"""
|
|
SELECT COUNT(*) AS n,
|
|
COALESCE(SUM(total_paid), 0) AS total,
|
|
SUM(CASE WHEN legible = 0 THEN 1 ELSE 0 END) AS ilegiveis,
|
|
SUM(CASE WHEN supplier_name IS NULL OR TRIM(supplier_name) = ''
|
|
OR total_paid IS NULL THEN 1 ELSE 0 END) AS incompletos
|
|
FROM detected_documents
|
|
WHERE batch_id = ? AND status = 'staged'
|
|
""",
|
|
(batch_id,),
|
|
).fetchone()
|
|
# "pendentes" = tudo que impede importar (ilegível OU faltando fornecedor/valor).
|
|
ilegiveis = int(row["ilegiveis"] or 0)
|
|
incompletos = int(row["incompletos"] or 0)
|
|
pendentes = conn.execute(
|
|
"""
|
|
SELECT COUNT(*) FROM detected_documents
|
|
WHERE batch_id = ? AND status = 'staged'
|
|
AND (legible = 0 OR supplier_name IS NULL OR TRIM(supplier_name) = '' OR total_paid IS NULL)
|
|
""",
|
|
(batch_id,),
|
|
).fetchone()[0]
|
|
return {
|
|
"count": int(row["n"]),
|
|
"total": float(row["total"]),
|
|
"ilegiveis": ilegiveis,
|
|
"incompletos": incompletos,
|
|
"pendentes": int(pendentes),
|
|
}
|
|
|
|
|
|
def update_staged(
|
|
conn: sqlite3.Connection,
|
|
detected_id: int,
|
|
*,
|
|
purchase_date: str,
|
|
supplier_name: str,
|
|
total_paid: float,
|
|
legible: bool = True,
|
|
) -> None:
|
|
conn.execute(
|
|
"""
|
|
UPDATE detected_documents
|
|
SET purchase_date = ?, supplier_name = ?, total_paid = ?,
|
|
legible = ?, updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ? AND status = 'staged'
|
|
""",
|
|
(purchase_date, supplier_name, total_paid, 1 if legible else 0, detected_id),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def discard_staged(conn: sqlite3.Connection, detected_id: int) -> None:
|
|
conn.execute(
|
|
"UPDATE detected_documents SET status = 'discarded', updated_at = CURRENT_TIMESTAMP WHERE id = ?",
|
|
(detected_id,),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def confirm_batch(conn: sqlite3.Connection, batch_id: int) -> int:
|
|
"""Promove todos os detectados 'staged' do lote para fiscal_documents."""
|
|
rows = staged_documents(conn, batch_id)
|
|
inserted = 0
|
|
for row in rows:
|
|
if not row["purchase_date"] or not row["supplier_name"] or row["total_paid"] is None:
|
|
# Sem os três campos essenciais não confirma — permanece staged.
|
|
continue
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO fiscal_documents (
|
|
detected_document_id, source_file_name, source_location,
|
|
purchase_date, supplier_name, total_paid, confidence
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
row["id"],
|
|
row["source_file_name"],
|
|
row["source_location"],
|
|
row["purchase_date"],
|
|
row["supplier_name"],
|
|
float(row["total_paid"]),
|
|
row["confidence"],
|
|
),
|
|
)
|
|
conn.execute(
|
|
"UPDATE detected_documents SET status = 'confirmed', updated_at = CURRENT_TIMESTAMP WHERE id = ?",
|
|
(row["id"],),
|
|
)
|
|
inserted += 1
|
|
set_batch_status(conn, batch_id, "confirmed")
|
|
conn.commit()
|
|
return inserted
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# CRUD de fiscal_documents
|
|
# --------------------------------------------------------------------------- #
|
|
def create_fiscal(
|
|
conn: sqlite3.Connection,
|
|
*,
|
|
purchase_date: str,
|
|
supplier_name: str,
|
|
total_paid: float,
|
|
source_file_name: str = "lançamento manual",
|
|
source_location: str = "manual",
|
|
confidence: str = "high",
|
|
detected_document_id: int | None = None,
|
|
) -> int:
|
|
cur = conn.execute(
|
|
"""
|
|
INSERT INTO fiscal_documents (
|
|
detected_document_id, source_file_name, source_location,
|
|
purchase_date, supplier_name, total_paid, confidence
|
|
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
detected_document_id,
|
|
source_file_name,
|
|
source_location,
|
|
purchase_date,
|
|
supplier_name,
|
|
total_paid,
|
|
confidence,
|
|
),
|
|
)
|
|
conn.commit()
|
|
return int(cur.lastrowid)
|
|
|
|
|
|
def get_fiscal(conn: sqlite3.Connection, doc_id: int) -> sqlite3.Row | None:
|
|
return conn.execute(
|
|
"SELECT * FROM fiscal_documents WHERE id = ?", (doc_id,)
|
|
).fetchone()
|
|
|
|
|
|
def update_fiscal(
|
|
conn: sqlite3.Connection,
|
|
doc_id: int,
|
|
*,
|
|
purchase_date: str,
|
|
supplier_name: str,
|
|
total_paid: float,
|
|
) -> None:
|
|
conn.execute(
|
|
"""
|
|
UPDATE fiscal_documents
|
|
SET purchase_date = ?, supplier_name = ?, total_paid = ?, updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""",
|
|
(purchase_date, supplier_name, total_paid, doc_id),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def delete_fiscal(conn: sqlite3.Connection, doc_id: int) -> None:
|
|
conn.execute("DELETE FROM fiscal_documents WHERE id = ?", (doc_id,))
|
|
conn.commit()
|
|
|
|
|
|
def list_fiscal(
|
|
conn: sqlite3.Connection,
|
|
*,
|
|
start: str | None = None,
|
|
end: str | None = None,
|
|
supplier: str | None = None,
|
|
) -> list[sqlite3.Row]:
|
|
clauses: list[str] = []
|
|
params: list[Any] = []
|
|
if start:
|
|
clauses.append("purchase_date >= ?")
|
|
params.append(start)
|
|
if end:
|
|
clauses.append("purchase_date <= ?")
|
|
params.append(end)
|
|
if supplier:
|
|
clauses.append("supplier_name LIKE ?")
|
|
params.append(f"%{supplier}%")
|
|
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
|
|
return list(
|
|
conn.execute(
|
|
f"SELECT * FROM fiscal_documents {where} ORDER BY purchase_date DESC, id DESC",
|
|
params,
|
|
)
|
|
)
|
|
|
|
|
|
def monthly_totals(conn: sqlite3.Connection) -> list[sqlite3.Row]:
|
|
return list(
|
|
conn.execute(
|
|
"""
|
|
SELECT substr(purchase_date, 1, 7) AS month, SUM(total_paid) AS total, COUNT(*) AS count
|
|
FROM fiscal_documents
|
|
GROUP BY substr(purchase_date, 1, 7)
|
|
ORDER BY month ASC
|
|
"""
|
|
)
|
|
)
|
|
|
|
|
|
def supplier_totals(conn: sqlite3.Connection, limit: int = 10) -> list[sqlite3.Row]:
|
|
return list(
|
|
conn.execute(
|
|
"""
|
|
SELECT supplier_name, SUM(total_paid) AS total, COUNT(*) AS count
|
|
FROM fiscal_documents
|
|
GROUP BY supplier_name
|
|
ORDER BY total DESC
|
|
LIMIT ?
|
|
""",
|
|
(limit,),
|
|
)
|
|
)
|
|
|
|
|
|
def overall_totals(conn: sqlite3.Connection) -> dict[str, Any]:
|
|
row = conn.execute(
|
|
"SELECT COUNT(*) AS n, COALESCE(SUM(total_paid), 0) AS total FROM fiscal_documents"
|
|
).fetchone()
|
|
n = int(row["n"])
|
|
total = float(row["total"])
|
|
return {"count": n, "total": total, "avg": (total / n) if n else 0.0}
|