Commit inicial - upload de todos os arquivos da pasta
This commit is contained in:
+508
@@ -0,0 +1,508 @@
|
||||
"""Acesso ao SQLite: esquema, migrações leves e helpers de consulta.
|
||||
|
||||
Estende o esquema original (uploaded_files -> detected_documents -> fiscal_documents)
|
||||
com autenticação (users), lotes de importação (import_batches) e os campos de
|
||||
legibilidade usados pela extração por IA.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator
|
||||
|
||||
from .config import get_settings
|
||||
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS import_batches (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
status TEXT NOT NULL DEFAULT 'open',
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
confirmed_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS uploaded_files (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
batch_id INTEGER REFERENCES import_batches(id) ON DELETE SET NULL,
|
||||
original_name TEXT NOT NULL,
|
||||
stored_path TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes INTEGER NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
detected_count INTEGER NOT NULL DEFAULT 0,
|
||||
message TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS detected_documents (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
upload_id INTEGER NOT NULL REFERENCES uploaded_files(id) ON DELETE CASCADE,
|
||||
batch_id INTEGER REFERENCES import_batches(id) ON DELETE SET NULL,
|
||||
source_file_name TEXT NOT NULL,
|
||||
source_page INTEGER,
|
||||
source_location TEXT NOT NULL,
|
||||
raw_text TEXT NOT NULL DEFAULT '',
|
||||
purchase_date TEXT,
|
||||
supplier_name TEXT,
|
||||
total_paid REAL,
|
||||
confidence TEXT NOT NULL DEFAULT 'low',
|
||||
field_confidence_json TEXT NOT NULL DEFAULT '{}',
|
||||
legible INTEGER NOT NULL DEFAULT 1,
|
||||
uncertain_fields TEXT NOT NULL DEFAULT '[]',
|
||||
extractor TEXT NOT NULL DEFAULT 'local',
|
||||
status TEXT NOT NULL DEFAULT 'staged',
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS fiscal_documents (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
detected_document_id INTEGER REFERENCES detected_documents(id) ON DELETE SET NULL,
|
||||
source_file_name TEXT NOT NULL DEFAULT '',
|
||||
source_location TEXT NOT NULL DEFAULT '',
|
||||
purchase_date TEXT NOT NULL,
|
||||
supplier_name TEXT NOT NULL,
|
||||
total_paid REAL NOT NULL,
|
||||
confidence TEXT NOT NULL DEFAULT 'high',
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_fiscal_date ON fiscal_documents(purchase_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_detected_batch ON detected_documents(batch_id, status);
|
||||
"""
|
||||
|
||||
|
||||
def connect(db_path: Path | None = None) -> sqlite3.Connection:
|
||||
settings = get_settings()
|
||||
settings.ensure_storage()
|
||||
conn = sqlite3.connect(db_path or settings.db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("PRAGMA journal_mode = WAL")
|
||||
return conn
|
||||
|
||||
|
||||
def init_db(conn: sqlite3.Connection) -> None:
|
||||
conn.executescript(SCHEMA)
|
||||
conn.commit()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def session(db_path: Path | None = None) -> Iterator[sqlite3.Connection]:
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
init_db(conn)
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Usuários
|
||||
# --------------------------------------------------------------------------- #
|
||||
def count_users(conn: sqlite3.Connection) -> int:
|
||||
return int(conn.execute("SELECT COUNT(*) FROM users").fetchone()[0])
|
||||
|
||||
|
||||
def get_user(conn: sqlite3.Connection, username: str) -> sqlite3.Row | None:
|
||||
return conn.execute(
|
||||
"SELECT * FROM users WHERE username = ?", (username,)
|
||||
).fetchone()
|
||||
|
||||
|
||||
def create_user(conn: sqlite3.Connection, username: str, password_hash: str) -> int:
|
||||
cur = conn.execute(
|
||||
"INSERT INTO users (username, password_hash) VALUES (?, ?)",
|
||||
(username, password_hash),
|
||||
)
|
||||
conn.commit()
|
||||
return int(cur.lastrowid)
|
||||
|
||||
|
||||
def update_password(conn: sqlite3.Connection, username: str, password_hash: str) -> None:
|
||||
conn.execute(
|
||||
"UPDATE users SET password_hash = ? WHERE username = ?",
|
||||
(password_hash, username),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Lotes de importação
|
||||
# --------------------------------------------------------------------------- #
|
||||
def create_batch(conn: sqlite3.Connection) -> int:
|
||||
cur = conn.execute("INSERT INTO import_batches (status) VALUES ('open')")
|
||||
conn.commit()
|
||||
return int(cur.lastrowid)
|
||||
|
||||
|
||||
def get_batch(conn: sqlite3.Connection, batch_id: int) -> sqlite3.Row | None:
|
||||
return conn.execute(
|
||||
"SELECT * FROM import_batches WHERE id = ?", (batch_id,)
|
||||
).fetchone()
|
||||
|
||||
|
||||
def set_batch_status(conn: sqlite3.Connection, batch_id: int, status: str) -> None:
|
||||
if status == "confirmed":
|
||||
conn.execute(
|
||||
"UPDATE import_batches SET status = ?, confirmed_at = CURRENT_TIMESTAMP WHERE id = ?",
|
||||
(status, batch_id),
|
||||
)
|
||||
else:
|
||||
conn.execute(
|
||||
"UPDATE import_batches SET status = ? WHERE id = ?", (status, batch_id)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Uploads
|
||||
# --------------------------------------------------------------------------- #
|
||||
def insert_upload(
|
||||
conn: sqlite3.Connection,
|
||||
batch_id: int,
|
||||
original_name: str,
|
||||
stored_path: Path,
|
||||
content_type: str,
|
||||
size_bytes: int,
|
||||
) -> int:
|
||||
cur = conn.execute(
|
||||
"""
|
||||
INSERT INTO uploaded_files (batch_id, original_name, stored_path, content_type, size_bytes)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""",
|
||||
(batch_id, original_name, str(stored_path), content_type, size_bytes),
|
||||
)
|
||||
conn.commit()
|
||||
return int(cur.lastrowid)
|
||||
|
||||
|
||||
def update_upload_status(
|
||||
conn: sqlite3.Connection,
|
||||
upload_id: int,
|
||||
status: str,
|
||||
detected_count: int,
|
||||
message: str | None = None,
|
||||
) -> None:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE uploaded_files
|
||||
SET status = ?, detected_count = ?, message = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""",
|
||||
(status, detected_count, message, upload_id),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def get_upload(conn: sqlite3.Connection, upload_id: int) -> sqlite3.Row | None:
|
||||
return conn.execute(
|
||||
"SELECT * FROM uploaded_files WHERE id = ?", (upload_id,)
|
||||
).fetchone()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Documentos detectados (staging)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def insert_detected(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
upload_id: int,
|
||||
batch_id: int,
|
||||
source_file_name: str,
|
||||
source_page: int | None,
|
||||
source_location: str,
|
||||
raw_text: str,
|
||||
purchase_date: str | None,
|
||||
supplier_name: str | None,
|
||||
total_paid: float | None,
|
||||
confidence: str,
|
||||
field_confidence: dict[str, str],
|
||||
legible: bool,
|
||||
uncertain_fields: list[str],
|
||||
extractor: str,
|
||||
) -> int:
|
||||
cur = conn.execute(
|
||||
"""
|
||||
INSERT INTO detected_documents (
|
||||
upload_id, batch_id, source_file_name, source_page, source_location,
|
||||
raw_text, purchase_date, supplier_name, total_paid, confidence,
|
||||
field_confidence_json, legible, uncertain_fields, extractor, status
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'staged')
|
||||
""",
|
||||
(
|
||||
upload_id,
|
||||
batch_id,
|
||||
source_file_name,
|
||||
source_page,
|
||||
source_location,
|
||||
raw_text,
|
||||
purchase_date,
|
||||
supplier_name,
|
||||
total_paid,
|
||||
confidence,
|
||||
json.dumps(field_confidence, ensure_ascii=False),
|
||||
1 if legible else 0,
|
||||
json.dumps(uncertain_fields, ensure_ascii=False),
|
||||
extractor,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
return int(cur.lastrowid)
|
||||
|
||||
|
||||
def get_detected(conn: sqlite3.Connection, detected_id: int) -> sqlite3.Row | None:
|
||||
return conn.execute(
|
||||
"SELECT * FROM detected_documents WHERE id = ?", (detected_id,)
|
||||
).fetchone()
|
||||
|
||||
|
||||
def staged_documents(conn: sqlite3.Connection, batch_id: int) -> list[sqlite3.Row]:
|
||||
return list(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT * FROM detected_documents
|
||||
WHERE batch_id = ? AND status = 'staged'
|
||||
ORDER BY id ASC
|
||||
""",
|
||||
(batch_id,),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def batch_summary(conn: sqlite3.Connection, batch_id: int) -> dict[str, Any]:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*) AS n,
|
||||
COALESCE(SUM(total_paid), 0) AS total,
|
||||
SUM(CASE WHEN legible = 0 THEN 1 ELSE 0 END) AS ilegiveis,
|
||||
SUM(CASE WHEN supplier_name IS NULL OR TRIM(supplier_name) = ''
|
||||
OR total_paid IS NULL THEN 1 ELSE 0 END) AS incompletos
|
||||
FROM detected_documents
|
||||
WHERE batch_id = ? AND status = 'staged'
|
||||
""",
|
||||
(batch_id,),
|
||||
).fetchone()
|
||||
# "pendentes" = tudo que impede importar (ilegível OU faltando fornecedor/valor).
|
||||
ilegiveis = int(row["ilegiveis"] or 0)
|
||||
incompletos = int(row["incompletos"] or 0)
|
||||
pendentes = conn.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM detected_documents
|
||||
WHERE batch_id = ? AND status = 'staged'
|
||||
AND (legible = 0 OR supplier_name IS NULL OR TRIM(supplier_name) = '' OR total_paid IS NULL)
|
||||
""",
|
||||
(batch_id,),
|
||||
).fetchone()[0]
|
||||
return {
|
||||
"count": int(row["n"]),
|
||||
"total": float(row["total"]),
|
||||
"ilegiveis": ilegiveis,
|
||||
"incompletos": incompletos,
|
||||
"pendentes": int(pendentes),
|
||||
}
|
||||
|
||||
|
||||
def update_staged(
|
||||
conn: sqlite3.Connection,
|
||||
detected_id: int,
|
||||
*,
|
||||
purchase_date: str,
|
||||
supplier_name: str,
|
||||
total_paid: float,
|
||||
legible: bool = True,
|
||||
) -> None:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE detected_documents
|
||||
SET purchase_date = ?, supplier_name = ?, total_paid = ?,
|
||||
legible = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ? AND status = 'staged'
|
||||
""",
|
||||
(purchase_date, supplier_name, total_paid, 1 if legible else 0, detected_id),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def discard_staged(conn: sqlite3.Connection, detected_id: int) -> None:
|
||||
conn.execute(
|
||||
"UPDATE detected_documents SET status = 'discarded', updated_at = CURRENT_TIMESTAMP WHERE id = ?",
|
||||
(detected_id,),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def confirm_batch(conn: sqlite3.Connection, batch_id: int) -> int:
|
||||
"""Promove todos os detectados 'staged' do lote para fiscal_documents."""
|
||||
rows = staged_documents(conn, batch_id)
|
||||
inserted = 0
|
||||
for row in rows:
|
||||
if not row["purchase_date"] or not row["supplier_name"] or row["total_paid"] is None:
|
||||
# Sem os três campos essenciais não confirma — permanece staged.
|
||||
continue
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO fiscal_documents (
|
||||
detected_document_id, source_file_name, source_location,
|
||||
purchase_date, supplier_name, total_paid, confidence
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
row["id"],
|
||||
row["source_file_name"],
|
||||
row["source_location"],
|
||||
row["purchase_date"],
|
||||
row["supplier_name"],
|
||||
float(row["total_paid"]),
|
||||
row["confidence"],
|
||||
),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE detected_documents SET status = 'confirmed', updated_at = CURRENT_TIMESTAMP WHERE id = ?",
|
||||
(row["id"],),
|
||||
)
|
||||
inserted += 1
|
||||
set_batch_status(conn, batch_id, "confirmed")
|
||||
conn.commit()
|
||||
return inserted
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# CRUD de fiscal_documents
|
||||
# --------------------------------------------------------------------------- #
|
||||
def create_fiscal(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
purchase_date: str,
|
||||
supplier_name: str,
|
||||
total_paid: float,
|
||||
source_file_name: str = "lançamento manual",
|
||||
source_location: str = "manual",
|
||||
confidence: str = "high",
|
||||
detected_document_id: int | None = None,
|
||||
) -> int:
|
||||
cur = conn.execute(
|
||||
"""
|
||||
INSERT INTO fiscal_documents (
|
||||
detected_document_id, source_file_name, source_location,
|
||||
purchase_date, supplier_name, total_paid, confidence
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
detected_document_id,
|
||||
source_file_name,
|
||||
source_location,
|
||||
purchase_date,
|
||||
supplier_name,
|
||||
total_paid,
|
||||
confidence,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
return int(cur.lastrowid)
|
||||
|
||||
|
||||
def get_fiscal(conn: sqlite3.Connection, doc_id: int) -> sqlite3.Row | None:
|
||||
return conn.execute(
|
||||
"SELECT * FROM fiscal_documents WHERE id = ?", (doc_id,)
|
||||
).fetchone()
|
||||
|
||||
|
||||
def update_fiscal(
|
||||
conn: sqlite3.Connection,
|
||||
doc_id: int,
|
||||
*,
|
||||
purchase_date: str,
|
||||
supplier_name: str,
|
||||
total_paid: float,
|
||||
) -> None:
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE fiscal_documents
|
||||
SET purchase_date = ?, supplier_name = ?, total_paid = ?, updated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""",
|
||||
(purchase_date, supplier_name, total_paid, doc_id),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def delete_fiscal(conn: sqlite3.Connection, doc_id: int) -> None:
|
||||
conn.execute("DELETE FROM fiscal_documents WHERE id = ?", (doc_id,))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def list_fiscal(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
start: str | None = None,
|
||||
end: str | None = None,
|
||||
supplier: str | None = None,
|
||||
) -> list[sqlite3.Row]:
|
||||
clauses: list[str] = []
|
||||
params: list[Any] = []
|
||||
if start:
|
||||
clauses.append("purchase_date >= ?")
|
||||
params.append(start)
|
||||
if end:
|
||||
clauses.append("purchase_date <= ?")
|
||||
params.append(end)
|
||||
if supplier:
|
||||
clauses.append("supplier_name LIKE ?")
|
||||
params.append(f"%{supplier}%")
|
||||
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
|
||||
return list(
|
||||
conn.execute(
|
||||
f"SELECT * FROM fiscal_documents {where} ORDER BY purchase_date DESC, id DESC",
|
||||
params,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def monthly_totals(conn: sqlite3.Connection) -> list[sqlite3.Row]:
|
||||
return list(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT substr(purchase_date, 1, 7) AS month, SUM(total_paid) AS total, COUNT(*) AS count
|
||||
FROM fiscal_documents
|
||||
GROUP BY substr(purchase_date, 1, 7)
|
||||
ORDER BY month ASC
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def supplier_totals(conn: sqlite3.Connection, limit: int = 10) -> list[sqlite3.Row]:
|
||||
return list(
|
||||
conn.execute(
|
||||
"""
|
||||
SELECT supplier_name, SUM(total_paid) AS total, COUNT(*) AS count
|
||||
FROM fiscal_documents
|
||||
GROUP BY supplier_name
|
||||
ORDER BY total DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(limit,),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def overall_totals(conn: sqlite3.Connection) -> dict[str, Any]:
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) AS n, COALESCE(SUM(total_paid), 0) AS total FROM fiscal_documents"
|
||||
).fetchone()
|
||||
n = int(row["n"])
|
||||
total = float(row["total"])
|
||||
return {"count": n, "total": total, "avg": (total / n) if n else 0.0}
|
||||
Reference in New Issue
Block a user