318 lines
8.7 KiB
Python
318 lines
8.7 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import sqlite3
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
DATA_DIR = ROOT / "data"
|
|
UPLOAD_DIR = DATA_DIR / "uploads"
|
|
PREVIEW_DIR = DATA_DIR / "previews"
|
|
DB_PATH = DATA_DIR / "lernotafiscal.sqlite3"
|
|
|
|
|
|
def ensure_storage() -> None:
|
|
DATA_DIR.mkdir(exist_ok=True)
|
|
UPLOAD_DIR.mkdir(exist_ok=True)
|
|
PREVIEW_DIR.mkdir(exist_ok=True)
|
|
|
|
|
|
def connect(db_path: Path = DB_PATH) -> sqlite3.Connection:
|
|
ensure_storage()
|
|
conn = sqlite3.connect(db_path)
|
|
conn.row_factory = sqlite3.Row
|
|
conn.execute("PRAGMA foreign_keys = ON")
|
|
return conn
|
|
|
|
|
|
def init_db(conn: sqlite3.Connection) -> None:
|
|
conn.executescript(
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS uploaded_files (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
original_name TEXT NOT NULL,
|
|
stored_path TEXT NOT NULL,
|
|
content_type TEXT NOT NULL,
|
|
size_bytes INTEGER NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
detected_count INTEGER NOT NULL DEFAULT 0,
|
|
message TEXT,
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS detected_documents (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
upload_id INTEGER NOT NULL REFERENCES uploaded_files(id) ON DELETE CASCADE,
|
|
source_file_name TEXT NOT NULL,
|
|
source_page INTEGER,
|
|
source_location TEXT NOT NULL,
|
|
raw_text TEXT NOT NULL,
|
|
purchase_date TEXT,
|
|
supplier_name TEXT,
|
|
total_paid REAL,
|
|
confidence TEXT NOT NULL,
|
|
field_confidence_json TEXT NOT NULL,
|
|
status TEXT NOT NULL DEFAULT 'pending',
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS fiscal_documents (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
detected_document_id INTEGER REFERENCES detected_documents(id) ON DELETE SET NULL,
|
|
source_file_name TEXT NOT NULL,
|
|
source_location TEXT NOT NULL,
|
|
purchase_date TEXT NOT NULL,
|
|
supplier_name TEXT NOT NULL,
|
|
total_paid REAL NOT NULL,
|
|
confidence TEXT NOT NULL,
|
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
"""
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def insert_upload(
|
|
conn: sqlite3.Connection,
|
|
original_name: str,
|
|
stored_path: Path,
|
|
content_type: str,
|
|
size_bytes: int,
|
|
) -> int:
|
|
cur = conn.execute(
|
|
"""
|
|
INSERT INTO uploaded_files (original_name, stored_path, content_type, size_bytes)
|
|
VALUES (?, ?, ?, ?)
|
|
""",
|
|
(original_name, str(stored_path), content_type, size_bytes),
|
|
)
|
|
conn.commit()
|
|
return int(cur.lastrowid)
|
|
|
|
|
|
def update_upload_status(
|
|
conn: sqlite3.Connection,
|
|
upload_id: int,
|
|
status: str,
|
|
detected_count: int,
|
|
message: str | None = None,
|
|
) -> None:
|
|
conn.execute(
|
|
"""
|
|
UPDATE uploaded_files
|
|
SET status = ?, detected_count = ?, message = ?, updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""",
|
|
(status, detected_count, message, upload_id),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def insert_detected_document(
|
|
conn: sqlite3.Connection,
|
|
upload_id: int,
|
|
candidate: Any,
|
|
) -> int:
|
|
cur = conn.execute(
|
|
"""
|
|
INSERT INTO detected_documents (
|
|
upload_id, source_file_name, source_page, source_location, raw_text,
|
|
purchase_date, supplier_name, total_paid, confidence, field_confidence_json
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
upload_id,
|
|
candidate.source_file_name,
|
|
candidate.source_page,
|
|
candidate.source_location,
|
|
candidate.raw_text,
|
|
candidate.purchase_date,
|
|
candidate.supplier_name,
|
|
candidate.total_paid,
|
|
candidate.confidence,
|
|
json.dumps(candidate.field_confidence, ensure_ascii=True),
|
|
),
|
|
)
|
|
conn.commit()
|
|
return int(cur.lastrowid)
|
|
|
|
|
|
def pending_documents(conn: sqlite3.Connection) -> list[sqlite3.Row]:
|
|
return list(
|
|
conn.execute(
|
|
"""
|
|
SELECT d.*, u.original_name
|
|
FROM detected_documents d
|
|
JOIN uploaded_files u ON u.id = d.upload_id
|
|
WHERE d.status = 'pending'
|
|
ORDER BY d.created_at DESC, d.id DESC
|
|
"""
|
|
)
|
|
)
|
|
|
|
|
|
def uploads_summary(conn: sqlite3.Connection) -> list[sqlite3.Row]:
|
|
return list(
|
|
conn.execute(
|
|
"""
|
|
SELECT *
|
|
FROM uploaded_files
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 20
|
|
"""
|
|
)
|
|
)
|
|
|
|
|
|
def attention_uploads(conn: sqlite3.Connection) -> list[sqlite3.Row]:
|
|
return list(
|
|
conn.execute(
|
|
"""
|
|
SELECT *
|
|
FROM uploaded_files
|
|
WHERE status IN ('needs_attention', 'failed')
|
|
ORDER BY updated_at DESC, id DESC
|
|
LIMIT 20
|
|
"""
|
|
)
|
|
)
|
|
|
|
|
|
def review_counts(conn: sqlite3.Connection) -> dict[str, int]:
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT status, COUNT(*) AS total
|
|
FROM detected_documents
|
|
GROUP BY status
|
|
"""
|
|
).fetchall()
|
|
counts = {"pending": 0, "confirmed": 0, "ignored": 0}
|
|
for row in rows:
|
|
counts[row["status"]] = int(row["total"])
|
|
return counts
|
|
|
|
|
|
def get_detected(conn: sqlite3.Connection, detected_id: int) -> sqlite3.Row | None:
|
|
return conn.execute(
|
|
"SELECT * FROM detected_documents WHERE id = ?",
|
|
(detected_id,),
|
|
).fetchone()
|
|
|
|
|
|
def confirm_document(
|
|
conn: sqlite3.Connection,
|
|
detected_id: int,
|
|
purchase_date: str,
|
|
supplier_name: str,
|
|
total_paid: float,
|
|
) -> None:
|
|
row = get_detected(conn, detected_id)
|
|
if row is None:
|
|
raise ValueError("Documento detectado nao encontrado.")
|
|
|
|
conn.execute(
|
|
"""
|
|
UPDATE detected_documents
|
|
SET purchase_date = ?, supplier_name = ?, total_paid = ?,
|
|
status = 'confirmed', updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""",
|
|
(purchase_date, supplier_name, total_paid, detected_id),
|
|
)
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO fiscal_documents (
|
|
detected_document_id, source_file_name, source_location,
|
|
purchase_date, supplier_name, total_paid, confidence
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
(
|
|
detected_id,
|
|
row["source_file_name"],
|
|
row["source_location"],
|
|
purchase_date,
|
|
supplier_name,
|
|
total_paid,
|
|
row["confidence"],
|
|
),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def ignore_document(conn: sqlite3.Connection, detected_id: int) -> None:
|
|
conn.execute(
|
|
"""
|
|
UPDATE detected_documents
|
|
SET status = 'ignored', updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = ?
|
|
""",
|
|
(detected_id,),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def dashboard_documents(
|
|
conn: sqlite3.Connection,
|
|
start: str | None = None,
|
|
end: str | None = None,
|
|
supplier: str | None = None,
|
|
) -> list[sqlite3.Row]:
|
|
clauses = []
|
|
params: list[Any] = []
|
|
if start:
|
|
clauses.append("purchase_date >= ?")
|
|
params.append(start)
|
|
if end:
|
|
clauses.append("purchase_date <= ?")
|
|
params.append(end)
|
|
if supplier:
|
|
clauses.append("supplier_name LIKE ?")
|
|
params.append(f"%{supplier}%")
|
|
|
|
where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
|
|
return list(
|
|
conn.execute(
|
|
f"""
|
|
SELECT *
|
|
FROM fiscal_documents
|
|
{where}
|
|
ORDER BY purchase_date DESC, id DESC
|
|
""",
|
|
params,
|
|
)
|
|
)
|
|
|
|
|
|
def monthly_totals(conn: sqlite3.Connection) -> list[sqlite3.Row]:
|
|
return list(
|
|
conn.execute(
|
|
"""
|
|
SELECT substr(purchase_date, 1, 7) AS month, SUM(total_paid) AS total, COUNT(*) AS count
|
|
FROM fiscal_documents
|
|
GROUP BY substr(purchase_date, 1, 7)
|
|
ORDER BY month DESC
|
|
"""
|
|
)
|
|
)
|
|
|
|
|
|
def supplier_totals(conn: sqlite3.Connection) -> list[sqlite3.Row]:
|
|
return list(
|
|
conn.execute(
|
|
"""
|
|
SELECT supplier_name, SUM(total_paid) AS total, COUNT(*) AS count
|
|
FROM fiscal_documents
|
|
GROUP BY supplier_name
|
|
ORDER BY total DESC
|
|
LIMIT 10
|
|
"""
|
|
)
|
|
)
|