commit efa06def71bc6db05719186db552688baf83ba99 Author: WanderMotta Date: Fri Jul 24 15:51:10 2026 -0300 Commit inicial - upload de todos os arquivos da pasta diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..79beaae --- /dev/null +++ b/.env.example @@ -0,0 +1,24 @@ +# Copie para .env e ajuste. NUNCA versione o .env real. + +# --- Sessão / segurança --- +# Gere com: python -c "import secrets; print(secrets.token_urlsafe(48))" +SECRET_KEY=troque-por-um-valor-aleatorio-longo +# Em produção (HTTPS) marque o cookie como seguro: +SESSION_HTTPS_ONLY=true + +# --- Login admin (semeado na 1a execucao) --- +ADMIN_USERNAME=admin +ADMIN_PASSWORD=defina-uma-senha-forte + +# --- OpenAI (opcional; sem chave o app usa OCR/heuristica local) --- +OPENAI_API_KEY= +OPENAI_MODEL=gpt-4o +OPENAI_MAX_PAGES=8 + +# --- Banco / uploads --- +DB_PATH=data/app.sqlite3 +MAX_UPLOAD_BYTES=12582912 + +# --- Servidor (dev) --- +HOST=127.0.0.1 +PORT=8000 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1cbcf91 --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +# Segredos e ambiente +.env +*.env.local + +# Banco e dados de runtime +data/*.sqlite3 +data/*.sqlite3-wal +data/*.sqlite3-shm +data/uploads/ +data/previews/ +data/*.log + +# Python +__pycache__/ +*.py[cod] +.venv/ +venv/ +*.egg-info/ + +# Editores / SO +.DS_Store +.idea/ +.vscode/ diff --git a/DEPLOY.md b/DEPLOY.md new file mode 100644 index 0000000..2c8e296 --- /dev/null +++ b/DEPLOY.md @@ -0,0 +1,100 @@ +# Deploy — VPS Ubuntu + systemd + Nginx + HTTPS + +Guia para publicar o Lernotafiscal em uma VPS Ubuntu (22.04+). O app roda em +`127.0.0.1:8000` via Uvicorn sob systemd; o Nginx faz proxy reverso e TLS. + +## 1. Pacotes do sistema + +```bash +sudo apt update +sudo apt install -y python3-venv python3-pip nginx \ + tesseract-ocr tesseract-ocr-por \ + certbot python3-certbot-nginx +``` + +> `tesseract-ocr-por` habilita o OCR em português usado no fallback local. +> `PyMuPDF` (render de PDF) vem via pip, não precisa de pacote do sistema. + +## 2. Usuário e código + +```bash +sudo useradd --system --create-home --home-dir /opt/lernotafiscal lernotafiscal +sudo -u lernotafiscal -H bash +cd /opt/lernotafiscal +git clone . # ou copie os arquivos do projeto para cá +python3 -m venv .venv +.venv/bin/pip install -r requirements.txt +``` + +## 3. Configuração (`.env`) + +```bash +cp .env.example .env +# gere um SECRET_KEY forte: +python3 -c "import secrets; print(secrets.token_urlsafe(48))" +nano .env +``` + +Defina no mínimo: `SECRET_KEY`, `ADMIN_USERNAME`, `ADMIN_PASSWORD`, +`SESSION_HTTPS_ONLY=true` e, se for usar IA, `OPENAI_API_KEY` (+ `OPENAI_MODEL`). +Sem `OPENAI_API_KEY` o app funciona com OCR/heurística local. + +## 4. Dados iniciais (opcional) + +Para migrar o histórico existente (21 notas da skill) para o banco do app: + +```bash +.venv/bin/python scripts/migrate_notas.py +``` + +O banco é criado em `data/app.sqlite3` na primeira execução do app de qualquer forma. + +## 5. Serviço systemd + +```bash +sudo cp deploy/lernotafiscal.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now lernotafiscal +sudo systemctl status lernotafiscal +# a senha temporária (se ADMIN_PASSWORD estiver vazio) aparece no log: +sudo journalctl -u lernotafiscal -n 30 +``` + +## 6. Nginx + HTTPS + +```bash +sudo cp deploy/nginx.conf /etc/nginx/sites-available/lernotafiscal +sudo nano /etc/nginx/sites-available/lernotafiscal # ajuste server_name +sudo ln -s /etc/nginx/sites-available/lernotafiscal /etc/nginx/sites-enabled/ +sudo nginx -t && sudo systemctl reload nginx +sudo certbot --nginx -d seu.dominio.com # emite e configura o TLS +``` + +Após o certbot, confirme que o `.env` tem `SESSION_HTTPS_ONLY=true` e reinicie: +`sudo systemctl restart lernotafiscal`. + +## 7. Atualizações + +```bash +sudo -u lernotafiscal -H bash -c 'cd /opt/lernotafiscal && git pull && .venv/bin/pip install -r requirements.txt' +sudo systemctl restart lernotafiscal +``` + +## 8. Backup + +O estado vive em dois lugares — faça backup dos dois (ex.: cron diário): + +```bash +# banco (checkpoint do WAL antes de copiar) +sqlite3 /opt/lernotafiscal/data/app.sqlite3 "PRAGMA wal_checkpoint(TRUNCATE);" +cp /opt/lernotafiscal/data/app.sqlite3 /backup/app-$(date +%F).sqlite3 +# arquivos enviados +tar czf /backup/uploads-$(date +%F).tgz -C /opt/lernotafiscal/data uploads +``` + +## Notas de segurança + +- Os arquivos enviados **nunca** são servidos por static mount; o download passa + pela rota autenticada `GET /files/{id}`. +- Segredos ficam só no `.env` (fora do git via `.gitignore`). +- Rode o Uvicorn apenas em `127.0.0.1` — a exposição pública é só via Nginx/TLS. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3d1607d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + HOST=0.0.0.0 \ + PORT=8000 + +WORKDIR /app + +# Dependências de sistema para OCR em português +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + tesseract-ocr \ + tesseract-ocr-por \ + && rm -rf /var/lib/apt/lists/* + +# Instala dependências Python primeiro para aproveitar cache de build +COPY requirements.txt ./ + +RUN pip install --upgrade pip \ + && pip install -r requirements.txt + +# Copia o restante da aplicação +COPY . . + +# Prepara diretórios persistentes e usuário sem privilégios +RUN mkdir -p /app/data /app/data/uploads \ + && useradd --system --uid 10001 --create-home \ + --home-dir /home/lernotafiscal lernotafiscal \ + && chown -R lernotafiscal:lernotafiscal /app + +USER lernotafiscal + +EXPOSE 8000 + +# APP_MODULE pode ser sobrescrito pelo Coolify/docker-compose. +# Padrão assumido: app.main:app +CMD ["sh", "-c", "uvicorn ${APP_MODULE:-app.main:app} --host ${HOST:-0.0.0.0} --port ${PORT:-8000} --proxy-headers --forwarded-allow-ips='*'"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..e7b59af --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +# Lernotafiscal + +Aplicativo web para importar notas e cupons fiscais (PDF/imagem), revisar a +extração e acompanhar despesas confirmadas. A leitura dos documentos usa +**OpenAI Vision** quando há chave configurada, com **fallback** para OCR/heurística +local. Feito para rodar numa VPS. + +## Recursos + +- Login simples (usuário/senha, hash bcrypt, sessão por cookie assinado). +- Upload de PDF/JPG/PNG com **confirmação em lote** antes de gravar: mostra o + número de documentos e o valor total, e destaca itens ilegíveis para correção. +- Data ilegível/ausente → **1º dia do mês corrente** (regra única compartilhada + por IA e cadastro manual). +- **CRUD** completo de documentos (criar, editar, excluir, listar/filtrar). +- Dashboard com KPIs, gastos por mês e por fornecedor. +- Tema claro/escuro (persistido no navegador). +- Banco **SQLite** (WAL), sem serviço externo. + +## Stack + +FastAPI + Uvicorn · Jinja2 (server-rendered) · SQLite · OpenAI (Vision) com +fallback OCR (PyMuPDF + Tesseract) · bcrypt. + +## Rodar localmente + +```bash +python -m venv .venv +.venv/Scripts/activate # Windows; no Linux/Mac: source .venv/bin/activate +pip install -r requirements.txt +cp .env.example .env # defina ADMIN_PASSWORD e SECRET_KEY +python app.py # http://127.0.0.1:8000 +``` + +Sem `OPENAI_API_KEY` o app usa OCR/heurística local (o Tesseract precisa estar +instalado no sistema para OCR de imagens). Com a chave, a extração usa a IA. + +### Migrar histórico (opcional) + +```bash +python scripts/migrate_notas.py # importa Skill/dados/lernotafiscal.db -> app +``` + +## Testes + +```bash +python -m unittest +``` + +## Deploy em VPS + +Guia completo (Ubuntu + systemd + Nginx + HTTPS) em [DEPLOY.md](DEPLOY.md). + +## Estrutura + +``` +app/ aplicação FastAPI (config, database, auth, rotas, templates) +app/ai_extraction extração via OpenAI Vision +app/ingestion orquestra IA -> fallback OCR local +lernotafiscal/ motor de extração heurística/OCR reutilizado no fallback +scripts/ utilitários (migrate_notas.py) +deploy/ unit systemd + config nginx +``` + +## Variáveis de ambiente + +Veja [.env.example](.env.example). Principais: `SECRET_KEY`, `ADMIN_USERNAME`, +`ADMIN_PASSWORD`, `OPENAI_API_KEY`, `OPENAI_MODEL`, `DB_PATH`, `MAX_UPLOAD_BYTES`, +`SESSION_HTTPS_ONLY`. diff --git a/Skill/dados/lernotafiscal.db b/Skill/dados/lernotafiscal.db new file mode 100644 index 0000000..0df9863 Binary files /dev/null and b/Skill/dados/lernotafiscal.db differ diff --git a/app.py b/app.py new file mode 100644 index 0000000..aa42f48 --- /dev/null +++ b/app.py @@ -0,0 +1,16 @@ +"""Entrypoint de desenvolvimento: sobe o app FastAPI com Uvicorn. + +Produção usa `uvicorn app.main:app` via systemd (veja DEPLOY.md). +""" + +import os + +import uvicorn + +if __name__ == "__main__": + uvicorn.run( + "app.main:app", + host=os.environ.get("HOST", "127.0.0.1"), + port=int(os.environ.get("PORT", "8000")), + reload=bool(os.environ.get("RELOAD")), + ) diff --git a/app.zip b/app.zip new file mode 100644 index 0000000..196029d Binary files /dev/null and b/app.zip differ diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..d5b7478 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,3 @@ +"""Lernotafiscal — aplicativo web de controle de despesas fiscais.""" + +__version__ = "1.0.0" diff --git a/app/ai_extraction.py b/app/ai_extraction.py new file mode 100644 index 0000000..2532f84 --- /dev/null +++ b/app/ai_extraction.py @@ -0,0 +1,153 @@ +"""Extração de documentos fiscais via OpenAI Vision, com fallback local. + +`extract_with_ai` recebe as imagens de um arquivo enviado (páginas de PDF já +renderizadas em PNG, ou a própria imagem) e devolve uma lista de `RawExtraction` +— ou `None` para sinalizar que o chamador deve cair no OCR/heurística local. + +A superfície da SDK usada (chat.completions + response_format json_object + +entrada de imagem por data URI) foi verificada contra `openai` 2.x. +""" + +from __future__ import annotations + +import base64 +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path + +from .config import get_settings + +logger = logging.getLogger("lernotafiscal.ai") + + +@dataclass +class RawExtraction: + """Resultado bruto de UM documento fiscal, antes de normalizar a data.""" + + supplier_name: str | None + purchase_date_raw: str | None + total_paid: float | None + legible: bool + uncertain_fields: list[str] = field(default_factory=list) + extractor: str = "openai" + raw_text: str = "" + + +_MIME = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg"} + +_SYSTEM_PROMPT = ( + "Você é um extrator de dados de notas e cupons fiscais brasileiros. " + "Responda SEMPRE em JSON válido, sem texto fora do JSON." +) + +_USER_PROMPT = """As imagens a seguir são documentos fiscais (notas, cupons, recibos). Podem conter mais de um documento. +Para CADA documento distinto, extraia: +- "fornecedor": razão social ou nome fantasia do emissor (o mais destacado no cabeçalho), ou null. +- "data_compra": data da compra no formato "YYYY-MM-DD", ou null se ilegível/ausente. +- "valor_pago": número (ponto decimal) do TOTAL efetivamente pago. Use o "VALOR TOTAL"/"TOTAL A PAGAR"/"VALOR PAGO". NUNCA use "TROCO" nem "DINHEIRO RECEBIDO". Se parcelado, use o total da compra. Null se ilegível. +- "legivel": true se você leu os campos com confiança; false se o documento está borrado, cortado ou ilegível. +- "campos_incertos": lista dos campos que ficaram duvidosos (ex.: ["data_compra","valor_pago"]). + +Responda exatamente neste formato: +{"documentos": [{"fornecedor": ..., "data_compra": ..., "valor_pago": ..., "legivel": ..., "campos_incertos": [...]}]} +Se nenhum documento fiscal for identificável, responda {"documentos": []}.""" + + +def _data_uri(path: Path) -> str | None: + mime = _MIME.get(path.suffix.lower()) + if not mime: + return None + try: + encoded = base64.b64encode(path.read_bytes()).decode("ascii") + except OSError: + return None + return f"data:{mime};base64,{encoded}" + + +def _coerce_float(value: object) -> float | None: + if value is None: + return None + if isinstance(value, (int, float)): + return round(float(value), 2) + text = str(value).strip().replace("R$", "").replace(" ", "") + if not text: + return None + # aceita "1.234,56" e "1234.56" + if "," in text and "." in text: + text = text.replace(".", "").replace(",", ".") + elif "," in text: + text = text.replace(",", ".") + try: + return round(float(text), 2) + except ValueError: + return None + + +def _parse_response(content: str) -> list[RawExtraction]: + data = json.loads(content) + docs = data.get("documentos") if isinstance(data, dict) else None + if not isinstance(docs, list): + return [] + results: list[RawExtraction] = [] + for item in docs: + if not isinstance(item, dict): + continue + supplier = item.get("fornecedor") + supplier = str(supplier).strip()[:200] if supplier else None + date_raw = item.get("data_compra") + date_raw = str(date_raw).strip() if date_raw else None + total = _coerce_float(item.get("valor_pago")) + legible = bool(item.get("legivel", True)) + uncertain = item.get("campos_incertos") or [] + uncertain = [str(x) for x in uncertain if x] if isinstance(uncertain, list) else [] + # segurança: campos faltando são inerentemente incertos + for name, val in (("fornecedor", supplier), ("data_compra", date_raw), ("valor_pago", total)): + if val is None and name not in uncertain: + uncertain.append(name) + if uncertain and legible and len(uncertain) >= 2: + legible = False + results.append( + RawExtraction( + supplier_name=supplier, + purchase_date_raw=date_raw, + total_paid=total, + legible=legible, + uncertain_fields=uncertain, + ) + ) + return results + + +def extract_with_ai(image_paths: list[Path]) -> list[RawExtraction] | None: + """Extrai via OpenAI. Retorna None se IA indisponível ou em erro (=> fallback).""" + settings = get_settings() + if not settings.openai_enabled: + return None + + uris = [uri for p in image_paths[: settings.openai_max_pages] if (uri := _data_uri(p))] + if not uris: + return None + + try: + from openai import OpenAI + + client = OpenAI(api_key=settings.openai_api_key) + content: list[dict] = [{"type": "text", "text": _USER_PROMPT}] + for uri in uris: + content.append({"type": "image_url", "image_url": {"url": uri}}) + + response = client.chat.completions.create( + model=settings.openai_model, + messages=[ + {"role": "system", "content": _SYSTEM_PROMPT}, + {"role": "user", "content": content}, + ], + response_format={"type": "json_object"}, + temperature=0, + ) + message = response.choices[0].message.content or "{}" + return _parse_response(message) + except Exception as exc: # rede, cota, parsing, modelo indisponível... + logger.warning("Extração OpenAI falhou, usando fallback local: %s", exc) + return None diff --git a/app/auth.py b/app/auth.py new file mode 100644 index 0000000..04d0a80 --- /dev/null +++ b/app/auth.py @@ -0,0 +1,90 @@ +"""Autenticação: hash de senha (bcrypt), seed do admin, CSRF e guarda de sessão.""" + +from __future__ import annotations + +import secrets +import time + +import bcrypt +from starlette.requests import Request + +from . import database as db +from .config import get_settings + + +# --------------------------------------------------------------------------- # +# Senhas +# --------------------------------------------------------------------------- # +def hash_password(password: str) -> str: + return bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("ascii") + + +def verify_password(password: str, password_hash: str) -> bool: + try: + return bcrypt.checkpw(password.encode("utf-8"), password_hash.encode("ascii")) + except (ValueError, TypeError): + return False + + +def seed_admin() -> str | None: + """Cria o usuário admin na 1ª execução. Retorna uma mensagem de aviso, se houver.""" + settings = get_settings() + with db.session() as conn: + if db.count_users(conn) > 0: + return None + password = settings.admin_password + note = None + if not password: + # Sem ADMIN_PASSWORD definido: gera uma senha aleatória e a expõe UMA vez + # no log para o operador. Em produção defina ADMIN_PASSWORD no ambiente. + password = secrets.token_urlsafe(12) + note = ( + f"[AVISO] Nenhum ADMIN_PASSWORD definido. Usuário '{settings.admin_username}' " + f"criado com senha temporária: {password} (defina ADMIN_PASSWORD e reinicie)" + ) + db.create_user(conn, settings.admin_username, hash_password(password)) + return note + + +def authenticate(username: str, password: str) -> bool: + with db.session() as conn: + row = db.get_user(conn, username) + if row is None: + # Compara mesmo sem usuário para não vazar tempo (mitiga user enumeration). + verify_password(password, hash_password("dummy")) + return False + return verify_password(password, row["password_hash"]) + + +# --------------------------------------------------------------------------- # +# Sessão / guarda +# --------------------------------------------------------------------------- # +def current_user(request: Request) -> str | None: + return request.session.get("user") + + +def login_session(request: Request, username: str) -> None: + request.session["user"] = username + request.session["logged_at"] = int(time.time()) + # rotaciona o token CSRF a cada login + request.session["csrf"] = secrets.token_urlsafe(32) + + +def logout_session(request: Request) -> None: + request.session.clear() + + +# --------------------------------------------------------------------------- # +# CSRF +# --------------------------------------------------------------------------- # +def get_csrf_token(request: Request) -> str: + token = request.session.get("csrf") + if not token: + token = secrets.token_urlsafe(32) + request.session["csrf"] = token + return token + + +def check_csrf(request: Request, submitted: str | None) -> bool: + expected = request.session.get("csrf") + return bool(expected) and bool(submitted) and secrets.compare_digest(expected, submitted) diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..a5a574f --- /dev/null +++ b/app/config.py @@ -0,0 +1,81 @@ +"""Configuração central lida de variáveis de ambiente (com suporte a .env).""" + +from __future__ import annotations + +import os +import secrets +from functools import lru_cache +from pathlib import Path + +try: # carregamento opcional do .env + from dotenv import load_dotenv + + load_dotenv() +except Exception: # pragma: no cover - dotenv é opcional + pass + + +BASE_DIR = Path(__file__).resolve().parent.parent +DATA_DIR = BASE_DIR / "data" +UPLOAD_DIR = DATA_DIR / "uploads" +PREVIEW_DIR = DATA_DIR / "previews" + +ALLOWED_EXTENSIONS = {".pdf", ".jpg", ".jpeg", ".png"} + + +def _env_bool(name: str, default: bool) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + return raw.strip().lower() in {"1", "true", "yes", "on", "sim"} + + +class Settings: + """Configuração resolvida uma vez no start do processo.""" + + def __init__(self) -> None: + self.base_dir = BASE_DIR + self.data_dir = DATA_DIR + self.upload_dir = UPLOAD_DIR + self.preview_dir = PREVIEW_DIR + + self.db_path = Path(os.environ.get("DB_PATH", DATA_DIR / "app.sqlite3")) + + # Segredo da sessão. Em produção DEVE vir do ambiente; sem ele geramos um + # efêmero (as sessões caem a cada restart) e avisamos no log. + self.secret_key = os.environ.get("SECRET_KEY", "") + self.secret_key_is_ephemeral = not self.secret_key + if self.secret_key_is_ephemeral: + self.secret_key = secrets.token_urlsafe(48) + + # OpenAI (opcional — sem chave o app cai no OCR/heurística local). + self.openai_api_key = os.environ.get("OPENAI_API_KEY", "").strip() + self.openai_model = os.environ.get("OPENAI_MODEL", "gpt-4o").strip() or "gpt-4o" + self.openai_max_pages = int(os.environ.get("OPENAI_MAX_PAGES", "8")) + + # Admin semeado na primeira execução. + self.admin_username = os.environ.get("ADMIN_USERNAME", "admin").strip() or "admin" + self.admin_password = os.environ.get("ADMIN_PASSWORD", "").strip() + + self.max_upload_bytes = int(os.environ.get("MAX_UPLOAD_BYTES", str(12 * 1024 * 1024))) + self.allowed_extensions = set(ALLOWED_EXTENSIONS) + + self.session_cookie = os.environ.get("SESSION_COOKIE", "lnf_session") + self.session_https_only = _env_bool("SESSION_HTTPS_ONLY", False) + self.session_max_age = int(os.environ.get("SESSION_MAX_AGE", str(60 * 60 * 12))) + + @property + def openai_enabled(self) -> bool: + return bool(self.openai_api_key) + + def ensure_storage(self) -> None: + self.data_dir.mkdir(parents=True, exist_ok=True) + self.upload_dir.mkdir(parents=True, exist_ok=True) + self.preview_dir.mkdir(parents=True, exist_ok=True) + + +@lru_cache(maxsize=1) +def get_settings() -> Settings: + settings = Settings() + settings.ensure_storage() + return settings diff --git a/app/database.py b/app/database.py new file mode 100644 index 0000000..63541c7 --- /dev/null +++ b/app/database.py @@ -0,0 +1,508 @@ +"""Acesso ao SQLite: esquema, migrações leves e helpers de consulta. + +Estende o esquema original (uploaded_files -> detected_documents -> fiscal_documents) +com autenticação (users), lotes de importação (import_batches) e os campos de +legibilidade usados pela extração por IA. +""" + +from __future__ import annotations + +import json +import sqlite3 +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Iterator + +from .config import get_settings + + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS users ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + username TEXT NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS import_batches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + status TEXT NOT NULL DEFAULT 'open', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + confirmed_at TEXT +); + +CREATE TABLE IF NOT EXISTS uploaded_files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + batch_id INTEGER REFERENCES import_batches(id) ON DELETE SET NULL, + original_name TEXT NOT NULL, + stored_path TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + detected_count INTEGER NOT NULL DEFAULT 0, + message TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS detected_documents ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + upload_id INTEGER NOT NULL REFERENCES uploaded_files(id) ON DELETE CASCADE, + batch_id INTEGER REFERENCES import_batches(id) ON DELETE SET NULL, + source_file_name TEXT NOT NULL, + source_page INTEGER, + source_location TEXT NOT NULL, + raw_text TEXT NOT NULL DEFAULT '', + purchase_date TEXT, + supplier_name TEXT, + total_paid REAL, + confidence TEXT NOT NULL DEFAULT 'low', + field_confidence_json TEXT NOT NULL DEFAULT '{}', + legible INTEGER NOT NULL DEFAULT 1, + uncertain_fields TEXT NOT NULL DEFAULT '[]', + extractor TEXT NOT NULL DEFAULT 'local', + status TEXT NOT NULL DEFAULT 'staged', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS fiscal_documents ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + detected_document_id INTEGER REFERENCES detected_documents(id) ON DELETE SET NULL, + source_file_name TEXT NOT NULL DEFAULT '', + source_location TEXT NOT NULL DEFAULT '', + purchase_date TEXT NOT NULL, + supplier_name TEXT NOT NULL, + total_paid REAL NOT NULL, + confidence TEXT NOT NULL DEFAULT 'high', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_fiscal_date ON fiscal_documents(purchase_date); +CREATE INDEX IF NOT EXISTS idx_detected_batch ON detected_documents(batch_id, status); +""" + + +def connect(db_path: Path | None = None) -> sqlite3.Connection: + settings = get_settings() + settings.ensure_storage() + conn = sqlite3.connect(db_path or settings.db_path) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + conn.execute("PRAGMA journal_mode = WAL") + return conn + + +def init_db(conn: sqlite3.Connection) -> None: + conn.executescript(SCHEMA) + conn.commit() + + +@contextmanager +def session(db_path: Path | None = None) -> Iterator[sqlite3.Connection]: + conn = connect(db_path) + try: + init_db(conn) + yield conn + finally: + conn.close() + + +# --------------------------------------------------------------------------- # +# Usuários +# --------------------------------------------------------------------------- # +def count_users(conn: sqlite3.Connection) -> int: + return int(conn.execute("SELECT COUNT(*) FROM users").fetchone()[0]) + + +def get_user(conn: sqlite3.Connection, username: str) -> sqlite3.Row | None: + return conn.execute( + "SELECT * FROM users WHERE username = ?", (username,) + ).fetchone() + + +def create_user(conn: sqlite3.Connection, username: str, password_hash: str) -> int: + cur = conn.execute( + "INSERT INTO users (username, password_hash) VALUES (?, ?)", + (username, password_hash), + ) + conn.commit() + return int(cur.lastrowid) + + +def update_password(conn: sqlite3.Connection, username: str, password_hash: str) -> None: + conn.execute( + "UPDATE users SET password_hash = ? WHERE username = ?", + (password_hash, username), + ) + conn.commit() + + +# --------------------------------------------------------------------------- # +# Lotes de importação +# --------------------------------------------------------------------------- # +def create_batch(conn: sqlite3.Connection) -> int: + cur = conn.execute("INSERT INTO import_batches (status) VALUES ('open')") + conn.commit() + return int(cur.lastrowid) + + +def get_batch(conn: sqlite3.Connection, batch_id: int) -> sqlite3.Row | None: + return conn.execute( + "SELECT * FROM import_batches WHERE id = ?", (batch_id,) + ).fetchone() + + +def set_batch_status(conn: sqlite3.Connection, batch_id: int, status: str) -> None: + if status == "confirmed": + conn.execute( + "UPDATE import_batches SET status = ?, confirmed_at = CURRENT_TIMESTAMP WHERE id = ?", + (status, batch_id), + ) + else: + conn.execute( + "UPDATE import_batches SET status = ? WHERE id = ?", (status, batch_id) + ) + conn.commit() + + +# --------------------------------------------------------------------------- # +# Uploads +# --------------------------------------------------------------------------- # +def insert_upload( + conn: sqlite3.Connection, + batch_id: int, + original_name: str, + stored_path: Path, + content_type: str, + size_bytes: int, +) -> int: + cur = conn.execute( + """ + INSERT INTO uploaded_files (batch_id, original_name, stored_path, content_type, size_bytes) + VALUES (?, ?, ?, ?, ?) + """, + (batch_id, original_name, str(stored_path), content_type, size_bytes), + ) + conn.commit() + return int(cur.lastrowid) + + +def update_upload_status( + conn: sqlite3.Connection, + upload_id: int, + status: str, + detected_count: int, + message: str | None = None, +) -> None: + conn.execute( + """ + UPDATE uploaded_files + SET status = ?, detected_count = ?, message = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (status, detected_count, message, upload_id), + ) + conn.commit() + + +def get_upload(conn: sqlite3.Connection, upload_id: int) -> sqlite3.Row | None: + return conn.execute( + "SELECT * FROM uploaded_files WHERE id = ?", (upload_id,) + ).fetchone() + + +# --------------------------------------------------------------------------- # +# Documentos detectados (staging) +# --------------------------------------------------------------------------- # +def insert_detected( + conn: sqlite3.Connection, + *, + upload_id: int, + batch_id: int, + source_file_name: str, + source_page: int | None, + source_location: str, + raw_text: str, + purchase_date: str | None, + supplier_name: str | None, + total_paid: float | None, + confidence: str, + field_confidence: dict[str, str], + legible: bool, + uncertain_fields: list[str], + extractor: str, +) -> int: + cur = conn.execute( + """ + INSERT INTO detected_documents ( + upload_id, batch_id, source_file_name, source_page, source_location, + raw_text, purchase_date, supplier_name, total_paid, confidence, + field_confidence_json, legible, uncertain_fields, extractor, status + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'staged') + """, + ( + upload_id, + batch_id, + source_file_name, + source_page, + source_location, + raw_text, + purchase_date, + supplier_name, + total_paid, + confidence, + json.dumps(field_confidence, ensure_ascii=False), + 1 if legible else 0, + json.dumps(uncertain_fields, ensure_ascii=False), + extractor, + ), + ) + conn.commit() + return int(cur.lastrowid) + + +def get_detected(conn: sqlite3.Connection, detected_id: int) -> sqlite3.Row | None: + return conn.execute( + "SELECT * FROM detected_documents WHERE id = ?", (detected_id,) + ).fetchone() + + +def staged_documents(conn: sqlite3.Connection, batch_id: int) -> list[sqlite3.Row]: + return list( + conn.execute( + """ + SELECT * FROM detected_documents + WHERE batch_id = ? AND status = 'staged' + ORDER BY id ASC + """, + (batch_id,), + ) + ) + + +def batch_summary(conn: sqlite3.Connection, batch_id: int) -> dict[str, Any]: + row = conn.execute( + """ + SELECT COUNT(*) AS n, + COALESCE(SUM(total_paid), 0) AS total, + SUM(CASE WHEN legible = 0 THEN 1 ELSE 0 END) AS ilegiveis, + SUM(CASE WHEN supplier_name IS NULL OR TRIM(supplier_name) = '' + OR total_paid IS NULL THEN 1 ELSE 0 END) AS incompletos + FROM detected_documents + WHERE batch_id = ? AND status = 'staged' + """, + (batch_id,), + ).fetchone() + # "pendentes" = tudo que impede importar (ilegível OU faltando fornecedor/valor). + ilegiveis = int(row["ilegiveis"] or 0) + incompletos = int(row["incompletos"] or 0) + pendentes = conn.execute( + """ + SELECT COUNT(*) FROM detected_documents + WHERE batch_id = ? AND status = 'staged' + AND (legible = 0 OR supplier_name IS NULL OR TRIM(supplier_name) = '' OR total_paid IS NULL) + """, + (batch_id,), + ).fetchone()[0] + return { + "count": int(row["n"]), + "total": float(row["total"]), + "ilegiveis": ilegiveis, + "incompletos": incompletos, + "pendentes": int(pendentes), + } + + +def update_staged( + conn: sqlite3.Connection, + detected_id: int, + *, + purchase_date: str, + supplier_name: str, + total_paid: float, + legible: bool = True, +) -> None: + conn.execute( + """ + UPDATE detected_documents + SET purchase_date = ?, supplier_name = ?, total_paid = ?, + legible = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? AND status = 'staged' + """, + (purchase_date, supplier_name, total_paid, 1 if legible else 0, detected_id), + ) + conn.commit() + + +def discard_staged(conn: sqlite3.Connection, detected_id: int) -> None: + conn.execute( + "UPDATE detected_documents SET status = 'discarded', updated_at = CURRENT_TIMESTAMP WHERE id = ?", + (detected_id,), + ) + conn.commit() + + +def confirm_batch(conn: sqlite3.Connection, batch_id: int) -> int: + """Promove todos os detectados 'staged' do lote para fiscal_documents.""" + rows = staged_documents(conn, batch_id) + inserted = 0 + for row in rows: + if not row["purchase_date"] or not row["supplier_name"] or row["total_paid"] is None: + # Sem os três campos essenciais não confirma — permanece staged. + continue + conn.execute( + """ + INSERT INTO fiscal_documents ( + detected_document_id, source_file_name, source_location, + purchase_date, supplier_name, total_paid, confidence + ) VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + row["id"], + row["source_file_name"], + row["source_location"], + row["purchase_date"], + row["supplier_name"], + float(row["total_paid"]), + row["confidence"], + ), + ) + conn.execute( + "UPDATE detected_documents SET status = 'confirmed', updated_at = CURRENT_TIMESTAMP WHERE id = ?", + (row["id"],), + ) + inserted += 1 + set_batch_status(conn, batch_id, "confirmed") + conn.commit() + return inserted + + +# --------------------------------------------------------------------------- # +# CRUD de fiscal_documents +# --------------------------------------------------------------------------- # +def create_fiscal( + conn: sqlite3.Connection, + *, + purchase_date: str, + supplier_name: str, + total_paid: float, + source_file_name: str = "lançamento manual", + source_location: str = "manual", + confidence: str = "high", + detected_document_id: int | None = None, +) -> int: + cur = conn.execute( + """ + INSERT INTO fiscal_documents ( + detected_document_id, source_file_name, source_location, + purchase_date, supplier_name, total_paid, confidence + ) VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + detected_document_id, + source_file_name, + source_location, + purchase_date, + supplier_name, + total_paid, + confidence, + ), + ) + conn.commit() + return int(cur.lastrowid) + + +def get_fiscal(conn: sqlite3.Connection, doc_id: int) -> sqlite3.Row | None: + return conn.execute( + "SELECT * FROM fiscal_documents WHERE id = ?", (doc_id,) + ).fetchone() + + +def update_fiscal( + conn: sqlite3.Connection, + doc_id: int, + *, + purchase_date: str, + supplier_name: str, + total_paid: float, +) -> None: + conn.execute( + """ + UPDATE fiscal_documents + SET purchase_date = ?, supplier_name = ?, total_paid = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (purchase_date, supplier_name, total_paid, doc_id), + ) + conn.commit() + + +def delete_fiscal(conn: sqlite3.Connection, doc_id: int) -> None: + conn.execute("DELETE FROM fiscal_documents WHERE id = ?", (doc_id,)) + conn.commit() + + +def list_fiscal( + conn: sqlite3.Connection, + *, + start: str | None = None, + end: str | None = None, + supplier: str | None = None, +) -> list[sqlite3.Row]: + clauses: list[str] = [] + params: list[Any] = [] + if start: + clauses.append("purchase_date >= ?") + params.append(start) + if end: + clauses.append("purchase_date <= ?") + params.append(end) + if supplier: + clauses.append("supplier_name LIKE ?") + params.append(f"%{supplier}%") + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" + return list( + conn.execute( + f"SELECT * FROM fiscal_documents {where} ORDER BY purchase_date DESC, id DESC", + params, + ) + ) + + +def monthly_totals(conn: sqlite3.Connection) -> list[sqlite3.Row]: + return list( + conn.execute( + """ + SELECT substr(purchase_date, 1, 7) AS month, SUM(total_paid) AS total, COUNT(*) AS count + FROM fiscal_documents + GROUP BY substr(purchase_date, 1, 7) + ORDER BY month ASC + """ + ) + ) + + +def supplier_totals(conn: sqlite3.Connection, limit: int = 10) -> list[sqlite3.Row]: + return list( + conn.execute( + """ + SELECT supplier_name, SUM(total_paid) AS total, COUNT(*) AS count + FROM fiscal_documents + GROUP BY supplier_name + ORDER BY total DESC + LIMIT ? + """, + (limit,), + ) + ) + + +def overall_totals(conn: sqlite3.Connection) -> dict[str, Any]: + row = conn.execute( + "SELECT COUNT(*) AS n, COALESCE(SUM(total_paid), 0) AS total FROM fiscal_documents" + ).fetchone() + n = int(row["n"]) + total = float(row["total"]) + return {"count": n, "total": total, "avg": (total / n) if n else 0.0} diff --git a/app/dates.py b/app/dates.py new file mode 100644 index 0000000..ed8166f --- /dev/null +++ b/app/dates.py @@ -0,0 +1,56 @@ +"""Resolução de datas de compra com fallback único e compartilhado. + +Requisito: quando a data do documento está ilegível/ausente, atribuir o +**primeiro dia do mês corrente** (o mês da importação/lançamento). Tanto a +extração por IA quanto o cadastro manual passam por aqui, para nunca divergirem. +""" + +from __future__ import annotations + +import re +from datetime import date + + +_ISO_RE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})$") +_BR_RE = re.compile(r"^([0-3]?\d)[/.\-]([01]?\d)[/.\-]((?:19|20)?\d{2})$") + + +def first_of_current_month(reference: date | None = None) -> str: + ref = reference or date.today() + return f"{ref.year:04d}-{ref.month:02d}-01" + + +def _valid_iso(year: int, month: int, day: int) -> str | None: + try: + return date(year, month, day).isoformat() + except ValueError: + return None + + +def parse_date(value: str | None) -> str | None: + """Tenta interpretar uma data em ISO (YYYY-MM-DD) ou BR (dd/mm/aaaa). + + Retorna a data normalizada em ISO, ou None se não for uma data válida. + """ + if not value: + return None + text = value.strip() + m = _ISO_RE.match(text) + if m: + return _valid_iso(int(m.group(1)), int(m.group(2)), int(m.group(3))) + m = _BR_RE.match(text) + if m: + day, month, year = m.groups() + year_i = int(year) + if year_i < 100: + year_i += 2000 + return _valid_iso(year_i, int(month), int(day)) + return None + + +def resolve_purchase_date(value: str | None, *, reference: date | None = None) -> str: + """Data válida em ISO, ou o 1º dia do mês corrente quando ilegível/ausente.""" + parsed = parse_date(value) + if parsed: + return parsed + return first_of_current_month(reference) diff --git a/app/ingestion.py b/app/ingestion.py new file mode 100644 index 0000000..abc4dc1 --- /dev/null +++ b/app/ingestion.py @@ -0,0 +1,60 @@ +"""Orquestra a extração de um arquivo enviado: OpenAI Vision -> fallback local. + +Reutiliza `lernotafiscal.extraction` (normalização de PDF/imagem, render de +páginas via PyMuPDF, OCR Tesseract e a detecção heurística) como caminho de +contingência quando a IA está indisponível ou falha. +""" + +from __future__ import annotations + +from pathlib import Path + +from lernotafiscal.extraction import ( + DetectedDocumentCandidate, + detect_documents, + normalize_file, +) + +from .ai_extraction import RawExtraction, extract_with_ai + + +def _candidate_to_raw(candidate: DetectedDocumentCandidate) -> RawExtraction: + fields = candidate.field_confidence or {} + uncertain = [name for name, level in fields.items() if level == "low"] + for name, val in ( + ("fornecedor", candidate.supplier_name), + ("data_compra", candidate.purchase_date), + ("valor_pago", candidate.total_paid), + ): + if val is None and name not in uncertain: + uncertain.append(name) + legible = candidate.confidence != "low" and len(uncertain) < 2 + return RawExtraction( + supplier_name=candidate.supplier_name, + purchase_date_raw=candidate.purchase_date, + total_paid=candidate.total_paid, + legible=legible, + uncertain_fields=uncertain, + extractor="local", + raw_text=(candidate.raw_text or "")[:4000], + ) + + +def extract_file(stored_path: Path, original_name: str) -> list[RawExtraction]: + """Retorna os documentos extraídos de um arquivo (>=0). Nunca levanta exceção + de extração para o chamador — em último caso devolve lista vazia.""" + pages = normalize_file(stored_path, original_name) + + # Caminho preferencial: enviar as imagens (páginas renderizadas / imagem) à IA. + # Só aceitamos o resultado da IA quando ela retorna ao menos um documento. + # Uma resposta vazia (nada encontrado OU shape inesperado) cai no fallback + # local — evita "zero documentos" silencioso por uma resposta malformada. + image_paths = [p.image_path for p in pages if p.image_path is not None] + if image_paths: + ai_result = extract_with_ai(image_paths) + if ai_result: + return ai_result + + # Fallback: OCR/heurística sobre o texto já normalizado. + candidates = detect_documents(pages, original_name) + return [_candidate_to_raw(c) for c in candidates] diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..cc58ebe --- /dev/null +++ b/app/main.py @@ -0,0 +1,89 @@ +"""Ponto de entrada FastAPI do Lernotafiscal.""" + +from __future__ import annotations + +import logging +from contextlib import asynccontextmanager +from pathlib import Path + +from fastapi import FastAPI +from fastapi.staticfiles import StaticFiles +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.middleware.sessions import SessionMiddleware +from starlette.requests import Request +from starlette.responses import RedirectResponse + +from . import auth, database +from .config import get_settings +from .routes import ( + auth_routes, + dashboard_routes, + documents_routes, + files_routes, + upload_routes, +) + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +logger = logging.getLogger("lernotafiscal") + +# Caminhos liberados sem sessão (login e assets da própria UI). +PUBLIC_PREFIXES = ("/login", "/static/", "/favicon.ico", "/healthz") + + +class AuthGuardMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + path = request.url.path + if any(path == p or path.startswith(p) for p in PUBLIC_PREFIXES): + return await call_next(request) + if not request.session.get("user"): + return RedirectResponse("/login", status_code=303) + return await call_next(request) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings = get_settings() + with database.session() as conn: + database.init_db(conn) + if settings.secret_key_is_ephemeral: + logger.warning("SECRET_KEY não definido: sessões não sobrevivem a reinícios. Defina SECRET_KEY em produção.") + note = auth.seed_admin() + if note: + logger.warning(note) + logger.info("OpenAI %s.", "habilitado (" + settings.openai_model + ")" if settings.openai_enabled else "desabilitado — usando OCR/heurística local") + yield + + +def create_app() -> FastAPI: + settings = get_settings() + app = FastAPI(title="Lernotafiscal", lifespan=lifespan) + + # Ordem importa: SessionMiddleware é adicionado por último para ser o mais + # externo e popular request.session ANTES da guarda de autenticação. + app.add_middleware(AuthGuardMiddleware) + app.add_middleware( + SessionMiddleware, + secret_key=settings.secret_key, + session_cookie=settings.session_cookie, + https_only=settings.session_https_only, + max_age=settings.session_max_age, + same_site="lax", + ) + + static_dir = Path(__file__).resolve().parent / "static" + app.mount("/static", StaticFiles(directory=str(static_dir)), name="static") + + app.include_router(auth_routes.router) + app.include_router(dashboard_routes.router) + app.include_router(upload_routes.router) + app.include_router(documents_routes.router) + app.include_router(files_routes.router) + + @app.get("/healthz") + def healthz(): + return {"status": "ok"} + + return app + + +app = create_app() diff --git a/app/routes/__init__.py b/app/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/routes/auth_routes.py b/app/routes/auth_routes.py new file mode 100644 index 0000000..a7648df --- /dev/null +++ b/app/routes/auth_routes.py @@ -0,0 +1,45 @@ +"""Rotas de login/logout.""" + +from __future__ import annotations + +from fastapi import APIRouter, Form, Request +from starlette.responses import RedirectResponse + +from .. import auth +from ..templating import flash, render + +router = APIRouter() + + +@router.get("/login") +def login_form(request: Request): + if auth.current_user(request): + return RedirectResponse("/", status_code=303) + return render(request, "login.html", hide_nav=True) + + +@router.post("/login") +def login_submit( + request: Request, + username: str = Form(...), + password: str = Form(...), + csrf_token: str = Form(""), +): + if not auth.check_csrf(request, csrf_token): + flash(request, "Sessão expirada. Tente novamente.", "error") + return RedirectResponse("/login", status_code=303) + + if auth.authenticate(username.strip(), password): + auth.login_session(request, username.strip()) + flash(request, "Bem-vindo de volta.", "success") + return RedirectResponse("/", status_code=303) + + flash(request, "Usuário ou senha inválidos.", "error") + return RedirectResponse("/login", status_code=303) + + +@router.post("/logout") +def logout(request: Request, csrf_token: str = Form("")): + if auth.check_csrf(request, csrf_token): + auth.logout_session(request) + return RedirectResponse("/login", status_code=303) diff --git a/app/routes/dashboard_routes.py b/app/routes/dashboard_routes.py new file mode 100644 index 0000000..60705cb --- /dev/null +++ b/app/routes/dashboard_routes.py @@ -0,0 +1,48 @@ +"""Dashboard: KPIs, gastos por mês e por fornecedor, últimas notas.""" + +from __future__ import annotations + +from fastapi import APIRouter, Request + +from .. import database as db +from ..storage import money +from ..templating import render + +router = APIRouter() + +_MESES = ["jan", "fev", "mar", "abr", "mai", "jun", "jul", "ago", "set", "out", "nov", "dez"] + + +def _mes_label(ym: str) -> str: + try: + year, month = ym.split("-") + return f"{_MESES[int(month) - 1]}/{year}" + except (ValueError, IndexError): + return ym + + +@router.get("/") +def dashboard(request: Request): + with db.session() as conn: + totals = db.overall_totals(conn) + months = [dict(r) for r in db.monthly_totals(conn)] + suppliers = [dict(r) for r in db.supplier_totals(conn, limit=8)] + recent = [dict(r) for r in db.list_fiscal(conn)][:10] + + max_month = max((m["total"] for m in months), default=0) or 1 + for m in months: + m["label"] = _mes_label(m["month"]) + m["pct"] = round(m["total"] / max_month * 100, 1) + max_sup = max((s["total"] for s in suppliers), default=0) or 1 + for s in suppliers: + s["pct"] = round(s["total"] / max_sup * 100, 1) + + return render( + request, + "dashboard.html", + totals=totals, + months=months, + suppliers=suppliers, + recent=recent, + money=money, + ) diff --git a/app/routes/documents_routes.py b/app/routes/documents_routes.py new file mode 100644 index 0000000..dada051 --- /dev/null +++ b/app/routes/documents_routes.py @@ -0,0 +1,137 @@ +"""CRUD de documentos fiscais confirmados.""" + +from __future__ import annotations + +from fastapi import APIRouter, Form, Query, Request +from starlette.responses import RedirectResponse + +from .. import auth +from .. import database as db +from ..dates import resolve_purchase_date +from ..storage import money +from ..templating import flash, render + +router = APIRouter() + + +def _parse_money(value: str) -> float | None: + text = (value or "").strip().replace("R$", "").replace(" ", "") + if not text: + return None + if "," in text and "." in text: + text = text.replace(".", "").replace(",", ".") + elif "," in text: + text = text.replace(",", ".") + try: + return round(float(text), 2) + except ValueError: + return None + + +@router.get("/documents") +def list_documents( + request: Request, + start: str = Query(""), + end: str = Query(""), + supplier: str = Query(""), +): + with db.session() as conn: + rows = [dict(r) for r in db.list_fiscal(conn, start=start or None, end=end or None, supplier=supplier or None)] + total = sum(r["total_paid"] for r in rows) + return render( + request, + "documents_list.html", + rows=rows, + total=total, + filters={"start": start, "end": end, "supplier": supplier}, + money=money, + ) + + +@router.get("/documents/new") +def new_form(request: Request): + return render(request, "document_form.html", doc=None, mode="new") + + +@router.post("/documents/new") +def create_document( + request: Request, + csrf_token: str = Form(""), + purchase_date: str = Form(""), + supplier_name: str = Form(""), + total_paid: str = Form(""), +): + if not auth.check_csrf(request, csrf_token): + flash(request, "Sessão expirada.", "error") + return RedirectResponse("/documents/new", status_code=303) + + total = _parse_money(total_paid) + supplier = supplier_name.strip() + if not supplier or total is None: + flash(request, "Informe fornecedor e valor válidos.", "error") + return RedirectResponse("/documents/new", status_code=303) + + with db.session() as conn: + db.create_fiscal( + conn, + purchase_date=resolve_purchase_date(purchase_date), + supplier_name=supplier, + total_paid=total, + ) + flash(request, "Documento lançado.", "success") + return RedirectResponse("/documents", status_code=303) + + +@router.get("/documents/{doc_id}/edit") +def edit_form(request: Request, doc_id: int): + with db.session() as conn: + doc = db.get_fiscal(conn, doc_id) + if doc is None: + flash(request, "Documento não encontrado.", "error") + return RedirectResponse("/documents", status_code=303) + return render(request, "document_form.html", doc=dict(doc), mode="edit") + + +@router.post("/documents/{doc_id}/edit") +def update_document( + request: Request, + doc_id: int, + csrf_token: str = Form(""), + purchase_date: str = Form(""), + supplier_name: str = Form(""), + total_paid: str = Form(""), +): + if not auth.check_csrf(request, csrf_token): + flash(request, "Sessão expirada.", "error") + return RedirectResponse(f"/documents/{doc_id}/edit", status_code=303) + + total = _parse_money(total_paid) + supplier = supplier_name.strip() + if not supplier or total is None: + flash(request, "Informe fornecedor e valor válidos.", "error") + return RedirectResponse(f"/documents/{doc_id}/edit", status_code=303) + + with db.session() as conn: + if db.get_fiscal(conn, doc_id) is None: + flash(request, "Documento não encontrado.", "error") + return RedirectResponse("/documents", status_code=303) + db.update_fiscal( + conn, + doc_id, + purchase_date=resolve_purchase_date(purchase_date), + supplier_name=supplier, + total_paid=total, + ) + flash(request, "Documento atualizado.", "success") + return RedirectResponse("/documents", status_code=303) + + +@router.post("/documents/{doc_id}/delete") +def delete_document(request: Request, doc_id: int, csrf_token: str = Form("")): + if not auth.check_csrf(request, csrf_token): + flash(request, "Sessão expirada.", "error") + return RedirectResponse("/documents", status_code=303) + with db.session() as conn: + db.delete_fiscal(conn, doc_id) + flash(request, "Documento excluído.", "info") + return RedirectResponse("/documents", status_code=303) diff --git a/app/routes/files_routes.py b/app/routes/files_routes.py new file mode 100644 index 0000000..251adbe --- /dev/null +++ b/app/routes/files_routes.py @@ -0,0 +1,39 @@ +"""Servir arquivos enviados SOMENTE via rota autenticada. + +Documentos fiscais contêm CPF/CNPJ, endereços e valores — nunca ficam num +static mount público. O acesso passa pela guarda de sessão do middleware. +""" + +from __future__ import annotations + +from pathlib import Path + +from fastapi import APIRouter, Request +from starlette.responses import FileResponse, Response + +from .. import database as db +from ..config import get_settings + +router = APIRouter() + + +@router.get("/files/{upload_id}") +def serve_file(request: Request, upload_id: int): + settings = get_settings() + with db.session() as conn: + row = db.get_upload(conn, upload_id) + if row is None: + return Response("Arquivo não encontrado.", status_code=404) + + stored = Path(row["stored_path"]).resolve() + upload_root = settings.upload_dir.resolve() + # trava contra path traversal: o arquivo precisa estar dentro de data/uploads + if upload_root not in stored.parents or not stored.is_file(): + return Response("Arquivo indisponível.", status_code=404) + + return FileResponse( + str(stored), + media_type=row["content_type"] or "application/octet-stream", + filename=row["original_name"], + content_disposition_type="inline", + ) diff --git a/app/routes/upload_routes.py b/app/routes/upload_routes.py new file mode 100644 index 0000000..c3126df --- /dev/null +++ b/app/routes/upload_routes.py @@ -0,0 +1,169 @@ +"""Fluxo de importação: upload -> staging (revisão) -> confirmação em lote.""" + +from __future__ import annotations + +from pathlib import Path + +from fastapi import APIRouter, File, Form, Request, UploadFile +from starlette.responses import RedirectResponse + +from .. import auth +from .. import database as db +from ..config import get_settings +from ..dates import resolve_purchase_date +from ..ingestion import extract_file +from ..storage import money, safe_original_name, unique_storage_name +from ..templating import flash, render + +router = APIRouter() + + +def _confidence(legible: bool, uncertain: list[str]) -> str: + if legible and not uncertain: + return "high" + if len(uncertain) >= 2 or not legible: + return "low" + return "medium" + + +@router.post("/upload") +async def upload(request: Request, csrf_token: str = Form(""), files: list[UploadFile] = File(...)): + if not auth.check_csrf(request, csrf_token): + flash(request, "Sessão expirada. Tente novamente.", "error") + return RedirectResponse("/", status_code=303) + + settings = get_settings() + accepted = 0 + rejected = 0 + + with db.session() as conn: + batch_id = db.create_batch(conn) + for upload_file in files: + original = safe_original_name(upload_file.filename or "upload") + suffix = Path(original).suffix.lower() + content = await upload_file.read() + if suffix not in settings.allowed_extensions or len(content) > settings.max_upload_bytes: + rejected += 1 + continue + + stored_name = unique_storage_name(original) + stored_path = settings.upload_dir / stored_name + stored_path.write_bytes(content) + content_type = upload_file.content_type or "application/octet-stream" + upload_id = db.insert_upload(conn, batch_id, original, stored_path, content_type, len(content)) + + try: + extractions = extract_file(stored_path, original) + for raw in extractions: + purchase_date = resolve_purchase_date(raw.purchase_date_raw) + uncertain = list(raw.uncertain_fields) + db.insert_detected( + conn, + upload_id=upload_id, + batch_id=batch_id, + source_file_name=original, + source_page=None, + source_location=original, + raw_text=raw.raw_text, + purchase_date=purchase_date, + supplier_name=raw.supplier_name, + total_paid=raw.total_paid, + confidence=_confidence(raw.legible, uncertain), + field_confidence={}, + legible=raw.legible, + uncertain_fields=uncertain, + extractor=raw.extractor, + ) + accepted += 1 + status = "processed" if extractions else "needs_attention" + message = None if extractions else "Nenhum documento fiscal detectado." + db.update_upload_status(conn, upload_id, status, len(extractions), message) + except Exception as exc: # nunca deixa um arquivo derrubar o lote + db.update_upload_status(conn, upload_id, "failed", 0, str(exc)[:300]) + + if rejected: + flash(request, f"{rejected} arquivo(s) recusado(s). Use PDF/JPG/PNG até {settings.max_upload_bytes // (1024*1024)} MB.", "error") + if accepted == 0: + flash(request, "Nenhum documento foi extraído dos arquivos enviados.", "error") + return RedirectResponse("/", status_code=303) + return RedirectResponse(f"/import/{batch_id}/review", status_code=303) + + +@router.get("/import/{batch_id}/review") +def review(request: Request, batch_id: int): + with db.session() as conn: + batch = db.get_batch(conn, batch_id) + if batch is None: + flash(request, "Lote de importação não encontrado.", "error") + return RedirectResponse("/", status_code=303) + rows = [dict(r) for r in db.staged_documents(conn, batch_id)] + summary = db.batch_summary(conn, batch_id) + + return render( + request, + "staging.html", + batch=dict(batch), + rows=rows, + summary=summary, + money=money, + ) + + +@router.post("/import/{batch_id}/update/{detected_id}") +def update_row( + request: Request, + batch_id: int, + detected_id: int, + csrf_token: str = Form(""), + purchase_date: str = Form(""), + supplier_name: str = Form(""), + total_paid: str = Form(""), +): + if not auth.check_csrf(request, csrf_token): + flash(request, "Sessão expirada.", "error") + return RedirectResponse(f"/import/{batch_id}/review", status_code=303) + + try: + total = float(total_paid.replace(".", "").replace(",", ".")) if "," in total_paid else float(total_paid or 0) + except ValueError: + flash(request, "Valor inválido na correção.", "error") + return RedirectResponse(f"/import/{batch_id}/review", status_code=303) + + resolved_date = resolve_purchase_date(purchase_date) + with db.session() as conn: + db.update_staged( + conn, + detected_id, + purchase_date=resolved_date, + supplier_name=supplier_name.strip(), + total_paid=round(total, 2), + legible=True, + ) + flash(request, "Documento corrigido.", "success") + return RedirectResponse(f"/import/{batch_id}/review", status_code=303) + + +@router.post("/import/{batch_id}/discard/{detected_id}") +def discard_row(request: Request, batch_id: int, detected_id: int, csrf_token: str = Form("")): + if auth.check_csrf(request, csrf_token): + with db.session() as conn: + db.discard_staged(conn, detected_id) + flash(request, "Documento descartado do lote.", "info") + return RedirectResponse(f"/import/{batch_id}/review", status_code=303) + + +@router.post("/import/{batch_id}/confirm") +def confirm(request: Request, batch_id: int, csrf_token: str = Form("")): + if not auth.check_csrf(request, csrf_token): + flash(request, "Sessão expirada.", "error") + return RedirectResponse(f"/import/{batch_id}/review", status_code=303) + + with db.session() as conn: + summary = db.batch_summary(conn, batch_id) + if summary["pendentes"] > 0: + flash(request, f"Ainda há {summary['pendentes']} documento(s) com problema (ilegível ou sem fornecedor/valor). Corrija ou descarte antes de importar.", "error") + return RedirectResponse(f"/import/{batch_id}/review", status_code=303) + inserted = db.confirm_batch(conn, batch_id) + + flash(request, f"{inserted} documento(s) importado(s) com sucesso.", "success") + return RedirectResponse("/documents", status_code=303) diff --git a/app/static/app.js b/app/static/app.js new file mode 100644 index 0000000..bd98f6a --- /dev/null +++ b/app/static/app.js @@ -0,0 +1,59 @@ +(function () { + "use strict"; + + // ---- Alternância de tema (persistida em localStorage) ---- + function currentTheme() { + var explicit = document.documentElement.getAttribute("data-theme"); + if (explicit) return explicit; + return window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light"; + } + var toggle = document.getElementById("themeToggle"); + if (toggle) { + toggle.addEventListener("click", function () { + var next = currentTheme() === "dark" ? "light" : "dark"; + document.documentElement.setAttribute("data-theme", next); + try { localStorage.setItem("theme", next); } catch (e) {} + }); + } + + // ---- Área de upload (clique + arrastar/soltar + nome dos arquivos) ---- + var drop = document.getElementById("filedrop"); + var input = document.getElementById("fileInput"); + var label = document.getElementById("filedropText"); + var form = document.getElementById("uploadForm"); + var btn = document.getElementById("uploadBtn"); + + function describe(files) { + if (!files || !files.length) return "Clique ou arraste arquivos aqui"; + if (files.length === 1) return files[0].name; + return files.length + " arquivos selecionados"; + } + + if (input && label) { + input.addEventListener("change", function () { + label.textContent = describe(input.files); + }); + } + if (drop && input) { + ["dragenter", "dragover"].forEach(function (ev) { + drop.addEventListener(ev, function (e) { e.preventDefault(); drop.classList.add("drag"); }); + }); + ["dragleave", "drop"].forEach(function (ev) { + drop.addEventListener(ev, function (e) { e.preventDefault(); drop.classList.remove("drag"); }); + }); + drop.addEventListener("drop", function (e) { + if (e.dataTransfer && e.dataTransfer.files && e.dataTransfer.files.length) { + input.files = e.dataTransfer.files; + if (label) label.textContent = describe(input.files); + } + }); + } + if (form && btn) { + form.addEventListener("submit", function () { + if (input && input.files && input.files.length) { + btn.disabled = true; + btn.textContent = "Processando…"; + } + }); + } +})(); diff --git a/app/static/styles.css b/app/static/styles.css new file mode 100644 index 0000000..a0bb41a --- /dev/null +++ b/app/static/styles.css @@ -0,0 +1,206 @@ +:root { + --bg: #f3f5f4; + --surface: #ffffff; + --surface-2: #f7faf8; + --border: #e1e7e3; + --text: #16211c; + --muted: #5c6b63; + --faint: #8a978f; + --accent: #0f7a5a; + --accent-ink: #ffffff; + --accent-2: #2f6f8f; + --danger: #b23b3b; + --warn-bg: #fff4e0; + --warn-border: #e6bd76; + --warn-ink: #7a4d10; + --ok-bg: #e2f3ea; + --ok-ink: #14603f; + --err-bg: #fbe4e4; + --err-ink: #8f2626; + --shadow: 0 1px 2px rgba(16,33,26,.05), 0 10px 26px -18px rgba(16,33,26,.22); + --radius: 14px; + --radius-sm: 9px; +} +@media (prefers-color-scheme: dark) { + :root { + --bg: #0e1512; --surface: #141d19; --surface-2: #18231e; --border: #26332c; + --text: #e8efeb; --muted: #9db0a6; --faint: #6c7d74; --accent: #35c491; + --accent-ink: #06231a; --accent-2: #62b4d6; --danger: #e06a6a; + --warn-bg: #33270f; --warn-border: #6b5220; --warn-ink: #f0cd8a; + --ok-bg: #123528; --ok-ink: #7fe0b4; --err-bg: #3a1d1d; --err-ink: #f0a8a8; + --shadow: 0 1px 2px rgba(0,0,0,.3), 0 14px 30px -20px rgba(0,0,0,.7); + } +} +:root[data-theme="light"] { + --bg: #f3f5f4; --surface: #ffffff; --surface-2: #f7faf8; --border: #e1e7e3; + --text: #16211c; --muted: #5c6b63; --faint: #8a978f; --accent: #0f7a5a; + --accent-ink: #ffffff; --accent-2: #2f6f8f; --danger: #b23b3b; + --warn-bg: #fff4e0; --warn-border: #e6bd76; --warn-ink: #7a4d10; + --ok-bg: #e2f3ea; --ok-ink: #14603f; --err-bg: #fbe4e4; --err-ink: #8f2626; + --shadow: 0 1px 2px rgba(16,33,26,.05), 0 10px 26px -18px rgba(16,33,26,.22); +} +:root[data-theme="dark"] { + --bg: #0e1512; --surface: #141d19; --surface-2: #18231e; --border: #26332c; + --text: #e8efeb; --muted: #9db0a6; --faint: #6c7d74; --accent: #35c491; + --accent-ink: #06231a; --accent-2: #62b4d6; --danger: #e06a6a; + --warn-bg: #33270f; --warn-border: #6b5220; --warn-ink: #f0cd8a; + --ok-bg: #123528; --ok-ink: #7fe0b4; --err-bg: #3a1d1d; --err-ink: #f0a8a8; + --shadow: 0 1px 2px rgba(0,0,0,.3), 0 14px 30px -20px rgba(0,0,0,.7); +} + +* { box-sizing: border-box; } +html, body { margin: 0; padding: 0; } +body { + background: var(--bg); + color: var(--text); + font-family: system-ui, -apple-system, "Segoe UI", Roboto, sans-serif; + line-height: 1.5; + font-variant-numeric: tabular-nums; + -webkit-font-smoothing: antialiased; +} +h1, h2, h3 { text-wrap: balance; margin: 0; } +a { color: var(--accent); } +.money { font-variant-numeric: tabular-nums; } +.muted { color: var(--muted); } +.small { font-size: .82rem; } +.nowrap { white-space: nowrap; } +.r { text-align: right; } + +/* Topbar */ +.topbar { + display: flex; align-items: center; gap: 18px; + padding: 12px 22px; background: var(--surface); border-bottom: 1px solid var(--border); + position: sticky; top: 0; z-index: 10; +} +.brand { display: flex; align-items: center; gap: 9px; font-weight: 680; text-decoration: none; color: var(--text); font-size: 1.05rem; } +.brand-mark { font-size: 1.2rem; } +.brand-mark.lg { font-size: 2.2rem; } +.nav { display: flex; align-items: center; gap: 6px; margin-right: auto; } +.nav a { padding: 7px 12px; border-radius: var(--radius-sm); text-decoration: none; color: var(--muted); font-weight: 550; font-size: .92rem; } +.nav a:hover { background: var(--surface-2); color: var(--text); } +.nav a.active { color: var(--accent); background: var(--surface-2); } +.topbar-right { display: flex; align-items: center; gap: 10px; } +.user { font-size: .85rem; color: var(--muted); } +.theme-toggle { background: var(--surface-2); border: 1px solid var(--border); border-radius: 8px; width: 36px; height: 36px; cursor: pointer; color: var(--text); font-size: 1rem; } +.theme-toggle:hover { border-color: var(--accent); } +.inline { display: inline; } + +/* Layout */ +.main { max-width: 1080px; margin: 0 auto; padding: 26px 20px 70px; display: flex; flex-direction: column; gap: 20px; } +.main-centered { min-height: 100vh; display: grid; place-items: center; padding: 24px; } +.page-head { display: flex; align-items: flex-end; justify-content: space-between; gap: 16px; } +.card { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); box-shadow: var(--shadow); padding: 18px 20px; } +.card-head { display: flex; align-items: center; justify-content: space-between; margin-bottom: 12px; } +.card h3 { font-size: 1rem; margin-bottom: 12px; } +.grid-2 { display: grid; grid-template-columns: 1fr 1fr; gap: 18px; } + +/* Buttons */ +.btn { display: inline-flex; align-items: center; justify-content: center; gap: 6px; border: 1px solid transparent; border-radius: var(--radius-sm); padding: 9px 15px; font: inherit; font-weight: 600; font-size: .9rem; cursor: pointer; text-decoration: none; } +.btn-primary { background: var(--accent); color: var(--accent-ink); } +.btn-primary:hover { filter: brightness(1.06); } +.btn-primary:disabled { opacity: .5; cursor: not-allowed; } +.btn-ghost { background: transparent; border-color: var(--border); color: var(--text); } +.btn-ghost:hover { background: var(--surface-2); } +.btn-danger { background: transparent; border-color: var(--danger); color: var(--danger); } +.btn-danger:hover { background: var(--danger); color: #fff; } +.btn-sm { padding: 6px 11px; font-size: .84rem; } +.btn-lg { padding: 12px 20px; font-size: 1rem; } +.btn-block { width: 100%; } +.link { color: var(--accent); text-decoration: none; font-weight: 550; } +.link:hover { text-decoration: underline; } +.linkbtn { background: none; border: 0; padding: 0 0 0 10px; color: var(--accent); font: inherit; font-weight: 550; cursor: pointer; } +.linkbtn.danger { color: var(--danger); } + +/* Forms */ +.form { display: flex; flex-direction: column; gap: 14px; } +.field { display: flex; flex-direction: column; gap: 5px; } +.field > span { font-size: .78rem; font-weight: 600; color: var(--muted); text-transform: uppercase; letter-spacing: .03em; } +.field.grow { flex: 1; } +input[type=text], input[type=password], input[type=date], input[type=file] { + font: inherit; color: var(--text); background: var(--surface-2); + border: 1px solid var(--border); border-radius: var(--radius-sm); padding: 9px 11px; min-height: 40px; width: 100%; +} +input:focus-visible, button:focus-visible, a:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; } +.form-actions, .filters-actions { display: flex; gap: 10px; align-items: center; } +.form-card { max-width: 520px; } + +/* Login */ +.login-card { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); box-shadow: var(--shadow); padding: 30px; width: 100%; max-width: 380px; } +.login-head { text-align: center; margin-bottom: 20px; display: flex; flex-direction: column; align-items: center; gap: 2px; } +.login-head h1 { font-size: 1.4rem; } + +/* Flash */ +.flash-stack { display: flex; flex-direction: column; gap: 8px; } +.flash { padding: 11px 14px; border-radius: var(--radius-sm); font-size: .9rem; border: 1px solid transparent; } +.flash-success { background: var(--ok-bg); color: var(--ok-ink); } +.flash-error { background: var(--err-bg); color: var(--err-ink); } +.flash-info { background: var(--surface-2); color: var(--muted); border-color: var(--border); } + +/* Upload panel */ +.upload-panel { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); box-shadow: var(--shadow); padding: 20px; display: grid; grid-template-columns: 1fr auto; gap: 18px; align-items: center; } +.upload-panel h2 { font-size: 1.1rem; margin-bottom: 4px; } +.upload-form { display: flex; gap: 12px; align-items: center; } +.filedrop { display: flex; flex-direction: column; align-items: center; justify-content: center; gap: 4px; border: 1.5px dashed var(--border); border-radius: var(--radius-sm); padding: 14px 22px; cursor: pointer; background: var(--surface-2); color: var(--muted); min-width: 240px; text-align: center; } +.filedrop.drag { border-color: var(--accent); color: var(--accent); } +.filedrop-icon { font-size: 1.2rem; } +.filedrop-text { font-size: .85rem; } + +/* KPIs */ +.kpis { display: grid; grid-template-columns: repeat(3, 1fr); gap: 14px; } +.kpi { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); box-shadow: var(--shadow); padding: 16px 18px; } +.kpi-label { font-size: .7rem; text-transform: uppercase; letter-spacing: .06em; color: var(--faint); font-weight: 600; } +.kpi-value { display: block; font-size: 1.55rem; font-weight: 680; margin-top: 6px; letter-spacing: -.02em; } +.kpi-value.money { color: var(--accent); } +.kpi-foot { font-size: .78rem; color: var(--muted); } + +/* Bar lists */ +.barlist { display: flex; flex-direction: column; } +.barrow { display: grid; grid-template-columns: 92px 1fr auto; gap: 12px; align-items: center; padding: 7px 0; } +.barrow + .barrow { border-top: 1px solid var(--surface-2); } +.barlabel { font-size: .84rem; color: var(--muted); overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.bartrack { height: 10px; background: var(--surface-2); border-radius: 999px; overflow: hidden; } +.barfill { display: block; height: 100%; border-radius: 999px; background: linear-gradient(90deg, var(--accent-2), var(--accent)); } +.barfill.alt { background: var(--accent); } +.barval { font-size: .84rem; font-weight: 600; white-space: nowrap; } + +/* Tables */ +.table-scroll { overflow-x: auto; } +.table { width: 100%; border-collapse: collapse; font-size: .9rem; } +.table thead th { text-align: left; font-size: .7rem; text-transform: uppercase; letter-spacing: .05em; color: var(--faint); font-weight: 600; padding: 10px 12px; border-bottom: 1px solid var(--border); } +.table tbody td { padding: 11px 12px; border-bottom: 1px solid var(--surface-2); } +.table tbody tr:hover td { background: var(--surface-2); } +.table tfoot td { padding: 11px 12px; font-weight: 650; border-top: 2px solid var(--border); } + +/* Filters */ +.filters { display: flex; gap: 12px; align-items: flex-end; flex-wrap: wrap; } + +/* Staging */ +.summary-banner { display: flex; gap: 28px; background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); box-shadow: var(--shadow); padding: 18px 22px; } +.summary-item { display: flex; flex-direction: column; gap: 2px; } +.summary-label { font-size: .72rem; text-transform: uppercase; letter-spacing: .06em; color: var(--faint); font-weight: 600; } +.summary-value { font-size: 1.5rem; font-weight: 700; } +.summary-value.money { color: var(--accent); } +.summary-item.warn .summary-value { color: var(--warn-ink); } +.staging-list { display: flex; flex-direction: column; gap: 12px; } +.staging-row { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); box-shadow: var(--shadow); padding: 14px 16px; display: grid; grid-template-columns: 1fr auto; gap: 8px 14px; align-items: end; } +.staging-row.row-warn { border-color: var(--warn-border); background: var(--warn-bg); } +.staging-badges { grid-column: 1 / -1; display: flex; flex-wrap: wrap; gap: 6px; } +.badge { font-size: .68rem; font-weight: 600; padding: 2px 8px; border-radius: 999px; border: 1px solid var(--border); color: var(--muted); background: var(--surface-2); text-transform: uppercase; letter-spacing: .03em; } +.badge-warn { background: var(--warn-bg); color: var(--warn-ink); border-color: var(--warn-border); } +.badge-high { color: var(--ok-ink); } +.badge-low { color: var(--err-ink); } +.badge-fields { text-transform: none; letter-spacing: 0; } +.staging-form { grid-column: 1; display: flex; gap: 12px; align-items: end; flex-wrap: wrap; } +.staging-form .field { min-width: 130px; } +.staging-actions { display: flex; gap: 8px; align-items: center; } +.staging-discard { grid-column: 2; } +.staging-source { grid-column: 1 / -1; font-size: .74rem; color: var(--faint); } +.confirm-bar { position: sticky; bottom: 0; display: flex; align-items: center; justify-content: space-between; gap: 16px; background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); box-shadow: var(--shadow); padding: 14px 18px; } + +@media (max-width: 820px) { + .grid-2, .kpis { grid-template-columns: 1fr; } + .upload-panel { grid-template-columns: 1fr; } + .staging-row { grid-template-columns: 1fr; } + .staging-discard { grid-column: 1; } + .nav a:not(.btn) { display: none; } +} diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 0000000..4bbebb9 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,33 @@ +"""Helpers de armazenamento seguro dos arquivos enviados.""" + +from __future__ import annotations + +import os +import re +from pathlib import Path + +from .config import get_settings + + +def safe_original_name(name: str) -> str: + return os.path.basename(name or "").replace("\x00", "") or "upload" + + +def unique_storage_name(original: str) -> str: + upload_dir = get_settings().upload_dir + stem = re.sub(r"[^A-Za-z0-9_.-]+", "_", Path(original).stem).strip("._") or "upload" + suffix = Path(original).suffix.lower() + index = 0 + while True: + candidate = f"{stem}{'-' + str(index) if index else ''}{suffix}" + if not (upload_dir / candidate).exists(): + return candidate + index += 1 + + +def money(value: object) -> str: + try: + amount = float(value or 0) + except (TypeError, ValueError): + amount = 0.0 + return f"R$ {amount:,.2f}".replace(",", "X").replace(".", ",").replace("X", ".") diff --git a/app/templates/base.html b/app/templates/base.html new file mode 100644 index 0000000..dd02bac --- /dev/null +++ b/app/templates/base.html @@ -0,0 +1,61 @@ + + + + + + {% block title %}{{ app_name }}{% endblock %} · {{ app_name }} + + + + + + {% if not hide_nav %} +
+ + 🧾 + {{ app_name }} + + +
+ + {% if current_user %} + {{ current_user }} +
+ + +
+ {% endif %} +
+
+ {% endif %} + +
+ {% if flashes %} +
+ {% for f in flashes %} +
{{ f.message }}
+ {% endfor %} +
+ {% endif %} + + {% block content %}{% endblock %} +
+ + + {% block scripts %}{% endblock %} + + diff --git a/app/templates/dashboard.html b/app/templates/dashboard.html new file mode 100644 index 0000000..e4d14c2 --- /dev/null +++ b/app/templates/dashboard.html @@ -0,0 +1,97 @@ +{% extends "base.html" %} +{% block title %}Dashboard{% endblock %} +{% block content %} + +
+
+

Importar documentos

+

Envie PDFs ou imagens de notas e cupons fiscais. A leitura é feita + {% if openai_enabled %}por IA (OpenAI){% else %}por OCR/heurística local{% endif %}, + e você confirma antes de gravar.

+
+
+ + + +
+
+ +
+
+ Total gasto + {{ totals.total | money }} + {{ totals.count }} documento(s) +
+
+ Ticket médio + {{ totals.avg | money }} + por documento +
+
+ Meses com registro + {{ months | length }} + {{ suppliers | length }} fornecedor(es) no top +
+
+ +
+
+

Gastos por mês

+ {% if months %} +
+ {% for m in months %} +
+ {{ m.label }} + + {{ m.total | money }} +
+ {% endfor %} +
+ {% else %}

Sem dados ainda.

{% endif %} +
+ +
+

Gastos por fornecedor

+ {% if suppliers %} +
+ {% for s in suppliers %} +
+ {{ s.supplier_name }} + + {{ s.total | money }} +
+ {% endfor %} +
+ {% else %}

Sem dados ainda.

{% endif %} +
+
+ +
+
+

Últimos documentos

+ Ver todos +
+ {% if recent %} +
+ + + + {% for d in recent %} + + + + + + + {% endfor %} + +
DataFornecedorValor
{{ d.purchase_date }}{{ d.supplier_name }}{{ d.total_paid | money }}editar
+
+ {% else %}

Nenhum documento confirmado. Comece importando acima.

{% endif %} +
+ +{% endblock %} diff --git a/app/templates/document_form.html b/app/templates/document_form.html new file mode 100644 index 0000000..2798618 --- /dev/null +++ b/app/templates/document_form.html @@ -0,0 +1,36 @@ +{% extends "base.html" %} +{% block title %}{{ 'Editar documento' if mode == 'edit' else 'Novo lançamento' }}{% endblock %} +{% block content %} + +
+
+

{{ 'Editar documento' if mode == 'edit' else 'Novo lançamento' }}

+

{{ 'Ajuste os campos e salve.' if mode == 'edit' else 'Lançamento manual de uma despesa fiscal.' }}

+
+ Voltar +
+ +
+
+ + + + +
+ + Cancelar +
+
+
+ +{% endblock %} diff --git a/app/templates/documents_list.html b/app/templates/documents_list.html new file mode 100644 index 0000000..93256fd --- /dev/null +++ b/app/templates/documents_list.html @@ -0,0 +1,60 @@ +{% extends "base.html" %} +{% block title %}Documentos{% endblock %} +{% block content %} + +
+
+

Documentos

+

{{ rows | length }} registro(s) · total {{ total | money }}

+
+ + Novo lançamento +
+ +
+
+ + + +
+ + Limpar +
+
+
+ +
+ {% if rows %} +
+ + + + + + {% for d in rows %} + + + + + + + + {% endfor %} + + + + +
DataFornecedorValorOrigemAções
{{ d.purchase_date }}{{ d.supplier_name }}{{ d.total_paid | money }}{{ d.source_location }} + Editar +
+ + +
+
Total filtrado{{ total | money }}
+
+ {% else %} +

Nenhum documento encontrado. Lançar manualmente ou importar.

+ {% endif %} +
+ +{% endblock %} diff --git a/app/templates/login.html b/app/templates/login.html new file mode 100644 index 0000000..a777dbb --- /dev/null +++ b/app/templates/login.html @@ -0,0 +1,23 @@ +{% extends "base.html" %} +{% block title %}Entrar{% endblock %} +{% block content %} + +{% endblock %} diff --git a/app/templates/staging.html b/app/templates/staging.html new file mode 100644 index 0000000..33bc7eb --- /dev/null +++ b/app/templates/staging.html @@ -0,0 +1,92 @@ +{% extends "base.html" %} +{% block title %}Revisar importação{% endblock %} +{% block content %} + +
+
+

Revisar antes de importar

+

Confira os documentos extraídos. Corrija os ilegíveis e confirme.

+
+ Cancelar +
+ +
+
+ Documentos + {{ summary.count }} +
+
+ Valor total + {{ summary.total | money }} +
+ {% if summary.pendentes %} +
+ Precisam de correção + {{ summary.pendentes }} +
+ {% endif %} +
+ +{% if not rows %} +

Nenhum documento neste lote. Voltar.

+{% else %} + +
+ {% for r in rows %} +
+
+ {% if not r.legible %}Ilegível — revise{% endif %} + {% if r.legible and (not r.supplier_name or r.total_paid is none) %}Incompleto — preencha{% endif %} + {{ r.confidence }} + {{ 'IA' if r.extractor == 'openai' else 'OCR local' }} + {% if r.uncertain_fields and r.uncertain_fields != '[]' %} + incertos: {{ r.uncertain_fields | replace('[','') | replace(']','') | replace('"','') }} + {% endif %} +
+
+ + + + +
+ + Ver arquivo +
+
+
+ + +
+
{{ r.source_file_name }}
+
+ {% endfor %} +
+ +
+
+ {% if summary.pendentes %} + Corrija os {{ summary.pendentes }} documento(s) pendente(s) antes de importar. + {% else %} + Tudo pronto para importar. + {% endif %} +
+
+ + +
+
+ +{% endif %} +{% endblock %} diff --git a/app/templating.py b/app/templating.py new file mode 100644 index 0000000..70bb359 --- /dev/null +++ b/app/templating.py @@ -0,0 +1,41 @@ +"""Configuração do Jinja2 e helpers de renderização/flash.""" + +from __future__ import annotations + +from pathlib import Path + +from fastapi.templating import Jinja2Templates +from starlette.requests import Request +from starlette.responses import HTMLResponse + +from . import auth +from .config import get_settings +from .storage import money + +TEMPLATES_DIR = Path(__file__).resolve().parent / "templates" + +templates = Jinja2Templates(directory=str(TEMPLATES_DIR)) +templates.env.filters["money"] = money + + +def flash(request: Request, message: str, level: str = "info") -> None: + bucket = request.session.setdefault("_flash", []) + bucket.append({"message": message, "level": level}) + + +def pop_flash(request: Request) -> list[dict[str, str]]: + return request.session.pop("_flash", []) + + +def render(request: Request, name: str, status_code: int = 200, **context) -> HTMLResponse: + settings = get_settings() + base = { + "request": request, + "current_user": auth.current_user(request), + "csrf_token": auth.get_csrf_token(request), + "flashes": pop_flash(request), + "openai_enabled": settings.openai_enabled, + "app_name": "Lernotafiscal", + } + base.update(context) + return templates.TemplateResponse(request, name, base, status_code=status_code) diff --git a/deploy/lernotafiscal.service b/deploy/lernotafiscal.service new file mode 100644 index 0000000..210681a --- /dev/null +++ b/deploy/lernotafiscal.service @@ -0,0 +1,21 @@ +[Unit] +Description=Lernotafiscal (FastAPI/Uvicorn) +After=network.target + +[Service] +Type=simple +User=lernotafiscal +Group=lernotafiscal +WorkingDirectory=/opt/lernotafiscal +EnvironmentFile=/opt/lernotafiscal/.env +ExecStart=/opt/lernotafiscal/.venv/bin/uvicorn app.main:app --host 127.0.0.1 --port 8000 --workers 2 +Restart=on-failure +RestartSec=3 +# Hardening básico +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=full +ReadWritePaths=/opt/lernotafiscal/data + +[Install] +WantedBy=multi-user.target diff --git a/deploy/nginx.conf b/deploy/nginx.conf new file mode 100644 index 0000000..c731161 --- /dev/null +++ b/deploy/nginx.conf @@ -0,0 +1,21 @@ +# Nginx como proxy reverso do Lernotafiscal. +# Copie para /etc/nginx/sites-available/lernotafiscal, ajuste server_name e +# habilite com: ln -s .../sites-available/lernotafiscal /etc/nginx/sites-enabled/ +# Depois rode `certbot --nginx -d seu.dominio.com` para gerar o bloco TLS/443. + +server { + listen 80; + server_name seu.dominio.com; + + # Precisa acomodar MAX_UPLOAD_BYTES (12 MB por padrão) + folga do multipart. + client_max_body_size 16m; + + location / { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 120s; + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..1c1555c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,37 @@ +services: + app: + build: + context: . + dockerfile: Dockerfile + + restart: unless-stopped + + environment: + SECRET_KEY: ${SECRET_KEY:?SECRET_KEY obrigatoria} + SESSION_HTTPS_ONLY: ${SESSION_HTTPS_ONLY:-true} + + ADMIN_USERNAME: ${ADMIN_USERNAME:?ADMIN_USERNAME obrigatorio} + ADMIN_PASSWORD: ${ADMIN_PASSWORD:?ADMIN_PASSWORD obrigatoria} + + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + OPENAI_MODEL: ${OPENAI_MODEL:-gpt-4o} + OPENAI_MAX_PAGES: ${OPENAI_MAX_PAGES:-8} + + DB_PATH: ${DB_PATH:-data/app.sqlite3} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-12582912} + + HOST: ${HOST:-0.0.0.0} + PORT: ${PORT:-8000} + + # Se o Dockerfile já define o comando correto, esta variável pode + # permanecer no padrão ou ser removida se não for utilizada por ele. + APP_MODULE: ${APP_MODULE:-app.main:app} + + volumes: + - lernotafiscal-data:/app/data + + expose: + - "8000" + +volumes: + lernotafiscal-data: diff --git a/github.bat b/github.bat new file mode 100644 index 0000000..acc6a08 --- /dev/null +++ b/github.bat @@ -0,0 +1,30 @@ +@echo off +echo === INICIANDO UPLOAD PARA GITHUB === + +REM Inicializar repositório Git +echo Inicializando repositorio Git... +git init + +REM Adicionar todos os arquivos +echo Adicionando todos os arquivos... +git add . + +REM Fazer commit inicial +echo Realizando commit inicial... +git commit -m "Commit inicial - upload de todos os arquivos da pasta" + +REM Adicionar repositório remoto +echo Conectando ao repositorio remoto... +git remote add origin https://gitea.aplicativopro.com/wander/LerNotaFiscal.git + +REM Definir branch principal +echo Definindo branch principal como 'main'... +git branch -M main + +REM Fazer push para o GitHub +echo Fazendo upload para o GitHub... +git push -u origin main + +echo === UPLOAD CONCLUIDO COM SUCESSO! === + +pause \ No newline at end of file diff --git a/lernotafiscal/__init__.py b/lernotafiscal/__init__.py new file mode 100644 index 0000000..930eac8 --- /dev/null +++ b/lernotafiscal/__init__.py @@ -0,0 +1,2 @@ +"""Lernotafiscal application package.""" + diff --git a/lernotafiscal/db.py b/lernotafiscal/db.py new file mode 100644 index 0000000..b9067e0 --- /dev/null +++ b/lernotafiscal/db.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +DATA_DIR = ROOT / "data" +UPLOAD_DIR = DATA_DIR / "uploads" +PREVIEW_DIR = DATA_DIR / "previews" +DB_PATH = DATA_DIR / "lernotafiscal.sqlite3" + + +def ensure_storage() -> None: + DATA_DIR.mkdir(exist_ok=True) + UPLOAD_DIR.mkdir(exist_ok=True) + PREVIEW_DIR.mkdir(exist_ok=True) + + +def connect(db_path: Path = DB_PATH) -> sqlite3.Connection: + ensure_storage() + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + return conn + + +def init_db(conn: sqlite3.Connection) -> None: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS uploaded_files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + original_name TEXT NOT NULL, + stored_path TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + detected_count INTEGER NOT NULL DEFAULT 0, + message TEXT, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS detected_documents ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + upload_id INTEGER NOT NULL REFERENCES uploaded_files(id) ON DELETE CASCADE, + source_file_name TEXT NOT NULL, + source_page INTEGER, + source_location TEXT NOT NULL, + raw_text TEXT NOT NULL, + purchase_date TEXT, + supplier_name TEXT, + total_paid REAL, + confidence TEXT NOT NULL, + field_confidence_json TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS fiscal_documents ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + detected_document_id INTEGER REFERENCES detected_documents(id) ON DELETE SET NULL, + source_file_name TEXT NOT NULL, + source_location TEXT NOT NULL, + purchase_date TEXT NOT NULL, + supplier_name TEXT NOT NULL, + total_paid REAL NOT NULL, + confidence TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP + ); + """ + ) + conn.commit() + + +def insert_upload( + conn: sqlite3.Connection, + original_name: str, + stored_path: Path, + content_type: str, + size_bytes: int, +) -> int: + cur = conn.execute( + """ + INSERT INTO uploaded_files (original_name, stored_path, content_type, size_bytes) + VALUES (?, ?, ?, ?) + """, + (original_name, str(stored_path), content_type, size_bytes), + ) + conn.commit() + return int(cur.lastrowid) + + +def update_upload_status( + conn: sqlite3.Connection, + upload_id: int, + status: str, + detected_count: int, + message: str | None = None, +) -> None: + conn.execute( + """ + UPDATE uploaded_files + SET status = ?, detected_count = ?, message = ?, updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (status, detected_count, message, upload_id), + ) + conn.commit() + + +def insert_detected_document( + conn: sqlite3.Connection, + upload_id: int, + candidate: Any, +) -> int: + cur = conn.execute( + """ + INSERT INTO detected_documents ( + upload_id, source_file_name, source_page, source_location, raw_text, + purchase_date, supplier_name, total_paid, confidence, field_confidence_json + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + upload_id, + candidate.source_file_name, + candidate.source_page, + candidate.source_location, + candidate.raw_text, + candidate.purchase_date, + candidate.supplier_name, + candidate.total_paid, + candidate.confidence, + json.dumps(candidate.field_confidence, ensure_ascii=True), + ), + ) + conn.commit() + return int(cur.lastrowid) + + +def pending_documents(conn: sqlite3.Connection) -> list[sqlite3.Row]: + return list( + conn.execute( + """ + SELECT d.*, u.original_name + FROM detected_documents d + JOIN uploaded_files u ON u.id = d.upload_id + WHERE d.status = 'pending' + ORDER BY d.created_at DESC, d.id DESC + """ + ) + ) + + +def uploads_summary(conn: sqlite3.Connection) -> list[sqlite3.Row]: + return list( + conn.execute( + """ + SELECT * + FROM uploaded_files + ORDER BY created_at DESC, id DESC + LIMIT 20 + """ + ) + ) + + +def attention_uploads(conn: sqlite3.Connection) -> list[sqlite3.Row]: + return list( + conn.execute( + """ + SELECT * + FROM uploaded_files + WHERE status IN ('needs_attention', 'failed') + ORDER BY updated_at DESC, id DESC + LIMIT 20 + """ + ) + ) + + +def review_counts(conn: sqlite3.Connection) -> dict[str, int]: + rows = conn.execute( + """ + SELECT status, COUNT(*) AS total + FROM detected_documents + GROUP BY status + """ + ).fetchall() + counts = {"pending": 0, "confirmed": 0, "ignored": 0} + for row in rows: + counts[row["status"]] = int(row["total"]) + return counts + + +def get_detected(conn: sqlite3.Connection, detected_id: int) -> sqlite3.Row | None: + return conn.execute( + "SELECT * FROM detected_documents WHERE id = ?", + (detected_id,), + ).fetchone() + + +def confirm_document( + conn: sqlite3.Connection, + detected_id: int, + purchase_date: str, + supplier_name: str, + total_paid: float, +) -> None: + row = get_detected(conn, detected_id) + if row is None: + raise ValueError("Documento detectado nao encontrado.") + + conn.execute( + """ + UPDATE detected_documents + SET purchase_date = ?, supplier_name = ?, total_paid = ?, + status = 'confirmed', updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (purchase_date, supplier_name, total_paid, detected_id), + ) + conn.execute( + """ + INSERT INTO fiscal_documents ( + detected_document_id, source_file_name, source_location, + purchase_date, supplier_name, total_paid, confidence + ) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + detected_id, + row["source_file_name"], + row["source_location"], + purchase_date, + supplier_name, + total_paid, + row["confidence"], + ), + ) + conn.commit() + + +def ignore_document(conn: sqlite3.Connection, detected_id: int) -> None: + conn.execute( + """ + UPDATE detected_documents + SET status = 'ignored', updated_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + (detected_id,), + ) + conn.commit() + + +def dashboard_documents( + conn: sqlite3.Connection, + start: str | None = None, + end: str | None = None, + supplier: str | None = None, +) -> list[sqlite3.Row]: + clauses = [] + params: list[Any] = [] + if start: + clauses.append("purchase_date >= ?") + params.append(start) + if end: + clauses.append("purchase_date <= ?") + params.append(end) + if supplier: + clauses.append("supplier_name LIKE ?") + params.append(f"%{supplier}%") + + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" + return list( + conn.execute( + f""" + SELECT * + FROM fiscal_documents + {where} + ORDER BY purchase_date DESC, id DESC + """, + params, + ) + ) + + +def monthly_totals(conn: sqlite3.Connection) -> list[sqlite3.Row]: + return list( + conn.execute( + """ + SELECT substr(purchase_date, 1, 7) AS month, SUM(total_paid) AS total, COUNT(*) AS count + FROM fiscal_documents + GROUP BY substr(purchase_date, 1, 7) + ORDER BY month DESC + """ + ) + ) + + +def supplier_totals(conn: sqlite3.Connection) -> list[sqlite3.Row]: + return list( + conn.execute( + """ + SELECT supplier_name, SUM(total_paid) AS total, COUNT(*) AS count + FROM fiscal_documents + GROUP BY supplier_name + ORDER BY total DESC + LIMIT 10 + """ + ) + ) diff --git a/lernotafiscal/extraction.py b/lernotafiscal/extraction.py new file mode 100644 index 0000000..ef8e0ec --- /dev/null +++ b/lernotafiscal/extraction.py @@ -0,0 +1,369 @@ +from __future__ import annotations + +import re +import zlib +from dataclasses import dataclass +from pathlib import Path + +from .db import PREVIEW_DIR + + +PDF_RENDER_SCALE = 3.0 +MAX_BLOCKS_PER_PAGE = 12 +DATE_RE = re.compile(r"\b([0-3]?\d)[/-]([01]?\d)[/-]((?:20)?\d{2})\b") +MONEY_RE = re.compile(r"(? list[PageSource]: + suffix = path.suffix.lower() + if suffix == ".pdf": + return normalize_pdf(path) + if suffix in {".jpg", ".jpeg", ".png"}: + return [normalize_image(path, page_number=1)] + return [PageSource(1, "", None, "unsupported")] + + +def normalize_pdf(path: Path) -> list[PageSource]: + pages = extract_pdf_text_with_pypdf(path) + if not pages: + pages = extract_pdf_text_naive(path) + + preview_images = render_pdf_pages_to_images(path) + page_count = max(len(pages), len(preview_images), count_pdf_pages(path), 1) + normalized: list[PageSource] = [] + for index in range(page_count): + text = pages[index] if index < len(pages) else "" + if not has_usable_text(text) and index < len(preview_images): + ocr_text = extract_image_text(preview_images[index]) + if has_usable_text(ocr_text): + text = ocr_text + normalized.append( + PageSource( + page_number=index + 1, + text=text if has_usable_text(text) else "", + image_path=preview_images[index] if index < len(preview_images) else None, + source_kind="pdf", + ) + ) + return normalized + + +def normalize_image(path: Path, page_number: int) -> PageSource: + text = extract_image_text(path) + return PageSource(page_number, text, path, "image") + + +def extract_pdf_text_with_pypdf(path: Path) -> list[str]: + try: + from pypdf import PdfReader # type: ignore + except Exception: + return [] + + try: + reader = PdfReader(str(path)) + return [ + text if has_usable_text(text) else "" + for text in ((page.extract_text() or "") for page in reader.pages) + ] + except Exception: + return [] + + + +def extract_pdf_text_naive(path: Path) -> list[str]: + data = path.read_bytes() + chunks: list[bytes] = [] + for match in re.finditer(rb"stream\r?\n(.*?)\r?\nendstream", data, re.S): + stream = match.group(1) + try: + chunks.append(zlib.decompress(stream)) + except Exception: + chunks.append(stream) + + decoded = "\n".join(chunk.decode("latin-1", errors="ignore") for chunk in chunks) + if not decoded: + decoded = data.decode("latin-1", errors="ignore") + + text_tokens = re.findall(r"\(([^()]*)\)", decoded) + text = "\n".join(token.replace(r"\)", ")").replace(r"\(", "(") for token in text_tokens) + + page_count = count_pdf_pages(path) + if not has_usable_text(text): + return ["" for _ in range(max(page_count, 1))] + if "\f" in text and page_count > 1: + parts = [clean_text(part) for part in text.split("\f")] + if len(parts) <= page_count * 2 and any(has_usable_text(part) for part in parts): + return [part if has_usable_text(part) else "" for part in parts] + if page_count <= 1: + return [clean_text(text)] + return [clean_text(text)] + ["" for _ in range(max(page_count - 1, 0))] + +def render_pdf_pages_to_images(path: Path) -> list[Path]: + try: + import fitz # type: ignore + except Exception: + return [] + + rendered: list[Path] = [] + try: + PREVIEW_DIR.mkdir(exist_ok=True) + document = fitz.open(path) + for index, page in enumerate(document): + pix = page.get_pixmap(matrix=fitz.Matrix(PDF_RENDER_SCALE, PDF_RENDER_SCALE), alpha=False) + output = PREVIEW_DIR / f"{path.stem}-page-{index + 1}.png" + pix.save(output) + rendered.append(output) + except Exception: + return rendered + return rendered + + +def extract_image_text(path: Path) -> str: + try: + from PIL import Image # type: ignore + import pytesseract # type: ignore + except Exception: + return "" + + try: + image = Image.open(path) + for language in ("por+eng", "eng"): + try: + text = pytesseract.image_to_string(image, lang=language) + except Exception: + continue + if has_usable_text(text): + return text + return "" + except Exception: + return "" + + +def count_pdf_pages(path: Path) -> int: + try: + data = path.read_bytes() + except OSError: + return 0 + matches = re.findall(rb"/Type\s*/Page\b", data) + return len(matches) + + +def detect_documents(pages: list[PageSource], original_name: str) -> list[DetectedDocumentCandidate]: + candidates: list[DetectedDocumentCandidate] = [] + for page in pages: + blocks = segment_blocks(page.text) + if not blocks: + if page.source_kind == "pdf" and not page.text.strip(): + continue + location = f"pagina {page.page_number}" + if page.image_path: + location += f" ({page.image_path.name})" + candidates.append( + build_candidate( + original_name, + page.page_number, + location, + "", + forced_confidence="low", + ) + ) + continue + + for block_index, block in enumerate(blocks, start=1): + location = f"pagina {page.page_number}, bloco {block_index}" + candidates.append(build_candidate(original_name, page.page_number, location, block)) + return candidates + + +def segment_blocks(text: str) -> list[str]: + cleaned = clean_text(text) + if not has_usable_text(cleaned): + return [] + + lines = [line.strip() for line in cleaned.splitlines() if line.strip()] + anchor_indexes = [ + index for index, line in enumerate(lines) + if START_ANCHOR_RE.search(line) or CNPJ_RE.search(line) + ] + + starts: list[int] = [] + for index in anchor_indexes: + start = max(index - 1, 0) if CNPJ_RE.search(lines[index]) else index + if not starts or start - starts[-1] > 3: + starts.append(start) + + if len(starts) <= 1: + return [cleaned] + + blocks: list[str] = [] + for position, start in enumerate(starts): + end = starts[position + 1] if position + 1 < len(starts) else len(lines) + block = "\n".join(lines[start:end]).strip() + if block: + blocks.append(block) + return blocks[:MAX_BLOCKS_PER_PAGE] + + +def build_candidate( + original_name: str, + page_number: int | None, + location: str, + raw_text: str, + forced_confidence: str | None = None, +) -> DetectedDocumentCandidate: + purchase_date = extract_date(raw_text) + supplier = extract_supplier(raw_text) + total = extract_total(raw_text) + + fields = { + "purchase_date": "high" if purchase_date else "low", + "supplier_name": "medium" if supplier else "low", + "total_paid": "high" if total is not None and has_total_label(raw_text) else ("medium" if total is not None else "low"), + } + confidence = forced_confidence or overall_confidence(fields) + return DetectedDocumentCandidate( + source_file_name=original_name, + source_page=page_number, + source_location=location, + raw_text=raw_text, + purchase_date=purchase_date, + supplier_name=supplier, + total_paid=total, + confidence=confidence, + field_confidence=fields, + ) + + +def extract_date(text: str) -> str | None: + match = DATE_RE.search(text) + if not match: + return None + day, month, year = match.groups() + if len(year) == 2: + year = f"20{year}" + return f"{int(year):04d}-{int(month):02d}-{int(day):02d}" + + +def extract_supplier(text: str) -> str | None: + for line in clean_text(text).splitlines()[:8]: + value = line.strip(" :-") + if not value: + continue + if should_skip_supplier_line(value): + continue + if any(ch.isalpha() for ch in value): + return value[:120] + return None + + +def should_skip_supplier_line(line: str) -> bool: + upper = line.upper() + if CNPJ_RE.search(line) or DATE_RE.search(line) or MONEY_RE.search(line): + return True + blocked = ["CUPOM", "NFC", "SAT", "DANFE", "EXTRATO", "VALOR", "TOTAL", "CHAVE", "ENDERECO"] + return any(token in upper for token in blocked) + + +def extract_total(text: str) -> float | None: + lines = [line.strip() for line in clean_text(text).splitlines() if line.strip()] + labelled: list[float] = [] + all_values: list[float] = [] + for line in lines: + values = [parse_money(match.group(1)) for match in MONEY_RE.finditer(line)] + values = [value for value in values if value is not None] + all_values.extend(values) + if LABEL_RE.search(line): + labelled.extend(values) + + if labelled: + return labelled[-1] + if all_values: + return all_values[-1] + return None + + +def parse_money(value: str) -> float | None: + normalized = value.strip() + if "," in normalized: + normalized = normalized.replace(".", "").replace(",", ".") + try: + return round(float(normalized), 2) + except ValueError: + return None + + +def has_total_label(text: str) -> bool: + return LABEL_RE.search(text) is not None + + +def overall_confidence(fields: dict[str, str]) -> str: + values = list(fields.values()) + if all(value == "high" for value in values): + return "high" + if values.count("low") >= 2: + return "low" + return "medium" + + +def clean_text(text: str) -> str: + text = text.replace("\x00", " ") + lines = [re.sub(r"\s+", " ", line).strip() for line in text.splitlines()] + return "\n".join(line for line in lines if line) + + +def has_usable_text(text: str) -> bool: + cleaned = clean_text(text) + if len(cleaned) < 12: + return False + + allowed_extra = set("???????????????????????????????????????????????????$??") + ordinary = sum( + 1 + for ch in cleaned + if ch.isascii() or ch in allowed_extra + ) + printable = sum(1 for ch in cleaned if ch.isprintable()) + alnum = sum(1 for ch in cleaned if ch.isalnum()) + letters = sum(1 for ch in cleaned if ch.isalpha()) + length = max(len(cleaned), 1) + if ordinary / length < 0.85: + return False + if printable / length < 0.9: + return False + if alnum / length < 0.35: + return False + if letters < 3 and not (DATE_RE.search(cleaned) or MONEY_RE.search(cleaned) or CNPJ_RE.search(cleaned)): + return False + + return True diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..75effbf --- /dev/null +++ b/requirements.txt @@ -0,0 +1,15 @@ +# Web app +fastapi +uvicorn[standard] +jinja2 +python-multipart +itsdangerous +bcrypt +python-dotenv + +# Extração +openai +pypdf +PyMuPDF +Pillow +pytesseract diff --git a/scripts/migrate_notas.py b/scripts/migrate_notas.py new file mode 100644 index 0000000..dfe9666 --- /dev/null +++ b/scripts/migrate_notas.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Migra as notas da skill (tabela `notas`) para `fiscal_documents` do app. + +Idempotente: deduplica por (purchase_date, supplier_name, total_paid). + +Uso: + python scripts/migrate_notas.py [caminho_do_banco_origem] + +Origem padrão: Skill/dados/lernotafiscal.db +Destino: o banco do app (DB_PATH / data/app.sqlite3). +""" + +from __future__ import annotations + +import sqlite3 +import sys +from pathlib import Path + +# permite rodar de qualquer lugar +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from app import database as db # noqa: E402 + +DEFAULT_SOURCE = Path(__file__).resolve().parent.parent / "Skill" / "dados" / "lernotafiscal.db" + + +def read_notas(source: Path) -> list[tuple]: + conn = sqlite3.connect(source) + conn.row_factory = sqlite3.Row + try: + rows = conn.execute( + "SELECT data_compra, fornecedor, valor_pago, arquivo_origem FROM notas" + ).fetchall() + finally: + conn.close() + return [(r["data_compra"], r["fornecedor"], float(r["valor_pago"]), r["arquivo_origem"]) for r in rows] + + +def main() -> int: + source = Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_SOURCE + if not source.exists(): + print(f"Banco de origem não encontrado: {source}") + return 1 + + notas = read_notas(source) + inserted = skipped = 0 + with db.session() as conn: + for data_compra, fornecedor, valor, arquivo in notas: + exists = conn.execute( + """ + SELECT 1 FROM fiscal_documents + WHERE purchase_date = ? AND supplier_name = ? AND ROUND(total_paid, 2) = ROUND(?, 2) + LIMIT 1 + """, + (data_compra, fornecedor, valor), + ).fetchone() + if exists: + skipped += 1 + continue + db.create_fiscal( + conn, + purchase_date=data_compra, + supplier_name=fornecedor, + total_paid=valor, + source_file_name=arquivo or "migrado", + source_location="migrado", + confidence="high", + ) + inserted += 1 + totals = db.overall_totals(conn) + + print(f"Origem: {source}") + print(f"Inseridos: {inserted} · Ignorados (duplicados): {skipped}") + print(f"Total em fiscal_documents: {totals['count']} documento(s) · R$ {totals['total']:.2f}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e8d8fe6 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for Lernotafiscal.""" diff --git a/tests/test_app.py b/tests/test_app.py new file mode 100644 index 0000000..0ff5344 --- /dev/null +++ b/tests/test_app.py @@ -0,0 +1,141 @@ +"""Testes do app: fallback de data, autenticação e CRUD de documentos.""" + +import os +import tempfile +import unittest +from datetime import date +from pathlib import Path + + +class DateFallbackTests(unittest.TestCase): + def test_iso_date_is_kept(self): + from app.dates import resolve_purchase_date + + self.assertEqual(resolve_purchase_date("2026-06-09"), "2026-06-09") + + def test_br_date_is_normalized(self): + from app.dates import resolve_purchase_date + + self.assertEqual(resolve_purchase_date("09/06/2026"), "2026-06-09") + + def test_illegible_date_falls_back_to_first_of_month(self): + from app.dates import resolve_purchase_date + + ref = date(2026, 7, 24) + self.assertEqual(resolve_purchase_date("ilegível", reference=ref), "2026-07-01") + self.assertEqual(resolve_purchase_date(None, reference=ref), "2026-07-01") + self.assertEqual(resolve_purchase_date("", reference=ref), "2026-07-01") + + def test_invalid_calendar_date_falls_back(self): + from app.dates import resolve_purchase_date + + ref = date(2026, 7, 24) + self.assertEqual(resolve_purchase_date("2026-13-40", reference=ref), "2026-07-01") + + +class PasswordTests(unittest.TestCase): + def test_hash_and_verify(self): + from app import auth + + h = auth.hash_password("segredo-forte") + self.assertTrue(auth.verify_password("segredo-forte", h)) + self.assertFalse(auth.verify_password("errada", h)) + + +class CrudTests(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + os.environ["DB_PATH"] = str(Path(self._tmp.name) / "crud.sqlite3") + # zera o cache de settings para pegar o DB_PATH novo + from app.config import get_settings + + get_settings.cache_clear() + + def tearDown(self): + os.environ.pop("DB_PATH", None) + from app.config import get_settings + + get_settings.cache_clear() + self._tmp.cleanup() + + def test_create_update_delete_list(self): + from app import database as db + + with db.session() as conn: + doc_id = db.create_fiscal( + conn, purchase_date="2026-07-01", supplier_name="Padaria X", total_paid=10.0 + ) + self.assertEqual(db.overall_totals(conn)["count"], 1) + + db.update_fiscal( + conn, doc_id, purchase_date="2026-07-02", supplier_name="Padaria Y", total_paid=12.5 + ) + row = db.get_fiscal(conn, doc_id) + self.assertEqual(row["supplier_name"], "Padaria Y") + self.assertEqual(row["total_paid"], 12.5) + + rows = db.list_fiscal(conn, supplier="Padaria") + self.assertEqual(len(rows), 1) + + db.delete_fiscal(conn, doc_id) + self.assertEqual(db.overall_totals(conn)["count"], 0) + + +class StagingGateTests(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + os.environ["DB_PATH"] = str(Path(self._tmp.name) / "stage.sqlite3") + from app.config import get_settings + + get_settings.cache_clear() + + def tearDown(self): + os.environ.pop("DB_PATH", None) + from app.config import get_settings + + get_settings.cache_clear() + self._tmp.cleanup() + + def _stage(self, conn, batch_id, upload_id, **over): + base = dict( + upload_id=upload_id, batch_id=batch_id, source_file_name="f.pdf", + source_page=None, source_location="f.pdf", raw_text="", + purchase_date="2026-07-01", supplier_name="Loja", total_paid=10.0, + confidence="high", field_confidence={}, legible=True, + uncertain_fields=[], extractor="local", + ) + base.update(over) + from app import database as db + + return db.insert_detected(conn, **base) + + def test_incomplete_row_counts_as_pending_and_is_not_imported(self): + from app import database as db + + with db.session() as conn: + batch = db.create_batch(conn) + up = db.insert_upload(conn, batch, "f.pdf", Path("f.pdf"), "application/pdf", 1) + self._stage(conn, batch, up) # completo + # legível mas SEM fornecedor -> incompleto, deve contar como pendente + self._stage(conn, batch, up, supplier_name=None, legible=True) + + summary = db.batch_summary(conn, batch) + self.assertEqual(summary["count"], 2) + self.assertEqual(summary["incompletos"], 1) + self.assertEqual(summary["pendentes"], 1) + + def test_confirm_only_promotes_complete_rows(self): + from app import database as db + + with db.session() as conn: + batch = db.create_batch(conn) + up = db.insert_upload(conn, batch, "f.pdf", Path("f.pdf"), "application/pdf", 1) + self._stage(conn, batch, up, supplier_name="Loja A", total_paid=10.0) + self._stage(conn, batch, up, supplier_name="Loja B", total_paid=20.0) + inserted = db.confirm_batch(conn, batch) + self.assertEqual(inserted, 2) + self.assertEqual(db.overall_totals(conn)["total"], 30.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ingestion.py b/tests/test_ingestion.py new file mode 100644 index 0000000..35eec8f --- /dev/null +++ b/tests/test_ingestion.py @@ -0,0 +1,114 @@ +import tempfile +import unittest +from pathlib import Path + +from lernotafiscal import db +from lernotafiscal.extraction import detect_documents, extract_pdf_text_naive, segment_blocks +from lernotafiscal.extraction import PageSource + + +SINGLE_DOC = """ +MERCADO CENTRAL LTDA +CNPJ 12.345.678/0001-90 +DATA 13/07/2026 +ITEM ARROZ 10,00 +VALOR TOTAL R$ 42,50 +""" + + +MULTI_DOC = """ +MERCADO CENTRAL LTDA +CNPJ 12.345.678/0001-90 +DATA 13/07/2026 +VALOR TOTAL R$ 42,50 + +FARMACIA SAUDE +CNPJ 98.765.432/0001-10 +DATA 14/07/2026 +TOTAL A PAGAR R$ 18,90 +""" + + +class ExtractionTests(unittest.TestCase): + def test_single_image_text_creates_one_candidate(self) -> None: + pages = [PageSource(1, SINGLE_DOC, None, "image")] + docs = detect_documents(pages, "cupom.png") + + self.assertEqual(len(docs), 1) + self.assertEqual(docs[0].purchase_date, "2026-07-13") + self.assertEqual(docs[0].supplier_name, "MERCADO CENTRAL LTDA") + self.assertEqual(docs[0].total_paid, 42.50) + + def test_pdf_page_with_multiple_blocks_creates_multiple_candidates(self) -> None: + blocks = segment_blocks(MULTI_DOC) + self.assertEqual(len(blocks), 2) + + docs = detect_documents([PageSource(1, MULTI_DOC, None, "pdf")], "lote.pdf") + self.assertEqual(len(docs), 2) + self.assertEqual(docs[0].supplier_name, "MERCADO CENTRAL LTDA") + self.assertEqual(docs[1].supplier_name, "FARMACIA SAUDE") + + def test_pdf_with_one_document_per_page_creates_review_item_per_page(self) -> None: + pages = [ + PageSource(1, SINGLE_DOC, None, "pdf"), + PageSource(2, SINGLE_DOC.replace("MERCADO CENTRAL LTDA", "POSTO AVENIDA"), None, "pdf"), + ] + docs = detect_documents(pages, "duas-paginas.pdf") + + self.assertEqual(len(docs), 2) + self.assertEqual(docs[0].source_location, "pagina 1, bloco 1") + self.assertEqual(docs[1].source_location, "pagina 2, bloco 1") + self.assertEqual(docs[1].supplier_name, "POSTO AVENIDA") + + def test_naive_pdf_extraction_rejects_binary_streams_without_fake_pages(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + pdf_path = Path(tmp) / "scan.pdf" + pdf_path.write_bytes( + b"%PDF-1.4\n" + b"1 0 obj << /Type /Page >> endobj\n" + b"2 0 obj << /Length 28 >> stream\n" + b"\x01\x02binary\x0cnoise\x0cnot text\xff\n" + b"endstream\n%%EOF" + ) + + pages = extract_pdf_text_naive(pdf_path) + + self.assertEqual(pages, [""]) + + def test_low_confidence_candidate_can_be_corrected_and_confirmed(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + conn = db.connect(Path(tmp) / "test.sqlite3") + try: + db.init_db(conn) + upload_id = db.insert_upload(conn, "scan.png", Path(tmp) / "scan.png", "image/png", 12) + candidate = detect_documents([PageSource(1, "", None, "image")], "scan.png")[0] + detected_id = db.insert_detected_document(conn, upload_id, candidate) + + db.confirm_document(conn, detected_id, "2026-07-15", "PADARIA BOA", 9.75) + docs = db.dashboard_documents(conn) + + self.assertEqual(len(docs), 1) + self.assertEqual(docs[0]["supplier_name"], "PADARIA BOA") + self.assertEqual(docs[0]["total_paid"], 9.75) + finally: + conn.close() + + def test_ignored_documents_do_not_reach_dashboard(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + conn = db.connect(Path(tmp) / "test.sqlite3") + try: + db.init_db(conn) + upload_id = db.insert_upload(conn, "cupom.png", Path(tmp) / "cupom.png", "image/png", 12) + candidate = detect_documents([PageSource(1, SINGLE_DOC, None, "image")], "cupom.png")[0] + detected_id = db.insert_detected_document(conn, upload_id, candidate) + + db.ignore_document(conn, detected_id) + docs = db.dashboard_documents(conn) + + self.assertEqual(docs, []) + finally: + conn.close() + + +if __name__ == "__main__": + unittest.main()