421 lines
18 KiB
Python
421 lines
18 KiB
Python
"""Testes de categorização: palavra-chave, categorize_supplier (cache/limite/IA) e CRUD de categoria."""
|
|
|
|
import os
|
|
import re
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
|
|
class _DbTestCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self._tmp = tempfile.TemporaryDirectory()
|
|
os.environ["DB_PATH"] = str(Path(self._tmp.name) / "cat.sqlite3")
|
|
from app.config import get_settings
|
|
|
|
get_settings.cache_clear()
|
|
from app import categorization
|
|
|
|
categorization.reset_cache()
|
|
|
|
def tearDown(self):
|
|
os.environ.pop("DB_PATH", None)
|
|
os.environ.pop("OPENAI_API_KEY", None)
|
|
os.environ.pop("CATEGORIZATION_MAX_AI_CALLS_PER_BATCH", None)
|
|
os.environ.pop("ADMIN_USERNAME", None)
|
|
os.environ.pop("ADMIN_PASSWORD", None)
|
|
from app.config import get_settings
|
|
|
|
get_settings.cache_clear()
|
|
from app import categorization
|
|
|
|
categorization.reset_cache()
|
|
self._tmp.cleanup()
|
|
|
|
|
|
class FindCategoriaByKeywordTests(_DbTestCase):
|
|
def test_match_found(self):
|
|
from app import database as db
|
|
|
|
with db.session() as conn:
|
|
db.create_categoria(conn, "Alimentação", "MERCADO")
|
|
row = db.find_categoria_by_keyword(conn, "Supermercado Bom Preço")
|
|
self.assertIsNotNone(row)
|
|
self.assertEqual(row["categoria"], "Alimentação")
|
|
|
|
def test_no_match(self):
|
|
from app import database as db
|
|
|
|
with db.session() as conn:
|
|
db.create_categoria(conn, "Alimentação", "MERCADO")
|
|
row = db.find_categoria_by_keyword(conn, "Posto de Gasolina XYZ")
|
|
self.assertIsNone(row)
|
|
|
|
def test_case_insensitive(self):
|
|
from app import database as db
|
|
|
|
with db.session() as conn:
|
|
db.create_categoria(conn, "Transporte", "posto")
|
|
row = db.find_categoria_by_keyword(conn, "POSTO SHELL CENTRO")
|
|
self.assertIsNotNone(row)
|
|
self.assertEqual(row["categoria"], "Transporte")
|
|
|
|
def test_multiple_candidates_first_by_id_wins(self):
|
|
from app import database as db
|
|
|
|
with db.session() as conn:
|
|
first_id = db.create_categoria(conn, "Geral", "MERCADO")
|
|
db.create_categoria(conn, "Específico", "MERCADO CENTRAL")
|
|
row = db.find_categoria_by_keyword(conn, "Mercado Central Ltda")
|
|
self.assertEqual(row["id"], first_id)
|
|
self.assertEqual(row["categoria"], "Geral")
|
|
|
|
def test_reserved_row_never_returned_even_with_empty_keyword(self):
|
|
from app import database as db
|
|
|
|
with db.session() as conn:
|
|
# id=1 "Não Encontrado" já existe com palavra_chave='' (substring de tudo).
|
|
row = db.find_categoria_by_keyword(conn, "Qualquer Fornecedor")
|
|
self.assertIsNone(row)
|
|
self.assertEqual(db.get_categoria(conn, 1)["categoria"], "Não Encontrado")
|
|
|
|
|
|
class CategorizeSupplierTests(_DbTestCase):
|
|
def test_cache_hit_skips_ai(self):
|
|
from app import database as db
|
|
from app import categorization
|
|
|
|
os.environ["OPENAI_API_KEY"] = "test-key"
|
|
from app.config import get_settings
|
|
|
|
get_settings.cache_clear()
|
|
|
|
with db.session() as conn:
|
|
cat_id = db.create_categoria(conn, "Alimentação", "MERCADO")
|
|
batch_id = db.create_batch(conn)
|
|
with patch("app.categorization._call_ai", return_value="Alimentação") as mocked:
|
|
first = categorization.categorize_supplier("Padaria X", conn, batch_id)
|
|
second = categorization.categorize_supplier("padaria x", conn, batch_id)
|
|
self.assertEqual(first, cat_id)
|
|
self.assertEqual(second, cat_id)
|
|
mocked.assert_called_once()
|
|
|
|
def test_batch_limit_reached_skips_ai_and_uses_keyword_fallback(self):
|
|
from app import database as db
|
|
from app import categorization
|
|
|
|
os.environ["OPENAI_API_KEY"] = "test-key"
|
|
os.environ["CATEGORIZATION_MAX_AI_CALLS_PER_BATCH"] = "1"
|
|
from app.config import get_settings
|
|
|
|
get_settings.cache_clear()
|
|
|
|
with db.session() as conn:
|
|
cat_id = db.create_categoria(conn, "Alimentação", "PADARIA")
|
|
batch_id = db.create_batch(conn)
|
|
with patch("app.categorization._call_ai", return_value=None) as mocked:
|
|
categorization.categorize_supplier("Fornecedor Um", conn, batch_id)
|
|
self.assertEqual(mocked.call_count, 1)
|
|
result = categorization.categorize_supplier("Padaria Dois", conn, batch_id)
|
|
# 2º fornecedor: limite já atingido -> IA não é chamada, cai no fallback de palavra-chave
|
|
self.assertEqual(mocked.call_count, 1)
|
|
self.assertEqual(result, cat_id)
|
|
|
|
def test_ai_returns_valid_category(self):
|
|
from app import database as db
|
|
from app import categorization
|
|
|
|
os.environ["OPENAI_API_KEY"] = "test-key"
|
|
from app.config import get_settings
|
|
|
|
get_settings.cache_clear()
|
|
|
|
with db.session() as conn:
|
|
cat_id = db.create_categoria(conn, "Saúde", "FARMACIA")
|
|
batch_id = db.create_batch(conn)
|
|
with patch("app.categorization._call_ai", return_value="Saúde"):
|
|
result = categorization.categorize_supplier("Drogaria Central", conn, batch_id)
|
|
self.assertEqual(result, cat_id)
|
|
|
|
def test_ai_fails_or_invalid_falls_back_to_keyword(self):
|
|
from app import database as db
|
|
from app import categorization
|
|
|
|
os.environ["OPENAI_API_KEY"] = "test-key"
|
|
from app.config import get_settings
|
|
|
|
get_settings.cache_clear()
|
|
|
|
with db.session() as conn:
|
|
cat_id = db.create_categoria(conn, "Transporte", "POSTO")
|
|
batch_id = db.create_batch(conn)
|
|
with patch("app.categorization._call_ai", return_value=None):
|
|
result = categorization.categorize_supplier("Posto Ipiranga", conn, batch_id)
|
|
self.assertEqual(result, cat_id)
|
|
|
|
def test_no_method_finds_category_returns_reserved(self):
|
|
from app import database as db
|
|
from app import categorization
|
|
|
|
with db.session() as conn:
|
|
batch_id = db.create_batch(conn)
|
|
result = categorization.categorize_supplier("Fornecedor Desconhecido", conn, batch_id)
|
|
self.assertEqual(result, db.RESERVED_CATEGORIA_ID)
|
|
|
|
def test_unexpected_internal_failure_never_propagates_and_falls_back_to_reserved(self):
|
|
"""3.2: qualquer exceção não tratada dentro do módulo -> categoria_id=1,
|
|
nunca propaga para o chamador (que criaria fiscal_documents)."""
|
|
from app import database as db
|
|
from app import categorization
|
|
|
|
with db.session() as conn:
|
|
batch_id = db.create_batch(conn)
|
|
with patch("app.database.list_categorias", side_effect=RuntimeError("db explodiu")):
|
|
result = categorization.categorize_supplier("Fornecedor Y", conn, batch_id)
|
|
self.assertEqual(result, db.RESERVED_CATEGORIA_ID)
|
|
|
|
def test_repeated_suppliers_call_ai_once_each_and_respect_batch_limit(self):
|
|
"""Equivalente automatizado da validação manual 6.5: lote com fornecedores
|
|
repetidos chama a IA no máximo uma vez por fornecedor distinto, e respeita
|
|
o limite por lote (observável via logs)."""
|
|
from app import database as db
|
|
from app import categorization
|
|
|
|
os.environ["OPENAI_API_KEY"] = "test-key"
|
|
os.environ["CATEGORIZATION_MAX_AI_CALLS_PER_BATCH"] = "2"
|
|
from app.config import get_settings
|
|
|
|
get_settings.cache_clear()
|
|
|
|
with db.session() as conn:
|
|
db.create_categoria(conn, "Alimentação", "MERCADO")
|
|
batch_id = db.create_batch(conn)
|
|
suppliers = ["Fornecedor A", "Fornecedor A", "Fornecedor B", "Mercado C", "Fornecedor A"]
|
|
with patch("app.categorization._call_ai", return_value=None) as mocked, \
|
|
self.assertLogs("lernotafiscal.categorization", level="INFO") as logs:
|
|
for name in suppliers:
|
|
categorization.categorize_supplier(name, conn, batch_id)
|
|
# 2 fornecedores distintos com IA tentada (A, B) antes do limite=2 ser atingido
|
|
self.assertEqual(mocked.call_count, 2)
|
|
self.assertTrue(any("limite" in msg for msg in logs.output))
|
|
|
|
|
|
class CategoriaCrudTests(_DbTestCase):
|
|
def test_create_get_list_update(self):
|
|
from app import database as db
|
|
|
|
with db.session() as conn:
|
|
cat_id = db.create_categoria(conn, "Lazer", "CINEMA")
|
|
row = db.get_categoria(conn, cat_id)
|
|
self.assertEqual(row["categoria"], "Lazer")
|
|
|
|
db.update_categoria(conn, cat_id, "Entretenimento", "CINEMA")
|
|
row = db.get_categoria(conn, cat_id)
|
|
self.assertEqual(row["categoria"], "Entretenimento")
|
|
|
|
rows = db.list_categorias(conn)
|
|
self.assertTrue(any(r["id"] == cat_id for r in rows))
|
|
# linha reservada sempre presente
|
|
self.assertTrue(any(r["id"] == 1 for r in rows))
|
|
|
|
def test_delete_reassigns_referencing_documents_to_reserved(self):
|
|
from app import database as db
|
|
|
|
with db.session() as conn:
|
|
cat_id = db.create_categoria(conn, "Lazer", "CINEMA")
|
|
doc_id = db.create_fiscal(conn, mes=7, ano=2026, supplier_name="Cinema X", total_paid=30.0)
|
|
conn.execute("UPDATE fiscal_documents SET categoria_id = ? WHERE id = ?", (cat_id, doc_id))
|
|
conn.commit()
|
|
|
|
deleted = db.delete_categoria(conn, cat_id)
|
|
self.assertTrue(deleted)
|
|
self.assertIsNone(db.get_categoria(conn, cat_id))
|
|
self.assertEqual(db.get_fiscal(conn, doc_id)["categoria_id"], db.RESERVED_CATEGORIA_ID)
|
|
|
|
def test_delete_reserved_row_rejected(self):
|
|
from app import database as db
|
|
|
|
with db.session() as conn:
|
|
deleted = db.delete_categoria(conn, db.RESERVED_CATEGORIA_ID)
|
|
self.assertFalse(deleted)
|
|
self.assertIsNotNone(db.get_categoria(conn, db.RESERVED_CATEGORIA_ID))
|
|
|
|
def test_sem_categoria_group_and_filter_cover_legacy_null_rows(self):
|
|
"""Cobre o grupo/filtro 'Sem categoria' (categoria_id IS NULL, documento
|
|
legado nunca reprocessado), distinto da categoria reservada 'Não Encontrado'
|
|
(categoria_id = 1, usada pelo fluxo de categorização ativo)."""
|
|
from app import database as db
|
|
|
|
with db.session() as conn:
|
|
# Legado: create_fiscal nunca seta categoria_id -> permanece NULL.
|
|
legacy_id = db.create_fiscal(
|
|
conn, mes=7, ano=2026, supplier_name="Doc Legado", total_paid=40.0
|
|
)
|
|
self.assertIsNone(db.get_fiscal(conn, legacy_id)["categoria_id"])
|
|
|
|
# Documento processado pelo fluxo, sem match -> categoria_id = 1 (reservado).
|
|
found_id = db.create_fiscal(
|
|
conn, mes=7, ano=2026, supplier_name="Doc Não Encontrado", total_paid=15.0
|
|
)
|
|
conn.execute(
|
|
"UPDATE fiscal_documents SET categoria_id = ? WHERE id = ?",
|
|
(db.RESERVED_CATEGORIA_ID, found_id),
|
|
)
|
|
conn.commit()
|
|
|
|
totals = {r["categoria"]: r for r in db.category_totals(conn)}
|
|
self.assertIn("Sem categoria", totals)
|
|
self.assertEqual(totals["Sem categoria"]["total"], 40.0)
|
|
self.assertIsNone(totals["Sem categoria"]["categoria_id"])
|
|
self.assertIn("Não Encontrado", totals)
|
|
self.assertEqual(totals["Não Encontrado"]["total"], 15.0)
|
|
|
|
none_filtered = db.list_fiscal(conn, category="none")
|
|
self.assertEqual(len(none_filtered), 1)
|
|
self.assertEqual(none_filtered[0]["id"], legacy_id)
|
|
|
|
reserved_filtered = db.list_fiscal(conn, category=str(db.RESERVED_CATEGORIA_ID))
|
|
self.assertEqual(len(reserved_filtered), 1)
|
|
self.assertEqual(reserved_filtered[0]["id"], found_id)
|
|
|
|
self.assertEqual(db.overall_totals(conn, category="none")["count"], 1)
|
|
self.assertEqual(db.monthly_totals(conn, category="none")[0]["total"], 40.0)
|
|
self.assertEqual(db.supplier_totals(conn, category="none")[0]["supplier_name"], "Doc Legado")
|
|
|
|
|
|
class IngestionCategorizationIntegrationTests(_DbTestCase):
|
|
"""6.3: fluxo de ingestão ponta a ponta (confirm_batch) resulta em categoria_id
|
|
correto para os cenários de IA, palavra-chave e 'Não Encontrado'."""
|
|
|
|
def _stage(self, conn, batch_id, upload_id, supplier_name, total_paid):
|
|
from app import database as db
|
|
|
|
return db.insert_detected(
|
|
conn, upload_id=upload_id, batch_id=batch_id, source_file_name="f.pdf",
|
|
source_page=None, source_location="f.pdf", raw_text="",
|
|
mes=7, ano=2026, supplier_name=supplier_name, total_paid=total_paid,
|
|
confidence="high", field_confidence={}, legible=True, uncertain_fields=[], extractor="local",
|
|
)
|
|
|
|
def test_confirm_batch_assigns_categoria_via_ai_keyword_and_reserved(self):
|
|
from app import database as db
|
|
|
|
os.environ["OPENAI_API_KEY"] = "test-key"
|
|
from app.config import get_settings
|
|
|
|
get_settings.cache_clear()
|
|
|
|
with db.session() as conn:
|
|
saude_id = db.create_categoria(conn, "Saúde", "FARMACIA")
|
|
alimentacao_id = db.create_categoria(conn, "Alimentação", "MERCADO")
|
|
|
|
batch = db.create_batch(conn)
|
|
up = db.insert_upload(conn, batch, "f.pdf", Path("f.pdf"), "application/pdf", 1)
|
|
self._stage(conn, batch, up, "Drogaria Central", 10.0) # IA acerta -> Saúde
|
|
self._stage(conn, batch, up, "Mercado da Esquina", 20.0) # IA falha -> palavra-chave
|
|
self._stage(conn, batch, up, "Loja Sem Categoria Nenhuma", 30.0) # nada casa -> reservado
|
|
|
|
def fake_ai(supplier_name, known_categories):
|
|
return "Saúde" if supplier_name == "Drogaria Central" else None
|
|
|
|
with patch("app.categorization._call_ai", side_effect=fake_ai):
|
|
inserted = db.confirm_batch(conn, batch)
|
|
|
|
self.assertEqual(inserted, 3)
|
|
rows = {r["supplier_name"]: r for r in db.list_fiscal(conn)}
|
|
self.assertEqual(rows["Drogaria Central"]["categoria_id"], saude_id)
|
|
self.assertEqual(rows["Mercado da Esquina"]["categoria_id"], alimentacao_id)
|
|
self.assertEqual(rows["Loja Sem Categoria Nenhuma"]["categoria_id"], db.RESERVED_CATEGORIA_ID)
|
|
|
|
def test_confirm_batch_saves_document_with_reserved_category_when_nothing_matches(self):
|
|
"""3.2: nenhum erro bloqueia a criação do fiscal_documents; sem IA/palavra-chave
|
|
casando, o documento é salvo com categoria_id=1 ("Não Encontrado")."""
|
|
from app import database as db
|
|
|
|
with db.session() as conn:
|
|
batch = db.create_batch(conn)
|
|
up = db.insert_upload(conn, batch, "f.pdf", Path("f.pdf"), "application/pdf", 1)
|
|
self._stage(conn, batch, up, "Fornecedor Totalmente Novo", 10.0)
|
|
|
|
inserted = db.confirm_batch(conn, batch)
|
|
|
|
self.assertEqual(inserted, 1)
|
|
row = db.list_fiscal(conn)[0]
|
|
self.assertEqual(row["categoria_id"], db.RESERVED_CATEGORIA_ID)
|
|
|
|
|
|
class CategoriaFormValidationRouteTests(_DbTestCase):
|
|
"""Cobre, via requisição HTTP real (rota + CSRF + sessão), o cenário de spec
|
|
'User submits an invalid categoria form' que não tinha teste automatizado."""
|
|
|
|
def _login(self, client):
|
|
page = client.get("/login")
|
|
token = re.search(r'name="csrf_token" value="([^"]+)"', page.text).group(1)
|
|
client.post(
|
|
"/login",
|
|
data={"csrf_token": token, "username": "admin", "password": "test-pass-123"},
|
|
)
|
|
|
|
def _csrf(self, page_html: str) -> str:
|
|
return re.search(r'name="csrf_token" value="([^"]+)"', page_html).group(1)
|
|
|
|
def test_invalid_new_categoria_form_does_not_create_row(self):
|
|
from fastapi.testclient import TestClient
|
|
from app import database as db
|
|
from app.config import get_settings
|
|
from app.main import app
|
|
|
|
os.environ["ADMIN_USERNAME"] = "admin"
|
|
os.environ["ADMIN_PASSWORD"] = "test-pass-123"
|
|
get_settings.cache_clear()
|
|
|
|
with TestClient(app) as client:
|
|
self._login(client)
|
|
|
|
with db.session() as conn:
|
|
before = len(db.list_categorias(conn))
|
|
|
|
new_page = client.get("/categorias/new")
|
|
resp = client.post(
|
|
"/categorias/new",
|
|
data={"csrf_token": self._csrf(new_page.text), "categoria": "", "palavra_chave": ""},
|
|
)
|
|
self.assertIn("Informe categoria e palavra-chave", resp.text)
|
|
|
|
with db.session() as conn:
|
|
after = len(db.list_categorias(conn))
|
|
self.assertEqual(before, after)
|
|
|
|
def test_invalid_edit_categoria_form_does_not_change_row(self):
|
|
from fastapi.testclient import TestClient
|
|
from app import database as db
|
|
from app.config import get_settings
|
|
from app.main import app
|
|
|
|
os.environ["ADMIN_USERNAME"] = "admin"
|
|
os.environ["ADMIN_PASSWORD"] = "test-pass-123"
|
|
get_settings.cache_clear()
|
|
|
|
with TestClient(app) as client:
|
|
self._login(client)
|
|
|
|
with db.session() as conn:
|
|
cat_id = db.create_categoria(conn, "Lazer", "CINEMA")
|
|
|
|
edit_page = client.get(f"/categorias/{cat_id}/edit")
|
|
resp = client.post(
|
|
f"/categorias/{cat_id}/edit",
|
|
data={"csrf_token": self._csrf(edit_page.text), "categoria": "", "palavra_chave": ""},
|
|
)
|
|
self.assertIn("Informe categoria e palavra-chave", resp.text)
|
|
|
|
with db.session() as conn:
|
|
row = db.get_categoria(conn, cat_id)
|
|
self.assertEqual(row["categoria"], "Lazer")
|
|
self.assertEqual(row["palavra_chave"], "CINEMA")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|