defmodule HiiveRisk.TestModel do @behaviour HiiveRisk.Model def decision(snapshot), do: response(:decision, snapshot) def sft(snapshot), do: response(:sft, snapshot) defp response(stage, snapshot) do Agent.get_and_update(__MODULE__, fn state -> {Map.fetch!(state, stage), Map.update!(state, :calls, &[{stage, snapshot} | &1])} end) end end defmodule HiiveRisk.PipelineTest do use ExUnit.Case, async: false import Ecto.Query alias HiiveRisk.{Repo, RiskCase, TestModel} setup do :ok = Ecto.Adapters.SQL.Sandbox.checkout(Repo) Ecto.Adapters.SQL.Sandbox.mode(Repo, {:shared, self()}) previous = Application.fetch_env!(:hiive_risk, :model_client) Application.put_env(:hiive_risk, :model_client, TestModel) on_exit(fn -> Application.put_env(:hiive_risk, :model_client, previous) end) start_supervised!(%{ id: TestModel, start: {Agent, :start_link, [ fn -> %{ decision: {:ok, %{"p_review" => 0.9, "answer_mass" => 0.99}}, sft: {:ok, %{"risk_level" => 4, "reasons" => ["SHARED_BANK"]}}, calls: [] } end, [name: TestModel] ]} }) snapshot = "test/fixtures/case.json" |> File.read!() |> Jason.decode!() %{snapshot: snapshot} end test "intake is idempotent and rejects changed evidence under the same key", %{snapshot: s} do assert {:ok, receipt} = HiiveRisk.enqueue_review(s, audit_sample: false) assert {:ok, ^receipt} = HiiveRisk.enqueue_review(s, audit_sample: false) atom_key = s |> Map.delete("case_id") |> Map.put(:case_id, s["case_id"]) assert {:ok, ^receipt} = HiiveRisk.enqueue_review(atom_key, audit_sample: false) assert Repo.aggregate(RiskCase, :count) == 1 assert Repo.aggregate(Oban.Job, :count) == 1 refute Repo.get!(RiskCase, receipt.case_id).audit_sample changed = put_in(s, ["account", "age_days"], s["account"]["age_days"] + 1) assert {:error, :idempotency_conflict} = HiiveRisk.enqueue_review(changed, audit_sample: false) assert {:ok, other} = HiiveRisk.enqueue_review(changed, policy_version: "research-v2") refute other.case_id == receipt.case_id assert {:error, %Ecto.Changeset{}} = HiiveRisk.enqueue_review(%{"case_id" => "invalid"}) assert Repo.aggregate(RiskCase, :count) == 2 end test "marketplace transaction rolls back both snapshot and job", %{snapshot: s} do multi = Ecto.Multi.new() |> Ecto.Multi.put(:marketplace_case, s) |> HiiveRisk.enqueue_review(& &1.marketplace_case, audit_sample: false) |> Ecto.Multi.error(:order, :rejected) assert {:error, :order, :rejected, _} = Repo.transaction(multi) assert Repo.aggregate(RiskCase, :count) == 0 assert Repo.aggregate(Oban.Job, :count) == 0 assert Agent.get(TestModel, & &1.calls) == [] end test "committed jobs run both stages, publish results and sort the human worklist", %{ snapshot: s } do {:ok, receipt} = HiiveRisk.enqueue_review(s, audit_sample: false) :ok = HiiveRisk.subscribe(receipt.case_id) assert %{success: 1, failure: 0} = Oban.drain_queue(queue: :risk_decision) assert {:ok, %{status: "pending_sft"}} = HiiveRisk.get_assessment(receipt.case_id) assert_receive {:risk_updated, %{status: "pending_sft"}} assert %{success: 1, failure: 0} = Oban.drain_queue(queue: :risk_sft) assert_receive {:risk_updated, %{status: "completed", risk_level: 4}} Agent.update( TestModel, &Map.put(&1, :sft, {:ok, %{"risk_level" => 5, "reasons" => ["KNOWN_ATTACK_IP"]}}) ) {:ok, second} = HiiveRisk.enqueue_review(%{s | "case_id" => "second"}, audit_sample: false) Oban.drain_queue(queue: :risk_decision) Oban.drain_queue(queue: :risk_sft) assert Enum.map(HiiveRisk.list_reviews(), & &1.case_id) == [second.case_id, receipt.case_id] assert [%{case_id: id}] = HiiveRisk.list_reviews(priority: 4) assert id == receipt.case_id assert {:ok, %{snapshot: ^s, risk_level: 4, reasons: ["SHARED_BANK"]}} = HiiveRisk.get_assessment(receipt.case_id) assert {:ok, _} = HiiveRisk.enqueue_review(s, audit_sample: false) assert %{success: 0} = Oban.drain_queue(queue: :risk_decision) end test "clear boundary, audit, unusual input and low answer mass route safely", %{snapshot: s} do for {p, mass, opts, expected, audit} <- [ {0.019, 0.95, [], "completed", false}, {0.02, 0.95, [], "pending_sft", false}, {0.001, 0.949, [], "pending_sft", false}, {0.001, 0.99, [unusual_input: true], "pending_sft", false}, {0.001, 0.99, [audit_sample: true], "pending_sft", true} ] do Agent.update( TestModel, &Map.put(&1, :decision, {:ok, %{"p_review" => p, "answer_mass" => mass}}) ) item = %{s | "case_id" => Ecto.UUID.generate()} {:ok, receipt} = HiiveRisk.enqueue_review(item, Keyword.merge([audit_sample: false], opts)) Oban.drain_queue(queue: :risk_decision) assert {:ok, result} = HiiveRisk.get_assessment(receipt.case_id) assert result.status == expected assert result.audit == audit if expected == "completed", do: assert(result.disposition == "auto_clear") end end test "conflicts and model disagreement escalate instead of entering the ordinary worklist", %{ snapshot: s } do for {p, level, opts} <- [{0.9, 1, []}, {0.1, 4, []}, {0.9, 4, [conflicting_evidence: true]}] do Agent.update( TestModel, &Map.merge(&1, %{ decision: {:ok, %{"p_review" => p, "answer_mass" => 0.99}}, sft: {:ok, %{"risk_level" => level, "reasons" => ["SHARED_BANK"]}} }) ) {:ok, receipt} = HiiveRisk.enqueue_review(%{s | "case_id" => Ecto.UUID.generate()}, opts) Oban.drain_queue(queue: :risk_decision) Oban.drain_queue(queue: :risk_sft) assert {:ok, %{disposition: "escalate"}} = HiiveRisk.get_assessment(receipt.case_id) end assert HiiveRisk.list_reviews() == [] assert length(HiiveRisk.list_reviews(disposition: "escalate")) == 3 end test "invalid responses retry, exhaust visibly, and can be retried after repair", %{snapshot: s} do Agent.update(TestModel, &Map.put(&1, :decision, {:ok, %{"p_review" => 0}})) {:ok, receipt} = HiiveRisk.enqueue_review(s) :ok = HiiveRisk.subscribe(receipt.case_id) for _ <- 1..5, do: Oban.drain_queue(queue: :risk_decision, with_scheduled: true) assert {:ok, %{status: "failed", disposition: nil, last_error: "invalid_decision_response"}} = HiiveRisk.get_assessment(receipt.case_id) assert_receive {:risk_updated, %{status: "failed"}} assert Repo.aggregate(from(j in Oban.Job, where: j.queue == "risk_sft"), :count) == 0 Agent.update( TestModel, &Map.put(&1, :decision, {:ok, %{"p_review" => 0.9, "answer_mass" => 0.99}}) ) Oban.retry_job(receipt.job_id) Oban.drain_queue(queue: :risk_decision, with_scheduled: true) Agent.update(TestModel, &Map.put(&1, :sft, {:error, :model_request_failed})) Oban.drain_queue(queue: :risk_sft) assert {:ok, %{status: "pending_sft", disposition: nil, last_error: "model_request_failed"}} = HiiveRisk.get_assessment(receipt.case_id) for _ <- 1..4, do: Oban.drain_queue(queue: :risk_sft, with_scheduled: true) assert {:ok, %{status: "failed", disposition: nil}} = HiiveRisk.get_assessment(receipt.case_id) sft_job = Repo.one!(from(j in Oban.Job, where: j.queue == "risk_sft")) Oban.retry_job(sft_job.id) Agent.update( TestModel, &Map.put(&1, :sft, {:ok, %{"risk_level" => 4, "reasons" => ["SHARED_BANK"]}}) ) Oban.drain_queue(queue: :risk_sft, with_scheduled: true) assert {:ok, %{status: "completed", last_error: nil}} = HiiveRisk.get_assessment(receipt.case_id) end end