From 4a35db7b1504f788a2b7f62c81a82ed5cb6e2d7c Mon Sep 17 00:00:00 2001 From: Ayush Das <150268924+AyushDas4890@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:15:49 +0530 Subject: [PATCH 1/5] Fix RTDetrHungarianMatcher crash on infinite cost matrix --- src/transformers/loss/loss_rt_detr.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/transformers/loss/loss_rt_detr.py b/src/transformers/loss/loss_rt_detr.py index cf6d6ad05940..6a7e31e5a645 100644 --- a/src/transformers/loss/loss_rt_detr.py +++ b/src/transformers/loss/loss_rt_detr.py @@ -112,6 +112,9 @@ def forward(self, outputs, targets): giou_cost = -generalized_box_iou(center_to_corners_format(out_bbox), center_to_corners_format(target_bbox)) # Compute the final cost matrix cost_matrix = self.bbox_cost * bbox_cost + self.class_cost * class_cost + self.giou_cost * giou_cost + # eliminate infinite values in cost_matrix to avoid the error ``ValueError: cost matrix is infeasible`` + cost_matrix = torch.minimum(cost_matrix, torch.tensor(1e10)) + cost_matrix = torch.maximum(cost_matrix, torch.tensor(-1e10)) cost_matrix = cost_matrix.view(batch_size, num_queries, -1).cpu() sizes = [len(v["boxes"]) for v in targets] From 56a09bdaf3fd279f41762be2be8ef736b31ea9a5 Mon Sep 17 00:00:00 2001 From: Ayush Das <150268924+AyushDas4890@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:17:32 +0530 Subject: [PATCH 2/5] Add regression test for infinite cost matrix in matcher --- tests/models/rt_detr/test_modeling_rt_detr.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/models/rt_detr/test_modeling_rt_detr.py b/tests/models/rt_detr/test_modeling_rt_detr.py index 79d71105f27f..7a03cf431302 100644 --- a/tests/models/rt_detr/test_modeling_rt_detr.py +++ b/tests/models/rt_detr/test_modeling_rt_detr.py @@ -32,6 +32,7 @@ ) from transformers.testing_utils import ( Expectations, + require_scipy, require_torch, require_torch_accelerator, require_vision, @@ -757,3 +758,40 @@ def test_inference_object_detection_head(self): torch.testing.assert_close(results["scores"][:4], expected_scores, rtol=2e-4, atol=2e-4) self.assertSequenceEqual(results["labels"][:4].tolist(), expected_labels) torch.testing.assert_close(results["boxes"][:4], expected_slice_boxes, rtol=2e-4, atol=2e-4) + +@require_torch +@require_scipy +@require_vision +class RTDetrHungarianMatcherTest(unittest.TestCase): + def test_infinite_costs_do_not_crash_matcher(self): + """ + Regression test for #47000. Under AMP, fp16 sigmoid saturation makes the focal class cost + overflow to +/-inf (the 1e-8 epsilon underflows to 0 in fp16, so ``log(0)`` is hit), and + ``scipy.optimize.linear_sum_assignment`` raised ``ValueError: cost matrix is infeasible``. + """ + from transformers.loss.loss_rt_detr import RTDetrHungarianMatcher + + config = RTDetrConfig(num_labels=4) + matcher = RTDetrHungarianMatcher(config) + + num_queries, num_targets = 4, 3 + pred_boxes = torch.rand(1, num_queries, 4) * 0.5 + 0.25 + targets = [ + { + "class_labels": torch.tensor([0, 1, 2]), + "boxes": torch.rand(num_targets, 4) * 0.5 + 0.25, + } + ] + + for saturating_logit in (-30.0, 30.0): + # fp16 logits as produced under AMP: sigmoid saturates to exactly 0.0 (or 1.0), + # making ``pos_cost_class`` (or ``neg_cost_class``) infinite + logits = torch.full((1, num_queries, config.num_labels), saturating_logit, dtype=torch.float16) + outputs = {"logits": logits, "pred_boxes": pred_boxes} + + indices = matcher(outputs, targets) + + self.assertEqual(len(indices), 1) + row_indices, col_indices = indices[0] + self.assertEqual(len(row_indices), num_targets) + self.assertEqual(len(col_indices), num_targets) From 6bb234dc399d5e3a7f95ef157994d3a07976eb12 Mon Sep 17 00:00:00 2001 From: Ayush Das <150268924+AyushDas4890@users.noreply.github.com> Date: Thu, 2 Jul 2026 10:35:41 +0530 Subject: [PATCH 3/5] Fix formatting: add missing blank line before test class --- tests/models/rt_detr/test_modeling_rt_detr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/models/rt_detr/test_modeling_rt_detr.py b/tests/models/rt_detr/test_modeling_rt_detr.py index 7a03cf431302..460fdd27cb54 100644 --- a/tests/models/rt_detr/test_modeling_rt_detr.py +++ b/tests/models/rt_detr/test_modeling_rt_detr.py @@ -759,6 +759,7 @@ def test_inference_object_detection_head(self): self.assertSequenceEqual(results["labels"][:4].tolist(), expected_labels) torch.testing.assert_close(results["boxes"][:4], expected_slice_boxes, rtol=2e-4, atol=2e-4) + @require_torch @require_scipy @require_vision From 5a84ce21f7d78c837a2d12900c5c6b8c00ccd752 Mon Sep 17 00:00:00 2001 From: Ayush Das <150268924+AyushDas4890@users.noreply.github.com> Date: Fri, 3 Jul 2026 09:22:33 +0530 Subject: [PATCH 4/5] Replace invalid cost matrix values with finfo max per review --- src/transformers/loss/loss_rt_detr.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/transformers/loss/loss_rt_detr.py b/src/transformers/loss/loss_rt_detr.py index 6a7e31e5a645..616de93dd8b5 100644 --- a/src/transformers/loss/loss_rt_detr.py +++ b/src/transformers/loss/loss_rt_detr.py @@ -112,9 +112,11 @@ def forward(self, outputs, targets): giou_cost = -generalized_box_iou(center_to_corners_format(out_bbox), center_to_corners_format(target_bbox)) # Compute the final cost matrix cost_matrix = self.bbox_cost * bbox_cost + self.class_cost * class_cost + self.giou_cost * giou_cost - # eliminate infinite values in cost_matrix to avoid the error ``ValueError: cost matrix is infeasible`` - cost_matrix = torch.minimum(cost_matrix, torch.tensor(1e10)) - cost_matrix = torch.maximum(cost_matrix, torch.tensor(-1e10)) + # we assume any good match will not cause NaN or Inf, so we replace them with the maximum + # finite value to ensure these entries are never preferentially matched (avoids the error + # ``ValueError: cost matrix is infeasible``) + max_value = torch.finfo(cost_matrix.dtype).max + cost_matrix = torch.nan_to_num(cost_matrix, nan=max_value, posinf=max_value, neginf=max_value) cost_matrix = cost_matrix.view(batch_size, num_queries, -1).cpu() sizes = [len(v["boxes"]) for v in targets] From e68a33b0b08023f6c2cb2af55fa25c29f54b27a9 Mon Sep 17 00:00:00 2001 From: Ayush Das <150268924+AyushDas4890@users.noreply.github.com> Date: Fri, 3 Jul 2026 09:23:33 +0530 Subject: [PATCH 5/5] Extend matcher regression test with NaN cost case --- tests/models/rt_detr/test_modeling_rt_detr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/rt_detr/test_modeling_rt_detr.py b/tests/models/rt_detr/test_modeling_rt_detr.py index 460fdd27cb54..7498a8720b94 100644 --- a/tests/models/rt_detr/test_modeling_rt_detr.py +++ b/tests/models/rt_detr/test_modeling_rt_detr.py @@ -784,7 +784,7 @@ def test_infinite_costs_do_not_crash_matcher(self): } ] - for saturating_logit in (-30.0, 30.0): + for saturating_logit in (-30.0, 30.0, float("nan")): # fp16 logits as produced under AMP: sigmoid saturates to exactly 0.0 (or 1.0), # making ``pos_cost_class`` (or ``neg_cost_class``) infinite logits = torch.full((1, num_queries, config.num_labels), saturating_logit, dtype=torch.float16)