diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py
index 8a89fc0e6841..12931db1ba03 100644
--- a/src/transformers/modeling_utils.py
+++ b/src/transformers/modeling_utils.py
@@ -5135,6 +5135,9 @@ def caching_allocator_warmup(model: PreTrainedModel, expanded_device_map: dict,
             # This causes the warmup function to return a `RuntimeError: Invalid buffer size: XX.XX GiB`.
             # NOTE: not tested on intel macs
             continue
+        elif device.type == "neuron":
+            # Skip warmup on Neuron (AWS Trainium/Inferentia): it provides no benefit as there is no reusable memory pool
+            continue
         # We divide by 2 here as we allocate in fp16
         _ = torch.empty(int(byte_count // 2), dtype=torch.float16, device=device, requires_grad=False)