Range-check shift/rotate amounts in the bytestring wrappers

plutus-tx/changelog.d/20260717_101730_yuriy.lazaryev_issue_2342_shift_rotate.md

@@ -0,0 +1,5 @@
+### Fixed
+
+- The Haskell definitions of `shiftByteString` and `rotateByteString` now fail when the shift
+  or rotation amount does not fit in a machine `Int`, matching the builtin semantics that the
+  van Rossem HF activates (variants D and E); earlier variants accept any amount on-chain.

plutus-tx/src/PlutusTx/Builtins/Internal.hs

@@ -933,23 +933,29 @@ BITWISE
 
 {-| Shifts the bytestring to the left if the second argument is positive, and to the right otherwise.
 Right-shifts fill with 0s from the left (logical shift); left-shifts fill with 0s from the right.
-Never fails. -}
+Fails only if the shift amount does not fit in a machine 'Int', matching builtin semantics
+variants D and E; earlier variants accept any amount. -}
 shiftByteString
   :: BuiltinByteString
   -> BuiltinInteger
   -> BuiltinByteString
-shiftByteString (BuiltinByteString bs) =
-  BuiltinByteString . Bitwise.shiftByteString bs
+shiftByteString (BuiltinByteString bs) i
+  | toInteger (minBound :: Int) <= i && i <= toInteger (maxBound :: Int) =
+      BuiltinByteString (Bitwise.shiftByteString bs i)
+  | otherwise = Haskell.error "shift amount out of Int range"
 {-# OPAQUE shiftByteString #-}
 
 {-| Rotates the bytestring to the left if the second argument is positive, and to the right otherwise.
-Never fails. -}
+Fails only if the rotation amount does not fit in a machine 'Int', matching builtin semantics
+variants D and E; earlier variants accept any amount. -}
 rotateByteString
   :: BuiltinByteString
   -> BuiltinInteger
   -> BuiltinByteString
-rotateByteString (BuiltinByteString bs) =
-  BuiltinByteString . Bitwise.rotateByteString bs
+rotateByteString (BuiltinByteString bs) i
+  | toInteger (minBound :: Int) <= i && i <= toInteger (maxBound :: Int) =
+      BuiltinByteString (Bitwise.rotateByteString bs i)
+  | otherwise = Haskell.error "rotation amount out of Int range"
 {-# OPAQUE rotateByteString #-}
 
 -- | Counts the number of bits set to 1 in the bytestring and never fails.

plutus-tx/test/Builtins/Spec.hs

@@ -66,6 +66,24 @@ builtinsTests =
         , testCase "negative byte fails" $ throwsErrorCall (BI.replicateByte 3 (-1))
         , testCase "negative count fails" $ throwsErrorCall (BI.replicateByte (-1) 65)
         ]
+    , testGroup
+        "shiftByteString"
+        [ testCase "shift by zero is identity" $ unBS (BI.shiftByteString abc 0) @?= C8.pack "abc"
+        , testCase "overlong shift zeroes the bytestring" $
+            unBS (BI.shiftByteString abc 100) @?= C8.pack "\0\0\0"
+        , testCase "amount exceeding maxBound::Int fails" $
+            throwsErrorCall (BI.shiftByteString abc (2 ^ (64 :: Int)))
+        , testCase "amount below minBound::Int fails" $
+            throwsErrorCall (BI.shiftByteString abc (-(2 ^ (64 :: Int))))
+        ]
+    , testGroup
+        "rotateByteString"
+        [ testCase "rotation by zero is identity" $ unBS (BI.rotateByteString abc 0) @?= C8.pack "abc"
+        , testCase "rotation by the bit length is identity" $
+            unBS (BI.rotateByteString abc 24) @?= C8.pack "abc"
+        , testCase "amount exceeding maxBound::Int fails" $
+            throwsErrorCall (BI.rotateByteString abc (2 ^ (64 :: Int)))
+        ]
     , testGroup
         "caseInteger"
         [ testCase "in-range scrutinee" $ BI.caseInteger 1 [10 :: Integer, 20, 30] @?= 20