fix[venom]: make truthy or rule wrap-aware

tests/unit/compiler/venom/test_algebraic_binopt.py

@@ -619,6 +619,44 @@ def test_signextend_range_no_elimination():
     _check_pre_post(pre, post)
 
 
+def test_signextend_unwrapped_zero_byte_index_not_eliminated():
+    # 2**256 wraps to byte index 0. It must not take the raw n >= 31 no-op path.
+    zero_mod_uint256 = 2**256
+    pre = f"""
+    _global:
+        %x = source
+        %y = signextend {zero_mod_uint256}, %x
+        sink %y
+    """
+    post = f"""
+    _global:
+        %x = source
+        %y = signextend {zero_mod_uint256}, %x
+        sink %y
+    """
+    _check_pre_post(pre, post, hevm=False)
+
+
+def test_signextend_chain_uses_wrapped_byte_indexes():
+    # The outer index wraps to 0, so it is not wider than the inner index 1.
+    zero_mod_uint256 = 2**256
+    pre = f"""
+    _global:
+        %x = source
+        %inner = signextend 1, %x
+        %outer = signextend {zero_mod_uint256}, %inner
+        sink %outer
+    """
+    post = f"""
+    _global:
+        %x = source
+        %inner = signextend 1, %x
+        %outer = signextend {zero_mod_uint256}, %inner
+        sink %outer
+    """
+    _check_pre_post(pre, post, hevm=False)
+
+
 @pytest.mark.skip(reason="Range-based comparison needs investigation - flip timing issue")
 def test_comparison_range_always_true():
     # When range proves comparison is always true

tests/unit/compiler/venom/test_algebraic_optimizer.py

@@ -431,6 +431,33 @@ def test_assert_unreachable_iszero_chain(iszero_count):
     _check_pre_post(pre, post)
 
 
+# Test the case of https://github.com/vyperlang/vyper/issues/5072
+def test_or_truthy_unwrapped_zero_literal():
+    """
+    A literal equal to 2**256 wraps to 0 as an EVM word, so
+    `or 2**256, %x` in a truthy position must not be folded to 1 --
+    it still depends on %x.
+    """
+    pre = f"""
+    main:
+        %x = source
+        %y = or {2**256}, %x
+        assert %y
+        sink %x
+    """
+
+    # the literal wraps to 0, so the `or` reduces to %x
+    post = """
+    main:
+        %x = source
+        %y = %x
+        assert %y
+        sink %x
+    """
+
+    _check_pre_post(pre, post, hevm=False)
+
+
 # Test the case of https://github.com/vyperlang/vyper/issues/4288
 def test_ssa_after_algebraic_optimization():
     code = """

vyper/venom/passes/algebraic_optimization.py

@@ -20,6 +20,10 @@ def lit_eq(op: IROperand, val: int) -> bool:
     return isinstance(op, IRLiteral) and wrap256(op.value) == wrap256(val)
 
 
+def lit_word_value(op: IROperand) -> int | None:
+    return wrap256(op.value) if isinstance(op, IRLiteral) else None
+
+
 class AlgebraicOptimizationPass(IRPass):
     """
     This pass reduces algebraic evaluatable expressions.
@@ -136,13 +140,13 @@ def _rewrite_or_skip_producer(self, inst: IRInstruction) -> bool:
 
         # signextend(n, signextend(m, x)) where n >= m -> signextend(m, x)
         if inst.opcode == "signextend":
-            n_op = operands[-1]
+            n = lit_word_value(operands[-1])
             x_op = operands[-2]
-            if isinstance(x_op, IRVariable) and self._is_lit(n_op):
+            if isinstance(x_op, IRVariable) and n is not None:
                 producer = self.dfg.get_producing_instruction(x_op)
                 if producer is not None and producer.opcode == "signextend":
-                    inner_n = producer.operands[-1]
-                    if self._is_lit(inner_n) and n_op.value >= inner_n.value:
+                    inner_n = lit_word_value(producer.operands[-1])
+                    if inner_n is not None and n >= inner_n:
                         self.updater.mk_assign(inst, x_op)
                         return True
 
@@ -185,21 +189,18 @@ def _rule_shift(self, inst: IRInstruction):
             self.updater.mk_assign(inst, inst.operands[0])
 
     def _rule_signextend(self, inst: IRInstruction):
-        n_op = inst.operands[-1]  # byte count
+        n = lit_word_value(inst.operands[-1])  # byte count
         x_op = inst.operands[-2]  # value
+        if n is None:
+            return
 
         # signextend(n, x) where n >= 31 is always a no-op
-        if self._is_lit(n_op) and n_op.value >= 31:
+        if n >= 31:
             self.updater.mk_assign(inst, x_op)
             return
 
         # range-based: if x is in the valid signed range for (n+1) bytes,
         # signextend is a no-op
-        if not self._is_lit(n_op):
-            return
-        n = n_op.value
-        if not (0 <= n < 31):
-            return
         x_range = self.range_analysis.get_range(x_op, inst)
         if x_range.is_top:
             return
@@ -295,7 +296,7 @@ def _rule_or(self, inst: IRInstruction):
         # x | n -> 1 in truthy positions (if n != 0)
         uses = self.dfg.get_uses(inst.output)
         is_truthy = all(i.opcode in TRUTHY_INSTRUCTIONS for i in uses)
-        if is_truthy and self._is_lit(ops[0]) and ops[0].value != 0:
+        if is_truthy and self._is_lit(ops[0]) and not lit_eq(ops[0], 0):
             self.updater.mk_assign(inst, IRLiteral(1))
             return