Support i64.mul_wide_s and i64.mul_wide_u from Wide Arithmetic proposal

scripts/gen-s-parser.py

@@ -150,6 +150,8 @@
     ("i64.rotr",       "makeBinary(BinaryOp::RotRInt64)"),
     ("i64.add128",     "makeWideIntAddSub(WideIntAddSubOp::AddInt128)"),
     ("i64.sub128",     "makeWideIntAddSub(WideIntAddSubOp::SubInt128)"),
+    ("i64.mul_wide_s", "makeWideIntMul(WideIntMulOp::MulWideSInt64)"),
+    ("i64.mul_wide_u", "makeWideIntMul(WideIntMulOp::MulWideUInt64)"),
     ("f32.abs",        "makeUnary(UnaryOp::AbsFloat32)"),
     ("f32.neg",        "makeUnary(UnaryOp::NegFloat32)"),
     ("f32.ceil",       "makeUnary(UnaryOp::CeilFloat32)"),

scripts/test/shared.py

@@ -404,7 +404,6 @@ def get_tests(test_dir, extensions=[], recursive=False):
     'threads/atomic.wast',
 ]
 SPEC_TESTSUITE_PROPOSALS_TO_SKIP = [
-    'wide-arithmetic',
 ]
 
 # Paths are relative to the test/spec/testsuite directory

src/gen-s-parser.inc

@@ -3991,12 +3991,34 @@ switch (buf[0]) {
                   default: goto parse_error;
                 }
               }
-              case 'm':
-                if (op == "i64.mul"sv) {
-                  CHECK_ERR(makeBinary(ctx, pos, annotations, BinaryOp::MulInt64));
-                  return Ok{};
+              case 'm': {
+                switch (buf[7]) {
+                  case '\0':
+                    if (op == "i64.mul"sv) {
+                      CHECK_ERR(makeBinary(ctx, pos, annotations, BinaryOp::MulInt64));
+                      return Ok{};
+                    }
+                    goto parse_error;
+                  case '_': {
+                    switch (buf[13]) {
+                      case 's':
+                        if (op == "i64.mul_wide_s"sv) {
+                          CHECK_ERR(makeWideIntMul(ctx, pos, annotations, WideIntMulOp::MulWideSInt64));
+                          return Ok{};
+                        }
+                        goto parse_error;
+                      case 'u':
+                        if (op == "i64.mul_wide_u"sv) {
+                          CHECK_ERR(makeWideIntMul(ctx, pos, annotations, WideIntMulOp::MulWideUInt64));
+                          return Ok{};
+                        }
+                        goto parse_error;
+                      default: goto parse_error;
+                    }
+                  }
+                  default: goto parse_error;
                 }
-                goto parse_error;
+              }
               case 'n':
                 if (op == "i64.ne"sv) {
                   CHECK_ERR(makeBinary(ctx, pos, annotations, BinaryOp::NeInt64));

src/interpreter/interpreter.cpp

@@ -217,6 +217,7 @@ struct ExpressionInterpreter : OverriddenVisitor<ExpressionInterpreter, Flow> {
     }
   }
   Flow visitWideIntAddSub(WideIntAddSub* curr) { WASM_UNREACHABLE("TODO"); }
+  Flow visitWideIntMul(WideIntMul* curr) { WASM_UNREACHABLE("TODO"); }
   Flow visitSelect(Select* curr) { WASM_UNREACHABLE("TODO"); }
   Flow visitDrop(Drop* curr) { WASM_UNREACHABLE("TODO"); }
   Flow visitReturn(Return* curr) { WASM_UNREACHABLE("TODO"); }

src/ir/ReFinalize.cpp

@@ -110,6 +110,7 @@ void ReFinalize::visitConst(Const* curr) { curr->finalize(); }
 void ReFinalize::visitUnary(Unary* curr) { curr->finalize(); }
 void ReFinalize::visitBinary(Binary* curr) { curr->finalize(); }
 void ReFinalize::visitWideIntAddSub(WideIntAddSub* curr) { curr->finalize(); }
+void ReFinalize::visitWideIntMul(WideIntMul* curr) { curr->finalize(); }
 void ReFinalize::visitSelect(Select* curr) { curr->finalize(); }
 void ReFinalize::visitDrop(Drop* curr) { curr->finalize(); }
 void ReFinalize::visitReturn(Return* curr) { curr->finalize(); }

src/ir/child-typer.h

@@ -712,6 +712,11 @@ template<typename Subtype> struct ChildTyper : OverriddenVisitor<Subtype> {
     note(&curr->rightHigh, Type::i64);
   }
 
+  void visitWideIntMul(WideIntMul* curr) {
+    note(&curr->left, Type::i64);
+    note(&curr->right, Type::i64);
+  }
+
   void visitSelect(Select* curr, std::optional<Type> type = std::nullopt) {
     if (type) {
       note(&curr->ifTrue, *type);

src/ir/cost.h

@@ -579,6 +579,9 @@ struct CostAnalyzer : public OverriddenVisitor<CostAnalyzer, CostType> {
     return 1 + visit(curr->leftLow) + visit(curr->leftHigh) +
            visit(curr->rightLow) + visit(curr->rightHigh);
   }
+  CostType visitWideIntMul(WideIntMul* curr) {
+    return 4 + visit(curr->left) + visit(curr->right);
+  }
   CostType visitSelect(Select* curr) {
     return 1 + visit(curr->condition) + visit(curr->ifTrue) +
            visit(curr->ifFalse);

src/ir/effects.h

@@ -946,6 +946,7 @@ class EffectAnalyzer {
       }
     }
     void visitWideIntAddSub(WideIntAddSub* curr) {}
+    void visitWideIntMul(WideIntMul* curr) {}
     void visitSelect(Select* curr) {}
     void visitDrop(Drop* curr) {}
     void visitReturn(Return* curr) { parent.branchesOut = true; }

src/ir/possible-contents.cpp

@@ -635,6 +635,7 @@ struct InfoCollector
   }
   void visitBinary(Binary* curr) { addRoot(curr); }
   void visitWideIntAddSub(WideIntAddSub* curr) { addRoot(curr); }
+  void visitWideIntMul(WideIntMul* curr) { addRoot(curr); }
   void visitSelect(Select* curr) {
     receiveChildValue(curr->ifTrue, curr);
     receiveChildValue(curr->ifFalse, curr);

src/ir/subtype-exprs.h

@@ -214,6 +214,7 @@ struct SubtypingDiscoverer : public OverriddenVisitor<SubType> {
   void visitUnary(Unary* curr) {}
   void visitBinary(Binary* curr) {}
   void visitWideIntAddSub(WideIntAddSub* curr) {}
+  void visitWideIntMul(WideIntMul* curr) {}
   void visitSelect(Select* curr) {
     self()->noteSubtype(curr->ifTrue, curr);
     self()->noteSubtype(curr->ifFalse, curr);

src/parser/contexts.h

@@ -481,6 +481,10 @@ struct NullInstrParserCtx {
     return Ok{};
   }
 
+  Result<> makeWideIntMul(Index, const std::vector<Annotation>&, WideIntMulOp) {
+    return Ok{};
+  }
+
   Result<> makeUnary(Index, const std::vector<Annotation>&, UnaryOp) {
     return Ok{};
   }
@@ -2170,6 +2174,12 @@ struct ParseDefsCtx : TypeParserCtx<ParseDefsCtx>, AnnotationParserCtx {
     return withLoc(pos, irBuilder.makeWideIntAddSub(op));
   }
 
+  Result<> makeWideIntMul(Index pos,
+                          const std::vector<Annotation>& annotations,
+                          WideIntMulOp op) {
+    return withLoc(pos, irBuilder.makeWideIntMul(op));
+  }
+
   Result<>
   makeUnary(Index pos, const std::vector<Annotation>& annotations, UnaryOp op) {
     return withLoc(pos, irBuilder.makeUnary(op));

src/parser/parsers.h

@@ -96,6 +96,9 @@ Result<> makeWideIntAddSub(Ctx&,
                            const std::vector<Annotation>&,
                            WideIntAddSubOp op);
 template<typename Ctx>
+Result<>
+makeWideIntMul(Ctx&, Index, const std::vector<Annotation>&, WideIntMulOp op);
+template<typename Ctx>
 Result<> makeUnary(Ctx&, Index, const std::vector<Annotation>&, UnaryOp op);
 template<typename Ctx>
 Result<> makeSelect(Ctx&, Index, const std::vector<Annotation>&);
@@ -1605,6 +1608,14 @@ Result<> makeWideIntAddSub(Ctx& ctx,
   return ctx.makeWideIntAddSub(pos, annotations, op);
 }
 
+template<typename Ctx>
+Result<> makeWideIntMul(Ctx& ctx,
+                        Index pos,
+                        const std::vector<Annotation>& annotations,
+                        WideIntMulOp op) {
+  return ctx.makeWideIntMul(pos, annotations, op);
+}
+
 template<typename Ctx>
 Result<> makeUnary(Ctx& ctx,
                    Index pos,

src/passes/I64ToI32Lowering.cpp

@@ -1557,6 +1557,10 @@ struct I64ToI32Lowering : public WalkerPass<PostWalker<I64ToI32Lowering>> {
     WASM_UNREACHABLE("TODO: wide arithmetic lowering");
   }
 
+  void visitWideIntMul(WideIntMul* curr) {
+    WASM_UNREACHABLE("TODO: wide arithmetic lowering");
+  }
+
   void visitSelect(Select* curr) {
     if (handleUnreachable(curr)) {
       return;

src/passes/Print.cpp

@@ -2045,6 +2045,20 @@ struct PrintExpressionContents
     }
     restoreNormalColor(o);
   }
+  void visitWideIntMul(WideIntMul* curr) {
+    prepareColor(o);
+    switch (curr->op) {
+      case MulWideSInt64: {
+        o << "i64.mul_wide_s";
+        break;
+      }
+      case MulWideUInt64: {
+        o << "i64.mul_wide_u";
+        break;
+      }
+    }
+    restoreNormalColor(o);
+  }
   void visitSelect(Select* curr) {
     prepareColor(o) << "select";
     restoreNormalColor(o);

src/passes/TypeGeneralizing.cpp

@@ -434,6 +434,7 @@ struct TransferFn : OverriddenVisitor<TransferFn> {
   void visitUnary(Unary* curr) {}
   void visitBinary(Binary* curr) {}
   void visitWideIntAddSub(WideIntAddSub* curr) {}
+  void visitWideIntMul(WideIntMul* curr) {}
 
   void visitSelect(Select* curr) {
     if (curr->type.isRef()) {

src/support/CMakeLists.txt

@@ -7,6 +7,7 @@ set(support_SOURCES
   debug.cpp
   dfa_minimization.cpp
   file.cpp
+  int128.cpp
   intervals.cpp
   istring.cpp
   json.cpp

src/support/int128.cpp

@@ -0,0 +1,89 @@
+// Copyright 2024 WebAssembly Community Group participants
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "support/int128.h"
+
+namespace wasm {
+
+#ifdef __SIZEOF_INT128__
+
+Int128 mul_wide_s(uint64_t lhs, uint64_t rhs) {
+  __int128 result = static_cast<__int128>(static_cast<int64_t>(lhs)) *
+                    static_cast<__int128>(static_cast<int64_t>(rhs));
+  return {static_cast<uint64_t>(result >> 64), static_cast<uint64_t>(result)};
+}
+
+Int128 mul_wide_u(uint64_t lhs, uint64_t rhs) {
+  unsigned __int128 result =
+    static_cast<unsigned __int128>(lhs) * static_cast<unsigned __int128>(rhs);
+  return {static_cast<uint64_t>(result >> 64), static_cast<uint64_t>(result)};
+}
+
+#else
+
+Int128 mul_wide_s(uint64_t lhs, uint64_t rhs) {
+  return detail::mul_wide_s_fallback(lhs, rhs);
+}
+
+Int128 mul_wide_u(uint64_t lhs, uint64_t rhs) {
+  return detail::mul_wide_u_fallback(lhs, rhs);
+}
+
+#endif
+
+namespace detail {
+
+Int128 mul_wide_s_fallback(uint64_t lhs, uint64_t rhs) {
+  auto [high, low] = mul_wide_u_fallback(lhs, rhs);
+
+  if (static_cast<int64_t>(lhs) < 0) {
+    high -= rhs;
+  }
+  if (static_cast<int64_t>(rhs) < 0) {
+    high -= lhs;
+  }
+
+  return {high, low};
+}
+
+Int128 mul_wide_u_fallback(uint64_t lhs, uint64_t rhs) {
+  uint32_t lhsLow = lhs & 0xffffffff;
+  uint32_t lhsHigh = lhs >> 32;
+  uint32_t rhsLow = rhs & 0xffffffff;
+  uint32_t rhsHigh = rhs >> 32;
+
+  uint64_t mulLowLow = static_cast<uint64_t>(lhsLow) * rhsLow;
+  uint64_t mulLowHigh = static_cast<uint64_t>(lhsLow) * rhsHigh;
+  uint64_t mulHighLow = static_cast<uint64_t>(lhsHigh) * rhsLow;
+  uint64_t mulHighHigh = static_cast<uint64_t>(lhsHigh) * rhsHigh;
+
+  uint64_t cross = mulLowHigh + (mulLowLow >> 32);
+  uint64_t carry = cross >> 32;
+  cross = (cross & 0xffffffff) + mulHighLow;
+
+  // The lower 64 bits of the 128-bit result are formed by:
+  //   (low * low) + ((low * high) << 32) + ((high * low) << 32)
+  //
+  // The upper 64 bits of the 128-bit result are formed by:
+  //   (high * high) + the upper 32-bits of (low * high) + the upper 32-bits of
+  //   (high * low) + carries from the lower 64 bits
+  uint64_t lowResult = (cross << 32) | (mulLowLow & 0xffffffff);
+  uint64_t highResult = mulHighHigh + carry + (cross >> 32);
+
+  return {highResult, lowResult};
+}
+
+} // namespace detail
+
+} // namespace wasm

src/support/int128.h

@@ -0,0 +1,45 @@
+// Copyright 2024 WebAssembly Community Group participants
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef wasm_support_int128_h
+#define wasm_support_int128_h
+
+#include <cstdint>
+
+namespace wasm {
+
+struct Int128 {
+  uint64_t high;
+  uint64_t low;
+
+  bool operator==(const Int128& other) const {
+    return high == other.high && low == other.low;
+  }
+};
+
+// Computes the 128-bit product of two signed 64-bit integers.
+Int128 mul_wide_s(uint64_t lhs, uint64_t rhs);
+
+// Computes the 128-bit product of two unsigned 64-bit integers.
+Int128 mul_wide_u(uint64_t lhs, uint64_t rhs);
+
+namespace detail {
+// Fallback implementations exposed for testing.
+Int128 mul_wide_s_fallback(uint64_t lhs, uint64_t rhs);
+Int128 mul_wide_u_fallback(uint64_t lhs, uint64_t rhs);
+} // namespace detail
+
+} // namespace wasm
+
+#endif // wasm_support_i128_h

src/wasm-binary.h

@@ -1142,6 +1142,8 @@ enum ASTNodes {
 
   I64Add128 = 0x13,
   I64Sub128 = 0x14,
+  I64MulWideS = 0x15,
+  I64MulWideU = 0x16,
 
   // reference types opcodes
 

src/wasm-builder.h

@@ -674,6 +674,16 @@ class Builder {
     ret->finalize();
     return ret;
   }
+
+  WideIntMul*
+  makeWideIntMul(WideIntMulOp op, Expression* left, Expression* right) {
+    auto* ret = wasm.allocator.alloc<WideIntMul>();
+    ret->op = op;
+    ret->left = left;
+    ret->right = right;
+    ret->finalize();
+    return ret;
+  }
   Select*
   makeSelect(Expression* condition, Expression* ifTrue, Expression* ifFalse) {
     auto* ret = wasm.allocator.alloc<Select>();

src/wasm-delegations-fields.def

@@ -939,6 +939,12 @@ DELEGATE_FIELD_CHILD(WideIntAddSub, leftHigh)
 DELEGATE_FIELD_CHILD(WideIntAddSub, leftLow)
 DELEGATE_FIELD_CASE_END(WideIntAddSub)
 
+DELEGATE_FIELD_CASE_START(WideIntMul)
+DELEGATE_FIELD_INT(WideIntMul, op)
+DELEGATE_FIELD_CHILD(WideIntMul, right)
+DELEGATE_FIELD_CHILD(WideIntMul, left)
+DELEGATE_FIELD_CASE_END(WideIntMul)
+
 DELEGATE_FIELD_MAIN_END
 
 #undef DELEGATE_ID

src/wasm-delegations.def

@@ -120,5 +120,6 @@ DELEGATE(StackSwitch);
 DELEGATE(StructWait);
 DELEGATE(StructNotify);
 DELEGATE(WideIntAddSub);
+DELEGATE(WideIntMul);
 
 #undef DELEGATE