mirror of
https://github.com/apple/swift.git
synced 2025-12-21 12:14:44 +01:00
It saved some boilerplate, but if it doesn't get inline (as in debug builds), Swift doesn't know how to legalize the type at the call boundary, and we crash.
373 lines
11 KiB
Swift
373 lines
11 KiB
Swift
//===--- SIMDConcreteOperations.swift -------------------------*- swift -*-===//
|
|
//
|
|
// This source file is part of the Swift.org open source project
|
|
//
|
|
// Copyright (c) 2021 Apple Inc. and the Swift project authors
|
|
// Licensed under Apache License v2.0 with Runtime Library Exception
|
|
//
|
|
// See https://swift.org/LICENSE.txt for license information
|
|
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
%{
|
|
from __future__ import division
|
|
from SwiftIntTypes import all_integer_types
|
|
word_bits = int(CMAKE_SIZEOF_VOID_P) * 8
|
|
storagescalarCounts = [2,4,8,16,32,64]
|
|
vectorscalarCounts = storagescalarCounts + [3]
|
|
}%
|
|
|
|
%for int in all_integer_types(word_bits):
|
|
% Scalar = int.stdlib_name
|
|
% for n in vectorscalarCounts:
|
|
% Vector = "SIMD" + str(n) + "<" + Scalar + ">"
|
|
% storageN = 4 if n == 3 else n
|
|
% s = "s" if int.is_signed else "u"
|
|
% Builtin = "Vec" + str(storageN) + "xInt" + str(int.bits)
|
|
% MaskExt = "Builtin.sext_Vec" + str(storageN) + "xInt1_" + Builtin
|
|
% if int.is_signed:
|
|
extension SIMDMask where Storage == ${Vector} {
|
|
@_alwaysEmitIntoClient
|
|
internal init(_ _builtin: Builtin.${Builtin}) {
|
|
_storage = ${Vector}(_builtin)
|
|
}
|
|
|
|
@_alwaysEmitIntoClient
|
|
internal static var allTrue: Self {
|
|
let zero = ${Vector}()
|
|
return zero .== zero
|
|
}
|
|
|
|
/// A vector mask that is the pointwise logical negation of the input.
|
|
///
|
|
/// Equivalent to:
|
|
/// ```
|
|
/// var result = SIMDMask<${Vector}>()
|
|
/// for i in result.indices {
|
|
/// result[i] = !a[i]
|
|
/// }
|
|
/// ```
|
|
@_alwaysEmitIntoClient
|
|
public static prefix func .!(a: Self) -> Self {
|
|
a .^ .allTrue
|
|
}
|
|
|
|
/// A vector mask that is the pointwise logical conjunction of the inputs.
|
|
///
|
|
/// Equivalent to:
|
|
/// ```
|
|
/// var result = SIMDMask<${Vector}>()
|
|
/// for i in result.indices {
|
|
/// result[i] = a[i] && b[i]
|
|
/// }
|
|
/// ```
|
|
///
|
|
/// Note that unlike the scalar `&&` operator, the SIMD `.&` operator
|
|
/// always fully evaluates both arguments.
|
|
@_alwaysEmitIntoClient
|
|
public static func .&(a: Self, b: Self) -> Self {
|
|
Self(${Vector}(Builtin.and_${Builtin}(
|
|
a._storage._storage._value,
|
|
b._storage._storage._value
|
|
)))
|
|
}
|
|
|
|
/// Replaces `a` with the pointwise logical conjuction of `a` and `b`.
|
|
///
|
|
/// Equivalent to:
|
|
/// ```
|
|
/// for i in a.indices {
|
|
/// a[i] = a[i] && b[i]
|
|
/// }
|
|
/// ```
|
|
@_alwaysEmitIntoClient
|
|
public static func .&=(a: inout Self, b: Self) {
|
|
a = a .& b
|
|
}
|
|
|
|
/// A vector mask that is the pointwise exclusive or of the inputs.
|
|
///
|
|
/// Equivalent to:
|
|
/// ```
|
|
/// var result = SIMDMask<${Vector}>()
|
|
/// for i in result.indices {
|
|
/// result[i] = a[i] != b[i]
|
|
/// }
|
|
/// ```
|
|
@_alwaysEmitIntoClient
|
|
public static func .^(a: Self, b: Self) -> Self {
|
|
Self(${Vector}(Builtin.xor_${Builtin}(
|
|
a._storage._storage._value,
|
|
b._storage._storage._value
|
|
)))
|
|
}
|
|
|
|
/// Replaces `a` with the pointwise exclusive or of `a` and `b`.
|
|
///
|
|
/// Equivalent to:
|
|
/// ```
|
|
/// for i in a.indices {
|
|
/// a[i] = a[i] != b[i]
|
|
/// }
|
|
/// ```
|
|
@_alwaysEmitIntoClient
|
|
public static func .^=(a: inout Self, b: Self) {
|
|
a = a .^ b
|
|
}
|
|
|
|
/// A vector mask that is the pointwise logical disjunction of the inputs.
|
|
///
|
|
/// Equivalent to:
|
|
/// ```
|
|
/// var result = SIMDMask<${Vector}>()
|
|
/// for i in result.indices {
|
|
/// result[i] = a[i] || b[i]
|
|
/// }
|
|
/// ```
|
|
///
|
|
/// Note that unlike the scalar `||` operator, the SIMD `.|` operator
|
|
/// always fully evaluates both arguments.
|
|
@_alwaysEmitIntoClient
|
|
public static func .|(a: Self, b: Self) -> Self {
|
|
Self(${Vector}(Builtin.or_${Builtin}(
|
|
a._storage._storage._value,
|
|
b._storage._storage._value
|
|
)))
|
|
}
|
|
|
|
/// Replaces `a` with the pointwise logical disjunction of `a` and `b`.
|
|
///
|
|
/// Equivalent to:
|
|
/// ```
|
|
/// for i in a.indices {
|
|
/// a[i] = a[i] || b[i]
|
|
/// }
|
|
/// ```
|
|
@_alwaysEmitIntoClient
|
|
public static func .|=(a: inout Self, b: Self) {
|
|
a = a .| b
|
|
}
|
|
|
|
/// A vector mask with the result of a pointwise equality comparison.
|
|
///
|
|
/// Equivalent to:
|
|
/// ```
|
|
/// var result = SIMDMask<${Vector}>()
|
|
/// for i in result.indices {
|
|
/// result[i] = a[i] == b[i]
|
|
/// }
|
|
/// ```
|
|
@_alwaysEmitIntoClient
|
|
public static func .==(a: Self, b: Self) -> Self {
|
|
.!(a .^ b)
|
|
}
|
|
|
|
/// A vector mask with the result of a pointwise inequality comparison.
|
|
///
|
|
/// Equivalent to:
|
|
/// ```
|
|
/// var result = SIMDMask<${Vector}>()
|
|
/// for i in result.indices {
|
|
/// result[i] = a[i] != b[i]
|
|
/// }
|
|
/// ```
|
|
@_alwaysEmitIntoClient
|
|
public static func .!=(a: Self, b: Self) -> Self {
|
|
a .^ b
|
|
}
|
|
|
|
/// Replaces elements of this vector with elements of `other` in the lanes
|
|
/// where `mask` is `true`.
|
|
///
|
|
/// Equivalent to:
|
|
/// ```
|
|
/// for i in indices {
|
|
/// if mask[i] { self[i] = other[i] }
|
|
/// }
|
|
/// ```
|
|
@_alwaysEmitIntoClient
|
|
public mutating func replace(with other: Self, where mask: Self) {
|
|
self = replacing(with: other, where: mask)
|
|
}
|
|
|
|
/// Returns a copy of this vector, with elements replaced by elements of
|
|
/// `other` in the lanes where `mask` is `true`.
|
|
///
|
|
/// Equivalent to:
|
|
/// ```
|
|
/// var result = Self()
|
|
/// for i in indices {
|
|
/// result[i] = mask[i] ? other[i] : self[i]
|
|
/// }
|
|
/// ```
|
|
@_alwaysEmitIntoClient
|
|
public func replacing(with other: Self, where mask: Self) -> Self {
|
|
(self .& .!mask) .| (other .& mask)
|
|
}
|
|
}
|
|
|
|
% end
|
|
extension SIMD${n} where Scalar == ${Scalar} {
|
|
@_alwaysEmitIntoClient
|
|
internal init(_ _builtin: Builtin.${Builtin}) {
|
|
_storage = ${Scalar}.SIMD${storageN}Storage(_builtin)
|
|
}
|
|
|
|
/// A vector mask with the result of a pointwise equality comparison.
|
|
@_alwaysEmitIntoClient
|
|
public static func .==(a: Self, b: Self) -> SIMDMask<MaskStorage> {
|
|
SIMDMask<MaskStorage>(${MaskExt}(
|
|
Builtin.cmp_eq_${Builtin}(a._storage._value, b._storage._value)
|
|
))
|
|
}
|
|
|
|
/// A vector mask with the result of a pointwise inequality comparison.
|
|
@_alwaysEmitIntoClient
|
|
public static func .!=(a: Self, b: Self) -> SIMDMask<MaskStorage> {
|
|
SIMDMask<MaskStorage>(${MaskExt}(
|
|
Builtin.cmp_ne_${Builtin}(a._storage._value, b._storage._value)
|
|
))
|
|
}
|
|
|
|
/// A vector mask with the result of a pointwise less-than comparison.
|
|
@_alwaysEmitIntoClient
|
|
public static func .<(a: Self, b: Self) -> SIMDMask<MaskStorage> {
|
|
SIMDMask<MaskStorage>(${MaskExt}(
|
|
Builtin.cmp_${s}lt_${Builtin}(a._storage._value, b._storage._value)
|
|
))
|
|
}
|
|
|
|
/// A vector mask with the result of a pointwise less-than-or-equal-to comparison.
|
|
@_alwaysEmitIntoClient
|
|
public static func .<=(a: Self, b: Self) -> SIMDMask<MaskStorage> {
|
|
SIMDMask<MaskStorage>(${MaskExt}(
|
|
Builtin.cmp_${s}le_${Builtin}(a._storage._value, b._storage._value)
|
|
))
|
|
}
|
|
|
|
/// A vector mask with the result of a pointwise greater-than comparison.
|
|
@_alwaysEmitIntoClient
|
|
public static func .>(a: Self, b: Self) -> SIMDMask<MaskStorage> {
|
|
SIMDMask<MaskStorage>(${MaskExt}(
|
|
Builtin.cmp_${s}gt_${Builtin}(a._storage._value, b._storage._value)
|
|
))
|
|
}
|
|
|
|
/// A vector mask with the result of a pointwise greater-than-or-equal-to comparison.
|
|
@_alwaysEmitIntoClient
|
|
public static func .>=(a: Self, b: Self) -> SIMDMask<MaskStorage> {
|
|
SIMDMask<MaskStorage>(${MaskExt}(
|
|
Builtin.cmp_${s}ge_${Builtin}(a._storage._value, b._storage._value)
|
|
))
|
|
}
|
|
|
|
/// The wrapping sum of two vectors.
|
|
@_alwaysEmitIntoClient
|
|
public static func &+(a: Self, b: Self) -> Self {
|
|
Self(Builtin.add_${Builtin}(a._storage._value, b._storage._value))
|
|
}
|
|
|
|
/// The wrapping difference of two vectors.
|
|
@_alwaysEmitIntoClient
|
|
public static func &-(a: Self, b: Self) -> Self {
|
|
Self(Builtin.sub_${Builtin}(a._storage._value, b._storage._value))
|
|
}
|
|
|
|
/// The pointwise wrapping product of two vectors.
|
|
@_alwaysEmitIntoClient
|
|
public static func &*(a: Self, b: Self) -> Self {
|
|
Self(Builtin.mul_${Builtin}(a._storage._value, b._storage._value))
|
|
}
|
|
|
|
/// Updates the left hand side with the wrapping sum of the two
|
|
/// vectors.
|
|
@_alwaysEmitIntoClient
|
|
public static func &+=(a: inout Self, b: Self) { a = a &+ b }
|
|
|
|
/// Updates the left hand side with the wrapping difference of the two
|
|
/// vectors.
|
|
@_alwaysEmitIntoClient
|
|
public static func &-=(a: inout Self, b: Self) { a = a &- b }
|
|
|
|
/// Updates the left hand side with the pointwise wrapping product of two
|
|
/// vectors.
|
|
@_alwaysEmitIntoClient
|
|
public static func &*=(a: inout Self, b: Self) { a = a &* b }
|
|
}
|
|
|
|
% end
|
|
%end
|
|
|
|
%for (Scalar, bits) in [('Float16',16), ('Float',32), ('Double',64)]:
|
|
% for n in vectorscalarCounts:
|
|
% Vector = "SIMD" + str(n) + "<" + Scalar + ">"
|
|
% storageN = 4 if n == 3 else n
|
|
% Builtin = "Vec" + str(storageN) + "xFPIEEE" + str(bits)
|
|
% VecPre = "Vec" + str(storageN) + "x"
|
|
% MaskExt = "Builtin.sext_" + VecPre + "Int1_" + VecPre + "Int" + str(bits)
|
|
% if bits == 16:
|
|
#if !((os(macOS) || targetEnvironment(macCatalyst)) && arch(x86_64))
|
|
@available(macOS 11.0, iOS 14.0, watchOS 7.0, tvOS 14.0, *)
|
|
% end
|
|
extension SIMD${n} where Scalar == ${Scalar} {
|
|
@_alwaysEmitIntoClient
|
|
internal init(_ _builtin: Builtin.${Builtin}) {
|
|
_storage = ${Scalar}.SIMD${storageN}Storage(_builtin)
|
|
}
|
|
|
|
/// A vector mask with the result of a pointwise equality comparison.
|
|
@_alwaysEmitIntoClient
|
|
public static func .==(a: Self, b: Self) -> SIMDMask<MaskStorage> {
|
|
SIMDMask<MaskStorage>(${MaskExt}(
|
|
Builtin.fcmp_oeq_${Builtin}(a._storage._value, b._storage._value)
|
|
))
|
|
}
|
|
|
|
/// A vector mask with the result of a pointwise inequality comparison.
|
|
@_alwaysEmitIntoClient
|
|
public static func .!=(a: Self, b: Self) -> SIMDMask<MaskStorage> {
|
|
SIMDMask<MaskStorage>(${MaskExt}(
|
|
Builtin.fcmp_une_${Builtin}(a._storage._value, b._storage._value)
|
|
))
|
|
}
|
|
|
|
/// A vector mask with the result of a pointwise less-than comparison.
|
|
@_alwaysEmitIntoClient
|
|
public static func .<(a: Self, b: Self) -> SIMDMask<MaskStorage> {
|
|
SIMDMask<MaskStorage>(${MaskExt}(
|
|
Builtin.fcmp_olt_${Builtin}(a._storage._value, b._storage._value)
|
|
))
|
|
}
|
|
|
|
/// A vector mask with the result of a pointwise less-than-or-equal-to comparison.
|
|
@_alwaysEmitIntoClient
|
|
public static func .<=(a: Self, b: Self) -> SIMDMask<MaskStorage> {
|
|
SIMDMask<MaskStorage>(${MaskExt}(
|
|
Builtin.fcmp_ole_${Builtin}(a._storage._value, b._storage._value)
|
|
))
|
|
}
|
|
|
|
/// A vector mask with the result of a pointwise greater-than comparison.
|
|
@_alwaysEmitIntoClient
|
|
public static func .>(a: Self, b: Self) -> SIMDMask<MaskStorage> {
|
|
SIMDMask<MaskStorage>(${MaskExt}(
|
|
Builtin.fcmp_ogt_${Builtin}(a._storage._value, b._storage._value)
|
|
))
|
|
}
|
|
|
|
/// A vector mask with the result of a pointwise greater-than-or-equal-to comparison.
|
|
@_alwaysEmitIntoClient
|
|
public static func .>=(a: Self, b: Self) -> SIMDMask<MaskStorage> {
|
|
SIMDMask<MaskStorage>(${MaskExt}(
|
|
Builtin.fcmp_oge_${Builtin}(a._storage._value, b._storage._value)
|
|
))
|
|
}
|
|
}
|
|
% if bits == 16:
|
|
#endif
|
|
% end
|
|
|
|
% end
|
|
%end
|