diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td index b3b8afdd8b4c1..78def9cc78520 100644 --- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td +++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td @@ -2054,7 +2054,9 @@ def Vector_GatherOp : Variadic:$indices, VectorOfNonZeroRankOf<[AnyInteger, Index]>:$index_vec, VectorOfNonZeroRankOf<[I1]>:$mask, - AnyVectorOfNonZeroRank:$pass_thru)>, + AnyVectorOfNonZeroRank:$pass_thru, + ConfinedAttr, + [AllAttrOf<[IntPositive, IntPowerOf2]>]>:$alignment)>, Results<(outs AnyVectorOfNonZeroRank:$result)> { let summary = [{ @@ -2085,6 +2087,12 @@ def Vector_GatherOp : during progressively lowering to bring other memory operations closer to hardware ISA support for a gather. + An optional `alignment` attribute allows to specify the byte alignment of the + scatter operation. It must be a positive power of 2. The operation must access + memory at an address aligned to this boundary. Violations may lead to + architecture-specific faults or performance penalties. + A value of 0 indicates no specific alignment requirement. + Examples: ```mlir @@ -2111,6 +2119,20 @@ def Vector_GatherOp : "`into` type($result)"; let hasCanonicalizer = 1; let hasVerifier = 1; + + let builders = [ + OpBuilder<(ins "VectorType":$resultType, + "Value":$base, + "ValueRange":$indices, + "Value":$index_vec, + "Value":$mask, + "Value":$passthrough, + CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{ + return build($_builder, $_state, resultType, base, indices, index_vec, mask, passthrough, + alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) : + nullptr); + }]> + ]; } def Vector_ScatterOp : @@ -2119,7 +2141,9 @@ def Vector_ScatterOp : Variadic:$indices, VectorOfNonZeroRankOf<[AnyInteger, Index]>:$index_vec, VectorOfNonZeroRankOf<[I1]>:$mask, - AnyVectorOfNonZeroRank:$valueToStore)> { + AnyVectorOfNonZeroRank:$valueToStore, + ConfinedAttr, + [AllAttrOf<[IntPositive, IntPowerOf2]>]>:$alignment)> { let summary = [{ scatters elements from a vector into memory as defined by an index vector @@ -2153,6 +2177,12 @@ def Vector_ScatterOp : correspond to those of the `llvm.masked.scatter` [intrinsic](https://llvm.org/docs/LangRef.html#llvm-masked-scatter-intrinsics). + An optional `alignment` attribute allows to specify the byte alignment of the + scatter operation. It must be a positive power of 2. The operation must access + memory at an address aligned to this boundary. Violations may lead to + architecture-specific faults or performance penalties. + A value of 0 indicates no specific alignment requirement. + Examples: ```mlir @@ -2177,6 +2207,19 @@ def Vector_ScatterOp : "type($index_vec) `,` type($mask) `,` type($valueToStore)"; let hasCanonicalizer = 1; let hasVerifier = 1; + + let builders = [ + OpBuilder<(ins "Value":$base, + "ValueRange":$indices, + "Value":$index_vec, + "Value":$mask, + "Value":$valueToStore, + CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">: $alignment), [{ + return build($_builder, $_state, base, indices, index_vec, mask, valueToStore, + alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) : + nullptr); + }]> + ]; } def Vector_ExpandLoadOp : @@ -2184,7 +2227,9 @@ def Vector_ExpandLoadOp : Arguments<(ins Arg:$base, Variadic:$indices, FixedVectorOfNonZeroRankOf<[I1]>:$mask, - AnyVectorOfNonZeroRank:$pass_thru)>, + AnyVectorOfNonZeroRank:$pass_thru, + ConfinedAttr, + [AllAttrOf<[IntPositive, IntPowerOf2]>]>:$alignment)>, Results<(outs AnyVectorOfNonZeroRank:$result)> { let summary = "reads elements from memory and spreads them into a vector as defined by a mask"; @@ -2216,6 +2261,12 @@ def Vector_ExpandLoadOp : correspond to those of the `llvm.masked.expandload` [intrinsic](https://llvm.org/docs/LangRef.html#llvm-masked-expandload-intrinsics). + An optional `alignment` attribute allows to specify the byte alignment of the + load operation. It must be a positive power of 2. The operation must access + memory at an address aligned to this boundary. Violations may lead to + architecture-specific faults or performance penalties. + A value of 0 indicates no specific alignment requirement. + Note, at the moment this Op is only available for fixed-width vectors. Examples: @@ -2246,6 +2297,19 @@ def Vector_ExpandLoadOp : "type($base) `,` type($mask) `,` type($pass_thru) `into` type($result)"; let hasCanonicalizer = 1; let hasVerifier = 1; + + let builders = [ + OpBuilder<(ins "VectorType":$resultType, + "Value":$base, + "ValueRange":$indices, + "Value":$mask, + "Value":$passthrough, + CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{ + return build($_builder, $_state, resultType, base, indices, mask, passthrough, + alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) : + nullptr); + }]> + ]; } def Vector_CompressStoreOp : @@ -2253,7 +2317,9 @@ def Vector_CompressStoreOp : Arguments<(ins Arg:$base, Variadic:$indices, FixedVectorOfNonZeroRankOf<[I1]>:$mask, - AnyVectorOfNonZeroRank:$valueToStore)> { + AnyVectorOfNonZeroRank:$valueToStore, + ConfinedAttr, + [AllAttrOf<[IntPositive, IntPowerOf2]>]>:$alignment)> { let summary = "writes elements selectively from a vector as defined by a mask"; @@ -2284,6 +2350,12 @@ def Vector_CompressStoreOp : correspond to those of the `llvm.masked.compressstore` [intrinsic](https://llvm.org/docs/LangRef.html#llvm-masked-compressstore-intrinsics). + An optional `alignment` attribute allows to specify the byte alignment of the + store operation. It must be a positive power of 2. The operation must access + memory at an address aligned to this boundary. Violations may lead to + architecture-specific faults or performance penalties. + A value of 0 indicates no specific alignment requirement. + Note, at the moment this Op is only available for fixed-width vectors. Examples: @@ -2312,6 +2384,17 @@ def Vector_CompressStoreOp : "type($base) `,` type($mask) `,` type($valueToStore)"; let hasCanonicalizer = 1; let hasVerifier = 1; + let builders = [ + OpBuilder<(ins "Value":$base, + "ValueRange":$indices, + "Value":$mask, + "Value":$valueToStore, + CArg<"llvm::MaybeAlign", "llvm::MaybeAlign()">:$alignment), [{ + return build($_builder, $_state, base, indices, valueToStore, mask, + alignment.has_value() ? $_builder.getI64IntegerAttr(alignment->value()) : + nullptr); + }]> + ]; } def Vector_ShapeCastOp : diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir index 211e16db85a94..2e72bf036fa71 100644 --- a/mlir/test/Dialect/Vector/invalid.mlir +++ b/mlir/test/Dialect/Vector/invalid.mlir @@ -1317,7 +1317,7 @@ func.func @maskedload_negative_alignment(%base: memref<4xi32>, %mask: vector<32x // ----- -func.func @maskedload_nonpoweroftwo_alignment(%base: memref<4xi32>, %mask: vector<32xi1>, %pass: vector<1xi32>, %index: index) { +func.func @maskedload_non_power_of_2_alignment(%base: memref<4xi32>, %mask: vector<32xi1>, %pass: vector<1xi32>, %index: index) { // expected-error@below {{'vector.maskedload' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}} %val = vector.maskedload %base[%index], %mask, %pass { alignment = 3 } : memref<4xi32>, vector<32xi1>, vector<1xi32> into vector<1xi32> return @@ -1368,7 +1368,7 @@ func.func @maskedstore_negative_alignment(%base: memref<4xi32>, %mask: vector<32 // ----- -func.func @maskedstore_nonpoweroftwo_alignment(%base: memref<4xi32>, %mask: vector<32xi1>, %value: vector<1xi32>, %index: index) { +func.func @maskedstore_non_power_of_2_alignment(%base: memref<4xi32>, %mask: vector<32xi1>, %value: vector<1xi32>, %index: index) { // expected-error@below {{'vector.maskedstore' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}} vector.maskedstore %base[%index], %mask, %value { alignment = 3 } : memref<4xi32>, vector<32xi1>, vector<1xi32> into vector<1xi32> return @@ -1470,6 +1470,24 @@ func.func @gather_pass_thru_type_mismatch(%base: memref, %indices: vector // ----- +func.func @gather_negative_alignment(%base: memref<16xf32>, %indices: vector<16xi32>, + %mask: vector<16xi1>, %pass_thru: vector<16xf32>, %c0 : index) { + // expected-error@+2 {{'vector.gather' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}} + %0 = vector.gather %base[%c0][%indices], %mask, %pass_thru + { alignment = -1 } : memref<16xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32> into vector<16xf32> +} + +// ----- + +func.func @gather_non_power_of_two_alignment(%base: memref<16xf32>, %indices: vector<16xi32>, + %mask: vector<16xi1>, %pass_thru: vector<16xf32>, %c0 : index) { + // expected-error@+2 {{'vector.gather' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}} + %0 = vector.gather %base[%c0][%indices], %mask, %pass_thru + { alignment = 3 } : memref<16xf32>, vector<16xi32>, vector<16xi1>, vector<16xf32> into vector<16xf32> +} + +// ----- + func.func @scatter_to_vector(%base: vector<16xf32>, %indices: vector<16xi32>, %mask: vector<16xi1>, %pass_thru: vector<16xf32>) { %c0 = arith.constant 0 : index @@ -1531,6 +1549,24 @@ func.func @scatter_dim_mask_mismatch(%base: memref, %indices: vector<16xi // ----- +func.func @scatter_negative_alignment(%base: memref, %indices: vector<16xi32>, + %mask: vector<16xi1>, %value: vector<16xf32>, %c0: index) { + // expected-error@+1 {{'vector.scatter' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}} + vector.scatter %base[%c0][%indices], %mask, %value { alignment = -1 } + : memref, vector<16xi32>, vector<16xi1>, vector<16xf32> +} + +// ----- + +func.func @scatter_non_power_of_2_alignment(%base: memref, %indices: vector<16xi32>, + %mask: vector<16xi1>, %value: vector<16xf32>, %c0: index) { + // expected-error@+1 {{'vector.scatter' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}} + vector.scatter %base[%c0][%indices], %mask, %value { alignment = 3 } + : memref, vector<16xi32>, vector<16xi1>, vector<16xf32> +} + +// ----- + func.func @expand_base_type_mismatch(%base: memref, %mask: vector<16xi1>, %pass_thru: vector<16xf32>) { %c0 = arith.constant 0 : index // expected-error@+1 {{'vector.expandload' op base and result element type should match}} @@ -1571,6 +1607,20 @@ func.func @expand_memref_mismatch(%base: memref, %mask: vector<16xi1>, // ----- +func.func @expand_negative_alignment(%base: memref, %mask: vector<16xi1>, %pass_thru: vector<16xf32>, %c0: index) { + // expected-error@+1 {{'vector.expandload' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}} + %0 = vector.expandload %base[%c0], %mask, %pass_thru { alignment = -1 } : memref, vector<16xi1>, vector<16xf32> into vector<16xf32> +} + +// ----- + +func.func @expand_non_power_of_2_alignment(%base: memref, %mask: vector<16xi1>, %pass_thru: vector<16xf32>, %c0: index) { + // expected-error@+1 {{'vector.expandload' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}} + %0 = vector.expandload %base[%c0], %mask, %pass_thru { alignment = 3 } : memref, vector<16xi1>, vector<16xf32> into vector<16xf32> +} + +// ----- + func.func @compress_base_type_mismatch(%base: memref, %mask: vector<16xi1>, %value: vector<16xf32>) { %c0 = arith.constant 0 : index // expected-error@+1 {{'vector.compressstore' op base and valueToStore element type should match}} @@ -1603,6 +1653,20 @@ func.func @compress_memref_mismatch(%base: memref, %mask: vector<16xi1> // ----- +func.func @compress_negative_alignment(%base: memref, %mask: vector<16xi1>, %value: vector<16xf32>, %c0: index) { + // expected-error @below {{'vector.compressstore' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}} + vector.compressstore %base[%c0], %mask, %value { alignment = -1 } : memref, vector<16xi1>, vector<16xf32> +} + +// ----- + +func.func @compress_non_power_of_2_alignment(%base: memref, %mask: vector<16xi1>, %value: vector<16xf32>, %c0: index) { + // expected-error @below {{'vector.compressstore' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}} + vector.compressstore %base[%c0], %mask, %value { alignment = 3 } : memref, vector<16xi1>, vector<16xf32> +} + +// ----- + func.func @scan_reduction_dim_constraint(%arg0: vector<2x3xi32>, %arg1: vector<3xi32>) -> vector<3xi32> { // expected-error@+1 {{'vector.scan' op reduction dimension 5 has to be less than 2}} %0:2 = vector.scan , %arg0, %arg1 {inclusive = true, reduction_dim = 5} : @@ -1952,7 +2016,7 @@ func.func @vector_load(%src : memref) { // ----- -func.func @invalid_load_alignment(%memref: memref<4xi32>, %c0: index) { +func.func @load_negative_alignment(%memref: memref<4xi32>, %c0: index) { // expected-error @below {{'vector.load' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}} %val = vector.load %memref[%c0] { alignment = -1 } : memref<4xi32>, vector<4xi32> return @@ -1960,7 +2024,7 @@ func.func @invalid_load_alignment(%memref: memref<4xi32>, %c0: index) { // ----- -func.func @invalid_load_alignment(%memref: memref<4xi32>, %c0: index) { +func.func @load_non_pow_of_2_alignment(%memref: memref<4xi32>, %c0: index) { // expected-error @below {{'vector.load' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}} %val = vector.load %memref[%c0] { alignment = 3 } : memref<4xi32>, vector<4xi32> return @@ -1981,7 +2045,7 @@ func.func @vector_store(%dest : memref, %vec : vector<16x16xi8>) { // ----- -func.func @invalid_store_alignment(%memref: memref<4xi32>, %val: vector<4xi32>, %c0: index) { +func.func @store_negative_alignment(%memref: memref<4xi32>, %val: vector<4xi32>, %c0: index) { // expected-error @below {{'vector.store' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}} vector.store %val, %memref[%c0] { alignment = -1 } : memref<4xi32>, vector<4xi32> return @@ -1989,7 +2053,7 @@ func.func @invalid_store_alignment(%memref: memref<4xi32>, %val: vector<4xi32>, // ----- -func.func @invalid_store_alignment(%memref: memref<4xi32>, %val: vector<4xi32>, %c0: index) { +func.func @store_non_pow_of_2_alignment(%memref: memref<4xi32>, %val: vector<4xi32>, %c0: index) { // expected-error @below {{'vector.store' op attribute 'alignment' failed to satisfy constraint: 64-bit signless integer attribute whose value is positive and whose value is a power of two > 0}} vector.store %val, %memref[%c0] { alignment = 3 } : memref<4xi32>, vector<4xi32> return