diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h index 7ad5d5f3118b6..e032455ff2ab3 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h @@ -139,6 +139,9 @@ enum { // 3 -> SEW * 4 DestEEWShift = ElementsDependOnMaskShift + 1, DestEEWMask = 3ULL << DestEEWShift, + + ReadsPastVLShift = DestEEWShift + 2, + ReadsPastVLMask = 1ULL << ReadsPastVLShift, }; // Helper functions to read TSFlags. @@ -195,6 +198,12 @@ static inline bool elementsDependOnMask(uint64_t TSFlags) { return TSFlags & ElementsDependOnMaskMask; } +/// \returns true if the instruction may read elements past VL, e.g. +/// vslidedown/vrgather +static inline bool readsPastVL(uint64_t TSFlags) { + return TSFlags & ReadsPastVLMask; +} + static inline unsigned getVLOpNum(const MCInstrDesc &Desc) { const uint64_t TSFlags = Desc.TSFlags; // This method is only called if we expect to have a VL operand, and all diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormats.td b/llvm/lib/Target/RISCV/RISCVInstrFormats.td index d9c6101478064..878a0ec938919 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrFormats.td +++ b/llvm/lib/Target/RISCV/RISCVInstrFormats.td @@ -261,6 +261,12 @@ class RVInstCommon; defm VSLIDE1UP_V : VSLD1_MV_X<"vslide1up", 0b001110>; } // Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp +let ReadsPastVL = 1 in defm VSLIDEDOWN_V : VSLD_IV_X_I<"vslidedown", 0b001111, /*slidesUp=*/false>; -let ElementsDependOn = EltDepsVL in +let ElementsDependOn = EltDepsVL, ReadsPastVL = 1 in defm VSLIDE1DOWN_V : VSLD1_MV_X<"vslide1down", 0b001111>; } // Predicates = [HasVInstructions] @@ -1712,19 +1713,19 @@ let Predicates = [HasVInstructionsAnyF] in { let Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp in { defm VFSLIDE1UP_V : VSLD1_FV_F<"vfslide1up", 0b001110>; } // Constraints = "@earlyclobber $vd", RVVConstraint = SlideUp -let ElementsDependOn = EltDepsVL in +let ElementsDependOn = EltDepsVL, ReadsPastVL = 1 in defm VFSLIDE1DOWN_V : VSLD1_FV_F<"vfslide1down", 0b001111>; } // Predicates = [HasVInstructionsAnyF] let Predicates = [HasVInstructions] in { // Vector Register Gather Instruction -let Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather in { +let Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather, ReadsPastVL = 1 in { defm VRGATHER_V : VGTR_IV_V_X_I<"vrgather", 0b001100>; def VRGATHEREI16_VV : VALUVV<0b001110, OPIVV, "vrgatherei16.vv">, SchedBinaryMC<"WriteVRGatherEI16VV", "ReadVRGatherEI16VV_data", "ReadVRGatherEI16VV_index">; -} // Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather +} // Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather, ReadsPastVL = 1 // Vector Compress Instruction let Constraints = "@earlyclobber $vd", RVVConstraint = Vcompress, ElementsDependOn = EltDepsVLMask in { diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXRivos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXRivos.td index 3912eb0d16c59..6e8e910249c82 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoXRivos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXRivos.td @@ -58,7 +58,7 @@ class CustomRivosXVI funct6, RISCVVFormat opv, dag outs, dag ins, let Predicates = [HasVendorXRivosVizip], DecoderNamespace = "XRivos", Constraints = "@earlyclobber $vd", RVVConstraint = Vrgather, - Inst<6-0> = OPC_CUSTOM_2.Value in { + Inst<6-0> = OPC_CUSTOM_2.Value, ReadsPastVL = 1 in { defm RI_VZIPEVEN_V : VALU_IV_V<"ri.vzipeven", 0b001100>; defm RI_VZIPODD_V : VALU_IV_V<"ri.vzipodd", 0b011100>; defm RI_VZIP2A_V : VALU_IV_V<"ri.vzip2a", 0b000100>; @@ -126,6 +126,7 @@ def RI_VINSERT : CustomRivosVXI<0b010000, OPMVX, (outs VR:$vd_wb), (ins VR:$vd, GPR:$rs1, uimm5:$imm), "ri.vinsert.v.x", "$vd, $rs1, $imm">; +let ReadsPastVL = 1 in def RI_VEXTRACT : CustomRivosXVI<0b010111, OPMVV, (outs GPR:$rd), (ins VR:$vs2, uimm5:$imm), "ri.vextract.x.v", "$rd, $vs2, $imm">; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td index 17fb75eb851c4..8b3b3ed6db188 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td @@ -74,6 +74,7 @@ class RVInstVCCustom2 funct6_hi4, bits<3> funct3, dag outs, dag ins, let Uses = [VL, VTYPE]; let RVVConstraint = NoConstraint; let ElementsDependOn = EltDepsVLMask; + let ReadsPastVL = 1; } class RVInstVCFCustom2 funct6_hi4, bits<3> funct3, dag outs, dag ins, @@ -98,6 +99,7 @@ class RVInstVCFCustom2 funct6_hi4, bits<3> funct3, dag outs, dag ins, let Uses = [VL, VTYPE]; let RVVConstraint = NoConstraint; let ElementsDependOn = EltDepsVLMask; + let ReadsPastVL = 1; } class VCIXInfo nf, RISCVOpcode opcode, let Inst{6-0} = opcode.Value; let Uses = [VTYPE, VL]; + let ReadsPastVL = 1; } let hasSideEffects = 0, mayLoad = 1, mayStore = 0 in @@ -94,6 +95,7 @@ class SFInstTileMoveOp funct6, dag outs, dag ins, string opcodestr, let Inst{6-0} = OPC_OP_V.Value; let Uses = [VTYPE, VL]; + let ReadsPastVL = 1; } let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in @@ -113,6 +115,7 @@ class SFInstMatmulF let Inst{6-0} = OPC_OP_VE.Value; let Uses = [VTYPE, VL]; + let ReadsPastVL = 1; } let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in @@ -135,6 +138,7 @@ class SFInstMatmulF8 { diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp index 15bd3466373a7..b16f817064ce2 100644 --- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp +++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp @@ -1227,34 +1227,6 @@ static bool isVectorOpUsedAsScalarOp(const MachineOperand &MO) { } } -/// Return true if MI may read elements past VL. -static bool mayReadPastVL(const MachineInstr &MI) { - const RISCVVPseudosTable::PseudoInfo *RVV = - RISCVVPseudosTable::getPseudoInfo(MI.getOpcode()); - if (!RVV) - return true; - - switch (RVV->BaseInstr) { - // vslidedown instructions may read elements past VL. They are handled - // according to current tail policy. - case RISCV::VSLIDEDOWN_VI: - case RISCV::VSLIDEDOWN_VX: - case RISCV::VSLIDE1DOWN_VX: - case RISCV::VFSLIDE1DOWN_VF: - - // vrgather instructions may read the source vector at any index < VLMAX, - // regardless of VL. - case RISCV::VRGATHER_VI: - case RISCV::VRGATHER_VV: - case RISCV::VRGATHER_VX: - case RISCV::VRGATHEREI16_VV: - return true; - - default: - return false; - } -} - bool RISCVVLOptimizer::isCandidate(const MachineInstr &MI) const { const MCInstrDesc &Desc = MI.getDesc(); if (!RISCVII::hasVLOp(Desc.TSFlags) || !RISCVII::hasSEWOp(Desc.TSFlags)) @@ -1315,7 +1287,8 @@ RISCVVLOptimizer::getMinimumVLForUser(const MachineOperand &UserOp) const { return std::nullopt; } - if (mayReadPastVL(UserMI)) { + if (RISCVII::readsPastVL( + TII->get(RISCV::getRVVMCOpcode(UserMI.getOpcode())).TSFlags)) { LLVM_DEBUG(dbgs() << " Abort because used by unsafe instruction\n"); return std::nullopt; }