diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index ea0fa0668ef6b..3aa88f53116d6 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -263,6 +263,13 @@ class VPBuilder { new VPInstruction(VPInstruction::PtrAdd, {Ptr, Offset}, GEPNoWrapFlags::inBounds(), DL, Name)); } + VPInstruction *createWidePtrAdd(VPValue *Ptr, VPValue *Offset, + DebugLoc DL = DebugLoc::getUnknown(), + const Twine &Name = "") { + return tryInsertInstruction( + new VPInstruction(VPInstruction::WidePtrAdd, {Ptr, Offset}, + GEPNoWrapFlags::none(), DL, Name)); + } VPPhi *createScalarPhi(ArrayRef IncomingValues, DebugLoc DL, const Twine &Name = "") { diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 2138b4154d2c2..8052e3199ec88 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -1038,21 +1038,6 @@ void VPlan::execute(VPTransformState *State) { if (isa(&R)) continue; - if (auto *WidenPhi = dyn_cast(&R)) { - assert(!WidenPhi->onlyScalarsGenerated(State->VF.isScalable()) && - "recipe generating only scalars should have been replaced"); - auto *GEP = cast(State->get(WidenPhi)); - PHINode *Phi = cast(GEP->getPointerOperand()); - - Phi->setIncomingBlock(1, VectorLatchBB); - - // Move the last step to the end of the latch block. This ensures - // consistent placement of all induction updates. - Instruction *Inc = cast(Phi->getIncomingValue(1)); - Inc->moveBefore(std::prev(VectorLatchBB->getTerminator()->getIterator())); - continue; - } - auto *PhiR = cast(&R); // VPInstructions currently model scalar Phis only. bool NeedsScalar = isa(PhiR) || diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index c13fe4548ff11..8dfb982a7d2f9 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -991,6 +991,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags, // operand). Only generates scalar values (either for the first lane only or // for all lanes, depending on its uses). PtrAdd, + // Add a vector offset in bytes (second operand) to a scalar base pointer + // (first operand). + WidePtrAdd, // Returns a scalar boolean value, which is true if any lane of its // (boolean) vector operands is true. It produces the reduced value across // all unrolled iterations. Unrolling will add all copies of its original @@ -1979,6 +1982,9 @@ class VPWidenInductionRecipe : public VPHeaderPHIRecipe { /// Update the step value of the recipe. void setStepValue(VPValue *V) { setOperand(1, V); } + VPValue *getVFValue() { return getOperand(2); } + const VPValue *getVFValue() const { return getOperand(2); } + /// Returns the number of incoming values, also number of incoming blocks. /// Note that at the moment, VPWidenPointerInductionRecipe only has a single /// incoming value, its start value. @@ -2068,9 +2074,6 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe { VPSlotTracker &SlotTracker) const override; #endif - VPValue *getVFValue() { return getOperand(2); } - const VPValue *getVFValue() const { return getOperand(2); } - VPValue *getSplatVFValue() { // If the recipe has been unrolled return the VPValue for the induction // increment. @@ -2106,8 +2109,7 @@ class VPWidenIntOrFpInductionRecipe : public VPWidenInductionRecipe { } }; -class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe, - public VPUnrollPartAccessor<4> { +class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe { bool IsScalarAfterVectorization; public: @@ -2136,18 +2138,14 @@ class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe, VP_CLASSOF_IMPL(VPDef::VPWidenPointerInductionSC) /// Generate vector values for the pointer induction. - void execute(VPTransformState &State) override; + void execute(VPTransformState &State) override { + llvm_unreachable("cannot execute this recipe, should be expanded via " + "expandVPWidenPointerInduction"); + }; /// Returns true if only scalar values will be generated. bool onlyScalarsGenerated(bool IsScalable); - /// Returns the VPValue representing the value of this induction at - /// the first unrolled part, if it exists. Returns itself if unrolling did not - /// take place. - VPValue *getFirstUnrolledPartOperand() { - return getUnrollPart(*this) == 0 ? this : getOperand(3); - } - #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// Print the recipe. void print(raw_ostream &O, const Twine &Indent, diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp index 16072f268a98c..4c3cdda338708 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp @@ -128,6 +128,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) { return IntegerType::get(Ctx, 1); case VPInstruction::Broadcast: case VPInstruction::PtrAdd: + case VPInstruction::WidePtrAdd: // Return the type based on first operand. return inferScalarType(R->getOperand(0)); case VPInstruction::BranchOnCond: diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 98d11f0bc7893..47a807794eb3d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -478,6 +478,7 @@ unsigned VPInstruction::getNumOperandsForOpcode(unsigned Opcode) { case VPInstruction::FirstOrderRecurrenceSplice: case VPInstruction::LogicalAnd: case VPInstruction::PtrAdd: + case VPInstruction::WidePtrAdd: case VPInstruction::WideIVStep: return 2; case Instruction::Select: @@ -858,6 +859,12 @@ Value *VPInstruction::generate(VPTransformState &State) { Value *Addend = State.get(getOperand(1), VPLane(0)); return Builder.CreatePtrAdd(Ptr, Addend, Name, getGEPNoWrapFlags()); } + case VPInstruction::WidePtrAdd: { + Value *Ptr = + State.get(getOperand(0), vputils::isSingleScalar(getOperand(0))); + Value *Addend = State.get(getOperand(1)); + return Builder.CreatePtrAdd(Ptr, Addend, Name, getGEPNoWrapFlags()); + } case VPInstruction::AnyOf: { Value *Res = State.get(getOperand(0)); for (VPValue *Op : drop_begin(operands())) @@ -1085,6 +1092,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const { case VPInstruction::Not: case VPInstruction::PtrAdd: case VPInstruction::WideIVStep: + case VPInstruction::WidePtrAdd: case VPInstruction::StepVector: case VPInstruction::ReductionStartVector: return false; @@ -1123,6 +1131,8 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const { return true; case VPInstruction::PtrAdd: return Op == getOperand(0) || vputils::onlyFirstLaneUsed(this); + case VPInstruction::WidePtrAdd: + return Op == getOperand(0); case VPInstruction::ComputeAnyOfResult: case VPInstruction::ComputeFindIVResult: return Op == getOperand(1); @@ -1231,6 +1241,9 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent, case VPInstruction::PtrAdd: O << "ptradd"; break; + case VPInstruction::WidePtrAdd: + O << "wide-ptradd"; + break; case VPInstruction::AnyOf: O << "any-of"; break; @@ -1817,7 +1830,8 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const { return Opcode == Instruction::AShr; case OperationType::GEPOp: return Opcode == Instruction::GetElementPtr || - Opcode == VPInstruction::PtrAdd; + Opcode == VPInstruction::PtrAdd || + Opcode == VPInstruction::WidePtrAdd; case OperationType::FPMathOp: return Opcode == Instruction::FAdd || Opcode == Instruction::FMul || Opcode == Instruction::FSub || Opcode == Instruction::FNeg || @@ -3682,87 +3696,6 @@ bool VPWidenPointerInductionRecipe::onlyScalarsGenerated(bool IsScalable) { (!IsScalable || vputils::onlyFirstLaneUsed(this)); } -void VPWidenPointerInductionRecipe::execute(VPTransformState &State) { - assert(getInductionDescriptor().getKind() == - InductionDescriptor::IK_PtrInduction && - "Not a pointer induction according to InductionDescriptor!"); - assert(State.TypeAnalysis.inferScalarType(this)->isPointerTy() && - "Unexpected type."); - assert(!onlyScalarsGenerated(State.VF.isScalable()) && - "Recipe should have been replaced"); - - unsigned CurrentPart = getUnrollPart(*this); - - // Build a pointer phi - Value *ScalarStartValue = getStartValue()->getLiveInIRValue(); - Type *ScStValueType = ScalarStartValue->getType(); - - BasicBlock *VectorPH = - State.CFG.VPBB2IRBB.at(getParent()->getCFGPredecessor(0)); - PHINode *NewPointerPhi = nullptr; - if (CurrentPart == 0) { - IRBuilder<>::InsertPointGuard Guard(State.Builder); - if (State.Builder.GetInsertPoint() != - State.Builder.GetInsertBlock()->getFirstNonPHIIt()) - State.Builder.SetInsertPoint( - State.Builder.GetInsertBlock()->getFirstNonPHIIt()); - NewPointerPhi = State.Builder.CreatePHI(ScStValueType, 2, "pointer.phi"); - NewPointerPhi->addIncoming(ScalarStartValue, VectorPH); - NewPointerPhi->setDebugLoc(getDebugLoc()); - } else { - // The recipe has been unrolled. In that case, fetch the single pointer phi - // shared among all unrolled parts of the recipe. - auto *GEP = - cast(State.get(getFirstUnrolledPartOperand())); - NewPointerPhi = cast(GEP->getPointerOperand()); - } - - // A pointer induction, performed by using a gep - BasicBlock::iterator InductionLoc = State.Builder.GetInsertPoint(); - Value *ScalarStepValue = State.get(getStepValue(), VPLane(0)); - Type *PhiType = State.TypeAnalysis.inferScalarType(getStepValue()); - Value *RuntimeVF = getRuntimeVF(State.Builder, PhiType, State.VF); - // Add induction update using an incorrect block temporarily. The phi node - // will be fixed after VPlan execution. Note that at this point the latch - // block cannot be used, as it does not exist yet. - // TODO: Model increment value in VPlan, by turning the recipe into a - // multi-def and a subclass of VPHeaderPHIRecipe. - if (CurrentPart == 0) { - // The recipe represents the first part of the pointer induction. Create the - // GEP to increment the phi across all unrolled parts. - Value *NumUnrolledElems = State.get(getOperand(2), true); - - Value *InductionGEP = GetElementPtrInst::Create( - State.Builder.getInt8Ty(), NewPointerPhi, - State.Builder.CreateMul( - ScalarStepValue, - State.Builder.CreateTrunc(NumUnrolledElems, PhiType)), - "ptr.ind", InductionLoc); - - NewPointerPhi->addIncoming(InductionGEP, VectorPH); - } - - // Create actual address geps that use the pointer phi as base and a - // vectorized version of the step value () as offset. - Type *VecPhiType = VectorType::get(PhiType, State.VF); - Value *StartOffsetScalar = State.Builder.CreateMul( - RuntimeVF, ConstantInt::get(PhiType, CurrentPart)); - Value *StartOffset = - State.Builder.CreateVectorSplat(State.VF, StartOffsetScalar); - // Create a vector of consecutive numbers from zero to VF. - StartOffset = State.Builder.CreateAdd( - StartOffset, State.Builder.CreateStepVector(VecPhiType)); - - assert(ScalarStepValue == State.get(getOperand(1), VPLane(0)) && - "scalar step must be the same across all parts"); - Value *GEP = State.Builder.CreateGEP( - State.Builder.getInt8Ty(), NewPointerPhi, - State.Builder.CreateMul(StartOffset, State.Builder.CreateVectorSplat( - State.VF, ScalarStepValue)), - "vector.gep"); - State.set(this, GEP); -} - #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPWidenPointerInductionRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { @@ -3921,11 +3854,6 @@ void VPWidenPHIRecipe::execute(VPTransformState &State) { Value *Op0 = State.get(getOperand(0)); Type *VecTy = Op0->getType(); Instruction *VecPhi = State.Builder.CreatePHI(VecTy, 2, Name); - // Manually move it with the other PHIs in case PHI recipes above this one - // also inserted non-phi instructions. - // TODO: Remove once VPWidenPointerInductionRecipe is also expanded in - // convertToConcreteRecipes. - VecPhi->moveBefore(State.Builder.GetInsertBlock()->getFirstNonPHIIt()); State.set(this, VecPhi); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 98c6b9a70405b..a7965a053e6e3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -963,6 +963,7 @@ static Value *tryToFoldLiveIns(const VPRecipeBase &R, unsigned Opcode, RFlags.getGEPNoWrapFlags()); } case VPInstruction::PtrAdd: + case VPInstruction::WidePtrAdd: return Folder.FoldGEP(IntegerType::getInt8Ty(TypeInfo.getContext()), Ops[0], Ops[1], cast(R).getGEPNoWrapFlags()); @@ -2750,6 +2751,70 @@ expandVPWidenIntOrFpInduction(VPWidenIntOrFpInductionRecipe *WidenIVR, WidenIVR->replaceAllUsesWith(WidePHI); } +/// Expand a VPWidenPointerInductionRecipe into executable recipes, for the +/// initial value, phi and backedge value. In the following example: +/// +/// vector loop: { +/// vector.body: +/// EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION %start, %step, %vf +/// ... +/// EMIT branch-on-count ... +/// } +/// +/// WIDEN-POINTER-INDUCTION will get expanded to: +/// +/// vector loop: { +/// vector.body: +/// EMIT-SCALAR %pointer.phi = phi %start, %ptr.ind +/// EMIT %mul = mul %stepvector, %step +/// EMIT %vector.gep = wide-ptradd %pointer.phi, %mul +/// ... +/// EMIT %ptr.ind = ptradd %pointer.phi, %vf +/// EMIT branch-on-count ... +/// } +static void expandVPWidenPointerInduction(VPWidenPointerInductionRecipe *R, + VPTypeAnalysis &TypeInfo) { + VPlan *Plan = R->getParent()->getPlan(); + VPValue *Start = R->getStartValue(); + VPValue *Step = R->getStepValue(); + VPValue *VF = R->getVFValue(); + + assert(R->getInductionDescriptor().getKind() == + InductionDescriptor::IK_PtrInduction && + "Not a pointer induction according to InductionDescriptor!"); + assert(TypeInfo.inferScalarType(R)->isPointerTy() && "Unexpected type."); + assert(!R->onlyScalarsGenerated(Plan->hasScalableVF()) && + "Recipe should have been replaced"); + + VPBuilder Builder(R); + DebugLoc DL = R->getDebugLoc(); + + // Build a scalar pointer phi. + VPPhi *ScalarPtrPhi = Builder.createScalarPhi(Start, DL, "pointer.phi"); + + // Create actual address geps that use the pointer phi as base and a + // vectorized version of the step value () as offset. + Builder.setInsertPoint(R->getParent(), R->getParent()->getFirstNonPhi()); + Type *StepTy = TypeInfo.inferScalarType(Step); + VPValue *Offset = Builder.createNaryOp(VPInstruction::StepVector, {}, StepTy); + Offset = Builder.createNaryOp(Instruction::Mul, {Offset, Step}); + VPValue *PtrAdd = Builder.createNaryOp( + VPInstruction::WidePtrAdd, {ScalarPtrPhi, Offset}, DL, "vector.gep"); + R->replaceAllUsesWith(PtrAdd); + + // Create the backedge value for the scalar pointer phi. + Builder.setInsertPoint(R->getParent(), R->getParent()->getFirstNonPhi()); + VF = Builder.createScalarZExtOrTrunc(VF, StepTy, TypeInfo.inferScalarType(VF), + DL); + VPValue *Inc = Builder.createNaryOp(Instruction::Mul, {Step, VF}); + + VPBasicBlock *ExitingBB = Plan->getVectorLoopRegion()->getExitingBasicBlock(); + Builder.setInsertPoint(ExitingBB, ExitingBB->getTerminator()->getIterator()); + VPValue *InductionGEP = + Builder.createPtrAdd(ScalarPtrPhi, Inc, DL, "ptr.ind"); + ScalarPtrPhi->addOperand(InductionGEP); +} + void VPlanTransforms::dissolveLoopRegions(VPlan &Plan) { // Replace loop regions with explicity CFG. SmallVector LoopRegions; @@ -2775,6 +2840,12 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan, continue; } + if (auto *WidenIVR = dyn_cast(&R)) { + expandVPWidenPointerInduction(WidenIVR, TypeInfo); + ToRemove.push_back(WidenIVR); + continue; + } + // Expand VPBlendRecipe into VPInstruction::Select. VPBuilder Builder(&R); if (auto *Blend = dyn_cast(&R)) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index 871e37ef3966a..fc072de8ff78e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -65,7 +65,7 @@ class UnrollState { /// Unroll a widen induction recipe \p IV. This introduces recipes to compute /// the induction steps for each part. - void unrollWidenInductionByUF(VPWidenIntOrFpInductionRecipe *IV, + void unrollWidenInductionByUF(VPWidenInductionRecipe *IV, VPBasicBlock::iterator InsertPtForPhi); VPValue *getConstantVPV(unsigned Part) { @@ -148,7 +148,7 @@ void UnrollState::unrollReplicateRegionByUF(VPRegionBlock *VPR) { } void UnrollState::unrollWidenInductionByUF( - VPWidenIntOrFpInductionRecipe *IV, VPBasicBlock::iterator InsertPtForPhi) { + VPWidenInductionRecipe *IV, VPBasicBlock::iterator InsertPtForPhi) { VPBasicBlock *PH = cast( IV->getParent()->getEnclosingLoopRegion()->getSinglePredecessor()); Type *IVTy = TypeInfo.inferScalarType(IV); @@ -159,9 +159,11 @@ void UnrollState::unrollWidenInductionByUF( VPValue *ScalarStep = IV->getStepValue(); VPBuilder Builder(PH); + Type *VectorStepTy = + IVTy->isPointerTy() ? TypeInfo.inferScalarType(ScalarStep) : IVTy; VPInstruction *VectorStep = Builder.createNaryOp( - VPInstruction::WideIVStep, {&Plan.getVF(), ScalarStep}, IVTy, Flags, - IV->getDebugLoc()); + VPInstruction::WideIVStep, {&Plan.getVF(), ScalarStep}, VectorStepTy, + Flags, IV->getDebugLoc()); ToSkip.insert(VectorStep); @@ -169,8 +171,8 @@ void UnrollState::unrollWidenInductionByUF( // remains the header phi. Parts > 0 are computed by adding Step to the // previous part. The header phi recipe will get 2 new operands: the step // value for a single part and the last part, used to compute the backedge - // value during VPWidenIntOrFpInductionRecipe::execute. %Part.0 = - // VPWidenIntOrFpInductionRecipe %Start, %ScalarStep, %VectorStep, %Part.3 + // value during VPWidenInductionRecipe::execute. + // %Part.0 = VPWidenInductionRecipe %Start, %ScalarStep, %VectorStep, %Part.3 // %Part.1 = %Part.0 + %VectorStep // %Part.2 = %Part.1 + %VectorStep // %Part.3 = %Part.2 + %VectorStep @@ -179,8 +181,13 @@ void UnrollState::unrollWidenInductionByUF( // again. VPValue *Prev = IV; Builder.setInsertPoint(IV->getParent(), InsertPtForPhi); - unsigned AddOpc = - IVTy->isFloatingPointTy() ? ID.getInductionOpcode() : Instruction::Add; + unsigned AddOpc; + if (IVTy->isPointerTy()) + AddOpc = VPInstruction::WidePtrAdd; + else if (IVTy->isFloatingPointTy()) + AddOpc = ID.getInductionOpcode(); + else + AddOpc = Instruction::Add; for (unsigned Part = 1; Part != UF; ++Part) { std::string Name = Part > 1 ? "step.add." + std::to_string(Part) : "step.add"; @@ -207,7 +214,7 @@ void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R, return; // Generate step vectors for each unrolled part. - if (auto *IV = dyn_cast(R)) { + if (auto *IV = dyn_cast(R)) { unrollWidenInductionByUF(IV, InsertPtForPhi); return; } @@ -221,10 +228,7 @@ void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R, VPRecipeBase *Copy = R->clone(); Copy->insertBefore(*R->getParent(), InsertPt); addRecipeForPart(R, Copy, Part); - if (isa(R)) { - Copy->addOperand(R); - Copy->addOperand(getConstantVPV(Part)); - } else if (RdxPhi) { + if (RdxPhi) { // If the start value is a ReductionStartVector, use the identity value // (second operand) for unrolled parts. If the scaling factor is > 1, // create a new ReductionStartVector with the scale factor and both @@ -450,8 +454,7 @@ void VPlanTransforms::unrollByUF(VPlan &Plan, unsigned UF, LLVMContext &Ctx) { Unroller.remapOperand(&H, 1, UF - 1); continue; } - if (Unroller.contains(H.getVPSingleValue()) || - isa(&H)) { + if (Unroller.contains(H.getVPSingleValue())) { Part = 1; continue; } diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll index 0754a3884c0e4..9929f35d47dac 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll @@ -35,15 +35,9 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START_2]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 2 ; CHECK-NEXT: [[TMP11:%.*]] = mul i64 1, [[TMP6]] -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP10]], 0 -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP12]], i64 0 -; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: [[TMP13:%.*]] = call @llvm.stepvector.nxv2i64() -; CHECK-NEXT: [[TMP14:%.*]] = add [[DOTSPLAT]], [[TMP13]] -; CHECK-NEXT: [[TMP15:%.*]] = mul [[TMP14]], splat (i64 1) +; CHECK-NEXT: [[TMP15:%.*]] = mul [[TMP13]], splat (i64 1) ; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], [[TMP15]] ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 8 ; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START_1]], i64 [[OFFSET_IDX]] @@ -125,15 +119,9 @@ define void @pointer_induction(ptr noalias %start, i64 %N) { ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX2:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2 ; CHECK-NEXT: [[TMP10:%.*]] = mul i64 1, [[TMP6]] -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP8]], 0 -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP11]], i64 0 -; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: [[TMP12:%.*]] = call @llvm.stepvector.nxv2i64() -; CHECK-NEXT: [[TMP13:%.*]] = add [[DOTSPLAT]], [[TMP12]] -; CHECK-NEXT: [[TMP14:%.*]] = mul [[TMP13]], splat (i64 1) +; CHECK-NEXT: [[TMP14:%.*]] = mul [[TMP12]], splat (i64 1) ; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], [[TMP14]] ; CHECK-NEXT: [[TMP15:%.*]] = extractelement [[VECTOR_GEP]], i32 0 ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP15]], align 1 diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll index 0a751699a0549..a7c45fc57ece9 100644 --- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll +++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll @@ -29,14 +29,14 @@ define void @fn(i32 noundef %n, ptr %in, ptr %out) #0 { ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[IN]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[POINTER_PHI2:%.*]] = phi ptr [ [[OUT]], %[[VECTOR_PH]] ], [ [[PTR_IND3:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[POINTER_PHI2:%.*]] = phi ptr [ [[IN]], %[[VECTOR_PH]] ], [ [[PTR_IND3:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[OUT]], %[[VECTOR_PH]] ], [ [[PTR_IND6:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> ; CHECK-NEXT: [[VECTOR_GEP4:%.*]] = getelementptr i8, ptr [[POINTER_PHI2]], <4 x i32> ; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]]) -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP]], i32 1 -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> [[VECTOR_GEP]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison), !alias.scope [[META0:![0-9]+]] -; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP]], i32 2 +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP4]], i32 1 +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> [[VECTOR_GEP4]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison), !alias.scope [[META0:![0-9]+]] +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP4]], i32 2 ; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> [[TMP1]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison), !alias.scope [[META0]] ; CHECK-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> [[TMP2]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison), !alias.scope [[META0]] ; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[WIDE_MASKED_GATHER]] to <4 x i32> @@ -66,14 +66,14 @@ define void @fn(i32 noundef %n, ptr %in, ptr %out) #0 { ; CHECK-NEXT: [[TMP27:%.*]] = add nuw <4 x i32> [[TMP26]], [[TMP24]] ; CHECK-NEXT: [[TMP28:%.*]] = lshr <4 x i32> [[TMP27]], splat (i32 16) ; CHECK-NEXT: [[TMP29:%.*]] = trunc <4 x i32> [[TMP28]] to <4 x i8> -; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP4]], i32 1 -; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP13]], <4 x ptr> [[VECTOR_GEP4]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]] -; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP4]], i32 2 +; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP]], i32 1 +; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP13]], <4 x ptr> [[VECTOR_GEP]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]] +; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP]], i32 2 ; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP21]], <4 x ptr> [[TMP30]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3]], !noalias [[META0]] ; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP29]], <4 x ptr> [[TMP31]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3]], !noalias [[META0]] ; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 -; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 12 ; CHECK-NEXT: [[PTR_IND3]] = getelementptr i8, ptr [[POINTER_PHI2]], i32 12 +; CHECK-NEXT: [[PTR_IND6]] = getelementptr i8, ptr [[POINTER_PHI]], i32 12 ; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: diff --git a/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll b/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll index 7c6e7053b4c32..e8811253847be 100644 --- a/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll +++ b/llvm/test/Transforms/LoopVectorize/ARM/pointer_iv.ll @@ -746,7 +746,7 @@ define hidden void @pointer_phi_v4i32_uf2(ptr noalias nocapture readonly %A, ptr ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], [[ENTRY]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, <4 x ptr> [[TMP0]], i32 96 ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 2 ; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX]] ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> [[TMP0]], i32 4, <4 x i1> splat (i1 true), <4 x i32> poison) @@ -811,9 +811,9 @@ define hidden void @pointer_phi_v4i32_uf4(ptr noalias nocapture readonly %A, ptr ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], [[ENTRY]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> -; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, <4 x ptr> [[TMP0]], i32 96 +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, <4 x ptr> [[TMP0]], i32 192 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, <4 x ptr> [[TMP0]], i32 288 ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i32 [[INDEX]], 2 ; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[B]], i32 [[OFFSET_IDX]] ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> [[TMP0]], i32 4, <4 x i1> splat (i1 true), <4 x i32> poison) @@ -883,8 +883,8 @@ define hidden void @mult_ptr_iv(ptr noalias nocapture readonly %x, ptr noalias n ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ], [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ], [ [[X]], [[ENTRY]] ] ; CHECK-NEXT: [[POINTER_PHI5:%.*]] = phi ptr [ [[PTR_IND6:%.*]], [[VECTOR_BODY]] ], [ [[Z]], [[ENTRY]] ] -; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[POINTER_PHI5]], <4 x i32> +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, <4 x ptr> [[TMP0]], i32 1 ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> [[TMP0]], i32 1, <4 x i1> splat (i1 true), <4 x i8> poison), !alias.scope [[META28:![0-9]+]] ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, <4 x ptr> [[TMP0]], i32 2 diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll index 80f027452c3c1..9e492c62a5577 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll @@ -170,15 +170,9 @@ define void @single_constant_stride_ptr_iv(ptr %p) { ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[P]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4 ; CHECK-NEXT: [[TMP12:%.*]] = mul i64 8, [[TMP8]] -; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP10]], 0 -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP13]], i64 0 -; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: [[TMP14:%.*]] = call @llvm.stepvector.nxv4i64() -; CHECK-NEXT: [[TMP15:%.*]] = add [[DOTSPLAT]], [[TMP14]] -; CHECK-NEXT: [[TMP16:%.*]] = mul [[TMP15]], splat (i64 8) +; CHECK-NEXT: [[TMP16:%.*]] = mul [[TMP14]], splat (i64 8) ; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], [[TMP16]] ; CHECK-NEXT: [[TMP17:%.*]] = extractelement [[VECTOR_GEP]], i32 0 ; CHECK-NEXT: [[WIDE_VEC:%.*]] = load , ptr [[TMP17]], align 4 @@ -759,34 +753,22 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) { ; STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; STRIDED-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[P]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] ; STRIDED-NEXT: [[POINTER_PHI11:%.*]] = phi ptr [ [[P2]], [[VECTOR_PH]] ], [ [[PTR_IND12:%.*]], [[VECTOR_BODY]] ] -; STRIDED-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; STRIDED-NEXT: [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 4 ; STRIDED-NEXT: [[TMP17:%.*]] = mul i64 [[STRIDE]], [[TMP13]] -; STRIDED-NEXT: [[TMP18:%.*]] = mul i64 [[TMP15]], 0 -; STRIDED-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP18]], i64 0 -; STRIDED-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; STRIDED-NEXT: [[TMP19:%.*]] = call @llvm.stepvector.nxv4i64() -; STRIDED-NEXT: [[TMP20:%.*]] = add [[DOTSPLAT]], [[TMP19]] ; STRIDED-NEXT: [[DOTSPLATINSERT9:%.*]] = insertelement poison, i64 [[STRIDE]], i64 0 ; STRIDED-NEXT: [[DOTSPLAT10:%.*]] = shufflevector [[DOTSPLATINSERT9]], poison, zeroinitializer -; STRIDED-NEXT: [[TMP21:%.*]] = mul [[TMP20]], [[DOTSPLAT10]] -; STRIDED-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], [[TMP21]] -; STRIDED-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() -; STRIDED-NEXT: [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 4 +; STRIDED-NEXT: [[TMP18:%.*]] = mul [[TMP19]], [[DOTSPLAT10]] +; STRIDED-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI11]], [[TMP18]] ; STRIDED-NEXT: [[TMP25:%.*]] = mul i64 [[STRIDE]], [[TMP13]] -; STRIDED-NEXT: [[TMP26:%.*]] = mul i64 [[TMP23]], 0 -; STRIDED-NEXT: [[DOTSPLATINSERT13:%.*]] = insertelement poison, i64 [[TMP26]], i64 0 -; STRIDED-NEXT: [[DOTSPLAT14:%.*]] = shufflevector [[DOTSPLATINSERT13]], poison, zeroinitializer ; STRIDED-NEXT: [[TMP27:%.*]] = call @llvm.stepvector.nxv4i64() -; STRIDED-NEXT: [[TMP28:%.*]] = add [[DOTSPLAT14]], [[TMP27]] -; STRIDED-NEXT: [[TMP29:%.*]] = mul [[TMP28]], [[DOTSPLAT10]] -; STRIDED-NEXT: [[VECTOR_GEP17:%.*]] = getelementptr i8, ptr [[POINTER_PHI11]], [[TMP29]] -; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[VECTOR_GEP]], i32 4, splat (i1 true), poison), !alias.scope [[META15:![0-9]+]] +; STRIDED-NEXT: [[TMP21:%.*]] = mul [[TMP27]], [[DOTSPLAT10]] +; STRIDED-NEXT: [[VECTOR_GEP7:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], [[TMP21]] +; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[VECTOR_GEP7]], i32 4, splat (i1 true), poison), !alias.scope [[META15:![0-9]+]] ; STRIDED-NEXT: [[TMP30:%.*]] = add [[WIDE_MASKED_GATHER]], splat (i32 1) -; STRIDED-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[TMP30]], [[VECTOR_GEP17]], i32 4, splat (i1 true)), !alias.scope [[META18:![0-9]+]], !noalias [[META15]] +; STRIDED-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[TMP30]], [[VECTOR_GEP]], i32 4, splat (i1 true)), !alias.scope [[META18:![0-9]+]], !noalias [[META15]] ; STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] -; STRIDED-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP17]] -; STRIDED-NEXT: [[PTR_IND12]] = getelementptr i8, ptr [[POINTER_PHI11]], i64 [[TMP25]] +; STRIDED-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP25]] +; STRIDED-NEXT: [[PTR_IND12]] = getelementptr i8, ptr [[POINTER_PHI11]], i64 [[TMP17]] ; STRIDED-NEXT: [[TMP31:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; STRIDED-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] ; STRIDED: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr48340.ll b/llvm/test/Transforms/LoopVectorize/X86/pr48340.ll index 8acf89abae2d7..b6acf387fb658 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/pr48340.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/pr48340.ll @@ -28,7 +28,9 @@ define ptr @foo(ptr %p, ptr %p.last) unnamed_addr #0 { ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[P]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> +; CHECK-NEXT: [[STEP_ADD:%.*]] = getelementptr i8, <4 x ptr> [[TMP5]], <4 x i64> splat (i64 4096) +; CHECK-NEXT: [[STEP_ADD_2:%.*]] = getelementptr i8, <4 x ptr> [[STEP_ADD]], <4 x i64> splat (i64 4096) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, <4 x ptr> [[STEP_ADD_2]], <4 x i64> splat (i64 4096) ; CHECK-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <4 x ptr> @llvm.masked.gather.v4p0.v4p0(<4 x ptr> [[TMP8]], i32 8, <4 x i1> splat (i1 true), <4 x ptr> poison) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 16384 @@ -86,7 +88,9 @@ define ptr @bar(ptr %p, ptr %p.last) unnamed_addr #0 { ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[P]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i64> +; CHECK-NEXT: [[STEP_ADD:%.*]] = getelementptr i8, <4 x ptr> [[TMP5]], <4 x i64> splat (i64 4096) +; CHECK-NEXT: [[STEP_ADD_2:%.*]] = getelementptr i8, <4 x ptr> [[STEP_ADD]], <4 x i64> splat (i64 4096) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, <4 x ptr> [[STEP_ADD_2]], <4 x i64> splat (i64 4096) ; CHECK-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <4 x ptr> @llvm.masked.gather.v4p0.v4p0(<4 x ptr> [[TMP8]], i32 8, <4 x i1> splat (i1 true), <4 x ptr> poison) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 16384 diff --git a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll index daa896796c974..69931a0774883 100644 --- a/llvm/test/Transforms/LoopVectorize/pointer-induction.ll +++ b/llvm/test/Transforms/LoopVectorize/pointer-induction.ll @@ -312,8 +312,8 @@ define void @outside_lattice(ptr noalias %p, ptr noalias %q, i32 %n) { ; DEFAULT-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[Q:%.*]], i32 [[OFFSET_IDX]] ; DEFAULT-NEXT: store <4 x i32> [[VEC_IND]], ptr [[TMP7]], align 4 ; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; DEFAULT-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4) ; DEFAULT-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 16 +; DEFAULT-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4) ; DEFAULT-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; DEFAULT-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; DEFAULT: middle.block: @@ -366,8 +366,8 @@ define void @outside_lattice(ptr noalias %p, ptr noalias %q, i32 %n) { ; STRIDED-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[Q:%.*]], i32 [[OFFSET_IDX]] ; STRIDED-NEXT: store <4 x i32> [[VEC_IND]], ptr [[TMP7]], align 4 ; STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; STRIDED-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4) ; STRIDED-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 16 +; STRIDED-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4) ; STRIDED-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; STRIDED-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; STRIDED: middle.block: