diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h index fb2d4d3cc50ed..d15b96db58fef 100644 --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -62,7 +62,8 @@ enum class sampleprof_error { uncompress_failed, zlib_unavailable, hash_mismatch, - illegal_line_offset + illegal_line_offset, + duplicate_vtable_type }; inline std::error_code make_error_code(sampleprof_error E) { @@ -91,6 +92,8 @@ struct is_error_code_enum : std::true_type {}; namespace llvm { namespace sampleprof { +constexpr char kVTableProfPrefix[] = "vtables "; + enum SampleProfileFormat { SPF_None = 0, SPF_Text = 0x1, @@ -204,6 +207,9 @@ enum class SecProfSummaryFlags : uint32_t { /// SecFlagIsPreInlined means this profile contains ShouldBeInlined /// contexts thus this is CS preinliner computed. SecFlagIsPreInlined = (1 << 4), + + /// SecFlagHasVTableTypeProf means this profile contains vtable type profiles. + SecFlagHasVTableTypeProf = (1 << 5), }; enum class SecFuncMetadataFlags : uint32_t { @@ -303,7 +309,7 @@ struct LineLocation { } uint64_t getHashCode() const { - return ((uint64_t) Discriminator << 32) | LineOffset; + return ((uint64_t)Discriminator << 32) | LineOffset; } uint32_t LineOffset; @@ -318,16 +324,28 @@ struct LineLocationHash { LLVM_ABI raw_ostream &operator<<(raw_ostream &OS, const LineLocation &Loc); +/// Key represents the id of a vtable and value represents its count. +/// TODO: Rename class FunctionId to SymbolId in a separate PR. +using TypeCountMap = std::map; + +/// Write \p Map to the output stream. Keys are linearized using \p NameTable +/// and written as ULEB128. Values are written as ULEB128 as well. +std::error_code +serializeTypeMap(const TypeCountMap &Map, + const MapVector &NameTable, + raw_ostream &OS); + /// Representation of a single sample record. /// /// A sample record is represented by a positive integer value, which /// indicates how frequently was the associated line location executed. /// /// Additionally, if the associated location contains a function call, -/// the record will hold a list of all the possible called targets. For -/// direct calls, this will be the exact function being invoked. For -/// indirect calls (function pointers, virtual table dispatch), this -/// will be a list of one or more functions. +/// the record will hold a list of all the possible called targets and the types +/// for virtual table dispatches. For direct calls, this will be the exact +/// function being invoked. For indirect calls (function pointers, virtual table +/// dispatch), this will be a list of one or more functions. For virtual table +/// dispatches, this record will also hold the type of the object. class SampleRecord { public: using CallTarget = std::pair; @@ -746,6 +764,7 @@ using BodySampleMap = std::map; // memory, which is *very* significant for large profiles. using FunctionSamplesMap = std::map; using CallsiteSampleMap = std::map; +using CallsiteTypeMap = std::map; using LocToLocMap = std::unordered_map; @@ -928,6 +947,14 @@ class FunctionSamples { return &Iter->second; } + /// Returns the TypeCountMap for inlined callsites at the given \p Loc. + const TypeCountMap *findCallsiteTypeSamplesAt(const LineLocation &Loc) const { + auto Iter = VirtualCallsiteTypeCounts.find(mapIRLocToProfileLoc(Loc)); + if (Iter == VirtualCallsiteTypeCounts.end()) + return nullptr; + return &Iter->second; + } + /// Returns a pointer to FunctionSamples at the given callsite location /// \p Loc with callee \p CalleeName. If no callsite can be found, relax /// the restriction to return the FunctionSamples at callsite location @@ -989,6 +1016,42 @@ class FunctionSamples { return CallsiteSamples; } + /// Return all the callsite type samples collected in the body of the + /// function. + const CallsiteTypeMap &getCallsiteTypeCounts() const { + return VirtualCallsiteTypeCounts; + } + + /// Returns the type samples for the un-drifted location of \p Loc. + TypeCountMap &getTypeSamplesAt(const LineLocation &Loc) { + return VirtualCallsiteTypeCounts[mapIRLocToProfileLoc(Loc)]; + } + + /// Scale \p Other sample counts by \p Weight and add the scaled result to the + /// type samples for the undrifted location of \p Loc. + template + sampleprof_error addCallsiteVTableTypeProfAt(const LineLocation &Loc, + const T &Other, + uint64_t Weight = 1) { + static_assert((std::is_same_v || + std::is_same_v) && + std::is_same_v, + "T must be a map with StringRef or FunctionId as key and " + "uint64_t as value"); + TypeCountMap &TypeCounts = getTypeSamplesAt(Loc); + bool Overflowed = false; + + for (const auto [Type, Count] : Other) { + FunctionId TypeId(Type); + bool RowOverflow = false; + TypeCounts[TypeId] = SaturatingMultiplyAdd( + Count, Weight, TypeCounts[TypeId], &RowOverflow); + Overflowed |= RowOverflow; + } + return Overflowed ? sampleprof_error::counter_overflow + : sampleprof_error::success; + } + /// Return the maximum of sample counts in a function body. When SkipCallSite /// is false, which is the default, the return count includes samples in the /// inlined functions. When SkipCallSite is true, the return count only @@ -1043,6 +1106,10 @@ class FunctionSamples { mergeSampleProfErrors(Result, FSMap[Rec.first].merge(Rec.second, Weight)); } + for (const auto &[Loc, OtherTypeMap] : Other.getCallsiteTypeCounts()) + mergeSampleProfErrors( + Result, addCallsiteVTableTypeProfAt(Loc, OtherTypeMap, Weight)); + return Result; } @@ -1286,6 +1353,23 @@ class FunctionSamples { /// collected in the call to baz() at line offset 8. CallsiteSampleMap CallsiteSamples; + /// Map virtual callsites to the vtable from which they are loaded. + /// + /// Each entry is a mapping from the location to the list of vtables and their + /// sampled counts. For example, given: + /// + /// void foo() { + /// ... + /// 5 inlined_vcall_bar(); + /// ... + /// 5 inlined_vcall_baz(); + /// ... + /// 200 inlined_vcall_qux(); + /// } + /// This map will contain two entries. One with two types for line offset 5 + /// and one with one type for line offset 200. + CallsiteTypeMap VirtualCallsiteTypeCounts; + /// IR to profile location map generated by stale profile matching. /// /// Each entry is a mapping from the location on current build to the matched diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h index bfe079fbe536f..fe8266247b2db 100644 --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -703,6 +703,14 @@ class LLVM_ABI SampleProfileReaderBinary : public SampleProfileReader { /// otherwise same as readStringFromTable, also return its hash value. ErrorOr> readSampleContextFromTable(); + /// Read all virtual functions' vtable access counts for \p FProfile. + std::error_code readCallsiteVTableProf(FunctionSamples &FProfile); + + /// Read bytes from the input buffer pointed by `Data` and decode them into + /// \p M. `Data` will be advanced to the end of the read bytes when this + /// function returns. Returns error if any. + std::error_code readVTableTypeCountMap(TypeCountMap &M); + /// Points to the current location in the buffer. const uint8_t *Data = nullptr; @@ -727,6 +735,10 @@ class LLVM_ABI SampleProfileReaderBinary : public SampleProfileReader { /// to the start of MD5SampleContextTable. const uint64_t *MD5SampleContextStart = nullptr; + /// If true, the profile has vtable profiles and reader should decode them + /// to parse profiles correctly. + bool ReadVTableProf = false; + private: std::error_code readSummaryEntry(std::vector &Entries); virtual std::error_code verifySPMagic(uint64_t Magic) = 0; diff --git a/llvm/include/llvm/ProfileData/SampleProfWriter.h b/llvm/include/llvm/ProfileData/SampleProfWriter.h index e84b2095efd7b..9dbeaf56509b0 100644 --- a/llvm/include/llvm/ProfileData/SampleProfWriter.h +++ b/llvm/include/llvm/ProfileData/SampleProfWriter.h @@ -217,13 +217,20 @@ class LLVM_ABI SampleProfileWriterBinary : public SampleProfileWriter { std::error_code writeBody(const FunctionSamples &S); inline void stablizeNameTable(MapVector &NameTable, std::set &V); - + MapVector NameTable; - + void addName(FunctionId FName); virtual void addContext(const SampleContext &Context); void addNames(const FunctionSamples &S); + /// Write \p CallsiteTypeMap to the output stream \p OS. + std::error_code + writeCallsiteVTableProf(const CallsiteTypeMap &CallsiteTypeMap, + raw_ostream &OS); + + bool WriteVTableProf = false; + private: LLVM_ABI friend ErrorOr> SampleProfileWriter::create(std::unique_ptr &OS, @@ -412,8 +419,7 @@ class LLVM_ABI SampleProfileWriterExtBinaryBase class LLVM_ABI SampleProfileWriterExtBinary : public SampleProfileWriterExtBinaryBase { public: - SampleProfileWriterExtBinary(std::unique_ptr &OS) - : SampleProfileWriterExtBinaryBase(OS) {} + SampleProfileWriterExtBinary(std::unique_ptr &OS); private: std::error_code writeDefaultLayout(const SampleProfileMap &ProfileMap); diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp index 60c1393616713..f8706186454af 100644 --- a/llvm/lib/ProfileData/SampleProf.cpp +++ b/llvm/lib/ProfileData/SampleProf.cpp @@ -47,6 +47,24 @@ bool FunctionSamples::ProfileIsPreInlined = false; bool FunctionSamples::UseMD5 = false; bool FunctionSamples::HasUniqSuffix = true; bool FunctionSamples::ProfileIsFS = false; + +std::error_code +serializeTypeMap(const TypeCountMap &Map, + const MapVector &NameTable, + raw_ostream &OS) { + encodeULEB128(Map.size(), OS); + for (const auto &[TypeName, SampleCount] : Map) { + if (auto NameIndexIter = NameTable.find(TypeName); + NameIndexIter != NameTable.end()) { + encodeULEB128(NameIndexIter->second, OS); + } else { + // If the type is not in the name table, we cannot serialize it. + return sampleprof_error::truncated_name_table; + } + encodeULEB128(SampleCount, OS); + } + return sampleprof_error::success; +} } // namespace sampleprof } // namespace llvm @@ -93,6 +111,8 @@ class SampleProfErrorCategoryType : public std::error_category { return "Function hash mismatch"; case sampleprof_error::illegal_line_offset: return "Illegal line offset in sample profile data"; + case sampleprof_error::duplicate_vtable_type: + return "Duplicate vtable type in one map"; } llvm_unreachable("A value of sampleprof_error has no message."); } @@ -126,6 +146,7 @@ sampleprof_error SampleRecord::merge(const SampleRecord &Other, for (const auto &I : Other.getCallTargets()) { mergeSampleProfErrors(Result, addCalledTarget(I.first, I.second, Weight)); } + return Result; } @@ -178,6 +199,17 @@ raw_ostream &llvm::sampleprof::operator<<(raw_ostream &OS, return OS; } +static void printTypeCountMap(raw_ostream &OS, LineLocation Loc, + const TypeCountMap &TypeCountMap) { + if (TypeCountMap.empty()) { + return; + } + OS << Loc << ": vtables: "; + for (const auto &[Type, Count] : TypeCountMap) + OS << Type << ":" << Count << " "; + OS << "\n"; +} + /// Print the samples collected for a function on stream \p OS. void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const { if (getFunctionHash()) @@ -192,7 +224,13 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const { SampleSorter SortedBodySamples(BodySamples); for (const auto &SI : SortedBodySamples.get()) { OS.indent(Indent + 2); + const auto &Loc = SI->first; OS << SI->first << ": " << SI->second; + if (const TypeCountMap *TypeCountMap = + this->findCallsiteTypeSamplesAt(Loc)) { + OS.indent(Indent + 2); + printTypeCountMap(OS, Loc, *TypeCountMap); + } } OS.indent(Indent); OS << "}\n"; @@ -214,6 +252,11 @@ void FunctionSamples::print(raw_ostream &OS, unsigned Indent) const { OS << Loc << ": inlined callee: " << FuncSample.getFunction() << ": "; FuncSample.print(OS, Indent + 4); } + auto TypeSamplesIter = VirtualCallsiteTypeCounts.find(Loc); + if (TypeSamplesIter != VirtualCallsiteTypeCounts.end()) { + OS.indent(Indent + 2); + printTypeCountMap(OS, Loc, TypeSamplesIter->second); + } } OS.indent(Indent); OS << "}\n"; diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp index cf7874041f3bf..b8b55f72935b5 100644 --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -197,8 +197,30 @@ enum class LineType { CallSiteProfile, BodyProfile, Metadata, + VirtualCallTypeProfile, }; +static bool parseTypeCountMap(StringRef Input, + DenseMap &TypeCountMap) { + for (size_t Index = Input.find_first_not_of(' '); Index != StringRef::npos;) { + size_t n1 = Input.find(':', Index); + if (n1 == StringRef::npos) + return false; // No colon found, invalid format. + StringRef TypeName = Input.substr(Index, n1 - Index); + // n2 is the start index of count. + size_t n2 = n1 + 1; + // n3 is the start index after the 'target:count' pair. + size_t n3 = Input.find_first_of(' ', n2); + uint64_t Count; + if (Input.substr(n2, n3 - n2).getAsInteger(10, Count)) + return false; // Invalid count. + TypeCountMap[TypeName] = Count; + Index = (n3 == StringRef::npos) ? StringRef::npos + : Input.find_first_not_of(' ', n3); + } + return true; +} + /// Parse \p Input as line sample. /// /// \param Input input line. @@ -215,6 +237,7 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth, uint64_t &NumSamples, uint32_t &LineOffset, uint32_t &Discriminator, StringRef &CalleeName, DenseMap &TargetCountMap, + DenseMap &TypeCountMap, uint64_t &FunctionHash, uint32_t &Attributes, bool &IsFlat) { for (Depth = 0; Input[Depth] == ' '; Depth++) @@ -289,6 +312,7 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth, n4 = AfterColon.find_first_of(' '); n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size(); StringRef WordAfterColon = Rest.substr(n3 + 1, n4 - n3 - 1); + // Break the loop if parsing integer succeeded. if (!WordAfterColon.getAsInteger(10, count)) break; @@ -306,6 +330,10 @@ static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth, // Change n3 to the next blank space after colon + integer pair. n3 = n4; } + } else if (Rest.starts_with(kVTableProfPrefix)) { + LineTy = LineType::VirtualCallTypeProfile; + return parseTypeCountMap(Rest.substr(strlen(kVTableProfPrefix)), + TypeCountMap); } else { LineTy = LineType::CallSiteProfile; size_t n3 = Rest.find_last_of(':'); @@ -374,14 +402,15 @@ std::error_code SampleProfileReaderText::readImpl() { uint64_t NumSamples; StringRef FName; DenseMap TargetCountMap; + DenseMap TypeCountMap; uint32_t Depth, LineOffset, Discriminator; LineType LineTy; uint64_t FunctionHash = 0; uint32_t Attributes = 0; bool IsFlat = false; if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset, - Discriminator, FName, TargetCountMap, FunctionHash, - Attributes, IsFlat)) { + Discriminator, FName, TargetCountMap, TypeCountMap, + FunctionHash, Attributes, IsFlat)) { reportError(LineIt.line_number(), "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " + *LineIt); @@ -410,6 +439,14 @@ std::error_code SampleProfileReaderText::readImpl() { DepthMetadata = 0; break; } + + case LineType::VirtualCallTypeProfile: { + mergeSampleProfErrors( + Result, InlineStack.back()->addCallsiteVTableTypeProfAt( + LineLocation(LineOffset, Discriminator), TypeCountMap)); + break; + } + case LineType::BodyProfile: { FunctionSamples &FProfile = *InlineStack.back(); for (const auto &name_count : TargetCountMap) { @@ -591,6 +628,59 @@ SampleProfileReaderBinary::readSampleContextFromTable() { return std::make_pair(Context, Hash); } +std::error_code +SampleProfileReaderBinary::readVTableTypeCountMap(TypeCountMap &M) { + auto NumVTableTypes = readNumber(); + if (std::error_code EC = NumVTableTypes.getError()) + return EC; + + for (uint32_t I = 0; I < *NumVTableTypes; ++I) { + auto VTableType(readStringFromTable()); + if (std::error_code EC = VTableType.getError()) + return EC; + + auto VTableSamples = readNumber(); + if (std::error_code EC = VTableSamples.getError()) + return EC; + + if (!M.insert(std::make_pair(*VTableType, *VTableSamples)).second) + return sampleprof_error::duplicate_vtable_type; + } + return sampleprof_error::success; +} + +std::error_code +SampleProfileReaderBinary::readCallsiteVTableProf(FunctionSamples &FProfile) { + if (!ReadVTableProf) + return sampleprof_error::success; + + // Read the vtable type profile for the callsite. + auto NumCallsites = readNumber(); + if (std::error_code EC = NumCallsites.getError()) + return EC; + + for (uint32_t I = 0; I < *NumCallsites; ++I) { + auto LineOffset = readNumber(); + if (std::error_code EC = LineOffset.getError()) + return EC; + + if (!isOffsetLegal(*LineOffset)) + return sampleprof_error::illegal_line_offset; + + auto Discriminator = readNumber(); + if (std::error_code EC = Discriminator.getError()) + return EC; + + // Here we handle FS discriminators: + const uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask(); + + if (std::error_code EC = readVTableTypeCountMap(FProfile.getTypeSamplesAt( + LineLocation(*LineOffset, DiscriminatorVal)))) + return EC; + } + return sampleprof_error::success; +} + std::error_code SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { auto NumSamples = readNumber(); @@ -671,7 +761,7 @@ SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { return EC; } - return sampleprof_error::success; + return readCallsiteVTableProf(FProfile); } std::error_code @@ -733,6 +823,8 @@ std::error_code SampleProfileReaderExtBinaryBase::readOneSection( FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true; if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator)) FunctionSamples::ProfileIsFS = ProfileIsFS = true; + if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagHasVTableTypeProf)) + ReadVTableProf = true; break; case SecNameTable: { bool FixedLengthMD5 = diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp index 9173a0f94f69d..dfab404a64eb2 100644 --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -41,6 +41,11 @@ using namespace llvm; using namespace sampleprof; +// To begin with, make this option off by default. +static cl::opt ExtBinaryWriteVTableTypeProf( + "extbinary-write-vtable-type-prof", cl::init(false), cl::Hidden, + cl::desc("Write vtable type profile in ext-binary sample profile writer")); + namespace llvm { namespace support { namespace endian { @@ -435,6 +440,9 @@ std::error_code SampleProfileWriterExtBinaryBase::writeOneSection( addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagIsPreInlined); if (Type == SecProfSummary && FunctionSamples::ProfileIsFS) addSectionFlag(SecProfSummary, SecProfSummaryFlags::SecFlagFSDiscriminator); + if (Type == SecProfSummary && ExtBinaryWriteVTableTypeProf) + addSectionFlag(SecProfSummary, + SecProfSummaryFlags::SecFlagHasVTableTypeProf); uint64_t SectionStart = markSectionStart(Type, LayoutIdx); switch (Type) { @@ -478,6 +486,12 @@ std::error_code SampleProfileWriterExtBinaryBase::writeOneSection( return sampleprof_error::success; } +SampleProfileWriterExtBinary::SampleProfileWriterExtBinary( + std::unique_ptr &OS) + : SampleProfileWriterExtBinaryBase(OS) { + WriteVTableProf = ExtBinaryWriteVTableTypeProf; +} + std::error_code SampleProfileWriterExtBinary::writeDefaultLayout( const SampleProfileMap &ProfileMap) { // The const indices passed to writeOneSection below are specifying the @@ -587,6 +601,19 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) { OS << " " << J.first << ":" << J.second; OS << "\n"; LineCount++; + + if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc); + Map && !Map->empty()) { + OS.indent(Indent + 1); + Loc.print(OS); + OS << ": "; + OS << kVTableProfPrefix; + for (const auto [TypeName, Count] : *Map) { + OS << TypeName << ":" << Count << " "; + } + OS << "\n"; + LineCount++; + } } SampleSorter SortedCallsiteSamples( @@ -603,7 +630,21 @@ std::error_code SampleProfileWriterText::writeSample(const FunctionSamples &S) { if (std::error_code EC = writeSample(CalleeSamples)) return EC; } + + if (const TypeCountMap *Map = S.findCallsiteTypeSamplesAt(Loc); + Map && !Map->empty()) { + OS.indent(Indent); + Loc.print(OS); + OS << ": "; + OS << kVTableProfPrefix; + for (const auto [TypeId, Count] : *Map) { + OS << TypeId << ":" << Count << " "; + } + OS << "\n"; + LineCount++; + } } + Indent -= 1; if (FunctionSamples::ProfileIsProbeBased) { @@ -663,6 +704,17 @@ void SampleProfileWriterBinary::addNames(const FunctionSamples &S) { addName(CalleeSamples.getFunction()); addNames(CalleeSamples); } + + if (!WriteVTableProf) + return; + // Add all the vtable names to NameTable. + for (const auto &VTableAccessCountMap : + llvm::make_second_range(S.getCallsiteTypeCounts())) { + // Add type name to NameTable. + for (const auto Type : llvm::make_first_range(VTableAccessCountMap)) { + addName(Type); + } + } } void SampleProfileWriterExtBinaryBase::addContext( @@ -801,6 +853,21 @@ std::error_code SampleProfileWriterExtBinaryBase::writeHeader( return sampleprof_error::success; } +std::error_code SampleProfileWriterBinary::writeCallsiteVTableProf( + const CallsiteTypeMap &CallsiteTypeMap, raw_ostream &OS) { + if (!WriteVTableProf) + return sampleprof_error::success; + + encodeULEB128(CallsiteTypeMap.size(), OS); + for (const auto &[Loc, TypeMap] : CallsiteTypeMap) { + Loc.serialize(OS); + if (std::error_code EC = serializeTypeMap(TypeMap, getNameTable(), OS)) + return EC; + } + + return sampleprof_error::success; +} + std::error_code SampleProfileWriterBinary::writeSummary() { auto &OS = *OutputStream; encodeULEB128(Summary->getTotalCount(), OS); @@ -838,15 +905,14 @@ std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) { for (const auto &J : S.getCallsiteSamples()) NumCallsites += J.second.size(); encodeULEB128(NumCallsites, OS); - for (const auto &[Loc, CalleeFunctionSampleMap] : S.getCallsiteSamples()) - for (const auto &FunctionSample : - llvm::make_second_range(CalleeFunctionSampleMap)) { - Loc.serialize(OS); - if (std::error_code EC = writeBody(FunctionSample)) + for (const auto &J : S.getCallsiteSamples()) + for (const auto &FS : J.second) { + J.first.serialize(OS); + if (std::error_code EC = writeBody(FS.second)) return EC; } - return sampleprof_error::success; + return writeCallsiteVTableProf(S.getCallsiteTypeCounts(), OS); } /// Write samples of a top-level function to a binary file. diff --git a/llvm/test/tools/llvm-profdata/Inputs/profile-symbol-list-ext.expected b/llvm/test/tools/llvm-profdata/Inputs/profile-symbol-list-ext.expected new file mode 100644 index 0000000000000..f7e7499a2c781 --- /dev/null +++ b/llvm/test/tools/llvm-profdata/Inputs/profile-symbol-list-ext.expected @@ -0,0 +1,44 @@ +Function: main: 368038, 0, 7 sampled lines +Samples collected in the function's body { + 4: 1068 + 4.2: 1068 + 5: 2150 + 5.1: 2150 + 6: 4160 + 7: 1068 + 9: 4128, calls: _Z3bari:2942 _Z3fooi:1262 + 9: vtables: _ZTVbar:2942 _ZTVfoo:1260 +} +Samples collected in inlined callsites { + 10: inlined callee: inline1: 2000, 0, 1 sampled lines + Samples collected in the function's body { + 1: 2000 + } + No inlined callsites in this function + 10: inlined callee: inline2: 4000, 0, 1 sampled lines + Samples collected in the function's body { + 1: 4000 + } + No inlined callsites in this function + 10: vtables: _ZTVinline1:2000 _ZTVinline2:4000 +} +Function: _Z3bari: 40602, 2874, 1 sampled lines +Samples collected in the function's body { + 1: 2874 +} +No inlined callsites in this function +Function: _Z3fooi: 15422, 1220, 1 sampled lines +Samples collected in the function's body { + 1: 1220 +} +No inlined callsites in this function +======== Dump profile symbol list ======== +_Z3goov +_Z3sumii +__libc_csu_fini +__libc_csu_init +_dl_relocate_static_pie +_fini +_init +_start +main diff --git a/llvm/test/tools/llvm-profdata/Inputs/sample-profile-ext.proftext b/llvm/test/tools/llvm-profdata/Inputs/sample-profile-ext.proftext new file mode 100644 index 0000000000000..100133fa17ccb --- /dev/null +++ b/llvm/test/tools/llvm-profdata/Inputs/sample-profile-ext.proftext @@ -0,0 +1,18 @@ +main:184019:0 + 4: 534 + 4.2: 534 + 5: 1075 + 5.1: 1075 + 6: 2080 + 7: 534 + 9: 2064 _Z3bari:1471 _Z3fooi:631 + 9: vtables _ZTVbar:1471 _ZTVfoo:630 + 10: inline1:1000 + 1: 1000 + 10: inline2:2000 + 1: 2000 + 10: vtables _ZTVinline1:1000 _ZTVinline2:2000 +_Z3bari:20301:1437 + 1: 1437 +_Z3fooi:7711:610 + 1: 610 diff --git a/llvm/test/tools/llvm-profdata/profile-symbol-list-compress.test b/llvm/test/tools/llvm-profdata/profile-symbol-list-compress.test index b445695c8e8e4..06a3ef45ed39a 100644 --- a/llvm/test/tools/llvm-profdata/profile-symbol-list-compress.test +++ b/llvm/test/tools/llvm-profdata/profile-symbol-list-compress.test @@ -4,3 +4,9 @@ REQUIRES: zlib ; RUN: llvm-profdata merge -sample -extbinary -compress-all-sections %t.1.output %t.2.output -o %t.3.output ; RUN: llvm-profdata show -sample -show-prof-sym-list %t.3.output > %t.4.output ; RUN: diff -b %S/Inputs/profile-symbol-list.expected %t.4.output + +; RUN: llvm-profdata merge -sample -extbinary -compress-all-sections -extbinary-write-vtable-type-prof -prof-sym-list=%S/Inputs/profile-symbol-list-1.text %S/Inputs/sample-profile-ext.proftext -o %t.1.output +; RUN: llvm-profdata merge -sample -extbinary -compress-all-sections -extbinary-write-vtable-type-prof -prof-sym-list=%S/Inputs/profile-symbol-list-2.text %S/Inputs/sample-profile-ext.proftext -o %t.2.output +; RUN: llvm-profdata merge -sample -extbinary -compress-all-sections -extbinary-write-vtable-type-prof %t.1.output %t.2.output -o %t.3.output +; RUN: llvm-profdata show -sample -show-prof-sym-list %t.3.output > %t.4.output +; RUN: diff -b %S/Inputs/profile-symbol-list-ext.expected %t.4.output diff --git a/llvm/test/tools/llvm-profdata/profile-symbol-list.test b/llvm/test/tools/llvm-profdata/profile-symbol-list.test index 39dcd11ec1db7..009208001fc61 100644 --- a/llvm/test/tools/llvm-profdata/profile-symbol-list.test +++ b/llvm/test/tools/llvm-profdata/profile-symbol-list.test @@ -7,3 +7,9 @@ ; RUN: llvm-profdata show -sample -show-sec-info-only %t.5.output | FileCheck %s -check-prefix=NOSYMLIST ; NOSYMLIST: ProfileSymbolListSection {{.*}} Size: 0 + +; RUN: llvm-profdata merge -sample -extbinary -extbinary-write-vtable-type-prof -prof-sym-list=%S/Inputs/profile-symbol-list-1.text %S/Inputs/sample-profile-ext.proftext -o %t.1.output +; RUN: llvm-profdata merge -sample -extbinary -extbinary-write-vtable-type-prof -prof-sym-list=%S/Inputs/profile-symbol-list-2.text %S/Inputs/sample-profile-ext.proftext -o %t.2.output +; RUN: llvm-profdata merge -sample -extbinary -extbinary-write-vtable-type-prof %t.1.output %t.2.output -o %t.3.output +; RUN: llvm-profdata show -sample -show-prof-sym-list %t.3.output > %t.4.output +; RUN: diff -b %S/Inputs/profile-symbol-list-ext.expected %t.4.output diff --git a/llvm/test/tools/llvm-profdata/roundtrip.test b/llvm/test/tools/llvm-profdata/roundtrip.test index 7af76e0a58224..eb55534763877 100644 --- a/llvm/test/tools/llvm-profdata/roundtrip.test +++ b/llvm/test/tools/llvm-profdata/roundtrip.test @@ -16,3 +16,9 @@ RUN: llvm-profdata merge --sample --binary -output=%t.4.profdata %S/Inputs/sampl RUN: llvm-profdata merge --sample --extbinary -output=%t.5.profdata %t.4.profdata RUN: llvm-profdata merge --sample --text -output=%t.4.proftext %t.5.profdata RUN: diff -b %t.4.proftext %S/Inputs/sample-profile.proftext +# Round trip from text --> extbinary --> text. +# The vtable profile is supported by ext-binary profile but not raw binary profile format, +# so we don't use raw binary profile format in this roundtrip. +RUN: llvm-profdata merge --sample --extbinary -extbinary-write-vtable-type-prof --output=%t.5.profdata %S/Inputs/sample-profile-ext.proftext +RUN: llvm-profdata merge --sample --text --output=%t.5.proftext %t.5.profdata +RUN: diff -b %t.5.proftext %S/Inputs/sample-profile-ext.proftext