From c8edc91d13c381910d0016481da747f8d730a39e Mon Sep 17 00:00:00 2001 From: Garima Dhaked Date: Thu, 30 Jul 2026 04:37:04 -0600 Subject: [PATCH 1/6] Enable coalesced JSON dtrace_dump output by default in aie_dtrace plugin. Apply Debug.dtrace_output_json_format and dtrace_coalesce_result before XRT initializes dtrace so results are emitted as coalesced JSON on hw context teardown. Co-authored-by: Cursor --- .../plugin/aie_dtrace/aie_dtrace_plugin.cpp | 6 ++++ .../aie_dtrace/util/aie_dtrace_util.cpp | 35 +++++++++++++++++++ .../plugin/aie_dtrace/util/aie_dtrace_util.h | 4 +++ .../plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp | 3 ++ 4 files changed, 48 insertions(+) diff --git a/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp b/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp index 0b7f4faa..3c5f1abc 100644 --- a/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp +++ b/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp @@ -4,6 +4,7 @@ #define XDP_PLUGIN_SOURCE #include "xdp/profile/plugin/aie_dtrace/aie_dtrace_plugin.h" +#include "xdp/profile/plugin/aie_dtrace/util/aie_dtrace_util.h" #include "core/common/api/hw_context_int.h" #include "core/common/config_reader.h" @@ -31,6 +32,9 @@ namespace xdp { { AieDtracePlugin::live = true; + if (xrt_core::config::get_aie_dtrace()) + aie::dtrace::initDtraceOutputConfig(); + db->registerPlugin(this); db->registerInfo(info::aie_dtrace); db->getStaticInfo().setAieApplication(); @@ -68,6 +72,8 @@ namespace xdp { if (!xrt_core::config::get_aie_dtrace()) return; + aie::dtrace::initDtraceOutputConfig(); + if (!handle) return; diff --git a/profile/plugin/aie_dtrace/util/aie_dtrace_util.cpp b/profile/plugin/aie_dtrace/util/aie_dtrace_util.cpp index 93352107..89749265 100644 --- a/profile/plugin/aie_dtrace/util/aie_dtrace_util.cpp +++ b/profile/plugin/aie_dtrace/util/aie_dtrace_util.cpp @@ -5,8 +5,43 @@ #include "xdp/profile/plugin/aie_dtrace/util/aie_dtrace_util.h" +#include "core/common/config_reader.h" +#include "core/common/message.h" + +#include + namespace xdp::aie::dtrace { + namespace { + using severity_level = xrt_core::message::severity_level; + + static constexpr unsigned int DEFAULT_COALESCE_RESULT_MEMORY_MB = 256; + } // anonymous namespace + + void + initDtraceOutputConfig() + { + static std::once_flag once; + std::call_once(once, []() { + try { + xrt_core::config::detail::set("Debug.dtrace_output_json_format", "true"); + xrt_core::config::detail::set("Debug.dtrace_coalesce_result", "true"); + xrt_core::config::detail::set("Debug.dtrace_coalesce_result_memory_mb", + std::to_string(DEFAULT_COALESCE_RESULT_MEMORY_MB)); + } + catch (const std::exception& e) { + xrt_core::message::send(severity_level::warning, "XRT", + std::string("AIE dtrace: could not apply default dtrace output settings: ") + + e.what()); + return; + } + + xrt_core::message::send(severity_level::info, "XRT", + "AIE dtrace: enabled JSON dtrace_dump output with coalesced results " + "(dtrace_dump_ctx__.json on hw context teardown)"); + }); + } + std::map> getBandwidthInterfaceTileEventSets(int hwGen) { diff --git a/profile/plugin/aie_dtrace/util/aie_dtrace_util.h b/profile/plugin/aie_dtrace/util/aie_dtrace_util.h index 2a49a334..a8210658 100644 --- a/profile/plugin/aie_dtrace/util/aie_dtrace_util.h +++ b/profile/plugin/aie_dtrace/util/aie_dtrace_util.h @@ -17,6 +17,10 @@ namespace xdp::aie::dtrace { // Shim bandwidth metric sets used for Debug.aie_dtrace (not part of standard aie_profile ini). std::map> getBandwidthInterfaceTileEventSets(int hwGen); + // Enable JSON dtrace_dump output with coalesced results by default. + // Must run before XRT creates the first dtrace module (config keys lock on first read). + void initDtraceOutputConfig(); + } // namespace xdp::aie::dtrace #endif diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp index cfab9211..b59c6226 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp @@ -6,6 +6,7 @@ #include "xdp/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h" #include "xdp/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.h" #include "xdp/profile/plugin/aie_dtrace/ve2/elf_helper.h" +#include "xdp/profile/plugin/aie_dtrace/util/aie_dtrace_util.h" #include "core/common/api/hw_context_int.h" #include "core/common/api/kernel_int.h" @@ -134,6 +135,8 @@ namespace xdp { if (!ctWriter.generateBandwidthCT(outputPath, hwctx, it->second, bandwidthMetricSet, bandwidthChannel)) return; + aie::dtrace::initDtraceOutputConfig(); + xrt_core::message::send(severity_level::debug, "XRT", "AIE dtrace: Bandwidth CT generated for kernel '" + kernel_name + "' with metric set '" + bandwidthMetricSet + "'"); From 3908f2d9f747d24e2b713764a314b2d9431293de Mon Sep 17 00:00:00 2001 From: Garima Dhaked Date: Mon, 3 Aug 2026 07:00:46 -0600 Subject: [PATCH 2/6] Inject counter metadata into JSON dtrace_dump from aie_dtrace plugin The dtrace engine drops the CT COUNTER_METADATA comment block from the JSON output, so the JSON dump lacked the per-counter metadata that the python dump carries. Rather than modifying the aiebu dtrace engine, keep the change in the xdp plugin: capture the metadata during CT generation and inject it into the JSON dump on hw context teardown (finish_flush_device -> endPollforDevice -> finalizeDtraceDump), which runs after core writes dtrace_dump_ctx__*.json. Also make the metadata a single source of truth so the python (CT comment) and JSON metadata are identical for every bandwidth metric set, and keep each counter as a distinct JSON key via self-describing read_reg variable names. Co-authored-by: Cursor --- profile/plugin/aie_dtrace/aie_dtrace_impl.h | 4 + .../plugin/aie_dtrace/aie_dtrace_plugin.cpp | 6 + .../aie_dtrace/ve2/aie_dtrace_ct_writer.cpp | 174 ++++++++++++------ .../aie_dtrace/ve2/aie_dtrace_ct_writer.h | 16 +- .../plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp | 110 ++++++++++- .../plugin/aie_dtrace/ve2/aie_dtrace_ve2.h | 10 + 6 files changed, 264 insertions(+), 56 deletions(-) diff --git a/profile/plugin/aie_dtrace/aie_dtrace_impl.h b/profile/plugin/aie_dtrace/aie_dtrace_impl.h index 7c505be7..4bc6b476 100644 --- a/profile/plugin/aie_dtrace/aie_dtrace_impl.h +++ b/profile/plugin/aie_dtrace/aie_dtrace_impl.h @@ -47,6 +47,10 @@ namespace xdp { const std::string& /*kernel_name*/, void* /*elf_handle*/) {} + // Post-process the dumped dtrace JSON (e.g. inject counter metadata) after the + // hw context has written it. Called on hw context teardown, after the dump. + virtual void finalizeDtraceDump() {} + uint64_t getDeviceID() { return deviceID; } }; diff --git a/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp b/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp index 3c5f1abc..e1ea1e8a 100644 --- a/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp +++ b/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp @@ -182,6 +182,12 @@ namespace xdp { if (itr == handleToAIEDtraceImpl.end()) return; + // Post-process the JSON dtrace dump for this context. This runs from + // finish_flush_device on hw context teardown, i.e. after core has written the + // dtrace_dump_ctx__*.json file, so the dump exists and can be augmented + // with the counter metadata captured during CT generation. + itr->second->finalizeDtraceDump(); + // Drop implementation without endPoll(): dtrace must not read/offload on hwctx teardown; // ~AieDtrace_VE2Impl releases FAL resources only. handleToAIEDtraceImpl.erase(itr); diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp index a5f313fa..ef1d5f3d 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp @@ -12,6 +12,7 @@ #include "core/common/message.h" #include +#include #include #include #include @@ -87,6 +88,41 @@ extendLastUcToMaxConfiguredColumn(std::vector& asmFileInfoList, last.colEnd = std::max(last.colEnd, maxCfgCol); } +// Build a unique, self-describing dtrace variable name for a counter read. +// +// The JSON dtrace dump keys every read action by its left-hand-side variable +// name (json[probe][lhs] = value). Using the throwaway "_" for all reads makes +// the 4-16 per-probe counters collapse into a single "_" key in the JSON, and +// the "# COUNTER_METADATA" block is a comment that the JSON serializer drops. +// Encoding col/row/counter/channel/direction/event into the LHS name keeps every +// counter as a distinct JSON key and embeds the per-counter metadata directly in +// that key (e.g. "c0_r0_n1_ch0_i_lock"). +std::string +makeCounterVarName(const CTCounterInfo& ctr) +{ + const std::string dir = (ctr.portDirection == "input") ? "i" + : (ctr.portDirection == "output") ? "o" + : "x"; + + std::stringstream ss; + ss << "c" << static_cast(ctr.column) + << "_r" << static_cast(ctr.row) + << "_n" << static_cast(ctr.counterNumber) + << "_ch" << static_cast(ctr.channel) + << "_" << dir; + if (!ctr.eventType.empty()) + ss << "_" << ctr.eventType; + + // The key doubles as a python identifier in the non-JSON dump, so replace any + // character that is not [A-Za-z0-9_] with '_'. + std::string name = ss.str(); + for (char& c : name) { + if (!(std::isalnum(static_cast(c)) || c == '_')) + c = '_'; + } + return name; +} + } // namespace using severity_level = xrt_core::message::severity_level; @@ -636,10 +672,13 @@ bool AieDtraceCTWriter::writeCTFile(const std::vector& asmFileInfoL ctFile << "{\n"; ctFile << " ts_" << asmFileInfo.asmId << " = timestamp32()\n"; - // Write counter reads using _ as throwaway variable + // Write counter reads using a unique, self-describing variable name per + // counter so the JSON dtrace dump keeps every counter as a distinct key + // (see makeCounterVarName). for (size_t i = 0; i < asmFileInfo.counters.size(); i++) { - ctFile << " _ = read_reg(" - << formatAddress(asmFileInfo.counters[i].address) << ")\n"; + const auto& ctr = asmFileInfo.counters[i]; + ctFile << " " << makeCounterVarName(ctr) + << " = read_reg(" << formatAddress(ctr.address) << ")\n"; } ctFile << "}\n\n"; @@ -1077,58 +1116,17 @@ bool AieDtraceCTWriter::writeBandwidthCTFile( ctFile << "@blockopen\n"; ctFile << "# COUNTER_METADATA_BEGIN\n"; - ctFile << "# {\n"; - - // Per-UC counter metadata groupings only - std::vector metaGroups; - for (const auto& asmFileInfo : asmFileInfoList) { - if (!asmFileInfo.counters.empty()) - metaGroups.push_back(&asmFileInfo); - } - - for (size_t g = 0; g < metaGroups.size(); g++) { - const auto& asmFileInfo = *metaGroups[g]; - ctFile << "# \"" << asmFileInfo.asmId << "\": [\n"; - - for (size_t c = 0; c < asmFileInfo.counters.size(); c++) { - const auto& ctr = asmFileInfo.counters[c]; - ctFile << "# {\"col\": " << static_cast(ctr.column) - << ", \"row\": " << static_cast(ctr.row) - << ", \"ctr\": " << static_cast(ctr.counterNumber) - << ", \"ch\": " << static_cast(ctr.channel) - << ", \"dir\": "; - - if (ctr.portDirection == "input") - ctFile << "\"i\""; - else if (ctr.portDirection == "output") - ctFile << "\"o\""; - else - ctFile << "null"; - - // Add event type for peak bandwidth metrics - if (!ctr.eventType.empty()) { - ctFile << ", \"event\": "; - if (ctr.eventType == "running") - ctFile << "\"r\""; - else if (ctr.eventType == "stalled") - ctFile << "\"s\""; - else - ctFile << "\"" << ctr.eventType << "\""; - } - - ctFile << "}"; - if (c < asmFileInfo.counters.size() - 1) - ctFile << ","; - ctFile << "\n"; - } - ctFile << "# ]"; - if (g < metaGroups.size() - 1) - ctFile << ","; - ctFile << "\n"; + // Render the metadata comment from the same builder used for the JSON dump, so the + // python (comment) and JSON metadata are identical for every metric set. + { + const std::string metadata = buildBandwidthMetadataJson(asmFileInfoList); + std::istringstream metaStream(metadata); + std::string line; + while (std::getline(metaStream, line)) + ctFile << "# " << line << "\n"; } - ctFile << "# }\n"; ctFile << "# COUNTER_METADATA_END\n"; ctFile << "@blockclose\n"; ctFile << "}\n\n"; @@ -1158,7 +1156,8 @@ bool AieDtraceCTWriter::writeBandwidthCTFile( for (size_t i = 0; i < asmFileInfo.counters.size(); i++) { const auto& ctr = asmFileInfo.counters[i]; - ctFile << " _ = read_reg(" << formatAddress(ctr.address) << ")\n"; + ctFile << " " << makeCounterVarName(ctr) + << " = read_reg(" << formatAddress(ctr.address) << ")\n"; } ctFile << "}\n\n"; @@ -1179,12 +1178,74 @@ bool AieDtraceCTWriter::writeBandwidthCTFile( return true; } +std::string AieDtraceCTWriter::buildBandwidthMetadataJson( + const std::vector& asmFileInfoList) +{ + // Single source of truth for the bandwidth counter metadata, shared by every + // metric set. The returned string is valid JSON (per-UC groups keyed by ASM id) + // and is used two ways: + // 1. rendered verbatim (with a "# " prefix) as the CT begin-block + // COUNTER_METADATA comment, which is what the python dtrace dump emits, and + // 2. parsed and injected into the JSON dtrace dump (the engine drops CT + // comments from JSON output). + // Because both consumers format from this one function, the JSON metadata is + // guaranteed to match the python metadata for every metric set. + std::vector metaGroups; + for (const auto& asmFileInfo : asmFileInfoList) { + if (!asmFileInfo.counters.empty()) + metaGroups.push_back(&asmFileInfo); + } + + std::stringstream ss; + ss << "{\n"; + for (size_t g = 0; g < metaGroups.size(); g++) { + const auto& asmFileInfo = *metaGroups[g]; + ss << " \"" << asmFileInfo.asmId << "\": [\n"; + for (size_t c = 0; c < asmFileInfo.counters.size(); c++) { + const auto& ctr = asmFileInfo.counters[c]; + ss << " {\"col\": " << static_cast(ctr.column) + << ", \"row\": " << static_cast(ctr.row) + << ", \"ctr\": " << static_cast(ctr.counterNumber) + << ", \"ch\": " << static_cast(ctr.channel) + << ", \"dir\": "; + if (ctr.portDirection == "input") + ss << "\"i\""; + else if (ctr.portDirection == "output") + ss << "\"o\""; + else + ss << "null"; + + if (!ctr.eventType.empty()) { + ss << ", \"event\": "; + if (ctr.eventType == "running") + ss << "\"r\""; + else if (ctr.eventType == "stalled") + ss << "\"s\""; + else + ss << "\"" << ctr.eventType << "\""; + } + + ss << "}"; + if (c + 1 < asmFileInfo.counters.size()) + ss << ","; + ss << "\n"; + } + ss << " ]"; + if (g + 1 < metaGroups.size()) + ss << ","; + ss << "\n"; + } + ss << "}"; + return ss.str(); +} + bool AieDtraceCTWriter::generateBandwidthCT( const std::string& outputPath, void* hwctx, const std::vector& opLocations, const std::string& metricSet, - uint8_t channel) + uint8_t channel, + std::string* outMetadataJson) { if (opLocations.empty()) { xrt_core::message::send(severity_level::debug, "XRT", @@ -1258,6 +1319,11 @@ bool AieDtraceCTWriter::generateBandwidthCT( asmFileInfo.counters = filterCountersByColumn(allCounters, asmFileInfo.colStart, asmFileInfo.colEnd); } + // Expose the counter metadata so the plugin can inject it into the JSON dtrace + // dump (the dtrace engine drops the CT COUNTER_METADATA comment from JSON output). + if (outMetadataJson) + *outMetadataJson = buildBandwidthMetadataJson(asmFileInfoList); + std::vector beginBlockWrites; for (uint8_t column : shimColumns) { // For detailed sets, counter 0 monitors PORT_RUNNING on the channel's diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.h b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.h index d353b165..45c68b6c 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.h +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.h @@ -164,7 +164,8 @@ class AieDtraceCTWriter { void* hwctx, const std::vector& opLocations, const std::string& metricSet = "ddr_bandwidth", - uint8_t channel = 0); + uint8_t channel = 0, + std::string* outMetadataJson = nullptr); private: /** @@ -302,6 +303,19 @@ class AieDtraceCTWriter { const std::vector& beginBlockWrites, const std::string& outputPath); + /** + * @brief Build the per-UC counter metadata as a compact JSON string + * + * Produces the same structure embedded in the CT begin-block + * COUNTER_METADATA comment (keyed by ASM id -> array of counter descriptors). + * Used to inject the metadata into the JSON dtrace dump during post-processing, + * since the dtrace engine drops CT comments from the JSON output. + * + * @param asmFileInfoList ASM file infos with filtered per-UC counters + * @return JSON object string, e.g. {"0":[{"col":0,"row":0,"ctr":0,...}]} + */ + std::string buildBandwidthMetadataJson(const std::vector& asmFileInfoList); + private: VPDatabase* db; std::shared_ptr metadata; diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp index b59c6226..8ca0f727 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp @@ -16,8 +16,12 @@ #include "xdp/profile/database/static_info/aie_util.h" +#include "core/common/json/nlohmann/json.hpp" + #include #include +#include +#include #include namespace xdp { @@ -132,9 +136,17 @@ namespace xdp { "AIE dtrace: No interface tile metrics configured, using default 'peak_read_bandwidth'"); } - if (!ctWriter.generateBandwidthCT(outputPath, hwctx, it->second, bandwidthMetricSet, bandwidthChannel)) + std::string metadataJson; + if (!ctWriter.generateBandwidthCT(outputPath, hwctx, it->second, bandwidthMetricSet, + bandwidthChannel, &metadataJson)) return; + // Remember slot + per-run counter metadata so it can be injected into the JSON + // dtrace dump on hw context teardown (see finalizeDtraceDump()). + m_dumpSlotIdx = static_cast(slotIdx); + if (!metadataJson.empty()) + m_runMetadataJson[run_uid] = metadataJson; + aie::dtrace::initDtraceOutputConfig(); xrt_core::message::send(severity_level::debug, "XRT", @@ -156,4 +168,100 @@ namespace xdp { } } + void AieDtrace_VE2Impl::finalizeDtraceDump() + { + // The dtrace engine drops the CT COUNTER_METADATA comment from the JSON dump, + // so inject the metadata captured at CT generation into the JSON file that core + // has just written for this hw context (before it is erased on teardown). + if (m_dumpSlotIdx < 0 || m_runMetadataJson.empty()) + return; + + // The JSON dump file only exists in coalesced JSON output mode. + if (!(xrt_core::config::get_dtrace_output_json_format() + && xrt_core::config::get_dtrace_coalesce_result())) + return; + + try { + // Locate the newest dump file for this context slot in the working directory. + // core writes: dtrace_dump_ctx__.json + const std::regex fileRe("^dtrace_dump_ctx_" + std::to_string(m_dumpSlotIdx) + + "_.*\\.json$"); + std::filesystem::path dumpFile; + std::filesystem::file_time_type newest{}; + for (const auto& entry : std::filesystem::directory_iterator(std::filesystem::current_path())) { + if (!entry.is_regular_file()) + continue; + if (!std::regex_match(entry.path().filename().string(), fileRe)) + continue; + auto mtime = entry.last_write_time(); + if (dumpFile.empty() || mtime > newest) { + dumpFile = entry.path(); + newest = mtime; + } + } + + if (dumpFile.empty()) { + xrt_core::message::send(severity_level::debug, "XRT", + "AIE dtrace: no JSON dump file found for slot " + + std::to_string(m_dumpSlotIdx) + "; skipping metadata injection."); + return; + } + + nlohmann::ordered_json root; + { + std::ifstream in(dumpFile); + if (!in) + return; + in >> root; + } + + if (!root.is_object()) + return; + + // A fallback metadata (same metric set across runs of a context): used when a + // run key's uid is not found in the captured map. + const std::string& fallbackMeta = m_runMetadataJson.begin()->second; + const std::regex runRe("_run_(\\d+)_"); + + for (auto& item : root.items()) { + auto& runObj = item.value(); + if (!runObj.is_object()) + continue; + + const std::string* metaStr = &fallbackMeta; + std::smatch m; + const std::string key = item.key(); + if (std::regex_search(key, m, runRe)) { + auto uid = static_cast(std::stoul(m[1].str())); + auto it = m_runMetadataJson.find(uid); + if (it != m_runMetadataJson.end()) + metaStr = &it->second; + } + + nlohmann::ordered_json meta = nlohmann::ordered_json::parse(*metaStr, nullptr, false); + if (meta.is_discarded()) + continue; + + // Mirror the python dump: metadata lives in the begin block. + if (!runObj.contains("begin") || !runObj["begin"].is_object()) + runObj["begin"] = nlohmann::ordered_json::object(); + runObj["begin"]["counter_metadata"] = std::move(meta); + } + + { + std::ofstream out(dumpFile, std::ios::trunc); + if (!out) + return; + out << root.dump(4) << "\n"; + } + + xrt_core::message::send(severity_level::debug, "XRT", + "AIE dtrace: injected counter metadata into " + dumpFile.string()); + } + catch (const std::exception& e) { + xrt_core::message::send(severity_level::debug, "XRT", + std::string{"AIE dtrace: metadata injection failed (ignored): "} + e.what()); + } + } + } diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h index 3fa6926a..3ac52b91 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h @@ -33,10 +33,20 @@ namespace xdp { const std::string& kernel_name, void* elf_handle) override; + void finalizeDtraceDump() override; + private: void computeOpLocations(void* elf_handle, const std::string& kernel_name); std::map> m_op_locations_cache; + + // Slot index of the hw context this impl serves (for locating the dump file); + // -1 until a CT has been generated for a run. + int m_dumpSlotIdx = -1; + + // Per-run counter metadata (run_uid -> JSON object string) captured at CT + // generation, injected into the JSON dtrace dump on teardown. + std::map m_runMetadataJson; }; } From b987bba283a53b4a99f982848fe9df941ae21606 Mon Sep 17 00:00:00 2001 From: Garima Dhaked Date: Tue, 4 Aug 2026 05:28:47 -0600 Subject: [PATCH 3/6] Consolidate and compact aie_dtrace JSON dump metadata and keys Move the JSON dtrace_dump counter-metadata injection out of the VE2 impl into aie_dtrace/util (invoked via the plugin callback) and rework the emitted format so post-processing stays cheap: - Emit a single shared "counters" array (deduped by counter number, per-tile "col" dropped) plus a compact per-microcontroller "uc" map (col_start/ col_end) instead of duplicating per-tile counter blocks. - Write counter_metadata once at the top level of the coalesced dump rather than into every inference's begin block (all inferences share the config). - Shorten counter variable names to c_n and shorten probe keys to uc:, reducing CT file size and dtrace runtime/output overhead. buildBandwidthMetadataJson remains the single source of truth for both the python CT comment and the injected JSON metadata. Co-authored-by: Cursor --- profile/plugin/aie_dtrace/aie_dtrace_cb.cpp | 9 + profile/plugin/aie_dtrace/aie_dtrace_impl.h | 4 - .../plugin/aie_dtrace/aie_dtrace_plugin.cpp | 6 - .../aie_dtrace/util/aie_dtrace_util.cpp | 162 ++++++++++++++++++ .../plugin/aie_dtrace/util/aie_dtrace_util.h | 13 ++ .../aie_dtrace/ve2/aie_dtrace_ct_writer.cpp | 141 +++++++++------ .../plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp | 111 +----------- .../plugin/aie_dtrace/ve2/aie_dtrace_ve2.h | 10 -- 8 files changed, 275 insertions(+), 181 deletions(-) diff --git a/profile/plugin/aie_dtrace/aie_dtrace_cb.cpp b/profile/plugin/aie_dtrace/aie_dtrace_cb.cpp index 127e8b05..c5fa25bd 100644 --- a/profile/plugin/aie_dtrace/aie_dtrace_cb.cpp +++ b/profile/plugin/aie_dtrace/aie_dtrace_cb.cpp @@ -5,6 +5,7 @@ #include "aie_dtrace_cb.h" #include "aie_dtrace_plugin.h" +#include "xdp/profile/plugin/aie_dtrace/util/aie_dtrace_util.h" namespace xdp { @@ -18,6 +19,14 @@ namespace xdp { static void endAIEDtracePoll(void* handle) { + // Inject the captured counter metadata into the coalesced JSON dtrace dump that + // core writes on hw context teardown. This is deliberately done unconditionally + // (not gated by AieDtracePlugin::alive()): the plugin's static instance may be + // destroyed before the hw context writes the dump, so relying on the plugin being + // alive here would silently skip the injection. The metadata lives in a + // process-lifetime registry, so this is safe even after plugin teardown. + aie::dtrace::injectPendingMetadata(); + if (AieDtracePlugin::alive()) aieDtracePluginInstance.endPollforDevice(handle); } diff --git a/profile/plugin/aie_dtrace/aie_dtrace_impl.h b/profile/plugin/aie_dtrace/aie_dtrace_impl.h index 4bc6b476..7c505be7 100644 --- a/profile/plugin/aie_dtrace/aie_dtrace_impl.h +++ b/profile/plugin/aie_dtrace/aie_dtrace_impl.h @@ -47,10 +47,6 @@ namespace xdp { const std::string& /*kernel_name*/, void* /*elf_handle*/) {} - // Post-process the dumped dtrace JSON (e.g. inject counter metadata) after the - // hw context has written it. Called on hw context teardown, after the dump. - virtual void finalizeDtraceDump() {} - uint64_t getDeviceID() { return deviceID; } }; diff --git a/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp b/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp index e1ea1e8a..3c5f1abc 100644 --- a/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp +++ b/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp @@ -182,12 +182,6 @@ namespace xdp { if (itr == handleToAIEDtraceImpl.end()) return; - // Post-process the JSON dtrace dump for this context. This runs from - // finish_flush_device on hw context teardown, i.e. after core has written the - // dtrace_dump_ctx__*.json file, so the dump exists and can be augmented - // with the counter metadata captured during CT generation. - itr->second->finalizeDtraceDump(); - // Drop implementation without endPoll(): dtrace must not read/offload on hwctx teardown; // ~AieDtrace_VE2Impl releases FAL resources only. handleToAIEDtraceImpl.erase(itr); diff --git a/profile/plugin/aie_dtrace/util/aie_dtrace_util.cpp b/profile/plugin/aie_dtrace/util/aie_dtrace_util.cpp index 89749265..ec15ce63 100644 --- a/profile/plugin/aie_dtrace/util/aie_dtrace_util.cpp +++ b/profile/plugin/aie_dtrace/util/aie_dtrace_util.cpp @@ -7,8 +7,14 @@ #include "core/common/config_reader.h" #include "core/common/message.h" +#include "core/common/json/nlohmann/json.hpp" +#include +#include +#include #include +#include +#include namespace xdp::aie::dtrace { @@ -16,6 +22,101 @@ namespace xdp::aie::dtrace { using severity_level = xrt_core::message::severity_level; static constexpr unsigned int DEFAULT_COALESCE_RESULT_MEMORY_MB = 256; + + // Process-lifetime registry for counter metadata. Kept as a leaky singleton + // (never destroyed) so it remains valid during static destruction, when the + // hw context may write the coalesced JSON dump after the XDP plugin instance + // has already been torn down. + struct MetadataRegistry { + std::mutex mtx; + std::map> bySlot; // slot -> uid -> json + std::set injectedFiles; + }; + + MetadataRegistry& + registry() + { + static MetadataRegistry* reg = new MetadataRegistry(); + return *reg; + } + + // Compact the verbose dtrace probe key. The engine emits the full probe + // specifier verbatim as the JSON key, e.g. + // "jprobe:aie_runtime_control.asm:uc0:line8" + // The "jprobe:" prefix and asm filename are constant boilerplate repeated in + // every probe block of every inference. Only the microcontroller and the ASM + // line (the op/layer correlation) are needed, so shorten to "uc0:8". + std::string + shortenProbeKey(const std::string& key) + { + static const std::regex probeRe(R"(^jprobe:.*:uc(\d+):line(\d+)$)"); + std::smatch m; + if (std::regex_match(key, m, probeRe)) + return "uc" + m[1].str() + ":" + m[2].str(); + return key; + } + + // Inject metadata into a single coalesced dump file. uidMap: run uid -> JSON string. + bool + injectFile(const std::filesystem::path& file, + const std::map& uidMap) + { + try { + nlohmann::ordered_json root; + { + std::ifstream in(file); + if (!in) + return false; + in >> root; + } + if (!root.is_object() || uidMap.empty()) + return false; + + // All inferences in a coalesced file share the same counter configuration, + // so the metadata is emitted once as a top-level "counter_metadata" entry + // (placed before the inferences) instead of being duplicated in every + // inference's begin block. Any captured metadata works since they are + // identical. + auto sharedMeta = + nlohmann::ordered_json::parse(uidMap.begin()->second, nullptr, false); + + // Rebuild the whole document: metadata first, then each inference with its + // probe keys shortened (order preserved). + nlohmann::ordered_json newRoot = nlohmann::ordered_json::object(); + if (!sharedMeta.is_discarded()) + newRoot["counter_metadata"] = std::move(sharedMeta); + + for (auto& item : root.items()) { + auto& runObj = item.value(); + if (!runObj.is_object()) { + newRoot[item.key()] = std::move(runObj); + continue; + } + + nlohmann::ordered_json newRun = nlohmann::ordered_json::object(); + for (auto& kv : runObj.items()) + newRun[shortenProbeKey(kv.key())] = std::move(kv.value()); + + newRoot[item.key()] = std::move(newRun); + } + + { + std::ofstream out(file, std::ios::trunc); + if (!out) + return false; + out << newRoot.dump(4) << "\n"; + } + + xrt_core::message::send(severity_level::debug, "XRT", + "AIE dtrace: injected counter metadata into " + file.string()); + return true; + } + catch (const std::exception& e) { + xrt_core::message::send(severity_level::debug, "XRT", + std::string{"AIE dtrace: metadata injection failed (ignored): "} + e.what()); + return false; + } + } } // anonymous namespace void @@ -42,6 +143,67 @@ namespace xdp::aie::dtrace { }); } + void + registerCounterMetadata(uint32_t slotIdx, uint32_t runUid, const std::string& metadataJson) + { + if (metadataJson.empty()) + return; + auto& reg = registry(); + std::lock_guard lk(reg.mtx); + reg.bySlot[slotIdx][runUid] = metadataJson; + } + + void + injectPendingMetadata() + { + // The coalesced JSON dump only exists when both JSON output and coalescing are on. + if (!(xrt_core::config::get_dtrace_output_json_format() + && xrt_core::config::get_dtrace_coalesce_result())) + return; + + auto& reg = registry(); + std::lock_guard lk(reg.mtx); + if (reg.bySlot.empty()) + return; + + std::error_code ec; + const auto cwd = std::filesystem::current_path(ec); + if (ec) + return; + + for (const auto& [slot, uidMap] : reg.bySlot) { + if (uidMap.empty()) + continue; + + // core writes dtrace_dump_ctx__.json; inject the newest one + // for this slot that has not been processed yet. + const std::regex fileRe("^dtrace_dump_ctx_" + std::to_string(slot) + "_.*\\.json$"); + std::filesystem::path newestPath; + std::filesystem::file_time_type newest{}; + for (const auto& entry : std::filesystem::directory_iterator(cwd, ec)) { + if (ec) + break; + if (!entry.is_regular_file()) + continue; + if (!std::regex_match(entry.path().filename().string(), fileRe)) + continue; + auto mtime = entry.last_write_time(ec); + if (newestPath.empty() || mtime > newest) { + newestPath = entry.path(); + newest = mtime; + } + } + + if (newestPath.empty()) + continue; + if (reg.injectedFiles.count(newestPath.string())) + continue; + + if (injectFile(newestPath, uidMap)) + reg.injectedFiles.insert(newestPath.string()); + } + } + std::map> getBandwidthInterfaceTileEventSets(int hwGen) { diff --git a/profile/plugin/aie_dtrace/util/aie_dtrace_util.h b/profile/plugin/aie_dtrace/util/aie_dtrace_util.h index a8210658..f570697a 100644 --- a/profile/plugin/aie_dtrace/util/aie_dtrace_util.h +++ b/profile/plugin/aie_dtrace/util/aie_dtrace_util.h @@ -4,6 +4,7 @@ #ifndef AIE_DTRACE_UTIL_DOT_H #define AIE_DTRACE_UTIL_DOT_H +#include #include #include #include @@ -21,6 +22,18 @@ namespace xdp::aie::dtrace { // Must run before XRT creates the first dtrace module (config keys lock on first read). void initDtraceOutputConfig(); + // Register per-run bandwidth counter metadata (slot -> run uid -> JSON object string), + // captured during CT generation. Stored in a process-lifetime registry so it survives + // XDP plugin teardown; the plugin's static instance may be destroyed before the hw + // context writes the coalesced JSON dump (static destruction order is not guaranteed). + void registerCounterMetadata(uint32_t slotIdx, uint32_t runUid, const std::string& metadataJson); + + // Inject the registered counter metadata into the coalesced JSON dtrace dump file(s) + // (dtrace_dump_ctx__*.json) that core writes to the cwd on hw context teardown. + // Safe to call even after the XDP plugin instance has been destroyed; idempotent + // (each dump file is injected at most once). + void injectPendingMetadata(); + } // namespace xdp::aie::dtrace #endif diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp index ef1d5f3d..f0c605e0 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp @@ -100,27 +100,17 @@ extendLastUcToMaxConfiguredColumn(std::vector& asmFileInfoList, std::string makeCounterVarName(const CTCounterInfo& ctr) { - const std::string dir = (ctr.portDirection == "input") ? "i" - : (ctr.portDirection == "output") ? "o" - : "x"; - + // The key only needs to (a) be unique within a probe block so the counters do + // not collapse into a single JSON entry, and (b) let post-processing recover + // the counter's identity. Column + counter number satisfy both: everything + // else (row, channel, direction, event) is constant per counter number across + // tiles and is carried once in the common "counter_metadata" array (keyed by + // "ctr"). Keeping the key short reduces CT file size and dtrace runtime/output + // overhead. Example: "c3_n2" -> column 3, counter 2. std::stringstream ss; - ss << "c" << static_cast(ctr.column) - << "_r" << static_cast(ctr.row) - << "_n" << static_cast(ctr.counterNumber) - << "_ch" << static_cast(ctr.channel) - << "_" << dir; - if (!ctr.eventType.empty()) - ss << "_" << ctr.eventType; - - // The key doubles as a python identifier in the non-JSON dump, so replace any - // character that is not [A-Za-z0-9_] with '_'. - std::string name = ss.str(); - for (char& c : name) { - if (!(std::isalnum(static_cast(c)) || c == '_')) - c = '_'; - } - return name; + ss << "c" << static_cast(ctr.column) + << "_n" << static_cast(ctr.counterNumber); + return ss.str(); } } // namespace @@ -1182,59 +1172,98 @@ std::string AieDtraceCTWriter::buildBandwidthMetadataJson( const std::vector& asmFileInfoList) { // Single source of truth for the bandwidth counter metadata, shared by every - // metric set. The returned string is valid JSON (per-UC groups keyed by ASM id) - // and is used two ways: + // metric set. The returned string is valid JSON and is used two ways: // 1. rendered verbatim (with a "# " prefix) as the CT begin-block // COUNTER_METADATA comment, which is what the python dtrace dump emits, and // 2. parsed and injected into the JSON dtrace dump (the engine drops CT // comments from JSON output). // Because both consumers format from this one function, the JSON metadata is // guaranteed to match the python metadata for every metric set. - std::vector metaGroups; + // + // The per-tile counter layout is identical for every shim tile/UC: the same + // counter configuration is replicated on each column, so only the column + // number differs between tiles and that is already encoded in each counter's + // JSON key (c_n). We therefore emit the counter descriptors once as + // a shared "counters" array (deduped by counter number, "col" dropped) instead + // of repeating the same block per UC. + // + // The per-microcontroller (UC) information that IS distinct - namely which + // columns each microcontroller owns - is emitted separately as a compact "uc" + // map keyed by UC number, without re-duplicating the counter descriptors: + // {"counters": [ ... ], "uc": {"0": {"col_start":0,"col_end":3}, ...}} + std::vector uniqueCounters; + std::vector seenCounterNumbers; + for (const auto& asmFileInfo : asmFileInfoList) { + for (const auto& ctr : asmFileInfo.counters) { + if (std::find(seenCounterNumbers.begin(), seenCounterNumbers.end(), + ctr.counterNumber) != seenCounterNumbers.end()) + continue; + seenCounterNumbers.push_back(ctr.counterNumber); + uniqueCounters.push_back(&ctr); + } + } + + std::sort(uniqueCounters.begin(), uniqueCounters.end(), + [](const CTCounterInfo* a, const CTCounterInfo* b) { + return a->counterNumber < b->counterNumber; + }); + + // Per-UC column ownership, ordered by UC number (skip UCs with no counters). + std::vector ucGroups; for (const auto& asmFileInfo : asmFileInfoList) { if (!asmFileInfo.counters.empty()) - metaGroups.push_back(&asmFileInfo); + ucGroups.push_back(&asmFileInfo); } + std::sort(ucGroups.begin(), ucGroups.end(), + [](const ASMFileInfo* a, const ASMFileInfo* b) { + return a->ucNumber < b->ucNumber; + }); std::stringstream ss; ss << "{\n"; - for (size_t g = 0; g < metaGroups.size(); g++) { - const auto& asmFileInfo = *metaGroups[g]; - ss << " \"" << asmFileInfo.asmId << "\": [\n"; - for (size_t c = 0; c < asmFileInfo.counters.size(); c++) { - const auto& ctr = asmFileInfo.counters[c]; - ss << " {\"col\": " << static_cast(ctr.column) - << ", \"row\": " << static_cast(ctr.row) - << ", \"ctr\": " << static_cast(ctr.counterNumber) - << ", \"ch\": " << static_cast(ctr.channel) - << ", \"dir\": "; - if (ctr.portDirection == "input") - ss << "\"i\""; - else if (ctr.portDirection == "output") - ss << "\"o\""; - else - ss << "null"; - - if (!ctr.eventType.empty()) { - ss << ", \"event\": "; - if (ctr.eventType == "running") - ss << "\"r\""; - else if (ctr.eventType == "stalled") - ss << "\"s\""; - else - ss << "\"" << ctr.eventType << "\""; - } - ss << "}"; - if (c + 1 < asmFileInfo.counters.size()) - ss << ","; - ss << "\n"; + ss << " \"counters\": [\n"; + for (size_t c = 0; c < uniqueCounters.size(); c++) { + const auto& ctr = *uniqueCounters[c]; + ss << " {\"row\": " << static_cast(ctr.row) + << ", \"ctr\": " << static_cast(ctr.counterNumber) + << ", \"ch\": " << static_cast(ctr.channel) + << ", \"dir\": "; + if (ctr.portDirection == "input") + ss << "\"i\""; + else if (ctr.portDirection == "output") + ss << "\"o\""; + else + ss << "null"; + + if (!ctr.eventType.empty()) { + ss << ", \"event\": "; + if (ctr.eventType == "running") + ss << "\"r\""; + else if (ctr.eventType == "stalled") + ss << "\"s\""; + else + ss << "\"" << ctr.eventType << "\""; } - ss << " ]"; - if (g + 1 < metaGroups.size()) + + ss << "}"; + if (c + 1 < uniqueCounters.size()) ss << ","; ss << "\n"; } + ss << " ],\n"; + + ss << " \"uc\": {\n"; + for (size_t u = 0; u < ucGroups.size(); u++) { + const auto& uc = *ucGroups[u]; + ss << " \"" << uc.ucNumber << "\": {\"col_start\": " << uc.colStart + << ", \"col_end\": " << uc.colEnd << "}"; + if (u + 1 < ucGroups.size()) + ss << ","; + ss << "\n"; + } + ss << " }\n"; + ss << "}"; return ss.str(); } diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp index 8ca0f727..489347f6 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp @@ -16,12 +16,8 @@ #include "xdp/profile/database/static_info/aie_util.h" -#include "core/common/json/nlohmann/json.hpp" - #include #include -#include -#include #include namespace xdp { @@ -141,11 +137,12 @@ namespace xdp { bandwidthChannel, &metadataJson)) return; - // Remember slot + per-run counter metadata so it can be injected into the JSON - // dtrace dump on hw context teardown (see finalizeDtraceDump()). - m_dumpSlotIdx = static_cast(slotIdx); - if (!metadataJson.empty()) - m_runMetadataJson[run_uid] = metadataJson; + // Register the per-run counter metadata in the process-lifetime registry so it can + // be injected into the JSON dtrace dump on hw context teardown. This must not live + // in the plugin/impl instance: the plugin's static instance can be destroyed before + // the hw context writes the coalesced dump, so the injection is driven from + // endAIEDtracePoll via aie::dtrace::injectPendingMetadata() instead. + aie::dtrace::registerCounterMetadata(static_cast(slotIdx), run_uid, metadataJson); aie::dtrace::initDtraceOutputConfig(); @@ -168,100 +165,4 @@ namespace xdp { } } - void AieDtrace_VE2Impl::finalizeDtraceDump() - { - // The dtrace engine drops the CT COUNTER_METADATA comment from the JSON dump, - // so inject the metadata captured at CT generation into the JSON file that core - // has just written for this hw context (before it is erased on teardown). - if (m_dumpSlotIdx < 0 || m_runMetadataJson.empty()) - return; - - // The JSON dump file only exists in coalesced JSON output mode. - if (!(xrt_core::config::get_dtrace_output_json_format() - && xrt_core::config::get_dtrace_coalesce_result())) - return; - - try { - // Locate the newest dump file for this context slot in the working directory. - // core writes: dtrace_dump_ctx__.json - const std::regex fileRe("^dtrace_dump_ctx_" + std::to_string(m_dumpSlotIdx) - + "_.*\\.json$"); - std::filesystem::path dumpFile; - std::filesystem::file_time_type newest{}; - for (const auto& entry : std::filesystem::directory_iterator(std::filesystem::current_path())) { - if (!entry.is_regular_file()) - continue; - if (!std::regex_match(entry.path().filename().string(), fileRe)) - continue; - auto mtime = entry.last_write_time(); - if (dumpFile.empty() || mtime > newest) { - dumpFile = entry.path(); - newest = mtime; - } - } - - if (dumpFile.empty()) { - xrt_core::message::send(severity_level::debug, "XRT", - "AIE dtrace: no JSON dump file found for slot " - + std::to_string(m_dumpSlotIdx) + "; skipping metadata injection."); - return; - } - - nlohmann::ordered_json root; - { - std::ifstream in(dumpFile); - if (!in) - return; - in >> root; - } - - if (!root.is_object()) - return; - - // A fallback metadata (same metric set across runs of a context): used when a - // run key's uid is not found in the captured map. - const std::string& fallbackMeta = m_runMetadataJson.begin()->second; - const std::regex runRe("_run_(\\d+)_"); - - for (auto& item : root.items()) { - auto& runObj = item.value(); - if (!runObj.is_object()) - continue; - - const std::string* metaStr = &fallbackMeta; - std::smatch m; - const std::string key = item.key(); - if (std::regex_search(key, m, runRe)) { - auto uid = static_cast(std::stoul(m[1].str())); - auto it = m_runMetadataJson.find(uid); - if (it != m_runMetadataJson.end()) - metaStr = &it->second; - } - - nlohmann::ordered_json meta = nlohmann::ordered_json::parse(*metaStr, nullptr, false); - if (meta.is_discarded()) - continue; - - // Mirror the python dump: metadata lives in the begin block. - if (!runObj.contains("begin") || !runObj["begin"].is_object()) - runObj["begin"] = nlohmann::ordered_json::object(); - runObj["begin"]["counter_metadata"] = std::move(meta); - } - - { - std::ofstream out(dumpFile, std::ios::trunc); - if (!out) - return; - out << root.dump(4) << "\n"; - } - - xrt_core::message::send(severity_level::debug, "XRT", - "AIE dtrace: injected counter metadata into " + dumpFile.string()); - } - catch (const std::exception& e) { - xrt_core::message::send(severity_level::debug, "XRT", - std::string{"AIE dtrace: metadata injection failed (ignored): "} + e.what()); - } - } - } diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h index 3ac52b91..3fa6926a 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h @@ -33,20 +33,10 @@ namespace xdp { const std::string& kernel_name, void* elf_handle) override; - void finalizeDtraceDump() override; - private: void computeOpLocations(void* elf_handle, const std::string& kernel_name); std::map> m_op_locations_cache; - - // Slot index of the hw context this impl serves (for locating the dump file); - // -1 until a CT has been generated for a run. - int m_dumpSlotIdx = -1; - - // Per-run counter metadata (run_uid -> JSON object string) captured at CT - // generation, injected into the JSON dtrace dump on teardown. - std::map m_runMetadataJson; }; } From 42ce20f35f7597e1f789311b279c850b2a9d0445 Mon Sep 17 00:00:00 2001 From: Garima Dhaked Date: Thu, 6 Aug 2026 03:37:22 -0600 Subject: [PATCH 4/6] Revert "Consolidate and compact aie_dtrace JSON dump metadata and keys" This reverts commit b987bba283a53b4a99f982848fe9df941ae21606. --- profile/plugin/aie_dtrace/aie_dtrace_cb.cpp | 9 - profile/plugin/aie_dtrace/aie_dtrace_impl.h | 4 + .../plugin/aie_dtrace/aie_dtrace_plugin.cpp | 6 + .../aie_dtrace/util/aie_dtrace_util.cpp | 162 ------------------ .../plugin/aie_dtrace/util/aie_dtrace_util.h | 13 -- .../aie_dtrace/ve2/aie_dtrace_ct_writer.cpp | 141 ++++++--------- .../plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp | 111 +++++++++++- .../plugin/aie_dtrace/ve2/aie_dtrace_ve2.h | 10 ++ 8 files changed, 181 insertions(+), 275 deletions(-) diff --git a/profile/plugin/aie_dtrace/aie_dtrace_cb.cpp b/profile/plugin/aie_dtrace/aie_dtrace_cb.cpp index c5fa25bd..127e8b05 100644 --- a/profile/plugin/aie_dtrace/aie_dtrace_cb.cpp +++ b/profile/plugin/aie_dtrace/aie_dtrace_cb.cpp @@ -5,7 +5,6 @@ #include "aie_dtrace_cb.h" #include "aie_dtrace_plugin.h" -#include "xdp/profile/plugin/aie_dtrace/util/aie_dtrace_util.h" namespace xdp { @@ -19,14 +18,6 @@ namespace xdp { static void endAIEDtracePoll(void* handle) { - // Inject the captured counter metadata into the coalesced JSON dtrace dump that - // core writes on hw context teardown. This is deliberately done unconditionally - // (not gated by AieDtracePlugin::alive()): the plugin's static instance may be - // destroyed before the hw context writes the dump, so relying on the plugin being - // alive here would silently skip the injection. The metadata lives in a - // process-lifetime registry, so this is safe even after plugin teardown. - aie::dtrace::injectPendingMetadata(); - if (AieDtracePlugin::alive()) aieDtracePluginInstance.endPollforDevice(handle); } diff --git a/profile/plugin/aie_dtrace/aie_dtrace_impl.h b/profile/plugin/aie_dtrace/aie_dtrace_impl.h index 7c505be7..4bc6b476 100644 --- a/profile/plugin/aie_dtrace/aie_dtrace_impl.h +++ b/profile/plugin/aie_dtrace/aie_dtrace_impl.h @@ -47,6 +47,10 @@ namespace xdp { const std::string& /*kernel_name*/, void* /*elf_handle*/) {} + // Post-process the dumped dtrace JSON (e.g. inject counter metadata) after the + // hw context has written it. Called on hw context teardown, after the dump. + virtual void finalizeDtraceDump() {} + uint64_t getDeviceID() { return deviceID; } }; diff --git a/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp b/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp index 3c5f1abc..e1ea1e8a 100644 --- a/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp +++ b/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp @@ -182,6 +182,12 @@ namespace xdp { if (itr == handleToAIEDtraceImpl.end()) return; + // Post-process the JSON dtrace dump for this context. This runs from + // finish_flush_device on hw context teardown, i.e. after core has written the + // dtrace_dump_ctx__*.json file, so the dump exists and can be augmented + // with the counter metadata captured during CT generation. + itr->second->finalizeDtraceDump(); + // Drop implementation without endPoll(): dtrace must not read/offload on hwctx teardown; // ~AieDtrace_VE2Impl releases FAL resources only. handleToAIEDtraceImpl.erase(itr); diff --git a/profile/plugin/aie_dtrace/util/aie_dtrace_util.cpp b/profile/plugin/aie_dtrace/util/aie_dtrace_util.cpp index ec15ce63..89749265 100644 --- a/profile/plugin/aie_dtrace/util/aie_dtrace_util.cpp +++ b/profile/plugin/aie_dtrace/util/aie_dtrace_util.cpp @@ -7,14 +7,8 @@ #include "core/common/config_reader.h" #include "core/common/message.h" -#include "core/common/json/nlohmann/json.hpp" -#include -#include -#include #include -#include -#include namespace xdp::aie::dtrace { @@ -22,101 +16,6 @@ namespace xdp::aie::dtrace { using severity_level = xrt_core::message::severity_level; static constexpr unsigned int DEFAULT_COALESCE_RESULT_MEMORY_MB = 256; - - // Process-lifetime registry for counter metadata. Kept as a leaky singleton - // (never destroyed) so it remains valid during static destruction, when the - // hw context may write the coalesced JSON dump after the XDP plugin instance - // has already been torn down. - struct MetadataRegistry { - std::mutex mtx; - std::map> bySlot; // slot -> uid -> json - std::set injectedFiles; - }; - - MetadataRegistry& - registry() - { - static MetadataRegistry* reg = new MetadataRegistry(); - return *reg; - } - - // Compact the verbose dtrace probe key. The engine emits the full probe - // specifier verbatim as the JSON key, e.g. - // "jprobe:aie_runtime_control.asm:uc0:line8" - // The "jprobe:" prefix and asm filename are constant boilerplate repeated in - // every probe block of every inference. Only the microcontroller and the ASM - // line (the op/layer correlation) are needed, so shorten to "uc0:8". - std::string - shortenProbeKey(const std::string& key) - { - static const std::regex probeRe(R"(^jprobe:.*:uc(\d+):line(\d+)$)"); - std::smatch m; - if (std::regex_match(key, m, probeRe)) - return "uc" + m[1].str() + ":" + m[2].str(); - return key; - } - - // Inject metadata into a single coalesced dump file. uidMap: run uid -> JSON string. - bool - injectFile(const std::filesystem::path& file, - const std::map& uidMap) - { - try { - nlohmann::ordered_json root; - { - std::ifstream in(file); - if (!in) - return false; - in >> root; - } - if (!root.is_object() || uidMap.empty()) - return false; - - // All inferences in a coalesced file share the same counter configuration, - // so the metadata is emitted once as a top-level "counter_metadata" entry - // (placed before the inferences) instead of being duplicated in every - // inference's begin block. Any captured metadata works since they are - // identical. - auto sharedMeta = - nlohmann::ordered_json::parse(uidMap.begin()->second, nullptr, false); - - // Rebuild the whole document: metadata first, then each inference with its - // probe keys shortened (order preserved). - nlohmann::ordered_json newRoot = nlohmann::ordered_json::object(); - if (!sharedMeta.is_discarded()) - newRoot["counter_metadata"] = std::move(sharedMeta); - - for (auto& item : root.items()) { - auto& runObj = item.value(); - if (!runObj.is_object()) { - newRoot[item.key()] = std::move(runObj); - continue; - } - - nlohmann::ordered_json newRun = nlohmann::ordered_json::object(); - for (auto& kv : runObj.items()) - newRun[shortenProbeKey(kv.key())] = std::move(kv.value()); - - newRoot[item.key()] = std::move(newRun); - } - - { - std::ofstream out(file, std::ios::trunc); - if (!out) - return false; - out << newRoot.dump(4) << "\n"; - } - - xrt_core::message::send(severity_level::debug, "XRT", - "AIE dtrace: injected counter metadata into " + file.string()); - return true; - } - catch (const std::exception& e) { - xrt_core::message::send(severity_level::debug, "XRT", - std::string{"AIE dtrace: metadata injection failed (ignored): "} + e.what()); - return false; - } - } } // anonymous namespace void @@ -143,67 +42,6 @@ namespace xdp::aie::dtrace { }); } - void - registerCounterMetadata(uint32_t slotIdx, uint32_t runUid, const std::string& metadataJson) - { - if (metadataJson.empty()) - return; - auto& reg = registry(); - std::lock_guard lk(reg.mtx); - reg.bySlot[slotIdx][runUid] = metadataJson; - } - - void - injectPendingMetadata() - { - // The coalesced JSON dump only exists when both JSON output and coalescing are on. - if (!(xrt_core::config::get_dtrace_output_json_format() - && xrt_core::config::get_dtrace_coalesce_result())) - return; - - auto& reg = registry(); - std::lock_guard lk(reg.mtx); - if (reg.bySlot.empty()) - return; - - std::error_code ec; - const auto cwd = std::filesystem::current_path(ec); - if (ec) - return; - - for (const auto& [slot, uidMap] : reg.bySlot) { - if (uidMap.empty()) - continue; - - // core writes dtrace_dump_ctx__.json; inject the newest one - // for this slot that has not been processed yet. - const std::regex fileRe("^dtrace_dump_ctx_" + std::to_string(slot) + "_.*\\.json$"); - std::filesystem::path newestPath; - std::filesystem::file_time_type newest{}; - for (const auto& entry : std::filesystem::directory_iterator(cwd, ec)) { - if (ec) - break; - if (!entry.is_regular_file()) - continue; - if (!std::regex_match(entry.path().filename().string(), fileRe)) - continue; - auto mtime = entry.last_write_time(ec); - if (newestPath.empty() || mtime > newest) { - newestPath = entry.path(); - newest = mtime; - } - } - - if (newestPath.empty()) - continue; - if (reg.injectedFiles.count(newestPath.string())) - continue; - - if (injectFile(newestPath, uidMap)) - reg.injectedFiles.insert(newestPath.string()); - } - } - std::map> getBandwidthInterfaceTileEventSets(int hwGen) { diff --git a/profile/plugin/aie_dtrace/util/aie_dtrace_util.h b/profile/plugin/aie_dtrace/util/aie_dtrace_util.h index f570697a..a8210658 100644 --- a/profile/plugin/aie_dtrace/util/aie_dtrace_util.h +++ b/profile/plugin/aie_dtrace/util/aie_dtrace_util.h @@ -4,7 +4,6 @@ #ifndef AIE_DTRACE_UTIL_DOT_H #define AIE_DTRACE_UTIL_DOT_H -#include #include #include #include @@ -22,18 +21,6 @@ namespace xdp::aie::dtrace { // Must run before XRT creates the first dtrace module (config keys lock on first read). void initDtraceOutputConfig(); - // Register per-run bandwidth counter metadata (slot -> run uid -> JSON object string), - // captured during CT generation. Stored in a process-lifetime registry so it survives - // XDP plugin teardown; the plugin's static instance may be destroyed before the hw - // context writes the coalesced JSON dump (static destruction order is not guaranteed). - void registerCounterMetadata(uint32_t slotIdx, uint32_t runUid, const std::string& metadataJson); - - // Inject the registered counter metadata into the coalesced JSON dtrace dump file(s) - // (dtrace_dump_ctx__*.json) that core writes to the cwd on hw context teardown. - // Safe to call even after the XDP plugin instance has been destroyed; idempotent - // (each dump file is injected at most once). - void injectPendingMetadata(); - } // namespace xdp::aie::dtrace #endif diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp index f0c605e0..ef1d5f3d 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp @@ -100,17 +100,27 @@ extendLastUcToMaxConfiguredColumn(std::vector& asmFileInfoList, std::string makeCounterVarName(const CTCounterInfo& ctr) { - // The key only needs to (a) be unique within a probe block so the counters do - // not collapse into a single JSON entry, and (b) let post-processing recover - // the counter's identity. Column + counter number satisfy both: everything - // else (row, channel, direction, event) is constant per counter number across - // tiles and is carried once in the common "counter_metadata" array (keyed by - // "ctr"). Keeping the key short reduces CT file size and dtrace runtime/output - // overhead. Example: "c3_n2" -> column 3, counter 2. + const std::string dir = (ctr.portDirection == "input") ? "i" + : (ctr.portDirection == "output") ? "o" + : "x"; + std::stringstream ss; - ss << "c" << static_cast(ctr.column) - << "_n" << static_cast(ctr.counterNumber); - return ss.str(); + ss << "c" << static_cast(ctr.column) + << "_r" << static_cast(ctr.row) + << "_n" << static_cast(ctr.counterNumber) + << "_ch" << static_cast(ctr.channel) + << "_" << dir; + if (!ctr.eventType.empty()) + ss << "_" << ctr.eventType; + + // The key doubles as a python identifier in the non-JSON dump, so replace any + // character that is not [A-Za-z0-9_] with '_'. + std::string name = ss.str(); + for (char& c : name) { + if (!(std::isalnum(static_cast(c)) || c == '_')) + c = '_'; + } + return name; } } // namespace @@ -1172,98 +1182,59 @@ std::string AieDtraceCTWriter::buildBandwidthMetadataJson( const std::vector& asmFileInfoList) { // Single source of truth for the bandwidth counter metadata, shared by every - // metric set. The returned string is valid JSON and is used two ways: + // metric set. The returned string is valid JSON (per-UC groups keyed by ASM id) + // and is used two ways: // 1. rendered verbatim (with a "# " prefix) as the CT begin-block // COUNTER_METADATA comment, which is what the python dtrace dump emits, and // 2. parsed and injected into the JSON dtrace dump (the engine drops CT // comments from JSON output). // Because both consumers format from this one function, the JSON metadata is // guaranteed to match the python metadata for every metric set. - // - // The per-tile counter layout is identical for every shim tile/UC: the same - // counter configuration is replicated on each column, so only the column - // number differs between tiles and that is already encoded in each counter's - // JSON key (c_n). We therefore emit the counter descriptors once as - // a shared "counters" array (deduped by counter number, "col" dropped) instead - // of repeating the same block per UC. - // - // The per-microcontroller (UC) information that IS distinct - namely which - // columns each microcontroller owns - is emitted separately as a compact "uc" - // map keyed by UC number, without re-duplicating the counter descriptors: - // {"counters": [ ... ], "uc": {"0": {"col_start":0,"col_end":3}, ...}} - std::vector uniqueCounters; - std::vector seenCounterNumbers; - for (const auto& asmFileInfo : asmFileInfoList) { - for (const auto& ctr : asmFileInfo.counters) { - if (std::find(seenCounterNumbers.begin(), seenCounterNumbers.end(), - ctr.counterNumber) != seenCounterNumbers.end()) - continue; - seenCounterNumbers.push_back(ctr.counterNumber); - uniqueCounters.push_back(&ctr); - } - } - - std::sort(uniqueCounters.begin(), uniqueCounters.end(), - [](const CTCounterInfo* a, const CTCounterInfo* b) { - return a->counterNumber < b->counterNumber; - }); - - // Per-UC column ownership, ordered by UC number (skip UCs with no counters). - std::vector ucGroups; + std::vector metaGroups; for (const auto& asmFileInfo : asmFileInfoList) { if (!asmFileInfo.counters.empty()) - ucGroups.push_back(&asmFileInfo); + metaGroups.push_back(&asmFileInfo); } - std::sort(ucGroups.begin(), ucGroups.end(), - [](const ASMFileInfo* a, const ASMFileInfo* b) { - return a->ucNumber < b->ucNumber; - }); std::stringstream ss; ss << "{\n"; - - ss << " \"counters\": [\n"; - for (size_t c = 0; c < uniqueCounters.size(); c++) { - const auto& ctr = *uniqueCounters[c]; - ss << " {\"row\": " << static_cast(ctr.row) - << ", \"ctr\": " << static_cast(ctr.counterNumber) - << ", \"ch\": " << static_cast(ctr.channel) - << ", \"dir\": "; - if (ctr.portDirection == "input") - ss << "\"i\""; - else if (ctr.portDirection == "output") - ss << "\"o\""; - else - ss << "null"; - - if (!ctr.eventType.empty()) { - ss << ", \"event\": "; - if (ctr.eventType == "running") - ss << "\"r\""; - else if (ctr.eventType == "stalled") - ss << "\"s\""; + for (size_t g = 0; g < metaGroups.size(); g++) { + const auto& asmFileInfo = *metaGroups[g]; + ss << " \"" << asmFileInfo.asmId << "\": [\n"; + for (size_t c = 0; c < asmFileInfo.counters.size(); c++) { + const auto& ctr = asmFileInfo.counters[c]; + ss << " {\"col\": " << static_cast(ctr.column) + << ", \"row\": " << static_cast(ctr.row) + << ", \"ctr\": " << static_cast(ctr.counterNumber) + << ", \"ch\": " << static_cast(ctr.channel) + << ", \"dir\": "; + if (ctr.portDirection == "input") + ss << "\"i\""; + else if (ctr.portDirection == "output") + ss << "\"o\""; else - ss << "\"" << ctr.eventType << "\""; - } + ss << "null"; + + if (!ctr.eventType.empty()) { + ss << ", \"event\": "; + if (ctr.eventType == "running") + ss << "\"r\""; + else if (ctr.eventType == "stalled") + ss << "\"s\""; + else + ss << "\"" << ctr.eventType << "\""; + } - ss << "}"; - if (c + 1 < uniqueCounters.size()) - ss << ","; - ss << "\n"; - } - ss << " ],\n"; - - ss << " \"uc\": {\n"; - for (size_t u = 0; u < ucGroups.size(); u++) { - const auto& uc = *ucGroups[u]; - ss << " \"" << uc.ucNumber << "\": {\"col_start\": " << uc.colStart - << ", \"col_end\": " << uc.colEnd << "}"; - if (u + 1 < ucGroups.size()) + ss << "}"; + if (c + 1 < asmFileInfo.counters.size()) + ss << ","; + ss << "\n"; + } + ss << " ]"; + if (g + 1 < metaGroups.size()) ss << ","; ss << "\n"; } - ss << " }\n"; - ss << "}"; return ss.str(); } diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp index 489347f6..8ca0f727 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp @@ -16,8 +16,12 @@ #include "xdp/profile/database/static_info/aie_util.h" +#include "core/common/json/nlohmann/json.hpp" + #include #include +#include +#include #include namespace xdp { @@ -137,12 +141,11 @@ namespace xdp { bandwidthChannel, &metadataJson)) return; - // Register the per-run counter metadata in the process-lifetime registry so it can - // be injected into the JSON dtrace dump on hw context teardown. This must not live - // in the plugin/impl instance: the plugin's static instance can be destroyed before - // the hw context writes the coalesced dump, so the injection is driven from - // endAIEDtracePoll via aie::dtrace::injectPendingMetadata() instead. - aie::dtrace::registerCounterMetadata(static_cast(slotIdx), run_uid, metadataJson); + // Remember slot + per-run counter metadata so it can be injected into the JSON + // dtrace dump on hw context teardown (see finalizeDtraceDump()). + m_dumpSlotIdx = static_cast(slotIdx); + if (!metadataJson.empty()) + m_runMetadataJson[run_uid] = metadataJson; aie::dtrace::initDtraceOutputConfig(); @@ -165,4 +168,100 @@ namespace xdp { } } + void AieDtrace_VE2Impl::finalizeDtraceDump() + { + // The dtrace engine drops the CT COUNTER_METADATA comment from the JSON dump, + // so inject the metadata captured at CT generation into the JSON file that core + // has just written for this hw context (before it is erased on teardown). + if (m_dumpSlotIdx < 0 || m_runMetadataJson.empty()) + return; + + // The JSON dump file only exists in coalesced JSON output mode. + if (!(xrt_core::config::get_dtrace_output_json_format() + && xrt_core::config::get_dtrace_coalesce_result())) + return; + + try { + // Locate the newest dump file for this context slot in the working directory. + // core writes: dtrace_dump_ctx__.json + const std::regex fileRe("^dtrace_dump_ctx_" + std::to_string(m_dumpSlotIdx) + + "_.*\\.json$"); + std::filesystem::path dumpFile; + std::filesystem::file_time_type newest{}; + for (const auto& entry : std::filesystem::directory_iterator(std::filesystem::current_path())) { + if (!entry.is_regular_file()) + continue; + if (!std::regex_match(entry.path().filename().string(), fileRe)) + continue; + auto mtime = entry.last_write_time(); + if (dumpFile.empty() || mtime > newest) { + dumpFile = entry.path(); + newest = mtime; + } + } + + if (dumpFile.empty()) { + xrt_core::message::send(severity_level::debug, "XRT", + "AIE dtrace: no JSON dump file found for slot " + + std::to_string(m_dumpSlotIdx) + "; skipping metadata injection."); + return; + } + + nlohmann::ordered_json root; + { + std::ifstream in(dumpFile); + if (!in) + return; + in >> root; + } + + if (!root.is_object()) + return; + + // A fallback metadata (same metric set across runs of a context): used when a + // run key's uid is not found in the captured map. + const std::string& fallbackMeta = m_runMetadataJson.begin()->second; + const std::regex runRe("_run_(\\d+)_"); + + for (auto& item : root.items()) { + auto& runObj = item.value(); + if (!runObj.is_object()) + continue; + + const std::string* metaStr = &fallbackMeta; + std::smatch m; + const std::string key = item.key(); + if (std::regex_search(key, m, runRe)) { + auto uid = static_cast(std::stoul(m[1].str())); + auto it = m_runMetadataJson.find(uid); + if (it != m_runMetadataJson.end()) + metaStr = &it->second; + } + + nlohmann::ordered_json meta = nlohmann::ordered_json::parse(*metaStr, nullptr, false); + if (meta.is_discarded()) + continue; + + // Mirror the python dump: metadata lives in the begin block. + if (!runObj.contains("begin") || !runObj["begin"].is_object()) + runObj["begin"] = nlohmann::ordered_json::object(); + runObj["begin"]["counter_metadata"] = std::move(meta); + } + + { + std::ofstream out(dumpFile, std::ios::trunc); + if (!out) + return; + out << root.dump(4) << "\n"; + } + + xrt_core::message::send(severity_level::debug, "XRT", + "AIE dtrace: injected counter metadata into " + dumpFile.string()); + } + catch (const std::exception& e) { + xrt_core::message::send(severity_level::debug, "XRT", + std::string{"AIE dtrace: metadata injection failed (ignored): "} + e.what()); + } + } + } diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h index 3fa6926a..3ac52b91 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h @@ -33,10 +33,20 @@ namespace xdp { const std::string& kernel_name, void* elf_handle) override; + void finalizeDtraceDump() override; + private: void computeOpLocations(void* elf_handle, const std::string& kernel_name); std::map> m_op_locations_cache; + + // Slot index of the hw context this impl serves (for locating the dump file); + // -1 until a CT has been generated for a run. + int m_dumpSlotIdx = -1; + + // Per-run counter metadata (run_uid -> JSON object string) captured at CT + // generation, injected into the JSON dtrace dump on teardown. + std::map m_runMetadataJson; }; } From 196024b0f2d67c0ed227ae1f3385e912a9374cfa Mon Sep 17 00:00:00 2001 From: Garima Dhaked Date: Thu, 6 Aug 2026 03:37:22 -0600 Subject: [PATCH 5/6] Revert "Inject counter metadata into JSON dtrace_dump from aie_dtrace plugin" This reverts commit 3908f2d9f747d24e2b713764a314b2d9431293de. --- profile/plugin/aie_dtrace/aie_dtrace_impl.h | 4 - .../plugin/aie_dtrace/aie_dtrace_plugin.cpp | 6 - .../aie_dtrace/ve2/aie_dtrace_ct_writer.cpp | 174 ++++++------------ .../aie_dtrace/ve2/aie_dtrace_ct_writer.h | 16 +- .../plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp | 110 +---------- .../plugin/aie_dtrace/ve2/aie_dtrace_ve2.h | 10 - 6 files changed, 56 insertions(+), 264 deletions(-) diff --git a/profile/plugin/aie_dtrace/aie_dtrace_impl.h b/profile/plugin/aie_dtrace/aie_dtrace_impl.h index 4bc6b476..7c505be7 100644 --- a/profile/plugin/aie_dtrace/aie_dtrace_impl.h +++ b/profile/plugin/aie_dtrace/aie_dtrace_impl.h @@ -47,10 +47,6 @@ namespace xdp { const std::string& /*kernel_name*/, void* /*elf_handle*/) {} - // Post-process the dumped dtrace JSON (e.g. inject counter metadata) after the - // hw context has written it. Called on hw context teardown, after the dump. - virtual void finalizeDtraceDump() {} - uint64_t getDeviceID() { return deviceID; } }; diff --git a/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp b/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp index e1ea1e8a..3c5f1abc 100644 --- a/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp +++ b/profile/plugin/aie_dtrace/aie_dtrace_plugin.cpp @@ -182,12 +182,6 @@ namespace xdp { if (itr == handleToAIEDtraceImpl.end()) return; - // Post-process the JSON dtrace dump for this context. This runs from - // finish_flush_device on hw context teardown, i.e. after core has written the - // dtrace_dump_ctx__*.json file, so the dump exists and can be augmented - // with the counter metadata captured during CT generation. - itr->second->finalizeDtraceDump(); - // Drop implementation without endPoll(): dtrace must not read/offload on hwctx teardown; // ~AieDtrace_VE2Impl releases FAL resources only. handleToAIEDtraceImpl.erase(itr); diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp index ef1d5f3d..a5f313fa 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp @@ -12,7 +12,6 @@ #include "core/common/message.h" #include -#include #include #include #include @@ -88,41 +87,6 @@ extendLastUcToMaxConfiguredColumn(std::vector& asmFileInfoList, last.colEnd = std::max(last.colEnd, maxCfgCol); } -// Build a unique, self-describing dtrace variable name for a counter read. -// -// The JSON dtrace dump keys every read action by its left-hand-side variable -// name (json[probe][lhs] = value). Using the throwaway "_" for all reads makes -// the 4-16 per-probe counters collapse into a single "_" key in the JSON, and -// the "# COUNTER_METADATA" block is a comment that the JSON serializer drops. -// Encoding col/row/counter/channel/direction/event into the LHS name keeps every -// counter as a distinct JSON key and embeds the per-counter metadata directly in -// that key (e.g. "c0_r0_n1_ch0_i_lock"). -std::string -makeCounterVarName(const CTCounterInfo& ctr) -{ - const std::string dir = (ctr.portDirection == "input") ? "i" - : (ctr.portDirection == "output") ? "o" - : "x"; - - std::stringstream ss; - ss << "c" << static_cast(ctr.column) - << "_r" << static_cast(ctr.row) - << "_n" << static_cast(ctr.counterNumber) - << "_ch" << static_cast(ctr.channel) - << "_" << dir; - if (!ctr.eventType.empty()) - ss << "_" << ctr.eventType; - - // The key doubles as a python identifier in the non-JSON dump, so replace any - // character that is not [A-Za-z0-9_] with '_'. - std::string name = ss.str(); - for (char& c : name) { - if (!(std::isalnum(static_cast(c)) || c == '_')) - c = '_'; - } - return name; -} - } // namespace using severity_level = xrt_core::message::severity_level; @@ -672,13 +636,10 @@ bool AieDtraceCTWriter::writeCTFile(const std::vector& asmFileInfoL ctFile << "{\n"; ctFile << " ts_" << asmFileInfo.asmId << " = timestamp32()\n"; - // Write counter reads using a unique, self-describing variable name per - // counter so the JSON dtrace dump keeps every counter as a distinct key - // (see makeCounterVarName). + // Write counter reads using _ as throwaway variable for (size_t i = 0; i < asmFileInfo.counters.size(); i++) { - const auto& ctr = asmFileInfo.counters[i]; - ctFile << " " << makeCounterVarName(ctr) - << " = read_reg(" << formatAddress(ctr.address) << ")\n"; + ctFile << " _ = read_reg(" + << formatAddress(asmFileInfo.counters[i].address) << ")\n"; } ctFile << "}\n\n"; @@ -1116,17 +1077,58 @@ bool AieDtraceCTWriter::writeBandwidthCTFile( ctFile << "@blockopen\n"; ctFile << "# COUNTER_METADATA_BEGIN\n"; + ctFile << "# {\n"; - // Render the metadata comment from the same builder used for the JSON dump, so the - // python (comment) and JSON metadata are identical for every metric set. - { - const std::string metadata = buildBandwidthMetadataJson(asmFileInfoList); - std::istringstream metaStream(metadata); - std::string line; - while (std::getline(metaStream, line)) - ctFile << "# " << line << "\n"; + // Per-UC counter metadata groupings only + std::vector metaGroups; + for (const auto& asmFileInfo : asmFileInfoList) { + if (!asmFileInfo.counters.empty()) + metaGroups.push_back(&asmFileInfo); + } + + for (size_t g = 0; g < metaGroups.size(); g++) { + const auto& asmFileInfo = *metaGroups[g]; + ctFile << "# \"" << asmFileInfo.asmId << "\": [\n"; + + for (size_t c = 0; c < asmFileInfo.counters.size(); c++) { + const auto& ctr = asmFileInfo.counters[c]; + ctFile << "# {\"col\": " << static_cast(ctr.column) + << ", \"row\": " << static_cast(ctr.row) + << ", \"ctr\": " << static_cast(ctr.counterNumber) + << ", \"ch\": " << static_cast(ctr.channel) + << ", \"dir\": "; + + if (ctr.portDirection == "input") + ctFile << "\"i\""; + else if (ctr.portDirection == "output") + ctFile << "\"o\""; + else + ctFile << "null"; + + // Add event type for peak bandwidth metrics + if (!ctr.eventType.empty()) { + ctFile << ", \"event\": "; + if (ctr.eventType == "running") + ctFile << "\"r\""; + else if (ctr.eventType == "stalled") + ctFile << "\"s\""; + else + ctFile << "\"" << ctr.eventType << "\""; + } + + ctFile << "}"; + if (c < asmFileInfo.counters.size() - 1) + ctFile << ","; + ctFile << "\n"; + } + + ctFile << "# ]"; + if (g < metaGroups.size() - 1) + ctFile << ","; + ctFile << "\n"; } + ctFile << "# }\n"; ctFile << "# COUNTER_METADATA_END\n"; ctFile << "@blockclose\n"; ctFile << "}\n\n"; @@ -1156,8 +1158,7 @@ bool AieDtraceCTWriter::writeBandwidthCTFile( for (size_t i = 0; i < asmFileInfo.counters.size(); i++) { const auto& ctr = asmFileInfo.counters[i]; - ctFile << " " << makeCounterVarName(ctr) - << " = read_reg(" << formatAddress(ctr.address) << ")\n"; + ctFile << " _ = read_reg(" << formatAddress(ctr.address) << ")\n"; } ctFile << "}\n\n"; @@ -1178,74 +1179,12 @@ bool AieDtraceCTWriter::writeBandwidthCTFile( return true; } -std::string AieDtraceCTWriter::buildBandwidthMetadataJson( - const std::vector& asmFileInfoList) -{ - // Single source of truth for the bandwidth counter metadata, shared by every - // metric set. The returned string is valid JSON (per-UC groups keyed by ASM id) - // and is used two ways: - // 1. rendered verbatim (with a "# " prefix) as the CT begin-block - // COUNTER_METADATA comment, which is what the python dtrace dump emits, and - // 2. parsed and injected into the JSON dtrace dump (the engine drops CT - // comments from JSON output). - // Because both consumers format from this one function, the JSON metadata is - // guaranteed to match the python metadata for every metric set. - std::vector metaGroups; - for (const auto& asmFileInfo : asmFileInfoList) { - if (!asmFileInfo.counters.empty()) - metaGroups.push_back(&asmFileInfo); - } - - std::stringstream ss; - ss << "{\n"; - for (size_t g = 0; g < metaGroups.size(); g++) { - const auto& asmFileInfo = *metaGroups[g]; - ss << " \"" << asmFileInfo.asmId << "\": [\n"; - for (size_t c = 0; c < asmFileInfo.counters.size(); c++) { - const auto& ctr = asmFileInfo.counters[c]; - ss << " {\"col\": " << static_cast(ctr.column) - << ", \"row\": " << static_cast(ctr.row) - << ", \"ctr\": " << static_cast(ctr.counterNumber) - << ", \"ch\": " << static_cast(ctr.channel) - << ", \"dir\": "; - if (ctr.portDirection == "input") - ss << "\"i\""; - else if (ctr.portDirection == "output") - ss << "\"o\""; - else - ss << "null"; - - if (!ctr.eventType.empty()) { - ss << ", \"event\": "; - if (ctr.eventType == "running") - ss << "\"r\""; - else if (ctr.eventType == "stalled") - ss << "\"s\""; - else - ss << "\"" << ctr.eventType << "\""; - } - - ss << "}"; - if (c + 1 < asmFileInfo.counters.size()) - ss << ","; - ss << "\n"; - } - ss << " ]"; - if (g + 1 < metaGroups.size()) - ss << ","; - ss << "\n"; - } - ss << "}"; - return ss.str(); -} - bool AieDtraceCTWriter::generateBandwidthCT( const std::string& outputPath, void* hwctx, const std::vector& opLocations, const std::string& metricSet, - uint8_t channel, - std::string* outMetadataJson) + uint8_t channel) { if (opLocations.empty()) { xrt_core::message::send(severity_level::debug, "XRT", @@ -1319,11 +1258,6 @@ bool AieDtraceCTWriter::generateBandwidthCT( asmFileInfo.counters = filterCountersByColumn(allCounters, asmFileInfo.colStart, asmFileInfo.colEnd); } - // Expose the counter metadata so the plugin can inject it into the JSON dtrace - // dump (the dtrace engine drops the CT COUNTER_METADATA comment from JSON output). - if (outMetadataJson) - *outMetadataJson = buildBandwidthMetadataJson(asmFileInfoList); - std::vector beginBlockWrites; for (uint8_t column : shimColumns) { // For detailed sets, counter 0 monitors PORT_RUNNING on the channel's diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.h b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.h index 45c68b6c..d353b165 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.h +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.h @@ -164,8 +164,7 @@ class AieDtraceCTWriter { void* hwctx, const std::vector& opLocations, const std::string& metricSet = "ddr_bandwidth", - uint8_t channel = 0, - std::string* outMetadataJson = nullptr); + uint8_t channel = 0); private: /** @@ -303,19 +302,6 @@ class AieDtraceCTWriter { const std::vector& beginBlockWrites, const std::string& outputPath); - /** - * @brief Build the per-UC counter metadata as a compact JSON string - * - * Produces the same structure embedded in the CT begin-block - * COUNTER_METADATA comment (keyed by ASM id -> array of counter descriptors). - * Used to inject the metadata into the JSON dtrace dump during post-processing, - * since the dtrace engine drops CT comments from the JSON output. - * - * @param asmFileInfoList ASM file infos with filtered per-UC counters - * @return JSON object string, e.g. {"0":[{"col":0,"row":0,"ctr":0,...}]} - */ - std::string buildBandwidthMetadataJson(const std::vector& asmFileInfoList); - private: VPDatabase* db; std::shared_ptr metadata; diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp index 8ca0f727..b59c6226 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.cpp @@ -16,12 +16,8 @@ #include "xdp/profile/database/static_info/aie_util.h" -#include "core/common/json/nlohmann/json.hpp" - #include #include -#include -#include #include namespace xdp { @@ -136,17 +132,9 @@ namespace xdp { "AIE dtrace: No interface tile metrics configured, using default 'peak_read_bandwidth'"); } - std::string metadataJson; - if (!ctWriter.generateBandwidthCT(outputPath, hwctx, it->second, bandwidthMetricSet, - bandwidthChannel, &metadataJson)) + if (!ctWriter.generateBandwidthCT(outputPath, hwctx, it->second, bandwidthMetricSet, bandwidthChannel)) return; - // Remember slot + per-run counter metadata so it can be injected into the JSON - // dtrace dump on hw context teardown (see finalizeDtraceDump()). - m_dumpSlotIdx = static_cast(slotIdx); - if (!metadataJson.empty()) - m_runMetadataJson[run_uid] = metadataJson; - aie::dtrace::initDtraceOutputConfig(); xrt_core::message::send(severity_level::debug, "XRT", @@ -168,100 +156,4 @@ namespace xdp { } } - void AieDtrace_VE2Impl::finalizeDtraceDump() - { - // The dtrace engine drops the CT COUNTER_METADATA comment from the JSON dump, - // so inject the metadata captured at CT generation into the JSON file that core - // has just written for this hw context (before it is erased on teardown). - if (m_dumpSlotIdx < 0 || m_runMetadataJson.empty()) - return; - - // The JSON dump file only exists in coalesced JSON output mode. - if (!(xrt_core::config::get_dtrace_output_json_format() - && xrt_core::config::get_dtrace_coalesce_result())) - return; - - try { - // Locate the newest dump file for this context slot in the working directory. - // core writes: dtrace_dump_ctx__.json - const std::regex fileRe("^dtrace_dump_ctx_" + std::to_string(m_dumpSlotIdx) - + "_.*\\.json$"); - std::filesystem::path dumpFile; - std::filesystem::file_time_type newest{}; - for (const auto& entry : std::filesystem::directory_iterator(std::filesystem::current_path())) { - if (!entry.is_regular_file()) - continue; - if (!std::regex_match(entry.path().filename().string(), fileRe)) - continue; - auto mtime = entry.last_write_time(); - if (dumpFile.empty() || mtime > newest) { - dumpFile = entry.path(); - newest = mtime; - } - } - - if (dumpFile.empty()) { - xrt_core::message::send(severity_level::debug, "XRT", - "AIE dtrace: no JSON dump file found for slot " - + std::to_string(m_dumpSlotIdx) + "; skipping metadata injection."); - return; - } - - nlohmann::ordered_json root; - { - std::ifstream in(dumpFile); - if (!in) - return; - in >> root; - } - - if (!root.is_object()) - return; - - // A fallback metadata (same metric set across runs of a context): used when a - // run key's uid is not found in the captured map. - const std::string& fallbackMeta = m_runMetadataJson.begin()->second; - const std::regex runRe("_run_(\\d+)_"); - - for (auto& item : root.items()) { - auto& runObj = item.value(); - if (!runObj.is_object()) - continue; - - const std::string* metaStr = &fallbackMeta; - std::smatch m; - const std::string key = item.key(); - if (std::regex_search(key, m, runRe)) { - auto uid = static_cast(std::stoul(m[1].str())); - auto it = m_runMetadataJson.find(uid); - if (it != m_runMetadataJson.end()) - metaStr = &it->second; - } - - nlohmann::ordered_json meta = nlohmann::ordered_json::parse(*metaStr, nullptr, false); - if (meta.is_discarded()) - continue; - - // Mirror the python dump: metadata lives in the begin block. - if (!runObj.contains("begin") || !runObj["begin"].is_object()) - runObj["begin"] = nlohmann::ordered_json::object(); - runObj["begin"]["counter_metadata"] = std::move(meta); - } - - { - std::ofstream out(dumpFile, std::ios::trunc); - if (!out) - return; - out << root.dump(4) << "\n"; - } - - xrt_core::message::send(severity_level::debug, "XRT", - "AIE dtrace: injected counter metadata into " + dumpFile.string()); - } - catch (const std::exception& e) { - xrt_core::message::send(severity_level::debug, "XRT", - std::string{"AIE dtrace: metadata injection failed (ignored): "} + e.what()); - } - } - } diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h index 3ac52b91..3fa6926a 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ve2.h @@ -33,20 +33,10 @@ namespace xdp { const std::string& kernel_name, void* elf_handle) override; - void finalizeDtraceDump() override; - private: void computeOpLocations(void* elf_handle, const std::string& kernel_name); std::map> m_op_locations_cache; - - // Slot index of the hw context this impl serves (for locating the dump file); - // -1 until a CT has been generated for a run. - int m_dumpSlotIdx = -1; - - // Per-run counter metadata (run_uid -> JSON object string) captured at CT - // generation, injected into the JSON dtrace dump on teardown. - std::map m_runMetadataJson; }; } From bd170f9bee84a3482514f648b6e57c575cc647de Mon Sep 17 00:00:00 2001 From: Garima Dhaked Date: Thu, 6 Aug 2026 06:05:14 -0600 Subject: [PATCH 6/6] Emit sequential per-counter result vars in aie_dtrace CT file The dtrace JSON dump keys each counter by the read_reg result variable name. The CT writer used "_" for every counter read, which collapses all counters into a single "_" JSON key. Assign a sequential result variable (_0, _1, _2, ...) per counter instead, so each counter appears as its own key ("_0", "_1", ...) in probe order. Counter semantics remain described by the COUNTER_METADATA block. Applied to both writeCTFile and writeBandwidthCTFile. Co-authored-by: Cursor --- .../plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp index a5f313fa..137a8039 100644 --- a/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp +++ b/profile/plugin/aie_dtrace/ve2/aie_dtrace_ct_writer.cpp @@ -636,9 +636,12 @@ bool AieDtraceCTWriter::writeCTFile(const std::vector& asmFileInfoL ctFile << "{\n"; ctFile << " ts_" << asmFileInfo.asmId << " = timestamp32()\n"; - // Write counter reads using _ as throwaway variable + // Write counter reads. Each counter is assigned a sequential result + // variable (_0, _1, _2, ...) so the dtrace JSON dump emits one key per + // counter ("_0", "_1", ...) in probe order; counter semantics are provided + // by the COUNTER_METADATA block above. for (size_t i = 0; i < asmFileInfo.counters.size(); i++) { - ctFile << " _ = read_reg(" + ctFile << " _" << i << " = read_reg(" << formatAddress(asmFileInfo.counters[i].address) << ")\n"; } @@ -1156,9 +1159,12 @@ bool AieDtraceCTWriter::writeBandwidthCTFile( ctFile << "{\n"; ctFile << " ts_" << asmFileInfo.asmId << " = timestamp32()\n"; + // Each counter is assigned a sequential result variable (_0, _1, _2, ...) + // so the dtrace JSON dump emits one key per counter ("_0", "_1", ...) in + // probe order; counter semantics live in the COUNTER_METADATA block above. for (size_t i = 0; i < asmFileInfo.counters.size(); i++) { const auto& ctr = asmFileInfo.counters[i]; - ctFile << " _ = read_reg(" << formatAddress(ctr.address) << ")\n"; + ctFile << " _" << i << " = read_reg(" << formatAddress(ctr.address) << ")\n"; } ctFile << "}\n\n";