diff --git a/CONTRIBUTORS.TXT b/CONTRIBUTORS.TXT index 43d688c8b7..83dbba3899 100644 --- a/CONTRIBUTORS.TXT +++ b/CONTRIBUTORS.TXT @@ -5,3 +5,4 @@ International Business Machines (IBM) 2016-2025 Cray Inc 2014-2025 Intel 2023 EP Analytics 2025 +Hewlett Packard Enterprise Development LP 2026 diff --git a/src/sst/elements/ember/Makefile.am b/src/sst/elements/ember/Makefile.am index 05fd3622b9..ff48228d6c 100644 --- a/src/sst/elements/ember/Makefile.am +++ b/src/sst/elements/ember/Makefile.am @@ -253,6 +253,14 @@ libember_la_SOURCES = \ shmem/motifs/emberShmemFAM_Gatherv.h \ shmem/motifs/emberShmemFAM_AtomicInc.h \ shmem/motifs/emberShmemFAM_Cswap.h \ + libs/networkIOEvents/emberNetworkIOEvents.h \ + libs/networkIOEvents/emberNetworkIOReadEvent.h \ + libs/networkIOEvents/emberNetworkIOWriteEvent.h \ + libs/emberNetworkIOLib.h \ + networkIO/emberNetworkIOGen.h \ + networkIO/emberNetworkIOGen.cc \ + networkIO/motifs/emberTestNetworkIO.h \ + networkIO/motifs/emberTestNetworkIO.cc \ sirius/include/sirius/siriusglobals.h \ pyember.py diff --git a/src/sst/elements/ember/libs/emberNetworkIOLib.h b/src/sst/elements/ember/libs/emberNetworkIOLib.h new file mode 100644 index 0000000000..4e41b4e866 --- /dev/null +++ b/src/sst/elements/ember/libs/emberNetworkIOLib.h @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + +#include "sst/elements/ember/embergen.h" +#include "sst/elements/ember/libs/emberLib.h" +#include "sst/elements/ember/libs/networkIOEvents/emberNetworkIOReadEvent.h" +#include "sst/elements/ember/libs/networkIOEvents/emberNetworkIOWriteEvent.h" +#include "sst/elements/hermes/networkIOapi.h" + +using namespace Hermes; + +namespace SST { +namespace Ember { + +class EmberNetworkIOLib : public EmberLib { +public: + SST_ELI_REGISTER_MODULE( + EmberNetworkIOLib, + "ember", + "networkIOLib", + SST_ELI_ELEMENT_VERSION(1,0,0), + "Network I/O Library for network-attached storage operations", + SST::Ember::EmberLib + ) + + SST_ELI_DOCUMENT_PARAMS() + + EmberNetworkIOLib(Params& params) {} + + void networkIORead(std::queue& q, Hermes::MemAddr dest, uint64_t offset, uint64_t length) + { + q.push(new EmberNetworkIOReadEvent(api(), m_output, dest, offset, length)); + } + + void networkIOWrite(std::queue& q, uint64_t offset, Hermes::MemAddr src, uint64_t length) + { + q.push(new EmberNetworkIOWriteEvent(api(), m_output, offset, src, length)); + } + +private: + NetworkIO::Interface& api() { return *static_cast(m_api); } +}; + +} +} diff --git a/src/sst/elements/ember/libs/networkIOEvents/emberNetworkIOEvent.h b/src/sst/elements/ember/libs/networkIOEvents/emberNetworkIOEvent.h new file mode 100644 index 0000000000..aa001499e5 --- /dev/null +++ b/src/sst/elements/ember/libs/networkIOEvents/emberNetworkIOEvent.h @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + +#include +#include + +using namespace Hermes; + +namespace SST { +namespace Ember { + +typedef Statistic EmberEventTimeStatistic; + +class EmberNetworkIOEvent : public EmberEvent { +public: + EmberNetworkIOEvent(NetworkIO::Interface& api, Output* output, + EmberEventTimeStatistic* stat = NULL) : + EmberEvent(output, stat), m_api(api) + { + m_state = IssueCallback; + } + +protected: + NetworkIO::Interface& m_api; +}; + +} +} + diff --git a/src/sst/elements/ember/libs/networkIOEvents/emberNetworkIOReadEvent.h b/src/sst/elements/ember/libs/networkIOEvents/emberNetworkIOReadEvent.h new file mode 100644 index 0000000000..9b5d1c1384 --- /dev/null +++ b/src/sst/elements/ember/libs/networkIOEvents/emberNetworkIOReadEvent.h @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + +#include "emberNetworkIOEvent.h" + +namespace SST { +namespace Ember { + +class EmberNetworkIOReadEvent : public EmberNetworkIOEvent { +public: + EmberNetworkIOReadEvent(NetworkIO::Interface& api, Output* output, Hermes::MemAddr dest, + uint64_t offset, uint32_t length, + EmberEventTimeStatistic* stat = NULL) : + EmberNetworkIOEvent(api, output, stat), + m_dest(dest), m_offset(offset), m_length(length) + {} + + ~EmberNetworkIOReadEvent() {} + + std::string getName() { return "NetworkIORead"; } + + virtual void issue(uint64_t time, Callback callback) { + EmberEvent::issue(time); + m_api.networkIORead(m_dest.getSimVAddr(), m_offset, m_length, callback); + } + +private: + Hermes::MemAddr m_dest; + uint64_t m_offset; + uint32_t m_length; +}; + +} +} diff --git a/src/sst/elements/ember/libs/networkIOEvents/emberNetworkIOWriteEvent.h b/src/sst/elements/ember/libs/networkIOEvents/emberNetworkIOWriteEvent.h new file mode 100644 index 0000000000..ee0a392f1f --- /dev/null +++ b/src/sst/elements/ember/libs/networkIOEvents/emberNetworkIOWriteEvent.h @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + +#include "emberNetworkIOEvent.h" + +namespace SST { +namespace Ember { + +class EmberNetworkIOWriteEvent : public EmberNetworkIOEvent { +public: + EmberNetworkIOWriteEvent(NetworkIO::Interface& api, Output* output, uint64_t offset, + Hermes::MemAddr src, uint32_t length, + EmberEventTimeStatistic* stat = NULL) : + EmberNetworkIOEvent(api, output, stat), + m_offset(offset), m_src(src), m_length(length) + {} + + ~EmberNetworkIOWriteEvent() {} + + std::string getName() { return "NetworkIOWrite"; } + + virtual void issue(uint64_t time, Callback callback) { + EmberEvent::issue(time); + m_api.networkIOWrite(m_offset, m_src.getSimVAddr(), m_length, callback); + } + +private: + uint64_t m_offset; + Hermes::MemAddr m_src; + uint32_t m_length; +}; + +} +} diff --git a/src/sst/elements/ember/networkIO/emberNetworkIOGen.cc b/src/sst/elements/ember/networkIO/emberNetworkIOGen.cc new file mode 100644 index 0000000000..632283d081 --- /dev/null +++ b/src/sst/elements/ember/networkIO/emberNetworkIOGen.cc @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include "emberNetworkIOGen.h" + +using namespace SST::Ember; + +EmberNetworkIOGenerator::EmberNetworkIOGenerator(ComponentId_t id, Params& params, std::string name) + : EmberGenerator(id, params, name), m_networkIOLib(nullptr) +{ + m_shmemLib = NULL; +} + +void EmberNetworkIOGenerator::setup() +{ + m_shmemLib = static_cast(getLib("shmem")); + m_networkIOLib = static_cast(getLib("networkIO")); + assert(m_shmemLib); + assert(m_networkIOLib); +} \ No newline at end of file diff --git a/src/sst/elements/ember/networkIO/emberNetworkIOGen.h b/src/sst/elements/ember/networkIO/emberNetworkIOGen.h new file mode 100644 index 0000000000..481261eadf --- /dev/null +++ b/src/sst/elements/ember/networkIO/emberNetworkIOGen.h @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once +#include "sst/elements/ember/embergen.h" +#include "sst/elements/ember/libs/emberLib.h" +#include "sst/elements/ember/libs/emberShmemLib.h" +#include "sst/elements/ember/libs/emberNetworkIOLib.h" + +using namespace Hermes; + +namespace SST { +namespace Ember { + +class EmberNetworkIOGenerator : public EmberGenerator { +public: + EmberNetworkIOGenerator(ComponentId_t id, Params& params, std::string name = ""); + ~EmberNetworkIOGenerator() {} + virtual void completed(const SST::Output*, uint64_t time) {} + virtual void setup(); + +protected: + EmberShmemLib* m_shmemLib; + + + EmberNetworkIOLib* m_networkIOLib; + EmberNetworkIOLib& networkIO() { return *m_networkIOLib; } + + EmberShmemLib& shmem() { return *m_shmemLib; } + + // Barrier macro (same as EmberShmemGen) + #define enQ_barrier_all shmem().barrier_all + #define enQ_malloc shmem().malloc + #define enQ_my_pe shmem().my_pe +}; + +} +} + diff --git a/src/sst/elements/ember/networkIO/motifs/emberTestNetworkIO.cc b/src/sst/elements/ember/networkIO/motifs/emberTestNetworkIO.cc new file mode 100644 index 0000000000..b367716776 --- /dev/null +++ b/src/sst/elements/ember/networkIO/motifs/emberTestNetworkIO.cc @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include "emberTestNetworkIO.h" +#include + +using namespace SST::Ember; + +EmberTestNetworkIOGenerator::EmberTestNetworkIOGenerator(SST::ComponentId_t id, Params& params) + : EmberNetworkIOGenerator(id, params, "TestNetworkIO"), m_phase(0) +{ + m_messageSize = params.find("arg.messageSize", 1024); + m_iterations = params.find("arg.iterations", 5); + m_opType = params.find("arg.op", "write"); + m_fileSize = params.find("arg.fileSize", 10485760); // 10MB default + + m_rng = new SST::RNG::MarsagliaRNG(); + m_startTime = 0; + m_stopTime = 0; +} + +bool EmberTestNetworkIOGenerator::generate( std::queue& evQ) +{ + bool ret = false; + switch(m_phase) + { + case 0: + memSetNotBacked(); + m_localBuffer = memAlloc(m_messageSize); + enQ_getTime(evQ, &m_startTime); + for (uint32_t i = 0; i < m_iterations; i++) + { + uint64_t offset = m_rng->generateNextUInt64() % m_fileSize; + + if (m_opType == "read") + networkIO().networkIORead(evQ, m_localBuffer, offset, m_messageSize); + else + networkIO().networkIOWrite(evQ, offset, m_localBuffer, m_messageSize); + } + enQ_getTime(evQ, &m_stopTime); + break; + + case 1: + double totalTime = (double)(m_stopTime - m_startTime)/1000000000.0; + double latency = (totalTime/m_iterations); + output("message-size %u, iterations %u, total-time %.3lf us\n", + m_messageSize, m_iterations, totalTime * 1000000.0) ; + ret = true; + break; + } + + ++m_phase; + return ret; +} + diff --git a/src/sst/elements/ember/networkIO/motifs/emberTestNetworkIO.h b/src/sst/elements/ember/networkIO/motifs/emberTestNetworkIO.h new file mode 100644 index 0000000000..94249af844 --- /dev/null +++ b/src/sst/elements/ember/networkIO/motifs/emberTestNetworkIO.h @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + +#include +#include +#include "networkIO/emberNetworkIOGen.h" + +namespace SST { +namespace Ember { + +class EmberTestNetworkIOGenerator : public EmberNetworkIOGenerator +{ +public: + SST_ELI_REGISTER_SUBCOMPONENT( + EmberTestNetworkIOGenerator, + "ember", + "TestNetworkIOMotif", + SST_ELI_ELEMENT_VERSION(1,0,0), + "Network IO Test", + SST::Ember::EmberGenerator + ) + + SST_ELI_DOCUMENT_PARAMS( + {"arg.messageSize","Message size in bytes","1024"}, + {"arg.iterations","Number of iterations to perform","1"}, + {"arg.op","Operation type: read or write","write"}, + {"arg.fileSize","Storage file size in bytes","10485760"} + ) + + EmberTestNetworkIOGenerator(SST::ComponentId_t id, Params& params); + + bool generate( std::queue& evQ); + +private: + // Simulation control + int m_phase; + + // Random number generation + SST::RNG::MarsagliaRNG* m_rng; // RNG instance for offset generation + + // Operation parameters + uint32_t m_messageSize; // Size of each read/write operation + uint32_t m_iterations; // Number of operations to perform + uint64_t m_fileSize; // Total file size for offset generation + std::string m_opType; // Operation type: "read" or "write" + + // Runtime state + Hermes::MemAddr m_localBuffer; // Buffer for data transfer + + // Performance measurement + uint64_t m_startTime; + uint64_t m_stopTime; +}; +} +} diff --git a/src/sst/elements/ember/test/loadNetworkIO b/src/sst/elements/ember/test/loadNetworkIO new file mode 100644 index 0000000000..6b7f0fbe16 --- /dev/null +++ b/src/sst/elements/ember/test/loadNetworkIO @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +# SPDX-License-Identifier: BSD-3-Clause + +# This file must be ordered +# +# [JOB_ID] +# [NID_LIST] +# [MOTIF] +# [MOTIF] +# +# keywords must be left justified and not contain white space +# all characters between keywords will be considered part of the leading keyword +# you can have multiple MOTIF keywords for a given JOB_ID +# you can have multiple jobs +# two jobs can not have the same NID + +[VAR] TOTAL_NODES=4 +[VAR] COMPUTE_OFFSET=0 +[VAR] COMPUTE_NODES=2 +[VAR] SSD_START_NODE=2 +[VAR] SSD_NODES=2 + +[JOB_ID] 1 +[NID_LIST] generateNidList=generateNidListRange({COMPUTE_OFFSET},{COMPUTE_NODES}) +[PARAM] ember:firefly.hadesNetworkIO.ssd_start_node={SSD_START_NODE} +[PARAM] ember:firefly.hadesNetworkIO.numStorageNodes={SSD_NODES} +[PARAM] ember:firefly.hadesNetworkIO.storageNodeCapacity=1GiB +[MOTIF] TestNetworkIO iterations=1 messageSize=1000 op=write + +[JOB_ID] 2 +[NID_LIST] generateNidList=generateNidListRange({SSD_START_NODE},{SSD_NODES}) +[MOTIF] Null \ No newline at end of file diff --git a/src/sst/elements/ember/test/networkIOParams.py b/src/sst/elements/ember/test/networkIOParams.py new file mode 100644 index 0000000000..ee65408775 --- /dev/null +++ b/src/sst/elements/ember/test/networkIOParams.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +# SPDX-License-Identifier: BSD-3-Clause + + +debug = 0 + +netConfig = { +} + +networkParams = { + "packetSize" : "2048B", + "link_bw" : "4GB/s", + "xbar_bw" : "4GB/s", + "link_lat" : "40ns", + "input_latency" : "50ns", + "output_latency" : "50ns", + "flitSize" : "8B", + "input_buf_size" : "14KB", + "output_buf_size" : "14KB", +} + +nicParams = { + "detailedCompute.name" : "thornhill.SingleThread", + "module" : "merlin.linkcontrol", + "packetSize" : networkParams['packetSize'], + "link_bw" : networkParams['link_bw'], + "input_buf_size" : networkParams['input_buf_size'], + "output_buf_size" : networkParams['output_buf_size'], + "rxMatchDelay_ns" : 100, + "txDelay_ns" : 50, + "nic2host_lat" : "150ns", + "useSimpleMemoryModel" : 0, +# simpleMemoryModel.verboseMask: values +#define BUS_WIDGET_MASK 1<<1 +#define CACHE_MASK 1<<2 +#define LOAD_MASK 1<<3 +#define MEM_MASK 1<<4 +#define MUX_MASK 1<<5 +#define STORE_MASK 1<<6 +#define THREAD_MASK 1<<7 +#define BUS_BRIDGE_MASK 1<<8 + "simpleMemoryModel.verboseLevel" : 0, + "simpleMemoryModel.verboseMask" : -1, + + "simpleMemoryModel.memNumSlots" : 32, + "simpleMemoryModel.memReadLat_ns" : 150, + "simpleMemoryModel.memWriteLat_ns" : 40, + + "simpleMemoryModel.hostCacheUnitSize" : 32, + "simpleMemoryModel.hostCacheNumMSHR" : 32, + "simpleMemoryModel.hostCacheLineSize" : 64, + + "simpleMemoryModel.widgetSlots" : 32, + + "simpleMemoryModel.nicNumLoadSlots" : 16, + "simpleMemoryModel.nicNumStoreSlots" : 16, + + "simpleMemoryModel.nicHostLoadSlots" : 1, + "simpleMemoryModel.nicHostStoreSlots" : 1, + + "simpleMemoryModel.busBandwidth_Gbs" : 7.8, + "simpleMemoryModel.busNumLinks" : 8, + "simpleMemoryModel.detailedModel.name" : "firefly.detailedInterface", + "maxRecvMachineQsize" : 100, + "maxSendMachineQsize" : 100, + + "useSimpleSSD": 1, + "simpleSSD.nSSDsPerNode": 2, + "simpleSSD.queuesCountPerSSD": 4, + "simpleSSD.readBandwidthPerSSD_GBps": 6.25, + "simpleSSD.writeBandwidthPerSSD_GBps": 6.25, + "simpleSSD.readOverheadLatency_ns": 500, + "simpleSSD.writeOverheadLatency_ns": 500, + + + #"numVNs" : 7, + + #"getHdrVN" : 1, + #"getRespSmallVN" : 2, + #"getRespLargeVN" : 3, + #"getRespSize" : 15000, + + #"shmemAckVN": 1 , + #"shmemGetReqVN": 2, + #"shmemGetLargeVN": 3, + #"shmemGetSmallVN": 4, + #"shmemGetThresholdLength": 8, + #"shmemPutLargeVN": 5, + #"shmemPutSmallVN": 6, + #"shmemPutThresholdLength": 8, + +} + +emberParams = { + "os.module" : "firefly.hades", + "os.name" : "hermesParams", + "api.0.module" : "firefly.hadesMP", + "api.1.module" : "firefly.hadesSHMEM", + "api.2.module" : "firefly.hadesMisc", + "api.3.module" : "firefly.hadesNetworkIO", + 'firefly.hadesSHMEM.verboseLevel' : 0, + 'firefly.hadesSHMEM.verboseMask' : -1, + 'firefly.hadesSHMEM.enterLat_ns' : 7, + 'firefly.hadesSHMEM.returnLat_ns' : 7, + 'firefly.hadesNetworkIO.verboseLevel' : 0, + 'firefly.hadesNetworkIO.verboseMask' : -1, + "verbose" : 0, +} + +hermesParams = { + "hermesParams.detailedCompute.name" : "thornhill.SingleThread", + "hermesParams.memoryHeapLink.name" : "thornhill.MemoryHeapLink", + "hermesParams.nicModule" : "firefly.VirtNic", + + "hermesParams.functionSM.defaultEnterLatency" : 30000, + "hermesParams.functionSM.defaultReturnLatency" : 30000, + + #"hermesParams.functionSM.smallCollectiveVN" : 1, + #"hermesParams.functionSM.smallCollectiveSize" : 8, + + #"hermesParams.ctrlMsg.rendezvousVN" : 1, + #"hermesParams.ctrlMsg.ackVN" : 1, + + "hermesParams.ctrlMsg.shortMsgLength" : 12000, + "hermesParams.ctrlMsg.matchDelay_ns" : 150, + + "hermesParams.ctrlMsg.txSetupMod" : "firefly.LatencyMod", + "hermesParams.ctrlMsg.txSetupModParams.range.0" : "0-:130ns", + + "hermesParams.ctrlMsg.rxSetupMod" : "firefly.LatencyMod", + "hermesParams.ctrlMsg.rxSetupModParams.range.0" : "0-:100ns", + + "hermesParams.ctrlMsg.txMemcpyMod" : "firefly.LatencyMod", + "hermesParams.ctrlMsg.txMemcpyModParams.op" : "Mult", + "hermesParams.ctrlMsg.txMemcpyModParams.range.0" : "0-:344ps", + + "hermesParams.ctrlMsg.rxMemcpyMod" : "firefly.LatencyMod", + "hermesParams.ctrlMsg.rxMemcpyModParams.op" : "Mult", + "hermesParams.ctrlMsg.rxMemcpyModParams.range.0" : "0-:344ps", + + "hermesParams.ctrlMsg.sendAckDelay_ns" : 0, + "hermesParams.ctrlMsg.regRegionBaseDelay_ns" : 3000, + "hermesParams.ctrlMsg.regRegionPerPageDelay_ns" : 100, + "hermesParams.ctrlMsg.regRegionXoverLength" : 4096, + "hermesParams.loadMap.0.start" : 0, + "hermesParams.loadMap.0.len" : 2, +} diff --git a/src/sst/elements/firefly/Makefile.am b/src/sst/elements/firefly/Makefile.am index 3d42986ca2..01f2a2e738 100644 --- a/src/sst/elements/firefly/Makefile.am +++ b/src/sst/elements/firefly/Makefile.am @@ -161,7 +161,16 @@ libfirefly_la_SOURCES = \ nicUnitPool.h \ thingHeap.h \ nodePerf.h \ - pyfirefly.py + storageModel/simpleSSD.h \ + storageModel/simpleSSD.cc \ + hadesNetworkIO.h \ + hadesNetworkIO.cc \ + nicNetworkSendEntry.h \ + nicNetworkIO.cc \ + nicNetworkIO.h \ + nicNetworkIOStream.cc \ + nicNetworkIOStream.h \ + pyfirefly.py libfirefly_la_LDFLAGS = -module -avoid-version diff --git a/src/sst/elements/firefly/hadesNetworkIO.cc b/src/sst/elements/firefly/hadesNetworkIO.cc new file mode 100644 index 0000000000..dea7b876da --- /dev/null +++ b/src/sst/elements/firefly/hadesNetworkIO.cc @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#include "sst_config.h" + +#include "hades.h" +#include "hadesNetworkIO.h" + +using namespace SST::Firefly; + +HadesNetworkIO::HadesNetworkIO(ComponentId_t id, Params& params) : + Hermes::NetworkIO::Interface(id), + m_nicPtr(NULL) +{ + m_dbg.init("@t:HadesNetworkIO::@p():@l ", + params.find("verboseLevel",0), + params.find("verboseMask",-1), + Output::STDOUT ); + + auto parse = [](const std::string& s) + { + std::vector v; + std::istringstream ss(s); + for(int i; ss >> i; ss.ignore()) v.push_back(i); + return v; + }; + m_numSsdNodes = params.find("numStorageNodes", 0); + m_ssd_start_node = params.find("ssd_start_node", 0); + + for (int64_t i = 0; i < m_numSsdNodes; i++) { + m_storageNodesList.push_back(m_ssd_start_node + i); + } + + m_storageNodeCapacity = params.find("storageNodeCapacity", "1GiB").getRoundedValue(); +} + +void HadesNetworkIO::setOS( Hermes::OS* os ) +{ + m_osPtr = dynamic_cast(os); + assert(m_osPtr); + m_nicPtr = m_osPtr->getNic(); +} + +void HadesNetworkIO::setup() +{ +} + + +void HadesNetworkIO::networkIORead(Hermes::Vaddr dest, uint64_t offset, uint64_t length, Callback callback) +{ + m_dbg.verbose(CALL_INFO, 1, 0, "network_read: dest=%lx offset=%lu length=%lu \n", + dest, offset, length); + int targetNid = calcTargetNid(offset); + m_nicPtr->networkIORead(targetNid, dest, length, callback); +} + +void HadesNetworkIO::networkIOWrite(uint64_t offset, Hermes::Vaddr src, uint64_t length, Callback callback) +{ + m_dbg.verbose(CALL_INFO, 1, 0, "network_write: offset=%lu src=%lx length=%lu \n", + offset, src, length); + int targetNid = calcTargetNid(offset); + m_nicPtr->networkIOWrite(targetNid, src, length, callback); + +} + +int64_t HadesNetworkIO::calcTargetNid(int64_t offset) +{ + assert(m_storageNodesList.size() > 0 && "No storage nodes defined in storageNodesList"); + + int nodeIndex = (offset/m_storageNodeCapacity)%m_storageNodesList.size(); + return m_storageNodesList.at(nodeIndex); +} \ No newline at end of file diff --git a/src/sst/elements/firefly/hadesNetworkIO.h b/src/sst/elements/firefly/hadesNetworkIO.h new file mode 100644 index 0000000000..6489212b18 --- /dev/null +++ b/src/sst/elements/firefly/hadesNetworkIO.h @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + +#include +#include +#include + +namespace SST { +namespace Firefly { + +class VirtNic; + +class HadesNetworkIO : public Hermes::NetworkIO::Interface { + + public: + SST_ELI_REGISTER_SUBCOMPONENT( + HadesNetworkIO, + "firefly", + "hadesNetworkIO", + SST_ELI_ELEMENT_VERSION(1,0,0), + "Network Storage I/O API", + SST::Hermes::Interface + ) + SST_ELI_DOCUMENT_PARAMS( + {"storageNodesList", "List of storage nodes Ids used in simulation", ""}, + {"storageNodeCapacity", "The capacity of a storage node", "1 GiB"}, + {"verboseLevel","Sets the level of debug verbosity","0"}, + {"verboseMask","Sets the debug mask","-1"}, + ) + + HadesNetworkIO(ComponentId_t id, Params& params); + + ~HadesNetworkIO() {} + + void setOS( Hermes::OS* os ) override; + + void setup() override; + + std::string getName() override { return "networkIO"; } + + std::string getType() override { return "networkIO"; } + + void networkIORead(Hermes::Vaddr dest, uint64_t offset, uint64_t length, + Callback callback) override; + + void networkIOWrite(uint64_t offset, Hermes::Vaddr src, uint64_t length, + Callback callback) override; + + private: + Hades* m_osPtr; + VirtNic* m_nicPtr; + SST::Output m_dbg; + std::vector m_storageNodesList; + int64_t m_numSsdNodes; + int64_t m_ssd_start_node; + int64_t m_storageNodeCapacity; + + int64_t calcTargetNid(int64_t offset); +}; + +} // namespace Firefly +} // namespace SST + diff --git a/src/sst/elements/firefly/nic.cc b/src/sst/elements/firefly/nic.cc index e6a896d2b9..4e5430beb5 100644 --- a/src/sst/elements/firefly/nic.cc +++ b/src/sst/elements/firefly/nic.cc @@ -9,6 +9,10 @@ // See the file CONTRIBUTORS.TXT in the top level directory // of the distribution for more information. // +// Portions copyright (c) 2026, Hewlett Packard Enterprise Development LP +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause +// // This file is part of the SST software package. For license // information, see the LICENSE file in the top level directory of the // distribution. @@ -179,6 +183,8 @@ Nic::Nic(ComponentId_t id, Params ¶ms) : Params shmemParams = params.get_scoped_params( "shmem" ); m_shmem = new Shmem( *this, shmemParams, m_myNodeId, m_num_vNics, m_dbg, getDelay_ns(), getDelay_ns() ); + Params networkIOParams = params.get_scoped_params( "network" ); + m_networkIO = new NetworkIO( *this, networkIOParams, m_dbg ); size_t FAM_memSizeBytes = params.find("FAM_memSize" ).getRoundedValue(); if ( FAM_memSizeBytes ) { if ( printConfig ) { @@ -309,6 +315,13 @@ Nic::Nic(ComponentId_t id, Params ¶ms) : m_useDetailedCompute = params.find("useDetailed", false ); } + if(params.find("useSimpleSSD", 0)) + { + Params ssdParams = params.get_scoped_params("simpleSSD"); + m_simpleSSDPtr = dynamic_cast(loadAnonymousSubComponent("firefly.SimpleSSD","SimpleSSD", 0, ComponentInfo::SHARE_NONE, ssdParams )); + assert(m_simpleSSDPtr && "Failed to load SimpleSSD subcomponent in NIC\n"); + } + m_sentByteCount = registerStatistic("sentByteCount"); m_rcvdByteCount = registerStatistic("rcvdByteCount"); m_sentPkts = registerStatistic("sentPkts"); @@ -328,6 +341,7 @@ Nic::Nic(ComponentId_t id, Params ¶ms) : Nic::~Nic() { delete m_shmem; + delete m_networkIO; delete m_unitPool; delete m_linkSendWidget; delete m_linkRecvWidget; @@ -392,6 +406,10 @@ void Nic::handleVnicEvent( Event* ev, int id ) m_shmem->handleEvent( static_cast(event), id ); break; + case NicCmdBaseEvent::NetworkIO: + m_networkIO->handleEvent( static_cast(event), id ); + break; + default: assert(0); } @@ -451,6 +469,9 @@ void Nic::handleVnicEvent2( Event* ev, int id ) case NicCmdBaseEvent::Shmem: m_shmem->handleNicEvent2( static_cast(event), id ); break; + case NicCmdBaseEvent::NetworkIO: + m_networkIO->handleEvent( static_cast(event), id ); + break; default: assert(0); } diff --git a/src/sst/elements/firefly/nic.h b/src/sst/elements/firefly/nic.h index c4d363b3c7..c154448015 100644 --- a/src/sst/elements/firefly/nic.h +++ b/src/sst/elements/firefly/nic.h @@ -9,6 +9,10 @@ // See the file CONTRIBUTORS.TXT in the top level directory // of the distribution for more information. // +// Portions copyright (c) 2026, Hewlett Packard Enterprise Development LP +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause +// // This file is part of the SST software package. For license // information, see the LICENSE file in the top level directory of the // distribution. @@ -27,6 +31,7 @@ #include #include "sst/elements/hermes/shmemapi.h" +#include "sst/elements/hermes/networkIOapi.h" #include "sst/elements/thornhill/detailedCompute.h" #include "ioVec.h" #include "merlinEvent.h" @@ -34,6 +39,8 @@ #include "memoryModel/simpleMemoryModel.h" #include "memoryModel/detailedInterface.h" +#include "storageModel/simpleSSD.h" + #define CALL_INFO_LAMBDA __LINE__, __FILE__ namespace SST { @@ -53,6 +60,7 @@ namespace Firefly { #define NIC_DBG_RECV_STREAM (1<<8) #define NIC_DBG_RECV_MOVE (1<<9) #define NIC_DBG_LINK_CTRL (1<<10) +#define NIC_DBG_NETWORKIO (1<<11) class Nic : public SST::Component { @@ -124,6 +132,8 @@ class Nic : public SST::Component { { "dmaContentionMult", "set the DMA contention mult", "100"}, { "useDetailed", "Use detailed compute model", "false"}, + + { "useSimpleSSD", "Use simple SSD model for Network I/O simulations", "false"}, ) /* PARAMS @@ -206,7 +216,7 @@ class Nic : public SST::Component { private: struct __attribute__ ((packed)) MsgHdr { - enum Op : unsigned char { Msg, Rdma, Shmem } op; + enum Op : unsigned char { Msg, Rdma, Shmem, NetworkIO } op; }; struct __attribute__ ((packed)) MatchMsgHdr { @@ -214,6 +224,11 @@ class Nic : public SST::Component { int tag; }; + struct __attribute__ ((packed)) NetworkIOMsgHdr { + enum Op { Read, Write }; + unsigned char op; + }; + struct __attribute__ ((packed)) ShmemMsgHdr { ShmemMsgHdr() : op2(0) {} uint64_t vaddr; @@ -323,12 +338,14 @@ class Nic : public SST::Component { }; + #include "nicNetworkIO.h" #include "nicVirtNic.h" #include "nicShmem.h" #include "nicShmemMove.h" #include "nicEntryBase.h" #include "nicSendEntry.h" #include "nicShmemSendEntry.h" + #include "nicNetworkIOSendEntry.h" #include "nicRecvEntry.h" #include "nicSendMachine.h" #include "nicRecvMachine.h" @@ -519,6 +536,7 @@ struct X { DetailedInterface* m_detailedInterface; bool m_useDetailedCompute; Shmem* m_shmem; + NetworkIO* m_networkIO; SimTime_t m_nic2host_lat_ns; SimTime_t m_shmemRxDelay_ns; @@ -665,6 +683,13 @@ struct X { int m_shmemPutLargeVN; int m_shmemPutSmallVN; size_t m_shmemPutThresholdLength; + + private: + SimpleSSD* m_simpleSSDPtr; + + public: + SimpleSSD* getSimpleSSDPtr() { return m_simpleSSDPtr; } + }; } // namesapce Firefly diff --git a/src/sst/elements/firefly/nicEvents.h b/src/sst/elements/firefly/nicEvents.h index f008bd88e3..5bc00ed053 100644 --- a/src/sst/elements/firefly/nicEvents.h +++ b/src/sst/elements/firefly/nicEvents.h @@ -9,6 +9,10 @@ // See the file CONTRIBUTORS.TXT in the top level directory // of the distribution for more information. // +// Portions copyright (c) 2026, Hewlett Packard Enterprise Development LP +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause +// // This file is part of the SST software package. For license // information, see the LICENSE file in the top level directory of the // distribution. @@ -50,7 +54,7 @@ class NicInitEvent : public Event { class NicCmdBaseEvent : public Event { public: - enum Type { Shmem, Msg } base_type; + enum Type { Shmem, Msg, NetworkIO } base_type; NicCmdBaseEvent( Type type ) : Event(), base_type(type) {} @@ -400,7 +404,7 @@ class NicCmdEvent : public NicCmdBaseEvent { class NicRespBaseEvent : public Event { public: - enum Type { Shmem, Msg } base_type; + enum Type { Shmem, Msg, NetworkIO } base_type; NicRespBaseEvent( Type type ) : Event(), base_type(type) {} NotSerializable(NicCmdEvent) @@ -451,6 +455,35 @@ class NicShmemValueRespEvent : public NicShmemRespBaseEvent { NotSerializable(NicShmemValueRespEvent) }; +class NicNetworkIORespBaseEvent : public NicRespBaseEvent { + public: + + NicNetworkIORespBaseEvent( ) : + NicRespBaseEvent( NetworkIO ) {} + + virtual ~NicNetworkIORespBaseEvent() {} + + virtual void callback() = 0; + NotSerializable(NicNetworkIORespBaseEvent) +}; + +class NicNetworkIORespEvent : public NicNetworkIORespBaseEvent { + + public: + typedef std::function Callback; + + NicNetworkIORespEvent( Callback callback, int retval ) : + NicNetworkIORespBaseEvent( ), m_callback(callback), m_retval(retval) {} + + void callback() override { m_callback(m_retval); } + + private: + Callback m_callback; + int m_retval; + + NotSerializable(NicNetworkIORespEvent) +}; + class NicRespEvent : public NicRespBaseEvent { public: @@ -494,6 +527,81 @@ class NicRespEvent : public NicRespBaseEvent { NotSerializable(NicRespEvent) }; +//============================================================================= +// NetworkIO Command Events +//============================================================================= + +class NicNetworkIOCmdEvent : public NicCmdBaseEvent { +public: + enum Type { + NetworkIORead, + NetworkIOWrite + }; + + NicNetworkIOCmdEvent(Type type, std::function callback) + : NicCmdBaseEvent(NetworkIO), type(type), m_callback(callback) {} + + virtual ~NicNetworkIOCmdEvent() {} + + Type type; + + std::string getTypeStr() { + switch(type) { + case NetworkIORead: return "NetworkIORead"; + case NetworkIOWrite: return "NetworkIOWrite"; + default: return "Unknown"; + } + } + + std::function getCallback() { return m_callback; } + +private: + std::function m_callback; + + NotSerializable(NicNetworkIOCmdEvent) +}; + + + +class NicNetworkIOReadCmdEvent : public NicNetworkIOCmdEvent { +public: + NicNetworkIOReadCmdEvent(int targetNid, Hermes::Vaddr dest, size_t len, std::function callback) + : NicNetworkIOCmdEvent(NetworkIORead, callback), m_targetNid(targetNid), m_dest(dest), m_len(len) {} + + ~NicNetworkIOReadCmdEvent() {} + + int getTargetNid() const { return m_targetNid; } + Hermes::Vaddr getDest() const { return m_dest; } + size_t getLen() const { return m_len; } + +private: + int m_targetNid; + Hermes::Vaddr m_dest; + size_t m_len; + + NotSerializable(NicNetworkIOReadCmdEvent) +}; + + +class NicNetworkIOWriteCmdEvent : public NicNetworkIOCmdEvent { +public: + NicNetworkIOWriteCmdEvent(int targetNid, Hermes::Vaddr src, size_t len, std::function callback) + : NicNetworkIOCmdEvent(NetworkIOWrite, callback), m_targetNid(targetNid), m_src(src), m_len(len) {} + + ~NicNetworkIOWriteCmdEvent() {} + + int getTargetNid() const { return m_targetNid; } + Hermes::Vaddr getSrc() const { return m_src; } + size_t getLen() const { return m_len; } + +private: + int m_targetNid; + Hermes::Vaddr m_src; + size_t m_len; + + NotSerializable(NicNetworkIOWriteCmdEvent) +}; + #if defined(__clang__) #pragma clang diagnostic pop #endif diff --git a/src/sst/elements/firefly/nicNetworkIO.cc b/src/sst/elements/firefly/nicNetworkIO.cc new file mode 100644 index 0000000000..d4102898cf --- /dev/null +++ b/src/sst/elements/firefly/nicNetworkIO.cc @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#include "sst_config.h" +#include "nic.h" + +using namespace SST; +using namespace SST::Firefly; + +// ======================================================================== +// NetworkIO Constructor - Initialize storage configuration from parameters +// ======================================================================== +Nic::NetworkIO::NetworkIO(Nic& nic, Params& params, Output& output) : + m_nic(nic), m_dbg(output) +{ + m_prefix = "@t:" + std::to_string(nic.getNodeId()) + ":Nic::NetworkIO::@p():@l "; + + + // Initialize per-core pending operation tracking (same size as vNIC count) + m_pendingOps.resize(nic.getNum_vNics()); + + m_dbg.verbosePrefix(prefix().c_str(), CALL_INFO, 1, NIC_DBG_NETWORKIO, + "storage config:cores=%d\n", nic.getNum_vNics()); +} + +// ======================================================================== +// Main Event Handler - Dispatch to READ or WRITE +// ======================================================================== +void Nic::NetworkIO::handleEvent(NicNetworkIOCmdEvent* event, int id) +{ + m_dbg.verbosePrefix(prefix().c_str(), CALL_INFO, 1, NIC_DBG_NETWORKIO, + "core=%d `%s`\n", id, event->getTypeStr().c_str()); + + // Dispatch based on operation type + switch (event->type) { + case NicNetworkIOCmdEvent::NetworkIORead: + handleNetworkIORead(static_cast(event), id); + break; + case NicNetworkIOCmdEvent::NetworkIOWrite: + handleNetworkIOWrite(static_cast(event), id); + break; + default: + m_dbg.fatal(CALL_INFO, -1, "core=%d Unknown NetworkIOCmdEvent type %d\n", id, event->type); + } +} + +// ======================================================================== +// NetworkIO READ Handler - Get data from remote storage node +// ======================================================================== +void Nic::NetworkIO::handleNetworkIORead(NicNetworkIOReadCmdEvent* event, int id) +{ + int targetNid = event->getTargetNid(); + Hermes::Vaddr destAddr = event->getDest(); + size_t length = event->getLen(); + m_dbg.verbosePrefix(prefix().c_str(), CALL_INFO, 1, NIC_DBG_NETWORKIO, + "READ core=%d targetNid=%d dest=%#lx len=%zu \n", + id, targetNid, destAddr, length); + + auto callback = event->getCallback(); + + NetworkIOStorageReadEntry* entry = new NetworkIOStorageReadEntry( + id, // local vNic + targetNid, // destination storage node + destAddr, // local buffer for received data + 0, // local offset (not used in new API) + length // transfer size + ); + + Nic::RespKey_t respKey = m_nic.genRespKey(new std::function([=]() { + m_dbg.verbosePrefix(m_prefix.c_str(), CALL_INFO_LAMBDA, "handleNetworkIORead", + 1, NIC_DBG_NETWORKIO, "core=%d ACK received\n", id); + m_nic.getVirtNic(id)->notifyNetworkIO(m_nic.m_nic2host_lat_ns, callback, 0); + })); + entry->setRespKey(respKey); + + m_nic.qSendEntry(entry); + + delete event; +} + +// ======================================================================== +// NetworkIO WRITE Handler - Put data to remote storage node +// ======================================================================== +void Nic::NetworkIO::handleNetworkIOWrite(NicNetworkIOWriteCmdEvent* event, int id) +{ + int targetNid = event->getTargetNid(); + Hermes::Vaddr srcAddr = event->getSrc(); + size_t length = event->getLen(); + m_dbg.verbosePrefix(prefix().c_str(), CALL_INFO, 1, NIC_DBG_NETWORKIO, + "WRITE core=%d targetNid=%d src=%#lx len=%zu \n", + id, targetNid, srcAddr, length); + + auto callback = event->getCallback(); + + NetworkIOStorageWriteEntry* entry = new NetworkIOStorageWriteEntry( + id, // local vNic + targetNid, // destination storage node + 0, // remote offset (not used in new API) + srcAddr, // local source buffer + length // transfer size + ); + + Nic::RespKey_t respKey = m_nic.genRespKey(new std::function([=]() { + m_dbg.verbosePrefix(m_prefix.c_str(), CALL_INFO_LAMBDA, "handleNetworkIOWrite", + 1, NIC_DBG_NETWORKIO, "core=%d ACK received\n", id); + m_nic.getVirtNic(id)->notifyNetworkIO(m_nic.m_nic2host_lat_ns, callback, 0); + })); + entry->setRespKey(respKey); + + m_nic.qSendEntry(entry); + + delete event; +} \ No newline at end of file diff --git a/src/sst/elements/firefly/nicNetworkIO.h b/src/sst/elements/firefly/nicNetworkIO.h new file mode 100644 index 0000000000..1a1f2a775a --- /dev/null +++ b/src/sst/elements/firefly/nicNetworkIO.h @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +class NetworkIO { + public: + NetworkIO(Nic& nic, Params& params, Output& output); + ~NetworkIO() {} + + // Main event handler - dispatches to READ, WRITE, or QUIET handlers + void handleEvent(NicNetworkIOCmdEvent* event, int id); + + private: + // Network storage operation handlers + void handleNetworkIORead(NicNetworkIOReadCmdEvent* event, int id); + void handleNetworkIOWrite(NicNetworkIOWriteCmdEvent* event, int id); + + struct PendingOps { + PendingOps() : readCount(0), writeCount(0) {} + int readCount; // Number of pending non-blocking reads + int writeCount; // Number of pending non-blocking writes + }; + + void incPendingReads(int id) { + ++m_pendingOps[id].readCount; + } + + void decPendingReads(int id) { + PendingOps& ops = m_pendingOps[id]; + assert(ops.readCount > 0); + --ops.readCount; + } + + void incPendingWrites(int id) { + ++m_pendingOps[id].writeCount; + } + + void decPendingWrites(int id) { + PendingOps& ops = m_pendingOps[id]; + assert(ops.writeCount > 0); + --ops.writeCount; + } + + // Core references + Nic& m_nic; + Output& m_dbg; + std::string m_prefix; + + // Per-core tracking of pending non-blocking operations + std::vector m_pendingOps; + + std::string prefix() { return m_prefix; } +}; diff --git a/src/sst/elements/firefly/nicNetworkIOSendEntry.h b/src/sst/elements/firefly/nicNetworkIOSendEntry.h new file mode 100644 index 0000000000..d2946e34ea --- /dev/null +++ b/src/sst/elements/firefly/nicNetworkIOSendEntry.h @@ -0,0 +1,153 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +// ======================================================================== +// NetworkIO Storage SendEntry Classes +// ======================================================================== +// Three types: READ, WRITE, and ACK + +// Base class for all NetworkIO storage SendEntry types +class NetworkIOSendEntryBase : public SendEntryBase { + public: + NetworkIOSendEntryBase(int local_vNic, int streamNum) : + SendEntryBase(local_vNic, streamNum) {} + ~NetworkIOSendEntryBase() {} + + MsgHdr::Op getOp() { return MsgHdr::NetworkIO; } // NetworkIO storage message type + void* hdr() { return nullptr; } + size_t hdrSize() { return 0; } + int dst_vNic() { return 0; } // Storage nodes use vNic 0 + int vn() { return 0; } // Virtual NetworkIO 0 +}; + +// ======================================================================== +// NetworkIO Storage READ Entry +// ======================================================================== +class NetworkIOStorageReadEntry : public NetworkIOSendEntryBase { + public: + NetworkIOStorageReadEntry(int local_vNic, int dst_node, Hermes::Vaddr dest, + Hermes::Vaddr src, size_t len) : + NetworkIOSendEntryBase(local_vNic, 0), + m_dst_node(dst_node), // Target storage node (calculated from offset) + m_dest(dest), // Local buffer to receive data + m_src(src), // Remote storage offset + m_len(len), // Transfer size + m_respKey(0) // Response key for ACK matching + {} + + // Build READ request packet + // Packet format: [NetworkIOMsgHdr::Read][src_offset][dest_addr][length][respKey] + void copyOut(Output& dbg, int numBytes, FireflyNetworkEvent& event, std::vector& vec) override + { + // READ request: NO MemOp - just sending request, no data to read from host + + // Append NetworkIOMsgHdr with READ operation type + Nic::NetworkIOMsgHdr netHdr; + netHdr.op = Nic::NetworkIOMsgHdr::Read; + event.bufAppend(&netHdr, sizeof(netHdr)); + + // Append operation parameters + event.bufAppend(&m_src, sizeof(m_src)); // Storage offset + event.bufAppend(&m_dest, sizeof(m_dest)); // Local buffer + event.bufAppend(&m_len, sizeof(m_len)); + event.bufAppend(&m_respKey, sizeof(m_respKey)); + } + + bool isDone() override { return true; } // Packet sent, awaiting ACK + + size_t totalBytes() override { return m_len; } + int dest() override { return m_dst_node; } // Target storage node + + // Set response key for blocking operations (ACK matching) + void setRespKey(uint32_t key) { m_respKey = key; } + + private: + int m_dst_node; // Target storage node ID + Hermes::Vaddr m_dest; // Local buffer address + Hermes::Vaddr m_src; // Remote storage offset + size_t m_len; // Transfer length + uint32_t m_respKey; // Response key for ACK matching +}; + +// ======================================================================== +// NetworkIO Storage WRITE Entry +// ======================================================================== +class NetworkIOStorageWriteEntry : public NetworkIOSendEntryBase { + public: + NetworkIOStorageWriteEntry(int local_vNic, int dst_node, Hermes::Vaddr dest, + Hermes::Vaddr src, size_t len) : + NetworkIOSendEntryBase(local_vNic, 0), + m_dst_node(dst_node), // Target storage node + m_dest(dest), // Remote storage offset + m_src(src), // Local source buffer + m_len(len), // Transfer size + m_respKey(0) // Response key for ACK matching + {} + + // Build WRITE request packet + // Packet format: [NetworkIOMsgHdr::Write][dest_offset][src_addr][length][respKey] + void copyOut(Output& dbg, int numBytes, FireflyNetworkEvent& event, std::vector& vec) override + { + // WRITE request: NO MemOp - just sending request + + // Append NetworkIOMsgHdr with WRITE operation type + Nic::NetworkIOMsgHdr netHdr; + netHdr.op = Nic::NetworkIOMsgHdr::Write; + event.bufAppend(&netHdr, sizeof(netHdr)); + + // Append operation parameters + event.bufAppend(&m_dest, sizeof(m_dest)); // Storage offset + event.bufAppend(&m_src, sizeof(m_src)); // Local buffer + event.bufAppend(&m_len, sizeof(m_len)); + event.bufAppend(&m_respKey, sizeof(m_respKey)); + } + + bool isDone() override { return true; } // Packet sent, awaiting ACK + + size_t totalBytes() override { return m_len; } + int dest() override { return m_dst_node; } + bool isWriteOp() override { return true; } // This is a WRITE operation + + void setRespKey(uint32_t key) { m_respKey = key; } + + private: + int m_dst_node; + Hermes::Vaddr m_dest; + Hermes::Vaddr m_src; + size_t m_len; + uint32_t m_respKey; +}; + +// ======================================================================== +// NetworkIO Storage ACK Entry +// ======================================================================== +// Sends ACK packet back to requester after storage DMA completes +// Small packet containing only: [isAck=true][respKey] +// Requester matches respKey to invoke stored callback +class NetworkIOAckSendEntry : public NetworkIOSendEntryBase { + public: + NetworkIOAckSendEntry(int local_vNic, int dest_node, uint32_t respKey) : + NetworkIOSendEntryBase(local_vNic, 0), + m_dest_node(dest_node), // Requester node to send ACK to + m_respKey(respKey) // Response key for callback matching + { + m_isAck = true; // Flag to identify ACK packets + } + + // Build ACK packet + // Packet format: [isAck=true][respKey] + void copyOut(Output& dbg, int numBytes, FireflyNetworkEvent& event, std::vector& vec) override + { + event.bufAppend(&m_isAck, sizeof(m_isAck)); + event.bufAppend(&m_respKey, sizeof(m_respKey)); + } + + bool isDone() override { return true; } // Single packet + size_t totalBytes() override { return 0; } // No data payload + int dest() override { return m_dest_node; } // Back to requester + + private: + int m_dest_node; // Original requester node + uint32_t m_respKey; // Response key to match with stored callback + bool m_isAck; // ACK flag +}; diff --git a/src/sst/elements/firefly/nicNetworkIOStream.cc b/src/sst/elements/firefly/nicNetworkIOStream.cc new file mode 100644 index 0000000000..aace405af3 --- /dev/null +++ b/src/sst/elements/firefly/nicNetworkIOStream.cc @@ -0,0 +1,189 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#include "sst_config.h" +#include "nic.h" + +using namespace SST; +using namespace SST::Firefly; + +// ======================================================================== +// Network Storage Stream Constructor +// ======================================================================== +// Handles incoming network storage packets on storage nodes +// Two packet types: +// 1. Storage operations (READ/WRITE) - perform DMA and send ACK +// 2. ACK responses - match respKey and invoke stored callback +Nic::RecvMachine::NetworkIOStream::NetworkIOStream( Output& output, Ctx* ctx, + int srcNode, int srcPid, int destPid, FireflyNetworkEvent* ev ) : + StreamBase( output, ctx, srcNode, srcPid, destPid ), + m_offset(0), + m_length(0) +{ + // Pop the MsgHdr to get to packet payload + ev->bufPop(sizeof(MsgHdr)); + + unsigned char* bufPtr = (unsigned char*)ev->bufPtr(); + + // Peek at first byte to determine packet type + // ACK packets are small: only [isAck=true][respKey] + bool isAck = false; + if (ev->bufSize() == sizeof(bool) + sizeof(uint32_t)) { + memcpy(&isAck, bufPtr, sizeof(isAck)); + + if (isAck) { + // Handle ACK packet (requester side) + processAck(ev, bufPtr); + delete ev; + setDone(); + return; + } + } + + // Handle storage operation packet (storage node side) + processStorageOp(ev, bufPtr); +} + +// ======================================================================== +// Process ACK Packet +// ======================================================================== +// Matches respKey to invoke stored callback from m_respKeyMap +void Nic::RecvMachine::NetworkIOStream::processAck(FireflyNetworkEvent* ev, unsigned char* bufPtr) +{ + bufPtr += sizeof(bool); + ev->bufPop(sizeof(bool)); + + // Extract response key from ACK packet + uint32_t respKey = 0; + memcpy(&respKey, bufPtr, sizeof(respKey)); + ev->bufPop(sizeof(respKey)); + + m_dbg.debug(CALL_INFO,1,NIC_DBG_RECV_STREAM, + "ACK received: respKey=%u\n", respKey); + + // Retrieve and invoke stored callback + // getRespKeyValue() removes entry from m_respKeyMap + auto* respCallback = static_cast*>( + m_ctx->nic().getRespKeyValue(respKey)); + if (respCallback) { + (*respCallback)(); // Invoke user callback + delete respCallback; + } +} + +// ======================================================================== +// Process Storage Operation (Storage Node Side) +// ======================================================================== +// Handles READ or WRITE operations sent to storage node +// +// Packet format: +// [NetworkMsgHdr][offset][src][length][respKey] +// +// Flow: +// 1. Extract operation parameters +// 2. Submit DMA: READ=BusDmaFromHost, WRITE=BusDmaToHost +// 3. Send ACK with respKey when DMA completes +void Nic::RecvMachine::NetworkIOStream::processStorageOp(FireflyNetworkEvent* ev, unsigned char* bufPtr) +{ + // Extract NetworkIOMsgHdr to determine operation type + Nic::NetworkIOMsgHdr netHdr; + memcpy(&netHdr, bufPtr, sizeof(netHdr)); + bufPtr += sizeof(netHdr); + ev->bufPop(sizeof(netHdr)); + + // Extract operation parameters from packet + memcpy(&m_offset, bufPtr, sizeof(m_offset)); + bufPtr += sizeof(m_offset); + memcpy(&m_src, bufPtr, sizeof(m_src)); + bufPtr += sizeof(m_src); + memcpy(&m_length, bufPtr, sizeof(m_length)); + bufPtr += sizeof(m_length); + ev->bufPop(sizeof(m_offset) + sizeof(m_src) + sizeof(m_length)); + + // Extract respKey (always present - ACKs always sent) + uint32_t respKey; + memcpy(&respKey, bufPtr, sizeof(respKey)); + ev->bufPop(sizeof(respKey)); + + m_dbg.debug(CALL_INFO,1,NIC_DBG_RECV_STREAM, + "op=%u offset=%lu length=%lu respKey=%u\n", + netHdr.op, m_offset, m_length, respKey); + + // Submit DMA operation based on operation type + std::vector* vec = new std::vector; + + if (netHdr.op == Nic::NetworkIOMsgHdr::Read) { + // READ: BusDmaFromHost (load data FROM storage to send to requester) + vec->push_back(MemOp(m_offset, m_length, MemOp::Op::BusDmaFromHost)); + } else { + // WRITE: BusDmaToHost (store data TO storage) + vec->push_back(MemOp(m_offset, m_length, MemOp::Op::BusDmaToHost)); + } + + // Use dmaRead() for READ, dmaWrite() for WRITE to select correct DMA engine + int unit = m_ctx->nic().allocNicRecvUnit(m_myPid); + + if (netHdr.op == Nic::NetworkIOMsgHdr::Read) { + // READ uses dmaRead() -> DetailedCompute[0] + auto callback = [=]() { + m_dbg.debug(CALL_INFO_LAMBDA,"NetworkIOStream",1,NIC_DBG_RECV_STREAM, + "DMA READ complete, sending ACK respKey=%u to node=%d\n", + respKey, m_srcNode); + + // Create ACK packet with matching respKey + NetworkIOAckSendEntry* ackEntry = new NetworkIOAckSendEntry( + m_myPid, m_srcNode, respKey + ); + m_ctx->nic().qSendEntry(ackEntry); + + delete ev; + setDone(); + }; + if(m_ctx->nic().getSimpleSSDPtr()!=NULL){ + std::cout<<"Current node id : "<nic().getNodeId()<nic().getSimpleSSDPtr(); + assert(m_ssd); + + std::cout<<"SSD Read for node Id : "<nic().getNodeId()<read(0, m_length, callback); + } + else{ + std::cout<<"DMA Read for node Id : "<nic().getNodeId()<nic().dmaRead(unit, m_myPid, vec,callback); + } + } + else { + // WRITE uses dmaWrite() -> DetailedCompute[1] + auto writeCallback = [=]() { + m_dbg.debug(CALL_INFO_LAMBDA,"NetworkIOStream",1,NIC_DBG_RECV_STREAM, + "DMA WRITE complete, sending ACK respKey=%u to node=%d\n", + respKey, m_srcNode); + + // Create ACK packet with matching respKey + NetworkIOAckSendEntry* ackEntry = new NetworkIOAckSendEntry( + m_myPid, m_srcNode, respKey + ); + m_ctx->nic().qSendEntry(ackEntry); + + delete ev; + setDone(); + }; + + if(m_ctx->nic().getSimpleSSDPtr()!=NULL){ + std::cout<<"Current node id : "<nic().getNodeId()<nic().getSimpleSSDPtr(); + assert(m_ssd); + + std::cout<<"SSD Write for node Id : "<nic().getNodeId()<write(0, m_length, writeCallback); + } + else{ + std::cout<<"DMA Write for node Id : "<nic().getNodeId()<nic().dmaWrite(unit, m_myPid, vec, writeCallback); + } + } +} diff --git a/src/sst/elements/firefly/nicNetworkIOStream.h b/src/sst/elements/firefly/nicNetworkIOStream.h new file mode 100644 index 0000000000..a94f534b25 --- /dev/null +++ b/src/sst/elements/firefly/nicNetworkIOStream.h @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +// NetworkStream: Handles incoming network storage operations +class NetworkIOStream : public StreamBase { + public: + NetworkIOStream( Output&, Ctx*, int srcNode, int srcPid, int destPid, FireflyNetworkEvent* ); + ~NetworkIOStream() { + m_dbg.debug(CALL_INFO,1,NIC_DBG_RECV_STREAM,"NetworkIOStream destroyed\n"); + } + + void setDone() { + m_ctx->deleteStream( this ); + } + + private: + void processAck(FireflyNetworkEvent* ev, unsigned char* bufPtr); + void processStorageOp(FireflyNetworkEvent* ev, unsigned char* bufPtr); + + uint64_t m_offset; // Storage offset + Hermes::Vaddr m_src; // Source buffer (from sender) + size_t m_length; // Operation length +}; diff --git a/src/sst/elements/firefly/nicRecvCtx.cc b/src/sst/elements/firefly/nicRecvCtx.cc index 2d0f412862..f670678576 100644 --- a/src/sst/elements/firefly/nicRecvCtx.cc +++ b/src/sst/elements/firefly/nicRecvCtx.cc @@ -9,6 +9,10 @@ // See the file CONTRIBUTORS.TXT in the top level directory // of the distribution for more information. // +// Portions copyright (c) 2026, Hewlett Packard Enterprise Development LP +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause +// // This file is part of the SST software package. For license // information, see the LICENSE file in the top level directory of the // distribution. @@ -38,6 +42,9 @@ Nic::RecvMachine::StreamBase* Nic::RecvMachine::Ctx::newStream( FireflyNetworkEv case MsgHdr::Shmem: return new ShmemStream( m_dbg, this, ev->getSrcNode(),ev->getSrcPid(), ev->getDestPid(), ev ); break; + case MsgHdr::NetworkIO: + return new NetworkIOStream( m_dbg, this, ev->getSrcNode(),ev->getSrcPid(), ev->getDestPid(), ev ); + break; } assert(0); } diff --git a/src/sst/elements/firefly/nicRecvMachine.h b/src/sst/elements/firefly/nicRecvMachine.h index 81d9661fb3..ed0557a9e6 100644 --- a/src/sst/elements/firefly/nicRecvMachine.h +++ b/src/sst/elements/firefly/nicRecvMachine.h @@ -9,6 +9,10 @@ // See the file CONTRIBUTORS.TXT in the top level directory // of the distribution for more information. // +// Portions copyright (c) 2026, Hewlett Packard Enterprise Development LP +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause +// // This file is part of the SST software package. For license // information, see the LICENSE file in the top level directory of the // distribution. @@ -43,6 +47,7 @@ class RecvMachine { #include "nicMsgStream.h" #include "nicRdmaStream.h" #include "nicShmemStream.h" + #include "nicNetworkIOStream.h" public: diff --git a/src/sst/elements/firefly/nicSendEntry.h b/src/sst/elements/firefly/nicSendEntry.h index bf332da1e1..17cbfef504 100644 --- a/src/sst/elements/firefly/nicSendEntry.h +++ b/src/sst/elements/firefly/nicSendEntry.h @@ -9,6 +9,10 @@ // See the file CONTRIBUTORS.TXT in the top level directory // of the distribution for more information. // +// Portions copyright (c) 2026, Hewlett Packard Enterprise Development LP +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause +// // This file is part of the SST software package. For license // information, see the LICENSE file in the top level directory of the // distribution. @@ -34,6 +38,8 @@ class SendEntryBase { virtual size_t hdrSize() = 0; virtual void copyOut( Output& dbg, int numBytes, FireflyNetworkEvent& event, std::vector& vec ) = 0; + + virtual bool isWriteOp() { return false; } // Override for WRITE operations virtual bool shouldDelete() { return true; } bool isCtrl() { return m_isCtrl; } bool isAck() { return m_isAck; } diff --git a/src/sst/elements/firefly/nicSendMachine.cc b/src/sst/elements/firefly/nicSendMachine.cc index a9a1516636..9d63b87254 100644 --- a/src/sst/elements/firefly/nicSendMachine.cc +++ b/src/sst/elements/firefly/nicSendMachine.cc @@ -9,6 +9,10 @@ // See the file CONTRIBUTORS.TXT in the top level directory // of the distribution for more information. // +// Portions copyright (c) 2026, Hewlett Packard Enterprise Development LP +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause +// // This file is part of the SST software package. For license // information, see the LICENSE file in the top level directory of the // distribution. @@ -63,9 +67,9 @@ void Nic::SendMachine::getPayload( SendEntryBase* entry, FireflyNetworkEvent* ev m_dbg.debug(CALL_INFO,2,NIC_DBG_SEND_MACHINE, "enque load from host, %lu bytes\n",ev->bufSize()); if ( entry->isDone() ) { ev->setTail(); - m_inQ->enque( m_unit, pid, vec, ev, entry->vn(), entry->dest(), std::bind( &Nic::SendMachine::streamFini, this, entry ) ); + m_inQ->enque( m_unit, pid, vec, ev, entry->vn(), entry->dest(), std::bind( &Nic::SendMachine::streamFini, this, entry ), entry); } else { - m_inQ->enque( m_unit, pid, vec, ev, entry->vn(), entry->dest() ); + m_inQ->enque( m_unit, pid, vec, ev, entry->vn(), entry->dest() , Callback(), entry ); m_nic.schedCallback( std::bind( &Nic::SendMachine::getPayload, this, entry, new FireflyNetworkEvent(m_pktOverhead) ), 0); } @@ -97,7 +101,7 @@ void Nic::SendMachine::streamFini( SendEntryBase* entry ) } void Nic::SendMachine::InQ::enque( int unit, int pid, std::vector< MemOp >* vec, - FireflyNetworkEvent* ev, int vn, int dest, Callback callback ) + FireflyNetworkEvent* ev, int vn, int dest, Callback callback , SendEntryBase* entry) { ++m_numPending; m_nic.m_sendStreamPending->addData( m_numPending ); @@ -105,9 +109,16 @@ void Nic::SendMachine::InQ::enque( int unit, int pid, std::vector< MemOp >* vec m_dbg.verbosePrefix(prefix(), CALL_INFO,2,NIC_DBG_SEND_MACHINE, "get timing for packet %" PRIu64 " size=%lu numPending=%d\n", m_pktNum,ev->bufSize(), m_numPending); - m_nic.dmaRead( unit, pid, vec, + bool isWrite=entry && entry->isWriteOp(); + if(isWrite){ + m_nic.dmaWrite( unit, pid, vec, + std::bind( &Nic::SendMachine::InQ::ready, this, ev, vn, dest, callback, m_pktNum++ ) + ); + } else { + m_nic.dmaRead( unit, pid, vec, std::bind( &Nic::SendMachine::InQ::ready, this, ev, vn, dest, callback, m_pktNum++ ) ); +} // don't put code after this, the callback may be called serially } diff --git a/src/sst/elements/firefly/nicSendMachine.h b/src/sst/elements/firefly/nicSendMachine.h index 0a0d10f406..9d2839e2b2 100644 --- a/src/sst/elements/firefly/nicSendMachine.h +++ b/src/sst/elements/firefly/nicSendMachine.h @@ -9,6 +9,10 @@ // See the file CONTRIBUTORS.TXT in the top level directory // of the distribution for more information. // +// Portions copyright (c) 2026, Hewlett Packard Enterprise Development LP +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause +// // This file is part of the SST software package. For license // information, see the LICENSE file in the top level directory of the // distribution. @@ -98,7 +102,7 @@ class SendMachine { return m_numPending == m_maxQsize; } - void enque( int unit, int pid, std::vector< MemOp >* vec, FireflyNetworkEvent* ev, int vn, int dest, Callback callback = NULL ); + void enque( int unit, int pid, std::vector< MemOp >* vec, FireflyNetworkEvent* ev, int vn, int dest, Callback callback = NULL, SendEntryBase* entry=NULL ); void wakeMeUp( Callback callback) { assert(!m_callback); diff --git a/src/sst/elements/firefly/nicVirtNic.h b/src/sst/elements/firefly/nicVirtNic.h index 6f3799b7a1..e864c5fcac 100644 --- a/src/sst/elements/firefly/nicVirtNic.h +++ b/src/sst/elements/firefly/nicVirtNic.h @@ -9,6 +9,10 @@ // See the file CONTRIBUTORS.TXT in the top level directory // of the distribution for more information. // +// Portions copyright (c) 2026, Hewlett Packard Enterprise Development LP +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause +// // This file is part of the SST software package. For license // information, see the LICENSE file in the top level directory of the // distribution. @@ -47,6 +51,9 @@ class VirtNic { m_toCoreLink->send( delay , event ); } + void sendNetworkIO( SimTime_t delay, SST::Event * event ) { + m_toCoreLink->send( delay , event ); + } Link* m_toCoreLink; int id; @@ -84,6 +91,9 @@ class VirtNic { void notifyShmem( SimTime_t delay, NicShmemValueRespEvent::Callback callback, Hermes::Value& value ) { sendShmem( delay, new NicShmemValueRespEvent( callback, value )); } + void notifyNetworkIO( SimTime_t delay, std::function callback, int retval ) { + sendNetworkIO( delay, new NicNetworkIORespEvent( callback, retval )); + } // Following functions needed so checkpointable handlers will // compile. They do not currently do anything else usefule, but diff --git a/src/sst/elements/firefly/storageModel/simpleSSD.cc b/src/sst/elements/firefly/storageModel/simpleSSD.cc new file mode 100644 index 0000000000..8693a8fd4b --- /dev/null +++ b/src/sst/elements/firefly/storageModel/simpleSSD.cc @@ -0,0 +1,88 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#include "sst_config.h" + +#include "simpleSSD.h" + +using namespace SST; +using namespace Firefly; + +SimpleSSD::SimpleSSD(ComponentId_t id, Params ¶ms) + : SimpleSSDAPI(id), m_pendingRequests(0) +{ + int nSSDsPerNode = params.find("nSSDsPerNode", 1); + int nQueuesPerSSD = params.find("queuesCountPerSSD", 4); + m_bus.lanes.resize(nQueuesPerSSD); + + m_readOverheadLatency_ns = params.find("readOverheadLatency_ns", 500); + m_writeOverheadLatency_ns = params.find("writeOverheadLatency_ns", 500); + + double readBandwidthPerSSD = params.find("readBandwidthPerSSD_GBps", 6.25); + m_readBandwidthPerQueue_GBps = readBandwidthPerSSD*nSSDsPerNode/nQueuesPerSSD; + + double writeBandwidthPerSSD = params.find("writeBandwidthPerSSD_GBps", 6.25); + m_writeBandwidthPerQueue_GBps = writeBandwidthPerSSD*nSSDsPerNode/nQueuesPerSSD; + + int verboseLevel = params.find("verboseLevel", 0); + int verboseMask = params.find("verboseMask", -1); + m_out.init("[SimpleSSD] ", verboseLevel, verboseMask, Output::STDOUT); + registerClock("1GHz", new Clock::Handler2(this)); + m_selfLink = configureSelfLink("ReadWriteLatency", "1 ns", new Event::Handler2(this)); +} + +void SimpleSSD::read(int64_t offset, size_t bytes, const SsdReqCallback &callback) +{ + enqueueRequest(offset, bytes, m_readBandwidthPerQueue_GBps, m_readOverheadLatency_ns, callback); +} + +void SimpleSSD::write(int64_t offset, size_t bytes, const SsdReqCallback &callback) +{ + enqueueRequest(offset, bytes, m_writeBandwidthPerQueue_GBps, m_writeOverheadLatency_ns, callback); +} + +void SimpleSSD::handleEvent(SST::Event *ev) +{ + DelayEvent *event = dynamic_cast(ev); + assert(event); + event->m_callback(); + delete ev; + --m_pendingRequests; +} + +bool SimpleSSD::clockTick(SST::Cycle_t n) +{ + if (m_pendingRequests == 0) + { + for (int i = 0; i < m_bus.lanes.size(); ++i) + { + if (!m_bus.lanes.at(i).empty()) + { + Request request = m_bus.lanes.at(i).front(); + DelayEvent *ev = new DelayEvent(request.callback); + m_selfLink->send(request.delay_ns, ev); + m_bus.lanes.at(i).pop(); + ++m_pendingRequests; + } + } + } + return false; +} + +int64_t SimpleSSD::calcDelay_ns(const size_t bytes, const double bandwidth_GBps, const int64_t latency_ns) +{ + double delay_ns = latency_ns + ((bytes / bandwidth_GBps)); + return int64_t(delay_ns); +} + +void SimpleSSD::enqueueRequest(const int64_t offset, const size_t bytes, const double bandwidth_GBps, const int64_t overheadLatency_ns, const SsdReqCallback& callback) +{ + Request request; + request.bytes = bytes; + request.delay_ns = this->calcDelay_ns(bytes, bandwidth_GBps, overheadLatency_ns); + request.offset = offset; + request.callback = callback; + m_bus.currentLane %= m_bus.lanes.size(); + m_bus.lanes.at(m_bus.currentLane).push(request); + m_bus.currentLane++; +} diff --git a/src/sst/elements/firefly/storageModel/simpleSSD.h b/src/sst/elements/firefly/storageModel/simpleSSD.h new file mode 100644 index 0000000000..e2678fbcd3 --- /dev/null +++ b/src/sst/elements/firefly/storageModel/simpleSSD.h @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + +#include +#include + +#include +#include +#include + +using SsdReqCallback = std::function; + +namespace SST +{ + namespace Firefly + { + struct Request + { + int64_t offset; + size_t bytes; + int64_t delay_ns; + SsdReqCallback callback; + }; + + struct Bus + { + std::vector> lanes; + int currentLane = 0; + }; + + class SimpleSSDAPI : public SST::SubComponent + { + public: + SST_ELI_REGISTER_SUBCOMPONENT_API(SST::Firefly::SimpleSSDAPI) + + /// Constructor + SimpleSSDAPI(ComponentId_t id) : SubComponent(id) {}; + + /// Destructor + virtual ~SimpleSSDAPI() {}; + + /// SSD read API to read bytes from offset + virtual void read(int64_t offset, size_t bytes, const SsdReqCallback &callback) = 0; + + /// SSD write API to write bytes to offset + virtual void write(int64_t offset, size_t bytes, const SsdReqCallback &callback) = 0; + + /// API that models the variation of delay with message size + virtual int64_t calcDelay_ns(size_t bytes, double bandwidth, int64_t latency) = 0; + + protected: + /// An instance of SSD bus abstracting connection to NIC + Bus m_bus; + + /// Overhead latency used to tune SSD read calls simulations with hardware + int64_t m_readOverheadLatency_ns; + + /// Overhead latency used to tune SSD write calls simulations with hardware + int64_t m_writeOverheadLatency_ns; + + /// SSD's read bandwidth per queue + double m_readBandwidthPerQueue_GBps; + + /// SSD's read bandwidth per queue + double m_writeBandwidthPerQueue_GBps; + }; + + class SimpleSSD : public SimpleSSDAPI + { + public: + SST_ELI_REGISTER_SUBCOMPONENT( + SimpleSSD, + "firefly", + "SimpleSSD", + SST_ELI_ELEMENT_VERSION(1, 0, 0), + "A simple server model to mimic SSD", + SST::Firefly::SimpleSSDAPI) + + SST_ELI_DOCUMENT_PARAMS({"nSSDsPerNode","The number of SSDs to simulate", "1"}, + {"queuesCountPerSSD", "The number of parallel paths per SSD", "4"}, + {"readBandwidthPerSSD_GBps", "", "6.25"}, + {"writeBandwidthPerSSD_GBps", "", "6.25"}, + {"readOverheadLatency_ns", "Latency(ns) used for tuning simulations to hardware experiments", "500"}, + {"writeOverheadLatency_ns", "Latency(ns) used for tuning simulations to hardware experiments", "500"}) + + /// Constructor + SimpleSSD(ComponentId_t id, Params ¶ms); + + /// Copy constructor deleted + SimpleSSD(const SimpleSSD &) = delete; + + /// Assignment operator deleted + SimpleSSD operator=(const SimpleSSD &) = delete; + + /// Destructor + ~SimpleSSD() = default; + + /// SSD read API to read bytes from offset + void read(int64_t offset, size_t bytes, const std::function &callback) override; + + /// SSD write API to write bytes to offset + void write(int64_t offset, size_t bytes, const std::function &callback) override; + + /// Caculating the delay of a particular message size + int64_t calcDelay_ns(size_t bytes, double bandwidth, int64_t latency) override; + + private: + /// Logging system + SST::Output m_out; + + /// Self link for delay + SST::Link *m_selfLink; + + /// Counter for the total number of pending requests + int64_t m_pendingRequests; + + /// Handle event function for the self-link + void handleEvent(SST::Event *ev); + + /// Clock tick function + virtual bool clockTick(SST::Cycle_t); + + /// API to queue the requests + void enqueueRequest(const int64_t offset, const size_t bytes, const double bandwidth, const int64_t latency, const SsdReqCallback& callback); + + /// Delay class for the delay events + class DelayEvent : public SST::Event + { + public: + SsdReqCallback m_callback; + DelayEvent(SsdReqCallback callback) : Event(), m_callback(callback) {} + NotSerializable(DelayEvent) + }; + }; + } +} diff --git a/src/sst/elements/firefly/virtNic.cc b/src/sst/elements/firefly/virtNic.cc index c75c7dc876..5a09771d0b 100644 --- a/src/sst/elements/firefly/virtNic.cc +++ b/src/sst/elements/firefly/virtNic.cc @@ -9,6 +9,10 @@ // See the file CONTRIBUTORS.TXT in the top level directory // of the distribution for more information. // +// Portions copyright (c) 2026, Hewlett Packard Enterprise Development LP +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause +// // This file is part of the SST software package. For license // information, see the LICENSE file in the top level directory of the // distribution. @@ -92,6 +96,9 @@ void VirtNic::handleEvent( Event* ev ) case NicRespBaseEvent::Shmem: handleShmemEvent( static_cast(ev ) ); break; + case NicRespBaseEvent::NetworkIO: + handleNetworkIOEvent( static_cast(ev ) ); + break; } delete ev; } @@ -133,6 +140,22 @@ void VirtNic::handleShmemEvent( NicShmemRespBaseEvent* event ) } } +void VirtNic::handleNetworkIOEvent( NicNetworkIORespBaseEvent* event ) +{ + NicNetworkIORespBaseEvent* ev = static_cast(event); + + m_dbg.debug(CALL_INFO,2,0,"[VirtNic] calling NetworkIO callback\n"); + ev->callback(); + + m_dbg.debug(CALL_INFO,2,0," %d %d\n", m_curNicQdepth, m_maxNicQdepth); + assert( m_curNicQdepth > 0 ); + --m_curNicQdepth; + if ( m_blockedCallback ) { + m_blockedCallback(); + m_blockedCallback = NULL; + } +} + bool VirtNic::canDmaSend() { m_dbg.debug(CALL_INFO,1,0,"\n"); @@ -279,3 +302,15 @@ void VirtNic::setNotifyNeedRecv( m_dbg.debug(CALL_INFO,2,0,"\n"); m_notifyNeedRecv = functor; } + +void VirtNic::networkIORead( int targetNid, Hermes::Vaddr dest, size_t len, std::function callback ) +{ + m_dbg.debug(CALL_INFO,2,0,"dest=%#lx len=%zu\n", dest, len); + sendCmd(0, new NicNetworkIOReadCmdEvent( targetNid, dest, len, callback ) ); +} + +void VirtNic::networkIOWrite( int targetNid, Hermes::Vaddr src, size_t len, std::function callback ) +{ + m_dbg.debug(CALL_INFO,2,0,"src=%#lx len=%zu\n", src, len); + sendCmd(0, new NicNetworkIOWriteCmdEvent( targetNid, src, len, callback ) ); +} \ No newline at end of file diff --git a/src/sst/elements/firefly/virtNic.h b/src/sst/elements/firefly/virtNic.h index 76a00a51b0..86b6585941 100644 --- a/src/sst/elements/firefly/virtNic.h +++ b/src/sst/elements/firefly/virtNic.h @@ -1,4 +1,3 @@ - // Copyright 2013-2025 NTESS. Under the terms // of Contract DE-NA0003525 with NTESS, the U.S. // Government retains certain rights in this software. @@ -10,6 +9,10 @@ // See the file CONTRIBUTORS.TXT in the top level directory // of the distribution for more information. // +// Portions copyright (c) 2026, Hewlett Packard Enterprise Development LP +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause +// // This file is part of the SST software package. For license // information, see the LICENSE file in the top level directory of the // distribution. @@ -20,14 +23,16 @@ #include #include #include "sst/elements/hermes/shmemapi.h" - +#include "sst/elements/hermes/networkIOapi.h" #include "ioVec.h" namespace SST { namespace Firefly { + class NicRespEvent; class NicShmemRespBaseEvent; +class NicNetworkIORespBaseEvent; class VirtNic : public SST::SubComponent { @@ -207,6 +212,9 @@ class VirtNic : public SST::SubComponent { void notifyRecvDmaDone( int src, int tag, size_t len, void* key ); void notifyNeedRecv( int src, int tag, size_t length ); + void networkIORead( int targetNid, Hermes::Vaddr dest, size_t len, std::function ); + void networkIOWrite( int targetNid, Hermes::Vaddr src, size_t len, std::function ); + bool isBlocked() { m_dbg.debug(CALL_INFO,2,0,"%d %d\n", m_curNicQdepth, m_maxNicQdepth); return m_curNicQdepth == m_maxNicQdepth; @@ -254,6 +262,7 @@ class VirtNic : public SST::SubComponent { void handleEvent( Event * ); void handleMsgEvent( NicRespEvent * ); void handleShmemEvent( NicShmemRespBaseEvent * ); + void handleNetworkIOEvent( NicNetworkIORespBaseEvent * ); int m_realNicId; int m_coreId; diff --git a/src/sst/elements/hermes/Makefile.am b/src/sst/elements/hermes/Makefile.am index 9558f7a3e5..799f185aed 100644 --- a/src/sst/elements/hermes/Makefile.am +++ b/src/sst/elements/hermes/Makefile.am @@ -19,7 +19,8 @@ nobase_sst_HEADERS = \ miscapi.h \ hermes.h \ functor.h \ - shmemapi.h + shmemapi.h \ + networkIOapi.h libhermes_la_LDFLAGS = -module -avoid-version diff --git a/src/sst/elements/hermes/networkIOapi.h b/src/sst/elements/hermes/networkIOapi.h new file mode 100644 index 0000000000..5d1d1e3ed4 --- /dev/null +++ b/src/sst/elements/hermes/networkIOapi.h @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright Hewlett Packard Enterprise Development LP +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + +#include +#include +#include +#include "hermes.h" + +namespace SST { +namespace Hermes { +namespace NetworkIO { + +// Network Storage I/O Interface +class Interface : public Hermes::Interface { + public: + typedef std::function Callback; + + Interface( ComponentId_t id ) : Hermes::Interface(id) {} + + virtual ~Interface() = default; + + // Network IO READ - reads from network storage to local buffer + // dest: local destination address + // offset: global byte offset in network storage (node ID calculated via interleaving) + // length: number of bytes to read + // blocking: whether to block until completion + // callback: completion callback + virtual void networkIORead(Vaddr dest, uint64_t offset, uint64_t length, Callback) { assert(0); } + + // Network IO WRITE - writes from local buffer to network storage + // offset: global byte offset in network storage (node ID calculated via interleaving) + // src: local source address + // length: number of bytes to write + // blocking: whether to block until completion + // callback: completion callback + virtual void networkIOWrite(uint64_t offset, Vaddr src, uint64_t length, Callback) { assert(0); } +}; + +} // namespace NetworkIO +} // namespace Hermes +} // namespace SST +