From f201cef2a8927bb1330fac854307852806fa73fa Mon Sep 17 00:00:00 2001 From: Ilia Ponomarev Date: Fri, 17 Nov 2023 16:08:04 +0100 Subject: [PATCH 1/2] new script for converting JSON dumpstat to Twiki format --- ConditionsConsumed/AlCaCondJSONtoTwiki.py | 87 +++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100755 ConditionsConsumed/AlCaCondJSONtoTwiki.py diff --git a/ConditionsConsumed/AlCaCondJSONtoTwiki.py b/ConditionsConsumed/AlCaCondJSONtoTwiki.py new file mode 100755 index 0000000..ff856eb --- /dev/null +++ b/ConditionsConsumed/AlCaCondJSONtoTwiki.py @@ -0,0 +1,87 @@ +############################################ +# get AlCa conditions consumed in the CMSSW +# https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideCalAliTrigger2023 +############################################ + +import json +import argparse + +def checkTagInJson(tag, data): + #processing string with "/" separator from tagFile + stripped_tag = tag.split('/')[0].strip() + requested_label = tag.split('/')[1].strip() + for record in data.get(stripped_tag, []): + if requested_label in record.get('label', '') and record.get('timeLookupPayloadIds', []): + return "%GREEN% Yes %ENDCOLOR%" + return "" + +def getOneRow(tag, jsonFiles): + rowName = "|" + tag.strip() + for file_ in jsonFiles: + with open(file_, 'r') as file: + data = json.load(file) + checkTagInFile_ = checkTagInJson(tag, data) + rowName = rowName + " | " + checkTagInFile_ + return rowName + " | \n" + +def printAllRow(tagFile, jsonFiles, output_table, tableTitle): + with open(f"outputForTwiki_{output_table}.txt", 'w') as outForTwiki, open(tagFile, 'r') as tag_file: + outForTwiki.write(tableTitle) + for tag in tag_file: + getOneRow_ = getOneRow(tag, jsonFiles) + outForTwiki.write(getOneRow_) + +def updateTagFile(tagFile, json_files): + added_rows = set() + with open(tagFile, 'w') as new_tagFile: + for file_ in json_files: + with open(file_, 'r') as file_content: + data = json.load(file_content) + for record, label_info_list in data.items(): + for label_info in label_info_list: + label = label_info.get('label', '') + row = f"{record} / {label}" + if row not in added_rows: + new_tagFile.write(f"{row}\n") + added_rows.add(row) + + + +def main(): + parser = argparse.ArgumentParser(description='Usage: python3 getAlCaCondInGT.py [options] \n') + parser.add_argument('-d', '--data', dest='isData', action='store_true', default=False, help='Enter --data -d for data, False for MC. Default: False') + args = parser.parse_args() + + + tagFile = "allAlCaTags.txt" + if args.isData: + jsonFiles = [ + "step1_output.json", #step1_L1 + "step2_output.json", #step2_HLT + "step3_output.json", #step3_AOD + "step4_output.json", #step4_MINIAOD + "step5_output.json", #step5_NANOAOD + ] + else: + jsonFiles = [ + "step1_output.json", #step1_GEN + "step2_output.json", #step2_SIM + "step3_output.json", #step3_DIGI + "step4_output.json", #step4_L1 + "step5_output.json", #step5_DIGI2RAW + "step6_output.json", #step6_HLT + "step7_output.json", #step7_AODSIM + "step8_output.json", #step8_MINIAODSIM + "step9_output.json", #step9_NANOAODSIM + #Add remaining JSON filenames if available + ] + + output_table = "DATA" if args.isData else "MC" + tableTitle = "|Tags|L1|HLT|AOD|MINIAOD|NANOAOD|\n" if args.isData else "|Tags|GEN|SIM|DIGI|L1|DIGI2RAW|HLT|AOD|MINIAOD|NANOAOD|\n" + + updateTagFile(tagFile, jsonFiles) + + printAllRow(tagFile, jsonFiles, output_table, tableTitle) + +if __name__ == '__main__': + main() From 4c361d2bb4cf0187a2ce9e99cddc96f174614f21 Mon Sep 17 00:00:00 2001 From: Ilia Ponomarev Date: Tue, 19 Dec 2023 13:06:57 +0100 Subject: [PATCH 2/2] Improved the function of updating the allAlCaTags.txt file. New names of source files. Added .sh scripts for generating JSON dumps. --- ConditionsConsumed/AlCaCondJSONtoTwiki.py | 77 +++++++++++-------- .../runCMSDrivers_data2023DwithJSON.sh | 30 ++++++++ .../runCMSDrivers_mc2023withJSON.sh | 65 ++++++++++++++++ 3 files changed, 139 insertions(+), 33 deletions(-) create mode 100755 ConditionsConsumed/runCMSDrivers_data2023DwithJSON.sh create mode 100755 ConditionsConsumed/runCMSDrivers_mc2023withJSON.sh diff --git a/ConditionsConsumed/AlCaCondJSONtoTwiki.py b/ConditionsConsumed/AlCaCondJSONtoTwiki.py index ff856eb..e1f0ae6 100755 --- a/ConditionsConsumed/AlCaCondJSONtoTwiki.py +++ b/ConditionsConsumed/AlCaCondJSONtoTwiki.py @@ -5,9 +5,10 @@ import json import argparse +from contextlib import ExitStack def checkTagInJson(tag, data): - #processing string with "/" separator from tagFile + # processing string with "/" separator from tagFile stripped_tag = tag.split('/')[0].strip() requested_label = tag.split('/')[1].strip() for record in data.get(stripped_tag, []): @@ -15,6 +16,7 @@ def checkTagInJson(tag, data): return "%GREEN% Yes %ENDCOLOR%" return "" + def getOneRow(tag, jsonFiles): rowName = "|" + tag.strip() for file_ in jsonFiles: @@ -24,64 +26,73 @@ def getOneRow(tag, jsonFiles): rowName = rowName + " | " + checkTagInFile_ return rowName + " | \n" + def printAllRow(tagFile, jsonFiles, output_table, tableTitle): - with open(f"outputForTwiki_{output_table}.txt", 'w') as outForTwiki, open(tagFile, 'r') as tag_file: + with open(f"outputForTwiki_{output_table}.txt", 'w') as outForTwiki, open(tagFile, 'r') as tag_file: outForTwiki.write(tableTitle) for tag in tag_file: getOneRow_ = getOneRow(tag, jsonFiles) outForTwiki.write(getOneRow_) def updateTagFile(tagFile, json_files): - added_rows = set() - with open(tagFile, 'w') as new_tagFile: - for file_ in json_files: - with open(file_, 'r') as file_content: - data = json.load(file_content) - for record, label_info_list in data.items(): - for label_info in label_info_list: - label = label_info.get('label', '') - row = f"{record} / {label}" - if row not in added_rows: - new_tagFile.write(f"{row}\n") - added_rows.add(row) + unique_rows = set() + +# Open files in advance using context managers + with ExitStack() as stack: + files = [stack.enter_context(open(file_, 'r')) for file_ in json_files] + data_list = [json.load(file_content) for file_content in files] + +# Process data + for data in data_list: + for record, label_info_list in data.items(): + for label_info in label_info_list: + label = label_info.get('label', '') + row = f"{record} / {label}" + unique_rows.add(row) +# Sort and write the sorted lines to a file + with open(tagFile, 'w') as new_tagFile: + new_tagFile.write('\n'.join(sorted(unique_rows, reverse=False))) def main(): - parser = argparse.ArgumentParser(description='Usage: python3 getAlCaCondInGT.py [options] \n') - parser.add_argument('-d', '--data', dest='isData', action='store_true', default=False, help='Enter --data -d for data, False for MC. Default: False') + parser = argparse.ArgumentParser( + description='Usage: python3 getAlCaCondInGT.py [options] \n') + parser.add_argument('-d', '--data', dest='isData', action='store_true', + default=False, help='Enter --data -d for data, False for MC. Default: False') args = parser.parse_args() - tagFile = "allAlCaTags.txt" if args.isData: jsonFiles = [ - "step1_output.json", #step1_L1 - "step2_output.json", #step2_HLT - "step3_output.json", #step3_AOD - "step4_output.json", #step4_MINIAOD - "step5_output.json", #step5_NANOAOD + "output_step1_L1.json", # step1_L1 + "output_step2_HLT.json", # step2_HLT + "output_step3_AOD.json", # step3_AOD + "output_step4_MINIAOD.json", # step4_MINIAOD + "output_step5_NANOAOD.json", # step5_NANOAOD ] else: jsonFiles = [ - "step1_output.json", #step1_GEN - "step2_output.json", #step2_SIM - "step3_output.json", #step3_DIGI - "step4_output.json", #step4_L1 - "step5_output.json", #step5_DIGI2RAW - "step6_output.json", #step6_HLT - "step7_output.json", #step7_AODSIM - "step8_output.json", #step8_MINIAODSIM - "step9_output.json", #step9_NANOAODSIM - #Add remaining JSON filenames if available + "output_step1_GEN.json", # step1_GEN + "output_step2_SIM.json", # step2_SIM + "output_step3_DIGI.json", # step3_DIGI + "output_step4_L1.json", # step4_L1 + "output_step5_DIGI2RAW.json", # step5_DIGI2RAW + "output_step6_HLT.json", # step6_HLT + "output_step7_AODSIM.json", # step7_AODSIM + "output_step8_MINIAODSIM.json", # step8_MINIAODSIM + "output_step9_NANOAODSIM.json", # step9_NANOAODSIM + + # Add remaining JSON filenames if available ] output_table = "DATA" if args.isData else "MC" tableTitle = "|Tags|L1|HLT|AOD|MINIAOD|NANOAOD|\n" if args.isData else "|Tags|GEN|SIM|DIGI|L1|DIGI2RAW|HLT|AOD|MINIAOD|NANOAOD|\n" - + updateTagFile(tagFile, jsonFiles) printAllRow(tagFile, jsonFiles, output_table, tableTitle) + if __name__ == '__main__': main() diff --git a/ConditionsConsumed/runCMSDrivers_data2023DwithJSON.sh b/ConditionsConsumed/runCMSDrivers_data2023DwithJSON.sh new file mode 100755 index 0000000..af50359 --- /dev/null +++ b/ConditionsConsumed/runCMSDrivers_data2023DwithJSON.sh @@ -0,0 +1,30 @@ +#1: L1 +echo "------------------------------" +echo "Running step-1/5: L1" +echo "------------------------------" +cmsDriver.py step1 --conditions auto:run3_hlt_relval -n 5 --era Run3_2023 -s L1REPACK:Full --data --scenario pp --datatier FEVTDEBUGHLT --eventcontent FEVTDEBUGHLT --filein /store/data/Run2023D/JetMET0/RAW/v1/000/369/978/00000/00b9eba7-c847-465b-a6de-98bceae93613.root --fileout output_step1_L1.root --customise_commands='process.load("Configuration.StandardSequences.Digi_cff") \n process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName = cms.untracked.string("output_step1_L1.json")' --outputCommands "keep *" |& tee output_step1_L1.log + +#2: HLTlt +echo "------------------------------" +echo "Running step-2/5: HLT" +echo "------------------------------" +cmsDriver.py step2 --conditions auto:run3_hlt_relval -n 5 --era Run3_2023 -s HLT:@relval2023 --processName HLT2 --data --scenario pp --datatier FEVTDEBUGHLT --eventcontent FEVTDEBUGHLT --filein file:output_step1_L1.root --fileout output_step2_HLT.root --customise_commands='process.GlobalTag.DumpStat=cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step2_HLT.json")' |& tee output_step2_HLT.log +#cmsDriver.py step2 --conditions auto:run3_hlt_relval -n 5 --era Run3_2023 -s HLT:@relval2023 --processName HLT2 --data --scenario pp --datatier FEVTDEBUGHLT --eventcontent FEVTDEBUGHLT --filein file:output_step1_L1.root --fileout output_step2_HLT.root --customise_commands='process.GlobalTag.DumpStat=cms.untracked.bool(True)' --outputCommands "keep *, drop *_hlt*Legacy_*_*" |& tee output_step2_HLT.log + +#3 AOD (RAW2DIGI,L1Reco,RECO) +echo "------------------------------" +echo "Running step-3/5: AOD" +echo "------------------------------" +cmsDriver.py step3 --conditions auto:run3_data_prompt_relval -n 5 --era Run3_2023 -s RAW2DIGI,L1Reco,RECO --data --datatier AOD --eventcontent AOD --filein file:output_step2_HLT.root --fileout output_step3_AOD.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step3_AOD.json")' --customise Configuration/DataProcessing/RecoTLR.customisePostEra_Run3 |& tee output_step3_AOD.log + +#4 MINIAOD +echo "------------------------------" +echo "Running step-4/5: MINIAOD" +echo "------------------------------" +cmsDriver.py step4 --runUnscheduled --conditions auto:run3_data_prompt_relval -n 5 --era Run3_2023 -s PAT --data --datatier MINIAOD --eventcontent MINIAOD --filein file:output_step3_AOD.root --fileout output_step4_MINIAOD.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step4_MINIAOD.json")' --customise Configuration/DataProcessing/RecoTLR.customisePostEra_Run3 |& tee output_step4_MINIAOD.log + +#5 NANOAOD +echo "------------------------------" +echo "Running step-5/5: NANOAOD" +echo "------------------------------" +cmsDriver.py step5 --runUnscheduled --conditions auto:run3_data_prompt_relval -n 5 --era Run3_2023 -s NANO --data --datatier NANOAOD --eventcontent NANOAOD --filein file:output_step4_MINIAOD.root --fileout output_step5_NANOAOD.root --customise_commands='process.GlobalTag.DumpStat=cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step5_NANOAOD.json")' |& tee output_step5_NANOAOD.log diff --git a/ConditionsConsumed/runCMSDrivers_mc2023withJSON.sh b/ConditionsConsumed/runCMSDrivers_mc2023withJSON.sh new file mode 100755 index 0000000..9e0bfea --- /dev/null +++ b/ConditionsConsumed/runCMSDrivers_mc2023withJSON.sh @@ -0,0 +1,65 @@ +#1: GEN +echo "------------------------------" +echo "Running step-1/9: GEN" +echo "------------------------------" +cmsDriver.py TTbar_13TeV_TuneCUETP8M1_cfi --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 --geometry DB:Extended -s GEN --fileout output_step1_GEN.root --beamspot Realistic25ns13p6TeVEarly2023Collision --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step1_GEN.json")'|& tee output_step1_GEN.log + +#2: SIM +echo "------------------------------" +echo "Running step-2/9: SIM" +echo "------------------------------" +cmsDriver.py step2 --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 --geometry DB:Extended -s SIM --datatier GEN-SIM --eventcontent FEVTDEBUG --filein file:output_step1_GEN.root --fileout output_step2_SIM.root --beamspot Realistic25ns13p6TeVEarly2023Collision --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step2_SIM.json")' --outputCommands "keep *" |& tee output_step2_SIM.log + +#3: DIGI +echo "------------------------------" +echo "Running step-3/9: DIGI" +echo "------------------------------" +cmsDriver.py step3 --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 --geometry DB:Extended -s DIGI:pdigi_valid --datatier GEN-SIM-DIGI-RAW --eventcontent FEVTDEBUGHLT --filein file:output_step2_SIM.root --fileout output_step3_DIGI.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step3_DIGI.json")' --outputCommands "keep *, drop *_mix_*_*" |& tee output_step3_DIGI.log + +#4: L1 +echo "------------------------------" +echo "Running step-4/9: L1" +echo "------------------------------" +cmsDriver.py step4 --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 --geometry DB:Extended -s L1 --datatier GEN-SIM-DIGI-RAW --eventcontent FEVTDEBUGHLT --filein file:output_step3_DIGI.root --fileout output_step4_L1.root --customise_commands='process.load("Configuration.StandardSequences.Digi_cff") \n process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step4_L1.json")' --outputCommands "keep *" |& tee output_step4_L1.log + +#5: DIGI2RAW +echo "------------------------------" +echo "Running step-5/9: DIGI2RAW" +echo "------------------------------" +cmsDriver.py step5 --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 --geometry DB:Extended -s DIGI2RAW --datatier GEN-SIM-DIGI-RAW --eventcontent FEVTDEBUGHLT --filein file:output_step4_L1.root --fileout output_step5_DIGI2RAW.root --customise_commands='process.GlobalTag.DumpStat=cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step5_DIGI2RAW.json")' --outputCommands "keep *" |& tee output_step5_DIGI2RAW.log + +#6: HLT +echo "------------------------------" +echo "Running step-6/9: HLT" +echo "------------------------------" +cmsDriver.py step6 --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 -s HLT:@relval2016 --datatier GEN-SIM-DIGI-RAW-HLTDEBUG --eventcontent FEVTDEBUGHLT --filein file:output_step5_DIGI2RAW.root --fileout output_step6_HLT.root --customise_commands='process.GlobalTag.DumpStat=cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step6_HLT.json")' --outputCommands "keep *" |& tee output_step6_HLT.log + +#7 AODSIM (RAW2DIGI,L1Reco,RECO,RECOSIM) +echo "------------------------------" +echo "Running step-7/9: AODSIM" +echo "------------------------------" +cmsDriver.py step7 --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 --geometry DB:Extended -s RAW2DIGI,L1Reco,RECO,RECOSIM --datatier AODSIM --eventcontent AODSIM --filein file:output_step6_HLT.root --fileout output_step7_AODSIM.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step7_AODSIM.json")' |& tee output_step7_AODSIM.log + +#8 MINIAODSIM +echo "------------------------------" +echo "Running step-8/9: MINIADOSIM" +echo "------------------------------" +cmsDriver.py step8 --runUnscheduled --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 -s PAT --datatier MINIAODSIM --eventcontent MINIAODSIM --filein file:output_step7_AODSIM.root --fileout output_step8_MINIAODSIM.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step8_MINIAODSIM.json")' |& tee output_step8_MINIAODSIM.log + +#9 NANOAODSIM +echo "------------------------------" +echo "Running step-9/9: NANOAODSIM" +echo "------------------------------" +cmsDriver.py step9 --runUnscheduled --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 --geometry DB:Extended -s NANO --datatier NANOAODSIM --eventcontent NANOAODSIM --filein file:output_step8_MINIAODSIM.root --fileout output_step9_NANOAODSIM.root --customise_commands='process.GlobalTag.DumpStat=cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step9_NANOAODSIM.json")' |& tee output_step9_NANOAODSIM.log + +################# step-7 can be splitted in sub steps ################### +#7a RAW2DIGI +#cmsDriver.py step7a --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 -s RAW2DIGI --datatier GEN-SIM-RECO --eventcontent RECOSIM --filein file:output_step6_HLT.root --fileout output_step7a_RAW2DIGI.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True)' --outputCommands "keep *" |& tee output_step7a_RAW2DIGI.log + +#7b L1Reco +#cmsDriver.py step7b --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 -s L1Reco --datatier GEN-SIM-RECO --eventcontent RECOSIM --filein file:output_step7a_RAW2DIGI.root --fileout output_step7b_L1Reco.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True)' --outputCommands "keep *" |& tee output_step7b_L1Reco.log + +#7c RECO +#cmsDriver.py step7c --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 -s RECO --datatier GEN-SIM-RECO --eventcontent RECO --filein file:output_step7b_L1Reco.root --fileout output_step7c_RECO.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True)' --outputCommands "keep *" |& tee output_step7c_RECO.log + +#7d, etc