diff --git a/MC/bin/o2dpg_dpl_config_tools.py b/MC/bin/o2dpg_dpl_config_tools.py index 388e4306f..6849ea3dc 100755 --- a/MC/bin/o2dpg_dpl_config_tools.py +++ b/MC/bin/o2dpg_dpl_config_tools.py @@ -309,7 +309,11 @@ def dpl_option_from_config(config, dpl_workflow, key, section = "filtered", defa """ if "Executables" in config: # new standard - return config["Executables"].get(dpl_workflow,{}).get(section,{}).get(key, default_value) + value = config["Executables"].get(dpl_workflow,{}).get(section,{}).get(key, None) + if value == None: + print (f"Could not lookup key/option {key} from {dpl_workflow}") + value = default_value + return value else: # backward compatible versions dpl_workflow_key = dpl_workflow + '-options' diff --git a/MC/bin/o2dpg_qc_finalization_workflow.py b/MC/bin/o2dpg_qc_finalization_workflow.py index 31f0b48a3..21d2bbd09 100755 --- a/MC/bin/o2dpg_qc_finalization_workflow.py +++ b/MC/bin/o2dpg_qc_finalization_workflow.py @@ -95,8 +95,9 @@ def add_QC_postprocessing(taskName, qcConfigPath, needs, runSpecific, prodSpecif add_QC_finalization('emcBCQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/emc-reco-tasks.json') #add_QC_finalization('tpcTrackingQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tpc-qc-tracking-direct.json') add_QC_finalization('tpcStandardQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tpc-qc-standard-direct.json') - add_QC_finalization('trdDigitsQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/trd-standalone-task.json') - add_QC_finalization('trdTrackingQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/trd-tracking-task.json') + if isActive('TRD'): + add_QC_finalization('trdDigitsQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/trd-standalone-task.json') + add_QC_finalization('trdTrackingQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/trd-tracking-task.json') add_QC_finalization('vertexQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/vertexing-qc-direct-mc.json') add_QC_finalization('ITSTPCmatchQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/ITSTPCmatchedTracks_direct_MC.json') add_QC_finalization('TOFMatchQC', 'json://${O2DPG_ROOT}/MC/config/QC/json/tofMatchedTracks_ITSTPCTOF_TPCTOF_direct_MC.json') diff --git a/MC/bin/o2dpg_sim_workflow.py b/MC/bin/o2dpg_sim_workflow.py index 5431429a3..54b2b747a 100755 --- a/MC/bin/o2dpg_sim_workflow.py +++ b/MC/bin/o2dpg_sim_workflow.py @@ -261,7 +261,7 @@ def load_external_config(configfile): # these are all detectors that should be assumed active readout_detectors = args.readoutDets # here are all detectors that have been set in an anchored script -activeDetectors = dpl_option_from_config(anchorConfig, 'o2-ctf-reader-workflow', key='onlyDet', default_value='all') +activeDetectors = dpl_option_from_config(anchorConfig, 'o2-ctf-reader-workflow', key='--onlyDet', default_value='all') if activeDetectors == 'all': # if "all" here, there was in fact nothing in the anchored script, set to what is passed to this script (which it either also "all" or a subset) activeDetectors = readout_detectors @@ -1122,7 +1122,8 @@ def putConfigValues(listOfMainKeys=[], localCF = {}): + ' --onlyDet TRD --interactionRate ' + str(INTRATE) + ' --incontext ' + str(CONTEXTFILE) + ' --disable-write-ini' \ + putConfigValues(localCF={"TRDSimParams.digithreads" : NWORKERS_TF, "DigiParams.seed" : str(TFSEED)}) + " --forceSelectedDets" TRDDigitask['cmd'] += ('',' --disable-mc')[args.no_mc_labels] - workflow['stages'].append(TRDDigitask) + if isActive("TRD"): + workflow['stages'].append(TRDDigitask) # these are digitizers which are single threaded def createRestDigiTask(name, det='ALLSMALLER'): @@ -1361,12 +1362,13 @@ def getDigiTaskName(det): getDPL_global_options(), putConfigValues(), ('',' --disable-mc')[args.no_mc_labels]]) - workflow['stages'].append(TRDTRACKINGtask) + if isActive("TRD"): + workflow['stages'].append(TRDTRACKINGtask) #<--------- TRD global tracking # FIXME This is so far a workaround to avoud a race condition for trdcalibratedtracklets.root TRDTRACKINGtask2 = createTask(name='trdreco2_'+str(tf), needs=[TRDTRACKINGtask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu='1', mem='2000') - trd_track_sources = cleanDetectorInputList(dpl_option_from_config(anchorConfig, 'o2-trd-global-tracking', 'track-sources', default_value='TPC,ITS-TPC')) + trd_track_sources = cleanDetectorInputList(dpl_option_from_config(anchorConfig, 'o2-trd-global-tracking', '--track-sources', default_value='TPC,ITS-TPC')) TRDTRACKINGtask2['cmd'] = task_finalizer([ '${O2_ROOT}/bin/o2-trd-global-tracking', getDPL_global_options(bigshm=True), @@ -1381,7 +1383,8 @@ def getDigiTaskName(det): '--track-sources ' + trd_track_sources, tpc_corr_scaling_options, tpc_corr_options_mc]) - workflow['stages'].append(TRDTRACKINGtask2) + if isActive("TRD"): + workflow['stages'].append(TRDTRACKINGtask2) #<--------- TOF reco task TOFRECOtask = createTask(name='tofmatch_'+str(tf), needs=[ITSTPCMATCHtask['name'], getDigiTaskName("TOF")], tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1500') @@ -1396,8 +1399,11 @@ def getDigiTaskName(det): workflow['stages'].append(TOFRECOtask) #<--------- TOF-TPC(-ITS) global track matcher workflow - toftpcmatchneeds = [TOFRECOtask['name'], TPCRECOtask['name'], ITSTPCMATCHtask['name'], TRDTRACKINGtask2['name']] - toftracksrcdefault = dpl_option_from_config(anchorConfig, 'o2-tof-matcher-workflow', 'track-sources', default_value='TPC,ITS-TPC,TPC-TRD,ITS-TPC-TRD') + toftpcmatchneeds = [TOFRECOtask['name'], + TPCRECOtask['name'], + ITSTPCMATCHtask['name'], + TRDTRACKINGtask2['name'] if isActive("TRD") else None] + toftracksrcdefault = dpl_option_from_config(anchorConfig, 'o2-tof-matcher-workflow', '--track-sources', default_value='TPC,ITS-TPC,TPC-TRD,ITS-TPC-TRD') tofusefit = option_if_available('o2-tof-matcher-workflow', '--use-fit', envfile=async_envfile) TOFTPCMATCHERtask = createTask(name='toftpcmatch_'+str(tf), needs=toftpcmatchneeds, tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1000') tofmatcher_cmd_parts = [ @@ -1591,9 +1597,9 @@ def getDigiTaskName(det): hmpmatchneeds = [HMPRECOtask['name'], ITSTPCMATCHtask['name'], TOFTPCMATCHERtask['name'] if isActive("TOF") else None, - TRDTRACKINGtask2['name']] + TRDTRACKINGtask2['name'] if isActive("TRD") else None] hmpmatchneeds = [ n for n in hmpmatchneeds if n != None ] - hmp_match_sources = cleanDetectorInputList(dpl_option_from_config(anchorConfig, 'o2-hmpid-matcher-workflow', 'track-sources', default_value='ITS-TPC,ITS-TPC-TRD,TPC-TRD')) + hmp_match_sources = cleanDetectorInputList(dpl_option_from_config(anchorConfig, 'o2-hmpid-matcher-workflow', '--track-sources', default_value='ITS-TPC,ITS-TPC-TRD,TPC-TRD')) HMPMATCHtask = createTask(name='hmpmatch_'+str(tf), needs=hmpmatchneeds, tf=tf, cwd=timeframeworkdir, lab=["RECO"], mem='1000') HMPMATCHtask['cmd'] = task_finalizer( ['${O2_ROOT}/bin/o2-hmpid-matcher-workflow', @@ -1606,17 +1612,17 @@ def getDigiTaskName(det): #<---------- primary vertex finding pvfinder_sources = dpl_option_from_config(anchorConfig, 'o2-primary-vertexing-workflow', - 'vertexing-sources', + '--vertexing-sources', default_value='ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,FDD,HMP,FV0,TRD,MCH,CTP') pvfinder_sources = cleanDetectorInputList(pvfinder_sources) pvfinder_matching_sources = dpl_option_from_config(anchorConfig, 'o2-primary-vertexing-workflow', - 'vertex-track-matching-sources', + '--vertex-track-matching-sources', default_value='ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,FDD,HMP,FV0,TRD,MCH,CTP') pvfinder_matching_sources = cleanDetectorInputList(pvfinder_matching_sources) - pvfinderneeds = [TRDTRACKINGtask2['name'], + pvfinderneeds = [TRDTRACKINGtask2['name'] if isActive("TRD") else None, FT0RECOtask['name'] if isActive("FT0") else None, FV0RECOtask['name'] if isActive("FV0") else None, EMCRECOtask['name'] if isActive("EMC") else None, @@ -1658,7 +1664,7 @@ def getDigiTaskName(det): svfinder_sources = dpl_option_from_config(anchorConfig, 'o2-primary-vertexing-workflow', - 'vertex-track-matching-sources', + '--vertex-track-matching-sources', default_value='ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,ZDC,FDD,HMP,FV0,TRD,MCH,CTP') svfinder_sources = cleanDetectorInputList(svfinder_sources) SVFINDERtask = createTask(name='svfinder_'+str(tf), needs=[PVFINDERtask['name'], FT0FV0EMCCTPDIGItask['name']], tf=tf, cwd=timeframeworkdir, lab=["RECO"], cpu=svfinder_cpu, mem='5000') @@ -1679,7 +1685,7 @@ def getDigiTaskName(det): #<------------- AOD producer # TODO This needs further refinement, sources and dependencies should be constructed dynamically aod_info_souces_default = 'ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,ZDC,FDD,HMP,FV0,TRD,MCH,CTP' - aodinfosources = dpl_option_from_config(anchorConfig, 'o2-aod-producer-workflow', 'info-sources', default_value=aod_info_souces_default) + aodinfosources = dpl_option_from_config(anchorConfig, 'o2-aod-producer-workflow', '--info-sources', default_value=aod_info_souces_default) aodinfosources = cleanDetectorInputList(aodinfosources) aodneeds = [PVFINDERtask['name'], SVFINDERtask['name']] @@ -1736,17 +1742,17 @@ def getDigiTaskName(det): #<------------- TPC residuals extraction scdcalib_vertex_sources = cleanDetectorInputList(dpl_option_from_config(anchorConfig, 'o2-tpc-scdcalib-interpolation-workflow', - 'vtx-sources', + '--vtx-sources', default_value='ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,FDD,HMP,FV0,TRD,MCH,CTP')) scdcalib_track_sources = cleanDetectorInputList(dpl_option_from_config(anchorConfig, 'o2-tpc-scdcalib-interpolation-workflow', - 'tracking-sources', + '--tracking-sources', default_value='ITS-TPC,TPC-TRD,ITS-TPC-TRD,TPC-TOF,ITS-TPC-TOF,TPC-TRD-TOF,ITS-TPC-TRD-TOF,MFT-MCH,MCH-MID,ITS,MFT,TPC,TOF,FT0,MID,EMC,PHS,CPV,FDD,HMP,FV0,TRD,MCH,CTP')) scdcalib_track_extraction = cleanDetectorInputList(dpl_option_from_config(anchorConfig, 'o2-tpc-scdcalib-interpolation-workflow', - 'tracking-sources-map-extraction', + '--tracking-sources-map-extraction', default_value='ITS-TPC')) SCDCALIBtask = createTask(name='scdcalib_'+str(tf), needs=[PVFINDERtask['name']], tf=tf, cwd=timeframeworkdir, lab=["CALIB"], mem='4000') @@ -1765,11 +1771,11 @@ def getDigiTaskName(det): #<------------- TPC residuals aggregator scdaggreg_secperslot = dpl_option_from_config(anchorConfig, 'o2-calibration-residual-aggregator', - 'sec-per-slot', + '--sec-per-slot', default_value='600') scdaggreg_outputtype = dpl_option_from_config(anchorConfig, 'o2-calibration-residual-aggregator', - 'output-type', + '--output-type', default_value='trackParams,unbinnedResid') SCDAGGREGtask = createTask(name='scdaggreg_'+str(tf), needs=[SCDCALIBtask['name']], tf=tf, cwd=timeframeworkdir, lab=["CALIB"], mem='1500') @@ -1849,12 +1855,13 @@ def remove_json_prefix(path): ### TRD # TODO: check if the readerCommand also reperforms tracklet construction (which already done in digitization) - addQCPerTF(taskName='trdDigitsQC', + if isActive('TRD'): + addQCPerTF(taskName='trdDigitsQC', needs=[TRDDigitask['name']], readerCommand='o2-trd-trap-sim --disable-root-output true', configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/trd-standalone-task.json') - addQCPerTF(taskName='trdTrackingQC', + addQCPerTF(taskName='trdTrackingQC', needs=[TRDTRACKINGtask2['name']], readerCommand='o2-global-track-cluster-reader --track-types "ITS-TPC-TRD,TPC-TRD" --cluster-types none', configFilePath='json://${O2DPG_ROOT}/MC/config/QC/json/trd-tracking-task.json') diff --git a/MC/bin/o2dpg_sim_workflow_anchored.py b/MC/bin/o2dpg_sim_workflow_anchored.py index 57e6676b6..9a7002bf2 100755 --- a/MC/bin/o2dpg_sim_workflow_anchored.py +++ b/MC/bin/o2dpg_sim_workflow_anchored.py @@ -122,6 +122,14 @@ def retrieve_Aggregated_RunInfos(run_number): detList = o2.detectors.DetID.getNames(runInfo.grpECS.getDetsReadOut()) assert (run_number == runInfo.runNumber) assert (run_number == runInfo.grpECS.getRun()) + + print (f"Detector list from RunInfo/GRPECS is {detList}") + # potential overwrite in detector list if special env variable ALIEN_JDL_WORKFLOWDETECTORS is set + detlist_overwrite = os.getenv("ALIEN_JDL_WORKFLOWDETECTORS") + if detlist_overwrite: + detList = detlist_overwrite + print (f"Detector list is overwritten to {detList} via JDL") + return {"SOR" : runInfo.sor, "EOR" : runInfo.eor, "FirstOrbit" : runInfo.orbitSOR, @@ -218,6 +226,12 @@ def retrieve_params_fromGRPECS_and_OrbitReset(ccdbreader, run_number, run_start, detList = o2.detectors.DetID.getNames(grp["mDetsReadout"]['v']) print ("Detector list is ", detList) + # potential reduction in detector list if special env variable ALIEN_JDL_WORKFLOWDETECTORS is set + detlist_overwrite = os.getenv("ALIEN_JDL_WORKFLOWDETECTORS") + if detlist_overwrite: + detList = detlist_overwrite + print ("Detector list is overwritten to ", detList) + # orbitReset.get(run_number) return {"FirstOrbit" : orbitFirst, "LastOrbit" : orbitLast, "OrbitsPerTF" : int(grp["mNHBFPerTF"]), "detList" : detList}