From 85686653397f79039c9c7596812d63bfc344d1b3 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 26 Apr 2022 17:29:55 +0200 Subject: [PATCH 1/3] TRD vDrift calib uses proper CCDB output mechanism --- .../include/TRDCalibration/CalibratorVdExB.h | 7 ++++ .../TRD/calibration/src/CalibratorVdExB.cxx | 23 ++++++------ .../include/TRDWorkflow/VdAndExBCalibSpec.h | 37 ++++++++++++++----- .../TRD/workflow/src/trd-calib-workflow.cxx | 3 +- 4 files changed, 47 insertions(+), 23 deletions(-) diff --git a/Detectors/TRD/calibration/include/TRDCalibration/CalibratorVdExB.h b/Detectors/TRD/calibration/include/TRDCalibration/CalibratorVdExB.h index 66cc75276ab4c..abccfd16fe191 100644 --- a/Detectors/TRD/calibration/include/TRDCalibration/CalibratorVdExB.h +++ b/Detectors/TRD/calibration/include/TRDCalibration/CalibratorVdExB.h @@ -20,6 +20,8 @@ #include "DetectorsCalibration/TimeSlot.h" #include "DataFormatsTRD/Constants.h" #include "DataFormatsTRD/AngularResidHistos.h" +#include "CCDB/CcdbObjectInfo.h" +#include "DataFormatsTRD/CalVdriftExB.h" #include "Rtypes.h" #include "TProfile.h" @@ -60,12 +62,17 @@ class CalibratorVdExB final : public o2::calibration::TimeSlotCalibration& getCcdbObjectVector() const { return mObjectVector; } + std::vector& getCcdbObjectInfoVector() { return mInfoVector; } + void initProcessing(); private: bool mInitDone{false}; ///< flag to avoid creating the TProfiles multiple times size_t mMinEntries; ///< minimum total number of angular deviations (on average ~3 entries per bin for each TRD chamber) FitFunctor mFitFunctor; ///< used for minimization procedure + std::vector mInfoVector; ///< vector of CCDB infos; each element is filled with CCDB description of accompanying CCDB calibration object + std::vector mObjectVector; ///< vector of CCDB calibration objects; the extracted vDrift and ExB per chamber for given slot ClassDefOverride(CalibratorVdExB, 1); }; diff --git a/Detectors/TRD/calibration/src/CalibratorVdExB.cxx b/Detectors/TRD/calibration/src/CalibratorVdExB.cxx index 643bcd0acae20..e11c5bd5a1d8b 100644 --- a/Detectors/TRD/calibration/src/CalibratorVdExB.cxx +++ b/Detectors/TRD/calibration/src/CalibratorVdExB.cxx @@ -14,7 +14,6 @@ /// \author Ole Schmidt #include "TRDCalibration/CalibratorVdExB.h" -#include "DataFormatsTRD/CalVdriftExB.h" #include "Fit/Fitter.h" #include "TStopwatch.h" #include "CCDB/CcdbApi.h" @@ -23,6 +22,7 @@ #include #include #include "CommonUtils/NameConf.h" +#include "CommonUtils/MemFileHelper.h" using namespace o2::trd::constants; @@ -89,8 +89,9 @@ using Slot = o2::calibration::TimeSlot; void CalibratorVdExB::initOutput() { - // prepare output objects which will go to CCDB - // nothing to be done + // reset the CCDB output vectors + mInfoVector.clear(); + mObjectVector.clear(); } void CalibratorVdExB::initProcessing() @@ -153,11 +154,7 @@ void CalibratorVdExB::finalizeSlot(Slot& slot) timer.Stop(); LOGF(info, "Done fitting angular residual histograms. CPU time: %f, real time: %f", timer.CpuTime(), timer.RealTime()); - // write results to CCDB - o2::ccdb::CcdbApi ccdb; - ccdb.init(o2::base::NameConf::getCCDBServer()); - // ccdb.init("http://localhost:8080"); - std::map metadata; // TODO: do we want to store any meta data? + // assemble CCDB object CalVdriftExB calObject; for (int iDet = 0; iDet < MAXCHAMBER; ++iDet) { // OS: what about chambers for which we had no data in this slot? should we use the initial parameters or something else? @@ -165,10 +162,12 @@ void CalibratorVdExB::finalizeSlot(Slot& slot) calObject.setVdrift(iDet, vdFitResults[iDet]); calObject.setExB(iDet, laFitResults[iDet]); } - auto timeStamp = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); - auto timeStampEnd = timeStamp; - timeStampEnd += 1e3 * 60 * 60 * 24 * 7; // set validity of 7 days - ccdb.storeAsTFileAny(&calObject, "TRD/Calib/CalVdriftExB", metadata, timeStamp, timeStampEnd); + auto clName = o2::utils::MemFileHelper::getClassName(calObject); + auto flName = o2::ccdb::CcdbApi::generateFileName(clName); + std::map metadata; // TODO: do we want to store any meta data? + long startValidity = slot.getStartTimeMS() - 10 * o2::ccdb::CcdbObjectInfo::SECOND; + mInfoVector.emplace_back("TRD/Calib/CalVdriftExB", clName, flName, metadata, startValidity, startValidity + o2::ccdb::CcdbObjectInfo::HOUR); + mObjectVector.push_back(calObject); } Slot& CalibratorVdExB::emplaceNewSlot(bool front, TFType tStart, TFType tEnd) diff --git a/Detectors/TRD/workflow/include/TRDWorkflow/VdAndExBCalibSpec.h b/Detectors/TRD/workflow/include/TRDWorkflow/VdAndExBCalibSpec.h index 6b39297b04469..4b29a0dc388d9 100644 --- a/Detectors/TRD/workflow/include/TRDWorkflow/VdAndExBCalibSpec.h +++ b/Detectors/TRD/workflow/include/TRDWorkflow/VdAndExBCalibSpec.h @@ -43,10 +43,10 @@ class VdAndExBCalibDevice : public o2::framework::Task { o2::base::GRPGeomHelper::instance().setRequest(mCCDBRequest); int minEnt = ic.options().get("min-entries"); - auto slotL = ic.options().get("tf-per-slot"); + auto slotL = ic.options().get("sec-per-slot"); auto delay = ic.options().get("max-delay"); mCalibrator = std::make_unique(minEnt); - mCalibrator->setSlotLength(slotL); + mCalibrator->setSlotLengthInSeconds(slotL); mCalibrator->setMaxSlotsDelay(delay); } @@ -78,9 +78,26 @@ class VdAndExBCalibDevice : public o2::framework::Task //________________________________________________________________ void sendOutput(DataAllocator& output) { - // See LHCClockCalibratorSpec.h - // Before this can be implemented the output CCDB objects need to be defined - // and added to CalibratorVdExB + // extract CCDB infos and calibration objects, convert it to TMemFile and send them to the output + // TODO in principle, this routine is generic, can be moved to Utils.h + + using clbUtils = o2::calibration::Utils; + const auto& payloadVec = mCalibrator->getCcdbObjectVector(); + auto& infoVec = mCalibrator->getCcdbObjectInfoVector(); // use non-const version as we update it + assert(payloadVec.size() == infoVec.size()); + + for (uint32_t i = 0; i < payloadVec.size(); i++) { + auto& w = infoVec[i]; + auto image = o2::ccdb::CcdbApi::createObjectImage(&payloadVec[i], &w); + LOG(info) << "Sending object " << w.getPath() << "/" << w.getFileName() << " of size " << image->size() + << " bytes, valid for " << w.getStartValidityTimestamp() << " : " << w.getEndValidityTimestamp(); + + output.snapshot(Output{clbUtils::gDataOriginCDBPayload, "VDRIFTEXB", i}, *image.get()); // vector + output.snapshot(Output{clbUtils::gDataOriginCDBWrapper, "VDRIFTEXB", i}, w); // root-serialized + } + if (payloadVec.size()) { + mCalibrator->initOutput(); // reset the outputs once they are already sent + } } }; @@ -95,8 +112,8 @@ DataProcessorSpec getTRDVdAndExBCalibSpec() using clbUtils = o2::calibration::Utils; std::vector outputs; - //outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCLB, clbUtils::gDataDescriptionCLBPayload}); - //outputs.emplace_back(ConcreteDataTypeMatcher{clbUtils::gDataOriginCLB, clbUtils::gDataDescriptionCLBInfo}); + outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBPayload, "VDRIFTEXB"}, Lifetime::Sporadic); + outputs.emplace_back(ConcreteDataTypeMatcher{o2::calibration::Utils::gDataOriginCDBWrapper, "VDRIFTEXB"}, Lifetime::Sporadic); std::vector inputs{{"input", "TRD", "ANGRESHISTS"}}; auto ccdbRequest = std::make_shared(true, // orbitResetTime true, // GRPECS=true @@ -111,9 +128,9 @@ DataProcessorSpec getTRDVdAndExBCalibSpec() outputs, AlgorithmSpec{adaptFromTask(ccdbRequest)}, Options{ - {"tf-per-slot", VariantType::UInt32, 5u, {"number of TFs per calibration time slot"}}, - {"max-delay", VariantType::UInt32, 90'000u, {"number of slots in past to consider"}}, // 15 minutes delay, 10ms TF - {"min-entries", VariantType::Int, 500, {"minimum number of entries to fit single time slot"}}}}; + {"sec-per-slot", VariantType::UInt32, 900u, {"number of seconds per calibration time slot"}}, + {"max-delay", VariantType::UInt32, 2u, {"number of slots in past to consider"}}, + {"min-entries", VariantType::Int, 40'000, {"minimum number of entries to fit single time slot"}}}}; // around 3 entries per bin per chamber } } // namespace framework diff --git a/Detectors/TRD/workflow/src/trd-calib-workflow.cxx b/Detectors/TRD/workflow/src/trd-calib-workflow.cxx index eba8461f9f590..b1fdd3eb996b1 100644 --- a/Detectors/TRD/workflow/src/trd-calib-workflow.cxx +++ b/Detectors/TRD/workflow/src/trd-calib-workflow.cxx @@ -20,7 +20,8 @@ void customize(std::vector& workflowOptions) { // option allowing to set parameters std::vector options{ - {"enable-root-input", o2::framework::VariantType::Bool, false, {"enable root-files input readers"}}}; + {"enable-root-input", o2::framework::VariantType::Bool, false, {"enable root-files input readers"}}, + {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}}; std::swap(workflowOptions, options); } From 3b0bccb6c137bd9e43df0de1e9a97af2d3cf8d87 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 26 Apr 2022 17:30:30 +0200 Subject: [PATCH 2/3] Enable TRD vDrift calib for data taking with beam --- prodtests/full-system-test/calib-workflow.sh | 1 + prodtests/full-system-test/dpl-workflow.sh | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/prodtests/full-system-test/calib-workflow.sh b/prodtests/full-system-test/calib-workflow.sh index 622e1baad4d0e..e4e00ee30d901 100644 --- a/prodtests/full-system-test/calib-workflow.sh +++ b/prodtests/full-system-test/calib-workflow.sh @@ -8,6 +8,7 @@ fi if [[ $BEAMTYPE != "cosmic" ]]; then has_detector_calib TPC && has_detectors TPC ITS TRD TOF && add_W o2-tpc-scdcalib-interpolation-workflow "$DISABLE_ROOT_OUTPUT --disable-root-input --pipeline $(get_N tpc-track-interpolation TPC REST)" "$ITSMFT_FILES" has_detector_calib ITS && has_detectors ITS && has_detectors_reco ITS && has_detector_matching PRIMVTX && [[ ! -z "$VERTEXING_SOURCES" ]] && add_W o2-calibration-mean-vertex-calibration-workflow + has_detector_calib TRD && has_detector ITS TPC TRD && add_W o2-calibration-trd-vdrift-exb fi true # everything OK up to this point, so the script should return 0 (it is !=0 already if a has_detector check fails) diff --git a/prodtests/full-system-test/dpl-workflow.sh b/prodtests/full-system-test/dpl-workflow.sh index 55bc80be80bc0..1bb6b8f962c57 100755 --- a/prodtests/full-system-test/dpl-workflow.sh +++ b/prodtests/full-system-test/dpl-workflow.sh @@ -118,10 +118,12 @@ if [[ $SYNCMODE == 1 ]]; then ITS_CONFIG_KEY+="fastMultConfig.cutMultClusLow=30;fastMultConfig.cutMultClusHigh=2000;fastMultConfig.cutMultVtxHigh=500;" [[ -z ${ITS_CONFIG+x} ]] && ITS_CONFIG=" --tracking-mode sync" [[ -z ${PVERTEXING_CONFIG_KEY+x} ]] && PVERTEXING_CONFIG_KEY+="pvertexer.maxChi2TZDebris=2000;" + workflow_has_parameter CALIB && TRD_CONFIG+=" --enable-trackbased-calib" elif [[ $BEAMTYPE == "pp" ]]; then ITS_CONFIG_KEY+="fastMultConfig.cutMultClusLow=-1;fastMultConfig.cutMultClusHigh=-1;fastMultConfig.cutMultVtxHigh=-1;ITSVertexerParam.phiCut=0.5;ITSVertexerParam.clusterContributorsCut=3;ITSVertexerParam.tanLambdaCut=0.2" [[ -z ${ITS_CONFIG+x} ]] && ITS_CONFIG=" --tracking-mode sync" [[ -z ${PVERTEXING_CONFIG_KEY+x} ]] && PVERTEXING_CONFIG_KEY+="pvertexer.maxChi2TZDebris=10;" + workflow_has_parameter CALIB && TRD_CONFIG+=" --enable-trackbased-calib" elif [[ $BEAMTYPE == "cosmic" ]]; then [[ -z ${ITS_CONFIG+x} ]] && ITS_CONFIG=" --tracking-mode cosmics" else @@ -134,10 +136,12 @@ else if [[ $BEAMTYPE == "PbPb" ]]; then [[ -z ${ITS_CONFIG+x} ]] && ITS_CONFIG=" --tracking-mode async" [[ -z ${PVERTEXING_CONFIG_KEY+x} ]] && PVERTEXING_CONFIG_KEY+="pvertexer.maxChi2TZDebris=2000;" + workflow_has_parameter CALIB && TRD_CONFIG+=" --enable-trackbased-calib" elif [[ $BEAMTYPE == "pp" ]]; then ITS_CONFIG_KEY+="ITSVertexerParam.phiCut=0.5;ITSVertexerParam.clusterContributorsCut=3;ITSVertexerParam.tanLambdaCut=0.2" [[ -z ${ITS_CONFIG+x} ]] && ITS_CONFIG=" --tracking-mode async" [[ -z ${PVERTEXING_CONFIG_KEY+x} ]] && PVERTEXING_CONFIG_KEY+="pvertexer.maxChi2TZDebris=10;" + workflow_has_parameter CALIB && TRD_CONFIG+=" --enable-trackbased-calib" elif [[ $BEAMTYPE == "cosmic" ]]; then [[ -z ${ITS_CONFIG+x} ]] && ITS_CONFIG=" --tracking-mode cosmics" else From bf8606d31e78cdba5ad2f63808e204301d91f580 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Thu, 5 May 2022 14:08:30 +0200 Subject: [PATCH 3/3] TrackBasedCalib sends output for every TF --- .../TRD/workflow/src/TrackBasedCalibSpec.cxx | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/Detectors/TRD/workflow/src/TrackBasedCalibSpec.cxx b/Detectors/TRD/workflow/src/TrackBasedCalibSpec.cxx index 143b7aa4138ab..f3f947fa33cd7 100644 --- a/Detectors/TRD/workflow/src/TrackBasedCalibSpec.cxx +++ b/Detectors/TRD/workflow/src/TrackBasedCalibSpec.cxx @@ -24,6 +24,7 @@ #include "DataFormatsParameters/GRPObject.h" #include "Headers/DataHeader.h" #include "DataFormatsGlobalTracking/RecoContainer.h" +#include "TStopwatch.h" using namespace o2::framework; using namespace o2::globaltracking; @@ -49,9 +50,7 @@ class TRDTrackBasedCalibDevice : public Task std::shared_ptr mDataRequest; TrackBasedCalib mCalibrator; // gather input data for calibration of vD, ExB and gain - std::unique_ptr mOutput; - uint32_t mNumberOfProcessedTFs{0}; - bool mDataHeaderSet{false}; + TStopwatch mTimer; }; void TRDTrackBasedCalibDevice::init(InitContext& ic) @@ -61,26 +60,21 @@ void TRDTrackBasedCalibDevice::init(InitContext& ic) o2::base::Propagator::initFieldFromGRP(); std::unique_ptr grp{o2::parameters::GRPObject::loadFrom()}; mCalibrator.init(); + mTimer.Stop(); + mTimer.Reset(); } void TRDTrackBasedCalibDevice::run(ProcessingContext& pc) { updateTimeDependentParams(pc); - if (!mDataHeaderSet) { - mOutput = std::make_unique(o2::header::gDataOriginTRD, "ANGRESHISTS", 0, Lifetime::Timeframe); - mDataHeaderSet = true; - } + mTimer.Start(false); RecoContainer recoData; recoData.collectData(pc, *mDataRequest.get()); mCalibrator.setInput(recoData); mCalibrator.calculateAngResHistos(); - ++mNumberOfProcessedTFs; - if (mNumberOfProcessedTFs % 200 == 0) { - pc.outputs().snapshot(*mOutput, mCalibrator.getAngResHistos()); - mDataHeaderSet = false; - mNumberOfProcessedTFs = 0; - mCalibrator.reset(); - } + pc.outputs().snapshot(Output{o2::header::gDataOriginTRD, "ANGRESHISTS", 0, Lifetime::Timeframe}, mCalibrator.getAngResHistos()); + mCalibrator.reset(); + mTimer.Stop(); } void TRDTrackBasedCalibDevice::updateTimeDependentParams(ProcessingContext& pc) @@ -98,11 +92,8 @@ void TRDTrackBasedCalibDevice::finaliseCCDB(ConcreteDataMatcher& matcher, void* void TRDTrackBasedCalibDevice::endOfStream(EndOfStreamContext& ec) { - if (mNumberOfProcessedTFs > 0) { - ec.outputs().snapshot(*mOutput, mCalibrator.getAngResHistos()); - } - LOGF(info, "Added in total %i entries to angular residual histograms", - mCalibrator.getAngResHistos().getNEntries()); + LOGF(info, "TRD track-based calibration total timing: Cpu: %.3e Real: %.3e s in %d slots", + mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1); } DataProcessorSpec getTRDTrackBasedCalibSpec(o2::dataformats::GlobalTrackID::mask_t src)