Subtract extra leap second from GCD file

scripts/icetray/step2gcd/AddSLCCalibration_fromjson.py

@@ -0,0 +1,96 @@
+##
+# SPDX-FileCopyrightText: 2024 The IceTray Contributors
+# SPDX-License-Identifier: BSD-2-Clause
+#
+# A module which fills an "I3IceTopSLCCalibrationCollection" structure in a C-frame,
+# This is a stopgap measure, for until this is done automatically during "new processing"
+# or for historical (i.e. pass3) data.
+# ..... SHOULD WE MOVE THIS TO INTO ICETRAY?
+##
+
+#from datetime import datetime
+import json
+#import numpy as np
+#import re
+
+from icecube import dataclasses, icetray
+from icecube.icetray.i3logging import log_debug, log_info, log_warn, log_fatal
+
+class AddSLCCalibrationCollection_fromjson(icetray.I3Module):
+    """Take a GCD file and modify it according to a json file (of the kind that would be injected into the DB)
+
+    Adds a I3IceTopSLCCalibrationCollection frame object to the C frame with the SLC calibrations constants for all IceTop DOMs
+    (also for the "chip unknown" case).
+    """
+
+    def __init__(self, ctx):
+        super().__init__(ctx)
+        self.AddParameter('SLCCalibFile', 'SLC calibration .json (of the kind that would be injected into the DB')
+        self.AddParameter('Provenance', 'What enum is the Provenance for these numbers (where are they from)?', dataclasses.ITSLCCalProvenance.Placeholder)
+        self.AddParameter('CollectionName', 'What the frame object should be named', "I3IceTopSLCCalibrationCollection")
+        self.AddOutBox('OutBox')
+
+    def Configure(self):
+        self.slccalibfile = self.GetParameter('SLCCalibFile')
+        self.prov = self.GetParameter('Provenance')
+        self.collection_name = self.GetParameter('CollectionName')
+
+    def Calibration(self, frame):
+        # The new object
+        calibration_collection = dataclasses.I3IceTopSLCCalibrationCollection()
+
+        # Load the big dictionary
+        with open(self.slccalibfile) as f:  # the default is read "r" mode
+            j = json.load(f)
+
+        # Fill the I3FrameObject structure
+        calibration_collection.start_run = j["StartRun"]
+        calibration_collection.end_run = j["EndRun"]
+        d = j["data"]
+        for m in d:  # Loop through all the maps: one per DOM
+            omkey = icetray.OMKey(m["StringId"], m["OmId"])
+            thisvalue = dataclasses.I3IceTopSLCCalibration()
+            thisvalue.SetIntercept(0, 0, m['Intercept_C0A0'])
+            thisvalue.SetIntercept(0, 1, m['Intercept_C0A1'])
+            thisvalue.SetIntercept(0, 2, m['Intercept_C0A2'])
+            thisvalue.SetIntercept(1, 0, m['Intercept_C1A0'])
+            thisvalue.SetIntercept(1, 1, m['Intercept_C1A1'])
+            thisvalue.SetIntercept(1, 2, m['Intercept_C1A2'])
+            thisvalue.SetIntercept(-1, 0, m['Intercept_CunkA0'])
+            thisvalue.SetIntercept(-1, 1, m['Intercept_CunkA1'])
+            thisvalue.SetIntercept(-1, 2, m['Intercept_CunkA2'])
+            thisvalue.SetSlope(0, 0, m['Slope_C0A0'])
+            thisvalue.SetSlope(0, 1, m['Slope_C0A1'])
+            thisvalue.SetSlope(0, 2, m['Slope_C0A2'])
+            thisvalue.SetSlope(1, 0, m['Slope_C1A0'])
+            thisvalue.SetSlope(1, 1, m['Slope_C1A1'])
+            thisvalue.SetSlope(1, 2, m['Slope_C1A2'])
+            thisvalue.SetSlope(-1, 0, m['Slope_CunkA0'])
+            thisvalue.SetSlope(-1, 1, m['Slope_CunkA1'])
+            thisvalue.SetSlope(-1, 2, m['Slope_CunkA2'])
+
+            calibration_collection.it_slc_cal[omkey] = thisvalue
+
+        calibration_collection.provenance = self.prov
+
+        # Add the I3IceTopSLCCalibrationCollection to the frame...
+        # ---------------------------------------------------
+        # First, check to see if it already exists. 
+        # We have to do this because is someone invokes gcdserver to make this object, and there is no DB info for the run yet,
+        # gcdserver will create an object, but it'll be empty.
+        if frame.Has(self.collection_name):
+            # It does exist! Check if it's empty.  (It should be!)
+            c = frame[self.collection_name]
+            if len(c.it_slc_cal) != 0:
+                log_fatal("I found a preexisting %s, of non-zero length %d"%(self.collection_name, len(c.it_slc_cal)))
+            if (c.start_run != 0) or (c.end_run != 0):
+                log_fatal("I found a preexisting %s, with non-zero start/end runs %d %d"%(c.start_run, c.end_run))
+            if (c.provenance != 0):
+                log_fatal("I found a preexisting %s, with non-zero provenance %d"%c.prov)
+            # If we survived all that checking, then the existint object is empty, and we can go ahead and delete it from the frame.
+            log_info("So you know, I found an empty %s and I'm deleting it to make a new one."%self.collection_name)
+            frame.Delete(self.collection_name)
+        # Now, put the new awesome one in there!
+        frame.Put(self.collection_name, calibration_collection)
+        self.PushFrame(frame)       # push the C-frame
+

scripts/icetray/step2gcd/condor_submit/submit_pass3step2gcd_leapfix.condor

@@ -0,0 +1,29 @@
+my_work = /data/user/i3filter
+my_scratch = /scratch/i3filter
+my_dir = IC86_OfflineProcessing/OfflineSubmitScripts_pass3step2gcd_leapfix_test
+
+initialdir = $(my_work)/$(my_dir)
+executable = $(initialdir)/src/pass3/leapfix/scripts/icetray/step2gcd/leapfix_gcdfile.sh 
+
+request_memory = 7000
+
+should_transfer_files = YES
+
+condor_file = $(my_scratch)/$(my_dir)/$Fn(Item).$(Cluster).$(Process).condor
+log    = $(condor_file).log
+output = $(condor_file).out
+error  = $(condor_file).err
+
+#notification = Error
+notify_user = rsnihur@icecube.wisc.edu
+
+arguments = $(Item)
+
+# 1 job
+#queue Item matching /data/ana/IceCube/2017/filtered/OfflinePass3.0/0301/Run00129239/OfflinePass3_IC86.2016_data_Run00129239_0301_90_299_GCD.i3.zst
+# 1 job
+#queue Item matching /data/ana/IceCube/2017/filtered/OfflinePass3.0/0309/Run00129271/OfflinePass3_IC86.2016_data_Run00129271_0309_90_300_GCD.i3.zst
+# 92 jobs
+#queue Item matching /data/ana/IceCube/2017/filtered/OfflinePass3.0/03??/Run00??????/*_data_Run00*_GCD.i3.zst
+# 435 jobs
+queue Item matching /data/ana/IceCube/2017/filtered/OfflinePass3.0/0[1-5]??/Run00??????/*2016_data_Run00*_GCD.i3.zst

scripts/icetray/step2gcd/icetopify_gcdfile.py

@@ -0,0 +1,147 @@
+# SPDX-FileCopyrightText: 2025 The IceTray Contributors
+#
+# SPDX-License-Identifier: BSD-2-Clause
+
+"""Support functions and tray segements for adding IceTop calibration objects to pass3step1 GCD files"""
+
+'''
+HOW TO RUN IT:
+python icetopify_gcdfile.py -i /data/ana/IceCube/2022/filtered/OnlinePass3.6/GCD/OnlinePass3_IC86.2021_data_Run00136124_81_647_GCD.i3.zst -o test_GCD.i3 --log-file test_GCD.log
+'''
+
+
+
+import re
+import argparse
+from icecube.icetray import I3Tray, OMKey
+from icecube import icetray, dataclasses, dataio  #noqa: F401
+from icecube.icetray import logging
+from icecube.offline_filterscripts.icetop_GCDmodification.overwrite_snowheights import ChangeSnowHeights_FromDB
+from icecube.offline_filterscripts.icetop_GCDmodification.add_ATWDcrossovers_to_Dframe import Add_ATWDCrossoverMap_Dframe
+from AddSLCCalibration_fromjson import AddSLCCalibrationCollection_fromjson
+
+## ------------------- HELPER FUNCTIONS --------------------------------
+## For parsing filenames to get the Run Number. 
+## This is stolen from GITHUB/wg-cosmic-rays/IceTop_Level4/filename_utils.py
+def get_run_from_filename(input_file):
+    result = None
+    m = re.search("Run([0-9]+)", input_file)
+    if not m:
+        raise ValueError("cannot parse %s for Run number" % input_file)
+    return int(m.group(1))
+
+## Create a ATWDCrossoverMap full of simple placeholder values
+def placeholder_for_Dframe(frame, objectname="IceTop_ATWDCrossoverMap"):
+    newobject = dataclasses.I3MapKeyVectorDouble()
+    # Loop through all the things in DetectorStatus (dead dom's will not be in there!)
+    ds = frame["I3DetectorStatus"].dom_status
+    for k, v in ds.items():
+        if (k.om >= 61) and (k.om <= 64):
+            # We found an IceTop DOM
+            #print(k, v.dom_gain_type)
+            # Pick some "typical" values, depending on whether it's HG or LG
+            if v.dom_gain_type == dataclasses.I3DOMStatus.DOMGain.High:
+                newobject[k] = [85, 750]
+            elif v.dom_gain_type == dataclasses.I3DOMStatus.DOMGain.Low:
+                newobject[k] = [4800, 48000]
+            else:
+                raise Exception("IceTop DOM with unknown gain type %s"%v.dom_gain_type)
+    # Put the new object in the frame
+    frame[objectname] = newobject
+
+## ------------------- THE TRAY --------------------------------
+def runme(infile, outfile, *, testmode=True, **kwargs): #, *, skip_audit=False, production_version=-1, **kwargs):
+    """Run procedure for adding IceTop calibration objects to a GCD file."""
+
+    # Pipe the log stuff somewhere?
+    if kwargs["logfile"]:
+        logging.rotating_files(kwargs["logfile"])
+
+    # Set log level
+    logging.set_level("INFO")
+
+    # Did we give it a run number?  If not, figure it out from the input filename
+    if not kwargs["run_id"]:
+        run_id = get_run_from_filename(infile)
+
+    # Set up the Tray.
+    tray = I3Tray()
+
+    # Read input
+    logging.log_info("Reading from: %s"%infile)
+    tray.AddModule("I3Reader", "readme", Filename=infile)
+
+    tray.AddModule("Dump", "dumpme")   # for testing!
+
+    # Deal with SNOW
+    tray.Add(ChangeSnowHeights_FromDB, "updateSnowHeights", Run = run_id)
+
+    # Deal with VEMCal
+    # <insert me when the time comes!!>
+
+    # Deal with SLCCal
+    ## This bit is temporary, for pass3step2 TESTING ONLY.  If Kath hasn't computed ITSLC calibration constants yet,
+    ## then we'll create some fake ones, designed to be really obvious (if looked at) that they are fake.
+    if testmode:
+        ## Create the C-frame object, using a placeholder .json file containing rounded-off numbers
+        if run_id >= 136442:  # Orc is dead
+            placeholder_filename = "placeholder_IceTopSLCCal_deadOrc.json"
+        else:  # Orc is alive
+            placeholder_filename = "placeholder_IceTopSLCCal_liveOrc.json"
+        logging.log_warn("I'm going to create a fake set of IceTop SLC constants, from %s"%placeholder_filename)
+        tray.Add(AddSLCCalibrationCollection_fromjson, 'add_Cframe_placeholder',
+             SLCCalibFile   = placeholder_filename,
+             Provenance = dataclasses.ITSLCCalProvenance.Placeholder  # This will tell future people that it's fake  
+         )
+        ## Create the D-frame object, with more simple placeholder values, so that future people will recognize it as fake
+        tray.Add(placeholder_for_Dframe, "add_Dframe_placeholder",
+                 Streams=[icetray.I3Frame.DetectorStatus])
+    else:
+        ## TO-DO: Create the C-frame object for realsies!
+        ## Create the D-frame object, for realsies.
+        tray.Add(Add_ATWDCrossoverMap_Dframe, "makeCrossoverMap")
+
+    # Write output
+    logging.log_info("Writing to: %s"%outfile)
+    streams = [icetray.I3Frame.Geometry, icetray.I3Frame.Calibration,
+               icetray.I3Frame.TrayInfo,
+               icetray.I3Frame.DetectorStatus]
+    tray.Add("I3Writer", filename=outfile, Streams=streams)
+
+    # Execute!
+    tray.Execute()
+
+    #tray.PrintUsage(fraction=1.0)
+    tray.Finish()
+    del tray
+
+
+## ------------------- THE EXECUTABLE ----------------------
+def main():
+    """Run setup arguments and run main function"""
+    description = """Update a step1 GCD file with juicy IceTop goodness for step2."""
+
+    # Parse the arguments
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument("-i", "--infile", type=str, required=True,
+                        help="Input GCD i3 filename")
+    parser.add_argument("-o", "--outfile", type=str, required=True,
+                        help="Output GCD i3 filename")
+    parser.add_argument("-r", "--run", type=int, dest = "run_id", required=False,
+                        help="Run Number")
+    '''
+    parser.add_argument("--skip_audit", default=False, action="store_true",
+                        help="Skip auditor")
+    parser.add_argument("--production-version", dest = "production_version",
+                        type=int, default = -1,
+                        help="Set production version")
+    '''
+    parser.add_argument("--log-file", dest = "logfile",
+                        type=str, help = "File where logs will be written")
+    args = parser.parse_args()
+
+    # Run it!
+    runme(**vars(args))
+
+if __name__ == "__main__":
+    main()

scripts/icetray/step2gcd/leapfix_gcdfile.py

@@ -0,0 +1,113 @@
+# SPDX-FileCopyrightText: 2025 The IceTray Contributors
+#
+# SPDX-License-Identifier: BSD-2-Clause
+
+"""Support functions and tray segements for subtracting extra leap second in pass3step2 GCD files"""
+
+'''
+HOW TO RUN IT:
+python leapfix_gcdfile.py -i /data/ana/IceCube/2017/filtered/OfflinePass3.0/0301/Run00129239/OfflinePass3_IC86.2016_data_Run00129239_0301_90_299_GCD.i3.zst -o test_GCD.i3.zst
+'''
+
+
+import copy
+import re
+import argparse
+from icecube.icetray import I3Tray
+from icecube.icetray import I3Units
+from icecube import icetray, dataclasses, dataio  #noqa: F401
+from icecube.icetray import logging
+
+## ------------------- HELPER FUNCTIONS --------------------------------
+## For parsing filenames to get the Run Number. 
+## This is stolen from GITHUB/wg-cosmic-rays/IceTop_Level4/filename_utils.py
+def get_run_from_filename(input_file):
+    result = None
+    m = re.search("Run([0-9]+)", input_file)
+    if not m:
+        raise ValueError("cannot parse %s for Run number" % input_file)
+    return int(m.group(1))
+
+def subtract_leap_second(frame, key=None):
+    grt = frame[key]
+    old_grt = frame[key]
+    grt -= I3Units.second
+    del frame[key]
+    frame[key] = grt
+    frame["Old" + key] = old_grt
+    return True
+
+## ------------------- THE TRAY --------------------------------
+def runme(infile, outfile, *, testmode=False, **kwargs):
+    """Procedure to subtract 1 second from GoodRunStartTime and GoodRunEndTime in a GCD file."""
+
+    # Pipe the log stuff somewhere?
+    if kwargs["logfile"]:
+        logging.rotating_files(kwargs["logfile"])
+
+    # Set log level
+    logging.set_level("INFO")
+
+    logging.log_info("Reading from: %s"%infile)
+    # Did we give it a run number?  If not, figure it out from the input filename
+    if not kwargs["run_id"]:
+        run_id = get_run_from_filename(infile)
+    else:
+        run_id = kwargs["run_id"]
+
+    run_id_min = 129004 # 2017-01-01 first run of calendar yeqar
+    run_id_max = 129519 # 2017-05-18 last run of IC86-2016
+    if (run_id < run_id_min or run_id > run_id_max):
+        raise Exception(f"This run number {run_id} is outside the range {run_id_min} to {run_id_max}")
+
+    # Set up the Tray.
+    tray = I3Tray()
+
+    tray.AddModule("I3Reader", "readme", Filename=infile)
+
+    tray.Add(subtract_leap_second,
+             key="GoodRunStartTime",
+             Streams=[icetray.I3Frame.DetectorStatus]
+             )
+
+    tray.Add(subtract_leap_second,
+             key="GoodRunEndTime",
+             Streams=[icetray.I3Frame.DetectorStatus]
+             )
+
+    # Write output
+    logging.log_info("Writing to: %s"%outfile)
+    streams = [icetray.I3Frame.Geometry, icetray.I3Frame.Calibration,
+               icetray.I3Frame.TrayInfo,
+               icetray.I3Frame.DetectorStatus]
+    icetray.logging.set_level_for_unit("I3Module", "WARN") # Prevents printing usage stats.
+    tray.Add("I3Writer", filename=outfile, Streams=streams)
+
+    tray.Execute()
+
+    tray.Finish()
+    del tray
+
+
+## ------------------- THE EXECUTABLE ----------------------
+def main():
+    """Run setup arguments and run main function"""
+    description = """Update a GCD file for pass3."""
+
+    # Parse the arguments
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument("-i", "--infile", type=str, required=True,
+                        help="Input GCD i3 filename")
+    parser.add_argument("-o", "--outfile", type=str, required=True,
+                        help="Output GCD i3 filename")
+    parser.add_argument("-r", "--run", type=int, dest = "run_id", required=False,
+                        help="Run Number")
+    parser.add_argument("--log-file", dest = "logfile",
+                        type=str, help = "File where logs will be written")
+    args = parser.parse_args()
+
+    # Run it!
+    runme(**vars(args))
+
+if __name__ == "__main__":
+    main()