diff --git a/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml index a45da11ae..005085e52 100644 --- a/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_global_cdf_attrs.yaml @@ -5,14 +5,15 @@ instrument_base: &instrument_base The Compact Dual Ion Composition Experiment (CoDICE) will measure the distributions and composition of interstellar pickup ions (PUIs), particles that make it through the heliosheath into the heliosphere. CoDICE also collects and characterizes solar wind ions including the mass and composition of highly energized particles (called suprathermal) - from the Sun. CoDICE combines an electrostatic analyzer(ESA) with a Time-Of-Flight versus Energy (TOF / E) subsystem - to simultaneously measure the velocity, arrival direction, ionic charge state, and mass of specific species of ions in - the LISM. CoDICE also has a path for higher energy particles to skip the ESA but still get measured by the common - TOF / E system. These measurements are critical in determining the Local Interstellar Medium (LISM) composition and - flow properties, the origin of the enigmatic suprathermal tails on the solar wind distributions and advance - understanding of the acceleration of particles in the heliosphere. + from the Sun. CoDICE combines an electrostatic analyzer (ESA) with a Time-Of-Flight versus Energy (TOF / E) + subsystem to simultaneously measure the velocity, arrival direction, ionic charge state, and mass of specific + species of ions in the LISM. CoDICE also has a path for higher energy particles to skip the ESA but still get + measured by the common TOF / E system. These measurements are critical in determining the Local Interstellar Medium + (LISM) composition and flow properties, the origin of the enigmatic suprathermal tails on the solar wind + distributions and advance understanding of the acceleration of particles in the heliosphere. Instrument_type: Particles (space) +# L1a imap_codice_l1a_hskp: <<: *instrument_base Data_level: 1A @@ -102,4 +103,96 @@ imap_codice_l1a_lo_nsw_species: Data_level: 1A Data_type: L1A_lo-nsw-species->Level-1A Lo Non-Sunward Species Counts Data Logical_source: imap_codice_l1a_lo-nsw-species - Logical_source_description: IMAP Mission CoDICE Lo Level-1A Non-Sunward Species Counts Data. \ No newline at end of file + Logical_source_description: IMAP Mission CoDICE Lo Level-1A Non-Sunward Species Counts Data. + +# L1b +imap_codice_l1b_hskp: + <<: *instrument_base + Data_level: 1B + Data_type: L1B_hskp->Level-1B Housekeeping Data + Logical_source: imap_codice_l1b_hskp + Logical_source_description: IMAP Mission CoDICE Instrument Level-1B Housekeeping Data. + +imap_codice_l1b_hi_counters_aggregated: + <<: *instrument_base + Data_level: 1B + Data_type: L1B_hi-counters-aggregated->Level-1B Hi Aggregated Instrument Rates Data + Logical_source: imap_codice_l1b_hi-counters-aggregated + Logical_source_description: IMAP Mission CoDICE Hi Level-1B Aggregated Instrument Rates Data. + +imap_codice_l1b_hi_counters_singles: + <<: *instrument_base + Data_level: 1B + Data_type: L1B_hi-counters-singles->Level-1B Hi Single Instrument Rates Data + Logical_source: imap_codice_l1b_hi-counters-singles + Logical_source_description: IMAP Mission CoDICE Hi Level-1B Single Instrument Rates Data. + +imap_codice_l1b_hi_omni: + <<: *instrument_base + Data_level: 1B + Data_type: L1B_hi-omni->Level-1B Hi Omnidirectional Data + Logical_source: imap_codice_l1b_hi-omni + Logical_source_description: IMAP Mission CoDICE Hi Level-1B Omnidirectional Data. + +imap_codice_l1b_hi_sectored: + <<: *instrument_base + Data_level: 1B + Data_type: L1B_hi-sectored->Level-1B Hi Sectored Data + Logical_source: imap_codice_l1b_hi-sectored + Logical_source_description: IMAP Mission CoDICE Hi Level-1B Sectored Data. + +imap_codice_l1b_lo_counters_aggregated: + <<: *instrument_base + Data_level: 1B + Data_type: L1B_lo-counters-aggregated->Level-1B Lo Aggregated Instrument Rates Data + Logical_source: imap_codice_l1b_lo-counters-aggregated + Logical_source_description: IMAP Mission CoDICE Lo Level-1B Aggregated Instrument Rates Data. + +imap_codice_l1b_lo_counters_singles: + <<: *instrument_base + Data_level: 1B + Data_type: L1B_lo-counters-aggregated->Level-1B Lo Single Instrument Rates Data + Logical_source: imap_codice_l1b_lo-counters-singles + Logical_source_description: IMAP Mission CoDICE Lo Level-1B Single Instrument Rates Data. + +imap_codice_l1b_lo_sw_angular: + <<: *instrument_base + Data_level: 1B + Data_type: L1B_lo-sw-angular->Level-1B Lo Sunward Angular Rates Data + Logical_source: imap_codice_l1b_lo-sw-angular + Logical_source_description: IMAP Mission CoDICE Lo Level-1B Sunward Angular Rates Data. + +imap_codice_l1b_lo_nsw_angular: + <<: *instrument_base + Data_level: 1B + Data_type: L1B_lo-nsw-angular->Level-1B Lo Non-Sunward Angular Rates Data + Logical_source: imap_codice_l1b_lo-nsw-angular + Logical_source_description: IMAP Mission CoDICE Lo Level-1B Non-Sunward Angular Rates Data. + +imap_codice_l1b_lo_sw_priority: + <<: *instrument_base + Data_level: 1B + Data_type: L1B_lo-sw-priority->Level-1B Lo Sunward Priority Rates Data + Logical_source: imap_codice_l1b_lo-sw-priority + Logical_source_description: IMAP Mission CoDICE Lo Level-1B Sunward Priority Rates Data. + +imap_codice_l1b_lo_nsw_priority: + <<: *instrument_base + Data_level: 1B + Data_type: L1B_lo-nsw-priority->Level-1B Lo Non-Sunward Priority Rates Data + Logical_source: imap_codice_l1b_lo-nsw-priority + Logical_source_description: IMAP Mission CoDICE Lo Level-1B Non-Sunward Priority Rates Data. + +imap_codice_l1b_lo_sw_species: + <<: *instrument_base + Data_level: 1B + Data_type: L1B_lo-sw-species->Level-1B Lo Sunward Species Rates Data + Logical_source: imap_codice_l1b_lo-sw-species + Logical_source_description: IMAP Mission CoDICE Lo Level-1B Sunward Species Rates Data. + +imap_codice_l1b_lo_nsw_species: + <<: *instrument_base + Data_level: 1B + Data_type: L1B_lo-nsw-species->Level-1B Lo Non-Sunward Species Rates Data + Logical_source: imap_codice_l1b_lo-nsw-species + Logical_source_description: IMAP Mission CoDICE Lo Level-1B Non-Sunward Species Rates Data. \ No newline at end of file diff --git a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml index b1eeafb38..20c18384a 100644 --- a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml @@ -50,6 +50,8 @@ energy_label: VAR_TYPE: metadata # <=== Dataset Attributes ===> +# TODO: resolve ISTP warning: Width of F10 FORMAT may be insufficient for +# values. Adjust FORMAT or VALIDMIN/VALIDMAX value to resolve. acquisition_times_attrs: <<: *default CATDESC: Time of acquisition for the energy step diff --git a/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml new file mode 100644 index 000000000..268a5b848 --- /dev/null +++ b/imap_processing/cdf/config/imap_codice_l1b_variable_attrs.yaml @@ -0,0 +1,371 @@ +# <=== Defaults ===> +default_attrs: &default + # Assumed values for all variable attrs unless overwritten + DISPLAY_TYPE: no_plot + FILLVAL: -9223372036854775808 + FORMAT: I12 + UNITS: dN + VALIDMIN: -9223372036854775808 + VALIDMAX: 9223372036854775807 + VAR_TYPE: data + SCALETYP: linear + +codice_support_attrs: &support_default + <<: *default + VAR_TYPE: support_data + +# <=== Coordinates ===> +energy_attrs: + <<: *default + CATDESC: Energy per charge (E/q) sweeping step + FIELDNAM: Energy step + FORMAT: I3 + LABLAXIS: energy step + UNITS: ' ' + VALIDMIN: 0 + VALIDMAX: 127 + VAR_TYPE: support_data + +epoch_attrs: + CATDESC: Time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: epoch + LABLAXIS: epoch + FILLVAL: -9223372036854775808 + FORMAT: " " # Supposedly not required, fails in xarray_to_cdf + VALIDMIN: -9223372036854775808 + VALIDMAX: 9223372036854775807 + UNITS: ns + VAR_TYPE: support_data + SCALETYP: linear + MONOTON: INCREASE + TIME_BASE: J2000 + TIME_SCALE: Terrestrial Time + REFERENCE_POSITION: Rotating Earth Geoid + +# <=== Labels ===> +energy_label: + CATDESC: Energy per charge (E/q) sweeping step + FIELDNAM: Energy step + FORMAT: A3 + VAR_TYPE: metadata + +# <=== Dataset Attributes ===> +# TODO: resolve ISTP warning: Width of F10 FORMAT may be insufficient for +# values. Adjust FORMAT or VALIDMIN/VALIDMAX value to resolve. +acquisition_times_attrs: + <<: *default + CATDESC: Time of acquisition for the energy step + FIELDNAM: Acquisition Time + FILLVAL: 1.7976931348623157e+308 + FORMAT: F10.3 + LABLAXIS: Acq Time + UNITS: ms + VALIDMIN: 0 + VALIDMAX: 1.7976931348623157e+308 + VAR_TYPE: support_data + +counters_attrs: &counters + <<: *default + CATDESC: Fill in at creation + DEPEND_0: epoch + DEPEND_1: energy + DISPLAY_TYPE: time_series + FIELDNAM: Fill in at creation + FILLVAL: 1.7976931348623157e+308 + FORMAT: F10.3 + LABL_PTR_1: energy_label + UNITS: rates + VALIDMIN: 0 + VALIDMAX: 1.7976931348623157e+308 + +esa_sweep_attrs: + <<: *default + CATDESC: ElectroStatic Analyzer Energy Values + FIELDNAM: ESA Voltage + FORMAT: I19 + LABLAXIS: ESA V + UNITS: V + VALIDMIN: 0 + VAR_TYPE: support_data + +# <=== Data Variable Attributes ===> +# hi-counters-aggregated +hi_counters_aggregated-aggregated: + <<: *counters + CATDESC: Aggregated Rates + FIELDNAM: Rates - Aggregated + +# hi-counters-singles +hi_counters_singles-tcr: + <<: *counters + CATDESC: TCR Rates + FIELDNAM: Rates - Event A (TCR) + +hi_counters_singles-ssdo: + <<: *counters + CATDESC: SSDO Rates + FIELDNAM: Rates - Event E (SSDO) + +hi_counters_singles-stssd: + <<: *counters + CATDESC: STSSD Rates + FIELDNAM: Rates - Event G (STSSD) + +# hi-omni +hi_omni-h: + <<: *counters + CATDESC: Omnidirectional H Rates + FIELDNAM: H + +hi_omni-he3: + <<: *counters + CATDESC: Omnidirectional He3 Rates + FIELDNAM: He3 + +hi_omni-he4: + <<: *counters + CATDESC: Omnidirectional He4 Rates + FIELDNAM: He4 + +hi_omni-c: + <<: *counters + CATDESC: Omnidirectional C Rates + FIELDNAM: C + +hi_omni-o: + <<: *counters + CATDESC: Omnidirectional O Rates + FIELDNAM: O + +hi_omni-ne_mg_si: + <<: *counters + CATDESC: Omnidirectional Ne_Mg_Si Rates + FIELDNAM: Ne_Mg_Si + +hi_omni-fe: + <<: *counters + CATDESC: Omnidirectional Fe Rates + FIELDNAM: Fe + +hi_omni-uh: + <<: *counters + CATDESC: Omnidirectional UH Rates + FIELDNAM: UH + +# hi-sectored +hi_sectored-h: + <<: *counters + CATDESC: Sectored H Rates + FIELDNAM: H + +hi_sectored-he3he4: + <<: *counters + CATDESC: Sectored He3He4 Rates + FIELDNAM: He3He4 + +hi_sectored-cno: + <<: *counters + CATDESC: Sectored CNO Rates + FIELDNAM: CNO + +hi_sectored-fe: + <<: *counters + CATDESC: Sectored Fe Rates + FIELDNAM: Fe + +# lo-counters-aggregated +lo_counters_aggregated-aggregated: + <<: *counters + CATDESC: Aggregated Rates + FIELDNAM: Rates - Aggregated + +# lo-counters-singles +lo_counters_singles-apd_singles: + <<: *counters + CATDESC: Single Rates (APD) + FIELDNAM: Rates - Single (APD) + +# lo-sw-angular +lo_sw_angular-hplus: + <<: *counters + CATDESC: Sunward H+ Species + FIELDNAM: SW - H+ + +lo_sw_angular-heplusplus: + <<: *counters + CATDESC: Sunward He++ Species + FIELDNAM: SW - He++ + +lo_sw_angular-oplus6: + <<: *counters + CATDESC: Sunward O+6 Species + FIELDNAM: SW - O+6 + +lo_sw_angular-fe_loq: + <<: *counters + CATDESC: Sunward Fe lowQ Species + FIELDNAM: SW - Fe lowQ + +lo_nsw_angular-heplusplus: + <<: *counters + CATDESC: Non-sunward He++ Species + FIELDNAM: NSW - He++ + +# lo-sw-priority +lo_sw_priority-p0_tcrs: + <<: *counters + CATDESC: Sunward Sector Triple Coincidence Pickup Ions Priority + FIELDNAM: SW Sector Triple Coincidence PUI's + +lo_sw_priority-p1_hplus: + <<: *counters + CATDESC: Sunward Sector H+ Priority + FIELDNAM: SW Sector H+ + +lo_sw_priority-p2_heplusplus: + <<: *counters + CATDESC: Sunward Sector He++ Priority + FIELDNAM: SW Sector He++ + +lo_sw_priority-p3_heavies: + <<: *counters + CATDESC: Sunward Sector High Charge State Heavies Priority + FIELDNAM: SW Sector High Charge State Heavies + +lo_sw_priority-p4_dcrs: + <<: *counters + CATDESC: Sunward Sector Double Coincidence Pickup Ions Priority + FIELDNAM: SW Sector Double Coincidence PUI's + +# lo-nsw-priority +lo_nsw_priority-p5_heavies: + <<: *counters + CATDESC: Non-sunward Sector Heavies Priority + FIELDNAM: NSW Sector Heavies + +lo_nsw_priority-p6_hplus_heplusplus: + <<: *counters + CATDESC: Non-sunward H+ and He++ Priority + FIELDNAM: NSW H+ and He++ + +# lo-sw-species +lo_sw_species-hplus: + <<: *counters + CATDESC: H+ Sunward Species + FIELDNAM: SW - H+ + +lo_sw_species-heplusplus: + <<: *counters + CATDESC: He++ Sunward Species + FIELDNAM: SW - He++ + +lo_sw_species-cplus4: + <<: *counters + CATDESC: C+4 Sunward Species + FIELDNAM: SW - C+4 + +lo_sw_species-cplus5: + <<: *counters + CATDESC: C+5 Sunward Species + FIELDNAM: SW - C+5 + +lo_sw_species-cplus6: + <<: *counters + CATDESC: C+6 Sunward Species + FIELDNAM: SW - C+6 + +lo_sw_species-oplus5: + <<: *counters + CATDESC: O+5 Sunward Species + FIELDNAM: SW - O+5 + +lo_sw_species-oplus6: + <<: *counters + CATDESC: O+6 Sunward Species + FIELDNAM: SW - O+6 + +lo_sw_species-oplus7: + <<: *counters + CATDESC: O+7 Sunward Species + FIELDNAM: SW - O+7 + +lo_sw_species-oplus8: + <<: *counters + CATDESC: O+8 Sunward Species + FIELDNAM: SW - O+8 + +lo_sw_species-ne: + <<: *counters + CATDESC: Ne Sunward Species + FIELDNAM: SW - Ne + +lo_sw_species-mg: + <<: *counters + CATDESC: Mg Sunward Species + FIELDNAM: SW - Mg + +lo_sw_species-si: + <<: *counters + CATDESC: Si Sunward Species + FIELDNAM: SW - Si + +lo_sw_species-fe_loq: + <<: *counters + CATDESC: Fe lowQ Sunward Species + FIELDNAM: SW - Fe lowQ + +lo_sw_species-fe_hiq: + <<: *counters + CATDESC: Fe highQ Sunward Species + FIELDNAM: SW - Fe highQ + +lo_sw_species-heplus: + <<: *counters + CATDESC: He+ Pickup Ion Sunward Species + FIELDNAM: SW - He+ (PUI) + +lo_sw_species-cnoplus: + <<: *counters + CATDESC: CNO+ Pickup Ion Sunward Species + FIELDNAM: SW - CNO+ (PUI) + +# lo-nsw-species +lo_nsw_species-hplus: + <<: *counters + CATDESC: H+ Non-sunward Species + FIELDNAM: NSW - H+ + +lo_nsw_species-heplusplus: + <<: *counters + CATDESC: He++ Non-sunward Species + FIELDNAM: NSW - He++ + +lo_nsw_species-c: + <<: *counters + CATDESC: C Non-sunward Species + FIELDNAM: NSW - C + +lo_nsw_species-o: + <<: *counters + CATDESC: O Non-sunward Species + FIELDNAM: NSW - O + +lo_nsw_species-ne_si_mg: + <<: *counters + CATDESC: Ne-Si-Mg Non-sunward Species + FIELDNAM: NSW - Ne_Si_Mg + +lo_nsw_species-fe: + <<: *counters + CATDESC: Fe Non-sunward Species + FIELDNAM: NSW - Fe + +lo_nsw_species-heplus: + <<: *counters + CATDESC: He+ Non-sunward Species + FIELDNAM: NSW - He+ + +lo_nsw_species-cnoplus: + <<: *counters + CATDESC: CNO+ Non-sunward Species + FIELDNAM: NSW - CNO+ \ No newline at end of file diff --git a/imap_processing/cli.py b/imap_processing/cli.py index 8d6ba7f87..58247d6c3 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -32,7 +32,7 @@ # from imap_processing import cdf # In code: # call cdf.utils.write_cdf -from imap_processing.codice import codice_l1a +from imap_processing.codice import codice_l1a, codice_l1b from imap_processing.glows.l1a.glows_l1a import glows_l1a from imap_processing.glows.l1b.glows_l1b import glows_l1b from imap_processing.hi.l1a import hi_l1a @@ -385,6 +385,17 @@ def do_processing(self, dependencies): cdf_file_path = dataset.attrs["cdf_filename"] return [cdf_file_path] + if self.data_level == "l1b": + if len(dependencies) > 1: + raise ValueError( + f"Unexpected dependencies found for CoDICE L1b:" + f"{dependencies}. Expected only one dependency." + ) + # process data + dataset = codice_l1b.process_codice_l1b(dependencies[0], self.version) + cdf_file_path = dataset.attrs["cdf_filename"] + return [cdf_file_path] + class Glows(ProcessInstrument): """Process GLOWS.""" diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index f5f4af0ac..c2ffef832 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -77,7 +77,9 @@ def __init__(self, table_id: int, plan_id: int, plan_step: int, view_id: int): self.plan_step = plan_step self.view_id = view_id - def create_science_dataset(self, start_time: np.datetime64) -> xr.Dataset: + def create_science_dataset( + self, start_time: np.datetime64, data_version: str + ) -> xr.Dataset: """Create an ``xarray`` dataset for the unpacked science data. The dataset can then be written to a CDF file. @@ -86,6 +88,9 @@ def create_science_dataset(self, start_time: np.datetime64) -> xr.Dataset: ---------- start_time : numpy.datetime64 The start time of the packet, used to determine epoch data variable + data_version : str + Version of the data product being created + Returns ------- @@ -96,6 +101,7 @@ def create_science_dataset(self, start_time: np.datetime64) -> xr.Dataset: cdf_attrs = ImapCdfAttributes() cdf_attrs.add_instrument_global_attrs("codice") cdf_attrs.add_instrument_variable_attrs("codice", "l1a") + cdf_attrs.add_global_attribute("Data_version", data_version) # Define coordinates epoch = xr.DataArray( @@ -360,7 +366,7 @@ def process_codice_l1a(file_path: Path | str, data_version: str) -> xr.Dataset: if apid == CODICEAPID.COD_NHK: packets = grouped_data[apid] sorted_packets = sort_by_time(packets, "SHCOARSE") - dataset = create_hskp_dataset(packets=sorted_packets) + dataset = create_hskp_dataset(sorted_packets, data_version) elif apid in apids_for_lo_science_processing: # Sort the packets by time @@ -384,7 +390,7 @@ def process_codice_l1a(file_path: Path | str, data_version: str) -> xr.Dataset: pipeline.get_acquisition_times() pipeline.get_data_products(apid) pipeline.unpack_science_data(science_values) - dataset = pipeline.create_science_dataset(start_time) + dataset = pipeline.create_science_dataset(start_time, data_version) # TODO: Temporary workaround in order to create hi data products in absence # of simulated data. This is essentially the same process as is for @@ -412,11 +418,10 @@ def process_codice_l1a(file_path: Path | str, data_version: str) -> xr.Dataset: pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id) pipeline.get_data_products(apid) pipeline.unpack_science_data(science_values) - dataset = pipeline.create_science_dataset(start_time) + dataset = pipeline.create_science_dataset(start_time, data_version) # Write dataset to CDF logger.info(f"\nFinal data product:\n{dataset}\n") - dataset.attrs["Data_version"] = data_version dataset.attrs["cdf_filename"] = write_cdf(dataset) logger.info(f"\tCreated CDF file: {dataset.cdf_filename}") diff --git a/imap_processing/codice/codice_l1b.py b/imap_processing/codice/codice_l1b.py new file mode 100644 index 000000000..bab6086e1 --- /dev/null +++ b/imap_processing/codice/codice_l1b.py @@ -0,0 +1,196 @@ +"""Perform CoDICE l1b processing. + +This module processes CoDICE l1a files and creates L1a data products. + +Use +--- + + from imap_processing.codice.codice_l0 import decom_packets + from imap_processing.codice.codice_l1b import process_codice_l1b + dataset = process_codice_l1b(l1a_file) +""" + +import logging +from pathlib import Path + +import xarray as xr + +from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes +from imap_processing.cdf.utils import load_cdf, write_cdf + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +# TODO: Fix ISTP compliance issues (revealed in SKTEditor) + + +def create_hskp_dataset(l1a_dataset, cdf_attrs) -> xr.Dataset: + """Create an ``xarray`` dataset for the housekeeping data. + + The dataset can then be written to a CDF file. + + Parameters + ---------- + l1a_dataset : xr.Dataset + The L1a dataset that is being processed + cdf_attrs : ImapCdfAttributes + The CDF attributes for the dataset + + Returns + ------- + xarray.Dataset + ``xarray`` dataset containing the science data and supporting metadata + """ + epoch = l1a_dataset.coords["epoch"] + l1b_dataset = xr.Dataset( + coords={"epoch": epoch}, + attrs=cdf_attrs.get_global_attributes("imap_codice_l1b_hskp"), + ) + for variable_name in l1a_dataset: + # Get the data array from the L1a data product + values = l1a_dataset[variable_name].values + + # Convert data array to "rates" + # TODO: For SIT-3, just convert value to float. Revisit after SIT-3. + variable_data_arr = values.astype(float) + + # TODO: Change 'TBD' catdesc and fieldname + # Once packet definition files are re-generated, can get this info from + # something like this: + # for key, value in (packet.header | packet.data).items(): + # fieldname = value.short_description + # catdesc = value.short_description + # I am holding off making this change until I acquire updated housekeeping + # packets/validation data that match the latest telemetry definitions + attrs = cdf_attrs.variable_attributes["codice_support_attrs"] + attrs["CATDESC"] = "TBD" + attrs["DEPEND_0"] = "epoch" + attrs["FIELDNAM"] = "TBD" + attrs["LABLAXIS"] = variable_name + + # Put the new data array into the dataset + l1b_dataset[variable_name] = xr.DataArray( + variable_data_arr, + name=variable_name, + dims=["epoch"], + attrs=attrs, + ) + + return l1b_dataset + + +def create_science_dataset( + l1a_dataset: xr.Dataset, cdf_attrs, dataset_name +) -> xr.Dataset: + """Create an ``xarray`` dataset for the science data. + + The dataset can then be written to a CDF file. + + Parameters + ---------- + l1a_dataset : xr.Dataset + The L1a dataset that is being processed + cdf_attrs : ImapCdfAttributes + The CDF attributes for the dataset + dataset_name : str + The name that is used to construct the data variable name and reference + the CDF attributes (e.g. ``imap_codice_l1b_hi_omni``) + + Returns + ------- + xarray.Dataset + ``xarray`` dataset containing the science data and supporting metadata + """ + # Retrieve the coordinates from the l1a dataset + epoch = l1a_dataset.coords["epoch"] + energy = l1a_dataset.coords["energy"] + energy_label = l1a_dataset.coords["energy_label"] + + # Create empty l1b dataset + l1b_dataset = xr.Dataset( + coords={"epoch": epoch, "energy": energy, "energy_label": energy_label}, + attrs=cdf_attrs.get_global_attributes(dataset_name), + ) + + # Get the data variables from l1a dataset + for variable_name in l1a_dataset: + if variable_name == "esa_sweep_values": + values = l1a_dataset["esa_sweep_values"] + l1b_dataset["esa_sweep_values"] = xr.DataArray( + values, + dims=["energy"], + attrs=cdf_attrs.get_variable_attributes("esa_sweep_attrs"), + ) + + elif variable_name == "acquisition_times": + values = l1a_dataset["acquisition_times"] + l1b_dataset["acquisition_times"] = xr.DataArray( + values, + dims=["energy"], + attrs=cdf_attrs.get_variable_attributes("acquisition_times_attrs"), + ) + + else: + # Get the data array from the L1a data product + values = l1a_dataset[variable_name].values + + # Convert data array to "rates" + # TODO: For SIT-3, just convert value to float. Revisit after SIT-3. + variable_data_arr = values.astype(float) + + # Put the new data array into the dataset + cdf_attrs_key = ( + f"{dataset_name.split('imap_codice_l1b_')[-1]}-{variable_name}" + ) + l1b_dataset[variable_name] = xr.DataArray( + variable_data_arr, + name=variable_name, + dims=["epoch", "energy"], + attrs=cdf_attrs.get_variable_attributes(cdf_attrs_key), + ) + + return l1b_dataset + + +def process_codice_l1b(file_path: Path, data_version: str) -> xr.Dataset: + """Process CoDICE l1a data to create l1b data products. + + Parameters + ---------- + file_path : pathlib.Path | str + Path to the CoDICE L1a file to process + data_version : str + Version of the data product being created + + Returns + ------- + dataset : xarray.Dataset + ``xarray`` dataset containing the science data and supporting metadata + """ + logger.info(f"\nProcessing {file_path.name} file.") + + # Load the L1a CDF + l1a_dataset = load_cdf(file_path) + + # Start constructing l1b dataset + cdf_attrs = ImapCdfAttributes() + cdf_attrs.add_instrument_global_attrs("codice") + cdf_attrs.add_instrument_variable_attrs("codice", "l1b") + cdf_attrs.add_global_attribute("Data_version", data_version) + + dataset_name = ( + l1a_dataset.attrs["Logical_source"].replace("-", "_").replace("l1a", "l1b") + ) + + if "hskp" in dataset_name: + l1b_dataset = create_hskp_dataset(l1a_dataset, cdf_attrs) + + else: + l1b_dataset = create_science_dataset(l1a_dataset, cdf_attrs, dataset_name) + + # Write the dataset to CDF + logger.info(f"\nFinal data product:\n{l1b_dataset}\n") + l1b_dataset.attrs["cdf_filename"] = write_cdf(l1b_dataset) + logger.info(f"\tCreated CDF file: {l1b_dataset.cdf_filename}") + + return l1b_dataset diff --git a/imap_processing/codice/utils.py b/imap_processing/codice/utils.py index cc22b8ce4..e3c51b43c 100644 --- a/imap_processing/codice/utils.py +++ b/imap_processing/codice/utils.py @@ -109,13 +109,15 @@ def add_metadata_to_array(packet, metadata_arrays: dict) -> dict: return metadata_arrays -def create_hskp_dataset(packets) -> xr.Dataset: +def create_hskp_dataset(packets, data_version: str) -> xr.Dataset: """Create dataset for each metadata field for housekeeping data. Parameters ---------- packets : list[space_packet_parser.parser.Packet] The list of packets to process + data_version : str + Version of the data product being created Returns ------- @@ -125,6 +127,7 @@ def create_hskp_dataset(packets) -> xr.Dataset: cdf_attrs = ImapCdfAttributes() cdf_attrs.add_instrument_global_attrs("codice") cdf_attrs.add_instrument_variable_attrs("codice", "l1a") + cdf_attrs.add_global_attribute("Data_version", data_version) metadata_arrays = collections.defaultdict(list) diff --git a/imap_processing/tests/codice/data/imap_codice_l1a_hi-counters-aggregated_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1a_hi-counters-aggregated_20240429_v001.cdf new file mode 100644 index 000000000..9989fd6e8 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1a_hi-counters-aggregated_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1a_hi-counters-singles_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1a_hi-counters-singles_20240429_v001.cdf new file mode 100644 index 000000000..8651d6314 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1a_hi-counters-singles_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1a_hi-omni_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1a_hi-omni_20240429_v001.cdf new file mode 100644 index 000000000..c544b648c Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1a_hi-omni_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1a_hi-sectored_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1a_hi-sectored_20240429_v001.cdf new file mode 100644 index 000000000..fca1a117b Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1a_hi-sectored_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1a_hskp_20100101_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1a_hskp_20100101_v001.cdf new file mode 100644 index 000000000..93b6c3705 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1a_hskp_20100101_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1a_lo-counters-aggregated_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1a_lo-counters-aggregated_20240429_v001.cdf new file mode 100644 index 000000000..8172a4ccb Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1a_lo-counters-aggregated_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1a_lo-counters-singles_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1a_lo-counters-singles_20240429_v001.cdf new file mode 100644 index 000000000..cbe013467 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1a_lo-counters-singles_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1a_lo-nsw-angular_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1a_lo-nsw-angular_20240429_v001.cdf new file mode 100644 index 000000000..8925da636 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1a_lo-nsw-angular_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1a_lo-nsw-priority_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1a_lo-nsw-priority_20240429_v001.cdf new file mode 100644 index 000000000..b3570928c Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1a_lo-nsw-priority_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1a_lo-nsw-species_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1a_lo-nsw-species_20240429_v001.cdf new file mode 100644 index 000000000..0d9619936 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1a_lo-nsw-species_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1a_lo-sw-angular_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1a_lo-sw-angular_20240429_v001.cdf new file mode 100644 index 000000000..2d6702154 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1a_lo-sw-angular_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1a_lo-sw-priority_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1a_lo-sw-priority_20240429_v001.cdf new file mode 100644 index 000000000..ac4473cc8 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1a_lo-sw-priority_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1a_lo-sw-species_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1a_lo-sw-species_20240429_v001.cdf new file mode 100644 index 000000000..0fc7dfbf4 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1a_lo-sw-species_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1b_hi-counters-aggregated_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1b_hi-counters-aggregated_20240429_v001.cdf new file mode 100644 index 000000000..4840467ac Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1b_hi-counters-aggregated_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1b_hi-counters-singles_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1b_hi-counters-singles_20240429_v001.cdf new file mode 100644 index 000000000..9e70051e9 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1b_hi-counters-singles_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1b_hi-omni_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1b_hi-omni_20240429_v001.cdf new file mode 100644 index 000000000..908cdaa24 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1b_hi-omni_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1b_hi-sectored_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1b_hi-sectored_20240429_v001.cdf new file mode 100644 index 000000000..e9778b72d Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1b_hi-sectored_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1b_hskp_20100101_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1b_hskp_20100101_v001.cdf new file mode 100644 index 000000000..5860b0fd0 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1b_hskp_20100101_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1b_lo-counters-aggregated_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1b_lo-counters-aggregated_20240429_v001.cdf new file mode 100644 index 000000000..d09d81721 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1b_lo-counters-aggregated_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1b_lo-counters-singles_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1b_lo-counters-singles_20240429_v001.cdf new file mode 100644 index 000000000..c4ade15a4 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1b_lo-counters-singles_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1b_lo-nsw-angular_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1b_lo-nsw-angular_20240429_v001.cdf new file mode 100644 index 000000000..45555c3d7 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1b_lo-nsw-angular_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1b_lo-nsw-priority_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1b_lo-nsw-priority_20240429_v001.cdf new file mode 100644 index 000000000..0f115662f Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1b_lo-nsw-priority_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1b_lo-nsw-species_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1b_lo-nsw-species_20240429_v001.cdf new file mode 100644 index 000000000..11b4e95f3 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1b_lo-nsw-species_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1b_lo-sw-angular_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1b_lo-sw-angular_20240429_v001.cdf new file mode 100644 index 000000000..5aece32e6 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1b_lo-sw-angular_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1b_lo-sw-priority_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1b_lo-sw-priority_20240429_v001.cdf new file mode 100644 index 000000000..75eacef51 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1b_lo-sw-priority_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/data/imap_codice_l1b_lo-sw-species_20240429_v001.cdf b/imap_processing/tests/codice/data/imap_codice_l1b_lo-sw-species_20240429_v001.cdf new file mode 100644 index 000000000..2dda54d77 Binary files /dev/null and b/imap_processing/tests/codice/data/imap_codice_l1b_lo-sw-species_20240429_v001.cdf differ diff --git a/imap_processing/tests/codice/test_codice_l0.py b/imap_processing/tests/codice/test_codice_l0.py index 81a15c9cb..47a1c9dd3 100644 --- a/imap_processing/tests/codice/test_codice_l0.py +++ b/imap_processing/tests/codice/test_codice_l0.py @@ -76,7 +76,7 @@ def test_eu_hk_data( The validation data to compare against """ - l1a_hk_ds = create_hskp_dataset(decom_test_data) + l1a_hk_ds = create_hskp_dataset(decom_test_data, "001") eu_hk_data = convert_raw_to_eu( l1a_hk_ds, imap_module_directory / "tests/codice/data/eu_unit_lookup_table.csv", diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index 3952f6d5d..77a9f035e 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -127,7 +127,6 @@ def test_l1a_data(request) -> xr.Dataset: A ``xarray`` dataset containing the test data """ - # TODO: add version to test inputs dataset = process_codice_l1a(file_path=request.param, data_version="001") return dataset diff --git a/imap_processing/tests/codice/test_codice_l1b.py b/imap_processing/tests/codice/test_codice_l1b.py new file mode 100644 index 000000000..2a5542d88 --- /dev/null +++ b/imap_processing/tests/codice/test_codice_l1b.py @@ -0,0 +1,101 @@ +"""Tests the L1b processing for CoDICE L1a data""" + +from pathlib import Path + +import pytest +import xarray as xr + +from imap_processing import imap_module_directory +from imap_processing.codice.codice_l1b import process_codice_l1b + +EXPECTED_FILENAMES = [ + "imap_codice_l1b_hskp_20100101_v001.cdf", + "imap_codice_l1b_hi-counters-aggregated_20240429_v001.cdf", + "imap_codice_l1b_hi-counters-singles_20240429_v001.cdf", + "imap_codice_l1b_hi-omni_20240429_v001.cdf", + "imap_codice_l1b_hi-sectored_20240429_v001.cdf", + "imap_codice_l1b_lo-counters-aggregated_20240429_v001.cdf", + "imap_codice_l1b_lo-counters-singles_20240429_v001.cdf", + "imap_codice_l1b_lo-sw-angular_20240429_v001.cdf", + "imap_codice_l1b_lo-nsw-angular_20240429_v001.cdf", + "imap_codice_l1b_lo-sw-priority_20240429_v001.cdf", + "imap_codice_l1b_lo-nsw-priority_20240429_v001.cdf", + "imap_codice_l1b_lo-sw-species_20240429_v001.cdf", + "imap_codice_l1b_lo-nsw-species_20240429_v001.cdf", +] +TEST_FILES = [ + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hskp_20100101_v001.pkts" + ), + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-counters-aggregated_20240429_v001.pkts" + ), + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-counters-singles_20240429_v001.pkts" + ), + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-omni_20240429_v001.pkts" + ), + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_hi-sectored_20240429_v001.pkts" + ), + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-counters-aggregated_20240429_v001.pkts" + ), + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-counters-singles_20240429_v001.pkts" + ), + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-angular_20240429_v001.pkts" + ), + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-angular_20240429_v001.pkts" + ), + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-priority_20240429_v001.pkts" + ), + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-priority_20240429_v001.pkts" + ), + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-sw-species_20240429_v001.pkts" + ), + Path( + f"{imap_module_directory}/tests/codice/data/imap_codice_l1a_lo-nsw-species_20240429_v001.pkts" + ), +] + + +@pytest.fixture(params=TEST_FILES) +def test_l1b_data(request) -> xr.Dataset: + """Return a ``xarray`` dataset containing test data. + + Returns + ------- + dataset : xr.Dataset + A ``xarray`` dataset containing the test data + """ + + dataset = process_codice_l1b(request.param, data_version="001") + return dataset + + +@pytest.mark.parametrize( + "test_l1b_data, expected_filename", + list(zip(TEST_FILES, EXPECTED_FILENAMES)), + indirect=["test_l1b_data"], +) +def test_l1b_cdf_filenames(test_l1b_data: xr.Dataset, expected_filename: str): + """Tests that the ``process_codice_l1b`` function generates CDF files with + expected filenames. + + Parameters + ---------- + test_l1b_data : xr.Dataset + A ``xarray`` dataset containing the test data + expected_filename : str + The expected CDF filename + """ + + dataset = test_l1b_data + assert dataset.cdf_filename.name == expected_filename