SWE updates to be compliant with new cdflib releases (IMAP-Science-Op…

…erations-Center#357)

* update cdflib version to 1.2.6

imap_processing/swe/l1a/swe_l1a.py

@@ -2,7 +2,6 @@
 
 import logging
 
-from imap_processing import imap_module_directory
 from imap_processing.swe.l0 import decom_swe
 from imap_processing.swe.l1a.swe_science import swe_science
 from imap_processing.swe.utils.swe_utils import (
@@ -13,18 +12,6 @@
 from imap_processing.utils import group_by_apid, sort_by_time
 
 
-def decom_data(file_path):
-    """Read test data from test folder."""
-    # TODO: replace test folder after demo
-    test_folder_path = "tests/swe/l0_data"
-    packet_files = list(imap_module_directory.glob(f"{test_folder_path}/*.bin"))
-
-    data_list = []
-    for packet_file in packet_files:
-        data_list.extend(decom_swe.decom_packets(packet_file))
-    return data_list
-
-
 def swe_l1a(file_path):
     """Process SWE l0 data into l1a data.
 
@@ -42,8 +29,8 @@ def swe_l1a(file_path):
     List
         List of xarray.Dataset
     """
-    # TODO: figure out how to do this better after demo
-    packets = decom_data(file_path)
+    packets = decom_swe.decom_packets(file_path)
+
     processed_data = []
     # group data by appId
     grouped_data = group_by_apid(packets)

imap_processing/swe/l1a/swe_science.py

@@ -7,6 +7,7 @@
 import xarray as xr
 
 from imap_processing.cdf.global_attrs import ConstantCoordinates
+from imap_processing.cdf.utils import calc_start_time
 from imap_processing.swe import swe_cdf_attrs
 from imap_processing.swe.utils.swe_utils import (
     add_metadata_to_array,
@@ -135,8 +136,11 @@ def swe_science(decom_data):
         raw_science_array.append(raw_counts.astype(np.int64))
         metadata_arrays = add_metadata_to_array(data_packet, metadata_arrays)
 
+    epoch_converted_time = [
+        calc_start_time(sc_time) for sc_time in metadata_arrays["SHCOARSE"]
+    ]
     epoch_time = xr.DataArray(
-        metadata_arrays["SHCOARSE"],
+        epoch_converted_time,
         name="Epoch",
         dims=["Epoch"],
         attrs=ConstantCoordinates.EPOCH,
@@ -169,16 +173,30 @@ def swe_science(decom_data):
         ).output(),
     )
 
+    science_attrs = dataclasses.replace(
+        swe_cdf_attrs.l1a_science_attrs,
+        catdesc="Uncompressed counts from SWE",
+        fieldname="Uncompressed counts from SWE",
+        label_axis="Uncompressed counts from SWE",
+        units="counts",
+    )
     science_xarray = xr.DataArray(
         science_array,
         dims=["Epoch", "Energy", "Counts"],
-        attrs=swe_cdf_attrs.l1a_science_attrs.output(),
+        attrs=science_attrs.output(),
     )
 
+    raw_science_attrs = dataclasses.replace(
+        swe_cdf_attrs.l1a_science_attrs,
+        catdesc="Raw counts from SWE",
+        fieldname="Raw counts from SWE",
+        label_axis="Raw counts from SWE",
+        units="counts",
+    )
     raw_science_xarray = xr.DataArray(
         raw_science_array,
         dims=["Epoch", "Energy", "Counts"],
-        attrs=swe_cdf_attrs.l1a_science_attrs.output(),
+        attrs=raw_science_attrs.output(),
     )
 
     dataset = xr.Dataset(
@@ -194,8 +212,6 @@ def swe_science(decom_data):
 
     # create xarray dataset for each metadata field
     for key, value in metadata_arrays.items():
-        if key == "SHCOARSE":
-            continue
         # TODO: figure out how to add more descriptive
         # description for each metadata field
         #

imap_processing/swe/utils/swe_utils.py

@@ -7,6 +7,7 @@
 import xarray as xr
 
 from imap_processing.cdf.global_attrs import ConstantCoordinates
+from imap_processing.cdf.utils import calc_start_time
 from imap_processing.swe import swe_cdf_attrs
 
 
@@ -80,8 +81,11 @@ def create_dataset(packets):
     for data_packet in packets:
         add_metadata_to_array(data_packet, metadata_arrays)
 
+    epoch_converted_time = [
+        calc_start_time(sc_time) for sc_time in metadata_arrays["SHCOARSE"]
+    ]
     epoch_time = xr.DataArray(
-        metadata_arrays["SHCOARSE"],
+        epoch_converted_time,
         name="Epoch",
         dims=["Epoch"],
         attrs=ConstantCoordinates.EPOCH,

imap_processing/tests/swe/test_swe_l1a.py

@@ -1,6 +1,9 @@
+from pathlib import Path
+
 import pytest
 
 from imap_processing import imap_module_directory
+from imap_processing.cdf.utils import write_cdf
 from imap_processing.swe.l0 import decom_swe
 from imap_processing.swe.l1a.swe_l1a import swe_l1a
 from imap_processing.swe.utils.swe_utils import (
@@ -45,10 +48,16 @@ def test_group_by_apid(decom_test_data):
     assert len(total_event_message_data) == 15
 
 
-@pytest.mark.xfail(reason="Need to update after refactor of function returns.")
 def test_cdf_creation(decom_test_data):
-    grouped_data = group_by_apid(decom_test_data)
-    sci_cdf_filepath = swe_l1a(grouped_data[SWEAPID.SWE_SCIENCE])
-    hk_cdf_filepath = swe_l1a(grouped_data[SWEAPID.SWE_APP_HK])
-    assert sci_cdf_filepath.name == "imap_swe_l1a_sci_20230927_v01.cdf"
-    assert hk_cdf_filepath.name == "imap_swe_l1a_lveng-hk_20230927_v01.cdf"
+    test_data_path = "tests/swe/l0_data/20230927100425_SWE_CEM_RAW_packet.bin"
+    processed_data = swe_l1a(imap_module_directory / test_data_path)
+
+    l1a_cdf_filename = "imap_swe_l1a_cemraw_20230927_20230927_v01.cdf"
+    current_directory = Path(__file__).parent
+
+    cdf_filepath = current_directory / l1a_cdf_filename
+    cem_raw_cdf_filepath = write_cdf(processed_data[0]["data"], cdf_filepath)
+
+    assert cem_raw_cdf_filepath.name == l1a_cdf_filename
+
+    Path.unlink(cem_raw_cdf_filepath)

imap_processing/tests/swe/test_swe_l1b.py

@@ -1,8 +1,11 @@
+from pathlib import Path
+
 import pandas as pd
 import pytest
 from cdflib.xarray import cdf_to_xarray
 
 from imap_processing import imap_module_directory
+from imap_processing.cdf.utils import write_cdf
 from imap_processing.swe.l0 import decom_swe
 from imap_processing.swe.l1a.swe_l1a import swe_l1a
 from imap_processing.swe.l1a.swe_science import swe_science
@@ -26,34 +29,6 @@ def decom_test_data():
     return data_list
 
 
-@pytest.fixture(scope="session")
-def l1a_test_data():
-    """Read test data from file"""
-    # NOTE: data was provided in this sequence in both bin and validation data
-    # from instrument team.
-    # Packet 1 has spin 4's data
-    # Packet 2 has spin 1's data
-    # Packet 3 has spin 2's data
-    # Packet 4 has spin 3's data
-    # moved packet 1 to bottom to show data in order.
-    packet_files = [
-        imap_module_directory
-        / "tests/swe/l0_data/20230927173253_SWE_SCIENCE_packet.bin",
-        imap_module_directory
-        / "tests/swe/l0_data/20230927173308_SWE_SCIENCE_packet.bin",
-        imap_module_directory
-        / "tests/swe/l0_data/20230927173323_SWE_SCIENCE_packet.bin",
-        imap_module_directory
-        / "tests/swe/l0_data/20230927173238_SWE_SCIENCE_packet.bin",
-    ]
-    data = []
-    for packet_file in packet_files:
-        data.extend(decom_swe.decom_packets(packet_file))
-    # Get unpacked science data
-    unpacked_data = swe_science(data)
-    return unpacked_data
-
-
 def test_swe_l1b(decom_test_data):
     """Test that calculate engineering unit(EU) is
     matches validation data.
@@ -139,17 +114,35 @@ def test_swe_l1b(decom_test_data):
         assert round(hk_l1b[field].data[1], 5) == round(validation_data[field], 5)
 
 
-@pytest.mark.xfail(reason="Need to update after refactor of function returns.")
-def test_cdf_creation(decom_test_data, l1a_test_data):
-    sci_l1b_filepath = swe_l1b(l1a_test_data)
+def test_cdf_creation(decom_test_data):
+    """Test that CDF file is created and has the correct name."""
+    current_directory = Path(__file__).parent
+
+    test_data_path = "tests/swe/l0_data/20230927100248_SWE_HK_packet.bin"
+    l1a_datasets = swe_l1a(imap_module_directory / test_data_path)
+    hk_l1a_cdf_file_path = (
+        current_directory / "imap_swe_l1a_lveng-hk_20230927_20230927_v01.cdf"
+    )
+
+    for i in range(len(l1a_datasets)):
+        if l1a_datasets[i]["descriptor"] == "lveng-hk":
+            hk_l1a_data = l1a_datasets[i]["data"]
+            break
+
+    hk_l1a_filepath = write_cdf(hk_l1a_data, hk_l1a_cdf_file_path)
+
+    assert hk_l1a_filepath.name == "imap_swe_l1a_lveng-hk_20230927_20230927_v01.cdf"
 
-    # process hk data to l1a and then pass to l1b
-    grouped_data = group_by_apid(decom_test_data)
-    # writes data to CDF file
-    hk_l1a_filepath = swe_l1a(grouped_data[SWEAPID.SWE_APP_HK])
     # reads data from CDF file and passes to l1b
-    l1a_dataset = cdf_to_xarray(hk_l1a_filepath)
-    hk_l1b_filepath = swe_l1b(l1a_dataset)
+    l1a_cdf_dataset = cdf_to_xarray(hk_l1a_filepath, to_datetime=True)
+    l1b_dataset = swe_l1b(l1a_cdf_dataset)
+    cdf_file_path = (
+        current_directory / "imap_swe_l1b_lveng-hk_20230927_20230927_v01.cdf"
+    )
+
+    hk_l1b_filepath = write_cdf(l1b_dataset, cdf_file_path)
 
-    assert hk_l1b_filepath.name == "imap_swe_l1b_lveng-hk_20230927_v01.cdf"
-    assert sci_l1b_filepath.name == "imap_swe_l1b_sci_20230927_v01.cdf"
+    assert hk_l1b_filepath.name == "imap_swe_l1b_lveng-hk_20230927_20230927_v01.cdf"
+    # remove the file after reading for local testing
+    Path.unlink(hk_l1a_filepath)
+    Path.unlink(hk_l1b_filepath)

pyproject.toml

@@ -30,7 +30,7 @@ classifiers = [
 
 [tool.poetry.dependencies]
 bitstring = ">=4.0.1"
-cdflib = "==1.2.4"
+cdflib = "==1.2.6"
 imap-data-access = ">=0.4.0"
 python = ">=3.9,<4"
 space_packet_parser = ">=4.1.0"