Differences

This shows you the differences between two versions of the page.

--- data:data_analysis_manual:read_catalog_python [2021/11/12 11:14]
eric buchlin How to use the catalogue
+++ data:data_analysis_manual:read_catalog_python [2024/03/29 14:11] (current)
eric buchlin Page is deprecated
@@ Line 1: / Line 1: @@
-====== How to read the UiO FITS files catalog in Python ======
+====== How to read and use the UiO FITS files catalog in Python ======
-<file python read_uio_cat.py>
+<note warning>These scripts are deprecated in favor of the current version, available in the [[https://sospice.readthedocs.io/en/stable/|sospice Python module]].</note>
+===== CSV catalog (new, recommended) =====
+<file python read_uio_cat_csv.py>
 from pathlib import Path
 import pandas as pd
@@ Line 7: / Line 11: @@
 # SPICE data tree path, to be changed to your SPICE data mirror
 data_path = "/archive/SOLAR-ORBITER/SPICE"      # example for IAS computing servers
+def date_parser(string):
+    try:
+        return pd.Timestamp(string)
+    except ValueError:
+         return pd.NaT
 def read_uio_cat():
     """
-    Read UiO text table SPICE FITS files catalog
+    Read UiO SPICE FITS files CSV catalog
-    http://astro-sdc-db.uio.no/vol/spice/fits/spice_catalog.txt
+    http://astro-sdc-db.uio.no/vol/spice/fits/spice_catalog.csv
     Return
@@ Line 17: / Line 29: @@
     pandas.DataFrame
         Table
+    """
+    cat_file = Path(data_path) / "fits" / "spice_catalog.csv"
+    if not cat_file.exists():
+        print(f'Error: Catalog file not available at {cat_file.as_posix()}')
+        sys.exit(1)
+    date_columns = ['DATE-BEG','DATE', 'TIMAQUTC']
+    df = pd.read_csv(cat_file, parse_dates=date_columns, date_parser=date_parser)
+    return df
+</file>
-    Example queries that can be done on the result:
+The same applies for the catalog included in the data releases (here: release 2.0), which can simply be read by:
-    * `df[(df.LEVEL == "L2") & (df["DATE-BEG"] >= "2020-11-17") & (df["DATE-BEG"] < "2020-11-18") & (df.XPOSURE > 60.)]`
+<file python read_release_cat.py>
-    * `df[(df.LEVEL == "L2") & (df.STUDYDES == "Standard dark for cruise phase")]`
+import pandas as pd
+def date_parser(string):
+    try:
+        return pd.Timestamp(string)
+    except ValueError:
+         return pd.NaT
+date_columns = ['DATE-BEG','DATE', 'TIMAQUTC']
+cat = pd.read_csv(
+    'https://spice.osups.universite-paris-saclay.fr/spice-data/release-2.0/catalog.csv',
+    date_parser=date_parser,
+    parse_dates=date_columns
+)
+# TODO interpret the JSON included in columns `proc_steps` and `windows`.
+</file>
+===== Text catalog =====
+<file python read_uio_cat_txt.py>
+from pathlib import Path
+import pandas as pd
+# SPICE data tree path, to be changed to your SPICE data mirror
+data_path = "/archive/SOLAR-ORBITER/SPICE"      # example for IAS computing servers
+def date_parser(string):
+    try:
+        return pd.Timestamp(string)
+    except ValueError:
+         return pd.NaT
+def read_uio_cat():
+    """
+    Read UiO text table SPICE FITS files catalog
+    http://astro-sdc-db.uio.no/vol/spice/fits/spice_catalog.txt
+    Return
+    ------
+    pandas.DataFrame
+        Table
     """
     cat_file = Path(data_path) / "fits" / "spice_catalog.txt"
+    if not cat_file.exists():
+        print(f'Error: Catalog file not available at {cat_file.as_posix()}')
+        sys.exit(1)
     columns = list(pd.read_csv(cat_file, nrows=0).keys())
     date_columns = ['DATE-BEG','DATE', 'TIMAQUTC']
-    df = pd.read_table(cat_file, skiprows=1, names=columns, na_values="MISSING",
+    df = pd.read_table(cat_file, skiprows=1, names=columns,
-                    parse_dates=date_columns, warn_bad_lines=True)
+                    parse_dates=date_columns, date_parser=date_parser,
-    df.LEVEL = df.LEVEL.apply(lambda string: string.strip())
+                    low_memory=False)
-    df.STUDYTYP = df.STUDYTYP.apply(lambda string: string.strip())
     return df
 </file>
-''na_values="MISSING"'' replaces the string "MISSING" by NaNs, it can be removed.
+===== Using the catalog =====
 Then we can read the catalog and filter it:

User Tools

Site Tools

Differences

Page Tools