Configuration
Configuration file
The config/config.json file configures all steps of the pipeline. You can use the config/config.template.json file as a template to create your own configuration file.
💡
The full schema specification and description of each parameter can be found in the API Reference section.
Template:
{
"version": "1.4",
"general": {
"metadata": {
"github_repository": "tum-esm/em27-metadata-storage",
"access_token": null
},
"data": {
"ground_pressure": {
"path": "path-to-ground-pressure-data",
"file_regex": "^ground-pressure-$(SENSOR_ID)-$(YYYY)-$(MM)-$(DD).csv$",
"separator": ",",
"datetime_column": null,
"datetime_column_format": null,
"date_column": "UTCdate_____",
"date_column_format": "%Y-%m-%d",
"time_column": "UTCtime_____",
"time_column_format": "%H:%M:%S",
"unix_timestamp_column": null,
"unix_timestamp_column_format": null,
"pressure_column": "pressure",
"pressure_column_format": "hPa"
},
"atmospheric_profiles": "path-to-atmospheric-profiles",
"interferograms": "path-to-ifg-upload-directory",
"results": "path-to-results-storage"
}
},
"profiles": {
"server": {
"email": "...@...",
"max_parallel_requests": 25
},
"scope": {
"from_date": "2022-01-01",
"to_date": "2022-01-05",
"models": [
"GGG2014",
"GGG2020"
]
},
"GGG2020_standard_sites": [
{
"identifier": "mu",
"lat": 48.151,
"lon": 11.569,
"from_date": "2019-01-01",
"to_date": "2099-12-31"
}
]
},
"retrieval": {
"general": {
"max_process_count": 9,
"ifg_file_regex": "^$(SENSOR_ID)$(DATE).*\\.\\d+$",
"queue_verbosity": "compact"
},
"jobs": [
{
"retrieval_algorithm": "proffast-1.0",
"atmospheric_profile_model": "GGG2014",
"sensor_ids": [
"ma",
"mb",
"mc",
"md",
"me"
],
"from_date": "2019-01-01",
"to_date": "2022-12-31",
"settings": {
"store_binary_spectra": true,
"dc_min_threshold": 0.05,
"dc_var_threshold": 0.1,
"use_local_pressure_in_pcxs": true,
"use_ifg_corruption_filter": false,
"custom_ils": {
"ma": {
"channel1_me": 0.9892,
"channel1_pe": -0.001082,
"channel2_me": 0.9892,
"channel2_pe": -0.001082
}
},
"output_suffix": "template_config"
}
},
{
"retrieval_algorithm": "proffast-2.3",
"atmospheric_profile_model": "GGG2020",
"sensor_ids": [
"ma",
"mb",
"mc",
"md",
"me"
],
"from_date": "2019-01-01",
"to_date": "2099-12-31",
"settings": {
"store_binary_spectra": false,
"dc_min_threshold": 0.05,
"dc_var_threshold": 0.1,
"use_local_pressure_in_pcxs": false,
"use_ifg_corruption_filter": true,
"custom_ils": null,
"output_suffix": null
}
}
]
},
"bundles": [
{
"dst_dir": "directory-to-write-the-bundles-to",
"output_formats": [
"csv",
"parquet"
],
"from_datetime": "2022-01-01T00:00:00Z",
"to_datetime": "2022-12-31T23:59:59Z",
"retrieval_algorithms": [
"proffast-1.0",
"proffast-2.4"
],
"atmospheric_profile_models": [
"GGG2014",
"GGG2020"
],
"sensor_ids": [
"ma",
"mb",
"mc",
"md",
"me"
],
"bundle_suffix": null,
"retrieval_job_output_suffix": null,
"parse_dc_timeseries": true
}
]
}The next section describes how the directories for the data sources (interferograms, ground pressure, atmospheric profiles), the outputs of this pipeline (retrieval results, bundles, logs), and the internal directories (containers, etc.) are structured.