Feature/mars2grib era6 reencoding support

share/metkit/params.yaml

@@ -3603,6 +3603,24 @@
     - 228240
     - 228246
     - 228247
+- - levtype: sfc
+  - - 254001
+    - 254002
+    - 254003
+    - 254004
+    - 254005
+    - 254006
+    - 254007
+    - 254008
+    - 254009
+    - 254010
+    - 254011
+    - 254012
+    - 254013
+    - 254014
+    - 254015
+    - 254016
+    - 254017
 - - class: od
     levtype: al
     stream: elda

src/metkit/mars2grib/backend/concepts/AllConcepts.h

@@ -99,9 +99,11 @@
 #include "metkit/mars2grib/backend/concepts/destine/destineConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/ensemble/ensembleConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/generating-process/generatingProcessConceptDescriptor.h"
+#include "metkit/mars2grib/backend/concepts/iteration/iterationConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/level/levelConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/longrange/longrangeConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/mars/marsConceptDescriptor.h"
+#include "metkit/mars2grib/backend/concepts/model-error/modelErrorConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/nil/nilConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/origin/originConceptDescriptor.h"
 #include "metkit/mars2grib/backend/concepts/packing/packingConceptDescriptor.h"
@@ -168,10 +170,10 @@ using TypeList = metkit::mars2grib::backend::compile_time_registry_engine::TypeL
 /// Higher-level code should interact with concepts exclusively through
 /// registry APIs, not by iterating this list directly.
 ///
-using AllConcepts =
-    TypeList<AnalysisConcept, CompositionConcept, DataTypeConcept, DerivedConcept, DestineConcept, EnsembleConcept,
-             GeneratingProcessConcept, LevelConcept, LongrangeConcept, MarsConcept, NilConcept, OriginConcept,
-             PackingConcept, ParamConcept, PointInTimeConcept, ReferenceTimeConcept, RepresentationConcept,
-             SatelliteConcept, ShapeOfTheEarthConcept, StatisticsConcept, TablesConcept, WaveConcept>;
+using AllConcepts = TypeList<AnalysisConcept, CompositionConcept, DataTypeConcept, DerivedConcept, DestineConcept,
+                             EnsembleConcept, GeneratingProcessConcept, LevelConcept, LongrangeConcept,
+                             IterationConcept, MarsConcept, NilConcept, OriginConcept, PackingConcept, ParamConcept,
+                             PointInTimeConcept, ReferenceTimeConcept, RepresentationConcept, SatelliteConcept,
+                             ShapeOfTheEarthConcept, StatisticsConcept, TablesConcept, WaveConcept, ModelErrorConcept>;
 
 }  // namespace metkit::mars2grib::backend::concepts_::detail

src/metkit/mars2grib/backend/concepts/analysis/analysisEncoding.h

@@ -150,7 +150,7 @@ void AnalysisOp(const MarsDict_t& mars, const ParDict_t& par, const OptDict_t& o
             MARS2GRIB_LOG_CONCEPT(analysis);
 
             // Structural validation
-            validation::match_LocalDefinitionNumber_or_throw(opt, out, {36L});
+            validation::match_LocalDefinitionNumber_or_throw(opt, out, {36L, 37L, 38L, 39L});
 
             // Deductions
             long offsetToEndOf4DvarWindowVal = deductions::resolve_offsetToEndOf4DvarWindow_or_throw(mars, par, opt);

src/metkit/mars2grib/backend/concepts/concepts.md

@@ -47,7 +47,60 @@ The ordered list of variants for a concept defines its **local variant index spa
 
 ---
 
-## 3. Concept Descriptor Contract
+## 3. New Concept vs New Variant
+
+When changing the concept system, first decide whether the requested behavior is
+a new concept or a new variant of an existing concept.
+
+Use a **new concept** when the feature is an independent semantic axis that must
+be composable with other concepts. Use a **new variant** when the feature is an
+alternative realization inside an existing semantic axis and does not require
+independent composability.
+
+This distinction is often a domain decision and is usually not reliably
+deducible from the code alone. If a request does not explicitly state whether a
+new concept or a new variant is required, ask before implementing.
+
+---
+
+## 4. Level Concept Guardrail
+
+The `level` concept is one of the most constrained concepts in mars2grib.
+Although GRIB ultimately represents vertical levels through six low-level
+fixed-surface keys:
+
+* `typeOfFirstFixedSurface`
+* `scaleFactorOfFirstFixedSurface`
+* `scaledValueOfFirstFixedSurface`
+* `typeOfSecondFixedSurface`
+* `scaleFactorOfSecondFixedSurface`
+* `scaledValueOfSecondFixedSurface`
+
+mars2grib must not set these keys directly. Many combinations of these keys are
+syntactically possible but semantically meaningless for ECMWF products.
+
+Instead, the encoder must rely on the official level abstraction:
+
+* `typeOfLevel`
+* `level`, when required
+* `topLevel` / `bottomLevel`, when required
+* PV-array data, when required
+
+Each supported `typeOfLevel` corresponds to a `LevelType` variant, apart from a
+few virtual type-of-level values kept in mars2grib because they cannot be added
+to ecCodes for backward-compatibility reasons. Each variant maps to a prescribed
+configuration of the low-level fixed-surface keys.
+
+Do not implement level fixes by injecting `typeOfFirstFixedSurface`,
+`scaleFactorOfFirstFixedSurface`, `scaledValueOfFirstFixedSurface`,
+`typeOfSecondFixedSurface`, `scaleFactorOfSecondFixedSurface`, or
+`scaledValueOfSecondFixedSurface`. If a new level behavior is required, add or
+adjust the appropriate `LevelType` variant, matcher mapping, or deduction so the
+level remains encoded through `typeOfLevel` and the official level interface.
+
+---
+
+## 5. Concept Descriptor Contract
 
 Each concept is implemented as a **descriptor type** that conforms to the
 `RegisterEntryDescriptor` interface.
@@ -68,7 +121,7 @@ The descriptor contains **no runtime state** and no virtual functions.
 
 ---
 
-## 4. Capabilities
+## 6. Capabilities
 
 Concepts may expose multiple independent *capabilities*.
 
@@ -87,7 +140,7 @@ independent dispatch planes.
 
 ---
 
-## 5. The Concept Universe (`AllConcepts`)
+## 7. The Concept Universe (`AllConcepts`)
 
 All concepts known to the system are aggregated into a single ordered typelist:
 
@@ -107,7 +160,7 @@ Changing this order is a **breaking structural change**.
 
 ---
 
-## 6. Concept Identifiers
+## 8. Concept Identifiers
 
 Each concept is assigned a **stable numeric identifier** based on its position
 in `AllConcepts`.
@@ -130,7 +183,7 @@ They are used as indices into:
 
 ---
 
-## 7. Variant Index Spaces
+## 9. Variant Index Spaces
 
 Variants are indexed in two ways:
 
@@ -158,7 +211,7 @@ The global variant index is the primary key used by:
 
 ---
 
-## 8. Matching Phase
+## 10. Matching Phase
 
 Matching determines **which concepts and variants are active** for a given
 input request.
@@ -179,7 +232,7 @@ The result is an `ActiveConceptsData` structure.
 
 ---
 
-## 9. Encoding Phases
+## 11. Encoding Phases
 
 Encoding is divided into **logical stages**, such as:
 
@@ -201,7 +254,7 @@ All dispatch tables are generated **entirely at compile time**.
 
 ---
 
-## 10. Design Principles
+## 12. Design Principles
 
 The concept system is designed around the following principles:
 
@@ -217,7 +270,7 @@ Execution code performs *only iteration and invocation*.
 
 ---
 
-## 11. Adding a New Concept
+## 13. Adding a New Concept
 
 To add a new concept:
 
@@ -231,7 +284,7 @@ No registry code needs to be modified.
 
 ---
 
-## 12. Summary
+## 14. Summary
 
 Concepts are the **semantic backbone** of the mars2grib backend.
 

src/metkit/mars2grib/backend/concepts/derived/derivedEncoding.h

@@ -51,8 +51,10 @@
 #include "metkit/mars2grib/utils/generalUtils.h"
 
 // Deductions
+#include "metkit/mars2grib/backend/deductions/channel.h"
 #include "metkit/mars2grib/backend/deductions/derivedForecast.h"
 #include "metkit/mars2grib/backend/deductions/numberOfForecastsInEnsemble.h"
+#include "metkit/mars2grib/backend/deductions/numberOfFrequencies.h"
 
 // Tables
 #include "metkit/mars2grib/backend/tables/derivedForecast.h"
@@ -96,7 +98,7 @@ namespace metkit::mars2grib::backend::concepts_ {
 ///
 template <std::size_t Stage, std::size_t Section, DerivedType Variant>
 constexpr bool derivedApplicable() {
-    return (Stage == StagePreset) && (Section == SecProductDefinitionSection);
+    return (Stage == StagePreset) && ((Section == SecProductDefinitionSection) || (Section == SecLocalUseSection));
 }
 
 ///
@@ -158,16 +160,34 @@ void DerivedOp(const MarsDict_t& mars, const ParDict_t& par, const OptDict_t& op
 
             MARS2GRIB_LOG_CONCEPT(derived);
 
-            // Structural validation
-            validation::check_DerivedProductDefinitionSection_or_throw(opt, out);
+            if constexpr (Section == SecLocalUseSection && Stage == StagePreset &&
+                          Variant == DerivedType::BrightnessTemperatureEnsembleMean) {
 
-            // Deductions
-            tables::DerivedForecast derivedForecast = deductions::resolve_DerivedForecast_or_throw(mars, par, opt);
-            long numberOfForecastsInEnsemble = deductions::resolve_NumberOfForecastsInEnsemble_or_throw(mars, par, opt);
+                // Check/Validation
+                validation::match_LocalDefinitionNumber_or_throw(opt, out, {37});
 
-            // Encoding
-            set_or_throw<long>(out, "derivedForecast", static_cast<long>(derivedForecast));
-            set_or_throw<long>(out, "numberOfForecastsInEnsemble", numberOfForecastsInEnsemble);
+                // Deductions
+                long channelNumber       = deductions::resolve_Channel_or_throw(mars, par, opt);
+                long numberOfFrequencies = deductions::resolve_NumberOfFrequencies_or_throw(mars, par, opt);
+
+                // Encoding
+                set_or_throw<long>(out, "channelNumber", channelNumber);
+                set_or_throw<long>(out, "numberOfFrequencies", numberOfFrequencies);
+            }
+
+            if constexpr (Section == SecProductDefinitionSection && Stage == StagePreset) {
+                // Structural validation
+                validation::check_DerivedProductDefinitionSection_or_throw(opt, out);
+
+                // Deductions
+                tables::DerivedForecast derivedForecast = deductions::resolve_DerivedForecast_or_throw(mars, par, opt);
+                long numberOfForecastsInEnsemble =
+                    deductions::resolve_NumberOfForecastsInEnsemble_or_throw(mars, par, opt);
+
+                // Encoding
+                set_or_throw<long>(out, "derivedForecast", static_cast<long>(derivedForecast));
+                set_or_throw<long>(out, "numberOfForecastsInEnsemble", numberOfForecastsInEnsemble);
+            }
         }
         catch (...) {
 

src/metkit/mars2grib/backend/concepts/derived/derivedEnum.h

@@ -83,21 +83,7 @@ inline constexpr std::string_view derivedName{"derived"};
 /// tables and registries.
 ///
 enum class DerivedType : std::size_t {
-    Individual = 0,
-    Derived,
-    PerturbedParameters,
-    RandomPatterns,
-    MeanUnweightedAll,
-    MeanWeightedAll,
-    StddevCluster,
-    StddevClusterNorm,
-    SpreadAll,
-    LargeAnomalyIndex,
-    MeanUnweightedCluster,
-    Iqr,
-    MinAll,
-    MaxAll,
-    VarianceAll,
+    BrightnessTemperatureEnsembleMean,  // Special variant for satellite brightness temperature products
     Default
 };
 
@@ -114,11 +100,7 @@ enum class DerivedType : std::size_t {
 /// The order of this list must match the intended iteration order
 /// for registry construction and diagnostics.
 ///
-using DerivedList = ValueList<DerivedType::Individual, DerivedType::Derived, DerivedType::PerturbedParameters,
-                              DerivedType::RandomPatterns, DerivedType::MeanUnweightedAll, DerivedType::MeanWeightedAll,
-                              DerivedType::StddevCluster, DerivedType::StddevClusterNorm, DerivedType::SpreadAll,
-                              DerivedType::LargeAnomalyIndex, DerivedType::MeanUnweightedCluster, DerivedType::Iqr,
-                              DerivedType::MinAll, DerivedType::MaxAll, DerivedType::VarianceAll, DerivedType::Default>;
+using DerivedList = ValueList<DerivedType::BrightnessTemperatureEnsembleMean, DerivedType::Default>;
 
 
 ///
@@ -148,21 +130,7 @@ constexpr std::string_view derivedTypeName();
         return NAME;                                  \
     }
 
-DEF(DerivedType::Individual, "individual");
-DEF(DerivedType::Derived, "derived");
-DEF(DerivedType::PerturbedParameters, "perturbedParameters");
-DEF(DerivedType::RandomPatterns, "randomPatterns");
-DEF(DerivedType::MeanUnweightedAll, "meanUnweightedAll");
-DEF(DerivedType::MeanWeightedAll, "meanWeightedAll");
-DEF(DerivedType::StddevCluster, "stddevCluster");
-DEF(DerivedType::StddevClusterNorm, "stddevClusterNorm");
-DEF(DerivedType::SpreadAll, "spreadAll");
-DEF(DerivedType::LargeAnomalyIndex, "largeAnomalyIndex");
-DEF(DerivedType::MeanUnweightedCluster, "meanUnweightedCluster");
-DEF(DerivedType::Iqr, "iqr");
-DEF(DerivedType::MinAll, "minAll");
-DEF(DerivedType::MaxAll, "maxAll");
-DEF(DerivedType::VarianceAll, "varianceAll");
+DEF(DerivedType::BrightnessTemperatureEnsembleMean, "brightnessTemperatureEnsembleMean");
 DEF(DerivedType::Default, "default");
 
 #undef DEF

src/metkit/mars2grib/backend/concepts/derived/derivedMatcher.h

@@ -13,11 +13,13 @@ namespace metkit::mars2grib::backend::concepts_ {
 
 template <class MarsDict_t, class OptDict_t>
 std::size_t derivedMatcher(const MarsDict_t& mars, const OptDict_t& opt) {
+
     using metkit::mars2grib::utils::dict_traits::get_or_throw;
+    using metkit::mars2grib::utils::dict_traits::has;
 
     const auto& type = get_or_throw<std::string>(mars, "type");
-    if (type == "em" ||    // Ensemble mean
-        type == "es" ||    // Ensemble standard deviation
+    if (type == "es" ||    // Ensemble standard deviation
+        type == "ses" ||   // Ensemble spread of estimation
         type == "taem" ||  // Time-averaged ensemble mean
         type == "taes" ||  // Time-averaged ensemble standard deviation
         type == "efi" ||   // Extreme forecast index
@@ -26,6 +28,16 @@ std::size_t derivedMatcher(const MarsDict_t& mars, const OptDict_t& opt) {
         return static_cast<std::size_t>(DerivedType::Default);
     }
 
+    if (type == "em") {  // Ensemble mean (special handling for brightness temperature products)
+        if (has(mars, "channel") && has(mars, "param") && get_or_throw<long>(mars, "param") == 194 &&
+            has(mars, "stream") && get_or_throw<std::string>(mars, "stream") == "elda") {
+            return static_cast<std::size_t>(DerivedType::BrightnessTemperatureEnsembleMean);
+        }
+        else {
+            return static_cast<std::size_t>(DerivedType::Default);
+        }
+    }
+
     return compile_time_registry_engine::MISSING;
 }
 

src/metkit/mars2grib/backend/concepts/ensemble/ensembleMatcher.h

@@ -2,6 +2,7 @@
 
 // System include
 #include <cstddef>
+#include <string>
 
 // Utils
 #include "metkit/mars2grib/backend/concepts/ensemble/ensembleEnum.h"
@@ -12,8 +13,15 @@ namespace metkit::mars2grib::backend::concepts_ {
 
 template <class MarsDict_t, class OptDict_t>
 std::size_t ensembleMatcher(const MarsDict_t& mars, const OptDict_t& opt) {
+    using metkit::mars2grib::utils::dict_traits::get_or_throw;
     using metkit::mars2grib::utils::dict_traits::has;
 
+    // Skip model-error products: in that case "number" identifies the
+    // model-error realization, not an ensemble member.
+    if (has(mars, "type") && get_or_throw<std::string>(mars, "type") == "eme") {
+        return compile_time_registry_engine::MISSING;
+    }
+
     if (has(mars, "number")) {
         return static_cast<std::size_t>(EnsembleType::Individual);
     }