cu29_runtime/planner/
score.rs

1//! Candidates measured: their profiles beside their predictions, ranked by
2//! the constraint-ranked score computed from what was measured.
3
4use super::CuContract;
5use super::CuPrediction;
6use super::CuProfile;
7use super::propose::rate_deficit;
8use alloc::collections::BTreeMap;
9use alloc::string::String;
10use alloc::vec::Vec;
11use core::fmt;
12use cu29_traits::CuError;
13use cu29_traits::CuResult;
14use serde::Deserialize;
15use serde::Serialize;
16
17/// One candidate's measurement beside its prediction.
18#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
19#[serde(deny_unknown_fields)]
20pub struct CuScoreRow {
21    pub candidate: String,
22    /// The constraint-ranked tiers from measured p99 latencies and delivered
23    /// rates, smallest first is better.
24    pub score: Vec<f64>,
25    pub chains: BTreeMap<String, CuChainScore>,
26    /// Delivered rate per contract source.
27    pub sources: BTreeMap<String, f64>,
28    /// The proposer's prediction, when the candidate has one.
29    #[serde(default, skip_serializing_if = "Option::is_none")]
30    pub predicted: Option<CuPrediction>,
31}
32
33#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
34#[serde(deny_unknown_fields)]
35pub struct CuChainScore {
36    pub deadline_ms: u32,
37    pub samples: u64,
38    pub p50_ns: u64,
39    pub p99_ns: u64,
40    pub max_ns: u64,
41    pub misses: u64,
42    #[serde(default, skip_serializing_if = "Option::is_none")]
43    pub predicted_ns: Option<u64>,
44}
45
46/// Every measured candidate, best first.
47#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
48#[serde(deny_unknown_fields)]
49pub struct CuScoreTable {
50    pub rows: Vec<CuScoreRow>,
51}
52
53impl CuScoreTable {
54    /// Ranks `measured` (candidate name to its profile) by the contract's
55    /// constraint-ranked score. `predictions` supplies the model's numbers
56    /// for comparison, keyed by the candidate's name up to a `/`.
57    pub fn new(
58        contract: &CuContract,
59        predictions: &BTreeMap<String, CuPrediction>,
60        measured: &BTreeMap<String, CuProfile>,
61    ) -> CuResult<Self> {
62        if let Some((first, reference)) = measured.iter().next() {
63            for (candidate, profile) in measured {
64                if profile.config_signature != reference.config_signature
65                    || profile.mission != reference.mission
66                {
67                    return Err(CuError::from(format!(
68                        "Profiles of '{first}' and '{candidate}' were recorded on different graphs or missions"
69                    )));
70                }
71            }
72        }
73        let mut rows = Vec::with_capacity(measured.len());
74        for (candidate, profile) in measured {
75            // Rounds of one candidate are named `plan-1/r2`.
76            let predicted = predictions
77                .get(candidate)
78                .or_else(|| predictions.get(candidate.split('/').next().unwrap_or(candidate)));
79            let mut chains = BTreeMap::new();
80            let mut ratios = Vec::with_capacity(contract.chains.len());
81            for chain in &contract.chains {
82                let stats = profile.chains.get(&chain.id).ok_or_else(|| {
83                    CuError::from(format!(
84                        "Profile of '{candidate}' has no chain '{}'",
85                        chain.id
86                    ))
87                })?;
88                if stats.latency.samples == 0 {
89                    return Err(CuError::from(format!(
90                        "Profile of '{candidate}' has no samples for chain '{}'",
91                        chain.id
92                    )));
93                }
94                let deadline_ns = u64::from(chain.deadline_ms) * 1_000_000;
95                ratios.push(stats.latency.p99_ns as f64 / deadline_ns as f64);
96                chains.insert(
97                    chain.id.clone(),
98                    CuChainScore {
99                        deadline_ms: chain.deadline_ms,
100                        samples: stats.latency.samples,
101                        p50_ns: stats.latency.p50_ns,
102                        p99_ns: stats.latency.p99_ns,
103                        max_ns: stats.latency.max_ns,
104                        misses: stats.misses,
105                        predicted_ns: predicted
106                            .and_then(|p| p.chains.get(&chain.id))
107                            .map(|c| c.latency_ns),
108                    },
109                );
110            }
111            let mut sources = BTreeMap::new();
112            let mut rate_deficit = 0.0f64;
113            for source in &contract.sources {
114                let rate = profile
115                    .sources
116                    .get(&source.task)
117                    .ok_or_else(|| {
118                        CuError::from(format!(
119                            "Profile of '{candidate}' has no source rate for '{}'",
120                            source.task
121                        ))
122                    })?
123                    .delivered_rate;
124                rate_deficit += self::rate_deficit(rate);
125                sources.insert(source.task.clone(), rate);
126            }
127            let sum: f64 = ratios.iter().sum();
128            let score = vec![
129                round6(rate_deficit),
130                ratios.iter().filter(|&&r| r > 1.0).count() as f64,
131                ratios
132                    .iter()
133                    .filter(|&&r| r > 1.0 - contract.headroom)
134                    .count() as f64,
135                round6(sum),
136            ];
137            rows.push(CuScoreRow {
138                candidate: candidate.clone(),
139                score,
140                chains,
141                sources,
142                predicted: predicted.cloned(),
143            });
144        }
145        rows.sort_by(|a, b| {
146            a.score
147                .iter()
148                .zip(&b.score)
149                .map(|(x, y)| x.total_cmp(y))
150                .find(|o| o.is_ne())
151                .unwrap_or_else(|| a.candidate.cmp(&b.candidate))
152        });
153        Ok(Self { rows })
154    }
155
156    /// The best measured candidate.
157    pub fn best(&self) -> Option<&CuScoreRow> {
158        self.rows.first()
159    }
160
161    pub fn serialize_ron(&self) -> CuResult<String> {
162        ron::ser::to_string_pretty(self, ron::ser::PrettyConfig::default())
163            .map_err(|e| CuError::new_with_cause("Could not serialize the score table", e))
164    }
165}
166
167fn round6(value: f64) -> f64 {
168    (value * 1e6 + 0.5) as u64 as f64 / 1e6
169}
170
171fn ms(ns: u64) -> f64 {
172    ns as f64 / 1e6
173}
174
175impl fmt::Display for CuScoreTable {
176    /// One block per candidate, best first: the score, then every chain's
177    /// predicted and measured latency against its deadline.
178    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
179        for (rank, row) in self.rows.iter().enumerate() {
180            writeln!(f, "{}. {} score {:?}", rank + 1, row.candidate, row.score)?;
181            for (id, chain) in &row.chains {
182                let predicted = chain
183                    .predicted_ns
184                    .map(|ns| format!("{:.2}", ms(ns)))
185                    .unwrap_or_else(|| "-".into());
186                writeln!(
187                    f,
188                    "   {id:<32} predicted {predicted:>8} | p50 {:>8.2} p99 {:>8.2} max {:>8.2} ms | {} misses / {} of {} ms",
189                    ms(chain.p50_ns),
190                    ms(chain.p99_ns),
191                    ms(chain.max_ns),
192                    chain.misses,
193                    chain.samples,
194                    chain.deadline_ms
195                )?;
196            }
197            for (task, rate) in &row.sources {
198                let predicted = row
199                    .predicted
200                    .as_ref()
201                    .and_then(|p| p.sources.get(task))
202                    .map(|r| format!("{r:.3}"))
203                    .unwrap_or_else(|| "-".into());
204                writeln!(
205                    f,
206                    "   {task:<32} rate predicted {predicted:>8} | delivered {rate:.3}"
207                )?;
208            }
209        }
210        Ok(())
211    }
212}
213
214#[cfg(test)]
215mod tests {
216    use super::*;
217    use crate::planner::CuChain;
218    use crate::planner::CuChainProfile;
219    use crate::planner::CuCostStats;
220    use crate::planner::CuSourceProfile;
221    use crate::planner::CuSourceRate;
222
223    fn contract() -> CuContract {
224        CuContract::deserialize_ron(
225            r#"(
226            chains: [(id: "hot", source: "src", sink: "sink", deadline_ms: 10)],
227            sources: [(task: "src", period_ms: 5)],
228            cpus: [0],
229            max_in_flight: 1,
230        )"#,
231        )
232        .unwrap()
233    }
234
235    fn profile(p99_ms: u64, rate: f64) -> CuProfile {
236        let mut profile = CuProfile::new("sig".into(), "default".into());
237        let mut samples = vec![p99_ms * 1_000_000; 4];
238        samples[0] /= 2;
239        profile.chains.insert(
240            "hot".into(),
241            CuChainProfile {
242                deadline_ms: 10,
243                latency: CuCostStats::from_samples(&mut samples),
244                misses: u64::from(p99_ms > 10),
245            },
246        );
247        profile.sources.insert(
248            "src".into(),
249            CuSourceProfile {
250                period_ms: 5,
251                fired: 10,
252                expected: 10.0,
253                delivered_rate: rate,
254            },
255        );
256        profile
257    }
258
259    #[test]
260    fn ranks_measured_candidates_by_deadlines_and_keeps_predictions() {
261        let contract = contract();
262        let _ = (CuChain::clone, CuSourceRate::clone);
263        let measured: BTreeMap<String, CuProfile> = [
264            ("late".to_string(), profile(12, 1.0)),
265            ("tight".to_string(), profile(9, 1.0)),
266            ("dropping".to_string(), profile(4, 0.5)),
267        ]
268        .into_iter()
269        .collect();
270        let table = CuScoreTable::new(&contract, &BTreeMap::new(), &measured).unwrap();
271        let order: Vec<&str> = table.rows.iter().map(|r| r.candidate.as_str()).collect();
272        assert_eq!(order, ["tight", "late", "dropping"]);
273        assert_eq!(table.best().unwrap().score, vec![0.0, 0.0, 1.0, 0.9]);
274        assert_eq!(table.rows[2].score[0], 0.5);
275        assert!(table.to_string().contains("tight score"));
276        let text = table.serialize_ron().unwrap();
277        assert!(text.contains("dropping"));
278    }
279}
cu29_runtime/planner/score.rs

cu29_runtime/planner/
score.rs
