cu29_export/
lib.rs

1mod fsck;
2
3use bincode::config::standard;
4use bincode::decode_from_std_read;
5use bincode::error::DecodeError;
6use clap::{Parser, Subcommand, ValueEnum};
7use cu29::prelude::*;
8use cu29::UnifiedLogType;
9use fsck::check;
10use std::fmt::{Display, Formatter};
11use std::io::Read;
12use std::path::{Path, PathBuf};
13
14#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)]
15pub enum ExportFormat {
16    Json,
17    Csv,
18}
19
20impl Display for ExportFormat {
21    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
22        match self {
23            ExportFormat::Json => write!(f, "json"),
24            ExportFormat::Csv => write!(f, "csv"),
25        }
26    }
27}
28
29/// This is a generator for a main function to build a log extractor.
30#[derive(Parser)]
31#[command(author, version, about)]
32pub struct LogReaderCli {
33    /// The base path is the name with no _0 _1 et the end.
34    /// for example for toto_0.copper, toto_1.copper ... the base name is toto.copper
35    pub unifiedlog_base: PathBuf,
36
37    #[command(subcommand)]
38    pub command: Command,
39}
40
41#[derive(Subcommand)]
42pub enum Command {
43    /// Extract logs
44    ExtractTextLog { log_index: PathBuf },
45    /// Extract copperlists
46    ExtractCopperlists {
47        #[arg(short, long, default_value_t = ExportFormat::Json)]
48        export_format: ExportFormat,
49    },
50    /// Check the log and dump info about it.
51    Fsck {
52        #[arg(short, long, action = clap::ArgAction::Count)]
53        verbose: u8,
54    },
55}
56
57/// This is a generator for a main function to build a log extractor.
58/// It depends on the specific type of the CopperList payload that is determined at compile time from the configuration.
59pub fn run_cli<P>() -> CuResult<()>
60where
61    P: CopperListTuple,
62{
63    let args = LogReaderCli::parse();
64    let unifiedlog_base = args.unifiedlog_base;
65
66    let UnifiedLogger::Read(mut dl) = UnifiedLoggerBuilder::new()
67        .file_base_name(&unifiedlog_base)
68        .build()
69        .expect("Failed to create logger")
70    else {
71        panic!("Failed to create logger");
72    };
73
74    match args.command {
75        Command::ExtractTextLog { log_index } => {
76            let reader = UnifiedLoggerIOReader::new(dl, UnifiedLogType::StructuredLogLine);
77            textlog_dump(reader, &log_index)?;
78        }
79        Command::ExtractCopperlists { export_format } => {
80            println!("Extracting copperlists with format: {export_format}");
81            let mut reader = UnifiedLoggerIOReader::new(dl, UnifiedLogType::CopperList);
82            let iter = copperlists_reader::<P>(&mut reader);
83
84            match export_format {
85                ExportFormat::Json => {
86                    for entry in iter {
87                        serde_json::to_writer_pretty(std::io::stdout(), &entry).unwrap();
88                    }
89                }
90                ExportFormat::Csv => {
91                    let mut first = true;
92                    for origin in P::get_all_task_ids() {
93                        if !first {
94                            print!(", ");
95                        } else {
96                            print!("id, ");
97                        }
98                        print!("{origin}_time, {origin}_tov, {origin},");
99                        first = false;
100                    }
101                    println!();
102                    for entry in iter {
103                        let mut first = true;
104                        for msg in entry.cumsgs() {
105                            if let Some(payload) = msg.payload() {
106                                if !first {
107                                    print!(", ");
108                                } else {
109                                    print!("{}, ", entry.id);
110                                }
111                                let metadata = msg.metadata();
112                                print!("{}, {}, ", metadata.process_time(), msg.tov());
113                                serde_json::to_writer(std::io::stdout(), payload).unwrap(); // TODO: escape for CSV
114                                first = false;
115                            }
116                        }
117                        println!();
118                    }
119                }
120            }
121        }
122        Command::Fsck { verbose } => {
123            if let Some(value) = check::<P>(&mut dl, verbose) {
124                return value;
125            }
126        }
127    }
128
129    Ok(())
130}
131/// Extracts the copper lists from a binary representation.
132/// P is the Payload determined by the configuration of the application.
133pub fn copperlists_reader<P: CopperListTuple>(
134    mut src: impl Read,
135) -> impl Iterator<Item = CopperList<P>> {
136    std::iter::from_fn(move || {
137        let entry = decode_from_std_read::<CopperList<P>, _, _>(&mut src, standard());
138        match entry {
139            Ok(entry) => Some(entry),
140            Err(e) => match e {
141                DecodeError::UnexpectedEnd { .. } => None,
142                DecodeError::Io { inner, additional } => {
143                    if inner.kind() == std::io::ErrorKind::UnexpectedEof {
144                        None
145                    } else {
146                        println!("Error {inner:?} additional:{additional}");
147                        None
148                    }
149                }
150                _ => {
151                    println!("Error {e:?}");
152                    None
153                }
154            },
155        }
156    })
157}
158
159/// Extracts the keyframes from the log.
160pub fn keyframes_reader(mut src: impl Read) -> impl Iterator<Item = KeyFrame> {
161    std::iter::from_fn(move || {
162        let entry = decode_from_std_read::<KeyFrame, _, _>(&mut src, standard());
163        match entry {
164            Ok(entry) => Some(entry),
165            Err(e) => match e {
166                DecodeError::UnexpectedEnd { .. } => None,
167                DecodeError::Io { inner, additional } => {
168                    if inner.kind() == std::io::ErrorKind::UnexpectedEof {
169                        None
170                    } else {
171                        println!("Error {inner:?} additional:{additional}");
172                        None
173                    }
174                }
175                _ => {
176                    println!("Error {e:?}");
177                    None
178                }
179            },
180        }
181    })
182}
183
184pub fn structlog_reader(mut src: impl Read) -> impl Iterator<Item = CuResult<CuLogEntry>> {
185    std::iter::from_fn(move || {
186        let entry = decode_from_std_read::<CuLogEntry, _, _>(&mut src, standard());
187
188        match entry {
189            Err(DecodeError::UnexpectedEnd { .. }) => None,
190            Err(DecodeError::Io {
191                inner,
192                additional: _,
193            }) => {
194                if inner.kind() == std::io::ErrorKind::UnexpectedEof {
195                    None
196                } else {
197                    Some(Err(CuError::new_with_cause("Error reading log", inner)))
198                }
199            }
200            Err(e) => Some(Err(CuError::new_with_cause("Error reading log", e))),
201            Ok(entry) => {
202                if entry.msg_index == 0 {
203                    None
204                } else {
205                    Some(Ok(entry))
206                }
207            }
208        }
209    })
210}
211
212/// Full dump of the copper structured log from its binary representation.
213/// This rebuilds a textual log.
214/// src: the source of the log data
215/// index: the path to the index file (containing the interned strings constructed at build time)
216pub fn textlog_dump(src: impl Read, index: &Path) -> CuResult<()> {
217    let all_strings = read_interned_strings(index).map_err(|e| {
218        CuError::new_with_cause(
219            "Failed to read interned strings from index",
220            std::io::Error::other(e),
221        )
222    })?;
223
224    for result in structlog_reader(src) {
225        match result {
226            Ok(entry) => match rebuild_logline(&all_strings, &entry) {
227                Ok(line) => println!("{}: {}", entry.time, line),
228                Err(e) => println!("Failed to rebuild log line: {e:?}"),
229            },
230            Err(e) => return Err(e),
231        }
232    }
233
234    Ok(())
235}
236
237// only for users opting into python interface, not supported on macOS at the moment
238#[cfg(all(feature = "python", not(target_os = "macos")))]
239mod python {
240    use bincode::config::standard;
241    use bincode::decode_from_std_read;
242    use bincode::error::DecodeError;
243    use cu29::prelude::*;
244    use pyo3::exceptions::PyIOError;
245    use pyo3::prelude::*;
246    use pyo3::types::{PyDelta, PyDict, PyList};
247    use std::io::Read;
248    use std::path::Path;
249
250    #[pyclass]
251    pub struct PyLogIterator {
252        reader: Box<dyn Read + Send + Sync>,
253    }
254
255    #[pymethods]
256    impl PyLogIterator {
257        fn __iter__(slf: PyRefMut<Self>) -> PyRefMut<Self> {
258            slf
259        }
260
261        fn __next__(mut slf: PyRefMut<Self>) -> Option<PyResult<PyCuLogEntry>> {
262            match decode_from_std_read::<CuLogEntry, _, _>(&mut slf.reader, standard()) {
263                Ok(entry) => {
264                    if entry.msg_index == 0 {
265                        None
266                    } else {
267                        Some(Ok(PyCuLogEntry { inner: entry }))
268                    }
269                }
270                Err(DecodeError::UnexpectedEnd { .. }) => None,
271                Err(DecodeError::Io { inner, .. })
272                    if inner.kind() == std::io::ErrorKind::UnexpectedEof =>
273                {
274                    None
275                }
276                Err(e) => Some(Err(PyIOError::new_err(e.to_string()))),
277            }
278        }
279    }
280
281    /// Creates an iterator of CuLogEntries from a bare binary structured log file (ie. not within a unified log).
282    /// This is mainly used for using the structured logging out of the Copper framework.
283    /// it returns a tuple with the iterator of log entries and the list of interned strings.
284    #[pyfunction]
285    pub fn struct_log_iterator_bare(
286        bare_struct_src_path: &str,
287        index_path: &str,
288    ) -> PyResult<(PyLogIterator, Vec<String>)> {
289        let file = std::fs::File::open(bare_struct_src_path)
290            .map_err(|e| PyIOError::new_err(e.to_string()))?;
291        let all_strings = read_interned_strings(Path::new(index_path))
292            .map_err(|e| PyIOError::new_err(e.to_string()))?;
293        Ok((
294            PyLogIterator {
295                reader: Box::new(file),
296            },
297            all_strings,
298        ))
299    }
300    /// Creates an iterator of CuLogEntries from a unified log file.
301    /// This function allows you to easily use python to datamind Copper's structured text logs.
302    /// it returns a tuple with the iterator of log entries and the list of interned strings.
303    #[pyfunction]
304    pub fn struct_log_iterator_unified(
305        unified_src_path: &str,
306        index_path: &str,
307    ) -> PyResult<(PyLogIterator, Vec<String>)> {
308        let all_strings = read_interned_strings(Path::new(index_path))
309            .map_err(|e| PyIOError::new_err(e.to_string()))?;
310
311        let UnifiedLogger::Read(dl) = UnifiedLoggerBuilder::new()
312            .file_base_name(Path::new(unified_src_path))
313            .build()
314            .expect("Failed to create logger")
315        else {
316            panic!("Failed to create logger");
317        };
318
319        let reader = UnifiedLoggerIOReader::new(dl, UnifiedLogType::StructuredLogLine);
320        Ok((
321            PyLogIterator {
322                reader: Box::new(reader),
323            },
324            all_strings,
325        ))
326    }
327
328    /// This is a python wrapper for CuLogEntries.
329    #[pyclass]
330    pub struct PyCuLogEntry {
331        pub inner: CuLogEntry,
332    }
333
334    #[pymethods]
335    impl PyCuLogEntry {
336        /// Returns the timestamp of the log entry.
337        pub fn ts<'a>(&self, py: Python<'a>) -> Bound<'a, PyDelta> {
338            let nanoseconds: u64 = self.inner.time.into();
339
340            // Convert nanoseconds to seconds and microseconds
341            let days = (nanoseconds / 86_400_000_000_000) as i32;
342            let seconds = (nanoseconds / 1_000_000_000) as i32;
343            let microseconds = ((nanoseconds % 1_000_000_000) / 1_000) as i32;
344
345            PyDelta::new(py, days, seconds, microseconds, false).unwrap()
346        }
347
348        /// Returns the index of the message in the vector of interned strings.
349        pub fn msg_index(&self) -> u32 {
350            self.inner.msg_index
351        }
352
353        /// Returns the index of the parameter names in the vector of interned strings.
354        pub fn paramname_indexes(&self) -> Vec<u32> {
355            self.inner.paramname_indexes.iter().copied().collect()
356        }
357
358        /// Returns the parameters of this log line
359        pub fn params(&self) -> Vec<PyObject> {
360            self.inner.params.iter().map(value_to_py).collect()
361        }
362    }
363
364    /// This needs to match the name of the generated '.so'
365    #[pymodule(name = "libcu29_export")]
366    fn cu29_export(m: &Bound<'_, PyModule>) -> PyResult<()> {
367        m.add_class::<PyCuLogEntry>()?;
368        m.add_class::<PyLogIterator>()?;
369        m.add_function(wrap_pyfunction!(struct_log_iterator_bare, m)?)?;
370        m.add_function(wrap_pyfunction!(struct_log_iterator_unified, m)?)?;
371        Ok(())
372    }
373
374    fn value_to_py(value: &cu29::prelude::Value) -> PyObject {
375        match value {
376            Value::String(s) => Python::with_gil(|py| s.into_pyobject(py).unwrap().into()),
377            Value::U64(u) => Python::with_gil(|py| u.into_pyobject(py).unwrap().into()),
378            Value::I64(i) => Python::with_gil(|py| i.into_pyobject(py).unwrap().into()),
379            Value::F64(f) => Python::with_gil(|py| f.into_pyobject(py).unwrap().into()),
380            Value::Bool(b) => Python::with_gil(|py| b.into_pyobject(py).unwrap().to_owned().into()),
381            Value::CuTime(t) => Python::with_gil(|py| t.0.into_pyobject(py).unwrap().into()),
382            Value::Bytes(b) => Python::with_gil(|py| b.into_pyobject(py).unwrap().into()),
383            Value::Char(c) => Python::with_gil(|py| c.into_pyobject(py).unwrap().into()),
384            Value::I8(i) => Python::with_gil(|py| i.into_pyobject(py).unwrap().into()),
385            Value::U8(u) => Python::with_gil(|py| u.into_pyobject(py).unwrap().into()),
386            Value::I16(i) => Python::with_gil(|py| i.into_pyobject(py).unwrap().into()),
387            Value::U16(u) => Python::with_gil(|py| u.into_pyobject(py).unwrap().into()),
388            Value::I32(i) => Python::with_gil(|py| i.into_pyobject(py).unwrap().into()),
389            Value::U32(u) => Python::with_gil(|py| u.into_pyobject(py).unwrap().into()),
390            Value::Map(m) => Python::with_gil(|py| {
391                let dict = PyDict::new(py);
392                for (k, v) in m.iter() {
393                    dict.set_item(value_to_py(k), value_to_py(v)).unwrap();
394                }
395                dict.into_pyobject(py).unwrap().into()
396            }),
397            Value::F32(f) => Python::with_gil(|py| f.into_pyobject(py).unwrap().into()),
398            Value::Option(o) => Python::with_gil(|py| {
399                if o.is_none() {
400                    py.None()
401                } else {
402                    o.clone().map(|v| value_to_py(&v)).unwrap()
403                }
404            }),
405            Value::Unit => Python::with_gil(|py| py.None()),
406            Value::Newtype(v) => value_to_py(v),
407            Value::Seq(s) => Python::with_gil(|py| {
408                let list = PyList::new(py, s.iter().map(value_to_py)).unwrap();
409                list.into_pyobject(py).unwrap().into()
410            }),
411        }
412    }
413}
414
415#[cfg(test)]
416mod tests {
417    use super::*;
418    use bincode::{encode_into_slice, Decode, Encode};
419    use fs_extra::dir::{copy, CopyOptions};
420    use std::io::Cursor;
421    use std::sync::{Arc, Mutex};
422    use tempfile::{tempdir, TempDir};
423
424    fn copy_stringindex_to_temp(tmpdir: &TempDir) -> PathBuf {
425        // for some reason using the index in real only locks it and generates a change in the file.
426        let temp_path = tmpdir.path();
427
428        let mut copy_options = CopyOptions::new();
429        copy_options.copy_inside = true;
430
431        copy("test/cu29_log_index", temp_path, &copy_options).unwrap();
432        temp_path.join("cu29_log_index")
433    }
434
435    #[test]
436    fn test_extract_low_level_cu29_log() {
437        let temp_dir = TempDir::new().unwrap();
438        let temp_path = copy_stringindex_to_temp(&temp_dir);
439        let entry = CuLogEntry::new(3, CuLogLevel::Info);
440        let bytes = bincode::encode_to_vec(&entry, standard()).unwrap();
441        let reader = Cursor::new(bytes.as_slice());
442        textlog_dump(reader, temp_path.as_path()).unwrap();
443    }
444
445    #[test]
446    fn end_to_end_datalogger_and_structlog_test() {
447        let dir = tempdir().expect("Failed to create temp dir");
448        let path = dir
449            .path()
450            .join("end_to_end_datalogger_and_structlog_test.copper");
451        {
452            // Write a couple log entries
453            let UnifiedLogger::Write(logger) = UnifiedLoggerBuilder::new()
454                .write(true)
455                .create(true)
456                .file_base_name(&path)
457                .preallocated_size(100000)
458                .build()
459                .expect("Failed to create logger")
460            else {
461                panic!("Failed to create logger")
462            };
463            let data_logger = Arc::new(Mutex::new(logger));
464            let stream = stream_write(data_logger.clone(), UnifiedLogType::StructuredLogLine, 1024);
465            let rt = LoggerRuntime::init(RobotClock::default(), stream, None::<NullLog>);
466
467            let mut entry = CuLogEntry::new(4, CuLogLevel::Info); // this is a "Just a String {}" log line
468            entry.add_param(0, Value::String("Parameter for the log line".into()));
469            log(&mut entry).expect("Failed to log");
470            let mut entry = CuLogEntry::new(2, CuLogLevel::Info); // this is a "Just a String {}" log line
471            entry.add_param(0, Value::String("Parameter for the log line".into()));
472            log(&mut entry).expect("Failed to log");
473
474            // everything is dropped here
475            drop(rt);
476        }
477        // Read back the log
478        let UnifiedLogger::Read(logger) = UnifiedLoggerBuilder::new()
479            .file_base_name(
480                &dir.path()
481                    .join("end_to_end_datalogger_and_structlog_test.copper"),
482            )
483            .build()
484            .expect("Failed to create logger")
485        else {
486            panic!("Failed to create logger")
487        };
488        let reader = UnifiedLoggerIOReader::new(logger, UnifiedLogType::StructuredLogLine);
489        let temp_dir = TempDir::new().unwrap();
490        textlog_dump(
491            reader,
492            Path::new(copy_stringindex_to_temp(&temp_dir).as_path()),
493        )
494        .expect("Failed to dump log");
495    }
496
497    // This is normally generated at compile time in CuPayload.
498    #[derive(Debug, PartialEq, Clone, Copy, Serialize, Encode, Decode)]
499    struct MyMsgs((u8, i32, f32));
500
501    impl ErasedCuStampedDataSet for MyMsgs {
502        fn cumsgs(&self) -> Vec<&dyn ErasedCuStampedData> {
503            Vec::new()
504        }
505    }
506
507    impl MatchingTasks for MyMsgs {
508        fn get_all_task_ids() -> &'static [&'static str] {
509            &[]
510        }
511    }
512
513    /// Checks if we can recover the copper lists from a binary representation.
514    #[test]
515    fn test_copperlists_dump() {
516        let mut data = vec![0u8; 10000];
517        let mypls: [MyMsgs; 4] = [
518            MyMsgs((1, 2, 3.0)),
519            MyMsgs((2, 3, 4.0)),
520            MyMsgs((3, 4, 5.0)),
521            MyMsgs((4, 5, 6.0)),
522        ];
523
524        let mut offset: usize = 0;
525        for pl in mypls.iter() {
526            let cl = CopperList::<MyMsgs>::new(1, *pl);
527            offset +=
528                encode_into_slice(&cl, &mut data.as_mut_slice()[offset..], standard()).unwrap();
529        }
530
531        let reader = Cursor::new(data);
532
533        let mut iter = copperlists_reader::<MyMsgs>(reader);
534        assert_eq!(iter.next().unwrap().msgs, MyMsgs((1, 2, 3.0)));
535        assert_eq!(iter.next().unwrap().msgs, MyMsgs((2, 3, 4.0)));
536        assert_eq!(iter.next().unwrap().msgs, MyMsgs((3, 4, 5.0)));
537        assert_eq!(iter.next().unwrap().msgs, MyMsgs((4, 5, 6.0)));
538    }
539}
cu29_export/lib.rs

cu29_export/
lib.rs
