cu29_export/
lib.rs

1use std::fmt::{Display, Formatter};
2use std::io::Read;
3use std::path::{Path, PathBuf};
4
5use bincode::config::standard;
6use bincode::decode_from_std_read;
7use bincode::error::DecodeError;
8use clap::{Parser, Subcommand, ValueEnum};
9use cu29::prelude::*;
10
11#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)]
12pub enum ExportFormat {
13    Json,
14    Csv,
15}
16
17impl Display for ExportFormat {
18    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
19        match self {
20            ExportFormat::Json => write!(f, "json"),
21            ExportFormat::Csv => write!(f, "csv"),
22        }
23    }
24}
25
26/// This is a generator for a main function to build a log extractor.
27#[derive(Parser)]
28#[command(author, version, about)]
29pub struct LogReaderCli {
30    /// The base path is the name with no _0 _1 et the end.
31    /// for example for toto_0.copper, toto_1.copper ... the base name is toto.copper
32    pub unifiedlog_base: PathBuf,
33
34    #[command(subcommand)]
35    pub command: Command,
36}
37
38#[derive(Subcommand)]
39pub enum Command {
40    /// Extract logs
41    ExtractLog { log_index: PathBuf },
42    /// Extract copperlists
43    ExtractCopperlist {
44        #[arg(short, long, default_value_t = ExportFormat::Json)]
45        export_format: ExportFormat,
46    },
47}
48
49/// This is a generator for a main function to build a log extractor.
50/// It depends on the specific type of the CopperList payload that is determined at compile time from the configuration.
51pub fn run_cli<P>() -> CuResult<()>
52where
53    P: CopperListTuple,
54{
55    let args = LogReaderCli::parse();
56    let unifiedlog_base = args.unifiedlog_base;
57
58    let UnifiedLogger::Read(dl) = UnifiedLoggerBuilder::new()
59        .file_base_name(&unifiedlog_base)
60        .build()
61        .expect("Failed to create logger")
62    else {
63        panic!("Failed to create logger");
64    };
65
66    match args.command {
67        Command::ExtractLog { log_index } => {
68            let reader = UnifiedLoggerIOReader::new(dl, UnifiedLogType::StructuredLogLine);
69            textlog_dump(reader, &log_index)?;
70        }
71        Command::ExtractCopperlist { export_format } => {
72            println!("Extracting copperlists with format: {export_format}");
73            let mut reader = UnifiedLoggerIOReader::new(dl, UnifiedLogType::CopperList);
74            let iter = copperlists_dump::<P>(&mut reader);
75            for entry in iter {
76                println!("{entry:#?}");
77            }
78        }
79    }
80
81    Ok(())
82}
83
84/// Extracts the copper lists from a binary representation.
85/// P is the Payload determined by the configuration of the application.
86pub fn copperlists_dump<P: CopperListTuple>(
87    mut src: impl Read,
88) -> impl Iterator<Item = CopperList<P>> {
89    std::iter::from_fn(move || {
90        let entry = decode_from_std_read::<CopperList<P>, _, _>(&mut src, standard());
91        match entry {
92            Ok(entry) => Some(entry),
93            Err(e) => match e {
94                DecodeError::UnexpectedEnd { .. } => None,
95                DecodeError::Io { inner, additional } => {
96                    if inner.kind() == std::io::ErrorKind::UnexpectedEof {
97                        None
98                    } else {
99                        println!("Error {inner:?} additional:{additional}");
100                        None
101                    }
102                }
103                _ => {
104                    println!("Error {e:?}");
105                    None
106                }
107            },
108        }
109    })
110}
111
112/// Full dump of the copper structured log from its binary representation.
113/// This rebuilds a textual log.
114/// src: the source of the log data
115/// index: the path to the index file (containing the interned strings constructed at build time)
116pub fn textlog_dump(mut src: impl Read, index: &Path) -> CuResult<()> {
117    let all_strings = read_interned_strings(index)?;
118    loop {
119        let entry = decode_from_std_read::<CuLogEntry, _, _>(&mut src, standard());
120
121        match entry {
122            Err(DecodeError::UnexpectedEnd { .. }) => return Ok(()),
123            Err(DecodeError::Io { inner, additional }) => {
124                if inner.kind() == std::io::ErrorKind::UnexpectedEof {
125                    return Ok(());
126                } else {
127                    println!("Error {inner:?} additional:{additional}");
128                    return Err(CuError::new_with_cause("Error reading log", inner));
129                }
130            }
131            Err(e) => {
132                println!("Error {e:?}");
133                return Err(CuError::new_with_cause("Error reading log", e));
134            }
135            Ok(entry) => {
136                if entry.msg_index == 0 {
137                    break;
138                }
139
140                let result = rebuild_logline(&all_strings, &entry);
141                if result.is_err() {
142                    println!("Failed to rebuild log line: {result:?}");
143                    continue;
144                }
145                println!("{}: {}", entry.time, result.unwrap());
146            }
147        };
148    }
149    Ok(())
150}
151
152// only for users opting into python interface, not supported on macOS at the moment
153#[cfg(all(feature = "python", not(target_os = "macos")))]
154mod python {
155    use bincode::config::standard;
156    use bincode::decode_from_std_read;
157    use bincode::error::DecodeError;
158    use cu29::prelude::*;
159    use pyo3::exceptions::PyIOError;
160    use pyo3::prelude::*;
161    use pyo3::types::{PyDelta, PyDict, PyList};
162    use std::io::Read;
163    use std::path::Path;
164
165    #[pyclass]
166    pub struct PyLogIterator {
167        reader: Box<dyn Read + Send + Sync>,
168    }
169
170    #[pymethods]
171    impl PyLogIterator {
172        fn __iter__(slf: PyRefMut<Self>) -> PyRefMut<Self> {
173            slf
174        }
175
176        fn __next__(mut slf: PyRefMut<Self>) -> Option<PyResult<PyCuLogEntry>> {
177            match decode_from_std_read::<CuLogEntry, _, _>(&mut slf.reader, standard()) {
178                Ok(entry) => {
179                    if entry.msg_index == 0 {
180                        None
181                    } else {
182                        Some(Ok(PyCuLogEntry { inner: entry }))
183                    }
184                }
185                Err(DecodeError::UnexpectedEnd { .. }) => None,
186                Err(DecodeError::Io { inner, .. })
187                    if inner.kind() == std::io::ErrorKind::UnexpectedEof =>
188                {
189                    None
190                }
191                Err(e) => Some(Err(PyIOError::new_err(e.to_string()))),
192            }
193        }
194    }
195
196    /// Creates an iterator of CuLogEntries from a bare binary structured log file (ie. not within a unified log).
197    /// This is mainly used for using the structured logging out of the Copper framework.
198    /// it returns a tuple with the iterator of log entries and the list of interned strings.
199    #[pyfunction]
200    pub fn struct_log_iterator_bare(
201        bare_struct_src_path: &str,
202        index_path: &str,
203    ) -> PyResult<(PyLogIterator, Vec<String>)> {
204        let file = std::fs::File::open(bare_struct_src_path)
205            .map_err(|e| PyIOError::new_err(e.to_string()))?;
206        let all_strings = read_interned_strings(Path::new(index_path))
207            .map_err(|e| PyIOError::new_err(e.to_string()))?;
208        Ok((
209            PyLogIterator {
210                reader: Box::new(file),
211            },
212            all_strings,
213        ))
214    }
215    /// Creates an iterator of CuLogEntries from a unified log file.
216    /// This function allows you to easily use python to datamind Copper's structured text logs.
217    /// it returns a tuple with the iterator of log entries and the list of interned strings.
218    #[pyfunction]
219    pub fn struct_log_iterator_unified(
220        unified_src_path: &str,
221        index_path: &str,
222    ) -> PyResult<(PyLogIterator, Vec<String>)> {
223        let all_strings = read_interned_strings(Path::new(index_path))
224            .map_err(|e| PyIOError::new_err(e.to_string()))?;
225
226        let UnifiedLogger::Read(dl) = UnifiedLoggerBuilder::new()
227            .file_base_name(Path::new(unified_src_path))
228            .build()
229            .expect("Failed to create logger")
230        else {
231            panic!("Failed to create logger");
232        };
233
234        let reader = UnifiedLoggerIOReader::new(dl, UnifiedLogType::StructuredLogLine);
235        Ok((
236            PyLogIterator {
237                reader: Box::new(reader),
238            },
239            all_strings,
240        ))
241    }
242
243    /// This is a python wrapper for CuLogEntries.
244    #[pyclass]
245    pub struct PyCuLogEntry {
246        pub inner: CuLogEntry,
247    }
248
249    #[pymethods]
250    impl PyCuLogEntry {
251        /// Returns the timestamp of the log entry.
252        pub fn ts<'a>(&self, py: Python<'a>) -> Bound<'a, PyDelta> {
253            let nanoseconds: u64 = self.inner.time.into();
254
255            // Convert nanoseconds to seconds and microseconds
256            let days = (nanoseconds / 86_400_000_000_000) as i32;
257            let seconds = (nanoseconds / 1_000_000_000) as i32;
258            let microseconds = ((nanoseconds % 1_000_000_000) / 1_000) as i32;
259
260            PyDelta::new(py, days, seconds, microseconds, false).unwrap()
261        }
262
263        /// Returns the index of the message in the vector of interned strings.
264        pub fn msg_index(&self) -> u32 {
265            self.inner.msg_index
266        }
267
268        /// Returns the index of the parameter names in the vector of interned strings.
269        pub fn paramname_indexes(&self) -> Vec<u32> {
270            self.inner.paramname_indexes.iter().copied().collect()
271        }
272
273        /// Returns the parameters of this log line
274        pub fn params(&self) -> Vec<PyObject> {
275            self.inner.params.iter().map(value_to_py).collect()
276        }
277    }
278
279    /// This needs to match the name of the generated '.so'
280    #[pymodule(name = "libcu29_export")]
281    fn cu29_export(m: &Bound<'_, PyModule>) -> PyResult<()> {
282        m.add_class::<PyCuLogEntry>()?;
283        m.add_class::<PyLogIterator>()?;
284        m.add_function(wrap_pyfunction!(struct_log_iterator_bare, m)?)?;
285        m.add_function(wrap_pyfunction!(struct_log_iterator_unified, m)?)?;
286        Ok(())
287    }
288
289    fn value_to_py(value: &cu29::prelude::Value) -> PyObject {
290        match value {
291            Value::String(s) => Python::with_gil(|py| s.into_pyobject(py).unwrap().into()),
292            Value::U64(u) => Python::with_gil(|py| u.into_pyobject(py).unwrap().into()),
293            Value::I64(i) => Python::with_gil(|py| i.into_pyobject(py).unwrap().into()),
294            Value::F64(f) => Python::with_gil(|py| f.into_pyobject(py).unwrap().into()),
295            Value::Bool(b) => Python::with_gil(|py| b.into_pyobject(py).unwrap().to_owned().into()),
296            Value::CuTime(t) => Python::with_gil(|py| t.0.into_pyobject(py).unwrap().into()),
297            Value::Bytes(b) => Python::with_gil(|py| b.into_pyobject(py).unwrap().into()),
298            Value::Char(c) => Python::with_gil(|py| c.into_pyobject(py).unwrap().into()),
299            Value::I8(i) => Python::with_gil(|py| i.into_pyobject(py).unwrap().into()),
300            Value::U8(u) => Python::with_gil(|py| u.into_pyobject(py).unwrap().into()),
301            Value::I16(i) => Python::with_gil(|py| i.into_pyobject(py).unwrap().into()),
302            Value::U16(u) => Python::with_gil(|py| u.into_pyobject(py).unwrap().into()),
303            Value::I32(i) => Python::with_gil(|py| i.into_pyobject(py).unwrap().into()),
304            Value::U32(u) => Python::with_gil(|py| u.into_pyobject(py).unwrap().into()),
305            Value::Map(m) => Python::with_gil(|py| {
306                let dict = PyDict::new(py);
307                for (k, v) in m.iter() {
308                    dict.set_item(value_to_py(k), value_to_py(v)).unwrap();
309                }
310                dict.into_pyobject(py).unwrap().into()
311            }),
312            Value::F32(f) => Python::with_gil(|py| f.into_pyobject(py).unwrap().into()),
313            Value::Option(o) => Python::with_gil(|py| {
314                if o.is_none() {
315                    py.None()
316                } else {
317                    o.clone().map(|v| value_to_py(&v)).unwrap()
318                }
319            }),
320            Value::Unit => Python::with_gil(|py| py.None()),
321            Value::Newtype(v) => value_to_py(v),
322            Value::Seq(s) => Python::with_gil(|py| {
323                let list = PyList::new(py, s.iter().map(value_to_py)).unwrap();
324                list.into_pyobject(py).unwrap().into()
325            }),
326        }
327    }
328}
329
330#[cfg(test)]
331mod tests {
332    use super::*;
333    use bincode::encode_into_slice;
334    use fs_extra::dir::{copy, CopyOptions};
335    use std::io::Cursor;
336    use std::sync::{Arc, Mutex};
337    use tempfile::{tempdir, TempDir};
338
339    fn copy_stringindex_to_temp(tmpdir: &TempDir) -> PathBuf {
340        // for some reason using the index in real only locks it and generates a change in the file.
341        let temp_path = tmpdir.path();
342
343        let mut copy_options = CopyOptions::new();
344        copy_options.copy_inside = true;
345
346        copy("test/cu29_log_index", temp_path, &copy_options).unwrap();
347        temp_path.join("cu29_log_index")
348    }
349
350    #[test]
351    fn test_extract_low_level_cu29_log() {
352        let temp_dir = TempDir::new().unwrap();
353        let temp_path = copy_stringindex_to_temp(&temp_dir);
354        let entry = CuLogEntry::new(3);
355        let bytes = bincode::encode_to_vec(&entry, standard()).unwrap();
356        let reader = Cursor::new(bytes.as_slice());
357        textlog_dump(reader, temp_path.as_path()).unwrap();
358    }
359
360    #[test]
361    fn end_to_end_datalogger_and_structlog_test() {
362        let dir = tempdir().expect("Failed to create temp dir");
363        let path = dir
364            .path()
365            .join("end_to_end_datalogger_and_structlog_test.copper");
366        {
367            // Write a couple log entries
368            let UnifiedLogger::Write(logger) = UnifiedLoggerBuilder::new()
369                .write(true)
370                .create(true)
371                .file_base_name(&path)
372                .preallocated_size(100000)
373                .build()
374                .expect("Failed to create logger")
375            else {
376                panic!("Failed to create logger")
377            };
378            let data_logger = Arc::new(Mutex::new(logger));
379            let stream = stream_write(data_logger.clone(), UnifiedLogType::StructuredLogLine, 1024);
380            let rt = LoggerRuntime::init(RobotClock::default(), stream, None::<NullLog>);
381
382            let mut entry = CuLogEntry::new(4); // this is a "Just a String {}" log line
383            entry.add_param(0, Value::String("Parameter for the log line".into()));
384            log(&mut entry).expect("Failed to log");
385            let mut entry = CuLogEntry::new(2); // this is a "Just a String {}" log line
386            entry.add_param(0, Value::String("Parameter for the log line".into()));
387            log(&mut entry).expect("Failed to log");
388
389            // everything is dropped here
390            drop(rt);
391        }
392        // Read back the log
393        let UnifiedLogger::Read(logger) = UnifiedLoggerBuilder::new()
394            .file_base_name(
395                &dir.path()
396                    .join("end_to_end_datalogger_and_structlog_test.copper"),
397            )
398            .build()
399            .expect("Failed to create logger")
400        else {
401            panic!("Failed to create logger")
402        };
403        let reader = UnifiedLoggerIOReader::new(logger, UnifiedLogType::StructuredLogLine);
404        let temp_dir = TempDir::new().unwrap();
405        textlog_dump(
406            reader,
407            Path::new(copy_stringindex_to_temp(&temp_dir).as_path()),
408        )
409        .expect("Failed to dump log");
410    }
411
412    // This is normally generated at compile time in CuPayload.
413    type MyCuPayload = (u8, i32, f32);
414
415    /// Checks if we can recover the copper lists from a binary representation.
416    #[test]
417    fn test_copperlists_dump() {
418        let mut data = vec![0u8; 10000];
419        let mypls: [MyCuPayload; 4] = [(1, 2, 3.0), (2, 3, 4.0), (3, 4, 5.0), (4, 5, 6.0)];
420
421        let mut offset: usize = 0;
422        for pl in mypls.iter() {
423            let cl = CopperList::<MyCuPayload>::new(1, *pl);
424            offset +=
425                encode_into_slice(&cl, &mut data.as_mut_slice()[offset..], standard()).unwrap();
426        }
427
428        let reader = Cursor::new(data);
429
430        let mut iter = copperlists_dump::<MyCuPayload>(reader);
431        assert_eq!(iter.next().unwrap().msgs, (1, 2, 3.0));
432        assert_eq!(iter.next().unwrap().msgs, (2, 3, 4.0));
433        assert_eq!(iter.next().unwrap().msgs, (3, 4, 5.0));
434        assert_eq!(iter.next().unwrap().msgs, (4, 5, 6.0));
435    }
436}
cu29_export/lib.rs

cu29_export/
lib.rs
