pjmsg_mcap_wrapper
Loading...
Searching...
No Matches
reader.hpp
Go to the documentation of this file.
1#pragma once
2
3#include "intervaltree.hpp"
4#include "read_job_queue.hpp"
5#include "types.hpp"
6#include "visibility.hpp"
7#include <cstdio>
8#include <fstream>
9#include <map>
10#include <memory>
11#include <optional>
12#include <string>
13#include <unordered_map>
14#include <unordered_set>
15#include <vector>
16
17namespace mcap {
18
19enum struct ReadSummaryMethod {
20 /**
21 * @brief Parse the Summary section to produce seeking indexes and summary
22 * statistics. If the Summary section is not present or corrupt, a failure
23 * Status is returned and the seeking indexes and summary statistics are not
24 * populated.
25 */
27 /**
28 * @brief If the Summary section is missing or incomplete, allow falling back
29 * to reading the file sequentially to produce seeking indexes and summary
30 * statistics.
31 */
33 /**
34 * @brief Read the file sequentially from Header to DataEnd to produce seeking
35 * indexes and summary statistics.
36 */
38};
39
40/**
41 * @brief An abstract interface for reading MCAP data.
42 */
44 virtual ~IReadable() = default;
45
46 /**
47 * @brief Returns the size of the file in bytes.
48 *
49 * @return uint64_t The total number of bytes in the MCAP file.
50 */
51 virtual uint64_t size() const = 0;
52 /**
53 * @brief This method is called by MCAP reader classes when they need to read
54 * a portion of the file.
55 *
56 * @param output A pointer to a pointer to the buffer to write to. This method
57 * is expected to either maintain an internal buffer, read data into it, and
58 * update this pointer to point at the internal buffer, or update this
59 * pointer to point directly at the source data if possible. The pointer and
60 * data must remain valid and unmodified until the next call to read().
61 * @param offset The offset in bytes from the beginning of the file to read.
62 * @param size The number of bytes to read.
63 * @return uint64_t Number of bytes actually read. This may be less than the
64 * requested size if the end of the file is reached. The output pointer must
65 * be readable from `output` to `output + size`. If the read fails, this
66 * method should return 0.
67 */
68 virtual uint64_t read(std::byte** output, uint64_t offset, uint64_t size) = 0;
69};
70
71/**
72 * @brief IReadable implementation wrapping a FILE* pointer created by fopen()
73 * and a read buffer.
74 */
75class MCAP_PUBLIC FileReader final : public IReadable {
76public:
77 FileReader(std::FILE* file);
78
79 uint64_t size() const override;
80 uint64_t read(std::byte** output, uint64_t offset, uint64_t size) override;
81
82private:
85 uint64_t size_;
86 uint64_t position_;
87};
88
89/**
90 * @brief IReadable implementation wrapping a std::ifstream input file stream.
91 */
93public:
95
96 uint64_t size() const override;
97 uint64_t read(std::byte** output, uint64_t offset, uint64_t size) override;
98
99private:
102 uint64_t size_;
103 uint64_t position_;
104};
105
106/**
107 * @brief An abstract interface for compressed readers.
108 */
110public:
111 virtual ~ICompressedReader() override = default;
112
113 /**
114 * @brief Reset the reader state, clearing any internal buffers and state, and
115 * initialize with new compressed data.
116 *
117 * @param data Compressed data to read from.
118 * @param size Size of the compressed data in bytes.
119 * @param uncompressedSize Size of the data in bytes after decompression. A
120 * buffer of this size will be allocated for the uncompressed data.
121 */
122 virtual void reset(const std::byte* data, uint64_t size, uint64_t uncompressedSize) = 0;
123 /**
124 * @brief Report the current status of decompression. A StatusCode other than
125 * `StatusCode::Success` after `reset()` is called indicates the decompression
126 * was not successful and the reader is in an invalid state.
127 */
128 virtual Status status() const = 0;
129};
130
131/**
132 * @brief A "null" compressed reader that directly passes through uncompressed
133 * data. No internal buffers are allocated.
134 */
136public:
137 void reset(const std::byte* data, uint64_t size, uint64_t uncompressedSize) override;
138 uint64_t read(std::byte** output, uint64_t offset, uint64_t size) override;
139 uint64_t size() const override;
140 Status status() const override;
141
142 BufferReader() = default;
143 BufferReader(const BufferReader&) = delete;
147
148private:
150 uint64_t size_;
151};
152
153#ifndef MCAP_COMPRESSION_NO_ZSTD
154/**
155 * @brief ICompressedReader implementation that decompresses Zstandard
156 * (https://facebook.github.io/zstd/) data.
157 */
159public:
160 void reset(const std::byte* data, uint64_t size, uint64_t uncompressedSize) override;
161 uint64_t read(std::byte** output, uint64_t offset, uint64_t size) override;
162 uint64_t size() const override;
163 Status status() const override;
164
165 /**
166 * @brief Decompresses an entire Zstd-compressed chunk into `output`.
167 *
168 * @param data The Zstd-compressed input chunk.
169 * @param compressedSize The size of the Zstd-compressed input.
170 * @param uncompressedSize The size of the data once uncompressed.
171 * @param output The output vector. This will be resized to `uncompressedSize` to fit the data,
172 * or 0 if the decompression encountered an error.
173 * @return Status
174 */
175 static Status DecompressAll(const std::byte* data, uint64_t compressedSize,
176 uint64_t uncompressedSize, ByteArray* output);
177 ZStdReader() = default;
178 ZStdReader(const ZStdReader&) = delete;
179 ZStdReader& operator=(const ZStdReader&) = delete;
182
183private:
186};
187#endif
188
189#ifndef MCAP_COMPRESSION_NO_LZ4
190/**
191 * @brief ICompressedReader implementation that decompresses LZ4
192 * (https://lz4.github.io/lz4/) data.
193 */
195public:
196 void reset(const std::byte* data, uint64_t size, uint64_t uncompressedSize) override;
197 uint64_t read(std::byte** output, uint64_t offset, uint64_t size) override;
198 uint64_t size() const override;
199 Status status() const override;
200
201 /**
202 * @brief Decompresses an entire LZ4-encoded chunk into `output`.
203 *
204 * @param data The LZ4-compressed input chunk.
205 * @param size The size of the LZ4-compressed input.
206 * @param uncompressedSize The size of the data once uncompressed.
207 * @param output The output vector. This will be resized to `uncompressedSize` to fit the data,
208 * or 0 if the decompression encountered an error.
209 * @return Status
210 */
211 Status decompressAll(const std::byte* data, uint64_t size, uint64_t uncompressedSize,
212 ByteArray* output);
213 LZ4Reader();
214 LZ4Reader(const LZ4Reader&) = delete;
215 LZ4Reader& operator=(const LZ4Reader&) = delete;
216 LZ4Reader(LZ4Reader&&) = delete;
218 ~LZ4Reader() override;
219
220private:
221 void* decompressionContext_ = nullptr; // LZ4F_dctx*
227};
228#endif
229
230struct LinearMessageView;
231
232/**
233 * @brief Options for reading messages out of an MCAP file.
234 */
236public:
237 /**
238 * @brief Only messages with log timestamps greater or equal to startTime will be included.
239 */
240 Timestamp startTime = 0;
241 /**
242 * @brief Only messages with log timestamps less than endTime will be included.
243 */
245 /**
246 * @brief If provided, `topicFilter` is called on all topics found in the MCAP file. If
247 * `topicFilter` returns true for a given channel, messages from that channel will be included.
248 * if not provided, messages from all channels are provided.
249 */
251 enum struct ReadOrder { FileOrder, LogTimeOrder, ReverseLogTimeOrder };
252 /**
253 * @brief Set the expected order that messages should be returned in.
254 * if readOrder == FileOrder, messages will be returned in the order they appear in the MCAP file.
255 * if readOrder == LogTimeOrder, messages will be returned in ascending log time order.
256 * if readOrder == ReverseLogTimeOrder, messages will be returned in descending log time order.
257 */
258 ReadOrder readOrder = ReadOrder::FileOrder;
259
261 : startTime(start)
262 , endTime(end) {}
263
265
266 /**
267 * @brief validate the configuration.
268 */
269 Status validate() const;
270};
271
272/**
273 * @brief Provides a read interface to an MCAP file.
274 */
276public:
277 ~McapReader();
278
279 /**
280 * @brief Opens an MCAP file for reading from an already constructed IReadable
281 * implementation.
282 *
283 * @param reader An implementation of the IReader interface that provides raw
284 * MCAP data.
285 * @return Status StatusCode::Success on success. If a non-success Status is
286 * returned, the data source is not considered open and McapReader is not
287 * usable until `open()` is called and a success response is returned.
288 */
289 Status open(IReadable& reader);
290 /**
291 * @brief Opens an MCAP file for reading from a given filename.
292 *
293 * @param filename Filename to open.
294 * @return Status StatusCode::Success on success. If a non-success Status is
295 * returned, the data source is not considered open and McapReader is not
296 * usable until `open()` is called and a success response is returned.
297 */
298 Status open(std::string_view filename);
299 /**
300 * @brief Opens an MCAP file for reading from a std::ifstream input file
301 * stream.
302 *
303 * @param stream Input file stream to read MCAP data from.
304 * @return Status StatusCode::Success on success. If a non-success Status is
305 * returned, the file is not considered open and McapReader is not usable
306 * until `open()` is called and a success response is returned.
307 */
308 Status open(std::ifstream& stream);
309
310 /**
311 * @brief Closes the MCAP file, clearing any internal data structures and
312 * state and dropping the data source reference.
313 *
314 */
315 void close();
316
317 /**
318 * @brief Read and parse the Summary section at the end of the MCAP file, if
319 * available. This will populate internal indexes to allow for efficient
320 * summarization and random access. This method will automatically be called
321 * upon requesting summary data or first seek if Summary section parsing is
322 * allowed by the configuration options.
323 */
324 Status readSummary(
325 ReadSummaryMethod method, const ProblemCallback& onProblem = [](const Status&) {});
326
327 /**
328 * @brief Returns an iterable view with `begin()` and `end()` methods for
329 * iterating Messages in the MCAP file. If a non-zero `startTime` is provided,
330 * this will first parse the Summary section (by calling `readSummary()`) if
331 * allowed by the configuration options and it has not been parsed yet.
332 *
333 * @param startTime Optional start time in nanoseconds. Messages before this
334 * time will not be returned.
335 * @param endTime Optional end time in nanoseconds. Messages equal to or after
336 * this time will not be returned.
337 */
338 LinearMessageView readMessages(Timestamp startTime = 0, Timestamp endTime = MaxTime);
339 /**
340 * @brief Returns an iterable view with `begin()` and `end()` methods for
341 * iterating Messages in the MCAP file. If a non-zero `startTime` is provided,
342 * this will first parse the Summary section (by calling `readSummary()`) if
343 * allowed by the configuration options and it has not been parsed yet.
344 *
345 * @param onProblem A callback that will be called when a parsing error
346 * occurs. Problems can either be recoverable, indicating some data could
347 * not be read, or non-recoverable, stopping the iteration.
348 * @param startTime Optional start time in nanoseconds. Messages before this
349 * time will not be returned.
350 * @param endTime Optional end time in nanoseconds. Messages equal to or after
351 * this time will not be returned.
352 */
353 LinearMessageView readMessages(const ProblemCallback& onProblem, Timestamp startTime = 0,
354 Timestamp endTime = MaxTime);
355
356 /**
357 * @brief Returns an iterable view with `begin()` and `end()` methods for
358 * iterating Messages in the MCAP file.
359 * Uses the options from `options` to select the messages that are yielded.
360 */
361 LinearMessageView readMessages(const ProblemCallback& onProblem,
362 const ReadMessageOptions& options);
363
364 /**
365 * @brief Returns starting and ending byte offsets that must be read to
366 * iterate all messages in the given time range. If `readSummary()` has been
367 * successfully called and the recording contains Chunk records, this range
368 * will be narrowed to Chunk records that contain messages in the given time
369 * range. Otherwise, this range will be the entire Data section if the Data
370 * End record has been found or the entire file otherwise.
371 *
372 * This method is automatically used by `readMessages()`, and only needs to be
373 * called directly if the caller is manually constructing an iterator.
374 *
375 * @param startTime Start time in nanoseconds.
376 * @param endTime Optional end time in nanoseconds.
377 * @return Start and end byte offsets.
378 */
380 Timestamp endTime = MaxTime) const;
381
382 /**
383 * @brief Returns a pointer to the IReadable data source backing this reader.
384 * Will return nullptr if the reader is not open.
385 */
386 IReadable* dataSource();
387
388 /**
389 * @brief Returns the parsed Header record, if it has been encountered.
390 */
391 const std::optional<Header>& header() const;
392 /**
393 * @brief Returns the parsed Footer record, if it has been encountered.
394 */
395 const std::optional<Footer>& footer() const;
396 /**
397 * @brief Returns the parsed Statistics record, if it has been encountered.
398 */
399 const std::optional<Statistics>& statistics() const;
400
401 /**
402 * @brief Returns all of the parsed Channel records. Call `readSummary()`
403 * first to fully populate this data structure.
404 */
405 const std::unordered_map<ChannelId, ChannelPtr> channels() const;
406 /**
407 * @brief Returns all of the parsed Schema records. Call `readSummary()`
408 * first to fully populate this data structure.
409 */
410 const std::unordered_map<SchemaId, SchemaPtr> schemas() const;
411
412 /**
413 * @brief Look up a Channel record by channel ID. If the Channel has not been
414 * encountered yet or does not exist in the file, this will return nullptr.
415 *
416 * @param channelId Channel ID to search for
417 * @return ChannelPtr A shared pointer to a Channel record, or nullptr
418 */
419 ChannelPtr channel(ChannelId channelId) const;
420 /**
421 * @brief Look up a Schema record by schema ID. If the Schema has not been
422 * encountered yet or does not exist in the file, this will return nullptr.
423 *
424 * @param schemaId Schema ID to search for
425 * @return SchemaPtr A shared pointer to a Schema record, or nullptr
426 */
427 SchemaPtr schema(SchemaId schemaId) const;
428
429 /**
430 * @brief Returns all of the parsed ChunkIndex records. Call `readSummary()`
431 * first to fully populate this data structure.
432 */
433 const std::vector<ChunkIndex>& chunkIndexes() const;
434
435 /**
436 * @brief Returns all of the parsed MetadataIndex records. Call `readSummary()`
437 * first to fully populate this data structure.
438 * The multimap's keys are the `name` field from each indexed Metadata.
439 */
440 const std::multimap<std::string, MetadataIndex>& metadataIndexes() const;
441
442 /**
443 * @brief Returns all of the parsed AttachmentIndex records. Call `readSummary()`
444 * first to fully populate this data structure.
445 * The multimap's keys are the `name` field from each indexed Attachment.
446 */
447 const std::multimap<std::string, AttachmentIndex>& attachmentIndexes() const;
448
449 // The following static methods are used internally for parsing MCAP records
450 // and do not need to be called directly unless you are implementing your own
451 // reader functionality or tests.
452
453 static Status ReadRecord(IReadable& reader, uint64_t offset, Record* record);
454 static Status ReadFooter(IReadable& reader, uint64_t offset, Footer* footer);
455
456 static Status ParseHeader(const Record& record, Header* header);
457 static Status ParseFooter(const Record& record, Footer* footer);
458 static Status ParseSchema(const Record& record, Schema* schema);
459 static Status ParseChannel(const Record& record, Channel* channel);
460 static Status ParseMessage(const Record& record, Message* message);
461 static Status ParseChunk(const Record& record, Chunk* chunk);
462 static Status ParseMessageIndex(const Record& record, MessageIndex* messageIndex);
463 static Status ParseChunkIndex(const Record& record, ChunkIndex* chunkIndex);
464 static Status ParseAttachment(const Record& record, Attachment* attachment);
465 static Status ParseAttachmentIndex(const Record& record, AttachmentIndex* attachmentIndex);
466 static Status ParseStatistics(const Record& record, Statistics* statistics);
467 static Status ParseMetadata(const Record& record, Metadata* metadata);
468 static Status ParseMetadataIndex(const Record& record, MetadataIndex* metadataIndex);
469 static Status ParseSummaryOffset(const Record& record, SummaryOffset* summaryOffset);
470 static Status ParseDataEnd(const Record& record, DataEnd* dataEnd);
471
472 /**
473 * @brief Converts a compression string ("", "zstd", "lz4") to the Compression enum.
474 */
475 static std::optional<Compression> ParseCompression(const std::string_view compression);
476
477private:
480
481 IReadable* input_ = nullptr;
482 std::FILE* file_ = nullptr;
494 ByteOffset dataStart_ = 0;
496 bool parsedSummary_ = false;
497
498 void reset_();
499 Status readSummarySection_(IReadable& reader);
500 Status readSummaryFromScan_(IReadable& reader);
501};
502
503/**
504 * @brief A low-level interface for parsing MCAP-style TLV records from a data
505 * source.
506 */
510
511 RecordReader(IReadable& dataSource, ByteOffset startOffset, ByteOffset endOffset = EndOffset);
512
513 void reset(IReadable& dataSource, ByteOffset startOffset, ByteOffset endOffset);
514
516
517 const Status& status() const;
518
519 ByteOffset curRecordOffset() const;
520
521private:
522 IReadable* dataSource_ = nullptr;
525};
526
532
538
539 void reset(const Chunk& chunk, Compression compression);
540
541 bool next();
542
543 ByteOffset offset() const;
544
545 const Status& status() const;
546
547private:
551#ifndef MCAP_COMPRESSION_NO_LZ4
553#endif
554#ifndef MCAP_COMPRESSION_NO_ZSTD
556#endif
557};
558
559/**
560 * @brief A mid-level interface for parsing and validating MCAP records from a
561 * data source.
562 */
581
582 TypedRecordReader(IReadable& dataSource, ByteOffset startOffset,
583 ByteOffset endOffset = EndOffset);
584
589
590 bool next();
591
592 ByteOffset offset() const;
593
594 const Status& status() const;
595
596private:
601};
602
603/**
604 * @brief Uses message indices to read messages out of an MCAP in log time order.
605 * The underlying MCAP must be chunked, with a summary section and message indexes.
606 * The required McapWriterOptions are:
607 * - noChunking: false
608 * - noMessageIndex: false
609 * - noSummary: false
610 */
612public:
613 IndexedMessageReader(McapReader& reader, const ReadMessageOptions& options,
614 const std::function<void(const Message&, RecordOffset)> onMessage);
615
616 /**
617 * @brief reads the next message out of the MCAP.
618 *
619 * @return true if a message was found.
620 * @return false if no more messages are to be read. If there was some error reading the MCAP,
621 * `status()` will return a non-Success status.
622 */
623 bool next();
624
625 /**
626 * @brief gets the status of the reader.
627 *
628 * @return Status
629 */
630 Status status() const;
631
632private:
638 size_t findFreeChunkSlot();
639 void decompressChunk(const Chunk& chunk, ChunkSlot& slot);
643#ifndef MCAP_COMPRESSION_NO_LZ4
645#endif
651};
652
653/**
654 * @brief An iterable view of Messages in an MCAP file.
655 */
659 using difference_type = int64_t;
661 using pointer = const MessageView*;
662 using reference = const MessageView&;
663
664 reference operator*() const;
665 pointer operator->() const;
666 Iterator& operator++();
667 void operator++(int);
668 MCAP_PUBLIC friend bool operator==(const Iterator& a, const Iterator& b);
669 MCAP_PUBLIC friend bool operator!=(const Iterator& a, const Iterator& b);
670
671 private:
673
674 Iterator() = default;
676
677 class Impl {
678 public:
679 Impl(LinearMessageView& view);
680
681 Impl(const Impl&) = delete;
682 Impl& operator=(const Impl&) = delete;
683 Impl(Impl&&) = delete;
684 Impl& operator=(Impl&&) = delete;
685
686 void increment();
687 reference dereference() const;
688 bool has_value() const;
689
691
696
697 private:
698 void onMessage(const Message& message, RecordOffset offset);
699 };
700
701 bool begun_ = false;
703 };
704
705 LinearMessageView(McapReader& mcapReader, const ProblemCallback& onProblem);
706 LinearMessageView(McapReader& mcapReader, ByteOffset dataStart, ByteOffset dataEnd,
707 Timestamp startTime, Timestamp endTime, const ProblemCallback& onProblem);
708 LinearMessageView(McapReader& mcapReader, const ReadMessageOptions& options, ByteOffset dataStart,
709 ByteOffset dataEnd, const ProblemCallback& onProblem);
710
715
716 Iterator begin();
717 Iterator end();
718
719private:
725};
726
727} // namespace mcap
728
729#ifdef MCAP_IMPLEMENTATION
730# include "reader.inl"
731#endif
A "null" compressed reader that directly passes through uncompressed data. No internal buffers are al...
Definition reader.hpp:135
const std::byte * data_
Definition reader.hpp:149
BufferReader()=default
BufferReader(const BufferReader &)=delete
BufferReader(BufferReader &&)=delete
BufferReader & operator=(BufferReader &&)=delete
BufferReader & operator=(const BufferReader &)=delete
IReadable implementation wrapping a FILE* pointer created by fopen() and a read buffer.
Definition reader.hpp:75
uint64_t size_
Definition reader.hpp:85
std::FILE * file_
Definition reader.hpp:83
uint64_t position_
Definition reader.hpp:86
std::vector< std::byte > buffer_
Definition reader.hpp:84
IReadable implementation wrapping a std::ifstream input file stream.
Definition reader.hpp:92
std::ifstream & stream_
Definition reader.hpp:100
std::vector< std::byte > buffer_
Definition reader.hpp:101
An abstract interface for compressed readers.
Definition reader.hpp:109
virtual Status status() const =0
Report the current status of decompression. A StatusCode other than StatusCode::Success after reset()...
virtual void reset(const std::byte *data, uint64_t size, uint64_t uncompressedSize)=0
Reset the reader state, clearing any internal buffers and state, and initialize with new compressed d...
virtual ~ICompressedReader() override=default
ICompressedReader implementation that decompresses LZ4 (https://lz4.github.io/lz4/) data.
Definition reader.hpp:194
LZ4Reader & operator=(LZ4Reader &&)=delete
LZ4Reader(LZ4Reader &&)=delete
LZ4Reader & operator=(const LZ4Reader &)=delete
uint64_t compressedSize_
Definition reader.hpp:225
ByteArray uncompressedData_
Definition reader.hpp:224
LZ4Reader(const LZ4Reader &)=delete
const std::byte * compressedData_
Definition reader.hpp:223
uint64_t uncompressedSize_
Definition reader.hpp:226
std::optional< MessageView > curMessageView_
Definition reader.hpp:695
Impl & operator=(const Impl &)=delete
std::optional< TypedRecordReader > recordReader_
Definition reader.hpp:692
std::optional< IndexedMessageReader > indexedMessageReader_
Definition reader.hpp:693
Provides a read interface to an MCAP file.
Definition reader.hpp:275
std::unique_ptr< FileReader > fileInput_
Definition reader.hpp:483
std::vector< ChunkIndex > chunkIndexes_
Definition reader.hpp:488
std::unordered_map< SchemaId, SchemaPtr > schemas_
Definition reader.hpp:492
std::optional< Footer > footer_
Definition reader.hpp:486
std::optional< Header > header_
Definition reader.hpp:485
std::unique_ptr< FileStreamReader > fileStreamInput_
Definition reader.hpp:484
friend LinearMessageView
Definition reader.hpp:479
internal::IntervalTree< ByteOffset, ChunkIndex > chunkRanges_
Definition reader.hpp:489
std::multimap< std::string, MetadataIndex > metadataIndexes_
Definition reader.hpp:491
std::optional< Statistics > statistics_
Definition reader.hpp:487
std::unordered_map< ChannelId, ChannelPtr > channels_
Definition reader.hpp:493
std::multimap< std::string, AttachmentIndex > attachmentIndexes_
Definition reader.hpp:490
ICompressedReader implementation that decompresses Zstandard (https://facebook.github....
Definition reader.hpp:158
ZStdReader(const ZStdReader &)=delete
ZStdReader()=default
ByteArray uncompressedData_
Definition reader.hpp:185
ZStdReader & operator=(const ZStdReader &)=delete
ZStdReader(ZStdReader &&)=delete
ZStdReader & operator=(ZStdReader &&)=delete
Definition crc32.hpp:5
constexpr Timestamp MaxTime
Definition types.hpp:32
uint16_t SchemaId
Definition types.hpp:19
Compression
Supported MCAP compression algorithms.
Definition types.hpp:37
uint64_t Timestamp
Definition types.hpp:21
constexpr ByteOffset EndOffset
Definition types.hpp:31
ReadSummaryMethod
Definition reader.hpp:19
@ AllowFallbackScan
If the Summary section is missing or incomplete, allow falling back to reading the file sequentially ...
@ ForceScan
Read the file sequentially from Header to DataEnd to produce seeking indexes and summary statistics.
@ NoFallbackScan
Parse the Summary section to produce seeking indexes and summary statistics. If the Summary section i...
uint16_t ChannelId
Definition types.hpp:20
bool operator==(const LinearMessageView::Iterator &a, const LinearMessageView::Iterator &b)
Definition reader.inl:1774
uint64_t ByteOffset
Definition types.hpp:22
Attachment Index records are found in the Summary section, providing summary information for a single...
Definition types.hpp:270
An Attachment is an arbitrary file embedded in an MCAP file, including a name, media type,...
Definition types.hpp:256
Describes a Channel that messages are written to. A Channel represents a single connection from a pub...
Definition types.hpp:159
Chunk Index records are found in the Summary section, providing summary information for a single Chun...
Definition types.hpp:239
An collection of Schemas, Channels, and Messages that supports compression and indexing.
Definition types.hpp:215
The final record in the Data section, signaling the end of Data and beginning of Summary....
Definition types.hpp:350
The final record in an MCAP file (before the trailing magic byte sequence). Contains byte offsets fro...
Definition types.hpp:115
Appears at the beginning of every MCAP file (after the magic byte sequence) and contains the recordin...
Definition types.hpp:103
An abstract interface for reading MCAP data.
Definition reader.hpp:43
virtual ~IReadable()=default
virtual uint64_t read(std::byte **output, uint64_t offset, uint64_t size)=0
This method is called by MCAP reader classes when they need to read a portion of the file.
virtual uint64_t size() const =0
Returns the size of the file in bytes.
Uses message indices to read messages out of an MCAP in log time order. The underlying MCAP must be c...
Definition reader.hpp:611
RecordReader recordReader_
Definition reader.hpp:642
std::vector< ChunkSlot > chunkSlots_
Definition reader.hpp:650
std::function< void(const Message &, RecordOffset)> onMessage_
Definition reader.hpp:648
internal::ReadJobQueue queue_
Definition reader.hpp:649
ReadMessageOptions options_
Definition reader.hpp:646
std::unordered_set< ChannelId > selectedChannels_
Definition reader.hpp:647
std::unique_ptr< Impl > impl_
Definition reader.hpp:702
An iterable view of Messages in an MCAP file.
Definition reader.hpp:656
ReadMessageOptions readMessageOptions_
Definition reader.hpp:723
LinearMessageView & operator=(const LinearMessageView &)=delete
LinearMessageView(const LinearMessageView &)=delete
McapReader & mcapReader_
Definition reader.hpp:720
LinearMessageView(LinearMessageView &&)=default
LinearMessageView & operator=(LinearMessageView &&)=delete
const ProblemCallback onProblem_
Definition reader.hpp:724
A list of timestamps to byte offsets for a single Channel. This record appears after each Chunk,...
Definition types.hpp:229
Returned when iterating over Messages in a file, MessageView contains a reference to one Message,...
Definition types.hpp:388
A single Message published to a Channel.
Definition types.hpp:182
Metadata Index records are found in the Summary section, providing summary information for a single M...
Definition types.hpp:325
Holds a named map of key/value strings containing arbitrary user data. Metadata records are found in ...
Definition types.hpp:316
Options for reading messages out of an MCAP file.
Definition reader.hpp:235
std::function< bool(std::string_view)> topicFilter
If provided, topicFilter is called on all topics found in the MCAP file. If topicFilter returns true ...
Definition reader.hpp:250
ReadMessageOptions(Timestamp start, Timestamp end)
Definition reader.hpp:260
A low-level interface for parsing MCAP-style TLV records from a data source.
Definition reader.hpp:507
ByteOffset offset
Definition reader.hpp:508
ByteOffset endOffset
Definition reader.hpp:509
A generic Type-Length-Value record using a uint8 type and uint64 length. This is the generic form of ...
Definition types.hpp:87
Describes a schema used for message encoding and decoding and/or describing the shape of messages....
Definition types.hpp:132
The Statistics record is found in the Summary section, providing counts and timestamp ranges for the ...
Definition types.hpp:300
Wraps a status code and string message carrying additional context.
Definition errors.hpp:36
Summary Offset records are found in the Summary Offset section. Records in the Summary section are gr...
Definition types.hpp:340
std::function< void(const Message &, ByteOffset)> onMessage
Definition reader.hpp:530
TypedChunkReader & operator=(const TypedChunkReader &)=delete
std::function< void(const SchemaPtr, ByteOffset)> onSchema
Definition reader.hpp:528
std::function< void(const Record &, ByteOffset)> onUnknownRecord
Definition reader.hpp:531
std::function< void(const ChannelPtr, ByteOffset)> onChannel
Definition reader.hpp:529
TypedChunkReader(TypedChunkReader &&)=delete
RecordReader reader_
Definition reader.hpp:548
BufferReader uncompressedReader_
Definition reader.hpp:550
ZStdReader zstdReader_
Definition reader.hpp:555
TypedChunkReader(const TypedChunkReader &)=delete
TypedChunkReader & operator=(TypedChunkReader &&)=delete
A mid-level interface for parsing and validating MCAP records from a data source.
Definition reader.hpp:563
std::function< void(const Message &, ByteOffset, std::optional< ByteOffset >)> onMessage
Definition reader.hpp:568
std::function< void(const Metadata &, ByteOffset)> onMetadata
Definition reader.hpp:575
std::function< void(ByteOffset)> onChunkEnd
Definition reader.hpp:580
std::function< void(const Record &, ByteOffset, std::optional< ByteOffset >)> onUnknownRecord
Definition reader.hpp:579
std::function< void(const SummaryOffset &, ByteOffset)> onSummaryOffset
Definition reader.hpp:577
std::function< void(const Footer &, ByteOffset)> onFooter
Definition reader.hpp:565
std::function< void(const DataEnd &, ByteOffset)> onDataEnd
Definition reader.hpp:578
std::function< void(const ChannelPtr, ByteOffset, std::optional< ByteOffset >)> onChannel
Definition reader.hpp:567
TypedRecordReader & operator=(TypedRecordReader &&)=delete
std::function< void(const Attachment &, ByteOffset)> onAttachment
Definition reader.hpp:572
RecordReader reader_
Definition reader.hpp:597
std::function< void(const ChunkIndex &, ByteOffset)> onChunkIndex
Definition reader.hpp:571
std::function< void(const MetadataIndex &, ByteOffset)> onMetadataIndex
Definition reader.hpp:576
TypedRecordReader(const TypedRecordReader &)=delete
TypedRecordReader & operator=(const TypedRecordReader &)=delete
std::function< void(const Chunk &, ByteOffset)> onChunk
Definition reader.hpp:569
TypedChunkReader chunkReader_
Definition reader.hpp:598
std::function< void(const Statistics &, ByteOffset)> onStatistics
Definition reader.hpp:574
std::function< void(const AttachmentIndex &, ByteOffset)> onAttachmentIndex
Definition reader.hpp:573
std::function< void(const SchemaPtr, ByteOffset, std::optional< ByteOffset >)> onSchema
Definition reader.hpp:566
std::function< void(const Header &, ByteOffset)> onHeader
Definition reader.hpp:564
TypedRecordReader(TypedRecordReader &&)=delete
std::function< void(const MessageIndex &, ByteOffset)> onMessageIndex
Definition reader.hpp:570
A priority queue of jobs for an indexed MCAP reader to execute.
#define MCAP_PUBLIC