Libosmium  2.14.0
Fast and flexible C++ library for working with OpenStreetMap data
buffer.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_MEMORY_BUFFER_HPP
2 #define OSMIUM_MEMORY_BUFFER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2018 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <osmium/memory/item.hpp>
38 #include <osmium/osm/entity.hpp>
40 
41 #include <algorithm>
42 #include <cassert>
43 #include <cstddef>
44 #include <cstring>
45 #include <functional>
46 #include <iterator>
47 #include <memory>
48 #include <stdexcept>
49 #include <utility>
50 
51 namespace osmium {
52 
58  struct buffer_is_full : public std::runtime_error {
59 
61  std::runtime_error{"Osmium buffer is full"} {
62  }
63 
64  }; // struct buffer_is_full
65 
69  namespace memory {
70 
97  class Buffer {
98 
99  public:
100 
101  // This is needed so we can call std::back_inserter() on a Buffer.
102  using value_type = Item;
103 
104  enum class auto_grow : bool {
105  yes = true,
106  no = false
107  }; // enum class auto_grow
108 
109  private:
110 
111  std::unique_ptr<unsigned char[]> m_memory;
112  unsigned char* m_data = nullptr;
113  std::size_t m_capacity = 0;
114  std::size_t m_written = 0;
115  std::size_t m_committed = 0;
116 #ifndef NDEBUG
117  uint8_t m_builder_count = 0;
118 #endif
120  std::function<void(Buffer&)> m_full;
121 
122  static std::size_t calculate_capacity(std::size_t capacity) noexcept {
123  // The majority of all Nodes will fit into this size.
124  constexpr static const std::size_t min_capacity = 64;
125  if (capacity < min_capacity) {
126  return min_capacity;
127  }
128  return padded_length(capacity);
129  }
130 
131  public:
132 
141  Buffer() noexcept :
142  m_memory() {
143  }
144 
155  explicit Buffer(unsigned char* data, std::size_t size) :
156  m_memory(),
157  m_data(data),
158  m_capacity(size),
159  m_written(size),
160  m_committed(size) {
161  if (size % align_bytes != 0) {
162  throw std::invalid_argument{"buffer size needs to be multiple of alignment"};
163  }
164  }
165 
178  explicit Buffer(unsigned char* data, std::size_t capacity, std::size_t committed) :
179  m_memory(),
180  m_data(data),
184  if (capacity % align_bytes != 0) {
185  throw std::invalid_argument{"buffer capacity needs to be multiple of alignment"};
186  }
187  if (committed % align_bytes != 0) {
188  throw std::invalid_argument{"buffer parameter 'committed' needs to be multiple of alignment"};
189  }
190  if (committed > capacity) {
191  throw std::invalid_argument{"buffer parameter 'committed' can not be larger than capacity"};
192  }
193  }
194 
207  explicit Buffer(std::size_t capacity, auto_grow auto_grow = auto_grow::yes) :
208  m_memory(new unsigned char[calculate_capacity(capacity)]),
209  m_data(m_memory.get()),
212  }
213 
214  // buffers can not be copied
215  Buffer(const Buffer&) = delete;
216  Buffer& operator=(const Buffer&) = delete;
217 
218  // buffers can be moved
219  Buffer(Buffer&&) = default;
220  Buffer& operator=(Buffer&&) = default;
221 
222  ~Buffer() = default;
223 
224 #ifndef NDEBUG
225  void increment_builder_count() noexcept {
226  ++m_builder_count;
227  }
228 
229  void decrement_builder_count() noexcept {
230  assert(m_builder_count > 0);
231  --m_builder_count;
232  }
233 
234  uint8_t builder_count() const noexcept {
235  return m_builder_count;
236  }
237 #endif
238 
244  unsigned char* data() const noexcept {
245  assert(m_data && "This must be a valid buffer");
246  return m_data;
247  }
248 
253  std::size_t capacity() const noexcept {
254  return m_capacity;
255  }
256 
261  std::size_t committed() const noexcept {
262  return m_committed;
263  }
264 
270  std::size_t written() const noexcept {
271  return m_written;
272  }
273 
280  bool is_aligned() const noexcept {
281  assert(m_data && "This must be a valid buffer");
282  return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0);
283  }
284 
300  OSMIUM_DEPRECATED void set_full_callback(const std::function<void(Buffer&)>& full) {
301  assert(m_data && "This must be a valid buffer");
302  m_full = full;
303  }
304 
320  void grow(std::size_t size) {
321  assert(m_data && "This must be a valid buffer");
322  if (!m_memory) {
323  throw std::logic_error{"Can't grow Buffer if it doesn't use internal memory management."};
324  }
325  size = calculate_capacity(size);
326  if (m_capacity < size) {
327  std::unique_ptr<unsigned char[]> memory(new unsigned char[size]);
328  std::copy_n(m_memory.get(), m_capacity, memory.get());
329  using std::swap;
330  swap(m_memory, memory);
331  m_data = m_memory.get();
332  m_capacity = size;
333  }
334  }
335 
348  std::size_t commit() {
349  assert(m_data && "This must be a valid buffer");
350  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
351  assert(is_aligned());
352 
353  const std::size_t offset = m_committed;
355  return offset;
356  }
357 
364  void rollback() {
365  assert(m_data && "This must be a valid buffer");
366  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
368  }
369 
379  std::size_t clear() {
380  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
381  const std::size_t committed = m_committed;
382  m_written = 0;
383  m_committed = 0;
384  return committed;
385  }
386 
397  template <typename T>
398  T& get(const std::size_t offset) const {
399  assert(m_data && "This must be a valid buffer");
400  return *reinterpret_cast<T*>(&m_data[offset]);
401  }
402 
436  unsigned char* reserve_space(const std::size_t size) {
437  assert(m_data && "This must be a valid buffer");
438  // try to flush the buffer empty first.
439  if (m_written + size > m_capacity && m_full) {
440  m_full(*this);
441  }
442  // if there's still not enough space, then try growing the buffer.
443  if (m_written + size > m_capacity) {
444  if (m_memory && (m_auto_grow == auto_grow::yes)) {
445  // double buffer size until there is enough space
446  std::size_t new_capacity = m_capacity * 2;
447  while (m_written + size > new_capacity) {
448  new_capacity *= 2;
449  }
450  grow(new_capacity);
451  } else {
452  throw osmium::buffer_is_full{};
453  }
454  }
455  unsigned char* data = &m_data[m_written];
456  m_written += size;
457  return data;
458  }
459 
475  template <typename T>
476  T& add_item(const T& item) {
477  assert(m_data && "This must be a valid buffer");
478  unsigned char* target = reserve_space(item.padded_size());
479  std::copy_n(reinterpret_cast<const unsigned char*>(&item), item.padded_size(), target);
480  return *reinterpret_cast<T*>(target);
481  }
482 
494  void add_buffer(const Buffer& buffer) {
495  assert(m_data && "This must be a valid buffer");
496  assert(buffer && "Buffer parameter must be a valid buffer");
497  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
498  unsigned char* target = reserve_space(buffer.committed());
499  std::copy_n(buffer.data(), buffer.committed(), target);
500  }
501 
511  void push_back(const osmium::memory::Item& item) {
512  assert(m_data && "This must be a valid buffer");
513  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
514  add_item(item);
515  commit();
516  }
517 
522  template <typename T>
524 
529  template <typename T>
531 
537 
543 
544  template <typename T>
547  }
548 
549  template <typename T>
552  }
553 
562  template <typename T>
564  assert(m_data && "This must be a valid buffer");
566  }
567 
577  assert(m_data && "This must be a valid buffer");
578  return {m_data, m_data + m_committed};
579  }
580 
590  template <typename T>
591  t_iterator<T> get_iterator(std::size_t offset) {
592  assert(m_data && "This must be a valid buffer");
593  return {m_data + offset, m_data + m_committed};
594  }
595 
605  iterator get_iterator(std::size_t offset) {
606  assert(m_data && "This must be a valid buffer");
607  return {m_data + offset, m_data + m_committed};
608  }
609 
618  template <typename T>
620  assert(m_data && "This must be a valid buffer");
621  return {m_data + m_committed, m_data + m_committed};
622  }
623 
633  assert(m_data && "This must be a valid buffer");
634  return {m_data + m_committed, m_data + m_committed};
635  }
636 
637  template <typename T>
639  assert(m_data && "This must be a valid buffer");
640  return {m_data, m_data + m_committed};
641  }
642 
644  assert(m_data && "This must be a valid buffer");
645  return {m_data, m_data + m_committed};
646  }
647 
648  template <typename T>
649  t_const_iterator<T> get_iterator(std::size_t offset) const {
650  assert(m_data && "This must be a valid buffer");
651  return {m_data + offset, m_data + m_committed};
652  }
653 
654  const_iterator get_iterator(std::size_t offset) const {
655  assert(m_data && "This must be a valid buffer");
656  return {m_data + offset, m_data + m_committed};
657  }
658 
659  template <typename T>
661  assert(m_data && "This must be a valid buffer");
662  return {m_data + m_committed, m_data + m_committed};
663  }
664 
666  assert(m_data && "This must be a valid buffer");
667  return {m_data + m_committed, m_data + m_committed};
668  }
669 
670  template <typename T>
672  return cbegin<T>();
673  }
674 
676  return cbegin();
677  }
678 
679  template <typename T>
681  return cend<T>();
682  }
683 
684  const_iterator end() const {
685  return cend();
686  }
687 
691  explicit operator bool() const noexcept {
692  return m_data != nullptr;
693  }
694 
695  void swap(Buffer& other) {
696  using std::swap;
697 
698  swap(m_memory, other.m_memory);
699  swap(m_data, other.m_data);
700  swap(m_capacity, other.m_capacity);
701  swap(m_written, other.m_written);
702  swap(m_committed, other.m_committed);
703  swap(m_auto_grow, other.m_auto_grow);
704  swap(m_full, other.m_full);
705  }
706 
723  template <typename TCallbackClass>
724  void purge_removed(TCallbackClass* callback) {
725  assert(m_data && "This must be a valid buffer");
726  if (begin() == end()) {
727  return;
728  }
729 
730  iterator it_write = begin();
731 
732  iterator next;
733  for (iterator it_read = begin(); it_read != end(); it_read = next) {
734  next = std::next(it_read);
735  if (!it_read->removed()) {
736  if (it_read != it_write) {
737  assert(it_read.data() >= data());
738  assert(it_write.data() >= data());
739  const auto old_offset = static_cast<std::size_t>(it_read.data() - data());
740  const auto new_offset = static_cast<std::size_t>(it_write.data() - data());
741  callback->moving_in_buffer(old_offset, new_offset);
742  std::memmove(it_write.data(), it_read.data(), it_read->padded_size());
743  }
744  it_write.advance_once();
745  }
746  }
747 
748  assert(it_write.data() >= data());
749  m_written = static_cast<std::size_t>(it_write.data() - data());
751  }
752 
753  }; // class Buffer
754 
755  inline void swap(Buffer& lhs, Buffer& rhs) {
756  lhs.swap(rhs);
757  }
758 
766  inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept {
767  if (!lhs || !rhs) {
768  return !lhs && !rhs;
769  }
770  return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed();
771  }
772 
773  inline bool operator!=(const Buffer& lhs, const Buffer& rhs) noexcept {
774  return !(lhs == rhs);
775  }
776 
777  } // namespace memory
778 
779 } // namespace osmium
780 
781 #endif // OSMIUM_MEMORY_BUFFER_HPP
void swap(Buffer &other)
Definition: buffer.hpp:695
std::size_t capacity() const noexcept
Definition: buffer.hpp:253
t_const_iterator< T > begin() const
Definition: buffer.hpp:671
std::size_t commit()
Definition: buffer.hpp:348
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:50
bool is_aligned() const noexcept
Definition: buffer.hpp:280
iterator get_iterator(std::size_t offset)
Definition: buffer.hpp:605
Definition: item_iterator.hpp:175
bool operator!=(const Buffer &lhs, const Buffer &rhs) noexcept
Definition: buffer.hpp:773
t_const_iterator< T > cend() const
Definition: buffer.hpp:660
const_iterator cend() const
Definition: buffer.hpp:665
Definition: item_iterator.hpp:59
OSMIUM_DEPRECATED void set_full_callback(const std::function< void(Buffer &)> &full)
Definition: buffer.hpp:300
unsigned char * m_data
Definition: buffer.hpp:112
std::size_t m_committed
Definition: buffer.hpp:115
void increment_builder_count() noexcept
Definition: buffer.hpp:225
Definition: location.hpp:550
ItemIteratorRange< const T > select() const
Definition: buffer.hpp:550
void swap(Buffer &lhs, Buffer &rhs)
Definition: buffer.hpp:755
t_const_iterator< T > get_iterator(std::size_t offset) const
Definition: buffer.hpp:649
std::size_t clear()
Definition: buffer.hpp:379
static std::size_t calculate_capacity(std::size_t capacity) noexcept
Definition: buffer.hpp:122
constexpr std::size_t padded_length(std::size_t length) noexcept
Definition: item.hpp:62
std::size_t committed() const noexcept
Definition: buffer.hpp:261
Buffer(std::size_t capacity, auto_grow auto_grow=auto_grow::yes)
Definition: buffer.hpp:207
constexpr const std::size_t align_bytes
Definition: item.hpp:60
const_iterator begin() const
Definition: buffer.hpp:675
bool operator==(const Buffer &lhs, const Buffer &rhs) noexcept
Definition: buffer.hpp:766
std::size_t m_capacity
Definition: buffer.hpp:113
t_iterator< T > end()
Definition: buffer.hpp:619
Definition: item.hpp:103
uint8_t m_builder_count
Definition: buffer.hpp:117
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
T & add_item(const T &item)
Definition: buffer.hpp:476
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:724
t_iterator< T > begin()
Definition: buffer.hpp:563
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:494
T & get(const std::size_t offset) const
Definition: buffer.hpp:398
std::size_t written() const noexcept
Definition: buffer.hpp:270
uint8_t builder_count() const noexcept
Definition: buffer.hpp:234
Buffer(unsigned char *data, std::size_t capacity, std::size_t committed)
Definition: buffer.hpp:178
Buffer(unsigned char *data, std::size_t size)
Definition: buffer.hpp:155
std::size_t m_written
Definition: buffer.hpp:114
ItemIterator< TMember > & advance_once() noexcept
Definition: item_iterator.hpp:114
unsigned char * data() const noexcept
Definition: buffer.hpp:244
Buffer() noexcept
Definition: buffer.hpp:141
void grow(std::size_t size)
Definition: buffer.hpp:320
unsigned char * reserve_space(const std::size_t size)
Definition: buffer.hpp:436
auto_grow m_auto_grow
Definition: buffer.hpp:119
void push_back(const osmium::memory::Item &item)
Definition: buffer.hpp:511
iterator end()
Definition: buffer.hpp:632
iterator begin()
Definition: buffer.hpp:576
const_iterator get_iterator(std::size_t offset) const
Definition: buffer.hpp:654
Definition: buffer.hpp:97
t_const_iterator< T > cbegin() const
Definition: buffer.hpp:638
Definition: buffer.hpp:58
void decrement_builder_count() noexcept
Definition: buffer.hpp:229
const_iterator end() const
Definition: buffer.hpp:684
auto_grow
Definition: buffer.hpp:104
t_const_iterator< T > end() const
Definition: buffer.hpp:680
data_type data() noexcept
Definition: item_iterator.hpp:135
t_iterator< T > get_iterator(std::size_t offset)
Definition: buffer.hpp:591
buffer_is_full()
Definition: buffer.hpp:60
ItemIteratorRange< T > select()
Definition: buffer.hpp:545
void rollback()
Definition: buffer.hpp:364
std::unique_ptr< unsigned char[]> m_memory
Definition: buffer.hpp:111
const_iterator cbegin() const
Definition: buffer.hpp:643
std::function< void(Buffer &)> m_full
Definition: buffer.hpp:120
Buffer & operator=(const Buffer &)=delete