Libosmium  2.8.0
Fast and flexible C++ library for working with OpenStreetMap data
buffer.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_MEMORY_BUFFER_HPP
2 #define OSMIUM_MEMORY_BUFFER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstring>
40 #include <functional>
41 #include <iterator>
42 #include <memory>
43 #include <stdexcept>
44 #include <utility>
45 
46 #include <osmium/memory/item.hpp>
48 #include <osmium/osm/entity.hpp>
50 
51 namespace osmium {
52 
58  struct buffer_is_full : public std::runtime_error {
59 
61  std::runtime_error("Osmium buffer is full") {
62  }
63 
64  }; // struct buffer_is_full
65 
69  namespace memory {
70 
97  class Buffer {
98 
99  public:
100 
101  // This is needed so we can call std::back_inserter() on a Buffer.
102  using value_type = Item;
103 
104  enum class auto_grow : bool {
105  yes = true,
106  no = false
107  }; // enum class auto_grow
108 
109  private:
110 
111  std::unique_ptr<unsigned char[]> m_memory;
112  unsigned char* m_data;
113  size_t m_capacity;
114  size_t m_written;
115  size_t m_committed;
116  auto_grow m_auto_grow {auto_grow::no};
117  std::function<void(Buffer&)> m_full;
118 
119  public:
120 
129  Buffer() noexcept :
130  m_memory(),
131  m_data(nullptr),
132  m_capacity(0),
133  m_written(0),
134  m_committed(0) {
135  }
136 
147  explicit Buffer(unsigned char* data, size_t size) :
148  m_memory(),
149  m_data(data),
150  m_capacity(size),
151  m_written(size),
152  m_committed(size) {
153  if (size % align_bytes != 0) {
154  throw std::invalid_argument("buffer size needs to be multiple of alignment");
155  }
156  }
157 
169  explicit Buffer(unsigned char* data, size_t capacity, size_t committed) :
170  m_memory(),
171  m_data(data),
172  m_capacity(capacity),
173  m_written(committed),
174  m_committed(committed) {
175  if (capacity % align_bytes != 0) {
176  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
177  }
178  if (committed % align_bytes != 0) {
179  throw std::invalid_argument("buffer parameter 'committed' needs to be multiple of alignment");
180  }
181  }
182 
197  explicit Buffer(size_t capacity, auto_grow auto_grow = auto_grow::yes) :
198  m_memory(new unsigned char[capacity]),
199  m_data(m_memory.get()),
200  m_capacity(capacity),
201  m_written(0),
202  m_committed(0),
203  m_auto_grow(auto_grow) {
204  if (capacity % align_bytes != 0) {
205  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
206  }
207  }
208 
209  // buffers can not be copied
210  Buffer(const Buffer&) = delete;
211  Buffer& operator=(const Buffer&) = delete;
212 
213  // buffers can be moved
214  Buffer(Buffer&&) = default;
215  Buffer& operator=(Buffer&&) = default;
216 
217  ~Buffer() = default;
218 
224  unsigned char* data() const noexcept {
225  assert(m_data);
226  return m_data;
227  }
228 
233  size_t capacity() const noexcept {
234  return m_capacity;
235  }
236 
241  size_t committed() const noexcept {
242  return m_committed;
243  }
244 
250  size_t written() const noexcept {
251  return m_written;
252  }
253 
260  bool is_aligned() const noexcept {
261  assert(m_data);
262  return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0);
263  }
264 
285  OSMIUM_DEPRECATED void set_full_callback(std::function<void(Buffer&)> full) {
286  assert(m_data);
287  m_full = full;
288  }
289 
307  void grow(size_t size) {
308  assert(m_data);
309  if (!m_memory) {
310  throw std::logic_error("Can't grow Buffer if it doesn't use internal memory management.");
311  }
312  if (m_capacity < size) {
313  if (size % align_bytes != 0) {
314  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
315  }
316  std::unique_ptr<unsigned char[]> memory(new unsigned char[size]);
317  std::copy_n(m_memory.get(), m_capacity, memory.get());
318  using std::swap;
319  swap(m_memory, memory);
320  m_data = m_memory.get();
321  m_capacity = size;
322  }
323  }
324 
335  size_t commit() {
336  assert(m_data);
337  assert(is_aligned());
338 
339  const size_t offset = m_committed;
340  m_committed = m_written;
341  return offset;
342  }
343 
349  void rollback() {
350  assert(m_data);
351  m_written = m_committed;
352  }
353 
361  size_t clear() {
362  const size_t committed = m_committed;
363  m_written = 0;
364  m_committed = 0;
365  return committed;
366  }
367 
378  template <typename T>
379  T& get(const size_t offset) const {
380  assert(m_data);
381  return *reinterpret_cast<T*>(&m_data[offset]);
382  }
383 
417  unsigned char* reserve_space(const size_t size) {
418  assert(m_data);
419  // try to flush the buffer empty first.
420  if (m_written + size > m_capacity && m_full) {
421  m_full(*this);
422  }
423  // if there's still not enough space, then try growing the buffer.
424  if (m_written + size > m_capacity) {
425  if (m_memory && (m_auto_grow == auto_grow::yes)) {
426  // double buffer size until there is enough space
427  size_t new_capacity = m_capacity * 2;
428  while (m_written + size > new_capacity) {
429  new_capacity *= 2;
430  }
431  grow(new_capacity);
432  } else {
433  throw osmium::buffer_is_full();
434  }
435  }
436  unsigned char* data = &m_data[m_written];
437  m_written += size;
438  return data;
439  }
440 
456  template <typename T>
457  T& add_item(const T& item) {
458  assert(m_data);
459  unsigned char* target = reserve_space(item.padded_size());
460  std::copy_n(reinterpret_cast<const unsigned char*>(&item), item.padded_size(), target);
461  return *reinterpret_cast<T*>(target);
462  }
463 
474  void add_buffer(const Buffer& buffer) {
475  assert(m_data && buffer);
476  unsigned char* target = reserve_space(buffer.committed());
477  std::copy_n(buffer.data(), buffer.committed(), target);
478  }
479 
488  void push_back(const osmium::memory::Item& item) {
489  assert(m_data);
490  add_item(item);
491  commit();
492  }
493 
498  template <typename T>
500 
505  template <typename T>
507 
513 
519 
520  template <typename T>
522  return ItemIteratorRange<T>{m_data, m_data + m_committed};
523  }
524 
525  template <typename T>
527  return ItemIteratorRange<const T>{m_data, m_data + m_committed};
528  }
529 
538  template <typename T>
540  assert(m_data);
541  return t_iterator<T>(m_data, m_data + m_committed);
542  }
543 
553  assert(m_data);
554  return iterator(m_data, m_data + m_committed);
555  }
556 
566  template <typename T>
567  t_iterator<T> get_iterator(size_t offset) {
568  assert(m_data);
569  return t_iterator<T>(m_data + offset, m_data + m_committed);
570  }
571 
581  iterator get_iterator(size_t offset) {
582  assert(m_data);
583  return iterator(m_data + offset, m_data + m_committed);
584  }
585 
594  template <typename T>
596  assert(m_data);
597  return t_iterator<T>(m_data + m_committed, m_data + m_committed);
598  }
599 
609  assert(m_data);
610  return iterator(m_data + m_committed, m_data + m_committed);
611  }
612 
613  template <typename T>
615  assert(m_data);
616  return t_const_iterator<T>(m_data, m_data + m_committed);
617  }
618 
620  assert(m_data);
621  return const_iterator(m_data, m_data + m_committed);
622  }
623 
624  template <typename T>
625  t_const_iterator<T> get_iterator(size_t offset) const {
626  assert(m_data);
627  return t_const_iterator<T>(m_data + offset, m_data + m_committed);
628  }
629 
630  const_iterator get_iterator(size_t offset) const {
631  assert(m_data);
632  return const_iterator(m_data + offset, m_data + m_committed);
633  }
634 
635  template <typename T>
637  assert(m_data);
638  return t_const_iterator<T>(m_data + m_committed, m_data + m_committed);
639  }
640 
642  assert(m_data);
643  return const_iterator(m_data + m_committed, m_data + m_committed);
644  }
645 
646  template <typename T>
648  return cbegin<T>();
649  }
650 
652  return cbegin();
653  }
654 
655  template <typename T>
657  return cend<T>();
658  }
659 
660  const_iterator end() const {
661  return cend();
662  }
663 
667  explicit operator bool() const noexcept {
668  return m_data != nullptr;
669  }
670 
671  void swap(Buffer& other) {
672  using std::swap;
673 
674  swap(m_memory, other.m_memory);
675  swap(m_data, other.m_data);
676  swap(m_capacity, other.m_capacity);
677  swap(m_written, other.m_written);
678  swap(m_committed, other.m_committed);
679  swap(m_auto_grow, other.m_auto_grow);
680  swap(m_full, other.m_full);
681  }
682 
699  template <typename TCallbackClass>
700  void purge_removed(TCallbackClass* callback) {
701  assert(m_data);
702  if (begin() == end()) {
703  return;
704  }
705 
706  iterator it_write = begin();
707 
708  iterator next;
709  for (iterator it_read = begin(); it_read != end(); it_read = next) {
710  next = std::next(it_read);
711  if (!it_read->removed()) {
712  if (it_read != it_write) {
713  assert(it_read.data() >= data());
714  assert(it_write.data() >= data());
715  size_t old_offset = static_cast<size_t>(it_read.data() - data());
716  size_t new_offset = static_cast<size_t>(it_write.data() - data());
717  callback->moving_in_buffer(old_offset, new_offset);
718  std::memmove(it_write.data(), it_read.data(), it_read->padded_size());
719  }
720  it_write.advance_once();
721  }
722  }
723 
724  assert(it_write.data() >= data());
725  m_written = static_cast<size_t>(it_write.data() - data());
726  m_committed = m_written;
727  }
728 
729  }; // class Buffer
730 
731  inline void swap(Buffer& lhs, Buffer& rhs) {
732  lhs.swap(rhs);
733  }
734 
742  inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept {
743  if (!lhs || !rhs) {
744  return !lhs && !rhs;
745  }
746  return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed();
747  }
748 
749  inline bool operator!=(const Buffer& lhs, const Buffer& rhs) noexcept {
750  return ! (lhs == rhs);
751  }
752 
753  } // namespace memory
754 
755 } // namespace osmium
756 
757 #endif // OSMIUM_MEMORY_BUFFER_HPP
size_t m_written
Definition: buffer.hpp:114
void swap(Buffer &other)
Definition: buffer.hpp:671
t_const_iterator< T > cend() const
Definition: buffer.hpp:636
const_iterator begin() const
Definition: buffer.hpp:651
size_t clear()
Definition: buffer.hpp:361
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:50
bool is_aligned() const noexcept
Definition: buffer.hpp:260
size_t written() const noexcept
Definition: buffer.hpp:250
OSMIUM_DEPRECATED void set_full_callback(std::function< void(Buffer &)> full)
Definition: buffer.hpp:285
Definition: item_iterator.hpp:235
iterator get_iterator(size_t offset)
Definition: buffer.hpp:581
void grow(size_t size)
Definition: buffer.hpp:307
const_iterator end() const
Definition: buffer.hpp:660
Definition: item_iterator.hpp:119
constexpr bool operator==(const Box &lhs, const Box &rhs) noexcept
Definition: box.hpp:222
unsigned char * m_data
Definition: buffer.hpp:112
constexpr item_size_type align_bytes
Definition: item.hpp:53
ItemIteratorRange< const T > select() const
Definition: buffer.hpp:526
Buffer(unsigned char *data, size_t capacity, size_t committed)
Definition: buffer.hpp:169
Definition: reader_iterator.hpp:39
void swap(Buffer &lhs, Buffer &rhs)
Definition: buffer.hpp:731
Buffer(size_t capacity, auto_grow auto_grow=auto_grow::yes)
Definition: buffer.hpp:197
t_iterator< T > end()
Definition: buffer.hpp:595
Definition: item.hpp:97
Namespace for everything in the Osmium library.
Definition: assembler.hpp:66
T & add_item(const T &item)
Definition: buffer.hpp:457
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:700
t_iterator< T > begin()
Definition: buffer.hpp:539
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:474
const_iterator cbegin() const
Definition: buffer.hpp:619
size_t m_committed
Definition: buffer.hpp:115
ItemIterator< TMember > & advance_once() noexcept
Definition: item_iterator.hpp:174
unsigned char * data() const noexcept
Definition: buffer.hpp:224
Buffer() noexcept
Definition: buffer.hpp:129
size_t capacity() const noexcept
Definition: buffer.hpp:233
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
unsigned char * reserve_space(const size_t size)
Definition: buffer.hpp:417
auto_grow m_auto_grow
Definition: buffer.hpp:116
void push_back(const osmium::memory::Item &item)
Definition: buffer.hpp:488
iterator end()
Definition: buffer.hpp:608
iterator begin()
Definition: buffer.hpp:552
const_iterator cend() const
Definition: buffer.hpp:641
size_t m_capacity
Definition: buffer.hpp:113
t_const_iterator< T > get_iterator(size_t offset) const
Definition: buffer.hpp:625
size_t committed() const noexcept
Definition: buffer.hpp:241
Buffer(unsigned char *data, size_t size)
Definition: buffer.hpp:147
t_const_iterator< T > end() const
Definition: buffer.hpp:656
Definition: buffer.hpp:97
Definition: buffer.hpp:58
const_iterator get_iterator(size_t offset) const
Definition: buffer.hpp:630
t_const_iterator< T > cbegin() const
Definition: buffer.hpp:614
t_const_iterator< T > begin() const
Definition: buffer.hpp:647
auto_grow
Definition: buffer.hpp:104
data_type data() noexcept
Definition: item_iterator.hpp:195
buffer_is_full()
Definition: buffer.hpp:60
ItemIteratorRange< T > select()
Definition: buffer.hpp:521
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
void rollback()
Definition: buffer.hpp:349
std::unique_ptr< unsigned char[]> m_memory
Definition: buffer.hpp:111
bool operator!=(const Changeset &lhs, const Changeset &rhs)
Definition: changeset.hpp:431
t_iterator< T > get_iterator(size_t offset)
Definition: buffer.hpp:567
std::function< void(Buffer &)> m_full
Definition: buffer.hpp:117
size_t commit()
Definition: buffer.hpp:335