Libosmium  2.15.2
Fast and flexible C++ library for working with OpenStreetMap data
memory_mapping.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_UTIL_MEMORY_MAPPING_HPP
2 #define OSMIUM_UTIL_MEMORY_MAPPING_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2019 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
37 #include <osmium/util/file.hpp>
38 
39 #include <cassert>
40 #include <cerrno>
41 #include <cstddef>
42 #include <stdexcept>
43 #include <system_error>
44 
45 #ifndef _WIN32
46 # include <sys/mman.h>
47 #else
48 # include <fcntl.h>
49 # include <io.h>
50 # include <windows.h>
51 # include <sys/types.h>
52 #endif
53 
54 namespace osmium {
55 
56  inline namespace util {
57 
95  class MemoryMapping {
96 
97  public:
98 
99  enum class mapping_mode {
100  readonly = 0,
101  write_private = 1,
102  write_shared = 2
103  };
104 
105  private:
106 
108  std::size_t m_size;
109 
111  off_t m_offset;
112 
114  int m_fd;
115 
118 
119 #ifdef _WIN32
120  HANDLE m_handle;
121 #endif
122 
124  void* m_addr;
125 
126  bool is_valid() const noexcept;
127 
128  void make_invalid() noexcept;
129 
130 #ifdef _WIN32
131  using flag_type = DWORD;
132 #else
133  using flag_type = int;
134 #endif
135 
136  flag_type get_protection() const noexcept;
137 
138  flag_type get_flags() const noexcept;
139 
140  static std::size_t check_size(std::size_t size) {
141  if (size == 0) {
142  return osmium::get_pagesize();
143  }
144  return size;
145  }
146 
147 #ifdef _WIN32
148  HANDLE get_handle() const noexcept;
149  HANDLE create_file_mapping() const noexcept;
150  void* map_view_of_file() const noexcept;
151 #endif
152 
153  int resize_fd(int fd) {
154  // Anonymous mapping doesn't need resizing.
155  if (fd == -1) {
156  return -1;
157  }
158 
159  // Make sure the file backing this mapping is large enough.
160  if (osmium::file_size(fd) < m_size + m_offset) {
162  }
163  return fd;
164  }
165 
166  public:
167 
184  MemoryMapping(std::size_t size, mapping_mode mode, int fd = -1, off_t offset = 0);
185 
191  OSMIUM_DEPRECATED MemoryMapping(std::size_t size, bool writable = true, int fd = -1, off_t offset = 0) : // NOLINT(google-explicit-constructor, hicpp-explicit-conversions)
192  MemoryMapping(size, writable ? mapping_mode::write_shared : mapping_mode::readonly, fd, offset) {
193  }
194 
196  MemoryMapping(const MemoryMapping&) = delete;
197 
199  MemoryMapping& operator=(const MemoryMapping&) = delete;
200 
205  MemoryMapping(MemoryMapping&& other) noexcept;
206 
210  MemoryMapping& operator=(MemoryMapping&& other) noexcept;
211 
216  ~MemoryMapping() noexcept {
217  try {
218  unmap();
219  } catch (const std::system_error&) {
220  // Ignore any exceptions because destructor must not throw.
221  }
222  }
223 
230  void unmap();
231 
243  void resize(std::size_t new_size);
244 
249  explicit operator bool() const noexcept {
250  return is_valid();
251  }
252 
258  std::size_t size() const noexcept {
259  return m_size;
260  }
261 
267  int fd() const noexcept {
268  return m_fd;
269  }
270 
274  bool writable() const noexcept {
276  }
277 
283  template <typename T = void>
284  T* get_addr() const noexcept {
285  return reinterpret_cast<T*>(m_addr);
286  }
287 
288  }; // class MemoryMapping
289 
301 
302  public:
303 
304  explicit AnonymousMemoryMapping(std::size_t size) :
305  MemoryMapping(size, mapping_mode::write_private) {
306  }
307 
308 #ifndef __linux__
309 
313  void resize(std::size_t) = delete;
314 #endif
315 
316  }; // class AnonymousMemoryMapping
317 
327  template <typename T>
329 
331 
332  public:
333 
340  explicit TypedMemoryMapping(std::size_t size) :
341  m_mapping(sizeof(T) * size, MemoryMapping::mapping_mode::write_private) {
342  }
343 
354  TypedMemoryMapping(std::size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset = 0) :
355  m_mapping(sizeof(T) * size, mode, fd, sizeof(T) * offset) {
356  }
357 
363  OSMIUM_DEPRECATED TypedMemoryMapping(std::size_t size, bool writable, int fd, off_t offset = 0) :
364  m_mapping(sizeof(T) * size,
365  writable ? MemoryMapping::mapping_mode::write_shared : MemoryMapping::mapping_mode::readonly,
366  fd,
367  sizeof(T) * offset) {
368  }
369 
371  TypedMemoryMapping(const TypedMemoryMapping&) = delete;
372 
375 
380  TypedMemoryMapping(TypedMemoryMapping&& other) noexcept = default;
381 
385  TypedMemoryMapping& operator=(TypedMemoryMapping&& other) noexcept = default;
386 
391  ~TypedMemoryMapping() noexcept = default;
392 
399  void unmap() {
400  m_mapping.unmap();
401  }
402 
413  void resize(std::size_t new_size) {
414  m_mapping.resize(sizeof(T) * new_size);
415  }
416 
421  explicit operator bool() const noexcept {
422  return !!m_mapping;
423  }
424 
430  std::size_t size() const noexcept {
431  assert(m_mapping.size() % sizeof(T) == 0);
432  return m_mapping.size() / sizeof(T);
433  }
434 
440  int fd() const noexcept {
441  return m_mapping.fd();
442  }
443 
447  bool writable() const noexcept {
448  return m_mapping.writable();
449  }
450 
456  T* begin() noexcept {
457  return m_mapping.get_addr<T>();
458  }
459 
465  T* end() noexcept {
466  return m_mapping.get_addr<T>() + size();
467  }
468 
474  const T* cbegin() const noexcept {
475  return m_mapping.get_addr<T>();
476  }
477 
483  const T* cend() const noexcept {
484  return m_mapping.get_addr<T>() + size();
485  }
486 
492  const T* begin() const noexcept {
493  return m_mapping.get_addr<T>();
494  }
495 
501  const T* end() const noexcept {
502  return m_mapping.get_addr<T>() + size();
503  }
504 
505  }; // class TypedMemoryMapping
506 
507  template <typename T>
509 
510  public:
511 
512  explicit AnonymousTypedMemoryMapping(std::size_t size) :
514  }
515 
516 #ifndef __linux__
517 
521  void resize(std::size_t) = delete;
522 #endif
523 
524  }; // class AnonymousTypedMemoryMapping
525 
526  } // namespace util
527 
528 } // namespace osmium
529 
530 #ifndef _WIN32
531 
532 // =========== Unix implementation =============
533 
534 // MAP_FAILED is often a macro containing an old style cast
535 #pragma GCC diagnostic push
536 #pragma GCC diagnostic ignored "-Wold-style-cast"
537 
538 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
539  return m_addr != MAP_FAILED; // NOLINT(cppcoreguidelines-pro-type-cstyle-cast)
540 }
541 
543  m_addr = MAP_FAILED; // NOLINT(cppcoreguidelines-pro-type-cstyle-cast)
544 }
545 
546 #pragma GCC diagnostic pop
547 
548 // for BSD systems
549 #ifndef MAP_ANONYMOUS
550 # define MAP_ANONYMOUS MAP_ANON
551 #endif
552 
553 inline int osmium::util::MemoryMapping::get_protection() const noexcept {
554  if (m_mapping_mode == mapping_mode::readonly) {
555  return PROT_READ;
556  }
557  return PROT_READ | PROT_WRITE; // NOLINT(hicpp-signed-bitwise)
558 }
559 
560 inline int osmium::util::MemoryMapping::get_flags() const noexcept {
561  if (m_fd == -1) {
562  return MAP_PRIVATE | MAP_ANONYMOUS; // NOLINT(hicpp-signed-bitwise)
563  }
564  if (m_mapping_mode == mapping_mode::write_shared) {
565  return MAP_SHARED;
566  }
567  return MAP_PRIVATE;
568 }
569 
570 inline osmium::util::MemoryMapping::MemoryMapping(std::size_t size, mapping_mode mode, int fd, off_t offset) :
571  m_size(check_size(size)),
572  m_offset(offset),
573  m_fd(resize_fd(fd)),
574  m_mapping_mode(mode),
575  m_addr(::mmap(nullptr, m_size, get_protection(), get_flags(), m_fd, m_offset)) {
576  assert(!(fd == -1 && mode == mapping_mode::readonly));
577  if (!is_valid()) {
578  throw std::system_error{errno, std::system_category(), "mmap failed"};
579  }
580 }
581 
583  m_size(other.m_size),
584  m_offset(other.m_offset),
585  m_fd(other.m_fd),
586  m_mapping_mode(other.m_mapping_mode),
587  m_addr(other.m_addr) {
588  other.make_invalid();
589 }
590 
592  try {
593  unmap();
594  } catch (const std::system_error&) {
595  // Ignore unmap error. It should never happen anyway and we can't do
596  // anything about it here.
597  }
598  m_size = other.m_size;
599  m_offset = other.m_offset;
600  m_fd = other.m_fd;
601  m_mapping_mode = other.m_mapping_mode;
602  m_addr = other.m_addr;
603  other.make_invalid();
604  return *this;
605 }
606 
608  if (is_valid()) {
609  if (::munmap(m_addr, m_size) != 0) {
610  throw std::system_error{errno, std::system_category(), "munmap failed"};
611  }
612  make_invalid();
613  }
614 }
615 
616 inline void osmium::util::MemoryMapping::resize(std::size_t new_size) {
617  assert(new_size > 0 && "can not resize to zero size");
618  if (m_fd == -1) { // anonymous mapping
619 #ifdef __linux__
620  m_addr = ::mremap(m_addr, m_size, new_size, MREMAP_MAYMOVE);
621  if (!is_valid()) {
622  throw std::system_error{errno, std::system_category(), "mremap failed"};
623  }
624  m_size = new_size;
625 #else
626  assert(false && "can't resize anonymous mappings on non-linux systems");
627 #endif
628  } else { // file-based mapping
629  unmap();
630  m_size = new_size;
631  resize_fd(m_fd);
632  m_addr = ::mmap(nullptr, new_size, get_protection(), get_flags(), m_fd, m_offset);
633  if (!is_valid()) {
634  throw std::system_error{errno, std::system_category(), "mmap (remap) failed"};
635  }
636  }
637 }
638 
639 #else
640 
641 // =========== Windows implementation =============
642 
643 /* References:
644  * CreateFileMapping: https://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx
645  * CloseHandle: https://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx
646  * MapViewOfFile: https://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx
647  * UnmapViewOfFile: https://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx
648  */
649 
650 namespace osmium {
651 
652  inline namespace util {
653 
654  inline DWORD dword_hi(uint64_t x) {
655  return static_cast<DWORD>(x >> 32);
656  }
657 
658  inline DWORD dword_lo(uint64_t x) {
659  return static_cast<DWORD>(x & 0xffffffff);
660  }
661 
662  } // namespace util
663 
664 } // namespace osmium
665 
666 inline DWORD osmium::util::MemoryMapping::get_protection() const noexcept {
667  switch (m_mapping_mode) {
668  case mapping_mode::readonly:
669  return PAGE_READONLY;
670  case mapping_mode::write_private:
671  return PAGE_WRITECOPY;
672  default: // mapping_mode::write_shared
673  break;
674  }
675  return PAGE_READWRITE;
676 }
677 
678 inline DWORD osmium::util::MemoryMapping::get_flags() const noexcept {
679  switch (m_mapping_mode) {
680  case mapping_mode::readonly:
681  return FILE_MAP_READ;
682  case mapping_mode::write_private:
683  return FILE_MAP_COPY;
684  default: // mapping_mode::write_shared
685  break;
686  }
687  return FILE_MAP_WRITE;
688 }
689 
690 inline HANDLE osmium::util::MemoryMapping::get_handle() const noexcept {
691  if (m_fd == -1) {
692  return INVALID_HANDLE_VALUE;
693  }
694  return reinterpret_cast<HANDLE>(_get_osfhandle(m_fd));
695 }
696 
697 inline HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept {
698  if (m_fd != -1) {
699  _setmode(m_fd, _O_BINARY);
700  }
701  return CreateFileMapping(get_handle(),
702  nullptr,
703  get_protection(),
704  osmium::dword_hi(static_cast<uint64_t>(m_size) + m_offset),
705  osmium::dword_lo(static_cast<uint64_t>(m_size) + m_offset),
706  nullptr);
707 }
708 
709 inline void* osmium::util::MemoryMapping::map_view_of_file() const noexcept {
710  return MapViewOfFile(m_handle,
711  get_flags(),
712  osmium::dword_hi(m_offset),
713  osmium::dword_lo(m_offset),
714  m_size);
715 }
716 
717 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
718  return m_addr != nullptr;
719 }
720 
721 inline void osmium::util::MemoryMapping::make_invalid() noexcept {
722  m_addr = nullptr;
723 }
724 
725 // GetLastError() returns a DWORD (A 32-bit unsigned integer), but the error
726 // code for std::system_error is an int. So we convert this here and hope
727 // it all works.
728 inline int last_error() noexcept {
729  return static_cast<int>(GetLastError());
730 }
731 
732 inline osmium::util::MemoryMapping::MemoryMapping(std::size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset) :
733  m_size(check_size(size)),
734  m_offset(offset),
735  m_fd(resize_fd(fd)),
736  m_mapping_mode(mode),
737  m_handle(create_file_mapping()),
738  m_addr(nullptr) {
739 
740  if (!m_handle) {
741  throw std::system_error{last_error(), std::system_category(), "CreateFileMapping failed"};
742  }
743 
744  m_addr = map_view_of_file();
745  if (!is_valid()) {
746  throw std::system_error{last_error(), std::system_category(), "MapViewOfFile failed"};
747  }
748 }
749 
750 inline osmium::util::MemoryMapping::MemoryMapping(MemoryMapping&& other) noexcept :
751  m_size(other.m_size),
752  m_offset(other.m_offset),
753  m_fd(other.m_fd),
754  m_mapping_mode(other.m_mapping_mode),
755  m_handle(std::move(other.m_handle)),
756  m_addr(other.m_addr) {
757  other.make_invalid();
758  other.m_handle = nullptr;
759 }
760 
762  unmap();
763  m_size = other.m_size;
764  m_offset = other.m_offset;
765  m_fd = other.m_fd;
766  m_mapping_mode = other.m_mapping_mode;
767  m_handle = std::move(other.m_handle);
768  m_addr = other.m_addr;
769  other.make_invalid();
770  other.m_handle = nullptr;
771  return *this;
772 }
773 
775  if (is_valid()) {
776  if (!UnmapViewOfFile(m_addr)) {
777  throw std::system_error{last_error(), std::system_category(), "UnmapViewOfFile failed"};
778  }
779  make_invalid();
780  }
781 
782  if (m_handle) {
783  if (!CloseHandle(m_handle)) {
784  throw std::system_error{last_error(), std::system_category(), "CloseHandle failed"};
785  }
786  m_handle = nullptr;
787  }
788 }
789 
790 inline void osmium::util::MemoryMapping::resize(std::size_t new_size) {
791  unmap();
792 
793  m_size = new_size;
794  resize_fd(m_fd);
795 
796  m_handle = create_file_mapping();
797  if (!m_handle) {
798  throw std::system_error{last_error(), std::system_category(), "CreateFileMapping failed"};
799  }
800 
801  m_addr = map_view_of_file();
802  if (!is_valid()) {
803  throw std::system_error{last_error(), std::system_category(), "MapViewOfFile failed"};
804  }
805 }
806 
807 #endif
808 
809 #endif // OSMIUM_UTIL_MEMORY_MAPPING_HPP
const T * begin() const noexcept
Definition: memory_mapping.hpp:492
~MemoryMapping() noexcept
Definition: memory_mapping.hpp:216
bool is_valid() const noexcept
Definition: memory_mapping.hpp:538
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:51
OSMIUM_DEPRECATED TypedMemoryMapping(std::size_t size, bool writable, int fd, off_t offset=0)
Definition: memory_mapping.hpp:363
flag_type get_protection() const noexcept
Definition: memory_mapping.hpp:553
MemoryMapping m_mapping
Definition: memory_mapping.hpp:330
int flag_type
Definition: memory_mapping.hpp:133
~TypedMemoryMapping() noexcept=default
OSMIUM_DEPRECATED MemoryMapping(std::size_t size, bool writable=true, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:191
std::size_t file_size(int fd)
Definition: file.hpp:109
const T * cend() const noexcept
Definition: memory_mapping.hpp:483
flag_type get_flags() const noexcept
Definition: memory_mapping.hpp:560
T * get_addr() const noexcept
Definition: memory_mapping.hpp:284
int fd() const noexcept
Definition: memory_mapping.hpp:267
void unmap()
Definition: memory_mapping.hpp:607
Definition: memory_mapping.hpp:95
Definition: location.hpp:550
mapping_mode
Definition: memory_mapping.hpp:99
int resize_fd(int fd)
Definition: memory_mapping.hpp:153
std::size_t size() const noexcept
Definition: memory_mapping.hpp:430
void * m_addr
The address where the memory is mapped.
Definition: memory_mapping.hpp:124
off_t m_offset
Offset into the file.
Definition: memory_mapping.hpp:111
#define MAP_ANONYMOUS
Definition: memory_mapping.hpp:550
int m_fd
File handle we got the mapping from.
Definition: memory_mapping.hpp:114
mapping_mode m_mapping_mode
Mapping mode.
Definition: memory_mapping.hpp:117
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
const T * cbegin() const noexcept
Definition: memory_mapping.hpp:474
Definition: memory_mapping.hpp:508
std::size_t get_pagesize()
Definition: file.hpp:193
T * begin() noexcept
Definition: memory_mapping.hpp:456
Definition: memory_mapping.hpp:300
TypedMemoryMapping(std::size_t size)
Definition: memory_mapping.hpp:340
void make_invalid() noexcept
Definition: memory_mapping.hpp:542
std::size_t m_size
The size of the mapping.
Definition: memory_mapping.hpp:108
int fd() const noexcept
Definition: memory_mapping.hpp:440
MemoryMapping & operator=(const MemoryMapping &)=delete
You can not copy a MemoryMapping.
Definition: memory_mapping.hpp:328
TypedMemoryMapping & operator=(const TypedMemoryMapping &)=delete
You can not copy a TypedMemoryMapping.
void unmap()
Definition: memory_mapping.hpp:399
MemoryMapping(std::size_t size, mapping_mode mode, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:570
bool writable() const noexcept
Definition: memory_mapping.hpp:274
AnonymousTypedMemoryMapping(std::size_t size)
Definition: memory_mapping.hpp:512
AnonymousMemoryMapping(std::size_t size)
Definition: memory_mapping.hpp:304
void resize_file(int fd, std::size_t new_size)
Definition: file.hpp:177
T * end() noexcept
Definition: memory_mapping.hpp:465
std::size_t size() const noexcept
Definition: memory_mapping.hpp:258
const T * end() const noexcept
Definition: memory_mapping.hpp:501
void resize(std::size_t new_size)
Definition: memory_mapping.hpp:616
static std::size_t check_size(std::size_t size)
Definition: memory_mapping.hpp:140
TypedMemoryMapping(std::size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset=0)
Definition: memory_mapping.hpp:354
void resize(std::size_t new_size)
Definition: memory_mapping.hpp:413
bool writable() const noexcept
Definition: memory_mapping.hpp:447