Libosmium  2.15.2
Fast and flexible C++ library for working with OpenStreetMap data
gzip_compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_GZIP_COMPRESSION_HPP
2 #define OSMIUM_IO_GZIP_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2019 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
46 #include <osmium/io/detail/read_write.hpp>
47 #include <osmium/io/error.hpp>
50 
51 #include <zlib.h>
52 
53 #include <cassert>
54 #include <cerrno>
55 #include <cstddef>
56 #include <limits>
57 #include <string>
58 
59 #ifndef _MSC_VER
60 # include <unistd.h>
61 #endif
62 
63 namespace osmium {
64 
69  struct gzip_error : public io_error {
70 
71  int gzip_error_code = 0;
72  int system_errno = 0;
73 
74  explicit gzip_error(const std::string& what) :
75  io_error(what) {
76  }
77 
78  gzip_error(const std::string& what, const int error_code) :
79  io_error(what),
80  gzip_error_code(error_code) {
81  if (error_code == Z_ERRNO) {
82  system_errno = errno;
83  }
84  }
85 
86  }; // struct gzip_error
87 
88  namespace io {
89 
90  namespace detail {
91 
92  [[noreturn]] inline void throw_gzip_error(gzFile gzfile, const char* msg) {
93  std::string error{"gzip error: "};
94  error += msg;
95  error += ": ";
96  int error_code = 0;
97  if (gzfile) {
98  error += ::gzerror(gzfile, &error_code);
99  }
100  throw osmium::gzip_error{error, error_code};
101  }
102 
103  } // namespace detail
104 
105  class GzipCompressor : public Compressor {
106 
107  int m_fd;
108  gzFile m_gzfile;
109 
110  public:
111 
112  explicit GzipCompressor(const int fd, const fsync sync) :
113  Compressor(sync),
114  m_fd(osmium::io::detail::reliable_dup(fd)) {
115 #ifdef _MSC_VER
116  osmium::detail::disable_invalid_parameter_handler diph;
117 #endif
118  m_gzfile = ::gzdopen(fd, "wb");
119  if (!m_gzfile) {
120  throw gzip_error{"gzip error: write initialization failed"};
121  }
122  }
123 
124  GzipCompressor(const GzipCompressor&) = delete;
125  GzipCompressor& operator=(const GzipCompressor&) = delete;
126 
127  GzipCompressor(GzipCompressor&&) = delete;
129 
130  ~GzipCompressor() noexcept final {
131  try {
132  close();
133  } catch (...) {
134  // Ignore any exceptions because destructor must not throw.
135  }
136  }
137 
138  void write(const std::string& data) final {
139 #ifdef _MSC_VER
140  osmium::detail::disable_invalid_parameter_handler diph;
141 #endif
142  assert(m_gzfile);
143  assert(data.size() < std::numeric_limits<unsigned int>::max());
144  if (!data.empty()) {
145  const int nwrite = ::gzwrite(m_gzfile, data.data(), static_cast<unsigned int>(data.size()));
146  if (nwrite == 0) {
147  detail::throw_gzip_error(m_gzfile, "write failed");
148  }
149  }
150  }
151 
152  void close() final {
153  if (m_gzfile) {
154 #ifdef _MSC_VER
155  osmium::detail::disable_invalid_parameter_handler diph;
156 #endif
157  const int result = ::gzclose_w(m_gzfile);
158  m_gzfile = nullptr;
159  if (result != Z_OK) {
160  throw gzip_error{"gzip error: write close failed", result};
161  }
162  if (do_fsync()) {
163  osmium::io::detail::reliable_fsync(m_fd);
164  }
165  osmium::io::detail::reliable_close(m_fd);
166  }
167  }
168 
169  }; // class GzipCompressor
170 
172 
173  gzFile m_gzfile = nullptr;
174 
175  public:
176 
177  explicit GzipDecompressor(const int fd) {
178 #ifdef _MSC_VER
179  osmium::detail::disable_invalid_parameter_handler diph;
180 #endif
181  m_gzfile = ::gzdopen(fd, "rb");
182  if (!m_gzfile) {
183  try {
184  osmium::io::detail::reliable_close(fd);
185  } catch (...) {
186  }
187  throw gzip_error{"gzip error: read initialization failed"};
188  }
189  }
190 
191  GzipDecompressor(const GzipDecompressor&) = delete;
192  GzipDecompressor& operator=(const GzipDecompressor&) = delete;
193 
196 
197  ~GzipDecompressor() noexcept final {
198  try {
199  close();
200  } catch (...) {
201  // Ignore any exceptions because destructor must not throw.
202  }
203  }
204 
205  std::string read() final {
206 #ifdef _MSC_VER
207  osmium::detail::disable_invalid_parameter_handler diph;
208 #endif
209  assert(m_gzfile);
210  std::string buffer(osmium::io::Decompressor::input_buffer_size, '\0');
211  assert(buffer.size() < std::numeric_limits<unsigned int>::max());
212  int nread = ::gzread(m_gzfile, &*buffer.begin(), static_cast<unsigned int>(buffer.size()));
213  if (nread < 0) {
214  detail::throw_gzip_error(m_gzfile, "read failed");
215  }
216  buffer.resize(static_cast<std::string::size_type>(nread));
217 #if ZLIB_VERNUM >= 0x1240
218  set_offset(static_cast<std::size_t>(::gzoffset(m_gzfile)));
219 #endif
220  return buffer;
221  }
222 
223  void close() final {
224  if (m_gzfile) {
225 #ifdef _MSC_VER
226  osmium::detail::disable_invalid_parameter_handler diph;
227 #endif
228  const int result = ::gzclose_r(m_gzfile);
229  m_gzfile = nullptr;
230  if (result != Z_OK) {
231  throw gzip_error{"gzip error: read close failed", result};
232  }
233  }
234  }
235 
236  }; // class GzipDecompressor
237 
239 
240  const char* m_buffer;
241  std::size_t m_buffer_size;
242  z_stream m_zstream;
243 
244  public:
245 
246  GzipBufferDecompressor(const char* buffer, const std::size_t size) :
247  m_buffer(buffer),
248  m_buffer_size(size),
249  m_zstream() {
250  m_zstream.next_in = reinterpret_cast<unsigned char*>(const_cast<char*>(buffer));
251  assert(size < std::numeric_limits<unsigned int>::max());
252  m_zstream.avail_in = static_cast<unsigned int>(size);
253  const int result = inflateInit2(&m_zstream, MAX_WBITS | 32); // NOLINT(hicpp-signed-bitwise)
254  if (result != Z_OK) {
255  std::string message{"gzip error: decompression init failed: "};
256  if (m_zstream.msg) {
257  message.append(m_zstream.msg);
258  }
259  throw osmium::gzip_error{message, result};
260  }
261  }
262 
265 
268 
269  ~GzipBufferDecompressor() noexcept final {
270  try {
271  close();
272  } catch (...) {
273  // Ignore any exceptions because destructor must not throw.
274  }
275  }
276 
277  std::string read() final {
278  std::string output;
279 
280  if (m_buffer) {
281  const std::size_t buffer_size = 10240;
282  output.append(buffer_size, '\0');
283  m_zstream.next_out = reinterpret_cast<unsigned char*>(&*output.begin());
284  m_zstream.avail_out = buffer_size;
285  const int result = inflate(&m_zstream, Z_SYNC_FLUSH);
286 
287  if (result != Z_OK) {
288  m_buffer = nullptr;
289  m_buffer_size = 0;
290  }
291 
292  if (result != Z_OK && result != Z_STREAM_END) {
293  std::string message{"gzip error: inflate failed: "};
294  if (m_zstream.msg) {
295  message.append(m_zstream.msg);
296  }
297  throw osmium::gzip_error{message, result};
298  }
299 
300  output.resize(static_cast<std::size_t>(m_zstream.next_out - reinterpret_cast<const unsigned char*>(output.data())));
301  }
302 
303  return output;
304  }
305 
306  void close() final {
307  inflateEnd(&m_zstream);
308  }
309 
310  }; // class GzipBufferDecompressor
311 
312  namespace detail {
313 
314  // we want the register_compression() function to run, setting
315  // the variable is only a side-effect, it will never be used
317  [](const int fd, const fsync sync) { return new osmium::io::GzipCompressor{fd, sync}; },
318  [](const int fd) { return new osmium::io::GzipDecompressor{fd}; },
319  [](const char* buffer, const std::size_t size) { return new osmium::io::GzipBufferDecompressor{buffer, size}; }
320  );
321 
322  // dummy function to silence the unused variable warning from above
323  inline bool get_registered_gzip_compression() noexcept {
324  return registered_gzip_compression;
325  }
326 
327  } // namespace detail
328 
329  } // namespace io
330 
331 } // namespace osmium
332 
333 #endif // OSMIUM_IO_GZIP_COMPRESSION_HPP
gzFile m_gzfile
Definition: gzip_compression.hpp:108
Definition: compression.hpp:95
GzipBufferDecompressor & operator=(const GzipBufferDecompressor &)=delete
Definition: gzip_compression.hpp:238
gzFile m_gzfile
Definition: gzip_compression.hpp:173
Definition: gzip_compression.hpp:105
std::size_t m_buffer_size
Definition: gzip_compression.hpp:241
z_stream m_zstream
Definition: gzip_compression.hpp:242
gzip_error(const std::string &what, const int error_code)
Definition: gzip_compression.hpp:78
static CompressionFactory & instance()
Definition: compression.hpp:180
GzipDecompressor & operator=(const GzipDecompressor &)=delete
Definition: gzip_compression.hpp:171
bool register_compression(osmium::io::file_compression compression, const create_compressor_type &create_compressor, const create_decompressor_type_fd &create_decompressor_fd, const create_decompressor_type_buffer &create_decompressor_buffer)
Definition: compression.hpp:185
~GzipDecompressor() noexcept final
Definition: gzip_compression.hpp:197
Definition: compression.hpp:87
void set_offset(const std::size_t offset) noexcept
Definition: compression.hpp:124
int m_fd
Definition: gzip_compression.hpp:107
bool do_fsync() const noexcept
Definition: compression.hpp:63
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
GzipCompressor(const int fd, const fsync sync)
Definition: gzip_compression.hpp:112
const char * m_buffer
Definition: gzip_compression.hpp:240
Definition: attr.hpp:333
void close() final
Definition: gzip_compression.hpp:306
fsync
Definition: writer_options.hpp:51
GzipDecompressor(const int fd)
Definition: gzip_compression.hpp:177
void close() final
Definition: gzip_compression.hpp:152
Definition: error.hpp:44
int system_errno
Definition: gzip_compression.hpp:72
gzip_error(const std::string &what)
Definition: gzip_compression.hpp:74
GzipCompressor & operator=(const GzipCompressor &)=delete
Definition: gzip_compression.hpp:69
std::string read() final
Definition: gzip_compression.hpp:205
int gzip_error_code
Definition: gzip_compression.hpp:71
Definition: compression.hpp:57
void write(const std::string &data) final
Definition: gzip_compression.hpp:138
~GzipCompressor() noexcept final
Definition: gzip_compression.hpp:130
~GzipBufferDecompressor() noexcept final
Definition: gzip_compression.hpp:269
GzipBufferDecompressor(const char *buffer, const std::size_t size)
Definition: gzip_compression.hpp:246
std::string read() final
Definition: gzip_compression.hpp:277
void close() final
Definition: gzip_compression.hpp:223