xrootd
XrdClZipArchive.hh
Go to the documentation of this file.
1 //------------------------------------------------------------------------------
2 // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN)
3 // Author: Michal Simon <michal.simon@cern.ch>
4 //------------------------------------------------------------------------------
5 // This file is part of the XRootD software suite.
6 //
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //
20 // In applying this licence, CERN does not waive the privileges and immunities
21 // granted to it by virtue of its status as an Intergovernmental Organization
22 // or submit itself to any jurisdiction.
23 //------------------------------------------------------------------------------
24 
25 #ifndef SRC_XRDZIP_XRDZIPARCHIVE_HH_
26 #define SRC_XRDZIP_XRDZIPARCHIVE_HH_
27 
28 #include "XrdCl/XrdClFile.hh"
30 #include "XrdCl/XrdClJobManager.hh"
31 #include "XrdCl/XrdClDefaultEnv.hh"
32 #include "XrdCl/XrdClPostMaster.hh"
33 #include "XrdZip/XrdZipEOCD.hh"
34 #include "XrdZip/XrdZipCDFH.hh"
36 #include "XrdZip/XrdZipLFH.hh"
37 #include "XrdCl/XrdClZipCache.hh"
38 
39 #include <memory>
40 #include <unordered_map>
41 
42 //-----------------------------------------------------------------------------
43 // Forward declaration needed for friendship
44 //-----------------------------------------------------------------------------
45 namespace XrdEc{ class StrmWriter; class Reader; template<bool> class OpenOnlyImpl; };
46 class MicroTest;
47 
48 namespace XrdCl
49 {
50 
51  using namespace XrdZip;
52 
53  //---------------------------------------------------------------------------
54  // ZipArchive provides following functionalities:
55  // - parsing of existing ZIP archive
56  // - reading data from existing ZIP archive
57  // - appending data to existing ZIP archive
58  // - querying stat info and checksum for given file in ZIP archive
59  //---------------------------------------------------------------------------
60  class ZipArchive
61  {
62  friend class XrdEc::StrmWriter;
63  friend class XrdEc::Reader;
64  template<bool>
65  friend class XrdEc::OpenOnlyImpl;
66  friend class ::MicroTest;
67 
68  public:
69  //-----------------------------------------------------------------------
71  //-----------------------------------------------------------------------
73 
74  //-----------------------------------------------------------------------
76  //-----------------------------------------------------------------------
77  virtual ~ZipArchive();
78 
79  //-----------------------------------------------------------------------
87  //-----------------------------------------------------------------------
88  XRootDStatus OpenArchive( const std::string &url,
89  OpenFlags::Flags flags,
90  ResponseHandler *handler,
91  uint16_t timeout = 0 );
92 
93  //-----------------------------------------------------------------------
101  //-----------------------------------------------------------------------
102  XRootDStatus OpenFile( const std::string &fn,
104  uint64_t size = 0,
105  uint32_t crc32 = 0 );
106 
107  //-----------------------------------------------------------------------
116  //-----------------------------------------------------------------------
117  inline
118  XRootDStatus Read( uint64_t offset,
119  uint32_t size,
120  void *buffer,
121  ResponseHandler *handler,
122  uint16_t timeout = 0 )
123  {
124  if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp );
125  return ReadFrom( openfn, offset, size, buffer, handler, timeout );
126  }
127 
128  //-----------------------------------------------------------------------
138  //-----------------------------------------------------------------------
139  XRootDStatus ReadFrom( const std::string &fn,
140  uint64_t offset,
141  uint32_t size,
142  void *buffer,
143  ResponseHandler *handler,
144  uint16_t timeout = 0 );
145 
146  //-----------------------------------------------------------------------
154  //-----------------------------------------------------------------------
155  inline XRootDStatus Write( uint32_t size,
156  const void *buffer,
157  ResponseHandler *handler,
158  uint16_t timeout = 0 )
159  {
160  if( openstage != Done || openfn.empty() )
162  errInvalidOp, "Archive not opened." );
163 
164  return WriteImpl( size, buffer, handler, timeout );
165  }
166 
167  //-----------------------------------------------------------------------
177  //-----------------------------------------------------------------------
178  XRootDStatus AppendFile( const std::string &fn,
179  uint32_t crc32,
180  uint32_t size,
181  const void *buffer,
182  ResponseHandler *handler,
183  uint16_t timeout = 0 );
184 
185  //-----------------------------------------------------------------------
191  //-----------------------------------------------------------------------
192  inline XRootDStatus Stat( const std::string &fn, StatInfo *&info )
193  { // make sure archive has been opened and CD has been parsed
194  if( openstage != Done )
195  return XRootDStatus( stError, errInvalidOp );
196  // make sure the file is part of the archive
197  auto cditr = cdmap.find( fn );
198  if( cditr == cdmap.end() )
199  return XRootDStatus( stError, errNotFound );
200  // create the result
201  info = make_stat( fn );
202  return XRootDStatus();
203  }
204 
205  //-----------------------------------------------------------------------
210  //-----------------------------------------------------------------------
211  inline XRootDStatus Stat( StatInfo *&info )
212  {
213  if( openfn.empty() )
214  return XRootDStatus( stError, errInvalidOp );
215  return Stat( openfn, info );
216  }
217 
218  //-----------------------------------------------------------------------
224  //-----------------------------------------------------------------------
225  inline XRootDStatus GetCRC32( const std::string &fn, uint32_t &cksum )
226  { // make sure archive has been opened and CD has been parsed
227  if( openstage != Done )
228  return XRootDStatus( stError, errInvalidOp );
229  // make sure the file is part of the archive
230  auto cditr = cdmap.find( fn );
231  if( cditr == cdmap.end() )
232  return XRootDStatus( stError, errNotFound );
233  cksum = cdvec[cditr->second]->ZCRC32;
234  return XRootDStatus();
235  }
236 
237  //-----------------------------------------------------------------------
239  //
243  //-----------------------------------------------------------------------
245  uint16_t timeout = 0 );
246 
247  //-----------------------------------------------------------------------
250  //-----------------------------------------------------------------------
252  {
253  if( openstage != Done || openfn.empty() )
255  errInvalidOp, "Archive not opened." );
256  openfn.clear();
257  lfh.reset();
258  return XRootDStatus();
259  }
260 
261  //-----------------------------------------------------------------------
264  //-----------------------------------------------------------------------
266 
267  //-----------------------------------------------------------------------
269  //-----------------------------------------------------------------------
270  inline bool IsOpen()
271  {
272  return openstage == Done;
273  }
274 
275  //-----------------------------------------------------------------------
277  //-----------------------------------------------------------------------
278  inline bool SetProperty( const std::string &name, const std::string &value )
279  {
280  return archive.SetProperty( name, value );
281  }
282 
283  private:
284 
285  //-----------------------------------------------------------------------
294  //-----------------------------------------------------------------------
295  XRootDStatus WriteImpl( uint32_t size,
296  const void *buffer,
297  ResponseHandler *handler,
298  uint16_t timeout );
299 
300  //-----------------------------------------------------------------------
308  //-----------------------------------------------------------------------
309  XRootDStatus OpenOnly( const std::string &url,
310  ResponseHandler *handler,
311  uint16_t timeout = 0 );
312 
313  //-----------------------------------------------------------------------
317  //-----------------------------------------------------------------------
319 
320  //-----------------------------------------------------------------------
324  //-----------------------------------------------------------------------
325  void SetCD( const buffer_t &buffer );
326 
327  //-----------------------------------------------------------------------
332  //-----------------------------------------------------------------------
333  template<typename Response>
334  inline static AnyObject* PkgRsp( Response *rsp )
335  {
336  if( !rsp ) return nullptr;
337  AnyObject *pkg = new AnyObject();
338  pkg->Set( rsp );
339  return pkg;
340  }
341 
342  //-----------------------------------------------------------------------
344  //-----------------------------------------------------------------------
345  template<typename Response>
346  inline static void Free( XRootDStatus *st, Response *rsp )
347  {
348  delete st;
349  delete rsp;
350  }
351 
352  //-----------------------------------------------------------------------
359  //-----------------------------------------------------------------------
360  template<typename Response>
361  inline static void Schedule( ResponseHandler *handler, XRootDStatus *st, Response *rsp = nullptr )
362  {
363  if( !handler ) return Free( st, rsp );
365  if( jobMgr->IsWorker() )
366  // this is a worker thread so we can simply call the handler
367  handler->HandleResponse( st, PkgRsp( rsp ) );
368  else
369  {
370  ResponseJob *job = new ResponseJob( handler, st, PkgRsp( rsp ), 0 );
372  }
373  }
374 
375  //-----------------------------------------------------------------------
381  //-----------------------------------------------------------------------
382  inline static StatInfo* make_stat( const StatInfo &starch, uint64_t size )
383  {
384  StatInfo *info = new StatInfo( starch );
385  uint32_t flags = info->GetFlags();
386  info->SetFlags( flags & ( ~StatInfo::IsWritable ) ); // make sure it is not listed as writable
387  info->SetSize( size );
388  return info;
389  }
390 
391  //-----------------------------------------------------------------------
396  //-----------------------------------------------------------------------
397  inline StatInfo* make_stat( const std::string &fn )
398  {
399  StatInfo *infoptr = 0;
400  XRootDStatus st = archive.Stat( false, infoptr );
401  std::unique_ptr<StatInfo> stinfo( infoptr );
402  auto itr = cdmap.find( fn );
403  if( itr == cdmap.end() ) return nullptr;
404  size_t index = itr->second;
405  return make_stat( *stinfo, cdvec[index]->uncompressedSize );
406  }
407 
408  //-----------------------------------------------------------------------
410  //-----------------------------------------------------------------------
411  inline static XRootDStatus* make_status( const XRootDStatus &status = XRootDStatus() )
412  {
413  return new XRootDStatus( status );
414  }
415 
416  //-----------------------------------------------------------------------
418  //-----------------------------------------------------------------------
419  inline void Clear()
420  {
421  buffer.reset();
422  eocd.reset();
423  cdvec.clear();
424  cdmap.clear();
425  zip64eocd.reset();
426  openstage = None;
427  }
428 
429  //-----------------------------------------------------------------------
431  //-----------------------------------------------------------------------
433  {
434  None = 0, //< opening/parsing not started
435  HaveEocdBlk, //< we have the End of Central Directory record
436  HaveZip64EocdlBlk, //< we have the ZIP64 End of Central Directory locator record
437  HaveZip64EocdBlk, //< we have the ZIP64 End of Central Directory record
438  HaveCdRecords, //< we have Central Directory records
439  Done, //< we are done parsing the Central Directory
440  Error, //< opening/parsing failed
441  NotParsed //< the ZIP archive has been opened but Central Directory is not parsed
442  };
443 
444  //-----------------------------------------------------------------------
446  //-----------------------------------------------------------------------
447  typedef std::unordered_map<std::string, ZipCache> zipcache_t;
448 
449  File archive; //> File object for handling the ZIP archive
450  uint64_t archsize; //> size of the ZIP archive
451  bool cdexists; //> true if Central Directory exists, false otherwise
452  bool updated; //> true if the ZIP archive has been updated, false otherwise
453  std::unique_ptr<char[]> buffer; //> buffer for keeping the data to be parsed or raw data
454  std::unique_ptr<EOCD> eocd; //> End of Central Directory record
455  cdvec_t cdvec; //> vector of Central Directory File Headers
456  cdmap_t cdmap; //> mapping of file name to CDFH index
457  uint64_t cdoff; //> Central Directory offset
458  uint32_t orgcdsz; //> original CD size
459  uint32_t orgcdcnt; //> original number CDFH records
460  buffer_t orgcdbuf; //> buffer with the original CDFH records
461  std::unique_ptr<ZIP64_EOCD> zip64eocd; //> ZIP64 End of Central Directory record
462  OpenStages openstage; //> stage of opening / parsing a ZIP archive
463  std::string openfn; //> file name of opened file
464  zipcache_t zipcache; //> cache for inflating compressed data
465  std::unique_ptr<LFH> lfh; //> Local File Header record for the newly appended file
466  };
467 
468 } /* namespace XrdZip */
469 
470 #endif /* SRC_XRDZIP_XRDZIPARCHIVE_HH_ */
Definition: XrdClAnyObject.hh:33
void Set(Type object, bool own=true)
Definition: XrdClAnyObject.hh:59
static PostMaster * GetPostMaster()
Get default post master.
Directory list.
Definition: XrdClXRootDResponses.hh:646
A file.
Definition: XrdClFile.hh:46
A synchronized queue.
Definition: XrdClJobManager.hh:51
void QueueJob(Job *job, void *arg=0)
Add a job to be run.
Definition: XrdClJobManager.hh:92
bool IsWorker()
Definition: XrdClJobManager.hh:102
JobManager * GetJobManager()
Get the job manager object user by the post master.
Handle an async response.
Definition: XrdClXRootDResponses.hh:1037
virtual void HandleResponse(XRootDStatus *status, AnyObject *response)
Definition: XrdClXRootDResponses.hh:1066
Call the user callback.
Definition: XrdClResponseJob.hh:31
Object stat info.
Definition: XrdClXRootDResponses.hh:396
@ IsWritable
Write access is allowed.
Definition: XrdClXRootDResponses.hh:410
uint32_t GetFlags() const
Get flags.
void SetSize(uint64_t size)
Set size.
void SetFlags(uint32_t flags)
Set flags.
Write operation (.
Definition: XrdClFileOperations.hh:369
Request status.
Definition: XrdClXRootDResponses.hh:215
Definition: XrdClZipArchive.hh:61
OpenStages
Stages of opening and parsing a ZIP archive.
Definition: XrdClZipArchive.hh:433
@ HaveCdRecords
Definition: XrdClZipArchive.hh:438
@ HaveZip64EocdlBlk
Definition: XrdClZipArchive.hh:436
@ Done
Definition: XrdClZipArchive.hh:439
@ HaveEocdBlk
Definition: XrdClZipArchive.hh:435
@ HaveZip64EocdBlk
Definition: XrdClZipArchive.hh:437
@ Error
Definition: XrdClZipArchive.hh:440
XRootDStatus ReadFrom(const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
buffer_t orgcdbuf
Definition: XrdClZipArchive.hh:460
bool cdexists
Definition: XrdClZipArchive.hh:451
static XRootDStatus * make_status(const XRootDStatus &status=XRootDStatus())
Allocate new XRootDStatus object.
Definition: XrdClZipArchive.hh:411
XRootDStatus Stat(const std::string &fn, StatInfo *&info)
Definition: XrdClZipArchive.hh:192
XRootDStatus Write(uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout=0)
Definition: XrdClZipArchive.hh:155
uint64_t archsize
Definition: XrdClZipArchive.hh:450
XRootDStatus WriteImpl(uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout)
static void Schedule(ResponseHandler *handler, XRootDStatus *st, Response *rsp=nullptr)
Definition: XrdClZipArchive.hh:361
std::unique_ptr< char[]> buffer
Definition: XrdClZipArchive.hh:453
static AnyObject * PkgRsp(Response *rsp)
Definition: XrdClZipArchive.hh:334
XRootDStatus OpenArchive(const std::string &url, OpenFlags::Flags flags, ResponseHandler *handler, uint16_t timeout=0)
XRootDStatus List(DirectoryList *&list)
XRootDStatus AppendFile(const std::string &fn, uint32_t crc32, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout=0)
bool SetProperty(const std::string &name, const std::string &value)
Set property on the underlying File object.
Definition: XrdClZipArchive.hh:278
std::unique_ptr< LFH > lfh
Definition: XrdClZipArchive.hh:465
XRootDStatus Stat(StatInfo *&info)
Definition: XrdClZipArchive.hh:211
void Clear()
Clear internal ZipArchive objects.
Definition: XrdClZipArchive.hh:419
cdvec_t cdvec
Definition: XrdClZipArchive.hh:455
uint32_t orgcdsz
Definition: XrdClZipArchive.hh:458
StatInfo * make_stat(const std::string &fn)
Definition: XrdClZipArchive.hh:397
OpenStages openstage
Definition: XrdClZipArchive.hh:462
zipcache_t zipcache
Definition: XrdClZipArchive.hh:464
XRootDStatus CloseArchive(ResponseHandler *handler, uint16_t timeout=0)
Create the central directory at the end of ZIP archive and close it.
std::unique_ptr< EOCD > eocd
Definition: XrdClZipArchive.hh:454
bool IsOpen()
Definition: XrdClZipArchive.hh:270
static StatInfo * make_stat(const StatInfo &starch, uint64_t size)
Definition: XrdClZipArchive.hh:382
virtual ~ZipArchive()
Destructor.
XRootDStatus CloseFile()
Definition: XrdClZipArchive.hh:251
File archive
Definition: XrdClZipArchive.hh:449
uint32_t orgcdcnt
Definition: XrdClZipArchive.hh:459
std::unique_ptr< ZIP64_EOCD > zip64eocd
Definition: XrdClZipArchive.hh:461
std::unordered_map< std::string, ZipCache > zipcache_t
Type that maps file name to its cache.
Definition: XrdClZipArchive.hh:447
bool updated
Definition: XrdClZipArchive.hh:452
XRootDStatus GetCRC32(const std::string &fn, uint32_t &cksum)
Definition: XrdClZipArchive.hh:225
buffer_t GetCD()
cdmap_t cdmap
Definition: XrdClZipArchive.hh:456
XRootDStatus OpenFile(const std::string &fn, OpenFlags::Flags flags=OpenFlags::None, uint64_t size=0, uint32_t crc32=0)
XRootDStatus Read(uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
Definition: XrdClZipArchive.hh:118
ZipArchive()
Constructor.
XRootDStatus OpenOnly(const std::string &url, ResponseHandler *handler, uint16_t timeout=0)
std::string openfn
Definition: XrdClZipArchive.hh:463
uint64_t cdoff
Definition: XrdClZipArchive.hh:457
static void Free(XRootDStatus *st, Response *rsp)
Free status and response.
Definition: XrdClZipArchive.hh:346
void SetCD(const buffer_t &buffer)
Definition: XrdClZipArchive.hh:45
Definition: XrdEcReader.hh:58
Definition: XrdEcStrmWriter.hh:53
Definition: XrdClAnyObject.hh:26
StatImpl< false > Stat(Ctx< File > file, Arg< bool > force, uint16_t timeout=0)
Definition: XrdClFileOperations.hh:358
const uint16_t stError
An error occurred that could potentially be retried.
Definition: XrdClStatus.hh:32
const uint16_t errNotFound
Definition: XrdClStatus.hh:98
ZipReadFromImpl< false > ReadFrom(Ctx< ZipArchive > zip, Arg< std::string > fn, Arg< uint64_t > offset, Arg< uint32_t > size, Arg< void * > buffer, uint16_t timeout=0)
Factory for creating ArchiveReadImpl objects.
Definition: XrdClZipOperations.hh:302
const uint16_t errInvalidOp
Definition: XrdClStatus.hh:50
Definition: XrdClZipArchive.hh:45
Definition: XrdZipCDFH.hh:39
std::vector< std::unique_ptr< CDFH > > cdvec_t
Definition: XrdZipCDFH.hh:43
std::vector< char > buffer_t
Definition: XrdZipUtils.hh:54
std::unordered_map< std::string, size_t > cdmap_t
Definition: XrdZipCDFH.hh:53
none object for initializing empty Optional
Definition: XrdClOptional.hh:35
Flags
Open flags, may be or'd when appropriate.
Definition: XrdClFileSystem.hh:76
@ None
Nothing.
Definition: XrdClFileSystem.hh:77