xrootd
XrdFileCacheInfo.hh
Go to the documentation of this file.
1 #ifndef __XRDFILECACHE_INFO_HH__
2 #define __XRDFILECACHE_INFO_HH__
3 //----------------------------------------------------------------------------------
4 // Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University
5 // Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman
6 //----------------------------------------------------------------------------------
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //----------------------------------------------------------------------------------
20 
21 #include <stdio.h>
22 #include <time.h>
23 #include <assert.h>
24 #include <vector>
25 
26 #include "XrdSys/XrdSysPthread.hh"
27 #include "XrdCl/XrdClConstants.hh"
28 #include "XrdCl/XrdClDefaultEnv.hh"
29 
30 class XrdOssDF;
31 class XrdCksCalc;
32 class XrdSysTrace;
33 
34 
35 namespace XrdCl
36 {
37 class Log;
38 }
39 
40 namespace XrdFileCache
41 {
42 class Stats;
43 
44 //----------------------------------------------------------------------------
46 //----------------------------------------------------------------------------
47 
48 class Info
49 {
50 public:
51  // !Access statistics
52  struct AStat
53  {
54  time_t AttachTime;
55  time_t DetachTime;
56  long long BytesDisk;
57  long long BytesRam;
58  long long BytesMissed;
59 
61  };
62 
63  struct Store {
64  int m_version;
65  long long m_bufferSize;
66  long long m_fileSize;
67  unsigned char *m_buff_synced;
68  char m_cksum[16];
69  time_t m_creationTime;
70  size_t m_accessCnt;
71  std::vector<AStat> m_astats;
72 
74  };
75 
76 
77  //------------------------------------------------------------------------
79  //------------------------------------------------------------------------
80  Info(XrdSysTrace* trace, bool prefetchBuffer = false);
81 
82  //------------------------------------------------------------------------
84  //------------------------------------------------------------------------
85  ~Info();
86 
87  //---------------------------------------------------------------------
91  //---------------------------------------------------------------------
92  void SetBitWritten(int i);
93 
94  //---------------------------------------------------------------------
98  //---------------------------------------------------------------------
99  void SetBitSynced(int i);
100 
101  //---------------------------------------------------------------------
103  //---------------------------------------------------------------------
104  void SetAllBitsSynced();
105 
106  //---------------------------------------------------------------------
110  //---------------------------------------------------------------------
111  void SetBitPrefetch(int i);
112 
113  void SetBufferSize(long long);
114 
115  void SetFileSize(long long);
116 
117  //---------------------------------------------------------------------
121  //---------------------------------------------------------------------
122  void ResizeBits(int n);
123 
124  //---------------------------------------------------------------------
131  //---------------------------------------------------------------------
132  bool Read(XrdOssDF* fp, const std::string &fname = "<unknown>");
133 
134  //---------------------------------------------------------------------
137  //---------------------------------------------------------------------
138  bool Write(XrdOssDF* fp, const std::string &fname = "<unknown>");
139 
140  //---------------------------------------------------------------------
142  //---------------------------------------------------------------------
143  void DisableDownloadStatus();
144 
145  //---------------------------------------------------------------------
147  //---------------------------------------------------------------------
148  void ResetAllAccessStats();
149 
150  //---------------------------------------------------------------------
152  //---------------------------------------------------------------------
153  void WriteIOStatAttach();
154 
156  //---------------------------------------------------------------------
157  void WriteIOStat(Stats& s);
158 
159  //---------------------------------------------------------------------
161  //---------------------------------------------------------------------
162  void WriteIOStatDetach(Stats& s);
163 
164  //---------------------------------------------------------------------
166  //---------------------------------------------------------------------
167  void WriteIOStatSingle(long long bytes_disk);
168 
169  //---------------------------------------------------------------------
171  //---------------------------------------------------------------------
172  void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc);
173 
174  //---------------------------------------------------------------------
176  //---------------------------------------------------------------------
177  bool IsAnythingEmptyInRng(int firstIdx, int lastIdx) const;
178 
179  //---------------------------------------------------------------------
181  //---------------------------------------------------------------------
182  int GetSizeInBytes() const;
183 
184  //---------------------------------------------------------------------
186  //---------------------------------------------------------------------
187  int GetSizeInBits() const;
188 
189  //---------------------------------------------------------------------
191  //---------------------------------------------------------------------
192  long long GetFileSize() const;
193 
194  //---------------------------------------------------------------------
196  //---------------------------------------------------------------------
197  bool GetLatestDetachTime(time_t& t) const;
198 
199  //---------------------------------------------------------------------
201  //---------------------------------------------------------------------
202  long long GetBufferSize() const;
203 
204  //---------------------------------------------------------------------
206  //---------------------------------------------------------------------
207  bool TestBit(int i) const;
208 
209  //---------------------------------------------------------------------
211  //---------------------------------------------------------------------
212  bool TestPrefetchBit(int i) const;
213 
214  //---------------------------------------------------------------------
216  //---------------------------------------------------------------------
217  bool IsComplete() const;
218 
219  //---------------------------------------------------------------------
221  //---------------------------------------------------------------------
222  int GetNDownloadedBlocks() const;
223 
224  //---------------------------------------------------------------------
226  //---------------------------------------------------------------------
227  long long GetNDownloadedBytes() const;
228 
229  //---------------------------------------------------------------------
231  //---------------------------------------------------------------------
232  int GetLastDownloadedBlock() const;
233 
234  //---------------------------------------------------------------------
236  //---------------------------------------------------------------------
237  long long GetExpectedDataFileSize() const;
238 
239  //---------------------------------------------------------------------
241  //---------------------------------------------------------------------
243 
244  //---------------------------------------------------------------------
246  //---------------------------------------------------------------------
247  size_t GetAccessCnt() { return m_store.m_accessCnt; }
248 
249  //---------------------------------------------------------------------
251  //---------------------------------------------------------------------
252  int GetVersion() { return m_store.m_version; }
253 
254  //---------------------------------------------------------------------
256  //---------------------------------------------------------------------
257  const Store& RefStoredData() const { return m_store; }
258 
259  //---------------------------------------------------------------------
261  //---------------------------------------------------------------------
262  void GetCksum( unsigned char* buff, char* digest);
263 
264  const static char* m_infoExtension;
265  const static char* m_traceID;
266  const static int m_defaultVersion;
267  const static size_t m_maxNumAccess;
268 
269  XrdSysTrace* GetTrace() const {return m_trace; }
270 
271  static size_t GetMaxNumAccess() { return m_maxNumAccess; }
272 
273 protected:
275 
278  unsigned char *m_buff_written;
279  unsigned char *m_buff_prefetch;
280 
282  bool m_complete;
283 
284 private:
285  inline unsigned char cfiBIT(int n) const { return 1 << n; }
286 
287  // split reading for V1
288  bool ReadV1(XrdOssDF* fp, const std::string &fname);
290 };
291 
292 //------------------------------------------------------------------------------
293 
294 inline bool Info::TestBit(int i) const
295 {
296  const int cn = i/8;
297  assert(cn < GetSizeInBytes());
298 
299  const int off = i - cn*8;
300  return (m_buff_written[cn] & cfiBIT(off)) == cfiBIT(off);
301 }
302 
303 inline bool Info::TestPrefetchBit(int i) const
304 {
305  if (!m_buff_prefetch) return false;
306 
307  const int cn = i/8;
308  assert(cn < GetSizeInBytes());
309 
310  const int off = i - cn*8;
311  return (m_buff_prefetch[cn] & cfiBIT(off)) == cfiBIT(off);
312 }
313 
314 inline int Info::GetNDownloadedBlocks() const
315 {
316  int cntd = 0;
317  for (int i = 0; i < m_sizeInBits; ++i)
318  if (TestBit(i)) cntd++;
319 
320  return cntd;
321 }
322 
323 inline long long Info::GetNDownloadedBytes() const
324 {
326 }
327 
329 {
330  for (int i = m_sizeInBits - 1; i >= 0; --i)
331  if (TestBit(i)) return i;
332 
333  return -1;
334 }
335 
336 inline long long Info::GetExpectedDataFileSize() const
337 {
338  int last_block = GetLastDownloadedBlock();
339  if (last_block == m_sizeInBits - 1)
340  return m_store.m_fileSize;
341  else
342  return (last_block + 1) * m_store.m_bufferSize;
343 }
344 
345 inline int Info::GetSizeInBytes() const
346 {
347  if (m_sizeInBits)
348  return ((m_sizeInBits - 1)/8 + 1);
349  else
350  return 0;
351 }
352 
353 inline int Info::GetSizeInBits() const
354 {
355  return m_sizeInBits;
356 }
357 
358 inline long long Info::GetFileSize() const
359 {
360  return m_store.m_fileSize;
361 }
362 
363 inline bool Info::IsComplete() const
364 {
365  return m_complete;
366 }
367 
368 inline bool Info::IsAnythingEmptyInRng(int firstIdx, int lastIdx) const
369 {
370  // TODO rewrite to use full byte comparisons outside of edges ?
371  // Also, it is always called with fisrtsdx = 0, lastIdx = m_sizeInBits.
372  for (int i = firstIdx; i < lastIdx; ++i)
373  if (! TestBit(i)) return true;
374 
375  return false;
376 }
377 
379 {
381 }
382 
383 inline void Info::SetBitSynced(int i)
384 {
385  const int cn = i/8;
386  assert(cn < GetSizeInBytes());
387 
388  const int off = i - cn*8;
389  m_store.m_buff_synced[cn] |= cfiBIT(off);
390 }
391 
392 inline void Info::SetBitWritten(int i)
393 {
394  const int cn = i/8;
395  assert(cn < GetSizeInBytes());
396 
397  const int off = i - cn*8;
398  m_buff_written[cn] |= cfiBIT(off);
399 }
400 
401 inline void Info::SetBitPrefetch(int i)
402 {
403  if (!m_buff_prefetch) return;
404 
405  const int cn = i/8;
406  assert(cn < GetSizeInBytes());
407 
408  const int off = i - cn*8;
409  m_buff_prefetch[cn] |= cfiBIT(off);
410 }
411 
412 inline long long Info::GetBufferSize() const
413 {
414  return m_store.m_bufferSize;
415 }
416 
417 }
418 #endif
int m_version
info version
Definition: XrdFileCacheInfo.hh:64
bool Read(XrdOssDF *fp, const std::string &fname="<unknown>")
Rea load content from cinfo file into this object.
unsigned char * m_buff_written
download state vector
Definition: XrdFileCacheInfo.hh:278
void WriteIOStat(Stats &s)
Write bytes missed, hits, and disk.
XrdCksCalc * m_cksCalc
Definition: XrdFileCacheInfo.hh:289
void SetBitWritten(int i)
Mark block as downloaded.
Definition: XrdFileCacheInfo.hh:392
Statistics of disk cache utilisation.
Definition: XrdFileCacheStats.hh:30
bool ReadV1(XrdOssDF *fp, const std::string &fname)
bool TestPrefetchBit(int i) const
Test if block at the given index is prewritten.
Definition: XrdFileCacheInfo.hh:303
long long GetExpectedDataFileSize() const
Get expected data file size.
Definition: XrdFileCacheInfo.hh:336
time_t AttachTime
Definition: XrdFileCacheInfo.hh:54
Definition: XrdFileCache.hh:40
int GetVersion()
Get version.
Definition: XrdFileCacheInfo.hh:252
AStat()
read remote client
Definition: XrdFileCacheInfo.hh:60
static const char * m_infoExtension
Definition: XrdFileCacheInfo.hh:264
time_t m_creationTime
time the info file was created
Definition: XrdFileCacheInfo.hh:69
int GetNDownloadedBlocks() const
Get number of downloaded blocks.
Definition: XrdFileCacheInfo.hh:314
Store m_store
Definition: XrdFileCacheInfo.hh:276
unsigned char * m_buff_prefetch
prefetch statistics
Definition: XrdFileCacheInfo.hh:279
bool IsAnythingEmptyInRng(int firstIdx, int lastIdx) const
Check download status in given block range.
Definition: XrdFileCacheInfo.hh:368
int GetSizeInBits() const
Get number of blocks represented in download-state bit-vector.
Definition: XrdFileCacheInfo.hh:353
Store()
Definition: XrdFileCacheInfo.hh:73
bool m_hasPrefetchBuffer
constains current prefetch score
Definition: XrdFileCacheInfo.hh:277
static const size_t m_maxNumAccess
Definition: XrdFileCacheInfo.hh:267
bool GetLatestDetachTime(time_t &t) const
Get latest detach time.
time_t DetachTime
open time
Definition: XrdFileCacheInfo.hh:55
void SetBitSynced(int i)
Mark block as disk written.
Definition: XrdFileCacheInfo.hh:383
long long BytesDisk
close time
Definition: XrdFileCacheInfo.hh:56
Status of cached file. Can be read from and written into a binary file.
Definition: XrdFileCacheInfo.hh:48
void ResetAllAccessStats()
Reset IO Stats.
void ResizeBits(int n)
Reserve buffer for fileSize/bufferSize bytes.
Definition: XrdSysTrace.hh:48
void WriteIOStatSingle(long long bytes_disk)
Write single open/close time for given bytes read from disk.
unsigned char * m_buff_synced
disk written state vector
Definition: XrdFileCacheInfo.hh:67
bool Write(XrdOssDF *fp, const std::string &fname="<unknown>")
int GetLastDownloadedBlock() const
Get number of the last downloaded block.
Definition: XrdFileCacheInfo.hh:328
void UpdateDownloadCompleteStatus()
Update complete status.
Definition: XrdFileCacheInfo.hh:378
bool TestBit(int i) const
Test if block at the given index is downlaoded.
Definition: XrdFileCacheInfo.hh:294
Definition: XrdCksCalc.hh:39
size_t m_accessCnt
number of written AStat structs
Definition: XrdFileCacheInfo.hh:70
Definition: XrdFileCacheInfo.hh:52
size_t GetAccessCnt()
Get number of accesses.
Definition: XrdFileCacheInfo.hh:247
long long GetFileSize() const
Get file size.
Definition: XrdFileCacheInfo.hh:358
XrdSysTrace * m_trace
Definition: XrdFileCacheInfo.hh:274
const Store & RefStoredData() const
Get stored data.
Definition: XrdFileCacheInfo.hh:257
Info(XrdSysTrace *trace, bool prefetchBuffer=false)
Constructor.
Definition: XrdClAnyObject.hh:25
long long GetBufferSize() const
Get prefetch buffer size.
Definition: XrdFileCacheInfo.hh:412
char m_cksum[16]
cksum of downloaded information
Definition: XrdFileCacheInfo.hh:68
static size_t GetMaxNumAccess()
Definition: XrdFileCacheInfo.hh:271
unsigned char cfiBIT(int n) const
Definition: XrdFileCacheInfo.hh:285
void WriteIOStatDetach(Stats &s)
Write close time together with bytes missed, hits, and disk.
bool m_complete
cached
Definition: XrdFileCacheInfo.hh:282
std::vector< AStat > m_astats
number of last m_maxAcessCnts
Definition: XrdFileCacheInfo.hh:71
void SetFileSize(long long)
void SetBufferSize(long long)
long long GetNDownloadedBytes() const
Get number of downloaded bytes.
Definition: XrdFileCacheInfo.hh:323
int m_sizeInBits
cached
Definition: XrdFileCacheInfo.hh:281
void SetBitPrefetch(int i)
Mark block as written from prefetchxs.
Definition: XrdFileCacheInfo.hh:401
static const char * m_traceID
Definition: XrdFileCacheInfo.hh:265
long long BytesMissed
read from ram
Definition: XrdFileCacheInfo.hh:58
Definition: XrdOss.hh:59
void SetAllBitsSynced()
Mark all blocks as writte.
long long BytesRam
read from disk
Definition: XrdFileCacheInfo.hh:57
bool IsComplete() const
Get complete status.
Definition: XrdFileCacheInfo.hh:363
~Info()
Destructor.
void GetCksum(unsigned char *buff, char *digest)
Get md5 cksum.
XrdSysTrace * GetTrace() const
Definition: XrdFileCacheInfo.hh:269
Definition: XrdFileCacheInfo.hh:63
int GetSizeInBytes() const
Get size of download-state bit-vector in bytes.
Definition: XrdFileCacheInfo.hh:345
void DisableDownloadStatus()
Disable allocating, writing, and reading of downlaod status.
long long m_bufferSize
prefetch buffer size
Definition: XrdFileCacheInfo.hh:65
void WriteIOStatAttach()
Write open time in the last entry of access statistics.
long long m_fileSize
number of file blocks
Definition: XrdFileCacheInfo.hh:66
static const int m_defaultVersion
Definition: XrdFileCacheInfo.hh:266