35 #include <sys/types.h>
62 #define MKDIR(a,b) _mkdir((a))
63 #define REMOVE(a) do { \
64 int s = remove((a)); \
66 throw InternalErr(__FILE__, __LINE__, "Cache error; could not remove file: " + long_to_string(s)); \
68 #define MKSTEMP(a) _open(_mktemp((a)),_O_CREAT,_S_IREAD|_S_IWRITE)
69 #define DIR_SEPARATOR_CHAR '\\'
70 #define DIR_SEPARATOR_STR "\\"
72 #define MKDIR(a,b) mkdir((a), (b))
73 #define REMOVE(a) remove((a))
74 #define MKSTEMP(a) mkstemp((a))
75 #define DIR_SEPARATOR_CHAR '/'
76 #define DIR_SEPARATOR_STR "/"
79 #define CACHE_META ".meta"
80 #define CACHE_INDEX ".index"
81 #define CACHE_EMPTY_ETAG "@cache@"
83 #define NO_LM_EXPIRATION 24*3600 // 24 hours
84 #define MAX_LM_EXPIRATION 48*3600 // Max expiration from LM
89 #define LM_EXPIRATION(t) (min((MAX_LM_EXPIRATION), static_cast<int>((t) / 10)))
106 for (
const char *ptr = url.c_str(); *ptr; ptr++)
112 HTTPCacheTable::HTTPCacheTable(
const string &cache_root,
int block_size) :
113 d_cache_root(cache_root), d_block_size(block_size), d_current_size(0), d_new_entries(0)
121 d_cache_table[i] = 0;
132 DBG2(cerr <<
"Deleting CacheEntry: " << e << endl);
142 for_each(cp->begin(), cp->end(), delete_cache_entry);
145 delete get_cache_table()[i];
146 get_cache_table()[i] = 0;
150 delete[] d_cache_table;
160 class DeleteExpired :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
166 d_time(t), d_table(table) {
171 void operator()(HTTPCacheTable::CacheEntry *&e) {
172 if (e && !e->readers && (e->freshness_lifetime
173 < (e->corrected_initial_age + (d_time - e->response_time)))) {
174 DBG(cerr <<
"Deleting expired cache entry: " << e->url << endl);
175 d_table.remove_cache_entry(e);
187 for_each(slot->begin(), slot->end(), DeleteExpired(*
this, time));
188 slot->erase(
remove(slot->begin(), slot->end(),
200 class DeleteByHits :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
206 d_table(table), d_hits(hits) {
209 void operator()(HTTPCacheTable::CacheEntry *&e) {
210 if (e && !e->readers && e->hits <= d_hits) {
211 DBG(cerr <<
"Deleting cache entry: " << e->url << endl);
212 d_table.remove_cache_entry(e);
221 if (get_cache_table()[cnt]) {
223 for_each(slot->begin(), slot->end(), DeleteByHits(*
this, hits));
224 slot->erase(
remove(slot->begin(), slot->end(),
236 class DeleteBySize :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
242 d_table(table), d_size(size) {
245 void operator()(HTTPCacheTable::CacheEntry *&e) {
246 if (e && !e->readers && e->size > d_size) {
247 DBG(cerr <<
"Deleting cache entry: " << e->url << endl);
248 d_table.remove_cache_entry(e);
256 if (get_cache_table()[cnt]) {
258 for_each(slot->begin(), slot->end(), DeleteBySize(*
this, size));
259 slot->erase(
remove(slot->begin(), slot->end(),
299 FILE *fp = fopen(d_cache_index.c_str(),
"r");
307 while (!feof(fp) && fgets(line, 1024, fp)) {
309 DBG2(cerr << line << endl);
312 int res = fclose(fp) ;
314 DBG(cerr <<
"HTTPCache::cache_index_read - Failed to close " << (
void *)fp << endl);
334 istringstream iss(line);
336 iss >> entry->cachename;
343 iss >> entry->expires;
349 iss >> entry->freshness_lifetime;
350 iss >> entry->response_time;
351 iss >> entry->corrected_initial_age;
353 iss >> entry->must_revalidate;
360 class WriteOneCacheEntry :
361 public unary_function<HTTPCacheTable::CacheEntry *, void>
367 WriteOneCacheEntry(FILE *fp) : d_fp(fp)
370 void operator()(HTTPCacheTable::CacheEntry *e)
372 if (e && fprintf(d_fp,
373 "%s %s %s %ld %ld %ld %c %d %d %ld %ld %ld %c\r\n",
375 e->cachename.c_str(),
380 e->range ?
'1' :
'0',
383 (long)(e->freshness_lifetime),
384 (long)(e->response_time),
385 (long)(e->corrected_initial_age),
386 e->must_revalidate ?
'1' :
'0') < 0)
387 throw Error(
"Cache Index. Error writing cache index\n");
403 DBG(cerr <<
"Cache Index. Writing index " << d_cache_index << endl);
407 if ((fp = fopen(d_cache_index.c_str(),
"wb")) == NULL) {
408 throw Error(
string(
"Cache Index. Can't open `") + d_cache_index
409 +
string(
"' for writing"));
418 for_each(cp->begin(), cp->end(), WriteOneCacheEntry(fp));
422 int res = fclose(fp);
424 DBG(cerr <<
"HTTPCache::cache_index_write - Failed to close "
425 << (
void *)fp << endl);
448 struct stat stat_info;
451 path << d_cache_root << hash;
452 string p = path.str();
454 if (stat(p.c_str(), &stat_info) == -1) {
455 DBG2(cerr <<
"Cache....... Create dir " << p << endl);
456 if (
MKDIR(p.c_str(), 0777) < 0) {
457 DBG2(cerr <<
"Cache....... Can't create..." << endl);
458 throw Error(
"Could not create cache slot to hold response! Check the write permissions on your disk cache directory. Cache root: " + d_cache_root +
".");
462 DBG2(cerr <<
"Cache....... Directory " << p <<
" already exists"
488 hash_dir +=
"\\dodsXXXXXX";
490 hash_dir +=
"/dodsXXXXXX";
495 vector<char> templat(hash_dir.size() + 1);
496 strncpy(&templat[0], hash_dir.c_str(), hash_dir.size() + 1);
510 throw Error(
"The HTTP Cache could not create a file to hold the response; it will not be cached.");
513 entry->cachename = &templat[0];
521 entry_disk_space(
int size,
unsigned int block_size)
523 unsigned int num_of_blocks = (size + block_size) / block_size;
525 DBG(cerr <<
"size: " << size <<
", block_size: " << block_size
526 <<
", num_of_blocks: " << num_of_blocks << endl);
528 return num_of_blocks * block_size;
543 unsigned int hash = entry->hash;
545 throw InternalErr(__FILE__, __LINE__,
"Hash value too large!");
547 if (!d_cache_table[hash])
550 d_cache_table[hash]->push_back(entry);
552 DBG(cerr <<
"add_entry_to_cache_table, current_size: " << d_current_size
553 <<
", entry->size: " << entry->size <<
", block size: " << d_block_size
556 d_current_size += entry_disk_space(entry->size, d_block_size);
558 DBG(cerr <<
"add_entry_to_cache_table, current_size: " << d_current_size << endl);
567 HTTPCacheTable::get_locked_entry_from_cache_table(
const string &url)
569 return get_locked_entry_from_cache_table(
get_hash(url), url);
580 HTTPCacheTable::get_locked_entry_from_cache_table(
int hash,
const string &url)
582 DBG(cerr <<
"url: " << url <<
"; hash: " << hash << endl);
583 DBG(cerr <<
"d_cache_table: " << hex << d_cache_table << dec << endl);
584 if (d_cache_table[hash]) {
589 if ((*i) && (*i)->url == url) {
590 (*i)->lock_read_response();
605 HTTPCacheTable::CacheEntry *
609 if (d_cache_table[hash]) {
614 if ((*i) && (*i)->url == url) {
615 (*i)->lock_write_response();
637 throw InternalErr(__FILE__, __LINE__,
"Tried to delete a cache entry that is in use.");
639 REMOVE(entry->cachename.c_str());
644 unsigned int eds = entry_disk_space(entry->size,
get_block_size());
652 class DeleteCacheEntry:
public unary_function<HTTPCacheTable::CacheEntry *&, void>
659 : d_url(url), d_cache_table(c)
662 void operator()(HTTPCacheTable::CacheEntry *&e)
664 if (e && e->url == d_url) {
665 e->lock_write_response();
666 d_cache_table->remove_cache_entry(e);
667 e->unlock_write_response();
683 if (d_cache_table[hash]) {
685 for_each(cp->begin(), cp->end(), DeleteCacheEntry(
this, url));
693 class DeleteUnlockedCacheEntry:
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
701 void operator()(HTTPCacheTable::CacheEntry *&e)
704 d_table.remove_cache_entry(e);
718 for_each(slot->begin(), slot->end(), DeleteUnlockedCacheEntry(*
this));
742 entry->response_time = time(NULL);
743 time_t apparent_age = max(0, static_cast<int>(entry->response_time - entry->date));
744 time_t corrected_received_age = max(apparent_age, entry->age);
745 time_t response_delay = entry->response_time - request_time;
746 entry->corrected_initial_age = corrected_received_age + response_delay;
751 time_t freshness_lifetime = entry->max_age;
752 if (freshness_lifetime < 0) {
753 if (entry->expires < 0) {
755 freshness_lifetime = default_expiration;
762 freshness_lifetime = entry->expires - entry->date;
765 entry->freshness_lifetime = max(0, static_cast<int>(freshness_lifetime));
767 DBG2(cerr <<
"Cache....... Received Age " << entry->age
768 <<
", corrected " << entry->corrected_initial_age
769 <<
", freshness lifetime " << entry->freshness_lifetime << endl);
784 const vector<string> &headers)
786 vector<string>::const_iterator i;
787 for (i = headers.begin(); i != headers.end(); ++i) {
792 string::size_type colon = (*i).find(
':');
795 if (colon == string::npos)
798 string header = (*i).substr(0, (*i).find(
':'));
799 string value = (*i).substr((*i).find(
": ") + 2);
800 DBG2(cerr <<
"Header: " << header << endl);
DBG2(cerr <<
"Value: " << value << endl);
802 if (header ==
"ETag") {
805 else if (header ==
"Last-Modified") {
808 else if (header ==
"Expires") {
811 else if (header ==
"Date") {
814 else if (header ==
"Age") {
817 else if (header ==
"Content-Length") {
818 unsigned long clength = strtoul(value.c_str(), 0, 0);
819 if (clength > max_entry_size)
822 else if (header ==
"Cache-Control") {
826 if (value ==
"no-cache" || value ==
"no-store")
831 else if (value ==
"must-revalidate")
832 entry->must_revalidate =
true;
833 else if (value.find(
"max-age") != string::npos) {
834 string max_age = value.substr(value.find(
"=" + 1));
846 d_locked_entries[body] = entry;
853 throw InternalErr(
"There is no cache entry for the response given.");
855 d_locked_entries.erase(body);
858 if (entry->readers < 0)
859 throw InternalErr(
"An unlocked entry was released");
863 return !d_locked_entries.empty();