00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026 #include "config.h"
00027
00028 #include <pthread.h>
00029 #include <limits.h>
00030 #include <unistd.h>
00031 #include <sys/types.h>
00032 #include <sys/stat.h>
00033
00034 #include <cstring>
00035 #include <iostream>
00036 #include <sstream>
00037 #include <algorithm>
00038 #include <iterator>
00039 #include <set>
00040
00041 #include "Error.h"
00042 #include "InternalErr.h"
00043 #include "ResponseTooBigErr.h"
00044 #ifndef WIN32
00045 #include "SignalHandler.h"
00046 #endif
00047 #include "HTTPCacheInterruptHandler.h"
00048 #include "HTTPCacheTable.h"
00049 #include "HTTPCache.h"
00050
00051 #include "util_mit.h"
00052 #include "debug.h"
00053
00054 using namespace std;
00055
00056 namespace libdap {
00057
00058 HTTPCache *HTTPCache::_instance = 0;
00059
00060
00061
00062
00063
00064
00065
00066
00067 static pthread_mutex_t instance_mutex;
00068 static pthread_once_t once_block = PTHREAD_ONCE_INIT;
00069
00070 #ifdef WIN32
00071 #include <direct.h>
00072 #include <time.h>
00073 #include <fcntl.h>
00074 #define MKDIR(a,b) _mkdir((a))
00075 #define REMOVE(a) remove((a))
00076 #define MKSTEMP(a) _open(_mktemp((a)),_O_CREAT,_S_IREAD|_S_IWRITE)
00077 #define DIR_SEPARATOR_CHAR '\\'
00078 #define DIR_SEPARATOR_STR "\\"
00079 #else
00080 #define MKDIR(a,b) mkdir((a), (b))
00081 #define REMOVE(a) remove((a))
00082 #define MKSTEMP(a) mkstemp((a))
00083 #define DIR_SEPARATOR_CHAR '/'
00084 #define DIR_SEPARATOR_STR "/"
00085 #endif
00086
00087 #ifdef WIN32
00088 #define CACHE_LOC "\\tmp\\"
00089 #define CACHE_ROOT "dods-cache\\"
00090 #else
00091 #define CACHE_LOC "/tmp/"
00092 #define CACHE_ROOT "dods-cache/"
00093 #endif
00094 #define CACHE_INDEX ".index"
00095 #define CACHE_LOCK ".lock"
00096 #define CACHE_META ".meta"
00097
00098
00099 #define NO_LM_EXPIRATION 24*3600 // 24 hours
00100
00101 #define DUMP_FREQUENCY 10 // Dump index every x loads
00102
00103 #define MEGA 0x100000L
00104 #define CACHE_TOTAL_SIZE 20 // Default cache size is 20M
00105 #define CACHE_FOLDER_PCT 10 // 10% of cache size for metainfo etc.
00106 #define CACHE_GC_PCT 10 // 10% of cache size free after GC
00107 #define MIN_CACHE_TOTAL_SIZE 5 // 5M Min cache size
00108 #define MAX_CACHE_ENTRY_SIZE 3 // 3M Max size of single cached entry
00109
00110 static void
00111 once_init_routine()
00112 {
00113 int status;
00114 status = INIT(&instance_mutex);
00115
00116 if (status != 0)
00117 throw InternalErr(__FILE__, __LINE__, "Could not initialize the HTTP Cache mutex. Exiting.");
00118 }
00119
00148 HTTPCache *
00149 HTTPCache::instance(const string &cache_root, bool force)
00150 {
00151 LOCK(&instance_mutex);
00152 DBG(cerr << "Entering instance(); (" << hex << _instance << dec << ")"
00153 << "... ");
00154
00155 try {
00156 if (!_instance) {
00157 _instance = new HTTPCache(cache_root, force);
00158
00159 DBG(cerr << "New instance: " << _instance << ", cache root: "
00160 << _instance->d_cache_root << endl);
00161
00162 atexit(delete_instance);
00163
00164 #ifndef WIN32
00165
00166
00167
00168
00169
00170
00171
00172
00173 EventHandler *old_eh = SignalHandler::instance()->register_handler
00174 (SIGINT, new HTTPCacheInterruptHandler);
00175 if (old_eh) {
00176 SignalHandler::instance()->register_handler(SIGINT, old_eh);
00177 throw SignalHandlerRegisteredErr(
00178 "Could not register event handler for SIGINT without superseding an existing one.");
00179 }
00180
00181 old_eh = SignalHandler::instance()->register_handler
00182 (SIGPIPE, new HTTPCacheInterruptHandler);
00183 if (old_eh) {
00184 SignalHandler::instance()->register_handler(SIGPIPE, old_eh);
00185 throw SignalHandlerRegisteredErr(
00186 "Could not register event handler for SIGPIPE without superseding an existing one.");
00187 }
00188
00189 old_eh = SignalHandler::instance()->register_handler
00190 (SIGTERM, new HTTPCacheInterruptHandler);
00191 if (old_eh) {
00192 SignalHandler::instance()->register_handler(SIGTERM, old_eh);
00193 throw SignalHandlerRegisteredErr(
00194 "Could not register event handler for SIGTERM without superseding an existing one.");
00195 }
00196 #endif
00197 }
00198 }
00199 catch (...) {
00200 DBG2(cerr << "The constructor threw an Error!" << endl);
00201 UNLOCK(&instance_mutex);
00202 throw;
00203 }
00204
00205 UNLOCK(&instance_mutex);
00206 DBGN(cerr << "returning " << hex << _instance << dec << endl);
00207
00208 return _instance;
00209 }
00210
00214 void
00215 HTTPCache::delete_instance()
00216 {
00217 DBG(cerr << "Entering delete_instance()..." << endl);
00218 if (HTTPCache::_instance) {
00219 DBG(cerr << "Deleting the cache: " << HTTPCache::_instance << endl);
00220 delete HTTPCache::_instance;
00221 HTTPCache::_instance = 0;
00222 }
00223
00224 DBG(cerr << "Exiting delete_instance()" << endl);
00225 }
00226
00241 HTTPCache::HTTPCache(string cache_root, bool force) :
00242 d_locked_open_file(0),
00243 d_cache_enabled(false),
00244 d_cache_protected(false),
00245 d_expire_ignored(false),
00246 d_always_validate(false),
00247 d_total_size(CACHE_TOTAL_SIZE * MEGA),
00248 d_folder_size(CACHE_TOTAL_SIZE / CACHE_FOLDER_PCT),
00249 d_gc_buffer(CACHE_TOTAL_SIZE / CACHE_GC_PCT),
00250 d_max_entry_size(MAX_CACHE_ENTRY_SIZE * MEGA),
00251 d_default_expiration(NO_LM_EXPIRATION),
00252 d_max_age(-1),
00253 d_max_stale(-1),
00254 d_min_fresh(-1),
00255 d_http_cache_table(0)
00256 {
00257 DBG(cerr << "Entering the constructor for " << this << "... ");
00258
00259 int status = pthread_once(&once_block, once_init_routine);
00260 if (status != 0)
00261 throw InternalErr(__FILE__, __LINE__, "Could not initialize the HTTP Cache mutex. Exiting.");
00262
00263 INIT(&d_cache_mutex);
00264
00265
00266
00267
00268
00269
00270
00271
00272
00273
00274
00275 set_cache_root(cache_root);
00276 int block_size;
00277
00278 if (!get_single_user_lock(force))
00279 throw Error("Could not get single user lock for the cache");
00280
00281 #ifdef WIN32
00282
00283
00284
00285
00286 block_size = 4096;
00287 #else
00288 struct stat s;
00289 if (stat(cache_root.c_str(), &s) == 0)
00290 block_size = s.st_blksize;
00291 else
00292 throw Error("Could not set file system block size.");
00293 #endif
00294 d_http_cache_table = new HTTPCacheTable(d_cache_root, block_size);
00295 d_cache_enabled = true;
00296
00297 DBGN(cerr << "exiting" << endl);
00298 }
00299
00312 HTTPCache::~HTTPCache()
00313 {
00314 DBG(cerr << "Entering the destructor for " << this << "... ");
00315
00316 try {
00317 if (startGC())
00318 perform_garbage_collection();
00319
00320 d_http_cache_table->cache_index_write();
00321 }
00322 catch (Error &e) {
00323
00324
00325
00326
00327 DBG(cerr << e.get_error_message() << endl);
00328 }
00329
00330 delete d_http_cache_table;
00331
00332 release_single_user_lock();
00333
00334 DBGN(cerr << "exiting destructor." << endl);
00335 DESTROY(&d_cache_mutex);
00336 }
00337
00338
00342
00346 bool
00347 HTTPCache::stopGC() const
00348 {
00349 return (d_http_cache_table->get_current_size() + d_folder_size < d_total_size - d_gc_buffer);
00350 }
00351
00358 bool
00359 HTTPCache::startGC() const
00360 {
00361 DBG(cerr << "startGC, current_size: " << d_http_cache_table->get_current_size() << endl);
00362 return (d_http_cache_table->get_current_size() + d_folder_size > d_total_size);
00363 }
00364
00379 void
00380 HTTPCache::perform_garbage_collection()
00381 {
00382 DBG(cerr << "Performing garbage collection" << endl);
00383
00384
00385 expired_gc();
00386
00387
00388 too_big_gc();
00389
00390
00391
00392 hits_gc();
00393 }
00394
00400 void
00401 HTTPCache::expired_gc()
00402 {
00403 if (!d_expire_ignored) {
00404 d_http_cache_table->delete_expired_entries();
00405 }
00406 }
00407
00424 void
00425 HTTPCache::hits_gc()
00426 {
00427 int hits = 0;
00428
00429 if (startGC()) {
00430 while (!stopGC()) {
00431 d_http_cache_table->delete_by_hits(hits);
00432 hits++;
00433 }
00434 }
00435 }
00436
00441 void HTTPCache::too_big_gc() {
00442 if (startGC())
00443 d_http_cache_table->delete_by_size(d_max_entry_size);
00444 }
00445
00447
00458 bool HTTPCache::get_single_user_lock(bool force) {
00459 if (!d_locked_open_file) {
00460 FILE * fp = NULL;
00461
00462 try {
00463
00464
00465 create_cache_root(d_cache_root);
00466 }
00467 catch (Error &e) {
00468
00469
00470
00471 return false;
00472 }
00473
00474
00475 string lock = d_cache_root + CACHE_LOCK;
00476 if ((fp = fopen(lock.c_str(), "r")) != NULL) {
00477 int res = fclose(fp);
00478 if (res) {
00479 DBG(cerr << "Failed to close " << (void *)fp << endl);
00480 }
00481 if (force)
00482 REMOVE(lock.c_str());
00483 else
00484 return false;
00485 }
00486
00487 if ((fp = fopen(lock.c_str(), "w")) == NULL)
00488 return false;
00489
00490 d_locked_open_file = fp;
00491 return true;
00492 }
00493
00494 return false;
00495 }
00496
00499 void
00500 HTTPCache::release_single_user_lock()
00501 {
00502 if (d_locked_open_file) {
00503 int res = fclose(d_locked_open_file);
00504 if (res) {
00505 DBG(cerr << "Failed to close " << (void *)d_locked_open_file << endl) ;
00506 }
00507 d_locked_open_file = 0;
00508 }
00509
00510 string lock = d_cache_root + CACHE_LOCK;
00511 REMOVE(lock.c_str());
00512 }
00513
00516
00520 string
00521 HTTPCache::get_cache_root() const
00522 {
00523 return d_cache_root;
00524 }
00525
00526
00535 void
00536 HTTPCache::create_cache_root(const string &cache_root)
00537 {
00538 struct stat stat_info;
00539 string::size_type cur = 0;
00540
00541 #ifdef WIN32
00542 cur = cache_root[1] == ':' ? 3 : 1;
00543 #else
00544 cur = 1;
00545 #endif
00546 while ((cur = cache_root.find(DIR_SEPARATOR_CHAR, cur)) != string::npos) {
00547 string dir = cache_root.substr(0, cur);
00548 if (stat(dir.c_str(), &stat_info) == -1) {
00549 DBG2(cerr << "Cache....... Creating " << dir << endl);
00550 if (MKDIR(dir.c_str(), 0777) < 0) {
00551 DBG2(cerr << "Error: can't create." << endl);
00552 throw Error(string("Could not create the directory for the cache. Failed when building path at ") + dir + string("."));
00553 }
00554 }
00555 else {
00556 DBG2(cerr << "Cache....... Found " << dir << endl);
00557 }
00558 cur++;
00559 }
00560 }
00561
00576 void
00577 HTTPCache::set_cache_root(const string &root)
00578 {
00579 if (root != "") {
00580 d_cache_root = root;
00581
00582 if (d_cache_root[d_cache_root.size()-1] != DIR_SEPARATOR_CHAR)
00583 d_cache_root += DIR_SEPARATOR_CHAR;
00584 }
00585 else {
00586
00587
00588 char * cr = (char *) getenv("DODS_CACHE");
00589 if (!cr) cr = (char *) getenv("TMP");
00590 if (!cr) cr = (char *) getenv("TEMP");
00591 if (!cr) cr = CACHE_LOC;
00592
00593 d_cache_root = cr;
00594 if (d_cache_root[d_cache_root.size()-1] != DIR_SEPARATOR_CHAR)
00595 d_cache_root += DIR_SEPARATOR_CHAR;
00596
00597 d_cache_root += CACHE_ROOT;
00598 }
00599
00600
00601
00602
00603 if (d_http_cache_table)
00604 d_http_cache_table->set_cache_root(d_cache_root);
00605 }
00606
00618 void
00619 HTTPCache::set_cache_enabled(bool mode)
00620 {
00621 lock_cache_interface();
00622
00623 d_cache_enabled = mode;
00624
00625 unlock_cache_interface();
00626 }
00627
00630 bool
00631 HTTPCache::is_cache_enabled() const
00632 {
00633 DBG2(cerr << "In HTTPCache::is_cache_enabled: (" << d_cache_enabled << ")"
00634 << endl);
00635 return d_cache_enabled;
00636 }
00637
00648 void
00649 HTTPCache::set_cache_disconnected(CacheDisconnectedMode mode)
00650 {
00651 lock_cache_interface();
00652
00653 d_cache_disconnected = mode;
00654
00655 unlock_cache_interface();
00656 }
00657
00660 CacheDisconnectedMode
00661 HTTPCache::get_cache_disconnected() const
00662 {
00663 return d_cache_disconnected;
00664 }
00665
00674 void
00675 HTTPCache::set_expire_ignored(bool mode)
00676 {
00677 lock_cache_interface();
00678
00679 d_expire_ignored = mode;
00680
00681 unlock_cache_interface();
00682 }
00683
00684
00685
00686
00687 bool
00688 HTTPCache::is_expire_ignored() const
00689 {
00690 return d_expire_ignored;
00691 }
00692
00708 void
00709 HTTPCache::set_max_size(unsigned long size)
00710 {
00711 lock_cache_interface();
00712
00713 try {
00714 unsigned long new_size = size < MIN_CACHE_TOTAL_SIZE ?
00715 MIN_CACHE_TOTAL_SIZE * MEGA :
00716 (size > ULONG_MAX ? ULONG_MAX : size * MEGA);
00717 unsigned long old_size = d_total_size;
00718 d_total_size = new_size;
00719 d_folder_size = d_total_size / CACHE_FOLDER_PCT;
00720 d_gc_buffer = d_total_size / CACHE_GC_PCT;
00721
00722 if (new_size < old_size && startGC()) {
00723 perform_garbage_collection();
00724 d_http_cache_table->cache_index_write();
00725 }
00726 }
00727 catch (...) {
00728 unlock_cache_interface();
00729 DBGN(cerr << "Unlocking interface." << endl);
00730 throw;
00731 }
00732
00733 DBG2(cerr << "Cache....... Total cache size: " << d_total_size
00734 << " with " << d_folder_size
00735 << " bytes for meta information and folders and at least "
00736 << d_gc_buffer << " bytes free after every gc" << endl);
00737
00738 unlock_cache_interface();
00739 }
00740
00743 unsigned long
00744 HTTPCache::get_max_size() const
00745 {
00746 return d_total_size / MEGA;
00747 }
00748
00757 void
00758 HTTPCache::set_max_entry_size(unsigned long size)
00759 {
00760 lock_cache_interface();
00761
00762 try {
00763 unsigned long new_size = size * MEGA;
00764 if (new_size > 0 && new_size < d_total_size - d_folder_size) {
00765 unsigned long old_size = d_max_entry_size;
00766 d_max_entry_size = new_size;
00767 if (new_size < old_size && startGC()) {
00768 perform_garbage_collection();
00769 d_http_cache_table->cache_index_write();
00770 }
00771 }
00772 }
00773 catch (...) {
00774 unlock_cache_interface();
00775 throw;
00776 }
00777
00778 DBG2(cerr << "Cache...... Max entry cache size is "
00779 << d_max_entry_size << endl);
00780
00781 unlock_cache_interface();
00782 }
00783
00788 unsigned long
00789 HTTPCache::get_max_entry_size() const
00790 {
00791 return d_max_entry_size / MEGA;
00792 }
00793
00804 void
00805 HTTPCache::set_default_expiration(const int exp_time)
00806 {
00807 lock_cache_interface();
00808
00809 d_default_expiration = exp_time;
00810
00811 unlock_cache_interface();
00812 }
00813
00816 int
00817 HTTPCache::get_default_expiration() const
00818 {
00819 return d_default_expiration;
00820 }
00821
00826 void
00827 HTTPCache::set_always_validate(bool validate)
00828 {
00829 d_always_validate = validate;
00830 }
00831
00835 bool
00836 HTTPCache::get_always_validate() const
00837 {
00838 return d_always_validate;
00839 }
00840
00857 void
00858 HTTPCache::set_cache_control(const vector<string> &cc)
00859 {
00860 lock_cache_interface();
00861
00862 try {
00863 d_cache_control = cc;
00864
00865 vector<string>::const_iterator i;
00866 for (i = cc.begin(); i != cc.end(); ++i) {
00867 string header = (*i).substr(0, (*i).find(':'));
00868 string value = (*i).substr((*i).find(": ") + 2);
00869 if (header != "Cache-Control") {
00870 throw InternalErr(__FILE__, __LINE__, "Expected cache control header not found.");
00871 }
00872 else {
00873 if (value == "no-cache" || value == "no-store")
00874 d_cache_enabled = false;
00875 else if (value.find("max-age") != string::npos) {
00876 string max_age = value.substr(value.find("=" + 1));
00877 d_max_age = parse_time(max_age.c_str());
00878 }
00879 else if (value == "max-stale")
00880 d_max_stale = 0;
00881 else if (value.find("max-stale") != string::npos) {
00882 string max_stale = value.substr(value.find("=" + 1));
00883 d_max_stale = parse_time(max_stale.c_str());
00884 }
00885 else if (value.find("min-fresh") != string::npos) {
00886 string min_fresh = value.substr(value.find("=" + 1));
00887 d_min_fresh = parse_time(min_fresh.c_str());
00888 }
00889 }
00890 }
00891 }
00892 catch (...) {
00893 unlock_cache_interface();
00894 throw;
00895 }
00896
00897 unlock_cache_interface();
00898 }
00899
00900
00905 vector<string>
00906 HTTPCache::get_cache_control()
00907 {
00908 return d_cache_control;
00909 }
00910
00912
00921 bool
00922 HTTPCache::is_url_in_cache(const string &url)
00923 {
00924 DBG(cerr << "Is this url in the cache? (" << url << ")" << endl);
00925
00926 HTTPCacheTable::CacheEntry *entry = d_http_cache_table->get_locked_entry_from_cache_table(url);
00927 bool status = entry != 0;
00928 if (entry) {
00929 #if 0
00930 entry->unlock();
00931 #endif
00932 entry->unlock_read_response();
00933 }
00934 return status;
00935 }
00936
00942 bool
00943 is_hop_by_hop_header(const string &header)
00944 {
00945 return header.find("Connection") != string::npos
00946 || header.find("Keep-Alive") != string::npos
00947 || header.find("Proxy-Authenticate") != string::npos
00948 || header.find("Proxy-Authorization") != string::npos
00949 || header.find("Transfer-Encoding") != string::npos
00950 || header.find("Upgrade") != string::npos;
00951 }
00952
00964 void
00965 HTTPCache::write_metadata(const string &cachename, const vector<string> &headers)
00966 {
00967 string fname = cachename + CACHE_META;
00968 d_open_files.push_back(fname);
00969
00970 FILE *dest = fopen(fname.c_str(), "w");
00971 if (!dest) {
00972 throw InternalErr(__FILE__, __LINE__,
00973 "Could not open named cache entry file.");
00974 }
00975
00976 vector<string>::const_iterator i;
00977 for (i = headers.begin(); i != headers.end(); ++i) {
00978 if (!is_hop_by_hop_header(*i)) {
00979 fwrite((*i).c_str(), (*i).size(), 1, dest);
00980 fwrite("\n", 1, 1, dest);
00981 }
00982 }
00983
00984 int res = fclose(dest);
00985 if (res) {
00986 DBG(cerr << "HTTPCache::write_metadata - Failed to close "
00987 << dest << endl);
00988 }
00989
00990 d_open_files.pop_back();
00991 }
00992
01003 void
01004 HTTPCache::read_metadata(const string &cachename, vector<string> &headers)
01005 {
01006 FILE *md = fopen(string(cachename + CACHE_META).c_str(), "r");
01007 if (!md) {
01008 throw InternalErr(__FILE__, __LINE__,
01009 "Could not open named cache entry meta data file.");
01010 }
01011
01012 char line[1024];
01013 while (!feof(md) && fgets(line, 1024, md)) {
01014 line[min(1024, static_cast<int>(strlen(line)))-1] = '\0';
01015 headers.push_back(string(line));
01016 }
01017
01018 int res = fclose(md);
01019 if (res) {
01020 DBG(cerr << "HTTPCache::read_metadata - Failed to close "
01021 << md << endl);
01022 }
01023 }
01024
01046 int
01047 HTTPCache::write_body(const string &cachename, const FILE *src)
01048 {
01049 d_open_files.push_back(cachename);
01050
01051 FILE *dest = fopen(cachename.c_str(), "wb");
01052 if (!dest) {
01053 throw InternalErr(__FILE__, __LINE__,
01054 "Could not open named cache entry file.");
01055 }
01056
01057
01058
01059 char line[1024];
01060 size_t n;
01061 int total = 0;
01062 while ((n = fread(line, 1, 1024, const_cast<FILE *>(src))) > 0) {
01063 total += fwrite(line, 1, n, dest);
01064 DBG2(sleep(3));
01065 }
01066
01067 if (ferror(const_cast<FILE *>(src)) || ferror(dest)) {
01068 int res = fclose(dest);
01069 res = res & unlink(cachename.c_str());
01070 if (res) {
01071 DBG(cerr << "HTTPCache::write_body - Failed to close/unlink "
01072 << dest << endl);
01073 }
01074 throw InternalErr(__FILE__, __LINE__,
01075 "I/O error transferring data to the cache.");
01076 }
01077
01078 rewind(const_cast<FILE *>(src));
01079
01080 int res = fclose(dest);
01081 if (res) {
01082 DBG(cerr << "HTTPCache::write_body - Failed to close "
01083 << dest << endl);
01084 }
01085
01086 d_open_files.pop_back();
01087
01088 return total;
01089 }
01090
01099 FILE *
01100 HTTPCache::open_body(const string &cachename)
01101 {
01102 FILE *src = fopen(cachename.c_str(), "rb");
01103 if (!src)
01104 throw InternalErr(__FILE__, __LINE__, "Could not open cache file.");
01105
01106 return src;
01107 }
01108
01134 bool
01135 HTTPCache::cache_response(const string &url, time_t request_time,
01136 const vector<string> &headers, const FILE *body)
01137 {
01138 lock_cache_interface();
01139
01140 DBG(cerr << "Caching url: " << url << "." << endl);
01141
01142 try {
01143
01144 if (url.find("http:") == string::npos &&
01145 url.find("https:") == string::npos) {
01146 unlock_cache_interface();
01147 return false;
01148 }
01149
01150
01151
01152
01153 d_http_cache_table->remove_entry_from_cache_table(url);
01154
01155 HTTPCacheTable::CacheEntry *entry = new HTTPCacheTable::CacheEntry(url);
01156 entry->lock_write_response();
01157
01158 try {
01159 d_http_cache_table->parse_headers(entry, d_max_entry_size, headers);
01160 if (entry->is_no_cache()) {
01161 DBG(cerr << "Not cache-able; deleting HTTPCacheTable::CacheEntry: " << entry
01162 << "(" << url << ")" << endl);
01163 entry->unlock_write_response();
01164 delete entry; entry = 0;
01165 unlock_cache_interface();
01166 return false;
01167 }
01168
01169
01170 d_http_cache_table->calculate_time(entry, d_default_expiration, request_time);
01171
01172 d_http_cache_table->create_location(entry);
01173
01174 entry->set_size(write_body(entry->get_cachename(), body));
01175 write_metadata(entry->get_cachename(), headers);
01176 d_http_cache_table->add_entry_to_cache_table(entry);
01177 entry->unlock_write_response();
01178 }
01179 catch (ResponseTooBigErr &e) {
01180
01181 DBG(cerr << e.get_error_message() << endl);
01182 REMOVE(entry->get_cachename().c_str());
01183 REMOVE(string(entry->get_cachename() + CACHE_META).c_str());
01184 DBG(cerr << "Too big; deleting HTTPCacheTable::CacheEntry: " << entry << "(" << url
01185 << ")" << endl);
01186 entry->unlock_write_response();
01187 delete entry; entry = 0;
01188 unlock_cache_interface();
01189 return false;
01190 }
01191
01192 if (d_http_cache_table->get_new_entries() > DUMP_FREQUENCY) {
01193 if (startGC())
01194 perform_garbage_collection();
01195
01196 d_http_cache_table->cache_index_write();
01197 }
01198 }
01199 catch (...) {
01200 unlock_cache_interface();
01201 throw;
01202 }
01203
01204 unlock_cache_interface();
01205
01206 return true;
01207 }
01208
01227 vector<string>
01228 HTTPCache::get_conditional_request_headers(const string &url)
01229 {
01230 lock_cache_interface();
01231
01232 HTTPCacheTable::CacheEntry *entry = 0;
01233 vector<string> headers;
01234
01235 DBG(cerr << "Getting conditional request headers for " << url << endl);
01236
01237 try {
01238 entry = d_http_cache_table->get_locked_entry_from_cache_table(url);
01239 if (!entry)
01240 throw Error("There is no cache entry for the URL: " + url);
01241
01242 if (entry->get_etag() != "")
01243 headers.push_back(string("If-None-Match: ") + entry->get_etag());
01244
01245 if (entry->get_lm() > 0) {
01246 time_t lm = entry->get_lm();
01247 headers.push_back(string("If-Modified-Since: ")
01248 + date_time_str(&lm));
01249 }
01250 else if (entry->get_max_age() > 0) {
01251 time_t max_age = entry->get_max_age();
01252 headers.push_back(string("If-Modified-Since: ")
01253 + date_time_str(&max_age));
01254 }
01255 else if (entry->get_expires() > 0) {
01256 time_t expires = entry->get_expires();
01257 headers.push_back(string("If-Modified-Since: ")
01258 + date_time_str(&expires));
01259 }
01260 #if 0
01261 entry->unlock();
01262 #endif
01263 entry->unlock_read_response();
01264 unlock_cache_interface();
01265 }
01266 catch (...) {
01267 unlock_cache_interface();
01268 if (entry) {
01269 #if 0
01270 entry->unlock();
01271 #endif
01272 entry->unlock_read_response();
01273 }
01274 throw;
01275 }
01276
01277 return headers;
01278 }
01279
01283 struct HeaderLess: binary_function<const string&, const string&, bool>
01284 {
01285 bool operator()(const string &s1, const string &s2) const {
01286 return s1.substr(0, s1.find(':')) < s2.substr(0, s2.find(':'));
01287 }
01288 };
01289
01303 void
01304 HTTPCache::update_response(const string &url, time_t request_time,
01305 const vector<string> &headers)
01306 {
01307 lock_cache_interface();
01308
01309 HTTPCacheTable::CacheEntry *entry = 0;
01310 DBG(cerr << "Updating the response headers for: " << url << endl);
01311
01312 try {
01313 entry = d_http_cache_table->get_write_locked_entry_from_cache_table(url);
01314 if (!entry)
01315 throw Error("There is no cache entry for the URL: " + url);
01316
01317
01318 d_http_cache_table->parse_headers(entry, d_max_entry_size, headers);
01319
01320
01321 d_http_cache_table->calculate_time(entry, d_default_expiration, request_time);
01322
01323
01324
01325
01326
01327
01328
01329 set<string, HeaderLess> merged_headers;
01330
01331
01332 copy(headers.begin(), headers.end(),
01333 inserter(merged_headers, merged_headers.begin()));
01334
01335
01336 vector<string> old_headers;
01337 read_metadata(entry->get_cachename(), old_headers);
01338 copy(old_headers.begin(), old_headers.end(),
01339 inserter(merged_headers, merged_headers.begin()));
01340
01341
01342
01343
01344 vector<string> result;
01345 copy(merged_headers.rbegin(), merged_headers.rend(),
01346 back_inserter(result));
01347
01348 write_metadata(entry->get_cachename(), result);
01349 #if 0
01350 entry->unlock();
01351 #endif
01352 entry->unlock_write_response();
01353 unlock_cache_interface();
01354 }
01355 catch (...) {
01356 if (entry) {
01357 #if 0
01358 entry->unlock();
01359 #endif
01360 entry->unlock_read_response();
01361 }
01362 unlock_cache_interface();
01363 throw;
01364 }
01365 }
01366
01378 bool
01379 HTTPCache::is_url_valid(const string &url)
01380 {
01381 lock_cache_interface();
01382
01383 bool freshness;
01384 HTTPCacheTable::CacheEntry *entry = 0;
01385
01386 DBG(cerr << "Is this URL valid? (" << url << ")" << endl);
01387
01388 try {
01389 if (d_always_validate) {
01390 unlock_cache_interface();
01391 return false;
01392 }
01393
01394 entry = d_http_cache_table->get_locked_entry_from_cache_table(url);
01395 if (!entry)
01396 throw Error("There is no cache entry for the URL: " + url);
01397
01398
01399
01400
01401
01402
01403 if (entry->get_must_revalidate()) {
01404 #if 0
01405 entry->unlock();
01406 #endif
01407 entry->unlock_read_response();
01408 unlock_cache_interface();
01409 return false;
01410 }
01411
01412 time_t resident_time = time(NULL) - entry->get_response_time();
01413 time_t current_age = entry->get_corrected_initial_age() + resident_time;
01414
01415
01416
01417 if (d_max_age >= 0 && current_age > d_max_age) {
01418 DBG(cerr << "Cache....... Max-age validation" << endl);
01419 #if 0
01420 entry->unlock();
01421 #endif
01422 entry->unlock_read_response();
01423 unlock_cache_interface();
01424 return false;
01425 }
01426 if (d_min_fresh >= 0
01427 && entry->get_freshness_lifetime() < current_age + d_min_fresh) {
01428 DBG(cerr << "Cache....... Min-fresh validation" << endl);
01429 #if 0
01430 entry->unlock();
01431 #endif
01432 entry->unlock_read_response();
01433 unlock_cache_interface();
01434 return false;
01435 }
01436
01437 freshness = (entry->get_freshness_lifetime()
01438 + (d_max_stale >= 0 ? d_max_stale : 0) > current_age);
01439 #if 0
01440 entry->unlock();
01441 #endif
01442 entry->unlock_read_response();
01443 unlock_cache_interface();
01444 }
01445 catch (...) {
01446 if (entry) {
01447 #if 0
01448 entry->unlock();
01449 #endif
01450 entry->unlock_read_response();
01451 }
01452 unlock_cache_interface();
01453 throw;
01454 }
01455
01456 return freshness;
01457 }
01458
01486 FILE * HTTPCache::get_cached_response(const string &url,
01487 vector<string> &headers, string &cacheName) {
01488 lock_cache_interface();
01489
01490 FILE *body;
01491 HTTPCacheTable::CacheEntry *entry = 0;
01492
01493 DBG(cerr << "Getting the cached response for " << url << endl);
01494
01495 try {
01496 entry = d_http_cache_table->get_locked_entry_from_cache_table(url);
01497 if (!entry) {
01498 unlock_cache_interface();
01499 return 0;
01500 }
01501
01502 cacheName = entry->get_cachename();
01503 read_metadata(entry->get_cachename(), headers);
01504
01505 DBG(cerr << "Headers just read from cache: " << endl);
01506 DBGN(copy(headers.begin(), headers.end(), ostream_iterator<string>(cerr, "\n")));
01507
01508 body = open_body(entry->get_cachename());
01509
01510 DBG(cerr << "Returning: " << url << " from the cache." << endl);
01511
01512 d_http_cache_table->bind_entry_to_data(entry, body);
01513 }
01514 catch (...) {
01515 if (entry)
01516 #if 0
01517 entry->unlock();
01518 #endif
01519 unlock_cache_interface();
01520 throw;
01521 }
01522
01523 unlock_cache_interface();
01524
01525 return body;
01526 }
01538 FILE *
01539 HTTPCache::get_cached_response(const string &url, vector<string> &headers)
01540 {
01541 string discard_name;
01542 return get_cached_response(url, headers, discard_name);
01543 }
01544
01555 FILE *
01556 HTTPCache::get_cached_response(const string &url)
01557 {
01558 string discard_name;
01559 vector<string> discard_headers;
01560 return get_cached_response(url, discard_headers, discard_name);
01561 }
01562
01575 void
01576 HTTPCache::release_cached_response(FILE *body)
01577 {
01578 lock_cache_interface();
01579
01580 try {
01581 d_http_cache_table->uncouple_entry_from_data(body);
01582 }
01583 catch (...) {
01584 unlock_cache_interface();
01585 throw;
01586 }
01587
01588 unlock_cache_interface();
01589 }
01590
01603 void
01604 HTTPCache::purge_cache()
01605 {
01606 lock_cache_interface();
01607
01608 try {
01609 if (d_http_cache_table->is_locked_read_responses())
01610 throw Error("Attempt to purge the cache with entries in use.");
01611
01612 d_http_cache_table->delete_all_entries();
01613 }
01614 catch (...) {
01615 unlock_cache_interface();
01616 throw;
01617 }
01618
01619 unlock_cache_interface();
01620 }
01621
01622 }