58 #include "mime_util.h" 59 #include "media_types.h" 61 #include "HTTPCache.h" 62 #include "HTTPConnect.h" 64 #include "HTTPResponse.h" 65 #include "HTTPCacheResponse.h" 80 int dods_keep_temps = 0;
82 #define CLIENT_ERR_MIN 400 83 #define CLIENT_ERR_MAX 417 84 static const char *http_client_errors[CLIENT_ERR_MAX - CLIENT_ERR_MIN +1] =
87 "Unauthorized: Contact the server administrator.",
89 "Forbidden: Contact the server administrator.",
90 "Not Found: The data source or server could not be found.\n\ 91 Often this means that the OPeNDAP server is missing or needs attention;\n\ 92 Please contact the server administrator.",
93 "Method Not Allowed.",
95 "Proxy Authentication Required.",
100 "Precondition Failed.",
101 "Request Entity Too Large.",
102 "Request URI Too Large.",
103 "Unsupported Media Type.",
104 "Requested Range Not Satisfiable.",
105 "Expectation Failed." 108 #define SERVER_ERR_MIN 500 109 #define SERVER_ERR_MAX 505 110 static const char *http_server_errors[SERVER_ERR_MAX - SERVER_ERR_MIN + 1] =
112 "Internal Server Error.",
115 "Service Unavailable.",
117 "HTTP Version Not Supported." 123 http_status_to_string(
int status)
125 if (status >= CLIENT_ERR_MIN && status <= CLIENT_ERR_MAX)
126 return string(http_client_errors[status - CLIENT_ERR_MIN]);
127 else if (status >= SERVER_ERR_MIN && status <= SERVER_ERR_MAX)
128 return string(http_server_errors[status - SERVER_ERR_MIN]);
130 return string(
"Unknown Error: This indicates a problem with libdap++.\nPlease report this to support@opendap.org.");
134 determine_object_type(
const string &header_value)
139 string::size_type plus = header_value.find(
'+');
141 string type_extension =
"";
142 if (plus != string::npos) {
143 base_type= header_value.substr(0, plus);
144 type_extension = header_value.substr(plus+1);
147 base_type = header_value;
149 if (base_type == DMR_Content_Type
150 || (base_type.find(
"application/") != string::npos
151 && base_type.find(
"dap4.dataset-metadata") != string::npos)) {
152 if (type_extension ==
"xml")
157 else if (base_type == DAP4_DATA_Content_Type
158 || (base_type.find(
"application/") != string::npos
159 && base_type.find(
"dap4.data") != string::npos)) {
162 else if (header_value.find(
"text/html") != string::npos) {
173 class ParseHeader :
public unary_function<const string &, void>
181 ParseHeader() : type(unknown_type), server(
"dods/0.0"), protocol(
"2.0")
184 void operator()(
const string &line)
189 DBG2(cerr << name <<
": " << value << endl);
194 if (type == unknown_type && name ==
"content-type") {
195 type = determine_object_type(value);
197 if (name ==
"content-description" && !(type == dap4_dmr || type == dap4_data || type == dap4_error)) {
203 else if (name ==
"xdods-server" && server ==
"dods/0.0") {
206 else if (name ==
"xopendap-server") {
209 else if (name ==
"xdap") {
212 else if (server ==
"dods/0.0" && name ==
"server") {
215 else if (name ==
"location") {
230 string get_protocol()
235 string get_location() {
256 save_raw_http_headers(
void *ptr,
size_t size,
size_t nmemb,
void *resp_hdrs)
258 DBG2(cerr <<
"Inside the header parser." << endl);
259 vector<string> *hdrs =
static_cast<vector<string> *
>(resp_hdrs);
262 string complete_line;
263 if (nmemb > 1 && *(static_cast<char*>(ptr) + size * (nmemb - 2)) ==
'\r')
264 complete_line.assign(static_cast<char *>(ptr), size * (nmemb - 2));
266 complete_line.assign(static_cast<char *>(ptr), size * (nmemb - 1));
269 if (complete_line !=
"" && complete_line.find(
"HTTP") == string::npos) {
270 DBG(cerr <<
"Header line: " << complete_line << endl);
271 hdrs->push_back(complete_line);
279 curl_debug(CURL *, curl_infotype info,
char *msg,
size_t size,
void *)
281 string message(msg, size);
285 cerr <<
"Text: " << message;
break;
286 case CURLINFO_HEADER_IN:
287 cerr <<
"Header in: " << message;
break;
288 case CURLINFO_HEADER_OUT:
289 cerr <<
"Header out: " << message;
break;
290 case CURLINFO_DATA_IN:
291 cerr <<
"Data in: " << message;
break;
292 case CURLINFO_DATA_OUT:
293 cerr <<
"Data out: " << message;
break;
295 cerr <<
"End: " << message;
break;
296 #ifdef CURLINFO_SSL_DATA_IN 297 case CURLINFO_SSL_DATA_IN:
298 cerr <<
"SSL Data in: " << message;
break;
300 #ifdef CURLINFO_SSL_DATA_OUT 301 case CURLINFO_SSL_DATA_OUT:
302 cerr <<
"SSL Data out: " << message;
break;
305 cerr <<
"Curl info: " << message;
break;
314 HTTPConnect::www_lib_init()
316 d_curl = curl_easy_init();
318 throw InternalErr(__FILE__, __LINE__,
"Could not initialize libcurl.");
324 if (!d_rcr->get_proxy_server_host().empty()) {
325 DBG(cerr <<
"Setting up a proxy server." << endl);
326 DBG(cerr <<
"Proxy host: " << d_rcr->get_proxy_server_host()
328 DBG(cerr <<
"Proxy port: " << d_rcr->get_proxy_server_port()
330 DBG(cerr <<
"Proxy pwd : " << d_rcr->get_proxy_server_userpw()
332 curl_easy_setopt(d_curl, CURLOPT_PROXY,
333 d_rcr->get_proxy_server_host().c_str());
334 curl_easy_setopt(d_curl, CURLOPT_PROXYPORT,
335 d_rcr->get_proxy_server_port());
338 #ifdef CURLOPT_PROXYAUTH 339 curl_easy_setopt(d_curl, CURLOPT_PROXYAUTH, (
long)CURLAUTH_ANY);
343 if (!d_rcr->get_proxy_server_userpw().empty())
344 curl_easy_setopt(d_curl, CURLOPT_PROXYUSERPWD,
345 d_rcr->get_proxy_server_userpw().c_str());
348 curl_easy_setopt(d_curl, CURLOPT_ERRORBUFFER, d_error_buffer);
351 curl_easy_setopt(d_curl, CURLOPT_FAILONERROR, 0);
356 curl_easy_setopt(d_curl, CURLOPT_HTTPAUTH, (
long)CURLAUTH_ANY);
358 curl_easy_setopt(d_curl, CURLOPT_NOPROGRESS, 1);
359 curl_easy_setopt(d_curl, CURLOPT_NOSIGNAL, 1);
360 curl_easy_setopt(d_curl, CURLOPT_HEADERFUNCTION, save_raw_http_headers);
365 curl_easy_setopt(d_curl, CURLOPT_FOLLOWLOCATION, 1);
366 curl_easy_setopt(d_curl, CURLOPT_MAXREDIRS, 5);
369 if (d_rcr->get_validate_ssl() == 0) {
370 curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYPEER, 0);
371 curl_easy_setopt(d_curl, CURLOPT_SSL_VERIFYHOST, 0);
378 if (!d_cookie_jar.empty()) {
379 DBG(cerr <<
"Setting the cookie jar to: " << d_cookie_jar << endl);
380 curl_easy_setopt(d_curl, CURLOPT_COOKIEJAR, d_cookie_jar.c_str());
381 curl_easy_setopt(d_curl, CURLOPT_COOKIESESSION, 1);
385 cerr <<
"Curl version: " << curl_version() << endl;
386 curl_easy_setopt(d_curl, CURLOPT_VERBOSE, 1);
387 curl_easy_setopt(d_curl, CURLOPT_DEBUGFUNCTION, curl_debug);
394 class BuildHeaders :
public unary_function<const string &, void>
396 struct curl_slist *d_cl;
399 BuildHeaders() : d_cl(0)
402 void operator()(
const string &header)
404 DBG(cerr <<
"Adding '" << header.c_str() <<
"' to the header list." 406 d_cl = curl_slist_append(d_cl, header.c_str());
409 struct curl_slist *get_headers()
430 HTTPConnect::read_url(
const string &url, FILE *stream, vector<string> *resp_hdrs,
const vector<string> *headers)
432 curl_easy_setopt(d_curl, CURLOPT_URL, url.c_str());
442 curl_easy_setopt(d_curl, CURLOPT_WRITEDATA, stream);
443 curl_easy_setopt(d_curl, CURLOPT_WRITEFUNCTION, &fwrite);
445 curl_easy_setopt(d_curl, CURLOPT_WRITEDATA, stream);
448 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
449 ostream_iterator<string>(cerr,
"\n")));
451 BuildHeaders req_hdrs;
452 req_hdrs = for_each(d_request_headers.begin(), d_request_headers.end(),
455 req_hdrs = for_each(headers->begin(), headers->end(), req_hdrs);
457 curl_easy_setopt(d_curl, CURLOPT_HTTPHEADER, req_hdrs.get_headers());
460 bool temporary_proxy =
false;
461 if ((temporary_proxy = url_uses_no_proxy_for(url))) {
462 DBG(cerr <<
"Suppress proxy for url: " << url << endl);
463 curl_easy_setopt(d_curl, CURLOPT_PROXY, 0);
466 string::size_type at_sign = url.find(
'@');
470 if (at_sign != url.npos)
471 d_upstring = url.substr(7, at_sign - 7);
473 if (!d_upstring.empty())
474 curl_easy_setopt(d_curl, CURLOPT_USERPWD, d_upstring.c_str());
479 curl_easy_setopt(d_curl, CURLOPT_WRITEHEADER, resp_hdrs);
484 CURLcode res = curl_easy_perform(d_curl);
487 curl_slist_free_all(req_hdrs.get_headers());
488 curl_easy_setopt(d_curl, CURLOPT_HTTPHEADER, 0);
491 if (temporary_proxy && !d_rcr->get_proxy_server_host().empty())
492 curl_easy_setopt(d_curl, CURLOPT_PROXY,
493 d_rcr->get_proxy_server_host().c_str());
496 throw Error(d_error_buffer);
499 res = curl_easy_getinfo(d_curl, CURLINFO_HTTP_CODE, &status);
501 throw Error(d_error_buffer);
504 res = curl_easy_getinfo(d_curl, CURLINFO_CONTENT_TYPE, &ct_ptr);
505 if (res == CURLE_OK && ct_ptr)
506 d_content_type = ct_ptr;
517 HTTPConnect::url_uses_proxy_for(
const string &url)
519 if (d_rcr->is_proxy_for_used()) {
520 Regex host_regex(d_rcr->get_proxy_for_regexp().c_str());
521 int index = 0, matchlen;
522 return host_regex.search(url.c_str(), url.size(), matchlen, index) != -1;
532 HTTPConnect::url_uses_no_proxy_for(
const string &url)
throw()
534 return d_rcr->is_no_proxy_for_used()
535 && url.find(d_rcr->get_no_proxy_for_host()) != string::npos;
546 HTTPConnect::HTTPConnect(
RCReader *rcr,
bool use_cpp) : d_username(
""), d_password(
""), d_cookie_jar(
""),
547 d_dap_client_protocol_major(2), d_dap_client_protocol_minor(0), d_use_cpp_streams(use_cpp)
550 d_accept_deflate = rcr->get_deflate();
557 d_request_headers.push_back(
string(
"Pragma:"));
558 string user_agent = string(
"User-Agent: ") + string(CNAME)
559 + string(
"/") + string(CVER);
560 d_request_headers.push_back(user_agent);
561 if (d_accept_deflate)
562 d_request_headers.push_back(
string(
"Accept-Encoding: deflate, gzip, compress"));
565 if (d_rcr->get_use_cache())
570 DBG2(cerr <<
"Cache object created (" << hex << d_http_cache << dec
576 d_http_cache->
set_max_size(d_rcr->get_max_cache_size());
582 d_cookie_jar = rcr->get_cookie_jar();
587 HTTPConnect::~HTTPConnect()
589 DBG2(cerr <<
"Entering the HTTPConnect dtor" << endl);
591 curl_easy_cleanup(d_curl);
593 DBG2(cerr <<
"Leaving the HTTPConnect dtor" << endl);
597 class HeaderMatch :
public unary_function<const string &, bool> {
598 const string &d_header;
600 HeaderMatch(
const string &header) : d_header(header) {}
601 bool operator()(
const string &arg) {
return arg.find(d_header) == 0; }
620 cout <<
"GET " << url <<
" HTTP/1.0" << endl;
626 stream = caching_fetch_url(url);
629 stream = plain_fetch_url(url);
634 ss <<
"HTTP/1.0 " << stream->get_status() <<
" -" << endl;
635 for (
size_t i = 0; i < stream->get_headers()->size(); i++) {
636 ss << stream->get_headers()->at(i) << endl;
647 if (!d_content_type.empty() && find_if(stream->get_headers()->begin(), stream->get_headers()->end(),
648 HeaderMatch(
"Content-Type:")) == stream->get_headers()->end())
649 stream->get_headers()->push_back(
"Content-Type: " + d_content_type);
651 parser = for_each(stream->get_headers()->begin(), stream->get_headers()->end(), ParseHeader());
654 cout << endl << endl;
658 if (parser.get_location() !=
"" &&
659 url.substr(0,url.find(
"?",0)).compare(parser.get_location().substr(0,url.find(
"?",0))) != 0) {
664 stream->set_type(parser.get_object_type());
666 stream->set_version(parser.get_server());
667 stream->set_protocol(parser.get_protocol());
669 if (d_use_cpp_streams) {
687 get_tempfile_template(
const string &file_template)
694 Regex directory(
"[-a-zA-Z0-9_:\\]*");
699 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
700 goto valid_temp_directory;
703 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
704 goto valid_temp_directory;
709 if (c && directory.match(c.c_str(), c.length()) && (access(c.c_str(), 6) == 0))
710 goto valid_temp_directory;
712 #else // Unix/Linux/OSX has another... 714 Regex directory(
"[-a-zA-Z0-9_/]*");
716 c = getenv(
"TMPDIR");
717 if (directory.match(c.c_str(), c.length()) && (access(c.c_str(), W_OK | R_OK) == 0))
718 goto valid_temp_directory;
723 if (access(P_tmpdir, W_OK | R_OK) == 0) {
725 goto valid_temp_directory;
731 if (directory.match(c.c_str(), c.length()) && (access(c.c_str(), W_OK | R_OK) == 0))
732 goto valid_temp_directory;
739 valid_temp_directory:
742 c +=
"\\" + file_template;
744 c +=
"/" + file_template;
771 string dods_temp = get_tempfile_template((
string)
"dodsXXXXXX");
773 vector<char> pathname(dods_temp.length() + 1);
775 strncpy(&pathname[0], dods_temp.c_str(), dods_temp.length());
777 DBG(cerr <<
"pathanme: " << &pathname[0] <<
" (" << dods_temp.length() + 1 <<
")" << endl);
780 #if defined(WIN32) || defined(TEST_WIN32_TEMPS) 781 stream = fopen(_mktemp(&pathname[0]),
"w+b");
784 int mask = umask(077);
786 throw Error(
"Could not set the file creation mask: " +
string(strerror(errno)));
787 int fd = mkstemp(&pathname[0]);
789 throw Error(
"Could not create a temporary file to store the response: " +
string(strerror(errno)));
791 stream = fdopen(fd,
"w+");
796 throw Error(
"Failed to open a temporary file for the data values (" + dods_temp +
")");
798 dods_temp = &pathname[0];
813 throw InternalErr(__FILE__, __LINE__,
"!FAIL! " + long_to_string(res));
815 res = unlink(name.c_str());
817 throw InternalErr(__FILE__, __LINE__,
"!FAIL! " + long_to_string(res));
842 HTTPConnect::caching_fetch_url(
const string &url)
844 DBG(cerr <<
"Is this URL (" << url <<
") in the cache?... ");
846 vector<string> *headers =
new vector<string>;
851 DBGN(cerr <<
"no; getting response and caching." << endl);
852 delete headers; headers = 0;
853 time_t now = time(0);
854 HTTPResponse *rs = plain_fetch_url(url);
855 d_http_cache->
cache_response(url, now, *(rs->get_headers()), rs->get_stream());
860 DBGN(cerr <<
"yes... ");
863 DBGN(cerr <<
"and it's valid; using cached response." << endl);
864 HTTPCacheResponse *crs =
new HTTPCacheResponse(s, 200, headers, file_name, d_http_cache);
868 DBGN(cerr <<
"but it's not valid; validating... ");
875 time_t now = time(0);
879 http_status = read_url(url, body, headers, &cond_hdrs);
888 switch (http_status) {
890 DBGN(cerr <<
"read a new response; caching." << endl);
893 HTTPResponse *rs =
new HTTPResponse(body, http_status, headers, dods_temp);
899 DBGN(cerr <<
"cached response valid; updating." << endl);
905 HTTPCacheResponse *crs =
new HTTPCacheResponse(hs, 304, headers, file_name, d_http_cache);
911 if (http_status >= 400) {
912 delete headers; headers = 0;
913 string msg =
"Error while reading the URL: ";
916 +=
".\nThe OPeNDAP server returned the following message:\n";
917 msg += http_status_to_string(http_status);
921 delete headers; headers = 0;
922 throw InternalErr(__FILE__, __LINE__,
923 "Bad response from the HTTP server: " + long_to_string(http_status));
930 throw InternalErr(__FILE__, __LINE__,
"Should never get here");
945 HTTPConnect::plain_fetch_url(
const string &url)
947 DBG(cerr <<
"Getting URL: " << url << endl);
950 vector<string> *resp_hdrs =
new vector<string>;
954 status = read_url(url, stream, resp_hdrs);
957 string msg =
"Error while reading the URL: ";
959 msg +=
".\nThe OPeNDAP server returned the following message:\n";
960 msg += http_status_to_string(status);
972 if (d_use_cpp_streams) {
974 fstream *in =
new fstream(dods_temp.c_str(), ios::in|ios::binary);
975 return new HTTPResponse(in, status, resp_hdrs, dods_temp);
980 return new HTTPResponse(stream, status, resp_hdrs, dods_temp);
1000 d_accept_deflate = deflate;
1002 if (d_accept_deflate) {
1003 if (find(d_request_headers.begin(), d_request_headers.end(),
1004 "Accept-Encoding: deflate, gzip, compress") == d_request_headers.end())
1005 d_request_headers.push_back(
string(
"Accept-Encoding: deflate, gzip, compress"));
1006 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
1007 ostream_iterator<string>(cerr,
"\n")));
1010 vector<string>::iterator i;
1011 i = remove_if(d_request_headers.begin(), d_request_headers.end(),
1012 bind2nd(equal_to<string>(),
1013 string(
"Accept-Encoding: deflate, gzip, compress")));
1014 d_request_headers.erase(i, d_request_headers.end());
1030 vector<string>::iterator i;
1031 i = find_if(d_request_headers.begin(), d_request_headers.end(),
1032 HeaderMatch(
"XDAP-Accept:"));
1033 if (i != d_request_headers.end())
1034 d_request_headers.erase(i);
1037 d_dap_client_protocol_major = major;
1038 d_dap_client_protocol_minor = minor;
1039 ostringstream xdap_accept;
1040 xdap_accept <<
"XDAP-Accept: " << major <<
"." << minor;
1042 d_request_headers.push_back(xdap_accept.str());
1044 DBG(copy(d_request_headers.begin(), d_request_headers.end(),
1045 ostream_iterator<string>(cerr,
"\n")));
1073 d_upstring = u +
":" + p;
vector< string > get_conditional_request_headers(const string &url)
void set_cache_enabled(bool mode)
bool is_url_valid(const string &url)
void set_credentials(const string &u, const string &p)
static HTTPCache * instance(const string &cache_root, bool force=false)
void set_max_size(unsigned long size)
FILE * get_cached_response(const string &url, vector< string > &headers, string &cacheName)
ObjectType
The type of object in the stream coming from the data server.
HTTPResponse * fetch_url(const string &url)
A class for software fault reporting.
void parse_mime_header(const string &header, string &name, string &value)
ObjectType get_description_type(const string &value)
void update_response(const string &url, time_t request_time, const vector< string > &headers)
void close_temp(FILE *s, const string &name)
string get_temp_file(FILE *&stream)
bool cache_response(const string &url, time_t request_time, const vector< string > &headers, const FILE *body)
void set_accept_deflate(bool defalte)
void set_always_validate(bool validate)
void set_xdap_protocol(int major, int minor)
void set_default_expiration(int exp_time)
void release_cached_response(FILE *response)
A class for error processing.
void set_expire_ignored(bool mode)
void set_max_entry_size(unsigned long size)