30 #include <uuid/uuid.h>
79 #define CRLF "\r\n" // Change here, expr-test.cc
80 #define FUNCTION_CACHE "/tmp/dap_functions_cache/"
81 #define FUNCTION_CACHE_PREFIX "f"
83 #define FUNCTION_CACHE_SIZE 20000
89 ResponseBuilder::~ResponseBuilder() {
94 void ResponseBuilder::initialize() {
117 DBG(cerr <<
"the FUNCTION_CACHE directory (" <<
FUNCTION_CACHE <<
") exists" << endl);
121 DBG(cerr <<
"the FUNCTION_CACHE directory (" <<
FUNCTION_CACHE <<
") does not exist - not caching" << endl);
128 _setmode(_fileno(stdout), _O_BINARY);
138 string ResponseBuilder::get_ce()
const {
142 void ResponseBuilder::set_ce(
string _ce) {
143 d_ce =
www2id(_ce,
"%",
"%20");
154 string ResponseBuilder::get_dataset_name()
const {
158 void ResponseBuilder::set_dataset_name(
const string ds) {
159 d_dataset =
www2id(ds,
"%",
"%20");
166 void ResponseBuilder::set_timeout(
int t) {
171 int ResponseBuilder::get_timeout()
const {
185 void ResponseBuilder::establish_timeout(ostream &stream)
const {
212 string btp_function_ce =
"";
213 string::size_type pos = 0;
214 DBG(cerr <<
"ce: " << ce << endl);
216 string::size_type first_paren = ce.find(
"(", pos);
217 string::size_type closing_paren = ce.find(
")", pos);
218 while (first_paren != string::npos && closing_paren != string::npos) {
220 string name = ce.substr(pos, first_paren-pos);
221 DBG(cerr <<
"name: " << name << endl);
226 if (!btp_function_ce.empty())
227 btp_function_ce +=
",";
228 btp_function_ce += ce.substr(pos, closing_paren+1-pos);
229 ce.erase(pos, closing_paren+1-pos);
234 pos = closing_paren + 1;
236 if (pos < ce.length() && ce.at(pos) ==
',')
240 first_paren = ce.find(
"(", pos);
241 closing_paren = ce.find(
")", pos);
244 DBG(cerr <<
"Modified constraint: " << ce << endl);
245 DBG(cerr <<
"BTP Function part: " << btp_function_ce << endl);
248 d_btp_func_ce = btp_function_ce;
259 build_cache_file_name(
const string &dataset,
const string &ce)
261 DBG(cerr <<
"build_cache_file_name: dataset: " << dataset <<
", ce: " << ce << endl);
263 string name = dataset +
"#" + ce;
264 string::size_type pos = name.find_first_of(
"/(),\"\'");
265 while (pos != string::npos) {
266 name.replace(pos, 1,
"#", 1);
267 pos = name.find_first_of(
"/()\"\'");
270 DBG(cerr <<
"build_cache_file_name: name: " << name << endl);
276 static bool cached_data_ddx_exists(
const string &cache_file_name)
278 ifstream icache_file(cache_file_name.c_str());
280 return !icache_file.fail() && !icache_file.bad() && !icache_file.eof();
292 bool ResponseBuilder::is_valid(
const string &cache_file_name)
297 off_t entry_size = 0;
298 time_t entry_time = 0;
300 if (stat(cache_file_name.c_str(), &buf) == 0) {
301 entry_size = buf.st_size;
302 entry_time = buf.st_mtime;
311 time_t dataset_time = entry_time;
312 if (stat(d_dataset.c_str(), &buf) == 0) {
313 dataset_time = buf.st_mtime;
321 if (dataset_time > entry_time)
343 DBG(cerr <<
"Found function(s) in CE: " << d_btp_func_ce << endl);
351 string cache_file_name = d_cache->get_cache_file_name(build_cache_file_name(d_dataset, d_btp_func_ce),
false);
358 if (!is_valid(cache_file_name))
359 d_cache->purge_file(cache_file_name);
361 if (d_cache->get_read_lock(cache_file_name, fd)) {
362 DBG(cerr <<
"function ce - cached hit: " << cache_file_name << endl );
363 fdds = get_cached_data_ddx(cache_file_name, &factory);
369 if (d_cache->create_and_lock(cache_file_name, fd)) {
370 DBG(cerr <<
"function ce - caching " << cache_file_name << endl );
377 cache_data_ddx(cache_file_name, *fdds);
382 d_cache->exclusive_to_shared_lock(fd);
387 unsigned long long size = d_cache->update_cache_info(cache_file_name);
388 if (d_cache->cache_too_big(size))
389 d_cache->update_and_purge(cache_file_name);
393 else if (d_cache->get_read_lock(cache_file_name, fd)) {
394 DBG(cerr <<
"function ce - cached hit: " << cache_file_name << endl );
395 fdds = get_cached_data_ddx(cache_file_name, &factory);
398 throw InternalErr(__FILE__, __LINE__,
"Cache error during function invocation.");
402 DBG(cerr <<
"caught exception, unlocking cache and re-throw." << endl );
404 d_cache->unlock_cache();
408 cache_token = cache_file_name;
422 void ResponseBuilder::send_das(ostream &out,
DAS &das,
bool with_mime_headers)
424 if (with_mime_headers)
448 bool with_mime_headers)
451 establish_timeout(out);
455 if (with_mime_headers)
469 if (!d_btp_func_ce.empty()) {
471 DBG(cerr <<
"Found function(s) in CE: " << d_btp_func_ce << endl);
479 string cache_file_name = d_cache->get_cache_file_name(build_cache_file_name(d_dataset, d_btp_func_ce),
false);
486 if (!is_valid(cache_file_name))
487 d_cache->purge_file(cache_file_name);
489 if (d_cache->get_read_lock(cache_file_name, fd)) {
490 DBG(cerr <<
"function ce - cached hit: " << cache_file_name << endl );
491 fdds = get_cached_data_ddx(cache_file_name, &factory);
497 if (d_cache->create_and_lock(cache_file_name, fd)) {
498 DBG(cerr <<
"function ce - caching " << cache_file_name << endl );
505 cache_data_ddx(cache_file_name, *fdds);
510 d_cache->exclusive_to_shared_lock(fd);
515 unsigned long long size = d_cache->update_cache_info(cache_file_name);
516 if (d_cache->cache_too_big(size))
517 d_cache->update_and_purge(cache_file_name);
519 else if (d_cache->get_read_lock(cache_file_name, fd)) {
520 DBG(cerr <<
"function ce - cached hit: " << cache_file_name << endl );
521 fdds = get_cached_data_ddx(cache_file_name, &factory);
524 throw InternalErr(__FILE__, __LINE__,
"Cache error during function invocation.");
528 DBG(cerr <<
"caught exception, unlocking cache and re-throw." << endl );
530 d_cache->unlock_cache();
535 string cache_token =
"";
538 DBG(cerr <<
"Using the cache for the server function CE" << endl);
539 fdds = read_cached_dataset(dds, eval, cache_token);
542 DBG(cerr <<
"Cache not found; (re)calculating" << endl);
547 if (with_mime_headers)
553 d_cache->unlock_and_close(cache_token);
558 DBG(cerr <<
"Simple constraint" << endl);
562 if (with_mime_headers)
589 bool with_mime_headers)
592 if (with_mime_headers)
601 establish_timeout(out);
610 if (!d_btp_func_ce.empty()) {
612 DBG(cerr <<
"Found function(s) in CE: " << d_btp_func_ce << endl);
620 string cache_file_name = d_cache->get_cache_file_name(build_cache_file_name(d_dataset, d_btp_func_ce),
false);
627 if (!is_valid(cache_file_name))
628 d_cache->purge_file(cache_file_name);
630 if (d_cache->get_read_lock(cache_file_name, fd)) {
631 DBG(cerr <<
"function ce - cached hit: " << cache_file_name << endl );
632 fdds = get_cached_data_ddx(cache_file_name, &factory);
637 if (d_cache->create_and_lock(cache_file_name, fd)) {
638 DBG(cerr <<
"function ce - caching " << cache_file_name << endl );
645 cache_data_ddx(cache_file_name, *fdds);
650 d_cache->exclusive_to_shared_lock(fd);
655 unsigned long long size = d_cache->update_cache_info(cache_file_name);
656 if (d_cache->cache_too_big(size))
657 d_cache->update_and_purge(cache_file_name);
659 else if (d_cache->get_read_lock(cache_file_name, fd)) {
660 DBG(cerr <<
"function ce - cached hit: " << cache_file_name << endl );
661 fdds = get_cached_data_ddx(cache_file_name, &factory);
664 throw InternalErr(__FILE__, __LINE__,
"Cache error during function invocation.");
668 DBG(cerr <<
"caught exception, unlocking cache and re-throw." << endl );
670 d_cache->unlock_cache();
674 string cache_token =
"";
678 DBG(cerr <<
"Using the cache for the server function CE" << endl);
679 fdds = read_cached_dataset(dds, eval, cache_token);
682 DBG(cerr <<
"Cache not found; (re)calculating" << endl);
696 if (with_mime_headers)
702 d_cache->unlock_and_close(cache_token);
707 DBG(cerr <<
"Simple constraint" << endl);
711 if (with_mime_headers)
722 DBG(cerr <<
"Inside dataset_constraint" << endl);
732 DBG(cerr <<
"Built stream encoder" << endl);
736 if ((*i)->send_p()) {
737 DBG(cerr <<
"Sending " << (*i)->name() << endl);
742 (*i)->serialize(eval, dds, m, ce_eval);
745 cerr << (*i)->name() <<
": " << m.
get_checksum() << endl;
747 (*i)->serialize(eval, dds, m, ce_eval);
756 void ResponseBuilder::dataset_constraint_ddx(ostream &out,
DDS & dds,
ConstraintEvaluator & eval,
const string &boundary,
const string &start,
bool ce_eval)
const
765 uuid_unparse(uu, &uuid[0]);
767 if (getdomainname(domain, 255) != 0 || strlen(domain) == 0)
768 strncpy(domain,
"opendap.org", 255);
770 string cid = string(&uuid[0]) +
"@" + string(&domain[0]);
785 if ((*i)->send_p()) {
786 DBG(cerr <<
"Sending " << (*i)->name() << endl);
787 (*i)->serialize(eval, dds, m, ce_eval);
815 establish_timeout(data_stream);
825 +
"KB is too large; requests for this user are limited to "
837 if (!d_btp_func_ce.empty()) {
838 DBG(cerr <<
"Found function(s) in CE: " << d_btp_func_ce << endl);
846 string cache_file_name = d_cache->get_cache_file_name(build_cache_file_name(d_dataset, d_btp_func_ce),
false);
853 if (!is_valid(cache_file_name))
854 d_cache->purge_file(cache_file_name);
856 if (d_cache->get_read_lock(cache_file_name, fd)) {
857 DBG(cerr <<
"function ce - cached hit: " << cache_file_name << endl );
858 fdds = get_cached_data_ddx(cache_file_name, &factory);
863 if (d_cache->create_and_lock(cache_file_name, fd)) {
864 DBG(cerr <<
"function ce - caching " << cache_file_name << endl );
873 cache_data_ddx(cache_file_name, *fdds);
878 d_cache->exclusive_to_shared_lock(fd);
883 unsigned long long size = d_cache->update_cache_info(cache_file_name);
884 if (d_cache->cache_too_big(size))
885 d_cache->update_and_purge(cache_file_name);
887 else if (d_cache->get_read_lock(cache_file_name, fd)) {
888 DBG(cerr <<
"function ce - cached hit: " << cache_file_name << endl );
889 fdds = get_cached_data_ddx(cache_file_name, &factory);
892 throw InternalErr(__FILE__, __LINE__,
"Cache error during function invocation.");
896 DBG(cerr <<
"caught exception, unlocking cache and re-throw." << endl );
898 d_cache->unlock_cache();
906 if (cached_data_ddx_exists(cache_file_name)) {
907 fdds = get_cached_data_ddx(cache_file_name, &factory);
910 DBG(cerr <<
"Reading cache for " << d_dataset +
"?" + d_btp_func_ce << endl);
913 fdds =
new DDS(&factory);
922 throw Error(
"The input source: " + cache_file_name +
" could not be opened");
930 for( ; i != e; i++ ) {
937 Ancillary::read_ancillary_das( *das, d_dataset ) ;
945 cache_data_ddx(cache_file_name, *fdds);
948 ofstream ocache_file(cache_file_name.c_str());
950 DBG(cerr <<
"Caching " << d_dataset +
"?" + d_btp_func_ce << endl);
951 cache_data_ddx(ocache_file, *fdds);
956 string cache_token =
"";
960 DBG(cerr <<
"Using the cache for the server function CE" << endl);
961 fdds = read_cached_dataset(dds, eval, cache_token);
964 DBG(cerr <<
"Cache not found; (re)calculating" << endl);
969 DBG(cerr <<
"Intermediate DDS: " << endl);
972 DBG(cerr <<
"Parsing remaining CE: " << d_ce << endl);
987 +
"KB is too large; requests for this user are limited to "
992 if (with_mime_headers)
995 DBG(cerr <<
"About to call dataset_constraint" << endl);
996 dataset_constraint(data_stream, *fdds, eval,
false);
999 d_cache->unlock_and_close(cache_token);
1004 DBG(cerr <<
"Simple constraint" << endl);
1012 +
"KB is too large; requests for this user are limited to "
1017 if (with_mime_headers)
1020 dataset_constraint(data_stream, dds, eval);
1029 if (with_mime_headers)
1032 dataset_constraint(data_stream, *fdds, eval,
false);
1036 if (with_mime_headers)
1039 dataset_constraint(data_stream, dds, eval);
1043 data_stream << flush;
1067 throw Error(
"Function calls can only be used with data requests. To see the structure of the underlying data source, reissue the URL without the function.");
1069 if (with_mime_headers)
1095 void ResponseBuilder::send_data_ddx(ostream & data_stream,
DDS & dds,
ConstraintEvaluator & eval,
const string &start,
const string &boundary,
bool with_mime_headers)
1099 establish_timeout(data_stream);
1117 if (with_mime_headers)
1119 data_stream << flush;
1121 dataset_constraint(data_stream, *fdds, eval,
false);
1125 if (with_mime_headers)
1127 data_stream << flush;
1128 dataset_constraint_ddx(data_stream, dds, eval, boundary, start);
1131 data_stream << flush;
1133 if (with_mime_headers)
1134 data_stream <<
CRLF <<
"--" << boundary <<
"--" <<
CRLF;
1149 void ResponseBuilder::cache_data_ddx(
const string &cache_file_name,
DDS &dds)
1151 DBG(cerr <<
"Caching " << d_dataset +
"?" + d_btp_func_ce << endl);
1153 ofstream data_stream(cache_file_name.c_str());
1156 string start=
"dataddx_cache_start", boundary=
"dataddx_cache_boundary";
1161 data_stream << flush;
1173 dataset_constraint_ddx(data_stream, dds, eval, boundary, start);
1174 data_stream << flush;
1176 data_stream <<
CRLF <<
"--" << boundary <<
"--" <<
CRLF;
1177 data_stream.close();
1198 void ResponseBuilder::read_data_from_cache(FILE *data,
DDS *fdds)
1202 while (!mime.empty()) {
1204 string header, value;
1215 DBG(cerr <<
"MPM Boundary: " << boundary << endl);
1222 ddx_parser.intern_stream(data, fdds, data_cid, boundary);
1226 DBG(cerr <<
"Data CID: " << data_cid << endl);
1236 (*i)->deserialize(um, fdds);
1244 ResponseBuilder::get_cached_data_ddx(
const string &cache_file_name,
BaseTypeFactory *factory)
1246 DBG(cerr <<
"Reading cache for " << d_dataset +
"?" + d_btp_func_ce << endl);
1248 DDS *fdds =
new DDS(factory);
1257 throw Error(
"The input source: " + cache_file_name +
" could not be opened");
1263 FILE *data = fopen( cache_file_name.c_str(),
"r" );
1264 read_data_from_cache(data, fdds);
1272 for( ; i != e; i++ ) {
1283 Ancillary::read_ancillary_das( *das, d_dataset ) ;
1289 static const char *descrip[] = {
"unknown",
"dods_das",
"dods_dds",
"dods_data",
"dods_error",
"web_error",
"dap4-ddx",
"dap4-data",
"dap4-error",
"dap4-data-ddx",
"dods_ddx" };
1291 static const char *encoding[] = {
"unknown",
"deflate",
"x-plain",
"gzip",
"binary" };
1307 strm <<
"HTTP/1.0 200 OK" <<
CRLF;
1309 strm <<
"XDODS-Server: " <<
DVR <<
CRLF;
1310 strm <<
"XOPeNDAP-Server: " <<
DVR <<
CRLF;
1313 strm <<
"XDAP: " << d_default_protocol <<
CRLF;
1315 strm <<
"XDAP: " << protocol <<
CRLF;
1317 const time_t t = time(0);
1320 strm <<
"Last-Modified: ";
1321 if (last_modified > 0)
1327 strm <<
"Content-Type: text/xml" <<
CRLF;
1329 strm <<
"Content-Type: text/plain" <<
CRLF;
1333 strm <<
"Content-Description: " << descrip[type] <<
CRLF;
1335 strm <<
"Cache-Control: no-cache" <<
CRLF;
1339 strm <<
"Content-Encoding: " << encoding[enc] <<
CRLF;
1355 strm <<
"HTTP/1.0 200 OK" <<
CRLF;
1357 strm <<
"XDODS-Server: " <<
DVR <<
CRLF;
1358 strm <<
"XOPeNDAP-Server: " <<
DVR <<
CRLF;
1361 strm <<
"XDAP: " << d_default_protocol <<
CRLF;
1363 strm <<
"XDAP: " << protocol <<
CRLF;
1365 const time_t t = time(0);
1368 strm <<
"Last-Modified: ";
1369 if (last_modified > 0)
1374 strm <<
"Content-type: text/html" <<
CRLF;
1376 strm <<
"Content-Description: " << descrip[type] <<
CRLF;
1378 strm <<
"Cache-Control: no-cache" <<
CRLF;
1382 strm <<
"Content-Encoding: " << encoding[enc] <<
CRLF;
1401 strm <<
"HTTP/1.0 200 OK" <<
CRLF;
1403 strm <<
"XDODS-Server: " <<
DVR <<
CRLF;
1404 strm <<
"XOPeNDAP-Server: " <<
DVR <<
CRLF;
1407 strm <<
"XDAP: " << d_default_protocol <<
CRLF;
1409 strm <<
"XDAP: " << protocol <<
CRLF;
1411 const time_t t = time(0);
1414 strm <<
"Last-Modified: ";
1415 if (last_modified > 0)
1420 strm <<
"Content-Type: application/octet-stream" <<
CRLF;
1421 strm <<
"Content-Description: " << descrip[type] <<
CRLF;
1423 strm <<
"Content-Encoding: " << encoding[enc] <<
CRLF;
1430 strm <<
"HTTP/1.0 200 OK" <<
CRLF;
1432 strm <<
"XDODS-Server: " <<
DVR <<
CRLF;
1433 strm <<
"XOPeNDAP-Server: " <<
DVR <<
CRLF;
1436 strm <<
"XDAP: " << d_default_protocol <<
CRLF;
1438 strm <<
"XDAP: " << protocol <<
CRLF;
1440 const time_t t = time(0);
1443 strm <<
"Last-Modified: ";
1444 if (last_modified > 0)
1449 strm <<
"Content-Type: Multipart/Related; boundary=" << boundary <<
"; start=\"<" << start <<
">\"; type=\"Text/xml\"" <<
CRLF;
1451 strm <<
"Content-Description: " << descrip[type] <<
CRLF;
1453 strm <<
"Content-Encoding: " << encoding[enc] <<
CRLF;
1460 strm <<
"--" << boundary <<
CRLF;
1461 strm <<
"Content-Type: Text/xml; charset=iso-8859-1" <<
CRLF;
1462 strm <<
"Content-Id: <" << cid <<
">" <<
CRLF;
1463 strm <<
"Content-Description: " << descrip[type] <<
CRLF;
1465 strm <<
"Content-Encoding: " << encoding[enc] <<
CRLF;
1472 strm <<
"--" << boundary <<
CRLF;
1473 strm <<
"Content-Type: application/octet-stream" <<
CRLF;
1474 strm <<
"Content-Id: <" << cid <<
">" <<
CRLF;
1475 strm <<
"Content-Description: " << descrip[type] <<
CRLF;
1477 strm <<
"Content-Encoding: " << encoding[enc] <<
CRLF;
1490 strm <<
"HTTP/1.0 " << code <<
" " << reason.c_str() <<
CRLF;
1492 strm <<
"XDODS-Server: " <<
DVR <<
CRLF;
1493 strm <<
"XOPeNDAP-Server: " <<
DVR <<
CRLF;
1496 strm <<
"XDAP: " << d_default_protocol <<
CRLF;
1498 strm <<
"XDAP: " << protocol <<
CRLF;
1500 const time_t t = time(0);
1502 strm <<
"Cache-Control: no-cache" <<
CRLF;