Orcus
parser_token_buffer.hpp
1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2 /*
3  * This Source Code Form is subject to the terms of the Mozilla Public
4  * License, v. 2.0. If a copy of the MPL was not distributed with this
5  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
6  */
7 
8 #ifndef INCLUDED_ORCUS_DETAIL_THREAD_PARSER_TOKEN_BUFFER_HPP
9 #define INCLUDED_ORCUS_DETAIL_THREAD_PARSER_TOKEN_BUFFER_HPP
10 
11 #include "orcus/exception.hpp"
12 
13 #include <mutex>
14 #include <condition_variable>
15 
16 namespace orcus { namespace detail { namespace thread {
17 
22 template<typename _TokensT>
24 {
25  typedef _TokensT tokens_type;
26 
27  mutable std::mutex m_mtx_tokens;
28  std::condition_variable m_cv_tokens_empty;
29  std::condition_variable m_cv_tokens_ready;
30 
31  tokens_type m_tokens; // token buffer used to hand over tokens to the client.
32 
33  size_t m_token_size_threshold;
34  const size_t m_max_token_size;
35 
36  bool m_parsing_progress;
37 
38  bool tokens_empty() const
39  {
40  std::lock_guard<std::mutex> lock(m_mtx_tokens);
41  return m_tokens.empty();
42  }
43 
50  void wait_until_tokens_empty()
51  {
52  std::unique_lock<std::mutex> lock(m_mtx_tokens);
53  while (!m_tokens.empty())
54  m_cv_tokens_empty.wait(lock);
55  }
56 
57 public:
58  parser_token_buffer(size_t min_token_size, size_t max_token_size) :
59  m_token_size_threshold(std::max<size_t>(min_token_size, 1)),
60  m_max_token_size(max_token_size),
61  m_parsing_progress(true)
62  {
63  if (m_token_size_threshold > m_max_token_size)
64  throw invalid_arg_error(
65  "initial token size threshold is already larger than the max token size.");
66  }
67 
76  void check_and_notify(tokens_type& parser_tokens)
77  {
78  if (parser_tokens.size() < m_token_size_threshold)
79  // Still below the threshold.
80  return;
81 
82  if (!tokens_empty())
83  {
84  if (m_token_size_threshold < (m_max_token_size/2))
85  {
86  // Double the threshold and continue to parse.
87  m_token_size_threshold *= 2;
88  return;
89  }
90 
91  // We cannot increase the threshold any more. Wait for the
92  // client to finish.
93  wait_until_tokens_empty();
94  }
95 
96  std::unique_lock<std::mutex> lock(m_mtx_tokens);
97  m_tokens.swap(parser_tokens);
98  lock.unlock();
99  m_cv_tokens_ready.notify_one();
100  }
101 
110  void notify_and_finish(tokens_type& parser_tokens)
111  {
112  // Wait until the client tokens get used up.
113  wait_until_tokens_empty();
114 
115  {
116  std::lock_guard<std::mutex> lock(m_mtx_tokens);
117  m_tokens.swap(parser_tokens);
118  m_parsing_progress = false;
119  }
120  m_cv_tokens_ready.notify_one();
121  }
122 
133  bool next_tokens(tokens_type& tokens)
134  {
135  tokens.clear();
136 
137  // Wait until the parser passes a new set of tokens.
138  std::unique_lock<std::mutex> lock(m_mtx_tokens);
139  while (m_tokens.empty() && m_parsing_progress)
140  m_cv_tokens_ready.wait(lock);
141 
142  // Get the new tokens and notify the parser.
143  tokens.swap(m_tokens);
144  bool parsing_progress = m_parsing_progress; // Make a copy so that lock can be released safely.
145 
146  lock.unlock();
147 
148  m_cv_tokens_empty.notify_one();
149 
150  return parsing_progress;
151  }
152 
159  size_t token_size_threshold() const
160  {
161  if (m_parsing_progress)
162  return 0;
163 
164  return m_token_size_threshold;
165  }
166 };
167 
168 }}}
169 
170 #endif
171 
172 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
Definition: parser_token_buffer.hpp:23
void check_and_notify(tokens_type &parser_tokens)
Definition: parser_token_buffer.hpp:76
size_t token_size_threshold() const
Definition: parser_token_buffer.hpp:159
Definition: tokens.hpp:21
Definition: exception.hpp:29
void notify_and_finish(tokens_type &parser_tokens)
Definition: parser_token_buffer.hpp:110
bool next_tokens(tokens_type &tokens)
Definition: parser_token_buffer.hpp:133