39 using namespace internal;
42 : DataFetcher(), parser_running(false)
44 REQUIRE(samples!=
nullptr,
"Samples cannot be null!\n");
50 StreamingDataFetcher::~StreamingDataFetcher()
56 void StreamingDataFetcher::set_num_samples(
index_t num_samples)
58 m_num_samples=num_samples;
61 void StreamingDataFetcher::shuffle_features()
65 void StreamingDataFetcher::unshuffle_features()
69 void StreamingDataFetcher::use_fold(
index_t i)
73 void StreamingDataFetcher::init_active_subset()
77 index_t StreamingDataFetcher::get_num_samples()
const 82 return m_num_samples*train_test_ratio/(train_test_ratio+1);
84 return m_num_samples/(train_test_ratio+1);
89 void StreamingDataFetcher::start()
91 REQUIRE(get_num_samples()>0,
"Number of samples is not set! It is MANDATORY for streaming features!\n");
92 if (m_block_details.m_full_data || m_block_details.m_blocksize>get_num_samples())
94 SG_SINFO(
"Fetching entire data (%d samples)!\n", get_num_samples());
95 m_block_details.with_blocksize(get_num_samples());
97 m_block_details.m_total_num_blocks=get_num_samples()/m_block_details.m_blocksize;
98 m_block_details.m_next_block_index=0;
101 m_samples->start_parser();
110 auto num_already_fetched=m_block_details.m_next_block_index*m_block_details.m_blocksize;
111 auto num_more_samples=get_num_samples()-num_already_fetched;
112 if (num_more_samples>0)
114 auto num_samples_this_burst=std::min(m_block_details.m_max_num_samples_per_burst, num_more_samples);
115 next_samples=m_samples->get_streamed_features(num_samples_this_burst);
117 m_block_details.m_next_block_index+=m_block_details.m_num_blocks_per_burst;
122 void StreamingDataFetcher::reset()
124 m_block_details.m_next_block_index=0;
125 m_samples->reset_stream();
128 void StreamingDataFetcher::end()
132 m_samples->end_parser();
133 parser_running=
false;
all of classes and functions are contained in the shogun namespace
The class Features is the base class of all feature objects.
Streaming features are features which are used for online algorithms.