#include <data_reader_imagenet.hpp>

Inheritance diagram for lbann::imagenet_reader:

Collaboration diagram for lbann::imagenet_reader:

Public Member Functions
	imagenet_reader (bool shuffle=true)

	imagenet_reader (const imagenet_reader &)=default

imagenet_reader &	operator= (const imagenet_reader &)=default

	~imagenet_reader () override

imagenet_reader *	copy () const override

std::string	get_type () const override

Public Member Functions inherited from lbann::image_data_reader
	image_data_reader (bool shuffle=true)

	image_data_reader (const image_data_reader &)

image_data_reader &	operator= (const image_data_reader &)

virtual void	set_input_params (const int width=0, const int height=0, const int num_ch=0, const int num_labels=0)

void	load () override

void	setup (int num_io_threads, observer_ptr< thread_pool > io_thread_pool) override

int	get_num_labels () const override
	Return the number of labels (classes) in this dataset. More...

virtual int	get_image_width () const

virtual int	get_image_height () const

virtual int	get_image_num_channels () const

int	get_linearized_data_size () const override
	Get the total number of channel values in a sample of image(s). More...

int	get_linearized_label_size () const override
	Get the linearized size (i.e. number of elements) in a label. More...

const std::vector< El::Int >	get_data_dims () const override
	Get the dimensions of the data. More...

const sample_list_t &	get_sample_list () const
	Allow read-only access to the entire sample list. More...

sample_t	get_sample (const size_t idx) const

void	do_preload_data_store () override

void	load_conduit_node_from_file (int data_id, conduit::Node &node)

Public Member Functions inherited from lbann::generic_data_reader
	generic_data_reader (bool shuffle=true)

	generic_data_reader (const generic_data_reader &)=default

generic_data_reader &	operator= (const generic_data_reader &)=default

virtual	~generic_data_reader ()

template<class Archive >
void	serialize (Archive &ar)

void	set_comm (lbann_comm *comm)
	set the comm object More...

lbann_comm *	get_comm () const
	returns a (possibly nullptr) to comm More...

virtual bool	has_conduit_output ()

void	set_file_dir (std::string s)

void	set_local_file_dir (std::string s)

void	set_max_files_to_load (size_t n)

std::string	get_file_dir () const

std::string	get_local_file_dir () const

void	set_data_sample_list (std::string s)

std::string	get_data_sample_list () const

void	keep_sample_order (bool same_order=false)

void	set_data_filename (std::string s)

std::string	get_data_filename () const

void	set_label_filename (std::string s)

std::string	get_label_filename () const

void	set_shuffle (bool b)

bool	is_shuffled () const

void	set_shuffled_indices (const std::vector< int > &indices)

const std::vector< int > &	get_shuffled_indices () const

void	set_first_n (int n)

void	set_absolute_sample_count (size_t s)

void	set_use_fraction (double s)

virtual void	set_execution_mode_split_fraction (execution_mode m, double s)

virtual void	set_role (std::string role)

std::string	get_role () const

int	fetch (std::map< data_field_type, CPUMat *> &input_buffers, El::Matrix< El::Int > &indices_fetched, size_t mb_size)
	Fetch a mini-batch worth of data, including samples, labels, responses (as appropriate) More...

int	fetch (std::vector< conduit::Node > &samples, El::Matrix< El::Int > &indices_fetched, size_t mb_size)

virtual bool	has_data_field (data_field_type data_field) const
	Check to see if the data reader supports this specific data field. More...

virtual bool	has_labels () const

virtual bool	has_responses () const

void	set_has_data_field (data_field_type const data_field, const bool b)
	Whether or not a data reader has a data field. More...

virtual void	set_has_labels (const bool b)
	Whether or not a data reader has labels. More...

virtual void	set_has_responses (const bool b)
	Whether or not a data reader has a response field. More...

void	start_data_store_mini_batch_exchange ()

void	finish_data_store_mini_batch_exchange ()

virtual bool	update (bool is_active_reader)

virtual int	get_num_responses () const
	Return the number of responses in this dataset. More...

virtual int	get_linearized_response_size () const
	Get the linearized size (i.e. number of elements) in a response. More...

virtual int	get_linearized_size (data_field_type const &data_field) const
	get the linearized size of what is identified by desc. More...

virtual std::vector< El::Int >	get_slice_points (const slice_points_mode var_category, bool &is_supported)

virtual bool	position_valid () const
	True if the data reader's current position is valid. More...

virtual bool	position_is_overrun () const

bool	at_new_epoch () const
	True if the data reader is at the start of an epoch. More...

void	set_mini_batch_size (const int s)
	Set the mini batch size. More...

int	get_mini_batch_size () const
	Get the mini batch size. More...

int	get_loaded_mini_batch_size () const
	Get the loaded mini-batch size. More...

int	get_current_mini_batch_size () const
	Get the current mini-batch size. More...

int	get_mini_batch_max () const
	Return the full mini_batch_size. More...

void	set_stride_to_next_mini_batch (const int s)
	Set the mini batch stride. More...

int	get_stride_to_next_mini_batch () const
	Return the mini batch stride. More...

void	set_sample_stride (const int s)
	Set the sample stride. More...

int	get_sample_stride () const
	Return the sample stride. More...

void	set_iteration_stride (const int s)
	Set the iteration stride. More...

int	get_iteration_stride () const
	Return the iteration stride. More...

virtual void	set_base_offset (const int s)
	Return the base offset. More...

int	get_base_offset () const
	Return the base offset. More...

void	set_last_mini_batch_size (const int s)
	Set the last mini batch size. More...

int	get_last_mini_batch_size () const
	Return the last mini batch size. More...

void	set_stride_to_last_mini_batch (const int s)
	Set the last mini batch stride. More...

int	get_stride_to_last_mini_batch () const
	Return the last mini batch stride. More...

void	set_num_parallel_readers (const int s)
	Set the number of parallel readers per model. More...

int	get_num_parallel_readers () const
	Return the number of parallel readers per model. More...

virtual void	set_reset_mini_batch_index (const int s)
	Set the starting mini-batch index for the epoch. More...

int	get_reset_mini_batch_index () const
	Return the starting mini-batch index for the epoch. More...

int	get_loaded_mini_batch_index () const
	Return the current mini-batch index for the epoch. More...

int	get_current_mini_batch_index () const
	Return the current mini-batch index for the epoch. More...

void	set_initial_position ()
	Set the current position based on the base and model offsets. More...

int	get_position () const
	Get the current position in the data reader. More...

int	get_next_position () const
	Get the next position in the data reader. More...

int *	get_indices ()
	Get a pointer to the start of the shuffled indices. More...

virtual int	get_num_data () const
	Get the number of samples in this dataset. More...

int	get_num_unused_data (execution_mode m) const
	Get the number of unused samples in this dataset. More...

int *	get_unused_data (execution_mode m)
	Get a pointer to the start of the unused sample indices. More...

const std::vector< int > &	get_unused_indices (execution_mode m)

void	set_num_iterations_per_epoch (int num_iterations_per_epoch)
	Set the number of iterations in each epoch. More...

int	get_num_iterations_per_epoch () const
	Get the number of iterations in each epoch. More...

int	get_current_step_in_epoch () const

void	resize_shuffled_indices ()

void	select_subset_of_data ()

virtual void	use_unused_index_set (execution_mode m)

virtual bool	has_list_per_model () const
	Does the data reader have a unique sample list per model. More...

virtual bool	has_list_per_trainer () const
	Does the data reader have a unique sample list per trainer. More...

bool	save_to_checkpoint_shared (persist &p, execution_mode mode)
	Given directory to store checkpoint files, write state to file and add to number of bytes written. More...

bool	load_from_checkpoint_shared (persist &p, execution_mode mode)
	Given directory to store checkpoint files, read state from file and add to number of bytes read. More...

bool	save_to_checkpoint_distributed (persist &p, execution_mode mode)

bool	load_from_checkpoint_distributed (persist &p, execution_mode mode)
	Given directory to store checkpoint files, read state from file and add to number of bytes read. More...

const data_store_conduit &	get_data_store () const
	returns a const ref to the data store More...

data_store_conduit &	get_data_store ()
	returns a non-const ref to the data store More...

data_store_conduit *	get_data_store_ptr () const

void	setup_data_store (int mini_batch_size)

void	instantiate_data_store ()

virtual void	preload_data_store ()

void	set_gan_labelling (bool has_gan_labelling)

void	set_gan_label_value (int gan_label_value)

void	set_data_store (data_store_conduit *g)
	support of data store functionality More...

virtual bool	data_store_active () const

virtual bool	priming_data_store () const

virtual void	post_update ()

void	set_transform_pipeline (transform::transform_pipeline &&tp)

void	print_get_methods (const std::string filename)
	Print the return values from various get_X methods to file. More...

size_t	get_num_indices_to_use () const

void	set_use_data_store (bool s)

Protected Member Functions
void	set_defaults () override

virtual CPUMat	create_datum_view (CPUMat &X, const int mb_idx) const

bool	fetch_datum (CPUMat &X, int data_id, int mb_idx) override

Protected Member Functions inherited from lbann::image_data_reader
void	copy_members (const image_data_reader &rhs)

bool	fetch_label (Mat &Y, int data_id, int mb_idx) override

void	set_linearized_image_size ()

void	dump_sample_label_list (const std::string &dump_file_name)

void	load_list_of_samples (const std::string filename)
	Rely on pre-determined list of samples. More...

void	load_list_of_samples_from_archive (const std::string &sample_list_archive)
	Load the sample list from a serialized archive from another rank. More...

void	gen_list_of_samples ()

void	load_labels (std::vector< char > &preloaded_buffer)
	Load the labels for samples. More...

void	read_labels (std::istream &istrm)
	Read the labels from an open input stream. More...

size_t	determine_num_of_samples (std::istream &istrm) const
	Return the number of lines in the input stream. More...

bool	load_conduit_nodes_from_file (const std::unordered_set< int > &data_ids)

Protected Member Functions inherited from lbann::generic_data_reader
size_t	get_absolute_sample_count () const

double	get_use_fraction () const

double	get_execution_mode_split_fraction (execution_mode m) const

virtual bool	fetch_data_block (std::map< data_field_type, CPUMat *> &input_buffers, El::Int block_offset, El::Int block_stride, El::Int mb_size, El::Matrix< El::Int > &indices_fetched)

bool	fetch_data_block_conduit (std::vector< conduit::Node > &samples, El::Int block_offset, El::Int block_stride, El::Int mb_size, El::Matrix< El::Int > &indices_fetched)

virtual bool	fetch_data_field (data_field_type data_field, CPUMat &Y, int data_id, int mb_idx)
	Called by fetch_data, fetch_label, fetch_response. More...

virtual bool	fetch_conduit_node (conduit::Node &sample, int data_id)

virtual bool	fetch_response (CPUMat &Y, int data_id, int mb_idx)

CPUMat	create_datum_view (CPUMat &X, const int mb_idx)

virtual void	preprocess_data_source (int tid)

virtual void	postprocess_data_source (int tid)

virtual void	shuffle_indices ()
	Shuffle indices (uses the data_seq_generator) More...

virtual void	shuffle_indices (rng_gen &gen)
	Shuffle indices and profide a random number generator. More...

void	error_check_counts () const

Additional Inherited Members
Public Types inherited from lbann::image_data_reader
using	img_src_t = std::string

using	label_t = int

using	sample_t = std::pair< img_src_t, label_t >

using	sample_name_t = img_src_t

using	sample_list_t = sample_list< sample_name_t >

using	sample_idx_t = sample_list_t::sample_idx_t

using	labels_t = std::vector< label_t >

Public Types inherited from lbann::generic_data_reader
using	unused_index_map_t = std::map< execution_mode, std::vector< int > >

Public Attributes inherited from lbann::generic_data_reader
int	m_mini_batch_size

int	m_current_pos

int	m_stride_to_next_mini_batch

int	m_base_offset

int	m_sample_stride

int	m_iteration_stride
	Stride used by parallel data readers within the model. More...

std::vector< int >	m_shuffled_indices

unused_index_map_t	m_unused_indices
	Record of the indicies that are not being used for training. More...

int	m_last_mini_batch_size

int	m_stride_to_last_mini_batch

int	m_reset_mini_batch_index
	The index at which this data reader starts its epoch. More...

int	m_loaded_mini_batch_idx
	The index of the current mini-batch that has been loaded. More...

int	m_current_mini_batch_idx

int	m_num_iterations_per_epoch

int	m_num_parallel_readers
	How many iterations all readers will execute. More...

size_t	m_max_files_to_load
	How many parallel readers are being used. More...

std::string	m_file_dir

std::string	m_local_file_dir

std::string	m_data_sample_list

std::string	m_data_fn

std::string	m_label_fn

bool	m_shuffle

size_t	m_absolute_sample_count

std::map< execution_mode, double >	m_execution_mode_split_fraction

double	m_use_fraction

int	m_first_n

std::string	m_role

Protected Attributes inherited from lbann::image_data_reader
std::string	m_image_dir
	where images are stored More...

int	m_image_width
	image width More...

int	m_image_height
	image height More...

int	m_image_num_channels
	number of image channels More...

int	m_image_linearized_size
	linearized image size More...

int	m_num_labels
	number of labels More...

sample_list_t	m_sample_list

labels_t	m_labels

Protected Attributes inherited from lbann::generic_data_reader
bool	m_verbose = false

std::unordered_set< int >	m_using_random_node

data_store_conduit *	m_data_store

lbann_comm *	m_comm

bool	m_use_data_store = false

std::map< data_field_type, bool >	m_supported_input_types
	Holds a true value for each input data type that is supported. Use an ordered map so that checkpoints are stable. More...

bool	m_gan_labelling

int	m_gan_label_value

observer_ptr< thread_pool >	m_io_thread_pool

bool	m_keep_sample_order

transform::transform_pipeline	m_transform_pipeline

bool	m_issue_warning

Detailed Description

Definition at line 35 of file data_reader_imagenet.hpp.

Constructor & Destructor Documentation

◆ imagenet_reader() [1/2]

lbann::imagenet_reader::imagenet_reader ( bool shuffle = true )

Here is the caller graph for this function:

◆ imagenet_reader() [2/2]

lbann::imagenet_reader::imagenet_reader ( const imagenet_reader & )

default

◆ ~imagenet_reader()

lbann::imagenet_reader::~imagenet_reader ( )

override

Member Function Documentation

◆ copy()

imagenet_reader* lbann::imagenet_reader::copy ( ) const

inlineoverridevirtual

Implements lbann::generic_data_reader.

Definition at line 43 of file data_reader_imagenet.hpp.

Here is the call graph for this function:

◆ create_datum_view()

virtual CPUMat lbann::imagenet_reader::create_datum_view	(	CPUMat &	X,
		const int	mb_idx
	)		const

protectedvirtual

Here is the caller graph for this function:

◆ fetch_datum()

bool lbann::imagenet_reader::fetch_datum	(	CPUMat &	X,
		int	data_id,
		int	mb_idx
	)

overrideprotectedvirtual

Fetch a single sample into a matrix.

Parameters

X	The matrix to load data into.
data_id	The index of the datum to fetch.
mb_idx	The index within the mini-batch.

Reimplemented from lbann::generic_data_reader.

Here is the caller graph for this function:

◆ get_type()

std::string lbann::imagenet_reader::get_type ( ) const

inlineoverridevirtual

Return this data_reader's type

Implements lbann::generic_data_reader.

Definition at line 45 of file data_reader_imagenet.hpp.

Here is the call graph for this function:

◆ operator=()

imagenet_reader& lbann::imagenet_reader::operator= ( const imagenet_reader & )

default

◆ set_defaults()

void lbann::imagenet_reader::set_defaults ( )

overrideprotectedvirtual

Set the default values for the width, the height, the number of channels, and the number of labels of an image

Reimplemented from lbann::image_data_reader.

Here is the caller graph for this function:

The documentation for this class was generated from the following file:

data_reader_imagenet.hpp

Public Member Functions

Protected Member Functions

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ imagenet_reader() [1/2]

◆ imagenet_reader() [2/2]

◆ ~imagenet_reader()

Member Function Documentation

◆ copy()

◆ create_datum_view()

◆ fetch_datum()

◆ get_type()

◆ operator=()

◆ set_defaults()