#include <data_reader_jag_conduit.hpp>

Inheritance diagram for lbann::data_reader_jag_conduit:

Collaboration diagram for lbann::data_reader_jag_conduit:

Public Types
enum	variable_t { Undefined = 0, JAG_Image, JAG_Scalar, JAG_Input }

using	ch_t = float
	jag output image channel type More...

using	conduit_ch_t = conduit::float32_array
	conduit type for ch_t array wrapper More...

using	scalar_t = double
	jag scalar output type More...

using	input_t = double

using	sample_locator_t = std::pair< std::string, hid_t >

using	sample_map_t = std::vector< sample_locator_t >
	valid sample map type More...

using	sample_name_t = std::string

using	sample_list_t = sample_list_hdf5< sample_name_t >

using	file_handle_t = sample_list_t::file_handle_t

using	sample_file_id_t = sample_list_t::sample_file_id_t

using	sample_t = std::pair< sample_file_id_t, sample_name_t >

using	linear_transform_t = std::pair< double, double >
	linear transform on X defined as: first * X + second => X' More...

using	TypeID = conduit::DataType::TypeID

using	prefix_t = std::pair< std::string, size_t >

Public Types inherited from lbann::generic_data_reader
using	unused_index_map_t = std::map< execution_mode, std::vector< int > >

Public Member Functions
	data_reader_jag_conduit (bool shuffle=true)

	data_reader_jag_conduit (const data_reader_jag_conduit &)

data_reader_jag_conduit &	operator= (const data_reader_jag_conduit &)

	~data_reader_jag_conduit () override

data_reader_jag_conduit *	copy () const override

void	setup (int num_io_threads, observer_ptr< thread_pool > io_thread_pool) override

std::string	get_type () const override

void	set_independent_variable_type (const std::vector< std::vector< variable_t >> &independent)
	Choose which data to use for independent variable. More...

void	set_dependent_variable_type (const std::vector< std::vector< variable_t >> &dependent)
	Choose which data to use for dependent variable. More...

std::vector< variable_t >	get_independent_variable_type () const
	Tell which data to use for independent variable. More...

std::vector< variable_t >	get_dependent_variable_type () const
	Tell which data to use for dependent variable. More...

void	set_output_scalar_prefix (const std::string &prefix)
	Set the common prefix path for any output scalar fields stored. More...

void	set_output_image_prefix (const std::string &prefix)
	Set the common prefix path for any output images stored. More...

void	set_input_prefix (const std::string &prefix)
	Set the common prefix path for any input variables stored. More...

void	set_image_dims (const int width, const int height, const int ch=1)
	Set the image dimension. More...

void	set_image_choices (const std::vector< std::string > image_keys)
	Choose images to use. e.g. by measurement views and time indices. More...

const std::vector< std::string > &	get_image_choices () const
	Report the image choices. More...

void	add_scalar_filter (const std::string &key)
	Add a scalar key to filter out. More...

void	add_scalar_prefix_filter (const prefix_t &p)
	Add a scalar key prefix to filter out. More...

void	add_input_filter (const std::string &key)
	Add an input key to filter out. More...

void	add_input_prefix_filter (const prefix_t &p)
	Add an input key prefix to filter out. More...

void	set_scalar_choices (const std::vector< std::string > &keys)
	Select the set of scalar output variables to use. More...

void	set_all_scalar_choices ()
	Set to use the entire set of scalar outputs. More...

const std::vector< std::string > &	get_scalar_choices () const
	Report the selected scalar outputs. More...

void	set_input_choices (const std::vector< std::string > &keys)
	Select the set of simulation input parameters to use. More...

void	set_all_input_choices ()
	Set to use the entire set of simulation input parameters. More...

const std::vector< std::string > &	get_input_choices () const
	Report the selected simulation input parameters. More...

void	load () override
	Load data and do data reader's chores. More...

void	set_local_id (const std::string role)
	Set the id of this local instance. More...

int	get_local_id (const std::string role) const
	Get the id of this local instance. More...

void	set_list_per_trainer (bool flag)
	Set the set of open hdf5 data files. More...

void	set_list_per_model (bool flag)
	Set every reader instances in a model to have an independent sample list. More...

bool	has_list_per_model () const override
	Does the data reader have a unique sample list per model. More...

bool	has_list_per_trainer () const override
	Does the data reader have a unique sample list per trainer. More...

unsigned int	get_num_img_srcs () const
	Return the number of measurement views. More...

size_t	get_linearized_image_size () const
	Return the linearized size of an image. More...

size_t	get_linearized_1ch_image_size () const
	Return the linearized size of a single channel image. More...

size_t	get_linearized_scalar_size () const
	Return the linearized size of scalar outputs. More...

size_t	get_linearized_input_size () const
	Return the linearized size of inputs. More...

int	get_linearized_data_size () const override
	Return the total linearized size of data. More...

int	get_linearized_response_size () const override
	Return the total linearized size of response. More...

std::vector< size_t >	get_linearized_data_sizes () const
	Return the per-source linearized sizes of composite data. More...

std::vector< size_t >	get_linearized_response_sizes () const
	Return the per-source linearized sizes of composite response. More...

const std::vector< El::Int >	get_data_dims () const override
	Return the dimension of data. More...

int	get_num_data () const override
	Get the number of samples in this dataset. More...

int	get_num_labels () const override
	Return the number of labels (classes) in this dataset. More...

int	get_linearized_label_size () const override
	Get the linearized size (i.e. number of elements) in a label. More...

std::vector< El::Int >	get_slice_points (const slice_points_mode var_category, bool &is_supported) override

void	set_split_image_channels ()

void	unset_split_image_channels ()

bool	check_split_image_channels () const

std::string	get_description () const
	Show the description. More...

std::vector< scalar_t >	get_scalars (const size_t i, conduit::Node &sample) const
	Return the scalar simulation output data of the i-th sample. More...

std::vector< input_t >	get_inputs (const size_t i, conduit::Node &sample) const
	Return the simulation input parameters of the i-th sample. More...

void	setup_data_store (int mini_batch_size)

void	print_schema (const size_t i) const
	print the schema of the specific sample identified by a given id More...

void	clear_image_normalization_params ()

void	clear_scalar_normalization_params ()

void	clear_input_normalization_params ()

void	add_image_normalization_param (const linear_transform_t &t)

void	add_scalar_normalization_param (const linear_transform_t &t)

void	add_input_normalization_param (const linear_transform_t &t)

Public Member Functions inherited from lbann::generic_data_reader
	generic_data_reader (bool shuffle=true)

	generic_data_reader (const generic_data_reader &)=default

generic_data_reader &	operator= (const generic_data_reader &)=default

virtual	~generic_data_reader ()

template<class Archive >
void	serialize (Archive &ar)

void	set_comm (lbann_comm *comm)
	set the comm object More...

lbann_comm *	get_comm () const
	returns a (possibly nullptr) to comm More...

virtual bool	has_conduit_output ()

void	set_file_dir (std::string s)

void	set_local_file_dir (std::string s)

void	set_max_files_to_load (size_t n)

std::string	get_file_dir () const

std::string	get_local_file_dir () const

void	set_data_sample_list (std::string s)

std::string	get_data_sample_list () const

void	keep_sample_order (bool same_order=false)

void	set_data_filename (std::string s)

std::string	get_data_filename () const

void	set_label_filename (std::string s)

std::string	get_label_filename () const

void	set_shuffle (bool b)

bool	is_shuffled () const

void	set_shuffled_indices (const std::vector< int > &indices)

const std::vector< int > &	get_shuffled_indices () const

void	set_first_n (int n)

void	set_absolute_sample_count (size_t s)

void	set_use_fraction (double s)

virtual void	set_execution_mode_split_fraction (execution_mode m, double s)

virtual void	set_role (std::string role)

std::string	get_role () const

int	fetch (std::map< data_field_type, CPUMat *> &input_buffers, El::Matrix< El::Int > &indices_fetched, size_t mb_size)
	Fetch a mini-batch worth of data, including samples, labels, responses (as appropriate) More...

int	fetch (std::vector< conduit::Node > &samples, El::Matrix< El::Int > &indices_fetched, size_t mb_size)

virtual bool	has_data_field (data_field_type data_field) const
	Check to see if the data reader supports this specific data field. More...

virtual bool	has_labels () const

virtual bool	has_responses () const

void	set_has_data_field (data_field_type const data_field, const bool b)
	Whether or not a data reader has a data field. More...

virtual void	set_has_labels (const bool b)
	Whether or not a data reader has labels. More...

virtual void	set_has_responses (const bool b)
	Whether or not a data reader has a response field. More...

void	start_data_store_mini_batch_exchange ()

void	finish_data_store_mini_batch_exchange ()

virtual bool	update (bool is_active_reader)

virtual int	get_num_responses () const
	Return the number of responses in this dataset. More...

virtual int	get_linearized_size (data_field_type const &data_field) const
	get the linearized size of what is identified by desc. More...

virtual bool	position_valid () const
	True if the data reader's current position is valid. More...

virtual bool	position_is_overrun () const

bool	at_new_epoch () const
	True if the data reader is at the start of an epoch. More...

void	set_mini_batch_size (const int s)
	Set the mini batch size. More...

int	get_mini_batch_size () const
	Get the mini batch size. More...

int	get_loaded_mini_batch_size () const
	Get the loaded mini-batch size. More...

int	get_current_mini_batch_size () const
	Get the current mini-batch size. More...

int	get_mini_batch_max () const
	Return the full mini_batch_size. More...

void	set_stride_to_next_mini_batch (const int s)
	Set the mini batch stride. More...

int	get_stride_to_next_mini_batch () const
	Return the mini batch stride. More...

void	set_sample_stride (const int s)
	Set the sample stride. More...

int	get_sample_stride () const
	Return the sample stride. More...

void	set_iteration_stride (const int s)
	Set the iteration stride. More...

int	get_iteration_stride () const
	Return the iteration stride. More...

virtual void	set_base_offset (const int s)
	Return the base offset. More...

int	get_base_offset () const
	Return the base offset. More...

void	set_last_mini_batch_size (const int s)
	Set the last mini batch size. More...

int	get_last_mini_batch_size () const
	Return the last mini batch size. More...

void	set_stride_to_last_mini_batch (const int s)
	Set the last mini batch stride. More...

int	get_stride_to_last_mini_batch () const
	Return the last mini batch stride. More...

void	set_num_parallel_readers (const int s)
	Set the number of parallel readers per model. More...

int	get_num_parallel_readers () const
	Return the number of parallel readers per model. More...

virtual void	set_reset_mini_batch_index (const int s)
	Set the starting mini-batch index for the epoch. More...

int	get_reset_mini_batch_index () const
	Return the starting mini-batch index for the epoch. More...

int	get_loaded_mini_batch_index () const
	Return the current mini-batch index for the epoch. More...

int	get_current_mini_batch_index () const
	Return the current mini-batch index for the epoch. More...

void	set_initial_position ()
	Set the current position based on the base and model offsets. More...

int	get_position () const
	Get the current position in the data reader. More...

int	get_next_position () const
	Get the next position in the data reader. More...

int *	get_indices ()
	Get a pointer to the start of the shuffled indices. More...

int	get_num_unused_data (execution_mode m) const
	Get the number of unused samples in this dataset. More...

int *	get_unused_data (execution_mode m)
	Get a pointer to the start of the unused sample indices. More...

const std::vector< int > &	get_unused_indices (execution_mode m)

void	set_num_iterations_per_epoch (int num_iterations_per_epoch)
	Set the number of iterations in each epoch. More...

int	get_num_iterations_per_epoch () const
	Get the number of iterations in each epoch. More...

int	get_current_step_in_epoch () const

void	resize_shuffled_indices ()

void	select_subset_of_data ()

virtual void	use_unused_index_set (execution_mode m)

bool	save_to_checkpoint_shared (persist &p, execution_mode mode)
	Given directory to store checkpoint files, write state to file and add to number of bytes written. More...

bool	load_from_checkpoint_shared (persist &p, execution_mode mode)
	Given directory to store checkpoint files, read state from file and add to number of bytes read. More...

bool	save_to_checkpoint_distributed (persist &p, execution_mode mode)

bool	load_from_checkpoint_distributed (persist &p, execution_mode mode)
	Given directory to store checkpoint files, read state from file and add to number of bytes read. More...

const data_store_conduit &	get_data_store () const
	returns a const ref to the data store More...

data_store_conduit &	get_data_store ()
	returns a non-const ref to the data store More...

data_store_conduit *	get_data_store_ptr () const

void	setup_data_store (int mini_batch_size)

void	instantiate_data_store ()

virtual void	preload_data_store ()

void	set_gan_labelling (bool has_gan_labelling)

void	set_gan_label_value (int gan_label_value)

void	set_data_store (data_store_conduit *g)
	support of data store functionality More...

virtual void	post_update ()

void	set_transform_pipeline (transform::transform_pipeline &&tp)

void	print_get_methods (const std::string filename)
	Print the return values from various get_X methods to file. More...

size_t	get_num_indices_to_use () const

void	set_use_data_store (bool s)

Static Public Member Functions
template<typename S >
static size_t	add_val (const std::string key, const conduit::Node &n, std::vector< S > &vals)

static std::string	to_string (const variable_t t)
	A utility function to convert a JAG variable type to name string. More...

Protected Types
typedef std::pair< std::string, std::string >	conduit_sample

Protected Member Functions
void	do_preload_data_store () override

virtual void	set_defaults ()

virtual void	copy_members (const data_reader_jag_conduit &rhs)

void	add_independent_variable_type (const variable_t independent)
	add data type for independent variable More...

void	add_dependent_variable_type (const variable_t dependent)
	add data type for dependent variable More...

bool	filter (const std::set< std::string > &key_filter, const std::vector< prefix_t > &prefix_filter, const std::string &name) const
	Check if a key is in the black lists to filter out. More...

size_t	get_linearized_size (const variable_t t) const
	Return the linearized size of a particular JAG variable type. More...

const std::vector< int >	get_dims (const variable_t t) const
	Return the dimension of a particular JAG variable type. More...

std::vector< El::Int >	get_slice_points_impl (const std::vector< std::vector< data_reader_jag_conduit::variable_t >> &var) const
	Return the slice points for linearized data or responses. More...

std::vector< El::Int >	get_slice_points_independent () const
	Return the slice points for linearized independent variables. More...

std::vector< El::Int >	get_slice_points_dependent () const
	Return the slice points for linearized dependent variables. More...

virtual std::vector< CPUMat >	create_datum_views (CPUMat &X, const std::vector< size_t > &sizes, const int mb_idx) const

bool	fetch (CPUMat &X, int data_id, conduit::Node &sample, int mb_idx, int tid, const variable_t vt, const std::string tag)
	Export cached data minibatch. More...

bool	fetch_datum (CPUMat &X, int data_id, int mb_idx) override

bool	fetch_response (CPUMat &Y, int data_id, int mb_idx) override

bool	fetch_label (CPUMat &X, int data_id, int mb_idx) override

void	shuffle_indices (rng_gen &gen) override
	Shuffle sammple indices using a different RNG. More...

int	compute_max_num_parallel_readers ()

bool	check_num_parallel_readers (long data_set_size)

void	sample_schema_check (const bool check_data)
	Check the consistency of the schema of the first sample. More...

void	load_list_of_samples (const std::string filename)
	Rely on pre-determined list of samples. More...

void	load_list_of_samples_from_archive (const std::string &sample_list_archive)
	Load the sample list from a serialized archive from another rank. More...

void	check_image_data ()
	See if the image size is consistent with the linearized size. More...

void	set_linearized_image_size ()
	Obtain the linearized size of images of a sample from the meta info. More...

void	check_scalar_keys ()
	Make sure that the keys to choose scalar outputs are valid. More...

void	check_input_keys ()
	Make sure that the keys to choose scalar outputs are valid. More...

bool	has_path (const file_handle_t &h, const std::string &path) const

void	read_node (const file_handle_t &h, const std::string &path, conduit::Node &n) const

bool	load_conduit_node (const size_t i, const std::string &key, conduit::Node &node) const

bool	has_conduit_path (const size_t i, const std::string &key) const
	Check if a key exist for sample i. More...

std::vector< std::vector< DataType > >	get_image_data (const size_t i, conduit::Node &sample) const
	Obtain image data. More...

bool	data_store_active () const override

bool	priming_data_store () const override

void	preload_helper (const hid_t &h, const std::string &sample_name, const std::string &field_name, int data_id, conduit::Node &node)

Protected Member Functions inherited from lbann::generic_data_reader
size_t	get_absolute_sample_count () const

double	get_use_fraction () const

double	get_execution_mode_split_fraction (execution_mode m) const

virtual bool	fetch_data_block (std::map< data_field_type, CPUMat *> &input_buffers, El::Int block_offset, El::Int block_stride, El::Int mb_size, El::Matrix< El::Int > &indices_fetched)

bool	fetch_data_block_conduit (std::vector< conduit::Node > &samples, El::Int block_offset, El::Int block_stride, El::Int mb_size, El::Matrix< El::Int > &indices_fetched)

virtual bool	fetch_data_field (data_field_type data_field, CPUMat &Y, int data_id, int mb_idx)
	Called by fetch_data, fetch_label, fetch_response. More...

virtual bool	fetch_conduit_node (conduit::Node &sample, int data_id)

CPUMat	create_datum_view (CPUMat &X, const int mb_idx)

virtual void	preprocess_data_source (int tid)

virtual void	postprocess_data_source (int tid)

virtual void	shuffle_indices ()
	Shuffle indices (uses the data_seq_generator) More...

void	error_check_counts () const

Static Protected Member Functions
static std::string	to_string (const std::vector< variable_t > &vec)
	A utility function to make a string to show all the variable types. More...

static std::string	to_string (const std::vector< std::vector< variable_t >> &vec)

static bool	check_non_numeric (const std::string key)

static const conduit::Node &	get_conduit_node (const conduit::Node &n_base, const std::string key)
	Allow const access to the conduit data structure. More...

Protected Attributes
std::vector< variable_t >	m_independent
	The flat list of independent variable types. More...

std::vector< std::vector< variable_t > >	m_independent_groups
	The list of independent variable types grouped for slicing. More...

std::vector< variable_t >	m_dependent
	The flat list of dependent variable types. More...

std::vector< std::vector< variable_t > >	m_dependent_groups
	The list of independent variable types grouped for slicing. More...

int	m_image_width
	image width More...

int	m_image_height
	image height More...

int	m_image_num_channels
	number of image channels More...

size_t	m_image_linearized_size
	The linearized size of an image. More...

size_t	m_1ch_image_linearized_size

unsigned int	m_num_img_srcs
	number of views result in images More...

bool	m_split_channels
	Whether to export a separate image per channel. More...

bool	m_is_data_loaded
	Whether data have been loaded. More...

int	m_num_labels
	number of labels More...

std::string	m_output_scalar_prefix
	Common prefix path to any output scalar fields in Conduit / HDF5. More...

std::string	m_output_image_prefix
	Common prefix path to any output image fields in Conduit / HDF5. More...

std::string	m_input_prefix
	Common prefix path to any input fields in Conduit / HDF5. More...

std::vector< std::string >	m_emi_image_keys
	Allow image selection by the view and the time index. More...

std::vector< std::string >	m_scalar_keys

std::vector< std::string >	m_input_keys

bool	m_uniform_input_type

std::set< std::string >	m_scalar_filter
	The set of scalar variables to filter out. More...

std::vector< prefix_t >	m_scalar_prefix_filter
	The list of scalar key prefixes to filter out. More...

std::set< std::string >	m_input_filter
	The set of input variables to filter out. More...

std::vector< prefix_t >	m_input_prefix_filter
	The list of input key prefixes to filter out. More...

int	m_local_reader_id

CPUMat	m_data_cache

CPUMat	m_response_cache

CPUMat	m_label_cache

int	m_cached_data_mb_size

int	m_cached_response_mb_size

int	m_cached_label_mb_size

std::vector< linear_transform_t >	m_image_normalization_params
	temporary normalization parameters based on linear transforms More...

std::vector< linear_transform_t >	m_scalar_normalization_params

std::vector< linear_transform_t >	m_input_normalization_params

sample_list_t	m_sample_list

bool	m_list_per_trainer

bool	m_list_per_model

Protected Attributes inherited from lbann::generic_data_reader
bool	m_verbose = false

std::unordered_set< int >	m_using_random_node

data_store_conduit *	m_data_store

lbann_comm *	m_comm

bool	m_use_data_store = false

std::map< data_field_type, bool >	m_supported_input_types
	Holds a true value for each input data type that is supported. Use an ordered map so that checkpoints are stable. More...

bool	m_gan_labelling

int	m_gan_label_value

observer_ptr< thread_pool >	m_io_thread_pool

bool	m_keep_sample_order

transform::transform_pipeline	m_transform_pipeline

bool	m_issue_warning

Static Protected Attributes
static const std::set< std::string >	non_numeric_vars

static std::unordered_map< std::string, int >	m_num_local_readers
	The number of local instances of this reader type. More...

Additional Inherited Members
Public Attributes inherited from lbann::generic_data_reader
int	m_mini_batch_size

int	m_current_pos

int	m_stride_to_next_mini_batch

int	m_base_offset

int	m_sample_stride

int	m_iteration_stride
	Stride used by parallel data readers within the model. More...

std::vector< int >	m_shuffled_indices

unused_index_map_t	m_unused_indices
	Record of the indicies that are not being used for training. More...

int	m_last_mini_batch_size

int	m_stride_to_last_mini_batch

int	m_reset_mini_batch_index
	The index at which this data reader starts its epoch. More...

int	m_loaded_mini_batch_idx
	The index of the current mini-batch that has been loaded. More...

int	m_current_mini_batch_idx

int	m_num_iterations_per_epoch

int	m_num_parallel_readers
	How many iterations all readers will execute. More...

size_t	m_max_files_to_load
	How many parallel readers are being used. More...

std::string	m_file_dir

std::string	m_local_file_dir

std::string	m_data_sample_list

std::string	m_data_fn

std::string	m_label_fn

bool	m_shuffle

size_t	m_absolute_sample_count

std::map< execution_mode, double >	m_execution_mode_split_fraction

double	m_use_fraction

int	m_first_n

std::string	m_role

Detailed Description

Loads JAG simulation parameters and results from hdf5 files using conduit interfaces

Definition at line 56 of file data_reader_jag_conduit.hpp.

Member Typedef Documentation

◆ ch_t

using lbann::data_reader_jag_conduit::ch_t = float

jag output image channel type

Definition at line 59 of file data_reader_jag_conduit.hpp.

◆ conduit_ch_t

using lbann::data_reader_jag_conduit::conduit_ch_t = conduit::float32_array

conduit type for ch_t array wrapper

Definition at line 61 of file data_reader_jag_conduit.hpp.

◆ conduit_sample

typedef std::pair<std::string, std::string> lbann::data_reader_jag_conduit::conduit_sample

protected

Definition at line 482 of file data_reader_jag_conduit.hpp.

◆ file_handle_t

using lbann::data_reader_jag_conduit::file_handle_t = sample_list_t::file_handle_t

Definition at line 74 of file data_reader_jag_conduit.hpp.

◆ input_t

using lbann::data_reader_jag_conduit::input_t = double

jag input parameter type

Definition at line 63 of file data_reader_jag_conduit.hpp.

◆ linear_transform_t

using lbann::data_reader_jag_conduit::linear_transform_t = std::pair<double, double>

linear transform on X defined as: first * X + second => X'

Definition at line 79 of file data_reader_jag_conduit.hpp.

◆ prefix_t

using lbann::data_reader_jag_conduit::prefix_t = std::pair<std::string, size_t>

Type to define a prefix string and the minimum length requirement to filter out a key

Definition at line 99 of file data_reader_jag_conduit.hpp.

◆ sample_file_id_t

using lbann::data_reader_jag_conduit::sample_file_id_t = sample_list_t::sample_file_id_t

Definition at line 75 of file data_reader_jag_conduit.hpp.

◆ sample_list_t

using lbann::data_reader_jag_conduit::sample_list_t = sample_list_hdf5<sample_name_t>

Definition at line 72 of file data_reader_jag_conduit.hpp.

◆ sample_locator_t

using lbann::data_reader_jag_conduit::sample_locator_t = std::pair<std::string, hid_t>

Type for the pair of the key string of a sample and the handle of the file that contains it

Definition at line 66 of file data_reader_jag_conduit.hpp.

◆ sample_map_t

using lbann::data_reader_jag_conduit::sample_map_t = std::vector<sample_locator_t>

valid sample map type

Definition at line 67 of file data_reader_jag_conduit.hpp.

◆ sample_name_t

using lbann::data_reader_jag_conduit::sample_name_t = std::string

Definition at line 68 of file data_reader_jag_conduit.hpp.

◆ sample_t

using lbann::data_reader_jag_conduit::sample_t = std::pair<sample_file_id_t, sample_name_t>

Definition at line 76 of file data_reader_jag_conduit.hpp.

◆ scalar_t

using lbann::data_reader_jag_conduit::scalar_t = double

jag scalar output type

Definition at line 62 of file data_reader_jag_conduit.hpp.

◆ TypeID

using lbann::data_reader_jag_conduit::TypeID = conduit::DataType::TypeID

Definition at line 95 of file data_reader_jag_conduit.hpp.

Member Enumeration Documentation

◆ variable_t

enum lbann::data_reader_jag_conduit::variable_t

Dependent/indepdendent variable types

JAG_Image: simulation output images
JAG_Scalar: simulation output scalars
JAG_Input: simulation input parameters
Undefined: the default

Enumerator
Undefined
JAG_Image
JAG_Scalar
JAG_Input

Definition at line 88 of file data_reader_jag_conduit.hpp.

Constructor & Destructor Documentation

◆ data_reader_jag_conduit() [1/2]

lbann::data_reader_jag_conduit::data_reader_jag_conduit ( bool shuffle = true )

Here is the caller graph for this function:

◆ data_reader_jag_conduit() [2/2]

lbann::data_reader_jag_conduit::data_reader_jag_conduit ( const data_reader_jag_conduit & )

◆ ~data_reader_jag_conduit()

lbann::data_reader_jag_conduit::~data_reader_jag_conduit ( )

override

Member Function Documentation

◆ add_dependent_variable_type()

void lbann::data_reader_jag_conduit::add_dependent_variable_type ( const variable_t dependent )

protected

add data type for dependent variable

Here is the caller graph for this function:

◆ add_image_normalization_param()

void lbann::data_reader_jag_conduit::add_image_normalization_param ( const linear_transform_t & t )

Here is the caller graph for this function:

◆ add_independent_variable_type()

void lbann::data_reader_jag_conduit::add_independent_variable_type ( const variable_t independent )

protected

add data type for independent variable

Here is the caller graph for this function:

◆ add_input_filter()

void lbann::data_reader_jag_conduit::add_input_filter ( const std::string & key )

Add an input key to filter out.

Here is the caller graph for this function:

◆ add_input_normalization_param()

void lbann::data_reader_jag_conduit::add_input_normalization_param ( const linear_transform_t & t )

Here is the caller graph for this function:

◆ add_input_prefix_filter()

void lbann::data_reader_jag_conduit::add_input_prefix_filter ( const prefix_t & p )

Add an input key prefix to filter out.

Here is the caller graph for this function:

◆ add_scalar_filter()

void lbann::data_reader_jag_conduit::add_scalar_filter ( const std::string & key )

Add a scalar key to filter out.

Here is the caller graph for this function:

◆ add_scalar_normalization_param()

void lbann::data_reader_jag_conduit::add_scalar_normalization_param ( const linear_transform_t & t )

Here is the caller graph for this function:

◆ add_scalar_prefix_filter()

void lbann::data_reader_jag_conduit::add_scalar_prefix_filter ( const prefix_t & p )

Add a scalar key prefix to filter out.

Here is the caller graph for this function:

◆ add_val()

template<typename S >

size_t lbann::data_reader_jag_conduit::add_val	(	const std::string	key,
		const conduit::Node &	n,
		std::vector< S > &	vals
	)

inlinestatic

Retrieve a value from the given node n, and add it to the vector of type S, vals. The first argument key is the name of the current node (i.e. the name reported by the node iterator to the node).

Definition at line 565 of file data_reader_jag_conduit.hpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ check_image_data()

void lbann::data_reader_jag_conduit::check_image_data ( )

protected

See if the image size is consistent with the linearized size.

Here is the caller graph for this function:

◆ check_input_keys()

void lbann::data_reader_jag_conduit::check_input_keys ( )

protected

Make sure that the keys to choose scalar outputs are valid.

Here is the caller graph for this function:

◆ check_non_numeric()

static bool lbann::data_reader_jag_conduit::check_non_numeric ( const std::string key )

staticprotected

Check if the key is associated with non-numeric value, that is not and cannot be converted to a numertic.

Here is the caller graph for this function:

◆ check_num_parallel_readers()

bool lbann::data_reader_jag_conduit::check_num_parallel_readers ( long data_set_size )

protected

Check if there are sufficient number of samples for the given number of data readers with distributed io buffer, based on the number of samples, the number of models and the mini batch size.

Here is the caller graph for this function:

◆ check_scalar_keys()

void lbann::data_reader_jag_conduit::check_scalar_keys ( )

protected

Make sure that the keys to choose scalar outputs are valid.

Here is the caller graph for this function:

◆ check_split_image_channels()

bool lbann::data_reader_jag_conduit::check_split_image_channels ( ) const

Here is the caller graph for this function:

◆ clear_image_normalization_params()

void lbann::data_reader_jag_conduit::clear_image_normalization_params ( )

Here is the caller graph for this function:

◆ clear_input_normalization_params()

void lbann::data_reader_jag_conduit::clear_input_normalization_params ( )

Here is the caller graph for this function:

◆ clear_scalar_normalization_params()

void lbann::data_reader_jag_conduit::clear_scalar_normalization_params ( )

Here is the caller graph for this function:

◆ compute_max_num_parallel_readers()

int lbann::data_reader_jag_conduit::compute_max_num_parallel_readers ( )

protected

Compute the number of parallel readers based on the type of io_buffer, the mini batch size, the requested number of parallel readers. This is done before populating the sample indices.

Here is the caller graph for this function:

◆ copy()

data_reader_jag_conduit* lbann::data_reader_jag_conduit::copy ( ) const

inlineoverridevirtual

Implements lbann::generic_data_reader.

Definition at line 105 of file data_reader_jag_conduit.hpp.

Here is the call graph for this function:

◆ copy_members()

virtual void lbann::data_reader_jag_conduit::copy_members ( const data_reader_jag_conduit & rhs )

protectedvirtual

Here is the caller graph for this function:

◆ create_datum_views()

virtual std::vector<CPUMat> lbann::data_reader_jag_conduit::create_datum_views	(	CPUMat &	X,
		const std::vector< size_t > &	sizes,
		const int	mb_idx
	)		const

protectedvirtual

Here is the caller graph for this function:

◆ data_store_active()

bool lbann::data_reader_jag_conduit::data_store_active ( ) const

inlineoverrideprotectedvirtual

Reimplemented from lbann::generic_data_reader.

Definition at line 389 of file data_reader_jag_conduit.hpp.

Here is the call graph for this function:

◆ do_preload_data_store()

void lbann::data_reader_jag_conduit::do_preload_data_store ( )

overrideprotectedvirtual

once the sample_list class and file formats are generalized and finalized, it should (may?) be possible to code a single preload_data_store method.

Reimplemented from lbann::generic_data_reader.

Here is the caller graph for this function:

◆ fetch()

bool lbann::data_reader_jag_conduit::fetch	(	CPUMat &	X,
		int	data_id,
		conduit::Node &	sample,
		int	mb_idx,
		int	tid,
		const variable_t	vt,
		const std::string	tag
	)

protected

Export cached data minibatch.

Here is the caller graph for this function:

◆ fetch_datum()

bool lbann::data_reader_jag_conduit::fetch_datum	(	CPUMat &	X,
		int	data_id,
		int	mb_idx
	)

overrideprotectedvirtual

Fetch a single sample into a matrix.

Parameters

X	The matrix to load data into.
data_id	The index of the datum to fetch.
mb_idx	The index within the mini-batch.

Reimplemented from lbann::generic_data_reader.

Here is the caller graph for this function:

◆ fetch_label()

bool lbann::data_reader_jag_conduit::fetch_label	(	CPUMat &	Y,
		int	data_id,
		int	mb_idx
	)

overrideprotectedvirtual

Fetch a single label into a matrix.

Parameters

Y	The matrix to load data into.
data_id	The index of the datum to fetch.
mb_idx	The index within the mini-batch.

Reimplemented from lbann::generic_data_reader.

Here is the caller graph for this function:

◆ fetch_response()

bool lbann::data_reader_jag_conduit::fetch_response	(	CPUMat &	Y,
		int	data_id,
		int	mb_idx
	)

overrideprotectedvirtual

Fetch a single response into a matrix.

Parameters

Y	The matrix to load data into.
data_id	The index of the datum to fetch.
mb_idx	The index within the mini-batch.

Reimplemented from lbann::generic_data_reader.

Here is the caller graph for this function:

◆ filter()

bool lbann::data_reader_jag_conduit::filter	(	const std::set< std::string > &	key_filter,
		const std::vector< prefix_t > &	prefix_filter,
		const std::string &	name
	)		const

protected

Check if a key is in the black lists to filter out.

Here is the caller graph for this function:

◆ get_conduit_node()

static const conduit::Node& lbann::data_reader_jag_conduit::get_conduit_node	(	const conduit::Node &	n_base,
		const std::string	key
	)

staticprotected

Allow const access to the conduit data structure.

Here is the caller graph for this function:

◆ get_data_dims()

const std::vector<El::Int> lbann::data_reader_jag_conduit::get_data_dims ( ) const

overridevirtual

Return the dimension of data.

Reimplemented from lbann::generic_data_reader.

Here is the caller graph for this function:

◆ get_dependent_variable_type()

std::vector<variable_t> lbann::data_reader_jag_conduit::get_dependent_variable_type ( ) const

Tell which data to use for dependent variable.

Here is the caller graph for this function:

◆ get_description()

std::string lbann::data_reader_jag_conduit::get_description ( ) const

Show the description.

Here is the caller graph for this function:

◆ get_dims()

const std::vector<int> lbann::data_reader_jag_conduit::get_dims ( const variable_t t ) const

protected

Return the dimension of a particular JAG variable type.

Here is the caller graph for this function:

◆ get_image_choices()

const std::vector<std::string>& lbann::data_reader_jag_conduit::get_image_choices ( ) const

Report the image choices.

Here is the caller graph for this function:

◆ get_image_data()

std::vector<std::vector<DataType> > lbann::data_reader_jag_conduit::get_image_data	(	const size_t	i,
		conduit::Node &	sample
	)		const

protected

Obtain image data.

Here is the caller graph for this function:

◆ get_independent_variable_type()

std::vector<variable_t> lbann::data_reader_jag_conduit::get_independent_variable_type ( ) const

Tell which data to use for independent variable.

Here is the caller graph for this function:

◆ get_input_choices()

const std::vector<std::string>& lbann::data_reader_jag_conduit::get_input_choices ( ) const

Report the selected simulation input parameters.

Here is the caller graph for this function:

◆ get_inputs()

std::vector<input_t> lbann::data_reader_jag_conduit::get_inputs	(	const size_t	i,
		conduit::Node &	sample
	)		const

Return the simulation input parameters of the i-th sample.

Here is the caller graph for this function:

◆ get_linearized_1ch_image_size()

size_t lbann::data_reader_jag_conduit::get_linearized_1ch_image_size ( ) const

Return the linearized size of a single channel image.

Here is the caller graph for this function:

◆ get_linearized_data_size()

int lbann::data_reader_jag_conduit::get_linearized_data_size ( ) const

overridevirtual

Return the total linearized size of data.

Reimplemented from lbann::generic_data_reader.

Here is the caller graph for this function:

◆ get_linearized_data_sizes()

std::vector<size_t> lbann::data_reader_jag_conduit::get_linearized_data_sizes ( ) const

Return the per-source linearized sizes of composite data.

Here is the caller graph for this function:

◆ get_linearized_image_size()

size_t lbann::data_reader_jag_conduit::get_linearized_image_size ( ) const

Return the linearized size of an image.

Here is the caller graph for this function:

◆ get_linearized_input_size()

size_t lbann::data_reader_jag_conduit::get_linearized_input_size ( ) const

Return the linearized size of inputs.

Here is the caller graph for this function:

◆ get_linearized_label_size()

int lbann::data_reader_jag_conduit::get_linearized_label_size ( ) const

overridevirtual

Get the linearized size (i.e. number of elements) in a label.

Reimplemented from lbann::generic_data_reader.

Here is the caller graph for this function:

◆ get_linearized_response_size()

int lbann::data_reader_jag_conduit::get_linearized_response_size ( ) const

overridevirtual

Return the total linearized size of response.

Reimplemented from lbann::generic_data_reader.

Here is the caller graph for this function:

◆ get_linearized_response_sizes()

std::vector<size_t> lbann::data_reader_jag_conduit::get_linearized_response_sizes ( ) const

Return the per-source linearized sizes of composite response.

Here is the caller graph for this function:

◆ get_linearized_scalar_size()

size_t lbann::data_reader_jag_conduit::get_linearized_scalar_size ( ) const

Return the linearized size of scalar outputs.

Here is the caller graph for this function:

◆ get_linearized_size()

size_t lbann::data_reader_jag_conduit::get_linearized_size ( const variable_t t ) const

protected

Return the linearized size of a particular JAG variable type.

Here is the caller graph for this function:

◆ get_local_id()

int lbann::data_reader_jag_conduit::get_local_id ( const std::string role ) const

Get the id of this local instance.

Here is the caller graph for this function:

◆ get_num_data()

int lbann::data_reader_jag_conduit::get_num_data ( ) const

overridevirtual

Get the number of samples in this dataset.

Reimplemented from lbann::generic_data_reader.

Here is the caller graph for this function:

◆ get_num_img_srcs()

unsigned int lbann::data_reader_jag_conduit::get_num_img_srcs ( ) const

Return the number of measurement views.

Here is the caller graph for this function:

◆ get_num_labels()

int lbann::data_reader_jag_conduit::get_num_labels ( ) const

overridevirtual

Return the number of labels (classes) in this dataset.

This is called at the end of update; it permits data readers to perform actions that are specific to their data sets, for example, data_reader_jag_conduit_hdf5 has the 'primary' data reader bcast its shuffled indices to the other data readers. In general most data readers will probably not overide this method. It may also be called outside of update.

Reimplemented from lbann::generic_data_reader.

Here is the caller graph for this function:

◆ get_scalar_choices()

const std::vector<std::string>& lbann::data_reader_jag_conduit::get_scalar_choices ( ) const

Report the selected scalar outputs.

Here is the caller graph for this function:

◆ get_scalars()

std::vector<scalar_t> lbann::data_reader_jag_conduit::get_scalars	(	const size_t	i,
		conduit::Node &	sample
	)		const

Return the scalar simulation output data of the i-th sample.

Here is the caller graph for this function:

◆ get_slice_points()

std::vector<El::Int> lbann::data_reader_jag_conduit::get_slice_points	(	const slice_points_mode	var_category,
		bool &	is_supported
	)

overridevirtual

Reimplemented from lbann::generic_data_reader.

Here is the caller graph for this function:

◆ get_slice_points_dependent()

std::vector<El::Int> lbann::data_reader_jag_conduit::get_slice_points_dependent ( ) const

protected

Return the slice points for linearized dependent variables.

Here is the caller graph for this function:

◆ get_slice_points_impl()

std::vector<El::Int> lbann::data_reader_jag_conduit::get_slice_points_impl ( const std::vector< std::vector< data_reader_jag_conduit::variable_t >> & var ) const

protected

Return the slice points for linearized data or responses.

Here is the caller graph for this function:

◆ get_slice_points_independent()

std::vector<El::Int> lbann::data_reader_jag_conduit::get_slice_points_independent ( ) const

protected

Return the slice points for linearized independent variables.

Here is the caller graph for this function:

◆ get_type()

std::string lbann::data_reader_jag_conduit::get_type ( ) const

inlineoverridevirtual

Return this data_reader's type

Implements lbann::generic_data_reader.

Definition at line 113 of file data_reader_jag_conduit.hpp.

Here is the call graph for this function:

◆ has_conduit_path()

bool lbann::data_reader_jag_conduit::has_conduit_path	(	const size_t	i,
		const std::string &	key
	)		const

protected

Check if a key exist for sample i.

Here is the caller graph for this function:

◆ has_list_per_model()

bool lbann::data_reader_jag_conduit::has_list_per_model ( ) const

inlineoverridevirtual

Does the data reader have a unique sample list per model.

Reimplemented from lbann::generic_data_reader.

Definition at line 192 of file data_reader_jag_conduit.hpp.

◆ has_list_per_trainer()

bool lbann::data_reader_jag_conduit::has_list_per_trainer ( ) const

inlineoverridevirtual

Does the data reader have a unique sample list per trainer.

Reimplemented from lbann::generic_data_reader.

Definition at line 193 of file data_reader_jag_conduit.hpp.

Here is the call graph for this function:

◆ has_path()

bool lbann::data_reader_jag_conduit::has_path	(	const file_handle_t &	h,
		const std::string &	path
	)		const

protected

Here is the caller graph for this function:

◆ load()

void lbann::data_reader_jag_conduit::load ( )

overridevirtual

Load data and do data reader's chores.

Implements lbann::generic_data_reader.

Here is the caller graph for this function:

◆ load_conduit_node()

bool lbann::data_reader_jag_conduit::load_conduit_node	(	const size_t	i,
		const std::string &	key,
		conduit::Node &	node
	)		const

protected

Load the conduit node with the data of the sample i identified by key from the file that contains the sample, and returm true. Upon failure to load from file, attempt to retrieve a random conduit node from the data_store (if –use_data_store) and return false.

Here is the caller graph for this function:

◆ load_list_of_samples()

void lbann::data_reader_jag_conduit::load_list_of_samples ( const std::string filename )

protected

Rely on pre-determined list of samples.

Here is the caller graph for this function:

◆ load_list_of_samples_from_archive()

void lbann::data_reader_jag_conduit::load_list_of_samples_from_archive ( const std::string & sample_list_archive )

protected

Load the sample list from a serialized archive from another rank.

Here is the caller graph for this function:

◆ operator=()

data_reader_jag_conduit& lbann::data_reader_jag_conduit::operator= ( const data_reader_jag_conduit & )

◆ preload_helper()

void lbann::data_reader_jag_conduit::preload_helper	(	const hid_t &	h,
		const std::string &	sample_name,
		const std::string &	field_name,
		int	data_id,
		conduit::Node &	node
	)

protected

◆ priming_data_store()

bool lbann::data_reader_jag_conduit::priming_data_store ( ) const

inlineoverrideprotectedvirtual

Reimplemented from lbann::generic_data_reader.

Definition at line 395 of file data_reader_jag_conduit.hpp.

Here is the call graph for this function:

◆ print_schema()

void lbann::data_reader_jag_conduit::print_schema ( const size_t i ) const

print the schema of the specific sample identified by a given id

Here is the caller graph for this function:

◆ read_node()

void lbann::data_reader_jag_conduit::read_node	(	const file_handle_t &	h,
		const std::string &	path,
		conduit::Node &	n
	)		const

protected

Here is the caller graph for this function:

◆ sample_schema_check()

void lbann::data_reader_jag_conduit::sample_schema_check ( const bool check_data )

protected

Check the consistency of the schema of the first sample.

Here is the caller graph for this function:

◆ set_all_input_choices()

void lbann::data_reader_jag_conduit::set_all_input_choices ( )

Set to use the entire set of simulation input parameters.

Here is the caller graph for this function:

◆ set_all_scalar_choices()

void lbann::data_reader_jag_conduit::set_all_scalar_choices ( )

Set to use the entire set of scalar outputs.

Here is the caller graph for this function:

◆ set_defaults()

virtual void lbann::data_reader_jag_conduit::set_defaults ( )

protectedvirtual

Here is the caller graph for this function:

◆ set_dependent_variable_type()

void lbann::data_reader_jag_conduit::set_dependent_variable_type ( const std::vector< std::vector< variable_t >> & dependent )

Choose which data to use for dependent variable.

Here is the caller graph for this function:

◆ set_image_choices()

void lbann::data_reader_jag_conduit::set_image_choices ( const std::vector< std::string > image_keys )

Choose images to use. e.g. by measurement views and time indices.

Here is the caller graph for this function:

◆ set_image_dims()

void lbann::data_reader_jag_conduit::set_image_dims	(	const int	width,
		const int	height,
		const int	ch = `1`
	)

Set the image dimension.

Here is the caller graph for this function:

◆ set_independent_variable_type()

void lbann::data_reader_jag_conduit::set_independent_variable_type ( const std::vector< std::vector< variable_t >> & independent )

Choose which data to use for independent variable.

Here is the caller graph for this function:

◆ set_input_choices()

void lbann::data_reader_jag_conduit::set_input_choices ( const std::vector< std::string > & keys )

Select the set of simulation input parameters to use.

Here is the caller graph for this function:

◆ set_input_prefix()

void lbann::data_reader_jag_conduit::set_input_prefix ( const std::string & prefix )

inline

Set the common prefix path for any input variables stored.

Definition at line 138 of file data_reader_jag_conduit.hpp.

Here is the call graph for this function:

◆ set_linearized_image_size()

void lbann::data_reader_jag_conduit::set_linearized_image_size ( )

protected

Obtain the linearized size of images of a sample from the meta info.

Here is the caller graph for this function:

◆ set_list_per_model()

void lbann::data_reader_jag_conduit::set_list_per_model ( bool flag )

inline

Set every reader instances in a model to have an independent sample list.

Definition at line 190 of file data_reader_jag_conduit.hpp.

◆ set_list_per_trainer()

void lbann::data_reader_jag_conduit::set_list_per_trainer ( bool flag )

inline

Set the set of open hdf5 data files.

Set every reader instances in a trainer to have an independent sample list

Definition at line 188 of file data_reader_jag_conduit.hpp.

◆ set_local_id()

void lbann::data_reader_jag_conduit::set_local_id ( const std::string role )

Set the id of this local instance.

Here is the caller graph for this function:

◆ set_output_image_prefix()

void lbann::data_reader_jag_conduit::set_output_image_prefix ( const std::string & prefix )

inline

Set the common prefix path for any output images stored.

Definition at line 133 of file data_reader_jag_conduit.hpp.

◆ set_output_scalar_prefix()

void lbann::data_reader_jag_conduit::set_output_scalar_prefix ( const std::string & prefix )

inline

Set the common prefix path for any output scalar fields stored.

Definition at line 128 of file data_reader_jag_conduit.hpp.

◆ set_scalar_choices()

void lbann::data_reader_jag_conduit::set_scalar_choices ( const std::vector< std::string > & keys )

Select the set of scalar output variables to use.

Here is the caller graph for this function:

◆ set_split_image_channels()

void lbann::data_reader_jag_conduit::set_split_image_channels ( )

Here is the caller graph for this function:

◆ setup()

void lbann::data_reader_jag_conduit::setup	(	int	num_io_threads,
		observer_ptr< thread_pool >	io_thread_pool
	)

overridevirtual

Prepare to start processing an epoch of data. If shuffle is true, then shuffle the indices of the data set If the base offset is not specified set it to 0 If the stride is not specified set it to batch size

Reimplemented from lbann::generic_data_reader.

Here is the caller graph for this function:

◆ setup_data_store()

void lbann::data_reader_jag_conduit::setup_data_store ( int mini_batch_size )

Here is the caller graph for this function:

◆ shuffle_indices()

void lbann::data_reader_jag_conduit::shuffle_indices ( rng_gen & gen )

overrideprotectedvirtual

Shuffle sammple indices using a different RNG.

Reimplemented from lbann::generic_data_reader.

◆ to_string() [1/3]

static std::string lbann::data_reader_jag_conduit::to_string ( const variable_t t )

static

A utility function to convert a JAG variable type to name string.

Here is the caller graph for this function:

◆ to_string() [2/3]

static std::string lbann::data_reader_jag_conduit::to_string ( const std::vector< variable_t > & vec )

staticprotected

A utility function to make a string to show all the variable types.

◆ to_string() [3/3]

static std::string lbann::data_reader_jag_conduit::to_string ( const std::vector< std::vector< variable_t >> & vec )

staticprotected

A utility function to make a string to show all the groups of variable types

◆ unset_split_image_channels()

void lbann::data_reader_jag_conduit::unset_split_image_channels ( )

Here is the caller graph for this function:

Member Data Documentation

◆ m_1ch_image_linearized_size

size_t lbann::data_reader_jag_conduit::m_1ch_image_linearized_size

protected

The linearized size of a single channel image

Definition at line 415 of file data_reader_jag_conduit.hpp.

◆ m_cached_data_mb_size

int lbann::data_reader_jag_conduit::m_cached_data_mb_size

protected

Definition at line 473 of file data_reader_jag_conduit.hpp.

◆ m_cached_label_mb_size

int lbann::data_reader_jag_conduit::m_cached_label_mb_size

protected

Definition at line 475 of file data_reader_jag_conduit.hpp.

◆ m_cached_response_mb_size

int lbann::data_reader_jag_conduit::m_cached_response_mb_size

protected

Definition at line 474 of file data_reader_jag_conduit.hpp.

◆ m_data_cache

CPUMat lbann::data_reader_jag_conduit::m_data_cache

protected

Definition at line 470 of file data_reader_jag_conduit.hpp.

◆ m_dependent

std::vector<variable_t> lbann::data_reader_jag_conduit::m_dependent

protected

The flat list of dependent variable types.

Definition at line 407 of file data_reader_jag_conduit.hpp.

◆ m_dependent_groups

std::vector<std::vector<variable_t> > lbann::data_reader_jag_conduit::m_dependent_groups

protected

The list of independent variable types grouped for slicing.

Definition at line 409 of file data_reader_jag_conduit.hpp.

◆ m_emi_image_keys

std::vector<std::string> lbann::data_reader_jag_conduit::m_emi_image_keys

protected

Allow image selection by the view and the time index.

Definition at line 433 of file data_reader_jag_conduit.hpp.

◆ m_image_height

int lbann::data_reader_jag_conduit::m_image_height

protected

image height

Definition at line 412 of file data_reader_jag_conduit.hpp.

◆ m_image_linearized_size

size_t lbann::data_reader_jag_conduit::m_image_linearized_size

protected

The linearized size of an image.

Definition at line 414 of file data_reader_jag_conduit.hpp.

◆ m_image_normalization_params

std::vector<linear_transform_t> lbann::data_reader_jag_conduit::m_image_normalization_params

protected

temporary normalization parameters based on linear transforms

Definition at line 478 of file data_reader_jag_conduit.hpp.

◆ m_image_num_channels

int lbann::data_reader_jag_conduit::m_image_num_channels

protected

number of image channels

Definition at line 413 of file data_reader_jag_conduit.hpp.

◆ m_image_width

int lbann::data_reader_jag_conduit::m_image_width

protected

image width

Definition at line 411 of file data_reader_jag_conduit.hpp.

◆ m_independent

std::vector<variable_t> lbann::data_reader_jag_conduit::m_independent

protected

The flat list of independent variable types.

Definition at line 403 of file data_reader_jag_conduit.hpp.

◆ m_independent_groups

std::vector<std::vector<variable_t> > lbann::data_reader_jag_conduit::m_independent_groups

protected

The list of independent variable types grouped for slicing.

Definition at line 405 of file data_reader_jag_conduit.hpp.

◆ m_input_filter

std::set<std::string> lbann::data_reader_jag_conduit::m_input_filter

protected

The set of input variables to filter out.

Definition at line 460 of file data_reader_jag_conduit.hpp.

◆ m_input_keys

std::vector<std::string> lbann::data_reader_jag_conduit::m_input_keys

protected

Keys to select a set of simulation input parameters to use. By default, use all.

Definition at line 439 of file data_reader_jag_conduit.hpp.

◆ m_input_normalization_params

std::vector<linear_transform_t> lbann::data_reader_jag_conduit::m_input_normalization_params

protected

Definition at line 480 of file data_reader_jag_conduit.hpp.

◆ m_input_prefix

std::string lbann::data_reader_jag_conduit::m_input_prefix

protected

Common prefix path to any input fields in Conduit / HDF5.

Definition at line 430 of file data_reader_jag_conduit.hpp.

◆ m_input_prefix_filter

std::vector<prefix_t> lbann::data_reader_jag_conduit::m_input_prefix_filter

protected

The list of input key prefixes to filter out.

Definition at line 462 of file data_reader_jag_conduit.hpp.

◆ m_is_data_loaded

bool lbann::data_reader_jag_conduit::m_is_data_loaded

protected

Whether data have been loaded.

Definition at line 421 of file data_reader_jag_conduit.hpp.

◆ m_label_cache

CPUMat lbann::data_reader_jag_conduit::m_label_cache

protected

Definition at line 472 of file data_reader_jag_conduit.hpp.

◆ m_list_per_model

bool lbann::data_reader_jag_conduit::m_list_per_model

protected

Definition at line 485 of file data_reader_jag_conduit.hpp.

◆ m_list_per_trainer

bool lbann::data_reader_jag_conduit::m_list_per_trainer

protected

Definition at line 484 of file data_reader_jag_conduit.hpp.

◆ m_local_reader_id

int lbann::data_reader_jag_conduit::m_local_reader_id

protected

locally addressable id in case of multiple data reader instances attached to a model

Definition at line 468 of file data_reader_jag_conduit.hpp.

◆ m_num_img_srcs

unsigned int lbann::data_reader_jag_conduit::m_num_img_srcs

protected

number of views result in images

Definition at line 417 of file data_reader_jag_conduit.hpp.

◆ m_num_labels

int lbann::data_reader_jag_conduit::m_num_labels

protected

number of labels

Definition at line 423 of file data_reader_jag_conduit.hpp.

◆ m_num_local_readers

std::unordered_map<std::string, int> lbann::data_reader_jag_conduit::m_num_local_readers

staticprotected

The number of local instances of this reader type.

Definition at line 465 of file data_reader_jag_conduit.hpp.

◆ m_output_image_prefix

std::string lbann::data_reader_jag_conduit::m_output_image_prefix

protected

Common prefix path to any output image fields in Conduit / HDF5.

Definition at line 428 of file data_reader_jag_conduit.hpp.

◆ m_output_scalar_prefix

std::string lbann::data_reader_jag_conduit::m_output_scalar_prefix

protected

Common prefix path to any output scalar fields in Conduit / HDF5.

Definition at line 426 of file data_reader_jag_conduit.hpp.

◆ m_response_cache

CPUMat lbann::data_reader_jag_conduit::m_response_cache

protected

Definition at line 471 of file data_reader_jag_conduit.hpp.

◆ m_sample_list

sample_list_t lbann::data_reader_jag_conduit::m_sample_list

protected

Definition at line 483 of file data_reader_jag_conduit.hpp.

◆ m_scalar_filter

std::set<std::string> lbann::data_reader_jag_conduit::m_scalar_filter

protected

The set of scalar variables to filter out.

Definition at line 456 of file data_reader_jag_conduit.hpp.

◆ m_scalar_keys

std::vector<std::string> lbann::data_reader_jag_conduit::m_scalar_keys

protected

Keys to select a set of scalar simulation outputs to use. By default, use all.

Definition at line 436 of file data_reader_jag_conduit.hpp.

◆ m_scalar_normalization_params

std::vector<linear_transform_t> lbann::data_reader_jag_conduit::m_scalar_normalization_params

protected

Definition at line 479 of file data_reader_jag_conduit.hpp.

◆ m_scalar_prefix_filter

std::vector<prefix_t> lbann::data_reader_jag_conduit::m_scalar_prefix_filter

protected

The list of scalar key prefixes to filter out.

Definition at line 458 of file data_reader_jag_conduit.hpp.

◆ m_split_channels

bool lbann::data_reader_jag_conduit::m_split_channels

protected

Whether to export a separate image per channel.

Definition at line 418 of file data_reader_jag_conduit.hpp.

◆ m_uniform_input_type

bool lbann::data_reader_jag_conduit::m_uniform_input_type

protected

indicate if all the input variables are of the input_t type, in which case we can rely on a data extraction method with lower overhead.

Definition at line 453 of file data_reader_jag_conduit.hpp.

◆ non_numeric_vars

const std::set<std::string> lbann::data_reader_jag_conduit::non_numeric_vars

staticprotected

Set of keys that are associated with non_numerical values. Such a variable requires a specific method for mapping to a numeric value. When a key is found in the set, the variable is ignored. Therefore, when a conversion is defined for such a key, remove it from the set.

Definition at line 447 of file data_reader_jag_conduit.hpp.

The documentation for this class was generated from the following file:

data_reader_jag_conduit.hpp

Public Types

Public Member Functions

Static Public Member Functions

Protected Types

Protected Member Functions

Static Protected Member Functions

Protected Attributes

Static Protected Attributes

Additional Inherited Members

Detailed Description

Member Typedef Documentation

◆ ch_t

◆ conduit_ch_t

◆ conduit_sample

◆ file_handle_t

◆ input_t

◆ linear_transform_t

◆ prefix_t

◆ sample_file_id_t

◆ sample_list_t

◆ sample_locator_t

◆ sample_map_t

◆ sample_name_t

◆ sample_t

◆ scalar_t

◆ TypeID

Member Enumeration Documentation

◆ variable_t

Constructor & Destructor Documentation

◆ data_reader_jag_conduit() [1/2]

◆ data_reader_jag_conduit() [2/2]

◆ ~data_reader_jag_conduit()

Member Function Documentation

◆ add_dependent_variable_type()

◆ add_image_normalization_param()

◆ add_independent_variable_type()

◆ add_input_filter()

◆ add_input_normalization_param()

◆ add_input_prefix_filter()

◆ add_scalar_filter()

◆ add_scalar_normalization_param()

◆ add_scalar_prefix_filter()

◆ add_val()

◆ check_image_data()

◆ check_input_keys()

◆ check_non_numeric()

◆ check_num_parallel_readers()

◆ check_scalar_keys()

◆ check_split_image_channels()

◆ clear_image_normalization_params()

◆ clear_input_normalization_params()

◆ clear_scalar_normalization_params()

◆ compute_max_num_parallel_readers()

◆ copy()

◆ copy_members()

◆ create_datum_views()

◆ data_store_active()

◆ do_preload_data_store()

◆ fetch()

◆ fetch_datum()

◆ fetch_label()

◆ fetch_response()

◆ filter()

◆ get_conduit_node()

◆ get_data_dims()

◆ get_dependent_variable_type()

◆ get_description()

◆ get_dims()

◆ get_image_choices()

◆ get_image_data()

◆ get_independent_variable_type()

◆ get_input_choices()

◆ get_inputs()

◆ get_linearized_1ch_image_size()

◆ get_linearized_data_size()

◆ get_linearized_data_sizes()

◆ get_linearized_image_size()

◆ get_linearized_input_size()

◆ get_linearized_label_size()

◆ get_linearized_response_size()