#include <weights.hpp>

Inheritance diagram for lbann::weights:

Collaboration diagram for lbann::weights:

Public Member Functions
	weights (lbann_comm &comm)

virtual	~weights ()=default

void	set_name (std::string name)

std::string	get_name () const

lbann_comm &	get_comm () const

description	get_description () const

virtual std::string	get_datatype_name () const =0

virtual bool	has_optimizer () const =0

std::vector< size_t >	get_dims () const

size_t	get_size () const

std::vector< size_t >	get_matrix_height_dims () const

std::vector< size_t >	get_matrix_width_dims () const

size_t	get_matrix_height () const

size_t	get_matrix_width () const

void	set_dims (std::vector< size_t > matrix_height_dims, std::vector< size_t > matrix_width_dims={})

void	set_dims (size_t size)

El::DistData	get_matrix_distribution () const

void	set_matrix_distribution (El::DistData dist)

virtual weights_initializer *	get_initializer ()=0

virtual const weights_initializer *	get_initializer () const =0

virtual void	set_initializer (std::unique_ptr< weights_initializer > &&init)=0

virtual optimizer *	get_optimizer ()=0

virtual const optimizer *	get_optimizer () const =0

virtual void	set_optimizer (std::unique_ptr< optimizer > &&opt)=0

void	setup ()

void	freeze ()

void	unfreeze ()

bool	is_frozen () const

virtual void	reconcile_values ()=0

virtual void	reconcile_values (Al::request &req)=0

virtual bool	load_from_save (std::string const &ckpt_dir, std::vector< std::string > const &weight_list)=0

virtual void	write_proto (lbann_data::Weights &proto) const =0

Matrix accessors
void	set_values (El::BaseDistMatrix const &values)
	Set the values matrix to the given matrix. More...

virtual El::BaseDistMatrix &	get_values ()=0
	Access the matrix of weights values. More...

virtual El::BaseDistMatrix const &	get_values () const =0

Serialization
template<typename ArchiveT >
void	serialize (ArchiveT &ar)
	Serialize the weights object to the archive. More...

Expert interface
void	steal_values (weights &other)
	Take the values matrix from another weights object. More...

Public Member Functions inherited from lbann::Cloneable< HasAbstractFunction< weights > >
std::unique_ptr< HasAbstractFunction< weights > >	clone () const
	Return an exception-safe, memory-safe copy of this object. More...

Protected Member Functions
	weights (const weights &other)=default

weights &	operator= (const weights &other)=default

Private Member Functions
	weights ()

void	set_comm (lbann_comm &comm)

void	setup_default_matrix_distribution ()

virtual void	do_augment_description_ (description &) const =0

virtual void	do_setup_ ()=0

virtual void	do_set_dims_ (std::vector< size_t > const &matrix_height_dims, std::vector< size_t > const &matrix_width_dims)=0

virtual void	do_steal_values_ (weights &other)=0

Private Attributes
std::string	m_name

lbann_comm *	m_comm

std::vector< size_t >	m_matrix_height_dims

std::vector< size_t >	m_matrix_width_dims

El::DistData	m_matrix_dist

bool	m_frozen

Detailed Description

Neural network weights. Weights are tensors that act as trainable parameters for a neural network. The values can be initialized with a weights initializer and are optimized with first-order methods (e.g. stochastic gradient descent).

Internally, the weight values are stored in a 2D distributed matrix. The "matrix height dimensions" are tensor dimensions that correspond to the matrix height. The remaining dimensions, the "matrix width dimensions," correspond to the matrix width.

Note that LBANN weights are similar to Tensorflow variables and Caffe parameters.

Definition at line 100 of file weights/weights.hpp.

Constructor & Destructor Documentation

◆ weights() [1/3]

lbann::weights::weights ( )

private

◆ weights() [2/3]

lbann::weights::weights ( lbann_comm & comm )

◆ ~weights()

virtual lbann::weights::~weights ( )

virtualdefault

◆ weights() [3/3]

lbann::weights::weights ( const weights & other )

protecteddefault

Member Function Documentation

◆ do_augment_description_()

virtual void lbann::weights::do_augment_description_ ( description & ) const

privatepure virtual

◆ do_set_dims_()

virtual void lbann::weights::do_set_dims_	(	std::vector< size_t > const &	matrix_height_dims,
		std::vector< size_t > const &	matrix_width_dims
	)

privatepure virtual

◆ do_setup_()

virtual void lbann::weights::do_setup_ ( )

privatepure virtual

◆ do_steal_values_()

virtual void lbann::weights::do_steal_values_ ( weights & other )

privatepure virtual

◆ freeze()

void lbann::weights::freeze ( )

inline

Disable weight optimization.

Definition at line 242 of file weights/weights.hpp.

◆ get_comm()

lbann_comm& lbann::weights::get_comm ( ) const

inline

Definition at line 123 of file weights/weights.hpp.

◆ get_datatype_name()

virtual std::string lbann::weights::get_datatype_name ( ) const

pure virtual

Get a string representing the weights's datatype.

◆ get_description()

description lbann::weights::get_description ( ) const

Human-readable description.

◆ get_dims()

std::vector<size_t> lbann::weights::get_dims ( ) const

Get weight tensor dimensions. The dimensions are sorted in decreasing order of the data strides. This is a generalization of the "NCHW/NHWC" notation commonly used to describe image data.

These dimensions are obtained by concatenating the matrix width dimensions with the matrix height dimensions (in that order). If the weight matrix is duplicated on all processes (i.e. in STAR,STAR layout), the tensor data is packed w.r.t. the matrix height dimensions. If the local matrices are also fully packed, the tensor data is fully packed.

◆ get_initializer() [1/2]

virtual weights_initializer* lbann::weights::get_initializer ( )

pure virtual

Get weights initializer.

◆ get_initializer() [2/2]

virtual const weights_initializer* lbann::weights::get_initializer ( ) const

pure virtual

Get weights initializer (const).

◆ get_matrix_distribution()

El::DistData lbann::weights::get_matrix_distribution ( ) const

Here is the caller graph for this function:

◆ get_matrix_height()

size_t lbann::weights::get_matrix_height ( ) const

Get weight matrix height. If there are no matrix height dimensions, the height is one.

Here is the caller graph for this function:

◆ get_matrix_height_dims()

std::vector<size_t> lbann::weights::get_matrix_height_dims ( ) const

Get tensor dimensions corresponding to weight matrix height. The dimensions are sorted in decreasing order of strides. Matrix rows are fully-packed w.r.t. the matrix height dimensions.

Here is the caller graph for this function:

◆ get_matrix_width()

size_t lbann::weights::get_matrix_width ( ) const

Get weight matrix width. If there are no matrix width dimensions, the width is one.

Here is the caller graph for this function:

◆ get_matrix_width_dims()

std::vector<size_t> lbann::weights::get_matrix_width_dims ( ) const

Get tensor dimensions corresponding to weight matrix width. The dimensions are sorted in decreasing order of strides. Matrix columns are fully-packed w.r.t. the matrix width dimensions.

◆ get_name()

std::string lbann::weights::get_name ( ) const

inline

Get weights name.

Definition at line 121 of file weights/weights.hpp.

Here is the caller graph for this function:

◆ get_optimizer() [1/2]

virtual optimizer* lbann::weights::get_optimizer ( )

pure virtual

Get weights optimizer. Returns a null pointer if the weights are frozen.

◆ get_optimizer() [2/2]

virtual const optimizer* lbann::weights::get_optimizer ( ) const

pure virtual

Get weights optimizer. Returns a null pointer if the weights are frozen.

◆ get_size()

size_t lbann::weights::get_size ( ) const

Get number of entries in weight tensor.

◆ get_values() [1/2]

virtual El::BaseDistMatrix& lbann::weights::get_values ( )

pure virtual

Access the matrix of weights values.

◆ get_values() [2/2]

virtual El::BaseDistMatrix const& lbann::weights::get_values ( ) const

pure virtual

◆ has_optimizer()

virtual bool lbann::weights::has_optimizer ( ) const

pure virtual

◆ is_frozen()

bool lbann::weights::is_frozen ( ) const

inline

Whether weight optimization is enabled.

Definition at line 246 of file weights/weights.hpp.

Here is the call graph for this function:

◆ load_from_save()

virtual bool lbann::weights::load_from_save	(	std::string const &	ckpt_dir,
		std::vector< std::string > const &	weight_list
	)

pure virtual

◆ operator=()

weights& lbann::weights::operator= ( const weights & other )

protecteddefault

◆ reconcile_values() [1/2]

virtual void lbann::weights::reconcile_values ( )

pure virtual

Reconcile weight values. If weight values are duplicated across multiple processes, they are set to the average across the processes.

◆ reconcile_values() [2/2]

virtual void lbann::weights::reconcile_values ( Al::request & req )

pure virtual

Asynchronously reconcile weight values. If weight values are duplicated across multiple processes, they are set to the average across the processes.

◆ serialize()

template<typename ArchiveT >

void lbann::weights::serialize ( ArchiveT & ar )

Serialize the weights object to the archive.

Template Parameters

ArchiveT (Inferred.) The archive type.

Parameters

[in,out] ar The archive to which to write or from which to read.

◆ set_comm()

void lbann::weights::set_comm ( lbann_comm & comm )

private

◆ set_dims() [1/2]

void lbann::weights::set_dims	(	std::vector< size_t >	matrix_height_dims,
		std::vector< size_t >	matrix_width_dims = `{}`
	)

Set weight tensor dimensions. See the 'get_dims' function for an explanation of the notation.

Here is the caller graph for this function:

◆ set_dims() [2/2]

void lbann::weights::set_dims ( size_t size )

inline

Set weight tensor dimensions as a 1D tensor.

Definition at line 175 of file weights/weights.hpp.

Here is the call graph for this function:

Here is the caller graph for this function:

◆ set_initializer()

virtual void lbann::weights::set_initializer ( std::unique_ptr< weights_initializer > && init )

pure virtual

Set weights initializer. The contents of 'init' are moved to a class member.

◆ set_matrix_distribution()

void lbann::weights::set_matrix_distribution ( El::DistData dist )

Here is the caller graph for this function:

◆ set_name()

void lbann::weights::set_name ( std::string name )

inline

Set weights name. Each set of weights in a model should have a unique, human-readable name.

Definition at line 118 of file weights/weights.hpp.

◆ set_optimizer()

virtual void lbann::weights::set_optimizer ( std::unique_ptr< optimizer > && opt )

pure virtual

Set weights optimizer. The contents of opt are moved to a class member.

◆ set_values()

void lbann::weights::set_values ( El::BaseDistMatrix const & values )

Set the values matrix to the given matrix.

The input matrix must be compatible with the established matrix dimensions. If the data type of the input matrix is different from that expected by the weights object, they will be cast to the data type expected by the weights object.

Exceptions

lbann::exception If the input matrix has incompatible dimensions.

Todo:: (trb 05/28/2020): Should this check the DistData of the input against the expected DistData for the weights object?

◆ setup()

void lbann::weights::setup ( )

◆ setup_default_matrix_distribution()

void lbann::weights::setup_default_matrix_distribution ( )

private

◆ steal_values()

void lbann::weights::steal_values ( weights & other )

Take the values matrix from another weights object.

If the other object has the same underlying DistData, the values matrix will simply be moved over. In this case, the other object will not have valid weights after this operation completes. Otherwise, the values will be copied.

Precondition: Other weights has the same dimenions.

Postcondition: Other weights objects values may be invalidated.

Parameters

[in,out] other The object from which to steal values.

Exceptions

lbann::exception If the weights objects don't have the same dimensions.

◆ unfreeze()

void lbann::weights::unfreeze ( )

inline

Enable weight optimization.

Definition at line 244 of file weights/weights.hpp.

◆ write_proto()

virtual void lbann::weights::write_proto ( lbann_data::Weights & proto ) const

pure virtual

Write weights to proto file

Member Data Documentation

◆ m_comm

lbann_comm* lbann::weights::m_comm

private

Reference to LBANN communicator.

Definition at line 327 of file weights/weights.hpp.

◆ m_frozen

bool lbann::weights::m_frozen

private

Whether weight optimization is disabled.

Definition at line 341 of file weights/weights.hpp.

◆ m_matrix_dist

El::DistData lbann::weights::m_matrix_dist

private

Distribution of weights matrix.

Definition at line 338 of file weights/weights.hpp.

◆ m_matrix_height_dims

std::vector<size_t> lbann::weights::m_matrix_height_dims

private

Tensor dimensions corresponding to matrix height. See the 'get_matrix_height_dims' function.

Definition at line 332 of file weights/weights.hpp.

◆ m_matrix_width_dims

std::vector<size_t> lbann::weights::m_matrix_width_dims

private

Tensor dimensions corresponding to matrix width. See the 'get_matrix_width_dims' function.

Definition at line 336 of file weights/weights.hpp.

◆ m_name

std::string lbann::weights::m_name

private

Weights name. Each set of weights in a model should have a unique, human-readable name.

Definition at line 324 of file weights/weights.hpp.

The documentation for this class was generated from the following file:

weights/weights.hpp

Public Member Functions

Protected Member Functions

Private Member Functions

Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ weights() [1/3]

◆ weights() [2/3]

◆ ~weights()

◆ weights() [3/3]

Member Function Documentation

◆ do_augment_description_()

◆ do_set_dims_()

◆ do_setup_()

◆ do_steal_values_()

◆ freeze()

◆ get_comm()

◆ get_datatype_name()

◆ get_description()

◆ get_dims()

◆ get_initializer() [1/2]

◆ get_initializer() [2/2]

◆ get_matrix_distribution()

◆ get_matrix_height()

◆ get_matrix_height_dims()

◆ get_matrix_width()

◆ get_matrix_width_dims()

◆ get_name()

◆ get_optimizer() [1/2]

◆ get_optimizer() [2/2]

◆ get_size()

◆ get_values() [1/2]

◆ get_values() [2/2]

◆ has_optimizer()

◆ is_frozen()

◆ load_from_save()

◆ operator=()

◆ reconcile_values() [1/2]

◆ reconcile_values() [2/2]

◆ serialize()

◆ set_comm()

◆ set_dims() [1/2]

◆ set_dims() [2/2]

◆ set_initializer()

◆ set_matrix_distribution()

◆ set_name()

◆ set_optimizer()

◆ set_values()

◆ setup()

◆ setup_default_matrix_distribution()

◆ steal_values()

◆ unfreeze()

◆ write_proto()

Member Data Documentation

◆ m_comm

◆ m_frozen

◆ m_matrix_dist

◆ m_matrix_height_dims

◆ m_matrix_width_dims

◆ m_name