Classes
class	KFACExecutionContext

Typedefs
using	BackendT = ::Al::MPIBackend

using	ReqT = typename BackendT::req_type

Enumerations
enum	kfac_inverse_strategy { kfac_inverse_strategy::ALL, kfac_inverse_strategy::EACH, kfac_inverse_strategy::ROOT }

enum	kfac_reduce_scatter_mode { kfac_reduce_scatter_mode::ALLREDUCE, kfac_reduce_scatter_mode::REDUCE_SCATTER, kfac_reduce_scatter_mode::REDUCE }

enum	kfac_allgather_mode { kfac_allgather_mode::ALLREDUCE, kfac_allgather_mode::ALLGATHER, kfac_allgather_mode::BROADCAST }

Functions
template<El::Device Device>
void	get_matrix_inverse (El::AbstractMatrix< DataType > &Ainv, El::AbstractMatrix< DataType > &Linv, const El::AbstractMatrix< DataType > &A, bool report_time, DataType damping, DataType damping_bn_err, bool is_bn, const El::SyncInfo< Device > &sync_info)
	Gets the inverse matrix of A. More...

template<El::Device Device>
void	get_matrix_inverse_eigen (El::AbstractMatrix< DataType > &Ainv, El::AbstractMatrix< DataType > &Linv, const El::AbstractMatrix< DataType > &A, bool report_time, DataType damping, DataType damping_bn_err, bool is_bn, const El::SyncInfo< Device > &sync_info)
	Gets the inverse matrix of A using Eigen Value Decomposition. More...

template<El::Device Device>
std::string	get_matrix_stat (const El::Matrix< DataType, Device > &X, const char *name)
	Gets statistics of a given matrix. More...

template<El::Device Device>
void	allreduce_lower_tri (El::AbstractMatrix< DataType > &A, El::AbstractMatrix< DataType > &AL, lbann_comm *comm, const El::SyncInfo< Device > &sync_info)
	Perform all-reduce on the lower triangular of a symmetric matrix. More...

bool	is_reduce_scatter_buffer_required (kfac_reduce_scatter_mode mode)
	Get whether a global buffer is needed. More...

template<El::Device Device>
void	reduce_scatter_blocks (const std::vector< std::pair< size_t, El::AbstractMatrix< DataType > >> &blocks, El::Matrix< DataType, Device > &global_buffer, lbann_comm comm, kfac_reduce_scatter_mode mode)
	Perform reduce-scatter on one or more blocks. More...

std::pair< bool, bool >	is_allgather_buffer_required (kfac_allgather_mode mode)
	Get whether local and global buffers are needed. More...

template<El::Device Device>
void	allgather_blocks (const std::vector< std::pair< size_t, El::AbstractMatrix< DataType > >> &blocks, El::Matrix< DataType, Device > &send_buffer, El::Matrix< DataType, Device > &recv_buffer, lbann_comm comm, kfac_allgather_mode mode)
	Perform reduce-scatter on one or more blocks. More...

template<El::Device Device>
void	allgather_inverse_matrices (const std::vector< std::shared_ptr< kfac_block< Device >>> &blocks, El::Matrix< DataType, Device > &global_buffer, lbann_comm *comm)
	Perform allgather for inverse matrices. More...

template<El::Device Device>
void	allgather_inverse_matrices_sizes (const std::vector< std::shared_ptr< kfac_block< Device >>> &blocks, El::Matrix< double, El::Device::CPU > &global_buffer, lbann_comm *comm)
	Perform allgather for inverse matrices size. More...

template<El::Device Device>
void	add_to_diagonal (El::Matrix< DataType, Device > &A, DataType value, DataType value_bn_err, bool is_bn, const El::SyncInfo< Device > &sync_info)
	Add the damping value to the diagonal elements of A. More...

template<El::Device Device>
void	make_diagonal (El::Matrix< DataType, Device > &A, El::Matrix< DataType, Device > &B, DataType value, DataType value_bn_err, bool is_bn, const El::SyncInfo< Device > &sync_info)
	Add the damping value to the diagonal elements of A from B. More...

template<El::Device Device>
void	get_matrix_entrywise_inverse (El::Matrix< DataType, Device > &input, El::Matrix< DataType, Device > &output, const El::SyncInfo< Device > &sync_info)
	Add the damping value to the diagonal elements of A. More...

template<El::Device Device>
void	fill_upper_tri (El::Matrix< DataType, Device > &A, const El::SyncInfo< Device > &sync_info)
	Fill the upper trianglar with the lower trianglar. More...

template<El::Device Device>
void	update_kronecker_average (El::Matrix< DataType, Device > &Aave, const El::Matrix< DataType, Device > &A, size_t count, double decay, const El::SyncInfo< Device > &sync_info)
	Update a Kronecker factor matrix using decay. More...

template<El::Device Device>
void	identity (El::Matrix< DataType, Device > &A, const El::SyncInfo< Device > &sync_info)
	Substitute the identity matrix. TODO: Replace with El::Identity<El::Device::GPU> once it gets supported. More...

template<El::Device Device>
void	pack_lower_tri (El::Matrix< DataType, Device > &L, const El::Matrix< DataType, Device > &A, const El::SyncInfo< Device > &sync_info)
	Pack the lower triangular of a symmetric matrix. More...

template<El::Device Device>
void	unpack_lower_tri (El::Matrix< DataType, Device > &A, const El::Matrix< DataType, Device > &L, const El::SyncInfo< Device > &sync_info)
	Unpack the lower triangular of a symmetric matrix. More...

template<typename T , El::Device Device>
void	TranslateBetweenGridsVCAsync (const El::DistMatrix< T, El::STAR, El::VC, El::ELEMENT, Device > &A, El::DistMatrix< T, El::STAR, El::VC, El::ELEMENT, Device > &B, El::DistMatrix< T, El::STAR, El::VC, El::ELEMENT, Device > &subset, std::vector< ReqT > &Requests)

template<typename T , El::Device Device>
void	TranslateBetweenGridsVCAsyncDirect (const El::DistMatrix< T, El::STAR, El::VC, El::ELEMENT, Device > &A, El::DistMatrix< T, El::STAR, El::VC, El::ELEMENT, Device > &B, El::Int featureSize, El::Int currentBatchSize, std::vector< ReqT > &Requests)

template<typename T , El::Device Device>
void	TranslateBetweenGridsSTARAsync (const El::DistMatrix< T, El::STAR, El::STAR, El::ELEMENT, Device > &A, El::DistMatrix< T, El::STAR, El::STAR, El::ELEMENT, Device > &B, std::vector< ReqT > &Requests)

template<typename T , El::Device Device>
void	TranslateBetweenGridsKFACAsync (const El::DistMatrix< T, El::STAR, El::VC, El::ELEMENT, Device > &A, El::DistMatrix< T, El::STAR, El::VC, El::ELEMENT, Device > &B, std::vector< ReqT > &Requests)

template<typename T , El::Device Device>
void	TranslateBetweenGridsVC (El::DistMatrix< T, El::STAR, El::VC, El::ELEMENT, Device > const &A, El::DistMatrix< T, El::STAR, El::VC, El::ELEMENT, Device > &B)

Variables
constexpr El::Device	Device = El::Device::CPU

Typedef Documentation

◆ BackendT

typedef::Al::MPIBackend lbann::kfac::BackendT

Definition at line 46 of file kfac_block.hpp.

◆ ReqT

typedef typename BackendT::req_type lbann::kfac::ReqT

Definition at line 49 of file kfac_block.hpp.

Enumeration Type Documentation

◆ kfac_allgather_mode

enum lbann::kfac::kfac_allgather_mode

strong

Enumerator
ALLREDUCE
ALLGATHER
BROADCAST

Definition at line 74 of file kfac_util.hpp.

◆ kfac_inverse_strategy

enum lbann::kfac::kfac_inverse_strategy

strong

Enumerator
ALL
EACH
ROOT

Definition at line 58 of file kfac_util.hpp.

◆ kfac_reduce_scatter_mode

enum lbann::kfac::kfac_reduce_scatter_mode

strong

Enumerator
ALLREDUCE
REDUCE_SCATTER
REDUCE

Definition at line 67 of file kfac_util.hpp.

Function Documentation

◆ add_to_diagonal()

template<El::Device Device>

void lbann::kfac::add_to_diagonal	(	El::Matrix< DataType, Device > &	A,
		DataType	value,
		DataType	value_bn_err,
		bool	is_bn,
		const El::SyncInfo< Device > &	sync_info
	)

Add the damping value to the diagonal elements of A.

◆ allgather_blocks()

template<El::Device Device>

void lbann::kfac::allgather_blocks	(	const std::vector< std::pair< size_t, El::AbstractMatrix< DataType > *>> &	blocks,
		El::Matrix< DataType, Device > &	send_buffer,
		El::Matrix< DataType, Device > &	recv_buffer,
		lbann_comm *	comm,
		kfac_allgather_mode	mode
	)

Perform reduce-scatter on one or more blocks.

◆ allgather_inverse_matrices()

template<El::Device Device>

void lbann::kfac::allgather_inverse_matrices	(	const std::vector< std::shared_ptr< kfac_block< Device >>> &	blocks,
		El::Matrix< DataType, Device > &	global_buffer,
		lbann_comm *	comm
	)

Perform allgather for inverse matrices.

◆ allgather_inverse_matrices_sizes()

template<El::Device Device>

void lbann::kfac::allgather_inverse_matrices_sizes	(	const std::vector< std::shared_ptr< kfac_block< Device >>> &	blocks,
		El::Matrix< double, El::Device::CPU > &	global_buffer,
		lbann_comm *	comm
	)

Perform allgather for inverse matrices size.

◆ allreduce_lower_tri()

template<El::Device Device>

void lbann::kfac::allreduce_lower_tri	(	El::AbstractMatrix< DataType > &	A,
		El::AbstractMatrix< DataType > &	AL,
		lbann_comm *	comm,
		const El::SyncInfo< Device > &	sync_info
	)

Perform all-reduce on the lower triangular of a symmetric matrix.

◆ fill_upper_tri()

template<El::Device Device>

void lbann::kfac::fill_upper_tri	(	El::Matrix< DataType, Device > &	A,
		const El::SyncInfo< Device > &	sync_info
	)

Fill the upper trianglar with the lower trianglar.

◆ get_matrix_entrywise_inverse()

template<El::Device Device>

void lbann::kfac::get_matrix_entrywise_inverse	(	El::Matrix< DataType, Device > &	input,
		El::Matrix< DataType, Device > &	output,
		const El::SyncInfo< Device > &	sync_info
	)

Add the damping value to the diagonal elements of A.

◆ get_matrix_inverse()

template<El::Device Device>

void lbann::kfac::get_matrix_inverse	(	El::AbstractMatrix< DataType > &	Ainv,
		El::AbstractMatrix< DataType > &	Linv,
		const El::AbstractMatrix< DataType > &	A,
		bool	report_time,
		DataType	damping,
		DataType	damping_bn_err,
		bool	is_bn,
		const El::SyncInfo< Device > &	sync_info
	)

Gets the inverse matrix of A.

◆ get_matrix_inverse_eigen()

template<El::Device Device>

void lbann::kfac::get_matrix_inverse_eigen	(	El::AbstractMatrix< DataType > &	Ainv,
		El::AbstractMatrix< DataType > &	Linv,
		const El::AbstractMatrix< DataType > &	A,
		bool	report_time,
		DataType	damping,
		DataType	damping_bn_err,
		bool	is_bn,
		const El::SyncInfo< Device > &	sync_info
	)

Gets the inverse matrix of A using Eigen Value Decomposition.

◆ get_matrix_stat()

template<El::Device Device>

std::string lbann::kfac::get_matrix_stat	(	const El::Matrix< DataType, Device > &	X,
		const char *	name
	)

Gets statistics of a given matrix.

◆ identity()

template<El::Device Device>

void lbann::kfac::identity	(	El::Matrix< DataType, Device > &	A,
		const El::SyncInfo< Device > &	sync_info
	)

Substitute the identity matrix. TODO: Replace with El::Identity<El::Device::GPU> once it gets supported.

◆ is_allgather_buffer_required()

std::pair<bool, bool> lbann::kfac::is_allgather_buffer_required ( kfac_allgather_mode mode )

Get whether local and global buffers are needed.

◆ is_reduce_scatter_buffer_required()

bool lbann::kfac::is_reduce_scatter_buffer_required ( kfac_reduce_scatter_mode mode )

Get whether a global buffer is needed.

◆ make_diagonal()

template<El::Device Device>

void lbann::kfac::make_diagonal	(	El::Matrix< DataType, Device > &	A,
		El::Matrix< DataType, Device > &	B,
		DataType	value,
		DataType	value_bn_err,
		bool	is_bn,
		const El::SyncInfo< Device > &	sync_info
	)

Add the damping value to the diagonal elements of A from B.

◆ pack_lower_tri()

template<El::Device Device>

void lbann::kfac::pack_lower_tri	(	El::Matrix< DataType, Device > &	L,
		const El::Matrix< DataType, Device > &	A,
		const El::SyncInfo< Device > &	sync_info
	)

Pack the lower triangular of a symmetric matrix.

◆ reduce_scatter_blocks()

template<El::Device Device>

void lbann::kfac::reduce_scatter_blocks	(	const std::vector< std::pair< size_t, El::AbstractMatrix< DataType > *>> &	blocks,
		El::Matrix< DataType, Device > &	global_buffer,
		lbann_comm *	comm,
		kfac_reduce_scatter_mode	mode
	)

Perform reduce-scatter on one or more blocks.