#include <timeline.hpp>

Inheritance diagram for lbann::callback::timeline:

Collaboration diagram for lbann::callback::timeline:

Public Member Functions
	timeline (std::string outdir)

	timeline (const timeline &)=default

timeline &	operator= (const timeline &)=default

timeline *	copy () const override

std::string	name () const override
	Return this callback's name. More...

void	on_train_begin (model *m) override
	Called at the beginning of training. More...

void	on_train_end (model *m) override
	Called at the end of training. More...

void	on_forward_prop_begin (model m, Layer l) override
	Called when a layer begins forward propagation. More...

void	on_forward_prop_end (model m, Layer l) override
	Called when a layer ends forward propagation. More...

void	on_backward_prop_begin (model m, Layer l) override
	Called when a layer begins backward propagation. More...

void	on_backward_prop_end (model m, Layer l) override
	Called when a layer ends backward propagation. More...

void	on_optimize_begin (model m, weights w) override
	Called when weights begins optimization. More...

void	on_optimize_end (model m, weights w) override
	Called when weights ends optimization. More...

Serialization
template<class Archive >
void	serialize (Archive &ar)
	Store state to archive for checkpoint and restart. More...

Public Member Functions inherited from lbann::callback_base
	callback_base (int batch_interval=1)
	Initialize a callback with an optional batch interval. More...

	callback_base (const callback_base &)=default

virtual	~callback_base ()=default

virtual void	setup (trainer *t)
	Called once to set up the callback on the trainer. More...

virtual void	setup (model *m)
	Called once to set up the callback on the model (after all layers are set up). More...

virtual void	on_setup_end (model *m)
	Called at the end of setup. More...

virtual void	on_phase_end (model *m)
	Called at the end of every phase (multiple epochs) in a layer-wise model training. More...

virtual void	on_epoch_begin (model *m)
	Called at the beginning of each epoch. More...

virtual void	on_epoch_end (model *m)
	Called immediate after the end of each epoch. More...

virtual void	on_batch_begin (model *m)
	Called at the beginning of a (mini-)batch. More...

virtual void	on_batch_end (model *m)
	Called immediately after the end of a (mini-)batch. More...

virtual void	on_test_begin (model *m)
	Called at the beginning of testing. More...

virtual void	on_test_end (model *m)
	Called immediately after the end of testing. More...

virtual void	on_validation_begin (model *m)
	Called at the beginning of validation. More...

virtual void	on_validation_end (model *m)
	Called immediately after the end of validation. More...

virtual void	on_forward_prop_begin (model *m)
	Called when a model begins forward propagation. More...

virtual void	on_forward_prop_end (model *m)
	Called when a model ends forward propagation. More...

virtual void	on_backward_prop_begin (model *m)
	Called when a model begins backward propagation. More...

virtual void	on_backward_prop_end (model *m)
	Called when a model ends backward propagation. More...

virtual void	on_optimize_begin (model *m)
	Called when a model begins optimization. More...

virtual void	on_optimize_end (model *m)
	Called when a model ends optimization. More...

virtual void	on_batch_evaluate_begin (model *m)
	Called at the beginning of a (mini-)batch evaluation (validation / testing). More...

virtual void	on_batch_evaluate_end (model *m)
	Called at the end of a (mini-)batch evaluation (validation / testing). More...

virtual void	on_evaluate_forward_prop_begin (model *m)
	Called when a model begins forward propagation for evaluation (validation / testing). More...

virtual void	on_evaluate_forward_prop_begin (model m, Layer l)
	Called when a layer begins forward propagation for evaluation (validation / testing). More...

virtual void	on_evaluate_forward_prop_end (model *m)
	Called when a model ends forward propagation for evaluation (validation / testing). More...

virtual void	on_evaluate_forward_prop_end (model m, Layer l)
	Called when a layer ends forward propagation for evaluation (validation / testing). More...

int	get_batch_interval () const
	Return the batch interval. More...

virtual description	get_description () const
	Human-readable description. More...

template<class Archive >
void	serialize (Archive &ar)
	Store state to archive for checkpoint and restart. More...

void	write_proto (lbann_data::Callback &proto) const
	Write a protobuf description of the callback. More...

Private Member Functions
void	write_specific_proto (lbann_data::Callback &proto) const final

	timeline ()

EvalType	get_rel_time () const
	Get time relative to the start time. More...

Private Attributes
std::string	m_outdir
	Directory to write output to. More...

EvalType	m_start_time = EvalType(0)
	Time training started; all times are relative to this. More...

EvalType	m_fp_start_time = EvalType(0)
	Time the current layer's forward pass started. More...

EvalType	m_bp_start_time = EvalType(0)
	Time the current layer's backward pass started. More...

EvalType	m_opt_start_time = EvalType(0)
	Time the current weights' optimization pass started. More...

std::unordered_map< std::string, std::vector< std::pair< EvalType, EvalType > > >	m_fp_times
	Store (relative) timing information. More...

std::unordered_map< std::string, std::vector< std::pair< EvalType, EvalType > > >	m_bp_times

std::unordered_map< std::string, std::vector< std::pair< EvalType, EvalType > > >	m_opt_times

Friends
class	cereal::access

Additional Inherited Members
Protected Member Functions inherited from lbann::callback_base
std::string	get_multi_trainer_path (const model &m, const std::string &root_dir)
	Build a standard directory hierarchy including trainer ID. More...

std::string	get_multi_trainer_ec_model_path (const model &m, const std::string &root_dir)
	Build a standard directory hierachy including trainer, execution context, and model information (in that order). More...

std::string	get_multi_trainer_model_path (const model &m, const std::string &root_dir)
	Build a standard directory hierachy including trainer, model information in that order. More...

callback_base &	operator= (const callback_base &)=default
	Copy-assignment operator. More...

Protected Attributes inherited from lbann::callback_base
int	m_batch_interval
	Batch methods should once every this many steps. More...

Detailed Description

Record a timeline of training runtime on each rank and output it to a logfile for external processing. The logfile is named timeline.m<model-rank>.<rank>.txt. Each line is a separate event, written as name:start-time:end-time. Times are relative to the beginning of training.

Definition at line 48 of file timeline.hpp.

Constructor & Destructor Documentation

◆ timeline() [1/3]

lbann::callback::timeline::timeline ( std::string outdir )

inline

Definition at line 51 of file timeline.hpp.

Here is the call graph for this function:

◆ timeline() [2/3]

lbann::callback::timeline::timeline ( const timeline & )

default

◆ timeline() [3/3]

lbann::callback::timeline::timeline ( )

private

Here is the caller graph for this function:

Member Function Documentation

◆ copy()

timeline* lbann::callback::timeline::copy ( ) const

inlineoverridevirtual

Implements lbann::callback_base.

Definition at line 54 of file timeline.hpp.

Here is the call graph for this function:

◆ get_rel_time()

EvalType lbann::callback::timeline::get_rel_time ( ) const

inlineprivate

Get time relative to the start time.

Definition at line 90 of file timeline.hpp.

Here is the call graph for this function:

◆ name()

std::string lbann::callback::timeline::name ( ) const

inlineoverridevirtual

Return this callback's name.

Implements lbann::callback_base.

Definition at line 55 of file timeline.hpp.

Here is the call graph for this function:

◆ on_backward_prop_begin()

void lbann::callback::timeline::on_backward_prop_begin	(	model *	m,
		Layer *	l
	)

overridevirtual

Called when a layer begins backward propagation.

Reimplemented from lbann::callback_base.

Here is the caller graph for this function:

◆ on_backward_prop_end()

void lbann::callback::timeline::on_backward_prop_end	(	model *	m,
		Layer *	l
	)

overridevirtual

Called when a layer ends backward propagation.

Reimplemented from lbann::callback_base.

Here is the caller graph for this function:

◆ on_forward_prop_begin()

void lbann::callback::timeline::on_forward_prop_begin	(	model *	m,
		Layer *	l
	)

overridevirtual

Called when a layer begins forward propagation.

Reimplemented from lbann::callback_base.

Here is the caller graph for this function:

◆ on_forward_prop_end()

void lbann::callback::timeline::on_forward_prop_end	(	model *	m,
		Layer *	l
	)

overridevirtual

Called when a layer ends forward propagation.

Reimplemented from lbann::callback_base.

Here is the caller graph for this function:

◆ on_optimize_begin()

void lbann::callback::timeline::on_optimize_begin	(	model *	m,
		weights *	w
	)

overridevirtual

Called when weights begins optimization.

Reimplemented from lbann::callback_base.

Here is the caller graph for this function:

◆ on_optimize_end()

void lbann::callback::timeline::on_optimize_end	(	model *	m,
		weights *	w
	)

overridevirtual

Called when weights ends optimization.

Reimplemented from lbann::callback_base.

Here is the caller graph for this function:

◆ on_train_begin()

void lbann::callback::timeline::on_train_begin ( model * m )

overridevirtual

Called at the beginning of training.

Reimplemented from lbann::callback_base.

Here is the caller graph for this function:

◆ on_train_end()

void lbann::callback::timeline::on_train_end ( model * m )

overridevirtual

Called at the end of training.

Reimplemented from lbann::callback_base.

Here is the caller graph for this function:

◆ operator=()

timeline& lbann::callback::timeline::operator= ( const timeline & )

default

Here is the caller graph for this function:

◆ serialize()

template<class Archive >

void lbann::callback::timeline::serialize ( Archive & ar )

Store state to archive for checkpoint and restart.

Here is the caller graph for this function:

◆ write_specific_proto()

void lbann::callback::timeline::write_specific_proto ( lbann_data::Callback & proto ) const

finalprivatevirtual

Add callback specific data to prototext

Implements lbann::callback_base.

Here is the caller graph for this function:

Friends And Related Function Documentation

◆ cereal::access

friend class cereal::access

friend

Definition at line 86 of file timeline.hpp.

Member Data Documentation

◆ m_bp_start_time

EvalType lbann::callback::timeline::m_bp_start_time = EvalType(0)

private

Time the current layer's backward pass started.

Definition at line 99 of file timeline.hpp.

◆ m_bp_times

std::unordered_map<std::string, std::vector<std::pair<EvalType, EvalType> > > lbann::callback::timeline::m_bp_times

private

Definition at line 106 of file timeline.hpp.

◆ m_fp_start_time

EvalType lbann::callback::timeline::m_fp_start_time = EvalType(0)

private

Time the current layer's forward pass started.

Definition at line 97 of file timeline.hpp.

◆ m_fp_times

std::unordered_map<std::string, std::vector<std::pair<EvalType, EvalType> > > lbann::callback::timeline::m_fp_times

private

Store (relative) timing information.

Definition at line 104 of file timeline.hpp.

◆ m_opt_start_time

EvalType lbann::callback::timeline::m_opt_start_time = EvalType(0)

private

Time the current weights' optimization pass started.

Definition at line 101 of file timeline.hpp.

◆ m_opt_times

std::unordered_map<std::string, std::vector<std::pair<EvalType, EvalType> > > lbann::callback::timeline::m_opt_times

private

Definition at line 108 of file timeline.hpp.

◆ m_outdir

std::string lbann::callback::timeline::m_outdir

private

Directory to write output to.

Definition at line 93 of file timeline.hpp.

◆ m_start_time

EvalType lbann::callback::timeline::m_start_time = EvalType(0)

private

Time training started; all times are relative to this.

Definition at line 95 of file timeline.hpp.

The documentation for this class was generated from the following file:

timeline.hpp

Public Member Functions

Private Member Functions

Private Attributes

Friends

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ timeline() [1/3]

◆ timeline() [2/3]

◆ timeline() [3/3]

Member Function Documentation

◆ copy()

◆ get_rel_time()

◆ name()

◆ on_backward_prop_begin()

◆ on_backward_prop_end()

◆ on_forward_prop_begin()

◆ on_forward_prop_end()

◆ on_optimize_begin()

◆ on_optimize_end()

◆ on_train_begin()

◆ on_train_end()

◆ operator=()

◆ serialize()

◆ write_specific_proto()

Friends And Related Function Documentation

◆ cereal::access

Member Data Documentation

◆ m_bp_start_time

◆ m_bp_times

◆ m_fp_start_time

◆ m_fp_times

◆ m_opt_start_time

◆ m_opt_times

◆ m_outdir

◆ m_start_time