+ echo 'Beginning trial 10 of 10'
Beginning trial 10 of 10
+ docker exec -it language_model python -c '
import mlperf_logger 
from mlperf_logging.mllog import constants 
mlperf_logger.mlperf_submission_log("bert")'
:::MLLOG {"namespace": "", "time_ms": 1623991652191, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "bert", "metadata": {"file": "/workspace/bert/mlperf_logger.py", "lineno": 66}}
:::MLLOG {"namespace": "", "time_ms": 1623991652215, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "Inspur", "metadata": {"file": "/workspace/bert/mlperf_logger.py", "lineno": 71}}
:::MLLOG {"namespace": "", "time_ms": 1623991652215, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/workspace/bert/mlperf_logger.py", "lineno": 75}}
:::MLLOG {"namespace": "", "time_ms": 1623991652215, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/workspace/bert/mlperf_logger.py", "lineno": 79}}
:::MLLOG {"namespace": "", "time_ms": 1623991652215, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "1xNF5488A5", "metadata": {"file": "/workspace/bert/mlperf_logger.py", "lineno": 83}}
+ '[' 1 -eq 1 ']'
+ sync
+ /sbin/sysctl vm.drop_caches=3
vm.drop_caches = 3
+ docker exec -it language_model python -c '
from mlperf_logging.mllog import constants 
from mlperf_logger import log_event 
log_event(key=constants.CACHE_CLEAR, value=True)'
:::MLLOG {"namespace": "", "time_ms": 1623991655343, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "<string>", "lineno": 4}}
+ export SEED=13548
+ SEED=13548
+ docker exec -it --env=BATCHSIZE --env=CHECKPOINTDIR --env=CHECKPOINTDIR_PHASE1 --env=DATADIR --env=DATADIR_PHASE2 --env=DGXHT --env=DGXNGPU --env=DGXNNODES --env=DGXNSOCKET --env=DGXSOCKETCORES --env=DGXSYSTEM --env=DWU_NUM_AR_PG --env=DWU_NUM_RS_PG --env=EVALDIR --env=EVAL_ITER_SAMPLES --env=EVAL_ITER_START_SAMPLES --env=EXTRA_PARAMS --env=GRADIENT_STEPS --env=LR --env=MAX_SAMPLES_TERMINATION --env=MAX_STEPS --env=OPT_LAMB_BETA_1 --env=OPT_LAMB_BETA_2 --env=PHASE --env=SLURM_NTASKS --env=START_WARMUP_STEP --env=UNITTESTDIR --env=WALLTIME --env=WARMUP_PROPORTION --env=SEED language_model sh -c './run_and_time.sh "    python -u -m bind_launch_nf5488a5 --nsockets_per_node=2 --ncores_per_socket=64 --nproc_per_node=8     /workspace/bert/run_pretraining.py         --train_batch_size=56     --learning_rate=3.5e-4     --opt_lamb_beta_1=0.9     --opt_lamb_beta_2=0.999     --warmup_proportion=0.0     --warmup_steps=0.0     --start_warmup_step=0     --max_steps=7100     --phase2     --max_seq_length=512     --max_predictions_per_seq=76     --input_dir=/workspace/data_phase2     --init_checkpoint=/workspace/phase1/model.ckpt-28252.pt          --do_train     --skip_checkpoint     --train_mlm_accuracy_window_size=0     --target_mlm_accuracy=0.720     --weight_decay_rate=0.01     --max_samples_termination=4500000     --eval_iter_start_samples=150000 --eval_iter_samples=150000     --eval_batch_size=16 --eval_dir=/workspace/evaldata     --cache_eval_data     --output_dir=/results     --fp16 --fused_gelu_bias --fused_mha --dense_seq_output --unpad --unpad_fmha --exchange_padding     --distributed_lamb   --dwu-num-rs-pg=1 --dwu-num-ar-pg=1 --dwu-num-blocks=1      --gradient_accumulation_steps=1     --log_freq=0     --bert_config_path=/workspace/phase1/bert_config.json --allreduce_post_accumulation --allreduce_post_accumulation_fp16" 13548'
Run vars: id 10344 gpus 8 mparams 
STARTING TIMING RUN AT 2021-06-18 04:47:35 AM
+ eval '     python -u -m bind_launch_nf5488a5 --nsockets_per_node=2 --ncores_per_socket=64 --nproc_per_node=8     /workspace/bert/run_pretraining.py         --train_batch_size=56     --learning_rate=3.5e-4     --opt_lamb_beta_1=0.9     --opt_lamb_beta_2=0.999     --warmup_proportion=0.0     --warmup_steps=0.0     --start_warmup_step=0     --max_steps=7100     --phase2     --max_seq_length=512     --max_predictions_per_seq=76     --input_dir=/workspace/data_phase2     --init_checkpoint=/workspace/phase1/model.ckpt-28252.pt          --do_train     --skip_checkpoint     --train_mlm_accuracy_window_size=0     --target_mlm_accuracy=0.720     --weight_decay_rate=0.01     --max_samples_termination=4500000     --eval_iter_start_samples=150000 --eval_iter_samples=150000     --eval_batch_size=16 --eval_dir=/workspace/evaldata     --cache_eval_data     --output_dir=/results     --fp16 --fused_gelu_bias --fused_mha --dense_seq_output --unpad --unpad_fmha --exchange_padding     --distributed_lamb   --dwu-num-rs-pg=1 --dwu-num-ar-pg=1 --dwu-num-blocks=1      --gradient_accumulation_steps=1     --log_freq=0     --bert_config_path=/workspace/phase1/bert_config.json --allreduce_post_accumulation --allreduce_post_accumulation_fp16 --seed=13548'
++ python -u -m bind_launch_nf5488a5 --nsockets_per_node=2 --ncores_per_socket=64 --nproc_per_node=8 /workspace/bert/run_pretraining.py --train_batch_size=56 --learning_rate=3.5e-4 --opt_lamb_beta_1=0.9 --opt_lamb_beta_2=0.999 --warmup_proportion=0.0 --warmup_steps=0.0 --start_warmup_step=0 --max_steps=7100 --phase2 --max_seq_length=512 --max_predictions_per_seq=76 --input_dir=/workspace/data_phase2 --init_checkpoint=/workspace/phase1/model.ckpt-28252.pt --do_train --skip_checkpoint --train_mlm_accuracy_window_size=0 --target_mlm_accuracy=0.720 --weight_decay_rate=0.01 --max_samples_termination=4500000 --eval_iter_start_samples=150000 --eval_iter_samples=150000 --eval_batch_size=16 --eval_dir=/workspace/evaldata --cache_eval_data --output_dir=/results --fp16 --fused_gelu_bias --fused_mha --dense_seq_output --unpad --unpad_fmha --exchange_padding --distributed_lamb --dwu-num-rs-pg=1 --dwu-num-ar-pg=1 --dwu-num-blocks=1 --gradient_accumulation_steps=1 --log_freq=0 --bert_config_path=/workspace/phase1/bert_config.json --allreduce_post_accumulation --allreduce_post_accumulation_fp16 --seed=13548
:::MLLOG {"namespace": "", "time_ms": 1623991658599, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 996}}
:::MLLOG {"namespace": "", "time_ms": 1623991658605, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 996}}
:::MLLOG {"namespace": "", "time_ms": 1623991658622, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 996}}
:::MLLOG {"namespace": "", "time_ms": 1623991658637, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 996}}
:::MLLOG {"namespace": "", "time_ms": 1623991658724, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 996}}
:::MLLOG {"namespace": "", "time_ms": 1623991658754, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 996}}
:::MLLOG {"namespace": "", "time_ms": 1623991658766, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 996}}
:::MLLOG {"namespace": "", "time_ms": 1623991658776, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 996}}
device: cuda:0 n_gpu: 8, distributed training: True, 16-bits training: True
:::MLLOG {"namespace": "", "time_ms": 1623991659840, "event_type": "POINT_IN_TIME", "key": "seed", "value": 13548, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1006}}
:::MLLOG {"namespace": "", "time_ms": 1623991659840, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 448, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1008}}
device: cuda:4 n_gpu: 8, distributed training: True, 16-bits training: True
:::MLLOG {"namespace": "", "time_ms": 1623991659841, "event_type": "POINT_IN_TIME", "key": "d_batch_size", "value": 56, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1010}}
:::MLLOG {"namespace": "", "time_ms": 1623991659841, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1012}}
:::MLLOG {"namespace": "", "time_ms": 1623991659841, "event_type": "POINT_IN_TIME", "key": "max_predictions_per_seq", "value": 76, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1014}}
:::MLLOG {"namespace": "", "time_ms": 1623991659841, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_training_steps", "value": 7100.0, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1016}}
:::MLLOG {"namespace": "", "time_ms": 1623991659841, "event_type": "POINT_IN_TIME", "key": "num_warmup_steps", "value": 0, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1018}}
parsed args:
Namespace(allreduce_post_accumulation=True, allreduce_post_accumulation_fp16=True, bert_config_path='/workspace/phase1/bert_config.json', bert_model='bert-large-uncased', bypass_amp=False, cache_eval_data=True, checkpoint_activations=False, cuda_graph_mode='segmented', ddp_type='apex', dense_seq_output=True, device=device(type='cuda', index=0), disable_apex_softmax=False, disable_fuse_mask=False, disable_fuse_qkv=False, disable_fuse_scale=False, distributed_lamb=True, do_train=True, dwu_e5m2_allgather=False, dwu_group_size=0, dwu_num_ag_pg=2, dwu_num_ar_pg=1, dwu_num_blocks=1, dwu_num_chunks=1, dwu_num_rs_pg=1, dwu_overlap_reductions=False, enable_fuse_dropout=False, enable_stream=False, eval_batch_size=16, eval_dir='/workspace/evaldata', eval_iter_samples=150000, eval_iter_start_samples=150000, exchange_padding=True, fp16=True, fused_dropout_add=False, fused_gelu_bias=True, fused_mha=True, gradient_accumulation_steps=1, init_checkpoint='/workspace/phase1/model.ckpt-28252.pt', init_tf_checkpoint=None, input_dir='/workspace/data_phase2', keep_n_most_recent_checkpoints=20, learning_rate=0.00035, local_rank=0, log_freq=0.0, loss_scale=0.0, max_iterations_per_graph=4, max_predictions_per_seq=76, max_samples_termination=4500000.0, max_seq_length=512, max_steps=7100.0, min_samples_to_start_checkpoints=3000000, n_gpu=8, num_epochs_to_generate_seeds_for=2, num_eval_examples=10000, num_samples_per_checkpoint=500000, opt_lamb_beta_1=0.9, opt_lamb_beta_2=0.999, output_dir='/results', pad=False, phase2=True, resume_from_checkpoint=False, seed=13548, skip_checkpoint=True, start_warmup_step=0.0, target_mlm_accuracy=0.72, train_batch_size=56, train_mlm_accuracy_window_size=0, unpad=True, unpad_fmha=True, use_cuda_graph=False, use_ddp=False, use_env=False, use_gradient_as_bucket_view=False, warmup_proportion=0.0, warmup_steps=0.0, weight_decay_rate=0.01)
device: cuda:3 n_gpu: 8, distributed training: True, 16-bits training: True
device: cuda:1 n_gpu: 8, distributed training: True, 16-bits training: True
device: cuda:5 n_gpu: 8, distributed training: True, 16-bits training: True
device: cuda:6 n_gpu: 8, distributed training: True, 16-bits training: True
device: cuda:7 n_gpu: 8, distributed training: True, 16-bits training: True
device: cuda:2 n_gpu: 8, distributed training: True, 16-bits training: True
:::MLLOG {"namespace": "", "time_ms": 1623991665937, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 0.00035, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 669}}
:::MLLOG {"namespace": "", "time_ms": 1623991666083, "event_type": "POINT_IN_TIME", "key": "opt_epsilon", "value": 1e-06, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 699}}
:::MLLOG {"namespace": "", "time_ms": 1623991666083, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_1", "value": 0.9, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 702}}
:::MLLOG {"namespace": "", "time_ms": 1623991666084, "event_type": "POINT_IN_TIME", "key": "opt_lamb_beta_2", "value": 0.999, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 703}}
:::MLLOG {"namespace": "", "time_ms": 1623991666084, "event_type": "POINT_IN_TIME", "key": "opt_lamb_weight_decay_rate", "value": 0.0, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 704}}
:::MLLOG {"namespace": "", "time_ms": 1623991666106, "event_type": "POINT_IN_TIME", "key": "opt_learning_rate_warmup_steps", "value": 0, "metadata": {"file": "/workspace/bert/schedulers.py", "lineno": 86}}
:::MLLOG {"namespace": "", "time_ms": 1623991666106, "event_type": "POINT_IN_TIME", "key": "opt_lamb_learning_rate_decay_poly_power", "value": 1.0, "metadata": {"file": "/workspace/bert/schedulers.py", "lineno": 87}}
:::MLLOG {"namespace": "", "time_ms": 1623991666106, "event_type": "POINT_IN_TIME", "key": "start_warmup_step", "value": 0, "metadata": {"file": "/workspace/bert/schedulers.py", "lineno": 88}}
Torch distributed is available.
Torch distributed is initialized.
Torch distributed is available.
Torch distributed is initialized.
Torch distributed is available.
Torch distributed is initialized.
Torch distributed is available.
Torch distributed is initialized.
Torch distributed is available.
Torch distributed is initialized.
Torch distributed is available.
Torch distributed is initialized.
Torch distributed is available.
Torch distributed is initialized.
Torch distributed is available.
Torch distributed is initialized.
:::MLLOG {"namespace": "", "time_ms": 1623991684534, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1264}}
:::MLLOG {"namespace": "", "time_ms": 1623991684602, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1265}}
:::MLLOG {"namespace": "", "time_ms": 1623991684623, "event_type": "INTERVAL_START", "key": "epoch_start", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1276, "epoch_num": 1}}
:::MLLOG {"namespace": "", "time_ms": 1623991684624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1278, "first_epoch_num": 1, "epoch_count": 1}}
parsed args:
Namespace(allreduce_post_accumulation=True, allreduce_post_accumulation_fp16=True, bert_config_path='/workspace/phase1/bert_config.json', bert_model='bert-large-uncased', bypass_amp=False, cache_eval_data=True, checkpoint_activations=False, cuda_graph_mode='segmented', ddp_type='apex', dense_seq_output=True, device=device(type='cuda', index=0), disable_apex_softmax=False, disable_fuse_mask=False, disable_fuse_qkv=False, disable_fuse_scale=False, distributed_lamb=True, do_train=True, dwu_e5m2_allgather=False, dwu_group_size=0, dwu_num_ag_pg=2, dwu_num_ar_pg=1, dwu_num_blocks=1, dwu_num_chunks=1, dwu_num_rs_pg=1, dwu_overlap_reductions=False, enable_fuse_dropout=False, enable_stream=False, eval_batch_size=16, eval_dir='/workspace/evaldata', eval_iter_samples=150000, eval_iter_start_samples=150000, exchange_padding=True, fp16=True, fused_dropout_add=False, fused_gelu_bias=True, fused_mha=True, gradient_accumulation_steps=1, init_checkpoint='/workspace/phase1/model.ckpt-28252.pt', init_tf_checkpoint=None, input_dir='/workspace/data_phase2', keep_n_most_recent_checkpoints=20, learning_rate=0.00035, local_rank=0, log_freq=0.0, loss_scale=0.0, max_iterations_per_graph=4, max_predictions_per_seq=76, max_samples_termination=4500000.0, max_seq_length=512, max_steps=7100.0, min_samples_to_start_checkpoints=3000000, n_gpu=8, num_epochs_to_generate_seeds_for=2, num_eval_examples=10000, num_samples_per_checkpoint=500000, opt_lamb_beta_1=0.9, opt_lamb_beta_2=0.999, output_dir='/results', pad=False, phase2=True, resume_from_checkpoint=False, resume_step=0, seed=13548, skip_checkpoint=True, start_warmup_step=0.0, target_mlm_accuracy=0.72, train_batch_size=56, train_mlm_accuracy_window_size=0, unpad=True, unpad_fmha=True, use_cuda_graph=False, use_ddp=False, use_env=False, use_gradient_as_bucket_view=False, warmup_proportion=0.0, warmup_steps=0.0, weight_decay_rate=0.01)
epoch: 1
:::MLLOG {"namespace": "", "time_ms": 1623991755074, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.37326234579086304, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 335, 'eval_loss': 4.117311954498291, 'eval_mlm_accuracy': 0.37326234579086304}
:::MLLOG {"namespace": "", "time_ms": 1623991822178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.3994792699813843, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 670, 'eval_loss': 3.8508710861206055, 'eval_mlm_accuracy': 0.3994792699813843}
:::MLLOG {"namespace": "", "time_ms": 1623991889211, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49081695079803467, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 1005, 'eval_loss': 3.0633938312530518, 'eval_mlm_accuracy': 0.49081695079803467}
:::MLLOG {"namespace": "", "time_ms": 1623991956689, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6365640163421631, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 1340, 'eval_loss': 1.8937674760818481, 'eval_mlm_accuracy': 0.6365640163421631}
:::MLLOG {"namespace": "", "time_ms": 1623992029633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7016054391860962, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 1675, 'eval_loss': 1.4263815879821777, 'eval_mlm_accuracy': 0.7016054391860962}
:::MLLOG {"namespace": "", "time_ms": 1623992102153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7092344164848328, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 2009, 'eval_loss': 1.3719865083694458, 'eval_mlm_accuracy': 0.7092344164848328}
:::MLLOG {"namespace": "", "time_ms": 1623992175039, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7125316858291626, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 2344, 'eval_loss': 1.353234052658081, 'eval_mlm_accuracy': 0.7125316858291626}
:::MLLOG {"namespace": "", "time_ms": 1623992247581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7141826152801514, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 2679, 'eval_loss': 1.3442305326461792, 'eval_mlm_accuracy': 0.7141826152801514}
:::MLLOG {"namespace": "", "time_ms": 1623992318718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7151703834533691, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 3014, 'eval_loss': 1.334442138671875, 'eval_mlm_accuracy': 0.7151703834533691}
:::MLLOG {"namespace": "", "time_ms": 1623992389322, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7161044478416443, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 3349, 'eval_loss': 1.3298249244689941, 'eval_mlm_accuracy': 0.7161044478416443}
:::MLLOG {"namespace": "", "time_ms": 1623992459832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7171599864959717, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 3684, 'eval_loss': 1.3228261470794678, 'eval_mlm_accuracy': 0.7171599864959717}
:::MLLOG {"namespace": "", "time_ms": 1623992529851, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7175055742263794, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 4018, 'eval_loss': 1.3200327157974243, 'eval_mlm_accuracy': 0.7175055742263794}
:::MLLOG {"namespace": "", "time_ms": 1623992602479, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7180566787719727, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 4353, 'eval_loss': 1.3146203756332397, 'eval_mlm_accuracy': 0.7180566787719727}
:::MLLOG {"namespace": "", "time_ms": 1623992675003, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7191004753112793, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 4688, 'eval_loss': 1.3092862367630005, 'eval_mlm_accuracy': 0.7191004753112793}
:::MLLOG {"namespace": "", "time_ms": 1623992747759, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.719796359539032, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 5023, 'eval_loss': 1.3082157373428345, 'eval_mlm_accuracy': 0.719796359539032}
:::MLLOG {"namespace": "", "time_ms": 1623992820665, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7201817035675049, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1441, "epoch_num": 1}}
{'global_steps': 5358, 'eval_loss': 1.304253101348877, 'eval_mlm_accuracy': 0.7201817035675049}
0.720182 > 0.720000, Target MLM Accuracy reached at 5358
(1, 5369.0) {'final_loss': 0.0}
:::MLLOG {"namespace": "", "time_ms": 1623992820756, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1567, "first_epoch_num": 1}}
:::MLLOG {"namespace": "", "time_ms": 1623992820756, "event_type": "INTERVAL_END", "key": "epoch_stop", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1570, "epoch_num": 1}}
:::MLLOG {"namespace": "", "time_ms": 1623992820756, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2400384, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1574}}
:::MLLOG {"namespace": "", "time_ms": 1623992820756, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 10000, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1577}}
:::MLLOG {"namespace": "", "time_ms": 1623992820756, "event_type": "INTERVAL_END", "key": "run_stop", "value": null, "metadata": {"file": "/workspace/bert/run_pretraining.py", "lineno": 1580, "status": "success"}}
{'e2e_time': 1162.061541557312, 'training_sequences_per_second': 2767.072704874139, 'final_loss': 0.0, 'raw_train_time': 1149.5180428028107}
++ date +%s
+ END=1623992824
++ date '+%Y-%m-%d %r'
+ END_FMT='2021-06-18 05:07:04 AM'
+ echo 'ENDING TIMING RUN AT 2021-06-18 05:07:04 AM'
ENDING TIMING RUN AT 2021-06-18 05:07:04 AM
+ RESULT=1169
+ RESULT_NAME=bert
+ echo 'RESULT,bert,13548,1169,,2021-06-18 04:47:35 AM'
RESULT,bert,13548,1169,,2021-06-18 04:47:35 AM
+ set +x